vmmon: use standard definition of MSR_TSX_FORCE_ABORT if available

The MSR_TSX_FORCE_ABORT macro is defined in mainline since commit 52f64909409c ("x86: Add TSX Force Abort CPUID/MSR") in v5.1-rc1.
vmnet: convert VNetBridge debugging code to ktime_get_ns()
2020-11-20 10:42:58 +01:00 · 2020-11-20 10:42:22 +01:00 · 2020-11-20 10:33:49 +01:00 · 2020-11-20 10:33:34 +01:00 · 2020-11-20 10:32:57 +01:00 · 2020-11-20 10:32:42 +01:00
103 changed files with 8402 additions and 6662 deletions
@@ -150,7 +150,7 @@ test build).
 When using a downloaded tarball, simply run

  tar -cf vmmon.tar vmmon-only
-  tar -cf vmnet.tar vmmon-only
+  tar -cf vmnet.tar vmnet-only

 to create the tarballs. In this case, there is no protection against
 unwanted local changes.
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -129,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -31,7 +31,7 @@ $(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, \
 		$(SRCROOT)/bootstrap/*.c)))

 clean:
-	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \
+	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions .cache.mk \
 		Module.symvers Modules.symvers Module.markers modules.order \
 		$(foreach dir,linux/ common/ vmcore/ bootstrap/ \
 		./,$(addprefix $(dir),.*.cmd .*.o.flags *.o)))
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998,2015,2017 VMware, Inc. All rights reserved.
+# Copyright (C) 1998,2015,2017,2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -72,7 +72,7 @@ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')

 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
-C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o
+C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o crosspage.o
 C_TARGETS_VMCORE := moduleloop.o
 C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,14 +24,7 @@
 *      See monLoader.h for a full description.
 */

-#ifdef VMX86_VMX
-
-#include <string.h>
-#include <limits.h>
-#include "vmx.h"
-#include "vm_assert.h"
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #include "vmkernel.h"
 #include "libc.h"
@@ -57,15 +50,15 @@

 #include "vm_assert.h"

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

 #include "vm_basic_types.h"
 #include "monLoader.h"
 #include "vcpuid.h"
-#include "x86types.h"
-#include "x86paging_64.h"
+#include "vm_pagetable.h"
+#include "address_defs.h"
 #include "monLoaderLog.h"

 #define CANONICAL_MASK MASK64(36)
@@ -75,12 +68,6 @@
 #define L2EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 18))
 #define L1EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 9))

-/*
- * Import address space from VMX rather than building it.  Once monitor loading
- * is removed from VMX entirely, code to import will be deleted.
- */
-#define IMPORT_AS_FROM_VMX (TRUE)
-
 /* The maximum (canonical-address) VPN */
 #define VPN_MAX MASK64(52)

@@ -111,15 +98,11 @@ typedef struct MonLoaderContext {
      VPN          ASLastVPN;   /* last VPN in the address space (inclusive) */
      uint64       ASPTEFlags;  /* PTE flags for L4->L1 connection. */
      Vcpuid       currentVCPU;
-      Bool         ASImported;  /* Was this AS imported or created anew? */
      Bool         hasAddrSpace;
   } vcpu;
 } MonLoaderContext;


-static const Bool importFromVMX = IMPORT_AS_FROM_VMX;
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -192,7 +175,7 @@ MonLoaderIsMapped(MonLoaderContext *ctx,    // IN/OUT
   if (ret != ML_OK) {
      return ret;
   }
-   *mapped = PTE_PRESENT(pte);
+   *mapped = ML_PERM_PRESENT(pte);
   return ML_OK;
 }

@@ -280,15 +263,40 @@ MonLoaderMapMPN(MonLoaderContext *ctx,    // IN/OUT
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderBuildsPtLevel --
+ *
+ *      Determines whether MonLoader allocates and maps the page table(s) for
+ *      the monitor at the given level.
+ *
+ * Result:
+ *      TRUE if MonLoader creates the page table at the given level, FALSE
+ *      otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+static Bool
+MonLoaderCreatesPtLevel(PT_Level level)
+{
+   ASSERT(level >= PT_LEVEL_STOP && level <= PT_MAX_LEVELS);
+   return FALSE;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
 * MonLoaderCreateAddressSpace --
 *
- *      Creates or (if importAS is set) verifies an address space.  The VPN
- *      range specified by firstVPN and size is used to determine page counts
- *      at each page table level to map every page.  Sufficient pages are then
- *      allocated or verified.
+ *      Creates or verifies an address space.  The VPN range specified by
+ *      firstVPN and size is used to determine page counts at each page table
+ *      level to map every page.  Sufficient pages are then allocated or
+ *      verified.
 *
 * Result:
 *      ML_OK if successful, some other value on error.
@@ -302,8 +310,7 @@ static MonLoaderError
 MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
                            VPN               firstVPN, // IN
                            uint64            flags,    // IN
-                            uint64            monPages, // IN
-                            Bool              importAS) // IN
+                            uint64            monPages) // IN
 {
   Vcpuid vcpu = ctx->vcpu.currentVCPU;
   VPN lastVPN = firstVPN + monPages - 1;
@@ -329,11 +336,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
   ctx->vcpu.ASFirstVPN = firstVPN;
   ctx->vcpu.ASLastVPN = lastVPN;
   ctx->vcpu.ASPTEFlags = flags;
-   ctx->vcpu.ASImported = importAS;

   ptMPNs = &ctx->vcpu.ptMPNs;

-   if (importAS) {
+   if (MonLoaderCreatesPtLevel(PT_LEVEL_4)) {
+      NOT_IMPLEMENTED();
+   } else {
+      /* Verify the VMX's allocation. */
      mpn = MonLoaderCallout_GetPageRoot(ctx->envCtx, vcpu);
      if (mpn == INVALID_MPN) {
         return ML_ERROR_CALLOUT_PAGEROOT_GET;
@@ -341,13 +350,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
      LOG(5, "%s: vcpu %u page root=0x%"FMT64"x\n", __FUNCTION__, vcpu, mpn);
      ptMPNs->L4MPNs[0] = mpn;
      MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-   } else {
-      NOT_IMPLEMENTED();
   }
   ptMPNs->L4MPNCount = 1;

   for (i = 0; i < L3MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_3)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L4E;
         unsigned L4Off = PT_LPN_2_L4OFF(monVPN);
         PT_L1E pte;
@@ -357,20 +366,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L4E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L3MPNs[i] = mpn;
   }
   ptMPNs->L3MPNCount = L3MPNsNeeded;
      
   for (i = 0; i < L2MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_2)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L3E;
         unsigned L3Off = PT_LPN_2_L3OFF(monVPN);
         unsigned L3Page = L3EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -381,20 +390,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L3E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L2MPNs[i] = mpn;
   }
   ptMPNs->L2MPNCount = L2MPNsNeeded;

   for (i = 0; i < L1MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_1)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L2E;
         unsigned L2Off = PT_LPN_2_L2OFF(monVPN);
         unsigned L2Page = L2EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -405,17 +414,15 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         if (!PTE_PRESENT(pte)) {
+         if (!ML_PERM_PRESENT(pte)) {
            ptMPNs->L1MPNs[i] = INVALID_MPN;
            continue;
         }
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L1MPNs[i] = mpn;
   }
@@ -446,12 +453,12 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
 */
 static MonLoaderError
 MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
-                       unsigned          level,    // IN
+                       PT_Level          level,    // IN
                       uint64            flags,    // IN
                       VPN               monVPN,   // IN
                       uint64            monPages) // IN
 {
-   Bool verify = ctx->vcpu.ASImported;
+   Bool verify = !MonLoaderCreatesPtLevel(level);
   uint64    i;
   unsigned  count;
   MPN      *ptMPNs;
@@ -460,25 +467,25 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
      if (!ctx->vcpu.hasAddrSpace) {
         return ML_ERROR_NO_ADDRSPACE;
      }
-      if (level < 1 || level > 4) {
+      if (level < PT_LEVEL_STOP || level > PT_MAX_LEVELS) {
         return ML_ERROR_ARGS;
      }
   }

   switch (level) {
-      case 4:
+      case PT_LEVEL_4:
         count = ctx->vcpu.ptMPNs.L4MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L4MPNs;
         break;
-      case 3:
+      case PT_LEVEL_3:
         count = ctx->vcpu.ptMPNs.L3MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L3MPNs;
         break;
-      case 2:
+      case PT_LEVEL_2:
         count = ctx->vcpu.ptMPNs.L2MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L2MPNs;
         break;
-      case 1:
+      case PT_LEVEL_1:
         count = ctx->vcpu.ptMPNs.L1MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L1MPNs;
         break;
@@ -494,11 +501,11 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
         VPN vpn = monVPN + i;
         if (verify) {
            PT_L1E l1e;
-            if (ptMPNs[i] == INVALID_MPN && level == 1) {
+            if (ptMPNs[i] == INVALID_MPN && level == PT_LEVEL_1) {
               continue;
            }
            if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK ||
-                LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
+                ML_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
               return ML_ERROR_PAGE_TABLE_VERIFY;
            }
         } else {
@@ -629,7 +636,7 @@ MonLoaderCopyFromBlob(MonLoaderContext *ctx,        // IN/OUT
 *      ML_OK if successful, some other value on error.
 *
 * Side effects:
- *      User pages may be mapped in.
+ *      User or host pages may be mapped in.
 *
 *----------------------------------------------------------------------
 */
@@ -660,6 +667,14 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
                                                  vcpu);
      }
      if (mpn == INVALID_MPN) {
+         if (subIndex == MONLOADER_HT_MAP_IDX) {
+            /*
+             * This item is tied to a vmkernel feature.
+             * When the feature is disabled, there is nothing to share.
+             */
+            ASSERT(vmx86_server);
+            return ML_OK;
+         }
         /* Partial sharing is allowed.  Return success if any occurred. */
         return i != 0 ? ML_OK : ML_ERROR_SHARE;
      }
@@ -672,45 +687,6 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
 }


-#ifndef VMMON
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderSetEntrypoint --
- *
- *      Calls out to set the code and stack entrypoint.
- *
- * Result:
- *      ML_OK if successful, some other value on error.
- *
- * Side effects:
- *      The monitor entrypoint for the current VCPU may be set.
- *
- *----------------------------------------------------------------------
- */
-static MonLoaderError
-MonLoaderSetEntrypoint(MonLoaderContext *ctx,           // IN
-                       uint16            codeSelector,  // IN
-                       VA64              code,          // IN
-                       uint16            stackSelector, // IN
-                       VA64              stack)         // IN
-{
-   if (VA_2_VPN(code) < ctx->vcpu.ASFirstVPN  ||
-       VA_2_VPN(code) > ctx->vcpu.ASLastVPN   ||
-       VA_2_VPN(stack) < ctx->vcpu.ASFirstVPN ||
-       VA_2_VPN(stack) > ctx->vcpu.ASLastVPN) {
-      return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
-   }
-   if (MonLoaderCallout_SetEntrypoint(ctx->envCtx, codeSelector, code,
-                                      stackSelector, stack)) {
-      return ML_OK;
-   } else {
-      return ML_ERROR_CALLOUT_ENTRYPOINT;
-   }
-}
-#endif
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -795,7 +771,7 @@ MonLoaderShareFromBlob(MonLoaderContext *ctx,        // IN/OUT

   if ((blobOffset & (PAGE_SIZE - 1)) != 0 ||
       (blobSize & (PAGE_SIZE - 1)) != 0 ||
-       (flags & PTE_RW) != 0) {
+       ML_PERM_WRITEABLE(flags)) {
      return ML_ERROR_SHARE;
   }
   if (blobSize > monBytes || blobSize == 0) {
@@ -866,7 +842,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
      return ML_ERROR_TABLE_MISSING;
   }

-   if (!MonLoaderCallout_Init(args, &ctx.envCtx)) {
+   if (!MonLoaderCallout_Init(args, &ctx.envCtx, numVCPUs)) {
      return ML_ERROR_CALLOUT_INIT;
   }
   ret = ML_OK;
@@ -887,7 +863,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         uint64               monBytes   = PAGES_2_BYTES(monPages);
         uint64               subIndex   = entry->subIndex;
         uint64               bspOnly    = entry->bspOnly;
-         uint64               isPriv     = vmx86_server || entry->procVmmon;
         *line = i;

         /*
@@ -896,17 +871,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         if (bspOnly && !IS_BOOT_VCPUID(*vcpu)) {
            continue;
         }
-         /*
-          * Entries marked as privileged (isPriv == TRUE) are processed
-          * by the privileged layer (VMKernel/vmmon). Others are
-          * processed by the VMX. One exception is the entry of type
-          * ML_CONTENT_ADDRSPACE: it is always processed by the VMX and,
-          * if marked as privileged, also by the privileged layer.
-          */
-         if (isPriv != MonLoaderCallout_IsPrivileged(ctx.envCtx) &&
-             (content != ML_CONTENT_ADDRSPACE || isPriv == 0)) {
-            continue;
-         }
         if (monPages == 0 || monVPN + monPages - 1 > VPN_MAX) {
            ret = ML_ERROR_SIZE;
            continue;
@@ -919,8 +883,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
                  continue;
               }
               /* Create or verify the address space and page table MPNs. */
-               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages,
-                                                 importFromVMX);
+               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages);
               break;
            case ML_CONTENT_PAGETABLE_L4:
            case ML_CONTENT_PAGETABLE_L3:
@@ -988,15 +951,10 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         }
      }
   }
-#ifndef VMMON
+#ifdef VMKERNEL
   if (ret == ML_OK) {
      if (ctx.vcpu.ASLastVPN == 0) {
         ret = ML_ERROR_NO_ADDRSPACE;
-      } else {
-         ret = MonLoaderSetEntrypoint(&ctx, header->codeSelector,
-                                      header->codeEntrypoint,
-                                      header->stackSelector,
-                                      header->stackEntrypoint);
      }
   }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,7 +65,7 @@ typedef struct MonLoaderEnvContext {
 */

 Bool
-MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx)
+MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx, unsigned numVCPUs)
 {
   MonLoaderEnvContext *c;
   MonLoaderArgs *mlArgs = (MonLoaderArgs *)args;
@@ -215,7 +215,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
   vpn = Vmx86_MapPage(mpn);
   if (vpn == 0) {
      Log("Failed to map MPN 0x%"FMT64"x\n", mpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   HostIF_VMLock(vm, 41);
@@ -225,7 +225,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
      Log("Failed to track mapping from VPN 0x%"FMTVPN"x to MPN 0x%"FMT64"x\n",
          vpn, mpn);
      Vmx86_UnmapPage(vpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   return mpn;
@@ -386,10 +386,8 @@ MonLoaderFindSharedRegion(MonLoaderEnvContext *ctx,
   int i;

   for (i = 0; i < ML_SHARED_REGIONS_MAX; i++) {
-      if (ctx->shRegions[i].baseVpn == INVALID_VPN) {
-         /* Not found. */
-         return NULL;
-      } else if (ctx->shRegions[i].index == index) {
+      if (ctx->shRegions[i].index == index &&
+          ctx->shRegions[i].baseVpn != INVALID_VPN) {
         return &ctx->shRegions[i];
      }
   }
@@ -425,10 +423,6 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,

   ASSERT(IS_BOOT_VCPUID(vcpuid));

-   if (index >= ML_SHARED_REGIONS_MAX) {
-      Log("Invalid shared region %"FMT64"x", index);
-      return INVALID_MPN;
-   }
   s = MonLoaderFindSharedRegion(ctx, index);
   if (s == NULL) {
      return INVALID_MPN;
@@ -453,7 +447,7 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 *
 * MonLoaderCallout_GetSharedUserPage --
 *
- *      Gets a user-provided shared page's MPN for a VCPU.
+ *      Gets a shared page's MPN for a VCPU.
 *
 * Returns:
 *      An MPN or INVALID_MPN.
@@ -465,10 +459,10 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 */

 MPN
-MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx,
-                                   uint64 subIndex, // IN
-                                   unsigned page,   // IN
-                                   Vcpuid vcpu)     // IN
+MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx, // IN
+                                   uint64 subIndex,          // IN
+                                   unsigned page,            // IN
+                                   Vcpuid vcpu)              // IN
 {
   if (subIndex == MONLOADER_HEADER_IDX) {
      return VmmBlob_GetHeaderMpn(ctx->vm);
@@ -484,30 +478,14 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   NOT_IMPLEMENTED();
-}
-
-
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderCallout_IsPrivileged --
- *
- *      Returns whether or not this is a privileged environment.
- *
- * Returns:
- *      TRUE if privileged, FALSE if not.
- *
- * Side effects:
- *      None.
- *
- *----------------------------------------------------------------------
- */
-
-Bool
-MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN
-{
-   return TRUE;
+   switch (subIndex) {
+   case MONLOADER_CROSS_PAGE_DATA_IDX:
+      return HostIF_GetCrossPageDataMPN(ctx->vm->crosspage[vcpu]);
+   case MONLOADER_CROSS_PAGE_CODE_IDX:
+      return HostIF_GetCrossPageCodeMPN();
+   default:
+      return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   }
 }


@@ -534,13 +512,3 @@ MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx,    // IN
   ASSERT((offset & (PAGE_SIZE - 1)) == 0);
   return VmmBlob_GetMpn(ctx->vm, BYTES_2_PAGES(offset));
 }
-
-Bool
-MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *ctx,           // IN
-                               uint16               codeSelector,  // IN
-                               VA64                 code,          // IN
-                               uint16               stackSelector, // IN
-                               VA64                 stack)         // IN
-{
-   NOT_IMPLEMENTED();
-}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,7 +31,8 @@
 #include "vmmblob.h"
 #include "monLoader.h"

-#define VMMBLOB_SIZE_MAX (12 * 1024 * 1024)
+#define VMMBLOB_SIZE_MAX (22 * 1024 * 1024) /* Ensure enough space for
+                                             * obj build with * GCOV_VMM=1. */

 /*
 *----------------------------------------------------------------------
@@ -65,7 +65,7 @@ APIC_GetMA(void)
    * Check if X2 APIC mode is enabled.
    */

-   if ((__GET_MSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
      return (MA)-1;
   }

@@ -75,7 +75,7 @@ APIC_GetMA(void)
    */

   // Mask out goo in the low 12 bits, which is unrelated to the address.
-   result = __GET_MSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);
+   result = X86MSR_GetMSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);

   /*
    * On Intel, the high bits are reserved so we mask.
@@ -122,7 +122,7 @@ APIC_Read(const APICDescriptor *desc, // IN
          int regNum)                 // IN
 {
   if (desc->isX2) {
-      return (uint32 )__GET_MSR(MSR_X2APIC_BASE + regNum);
+      return (uint32)X86MSR_GetMSR(MSR_X2APIC_BASE + regNum);
   } else {
      return desc->base[regNum][0];
   }
@@ -182,7 +182,7 @@ APIC_Write(const APICDescriptor *desc, // IN
           uint32 val)                 // IN
 {
   if (desc->isX2) {
-      __SET_MSR(MSR_X2APIC_BASE + regNum, val);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + regNum, val);
   } else {
      desc->base[regNum][0] = val;
   }
@@ -210,7 +210,7 @@ uint64
 APIC_ReadICR(const APICDescriptor *desc) // IN
 {
   if (desc->isX2) {
-      return __GET_MSR(MSR_X2APIC_BASE + APICR_ICRLO);
+      return X86MSR_GetMSR(MSR_X2APIC_BASE + APICR_ICRLO);
   } else {
      uint32 icrHi = desc->base[APICR_ICRHI][0];
      uint32 icrLo = desc->base[APICR_ICRLO][0];
@@ -244,7 +244,7 @@ APIC_WriteICR(const APICDescriptor *desc, // IN
 {
   if (desc->isX2) {
      uint64 icr = (uint64) id << 32 | icrLo;
-      __SET_MSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
   } else {
      ASSERT(!(id & ~(APIC_ICRHI_DEST_MASK >> APIC_ICRHI_DEST_OFFSET)));
      desc->base[APICR_ICRHI][0] = id << APIC_ICRHI_DEST_OFFSET;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -40,6 +40,7 @@ uint32      cpuidVersion;
 Bool        hostSupportsVT;
 Bool        hostSupportsSVM;
 Bool        hostHasSpecCtrl;
+Bool        hostSupportsXSave;


 /*
@@ -84,6 +85,7 @@ CPUID_Init(void)
   __GET_CPUID(1, &regs);
   cpuidVersion = regs.eax;
   cpuidFeatures = regs.edx;
+   hostSupportsXSave = CPUID_ISSET(1, ECX, XSAVE, regs.ecx);

   __GET_CPUID(0, &regs);
   ptr = (uint32 *)name;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,6 +37,7 @@ extern CpuidVendor cpuidVendor;
 extern Bool        hostSupportsVT;
 extern Bool        hostSupportsSVM;
 extern Bool        hostHasSpecCtrl;
+extern Bool        hostSupportsXSave;

 void CPUID_Init(void);
 Bool CPUID_AddressSizeSupported(void);
@@ -84,6 +85,12 @@ CPUID_HostSupportsSpecCtrl(void)
   return hostHasSpecCtrl;
 }

+static INLINE Bool
+CPUID_HostSupportsXSave(void)
+{
+   return hostSupportsXSave;
+}
+
 static INLINE Bool
 CPUID_SSE2Supported(void)
 {
@@ -0,0 +1,609 @@
+/*********************************************************
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * crosspage.c --
+ *
+ *    Cross page code and data.
+ *
+ *    The cross code page contains world switch code and interrupt/exception
+ *    handlers in support of world switch.  A separate data page is also mapped
+ *    in both the vmm and vmmon address spaces to describe the host and monitor
+ *    state.
+ *
+ *    Both the monitor and the vmmon driver call into the cross page to switch
+ *    worlds.  The world switch from host->vmm is able to refer directly to the
+ *    various symbols defined in CrossPage_CodePage.  On the vmm->host path
+ *    the VmmToHost entry point is stored in the data page.  Data references
+ *    must be handled as offsets from the base of the data page.
+ *
+ *    Use of inline assembly in C code is delicate as the compiler has many
+ *    possible reasons to emit unexpected instructions surrounding any assembly
+ *    in a C function.  Frame pointers are inconvenient to disable on a per-file
+ *    basis given structure of the Linux Makefile so they are being emitted
+ *    in the C wrapper function.  NORETURN and NOT_REACHED_MINIMAL are used to
+ *    omit a function epilogue.  The output/clobber lists of the assembly blocks
+ *    are not used as the compiler needs not compensate for the register use of
+ *    these hand-written assembly functions.
+ *    Each assembly function begins with a label denoting the proper entry point
+ *    for that function.  Only a handful of the functions need to be exported
+ *    out of this file, see crosspage.h.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#      include <linux/objtool.h>
+#   elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
+#      include <linux/frame.h>
+#   endif
+#endif
+
+#include "modulecall.h"
+#include "iocontrols.h"
+#include "segs.h"
+#include "x86_basic_defs.h"
+#include "vm_basic_defs.h"
+#include "vm_idt_x86.h"
+#include "crosspage.h"
+
+#ifdef __APPLE__
+/*
+ * The Mac compiler expects symbols to have a _ prefix.
+ * This arranges for any symbol being exported out of the assembly code
+ * to be accessible.
+ */
+#define ASM_PREFIX "_"
+/*
+ * OSX uses a segment,section style syntax for specifying output sections.
+ * https://developer.apple.com/library/archive/documentation/Performance/
+ * Conceptual/CodeFootprint/Articles/MachOOverview.html
+ */
+#define ASM_SECTION "__TEXT,cross"
+#else
+/*
+ * This is a COFF grouped section identifier.  This ensures the crosspage code
+ * is merged with the normal .text area but as a separate unit so it can have
+ * its own alignment properties.
+ * https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
+ *
+ * The Linux build will handle the section based on its contents (code).
+ */
+#define ASM_SECTION ".text$cross"
+#define ASM_PREFIX
+#endif
+
+#define EXPORTED_ASM_SYMBOL(fn) ".global " ASM_PREFIX #fn "\n"   \
+                                ASM_PREFIX #fn ":\n"
+
+/*
+ * Tag the crosspage code C wrapper with the crosspage section and page
+ * alignment.
+ */
+#define CPCODE __attribute__((section(ASM_SECTION)))   \
+               __attribute__((aligned(PAGE_SIZE)))
+
+/* No special handling is needed for the data. */
+#define CPDATA
+
+#define VMMDATALA(off) (LPN_2_LA(CROSS_PAGE_DATA_START) + (off))
+
+#define NOT_REACHED_MINIMAL __builtin_unreachable
+
+#ifdef ANNOTATE_INTRA_FUNCTION_CALL
+#define ANNOTATE_ASM_CALL_STR2(x) #x
+#define ANNOTATE_ASM_CALL_STR(x) ANNOTATE_ASM_CALL_STR2(x) "\n"
+#define ANNOTATE_ASM_CALL ANNOTATE_ASM_CALL_STR(ANNOTATE_INTRA_FUNCTION_CALL)
+#else
+#define ANNOTATE_ASM_CALL
+#endif
+
+void VmmToHost(void);
+
+CPDATA const VMCrossPageData cpDataTemplate = {
+   .version        = CROSSPAGE_VERSION,
+   .vmmonVersion   = VMMON_VERSION,
+
+   .monRSP         = VMM_LRET_STACK_TOP,
+   .monSS          = SYSTEM_DATA_SELECTOR,
+   .monDS          = SYSTEM_DATA_SELECTOR,
+   .monES          = SYSTEM_DATA_SELECTOR,
+
+   .vmmToHostLA    = (VA)VmmToHost,
+   .wsCR0          = CR0_PE | CR0_MP | CR0_EM | CR0_NE | CR0_WP | CR0_PG,
+   .wsCR4          = CR4_PAE | CR4_OSFXSR,
+
+   .monTask.rsp[0] = VMM_STACK_TOP,                   /* Monitor stack. */
+   .monTask.rsp[1] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL1 is not used. */
+   .monTask.rsp[2] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL2 is not used. */
+
+   .monTask.ist[IST_NONE]                 = 0, /* No stack switch. */
+   .monTask.ist[IST_VMM_DF]               = DF_STACK_TOP,
+   .monTask.ist[IST_VMM_NMI]              = NMI_STACK_TOP,
+   .monTask.ist[IST_VMM_MCE]              = MC_STACK_TOP,
+
+   .monTask.IOMapBase = sizeof(Task64),
+
+   .monGDTR.limit  = VMMON_GDT_LIMIT,
+   .monGDTR.offset = GDT_START_VA,
+
+   .shadowDR[6].ureg64 = DR6_DEFAULT,
+   .shadowDR[7].ureg64 = DR7_ONES,
+
+   /*
+    * switchHostIDT and switchHostIDTR are initialized in vmmon.
+    * switchMonIDTR is static and can be fully initialized at compile-time.
+    * switchMonIDT is mostly static with all non-present entries except for
+    * the 4 handled exceptions which are populated by the crosspage setup
+    * code in task.c
+    */
+   .switchMonIDTR  = { sizeof(Gate64) * NUM_EXCEPTIONS - 1,
+                       VMMDATALA(offsetof(VMCrossPageData, switchMonIDT)) },
+};
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * CrossPage_CodePage --
+ *
+ *      This function only serves as an anchor and wrapper for the code that
+ *      runs in the world switch's cross page.  The function must not be called
+ *      directly.  Its address may be used to locate the cross page code.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+CPCODE void
+CrossPage_CodePage(void)
+{
+   __asm__ __volatile__ (
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchDBHandler --
+    *
+    *      DB handler that operates during worldswitch (in both directions).
+    *
+    *      This handler is expected to be used in two cases:
+    *      1) If the host kernel has set the DR7_GD (guard) bit, when switch
+    *         code attempts to save debug registers, a #DB will be thrown.  In
+    *         this case, the handler sets wsException[EXC_DB] and returns.
+    *      2) When stress-testing the worldswitch code, RFLAGS.TF is set across
+    *         most switch paths.  This causes a #DB to be thrown before each
+    *         instruction is executed.  In this case, the handler must set
+    *         RFLAGS.RF in the return frame to guarantee progress while
+    *         RFLAGS.TF is set.  For added stress, the handler induces a
+    *         simulated #NMI.
+    *
+    *      In case (1), returning is accomplished via a simulated iret in order
+    *      to preserve any NMI-blocking.  In case (2), the inherent
+    *      incompatibility of RFLAGS.RF for single-stepping and simulating an
+    *      iret via popf and lretq necessitates the use of an actual iret.  As
+    *      a result, there is no strong NMI-blocking guarantee when
+    *      stress-testing the switch path.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_DB] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchDBHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionDB], %%rax\n"
+   "movb         $1,               (%%rax)\n" /* log EXC_DB */
+   "popq         %%rax\n"
+   "testl        %[EFLAGS_TF],     16(%%rsp)\n" /* check trap flag */
+   "jz           return_without_enabling_nmi\n"
+   "orl          %[EFLAGS_RF],     16(%%rsp)\n"
+   "int          $2\n"
+   "iretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchUDHandler --
+    *
+    *      UD handler that operates during worldswitch (in both directions).
+    *
+    *      It sets the wsException[EXC_UD] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_UD] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchUDHandler)
+   "pushq        %%rax\n"
+   "pushq        %%rbx\n"
+   "pushq        %%rcx\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "movl         %[wsExceptionUD],      %%ecx\n"    /* log EXC_UD */
+   "movb         $1,                    (%%rax, %%rcx)\n"
+   /* Check if the exception came from a monitor RIP. */
+   "cmpq         %[MONITOR_MINIMUM_VA], 24(%%rsp)\n"
+   "jae          monitor_context\n"
+   /* Check if the exception came from the crosspage code. */
+   "movq         24(%%rsp),              %%rbx\n"
+   "andq         %[PageAlignMask],       %%rbx\n"
+   "movl         %[crosspageCodeLA],     %%ecx\n"
+   "cmpq         (%%rax, %%rcx),         %%rbx\n"
+   "je           monitor_context\n"
+   /*
+    * Linux halts the processor on host #UD2.  Act similarly.  PR 1281662.
+    * Note that this can escape due to an #NMI, #MC, or #DB.
+    */
+   "cli\n"
+   "hlt\n"
+   /* Advance 2 bytes to skip past the #UD2 instruction. */
+   "monitor_context:\n"
+   "movq         24(%%rsp), %%rbx\n"
+   "movl         %[wsUD2],  %%ecx\n"
+   "movq         %%rbx,     (%%rax, %%rcx)\n"
+   "addq         $2,        24(%%rsp)\n"
+   "popq         %%rcx\n"
+   "popq         %%rbx\n"
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchNMIHandler --
+    *
+    *      NMI handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It 'simply' sets the wsException[EXC_NMI] flag then returns,
+    *      leaving further NMI delivery inhibited.
+    *
+    *      As long as we don't execute 'iret', the CPU will not allow further
+    *      NMIs to be delivered; this is how the CPU protects itself from
+    *      runaway NMIs eating up the stack and triple-faulting.
+    *
+    *      See Vol 3, 5.5.1, Handling multiple NMIs.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_NMI] = TRUE
+    *      further NMI delivery inhibited
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchNMIHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionNMI], %%rax\n"
+   "movb         $1,                (%%rax)\n" /* log EXC_NMI */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchMCEHandler --
+    *
+    *      MCE handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It simply sets the wsException[EXC_MC] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_MC] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchMCEHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionMC], %%rax\n"
+   "movb         $1,              (%%rax)\n" /* log EXC_MC */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchHandlerReturns --
+    *
+    *      Shared code to return without enabling NMIs.  Switch handlers call
+    *      directly to these labels as needed.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "return_without_enabling_nmi:\n"
+   "pushq        %%rbp\n"
+   "pushq        %%rax\n"
+   "movq         40(%%rsp), %%rbp\n"
+   "subq         $32, %%rbp\n"
+   "movq         24(%%rsp), %%rax\n"
+   "movq         %%rax, 24(%%rbp)\n"
+   "movq         16(%%rsp), %%rax\n"
+   "movq         %%rax, 16(%%rbp)\n"
+   "movq         32(%%rsp), %%rax\n"
+   "movq         %%rax, 8(%%rbp)\n"
+   "movq         8(%%rsp),  %%rax\n"
+   "movq         %%rax, (%%rbp)\n"
+   "popq         %%rax\n"
+   "movq         %%rbp, %%rsp\n"
+   "popq         %%rbp\n"
+   "popfq\n"
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * HostToVmm --
+    *
+    *      Switch from host to monitor.
+    *
+    * Entered with:
+    *      RCX = crosspage host address
+    *      (RSP) = return to host address
+    *
+    * Must preserve:
+    *      RBX, RSI, RDI, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(HostToVmm)
+   /* Create an lret frame on the host stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold host %rsp */
+
+   /* Start from the "empty" host context and save. */
+   "leaq            %c[hostContextEmpty](%%rcx), %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* host %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rdi\n"
+   "pushq           %%rsi\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" monitor context and load. */
+   "leaq            %c[monContextFull](%%rcx), %%rsp\n"
+   "popq            %%rsi\n" /* monitor %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+   "popq            %%rax\n" /* monitor %rsp */
+   "popw            %%dx\n"  /* monitor %ss */
+   /* Load the cross GDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%rsi, %%cr3\n"
+   /* Reload the monitor's %ss into %ds/%es. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * VmmToHost --
+    *
+    *      Switch from monitor to host.
+    *
+    * Must preserve:
+    *      RBX, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(VmmToHost)
+   "movq            %c[VMMCROSSPAGE] + %c[crosspageDataLA], %%rcx\n"
+   /* Create an lret frame on the monitor stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold monitor %rsp */
+
+   /* Start from the "empty" monitor context and save. */
+   "leaq            %c[VMMCROSSPAGE] + %c[monContextEmpty], %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* monitor %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" host context and load. */
+   "leaq            %c[hostContextFull](%%rcx), %%rsp\n"
+   "popq            %%r9\n" /* host %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%rsi\n" /* Not ABI-required. */
+   "popq            %%rdi\n" /* Not ABI-required. */
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+
+   "popq            %%rax\n" /* host %rsp */
+   "popw            %%dx\n"  /* host %ss */
+   /* Load the cross GDT and IDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   "lidtq           %c[switchHostIDTR](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%r9,  %%cr3\n"
+   /* Reload the host's %ss into %ds/%es.  Technically wrong. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   /* Microsoft RTL/codegen assumes EFLAGS<DF> = 0. */
+   "cld\n"
+   "lretq\n"
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchExcGetCrossPageData --
+    *
+    *      Common function for the exception handlers to locate the data
+    *      crosspage so they can record their respective events.  In order to
+    *      reach this code an exception had to vector through the IDT.  The IDT
+    *      is known to be in the data page.  Therefore, the data page can be
+    *      found by accessing IDTR and rounding down to page alignment.
+    *
+    * Input:
+    *     None
+    *
+    * Output:
+    *     %rax = Page aligned address of the current crosspage data area.
+    *
+    * Note:
+    *     %rax (return value) and %rflags are destroyed, all other registers
+    *     are preserved.  Since this is only called by exception handlers, the
+    *     CPU has already saved %rflags so no additional handling is required.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "SwitchExcGetCrossPageData:\n"
+   "subq            $0x10,            %%rsp\n"
+   "sidt            0(%%rsp)\n"
+   "movq            2(%%rsp),         %%rax\n" /* DTR.offset */
+   "addq            $0x10,            %%rsp\n"
+   "andq            %[PageAlignMask], %%rax\n"
+   "ret\n"
+
+   EXPORTED_ASM_SYMBOL(CrossPage_CodeEnd)
+
+   : /* No output list, this is not really C code. */
+   : [MONITOR_MINIMUM_VA] "i" (MONITOR_LINEAR_START),
+     [PageAlignMask]      "i" (~PAGE_MASK),
+     [EFLAGS_TF]          "i" (EFLAGS_TF),
+     [EFLAGS_RF]          "i" (EFLAGS_RF),
+     [VMMCROSSPAGE]       "i" (CROSS_PAGE_DATA_START * PAGE_SIZE),
+     [wsUD2]              "i" (offsetof(VMCrossPageData, wsUD2)),
+
+     [wsExceptionDB]      "i" (offsetof(VMCrossPageData, wsException[EXC_DB])),
+     [wsExceptionUD]      "i" (offsetof(VMCrossPageData, wsException[EXC_UD])),
+     [wsExceptionNMI]     "i" (offsetof(VMCrossPageData, wsException[EXC_NMI])),
+     [wsExceptionMC]      "i" (offsetof(VMCrossPageData, wsException[EXC_MC])),
+
+     [hostContextEmpty]   "i" (offsetof(VMCrossPageData, hostDS)),
+     [monContextEmpty]    "i" (offsetof(VMCrossPageData, monDS)),
+
+     [hostContextFull]    "i" (offsetof(VMCrossPageData, hostCR3)),
+     [monContextFull]     "i" (offsetof(VMCrossPageData, monCR3)),
+
+     [crosspageDataLA]    "i" (offsetof(VMCrossPageData, crosspageDataLA)),
+     [crosspageCodeLA]    "i" (offsetof(VMCrossPageData, crosspageCodeLA)),
+     [crossGDTHKLADesc]   "i" (offsetof(VMCrossPageData, crossGDTHKLADesc)),
+     [switchHostIDTR]     "i" (offsetof(VMCrossPageData, switchHostIDTR))
+   );
+   NOT_REACHED_MINIMAL();
+}
+
+#ifdef STACK_FRAME_NON_STANDARD
+STACK_FRAME_NON_STANDARD(CrossPage_CodePage);
+#endif
+
+/*
+ * The crosspage data must not exceed a single page.
+ * The code area also must fit in a single page but that size isn't available
+ * until link time.
+ */
+MY_ASSERTS(CROSSPAGE_DATA,
+   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
+)
@@ -0,0 +1,43 @@
+/*********************************************************
+ * Copyright (C) 2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _CROSSPAGE_H_
+#define _CROSSPAGE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct VMCrossPageData;
+
+extern void HostToVmm(struct VMCrossPageData *crosspageWIN, void *unused1,
+                      void *unused2, struct VMCrossPageData *crosspageSYSV);
+extern const VMCrossPageData cpDataTemplate;
+
+extern void SwitchDBHandler(void);
+extern void SwitchUDHandler(void);
+extern void SwitchNMIHandler(void);
+extern void SwitchMCEHandler(void);
+extern void CrossPage_CodePage(void);
+extern void CrossPage_CodeEnd(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,7 +27,7 @@
 #   define __HOST_KERNEL_H__


-#   ifdef linux
+#   ifdef __linux__
 /*
 * In some cases, this files needs to include Linux kernel header file
 * asm/page.h.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,9 +59,18 @@

 #define CROSSCALL_SLEEP_US 1000

-EXTERN int   HostIF_Init(VMDriver *vm, uint32 numVCPUs);
+typedef struct HostIFContigMemMap {
+   MPN mpn;
+   void *addr;
+   PageCnt pages;
+   struct HostIFContigMemMap *next;
+} HostIFContigMemMap;
+
+EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
-EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);
+
+EXTERN MPN   HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData);
+EXTERN MPN   HostIF_GetCrossPageCodeMPN(void);

 EXTERN void *HostIF_AllocPage(void);
 EXTERN void  HostIF_FreePage(void *ptr);
@@ -76,28 +85,32 @@ EXTERN Bool  HostIF_IsLockedByMPN(VMDriver *vm, MPN mpn);
 EXTERN void  HostIF_FreeAllResources(VMDriver *vm);
 EXTERN uint64 HostIF_ReadUptime(void);
 EXTERN uint64 HostIF_UptimeFrequency(void);
-EXTERN unsigned int HostIF_EstimateLockedPageLimit(const VMDriver *vm,
-                                                   unsigned int lockedPages);
+EXTERN PageCnt HostIF_EstimateLockedPageLimit(const VMDriver *vm,
+                                              PageCnt lockedPages);
 EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
-EXTERN void *HostIF_AllocKernelPages(unsigned numPages, MPN *mpns);
-EXTERN void  HostIF_FreeKernelPages(unsigned numPages, void *ptr);
+EXTERN void *HostIF_AllocKernelPages(PageCnt numPages, MPN *mpns);
+EXTERN void  HostIF_FreeKernelPages(PageCnt numPages, void *ptr);
+EXTERN HostIFContigMemMap *HostIF_AllocContigPages(VMDriver *vm,
+                                                   PageCnt numPages);
+EXTERN void  HostIF_FreeContigPages(VMDriver *vm, HostIFContigMemMap *mapping);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
 EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 #endif

-EXTERN Bool  HostIF_APICInit(VMDriver *vm, Bool setVMPtr, Bool probe);
+EXTERN void  HostIF_APICInit(VMDriver *vm);
 EXTERN uint8 HostIF_GetMonitorIPIVector(void);
 EXTERN uint8 HostIF_GetHVIPIVector(void);
+EXTERN uint8 HostIF_GetPerfCtrVector(void);
 EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);

 EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
                                  Vcpuid vcpuid,
                                  uint64 *args);

-EXTERN int   HostIF_SemaphoreSignal(uint64 *args);
+EXTERN int   HostIF_SemaphoreSignal(VMDriver *vm, uint64 *args);

 EXTERN void  HostIF_SemaphoreForceWakeup(VMDriver *vm, const VCPUSet *vcs);
 EXTERN void  HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
@@ -111,11 +124,11 @@ EXTERN void HostIF_WaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_CancelWaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_WakeUpYielders(VMDriver *vm, Vcpuid currVcpu);

-EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                   unsigned int numPages, Bool kernelMPNBuffer);
-EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned int numPages, Bool kernelMPNBuffer);
+EXTERN int64 HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                     PageCnt numPages, Bool kernelMPNBuffer);
+EXTERN int HostIF_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn);
+EXTERN PageCnt HostIF_GetNumAnonPages(VMDriver *vm);
 EXTERN MPN HostIF_AllocLowPage(VMDriver *vm);

 EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr,
@@ -29,9 +29,10 @@
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
+#include "vm_basic_types.h"


-void *HostIF_AllocKernelMem(size_t size, int nonPageable);
+void *HostIF_AllocKernelMem(size_t size, Bool nonPageable);
 void  HostIF_FreeKernelMem(void *ptr);

 #endif // ifdef _HOSTIFMEM_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -96,7 +96,7 @@
 * Modify this value to increase the maximum number of tracked pages
 * per MemTrack instance.
 */
-#define MEMTRACK_MAX_TRACKED        MBYTES_2_PAGES(4096)
+#define MEMTRACK_MAX_TRACKED        GBYTES_2_PAGES(119)

 /*
 * Linux uses a 3-level directory, because we want to keep allocations
@@ -148,7 +148,7 @@ typedef uint64 MemTrackHTKey;

 typedef struct MemTrack {
   VMDriver         *vm;            /* The VM instance. */
-   unsigned          numPages;      /* Number of pages tracked. */
+   PageCnt           numPages;      /* Number of pages tracked. */
   MemTrackDir1      dir1;          /* First level directory. */
   MemTrackHT        vpnHashTable;  /* VPN to entry hashtable. */
   MemTrackHT       *mpnHashTable;  /* MPN to entry hashtable. */
@@ -197,7 +197,7 @@ MemTrackAllocPage(void)

 #define MEMTRACK_ALLOCDFN(_name, _itype, _otype)   \
   static INLINE _otype *                          \
-   _name(_itype *arg, unsigned pos)                \
+   _name(_itype *arg, uint64 pos)                  \
   {                                               \
      if (arg->dir[pos] == NULL) {                 \
         arg->dir[pos] = MemTrackAllocPage();      \
@@ -273,8 +273,8 @@ MemTrackHTInsert(MemTrackHT *ht,          // IN
 static void
 MemTrackCleanup(MemTrack *mt)    // IN
 {
-   unsigned idx;
-   unsigned p1;
+   PageCnt idx;
+   uint64 p1;
   MemTrackDir1 *dir1;
   Bool freeBackMap = mt != NULL && mt->mpnHashTable != NULL;

@@ -284,7 +284,7 @@ MemTrackCleanup(MemTrack *mt)    // IN
   dir1 = &mt->dir1;

   for (p1 = 0; p1 < MEMTRACK_DIR1_ENTRIES; p1++) {
-      unsigned p2;
+      uint64 p2;
      MemTrackDir2 *dir2 = MEMTRACK_GETDIR2(dir1, p1);

      if (dir2 == NULL) {
@@ -339,7 +339,7 @@ MemTrack *
 MemTrack_Init(VMDriver *vm) // IN:
 {
   MemTrack *mt;
-   unsigned idx;
+   PageCnt idx;

 #if defined(MEMTRACK_3LEVEL)
   ASSERT_ON_COMPILE(sizeof *mt <= PAGE_SIZE);
@@ -359,7 +359,8 @@ MemTrack_Init(VMDriver *vm) // IN:
      MemTrackHTPage *htPage = MemTrackAllocPage();

      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate VPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate VPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->vpnHashTable.pages[idx] = htPage;
@@ -374,7 +375,8 @@ MemTrack_Init(VMDriver *vm) // IN:
   for (idx = 0; idx < MEMTRACK_HT_PAGES; idx++) {
      MemTrackHTPage *htPage = MemTrackAllocPage();
      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate MPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate MPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->mpnHashTable->pages[idx] = htPage;
@@ -409,8 +411,8 @@ MemTrack_Add(MemTrack *mt,    // IN
             VPN64 vpn,       // IN
             MPN mpn)         // IN
 {
-   unsigned idx = mt->numPages;
-   unsigned p1, p2, p3;
+   PageCnt idx = mt->numPages;
+   uint64 p1, p2, p3;
   MemTrackEntry *ent;
   MemTrackDir1 *dir1 = &mt->dir1;
   MemTrackDir2 *dir2;
@@ -535,16 +537,16 @@ MemTrack_LookupMPN(MemTrack *mt, // IN
 *----------------------------------------------------------------------
 */

-unsigned
+PageCnt
 MemTrack_Cleanup(MemTrack *mt,            // IN
                 MemTrackCleanupCb *cb,   // IN
                 void *cData)             // IN
 {
-   unsigned idx;
-   unsigned count = 0;
+   PageCnt idx;
+   PageCnt count = 0;

   for (idx = 0; idx < mt->numPages; idx++) {
-      unsigned p1, p2, p3;
+      uint64 p1, p2, p3;
      MemTrackEntry *ent;
      MemTrackDir1 *dir1 = &mt->dir1;
      MEMTRACK_IDX2DIR(idx, p1, p2, p3);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -43,8 +43,8 @@ typedef struct MemTrackEntry {
 typedef void (MemTrackCleanupCb)(void *cData, MemTrackEntry *entry);

 extern struct MemTrack *MemTrack_Init(VMDriver *vm);
-extern unsigned MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
-                                 void *cbData);
+extern PageCnt MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
+                                void *cbData);
 extern MemTrackEntry *MemTrack_Add(struct MemTrack *mt, VPN64 vpn, MPN mpn);
 extern MemTrackEntry *MemTrack_LookupVPN(struct MemTrack *mt, VPN64 vpn);
 extern MemTrackEntry *MemTrack_LookupMPN(struct MemTrack *mt, MPN mpn);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,16 +24,15 @@
 *    Depending on configuration phystracker provides either 2-level or
 *    3-level structure to track whether page (specified by its MPN) is
 *    locked or no.  Linux uses 3-level structures with top limit of
- *    1TB (32bit) or 16TB (64bit).  Windows use 2-level structures
- *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.  On Mac
- *    limit is 2-level 128GB.
+ *    1TB (32bit) or 16TB (64bit).  Windows and Mac use 2-level structures
+ *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.
 *
 *    2-level phystracker is built on top of 3-level one by collapsing
 *    middle level.
 */


-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -50,9 +49,7 @@
 #define BYTES_PER_ENTRY      (PAGE_SIZE)
 #define PHYSTRACK_L3_ENTRIES (8 * BYTES_PER_ENTRY) /* 128MB */

-#if defined(WINNT_DDK)
-#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
-#elif defined(__linux__)
+#if defined(__linux__)
 #define PHYSTRACK_L2_ENTRIES (BYTES_PER_ENTRY / sizeof(void *)) /* 64GB or 128GB */
 /*
 * Currently MPN is 32 bits.  15 bits are in L3, 9 bits are in L2,
@@ -60,7 +57,7 @@
 */
 #define PHYSTRACK_L1_ENTRIES (256) /* 16TB. */
 #else
-#define PHYSTRACK_L1_ENTRIES ((128 + 4) * 8) /* 128 GB */
+#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
 #endif

 #ifndef PHYSTRACK_L2_ENTRIES
@@ -493,3 +490,32 @@ PhysTrack_GetNext(const PhysTracker *tracker, // IN
   return INVALID_MPN;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PhysTrack_GetNumTrackedPages --
+ *
+ *      Returns the total number of tracked pages
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+PageCnt
+PhysTrack_GetNumTrackedPages(const PhysTracker *tracker)
+{
+   PageCnt numTrackedMPNs = 0;
+   MPN nextMPN;
+   ASSERT(tracker);
+   ASSERT(HostIF_VMLockIsHeld(tracker->vm));
+   nextMPN = PhysTrack_GetNext(tracker, INVALID_MPN);
+   while (nextMPN != INVALID_MPN) {
+      numTrackedMPNs++;
+      nextMPN = PhysTrack_GetNext(tracker, nextMPN);
+   }
+   return numTrackedMPNs;
+}
+
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined(WINNT_DDK)
+#if !defined (__linux__)
 /* XXX: can be more efficient based on server vs. desktop and version of Windows */
 #define PHYSTRACK_MAX_SUPPORTED_GB (2048 + 4) /* 2 TB 64-bit W2k8 + 4 GB PCI */
 #endif
@@ -44,6 +44,7 @@ EXTERN void PhysTrack_Add(struct PhysTracker *, MPN);
 EXTERN void PhysTrack_Remove(struct PhysTracker *, MPN);
 EXTERN Bool PhysTrack_Test(const struct PhysTracker *, MPN);
 EXTERN MPN PhysTrack_GetNext(const struct PhysTracker *, MPN);
+EXTERN PageCnt PhysTrack_GetNumTrackedPages(const struct PhysTracker *);

 #endif

@@ -0,0 +1,299 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.c --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "sharedAreaVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmonIsMultiVCPU --
+ *
+ *      Helper function for determining if a shared area type represents
+ *      a region that has VCPU-specific memory.
+ *
+ * Results:
+ *      TRUE if type has VCPU-specific memory, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+SharedAreaVmmonIsMultiVCPU(SharedAreaType type)
+{
+   switch (type) {
+   case SHARED_AREA_PER_VM:
+   case SHARED_AREA_PER_VM_VMX:
+      return FALSE;
+   default:
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Init --
+ *
+ *      Initializes the shared area component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon shared area.
+ *
+ *----------------------------------------------------------------------
+ */
+
+SharedAreaVmmon *
+SharedAreaVmmon_Init(VMDriver *vm)
+{
+   SharedAreaVmmon *sa;
+
+   /*
+    * Allocate the shared area pointer here. The pages in each region are
+    * lazily allocated when a region is registered.
+    */
+   sa = HostIF_AllocKernelMem(sizeof *sa, FALSE);
+   if (sa == NULL) {
+      Warning("SharedArea failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(sa, 0, sizeof *sa);
+   return sa;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Cleanup --
+ *
+ *      Cleans up the shared area component of the VM Driver by freeing all
+ *      previously allocated VMMon shared area memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+SharedAreaVmmon_Cleanup(SharedAreaVmmon *area)
+{
+   SharedAreaType t;
+
+   if (area == NULL) {
+      return;
+   }
+
+   for (t = 0; t < NUM_SHARED_AREAS; t++) {
+      SharedAreaVmmonRegion *region = &area->regions[t];
+
+      if (region->pages != NULL) {
+         HostIF_FreeKernelMem(region->pages);
+      }
+   }
+   HostIF_FreeKernelMem(area);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_ValidateRegionArgs --
+ *
+ *      Validate the given VMMon shared area region registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register a shared
+ *      area region for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_ValidateRegionArgs(VMDriver *vm,
+                                   VMSharedAreaRegistrationBlock *block)
+{
+   SharedAreaVmmonRegion *region;
+   Vcpuid vcpu         = block->vcpu;
+   SharedAreaType type = block->region.index;
+   PageCnt numPages    = block->region.numPages;
+
+   if (vm == NULL               ||
+       vm->sharedArea == NULL   ||
+       vcpu >= vm->numVCPUs     ||
+       numPages == 0            ||
+       type >= NUM_SHARED_AREAS ||
+       (!SharedAreaVmmonIsMultiVCPU(type) && vcpu > 0)) {
+      return FALSE;
+   }
+
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu != 0) {
+
+      /*
+       * A region that was previously registered for a given VCPU should have
+       * the same number of pages as originally specified. Also, a region can
+       * only be reserved once for a given VCPU throughout the runtime of a VM.
+       */
+      return region->pagesPerVcpu == numPages &&
+             region->pages[vcpu * region->pagesPerVcpu] == INVALID_MPN;
+   } else {
+      /* First registration for this region. */
+      ASSERT(region->pages == NULL);
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_RegisterRegion --
+ *
+ *      Register a VMMon shared area region for a given VCPU with the VM Driver.
+ *      A successful return value indicates that the backing pages of the
+ *      shared area region are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the region was successfully registered for the given VCPU,
+ *      FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_RegisterRegion(VMDriver *vm,
+                               VMSharedAreaRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   SharedAreaType type = block->region.index;
+   SharedAreaVmmonRegion *region;
+   MPN *pages;
+   PageCnt page;
+
+   ASSERT(SharedAreaVmmon_ValidateRegionArgs(vm, block));
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu == 0) {
+      PageCnt pagesInRegion = block->region.numPages;
+
+      if (SharedAreaVmmonIsMultiVCPU(type)) {
+         pagesInRegion *= vm->numVCPUs;
+      }
+      region->pages = HostIF_AllocKernelMem(sizeof *pages * pagesInRegion,
+                                            FALSE);
+      if (region->pages == NULL) {
+         Warning("Failed to allocate pages array for region %u\n", type);
+         return FALSE;
+      }
+      for (page = 0; page < pagesInRegion; page++) {
+         region->pages[page] = INVALID_MPN;
+      }
+      region->pagesPerVcpu = block->region.numPages;
+   }
+
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+
+   /*
+    * Lock all shared area backing pages throughout the runtime of the monitor.
+    * The pages remain locked until they are freed by HostIF during VM Driver
+    * tear-down.
+    */
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      VPN vmxVPN = block->region.baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, pages + page);
+
+      if (status != PAGE_LOCK_SUCCESS) {
+         PageCnt resetPage;
+
+         /* Reset the region pages for this VCPU to their original state. */
+         for (resetPage = 0; resetPage < page; resetPage++) {
+            status = Vmx86_UnlockPage(vm, uAddr);
+            ASSERT(status == PAGE_UNLOCK_SUCCESS);
+            pages[resetPage] = INVALID_MPN;
+         }
+         return FALSE;
+      }
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_GetRegionMPN --
+ *
+ *      For the shared area region corresponding to the given type and VCPU,
+ *      get the MPN backing the region at the given offset.
+ *
+ * Results:
+ *      The MPN backing the region, or INVALID_MPN if no MPN is found.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+SharedAreaVmmon_GetRegionMPN(VMDriver *vm, SharedAreaVmmonRequest *request)
+{
+   SharedAreaType type = request->type;
+   Vcpuid vcpu = request->vcpu;
+   SharedAreaVmmonRegion *region;
+   PageCnt pgOffset = request->offset;
+   MPN *pages;
+
+   ASSERT(vcpu < vm->numVCPUs);
+   ASSERT(type < NUM_SHARED_AREAS);
+   ASSERT(IMPLIES(!SharedAreaVmmonIsMultiVCPU(type), vcpu == 0));
+   region = &vm->sharedArea->regions[type];
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   if (region->pages == NULL || region->pagesPerVcpu == 0) {
+      Warning("(%s) Requested unregistered region %u, VCPU %u\n", __FUNCTION__,
+              type, vcpu);
+      return INVALID_MPN;
+   }
+   if (pgOffset >= region->pagesPerVcpu) {
+      Warning("(%s) Offset %"FMT64"u (type %u, per-VCPU size %"FMT64"u)\n",
+              __FUNCTION__, pgOffset, type, region->pagesPerVcpu);
+      return INVALID_MPN;
+   }
+
+   return pages[pgOffset];
+}
@@ -0,0 +1,282 @@
+/*********************************************************
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * statVarsVmmon.c --
+ *
+ *     VMMon stat variables management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "statVarsVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Init --
+ *
+ *      Initializes the stat vars component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon stat vars struct.
+ *
+ *----------------------------------------------------------------------
+ */
+
+StatVarsVmmon *
+StatVarsVmmon_Init(VMDriver *vm)
+{
+   StatVarsVmmon *statVars;
+
+   statVars = HostIF_AllocKernelMem(sizeof *statVars, FALSE);
+   if (statVars == NULL) {
+      Warning("StatVars failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(statVars, 0, sizeof *statVars);
+   return statVars;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Cleanup --
+ *
+ *      Cleans up the stat vars component of the VM Driver by freeing all
+ *      previously allocated VMMon stat vars memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+StatVarsVmmon_Cleanup(StatVarsVmmon *statVars)
+{
+   if (statVars == NULL) {
+      return;
+   }
+
+   if (statVars->pages != NULL) {
+      HostIF_FreeKernelMem(statVars->pages);
+   }
+   HostIF_FreeKernelMem(statVars);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmonValidateRegistration --
+ *
+ *      Validate the given VMMon stat vars registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register stat vars
+ *      backing pages for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+StatVarsVmmonValidateRegistration(VMDriver *vm,
+                                   VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu      = block->vcpu;
+   PageCnt numPages = block->numPages;
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+
+   if (vm == NULL           ||
+       vm->statVars == NULL ||
+       numPages == 0        ||
+       vcpu >= vm->numVCPUs) {
+      return FALSE;
+   }
+
+   pagesPerVcpu = vm->statVars->pagesPerVcpu;
+   pages = vm->statVars->pages;
+   if (pagesPerVcpu == 0) {
+      ASSERT(pages == NULL);
+      return TRUE;
+   } else {
+      return pagesPerVcpu == numPages &&
+             pages[vcpu * pagesPerVcpu] == INVALID_MPN;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_RegisterVCPU --
+ *
+ *      Register stat vars backing pages for the given VCPU with VMMon.
+ *      A successful return value indicates that stat vars backing pages
+ *      for the given VCPU are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the backing pages were successfully registered for the
+ *      given VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+StatVarsVmmon_RegisterVCPU(VMDriver *vm,
+                           VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   PageCnt pagesPerVcpu = block->numPages;
+   StatVarsVmmon *statVars = vm->statVars;
+   MPN *pages;
+   PageCnt page;
+
+   HostIF_VMLock(vm, 45);
+   if (!StatVarsVmmonValidateRegistration(vm, block)) {
+      HostIF_VMUnlock(vm, 45);
+      return FALSE;
+   }
+   if (statVars->pagesPerVcpu == 0) {
+      PageCnt totalPages = pagesPerVcpu * vm->numVCPUs;
+      statVars->pages = HostIF_AllocKernelMem(totalPages * sizeof *pages,
+                                              FALSE);
+      if (statVars->pages == NULL) {
+         HostIF_VMUnlock(vm, 45);
+         return FALSE;
+      }
+      for (page = 0; page < totalPages; page++) {
+         statVars->pages[page] = INVALID_MPN;
+      }
+      statVars->pagesPerVcpu = pagesPerVcpu;
+   }
+
+   /*
+    * Validate that all pages in this VCPU's region are still invalid. Before
+    * unlocking, temporarily set the first page to 0 to prevent a malicious user
+    * from triggering an ASSERT by firing parallel ioctls for the same VCPU.
+    */
+   pages = statVars->pages + vcpu * pagesPerVcpu;
+   for (page = 0; page < pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+   ASSERT_ON_COMPILE(INVALID_MPN != 0);
+   pages[0] = 0;
+   HostIF_VMUnlock(vm, 45);
+
+   /*
+    * Verify that all stat vars backing pages are locked.  The pages are
+    * assumed to remain locked either until they are unlocked by the VMX
+    * following a NUMA migration, or by HostIF during VM Driver tear-down.
+    */
+   for (page = 0; page < pagesPerVcpu; page++) {
+      VPN vmxVPN = block->baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      PageCnt resetPage;
+      MPN *currPage = pages + page;
+      Bool verifiedLocked = FALSE;
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, currPage);
+
+      /*
+       * Backing pages are expected to be locked by the VMX before being
+       * registered.
+       */
+      if (status == PAGE_LOCK_ALREADY_LOCKED) {
+         verifiedLocked = TRUE;
+         status = Vmx86_LookupUserMPN(vm, uAddr, currPage);
+         /*
+          * PR 2260615: Some platforms return PAGE_LOCK_SUCCESS for a successful
+          * user MPN lookup, others return PAGE_LOOKUP_SUCCESS, but they both
+          * happen to equal the same value.
+          */
+         ASSERT_ON_COMPILE(PAGE_LOOKUP_SUCCESS == PAGE_LOCK_SUCCESS);
+         if (status == PAGE_LOOKUP_SUCCESS) {
+            /*
+             * The backing page was verified as previously locked, and was
+             * successfully recorded.
+             */
+            continue;
+         }
+      }
+      /*
+       * The backing page was either not previously locked, or was not recorded.
+       * In either case, reset all previously recorded pages to INVALID_MPN.
+       */
+      if (!verifiedLocked && status == PAGE_LOCK_SUCCESS) {
+         status = Vmx86_UnlockPage(vm, uAddr);
+         ASSERT(status == PAGE_UNLOCK_SUCCESS);
+      }
+      for (resetPage = 0; resetPage <= page; resetPage++) {
+         pages[resetPage] = INVALID_MPN;
+      }
+      return FALSE;
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_GetRegionMPN --
+ *
+ *      Obtain the backing MPN for stat vars for the given VCPU at
+ *      the given offset.
+ *
+ * Results:
+ *      The backing stat vars MPN for the given VCPU at the given offset
+ *      if it has been registered, or INVALID_MPN otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpu, PageCnt offset)
+{
+   MPN *pages;
+   MPN backingPage;
+   StatVarsVmmon *statVars = vm->statVars;
+
+   ASSERT(vcpu < vm->numVCPUs);
+
+   HostIF_VMLock(vm, 46);
+   if (offset >= statVars->pagesPerVcpu) {
+      HostIF_VMUnlock(vm, 46);
+      Warning("(%s) Requested MPN at invalid offset %"FMT64"u for VCPU %u\n",
+              __FUNCTION__, offset, vcpu);
+      return INVALID_MPN;
+   }
+
+   pages = statVars->pages + vcpu * statVars->pagesPerVcpu;
+   backingPage = pages[offset];
+   HostIF_VMUnlock(vm, 46);
+   return backingPage;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,11 +24,12 @@
 #include "includeCheck.h"
 #include "bootstrap_vmm.h"

-struct InitBlock;
+struct MonLoaderHeader;

 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
-extern int Task_InitCrosspage(VMDriver *vm, struct InitBlock *params);
-extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
+extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
+                               PerVcpuPages *perVcpuPages);
+extern Bool Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
 extern MPN Task_GetHVRootPageForPCPU(uint32 pCPU);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,13 +32,14 @@
 #include "x86msr.h"
 #include "modulecall.h"
 #include "vcpuid.h"
-#include "initblock.h"
 #include "iocontrols.h"
 #include "numa_defs.h"
 #include "rateconv.h"
 #include "vmmem_shared.h"
 #include "apic.h"
 #include "bootstrap_vmm.h"
+#include "sharedAreaVmmon.h"
+#include "statVarsVmmon.h"

 typedef struct TSCDelta {
   Atomic_uint64 min;
@@ -47,6 +48,10 @@ typedef struct TSCDelta {

 struct VmmBlobInfo;

+struct HostIFContigMemMap;
+
+extern struct HostIFContigMemMap *hvIOBitmap;
+
 /*
 * VMDriver - the main data structure for the driver side of a
 *            virtual machine.
@@ -55,23 +60,19 @@ struct VmmBlobInfo;
 typedef struct VMDriver {
   /* Unique (in the driver), strictly positive, VM ID used by userland. */
   int                     userID;
-
   Vcpuid                  numVCPUs;         /* Number of vcpus in VM. */
-
   struct VMDriver        *nextDriver;       /* Next on list of all VMDrivers */
-
   struct VMHost          *vmhost;           /* Host-specific fields. */
-
   MPN                    *ptRootMpns;       /* numVCPUs-sized array. */
   struct VmmBlobInfo     *blobInfo;         /* VMM bootstrap blob info. */
-
+   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
+   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
-   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
+   struct VMCrossPageData **crosspage;        /* numVCPUs-sized array. */
   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
   APICDescriptor          hostAPIC;
-
   struct MemTrack        *memtracker;       /* Memory tracker pointer */
   Bool                    checkFuncFailed;
   struct PerfCounter     *perfCounter;
@@ -81,6 +82,13 @@ typedef struct VMDriver {
   Atomic_uint64           ptscLatest;
   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
+   PageCnt                 numPTPPages;      /* Num PTP pages allocated. */
+   /*
+    * List of physically contiguous allocations associated with this VM.
+    * Access is protected by the VM lock.
+    */
+   struct HostIFContigMemMap     *contigMappings;
+
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -110,7 +118,7 @@ typedef struct PseudoTSC {

 extern PseudoTSC pseudoTSC;

-#define MAX_LOCKED_PAGES (-1)
+#define MAX_LOCKED_PAGES MAX_PPN

 extern void Vmx86_CacheNXState(void);
 extern VMDriver *Vmx86_CreateVM(VA64 bsBlob,
@@ -121,11 +129,10 @@ extern Bool Vmx86_ProcessBootstrap(VMDriver *vm,
                                   uint32 numBytes,
                                   uint32 headerOffset,
                                   uint16 numVCPUs,
-                                   UserVA64 *ptRootVAs,
+                                   PerVcpuPages *perVcpuPages,
                                   VMSharedRegion *shRegions);
 extern int Vmx86_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 extern int Vmx86_ReleaseVM(VMDriver *vm);
-extern int Vmx86_InitVM(VMDriver *vm, InitBlock *initParams);
 extern int Vmx86_LateInitVM(VMDriver *vm);
 extern int Vmx86_RunVM(VMDriver *vm, Vcpuid vcpuid);
 extern void Vmx86_YieldToSet(VMDriver *vm, Vcpuid currVcpu, const VCPUSet *req,
@@ -142,22 +149,21 @@ extern int Vmx86_UnlockPage(VMDriver *vm, VA64 uAddr);
 extern int Vmx86_UnlockPageByMPN(VMDriver *vm, MPN mpn, VA64 uAddr);
 extern MPN Vmx86_GetRecycledPage(VMDriver *vm);
 extern int Vmx86_ReleaseAnonPage(VMDriver *vm, MPN mpn);
-extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned numPages, Bool kernelMPNBuffer,
-                                  Bool ignoreLimits);
-extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                 unsigned numPages, Bool kernelMPNBuffer);
+extern int64 Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                    PageCnt numPages, Bool kernelMPNBuffer,
+                                    Bool ignoreLimits);
+extern int Vmx86_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn);
+extern MPN Vmx86_GetNumAnonPages(VMDriver *vm);
 extern MPN Vmx86_AllocLowPage(VMDriver *vm, Bool ignoreLimits);
 extern void *Vmx86_Calloc(size_t numElements,
                          size_t elementSize,
-                          int nonPageable);
+                          Bool nonPageable);
 extern void  Vmx86_Free(void *ptr);

 extern int32 Vmx86_GetNumVMs(void);
-extern int32 Vmx86_GetTotalMemUsage(void);
-extern Bool Vmx86_SetConfiguredLockedPagesLimit(unsigned limit);
-extern void Vmx86_SetDynamicLockedPagesLimit(unsigned limit);
+extern Bool Vmx86_SetConfiguredLockedPagesLimit(PageCnt limit);
+extern void Vmx86_SetDynamicLockedPagesLimit(PageCnt limit);
 extern Bool Vmx86_GetMemInfo(VMDriver *curVM,
                             Bool curVMOnly,
                             VMMemInfoArgs *args,
@@ -169,9 +175,12 @@ extern void Vmx86_UpdateMemInfo(VMDriver *curVM,
 extern void Vmx86_Add2MonPageTable(VMDriver *vm, VPN vpn, MPN mpn,
                                   Bool readOnly);
 extern Bool Vmx86_GetAllMSRs(MSRQuery *query);
+extern Bool Vmx86_CheckMSRUniformity(void);
 extern void Vmx86_FlushVMCSAllCPUs(MA vmcs);
 extern void Vmx86_MonTimerIPI(void);
 extern void Vmx86_InitIDList(void);
+extern Bool Vmx86_CreateHVIOBitmap(void);
+extern void Vmx86_CleanupHVIOBitmap(void);
 extern Bool Vmx86_GetPageRoot(VMDriver *vm, Vcpuid vcpuid, MPN *mpn);
 extern void Vmx86_Open(void);
 extern void Vmx86_Close(void);
@@ -0,0 +1,319 @@
+/*********************************************************
+ * Copyright (C) 2007-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _ADDRLAYOUT_H_
+#define _ADDRLAYOUT_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "address_defs.h"
+#include "x86desc.h"
+#include "x86types.h"
+#include "vmm_constants.h"
+
+#define DIRECT_EXEC_USER_RPL    3
+#define BINARY_TRANSLATION_RPL  1
+typedef char x86_PAGE[PAGE_SIZE];
+
+#define MAX_VADDR                    CONST64U(0xffffffffffffffff)
+#define MONITOR_SIZE                 (64 * 1024 * 1024)
+#define MONITOR_LINEAR_START         (MAX_VADDR - MONITOR_SIZE + 1)
+#define MONITOR_LINEAR_END           (MAX_VADDR)
+#define NUM_MONITOR_PAGES            (MONITOR_SIZE / PAGE_SIZE)
+#define MONITOR_BASE_VPN             (MONITOR_LINEAR_START >> PAGE_SHIFT)
+
+#define MONITOR_AS_START             MONITOR_BASE_VPN
+#define MONITOR_AS_LEN               NUM_MONITOR_PAGES
+
+/*
+ * Regions and items necessarily overlap.  Build parallel structures for each.
+ * These 'field' structures are used to determine page numbers and offsets.
+ * AddrLayout contains zero-length regions used for alignment-checking.  (These
+ * are used by gcc as Windows prohibits zero-length arrays.)
+ */
+
+/* Item layout. */
+typedef struct {
+#if defined __GNUC__
+#define REGION(name, length) x86_PAGE name##_REGION_MARKER_FIELD[0];
+#else
+#define REGION(name, length)
+#endif
+#define ITEM(name, length) x86_PAGE name##_FIELD[length];
+#include "addrlayout_table.h"
+} AddrLayout;
+#undef REGION
+#undef ITEM
+#define FIELD_PAGE_NUMBER(name) PAGE_NUMBER(&(((AddrLayout *)0)->name##_FIELD))
+
+
+/* Region layout. */
+typedef struct {
+#define REGION(name, length) x86_PAGE name##_REGION_FIELD[length];
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+} RegionAddrLayout;
+#undef REGION
+#undef ITEM
+#define REGION_FIELD_PAGE_NUMBER(name) \
+   PAGE_NUMBER(&(((RegionAddrLayout *)0)->name##_REGION_FIELD))
+
+#if defined __GNUC__
+/*
+ * The sum of sizes of items in a region must add up to the region's size.
+ * AddrLayout contains zero-sized region markers at their correct offsets.
+ * RegionAddrLayout contains full-sized regions (but no items).  Comparing the
+ * marker and region offsets ensures that all items sizes in the previous
+ * region sum to the size of the previous region.  To check the final region
+ * (as it has no successor) we assert the sizes of the tables match.
+ *
+ * We also assert 2MB region granularity.
+ *
+ * This checking is only performed when using gcc.
+ */
+#define REGION(name, length) static INLINE void name(void) { \
+   ASSERT_ON_COMPILE(FIELD_PAGE_NUMBER(name##_REGION_MARKER) == \
+                     REGION_FIELD_PAGE_NUMBER(name)); \
+   ASSERT_ON_COMPILE((length % 512) == 0); \
+   ASSERT_ON_COMPILE(sizeof(AddrLayout) == sizeof(RegionAddrLayout)); \
+   ASSERT_ON_COMPILE(sizeof(RegionAddrLayout) == MONITOR_SIZE); \
+}
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+/*
+ * Sizes.
+ */
+
+enum {
+#define REGION(name, length) name##_REGION_LEN = length,
+#define ITEM(name, length) name##_LEN = length,
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+/*
+ * Starts.
+ */
+
+#if defined __GNUC__
+/* Windows compiler and gdb do not accept 64-bit enumerations, but gcc
+ * does.  These are used by geninfo.c.
+ */
+enum {
+#define REGION(name, length) _asm_##name##_REGION_START = \
+    0ULL + MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) _asm_##name##_START = \
+    0ULL + MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+enum {
+#define REGION(name, length) name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+#else
+#define REGION(name, length) \
+   static const uint64 name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name);
+#define ITEM(name, length) \
+   static const uint64 name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name);
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+enum {
+#define REGION(name, length) name##_REGION_start_page = \
+   REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_start_page = \
+   FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+
+/* Derive the bootstrap start address from table contents. */
+#define MONITOR_BOOTSTRAP_START_LA   VPN_2_VA(BS_TXT_START)
+
+#ifndef VMX86_SERVER
+#define SWITCH_PAGE_TABLE_LEN          12
+#endif
+
+#define VMM_STACK_TOP_HW          VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+/*
+ * VMM_STKTOP_HW_LEN needs to contain an ExcFrame64ForCopy (72 bytes) and
+ * some more bytes for a "no man's land" region.  We make sure the "no
+ * man's land" region is 0 on BT_Resume and HV_Resume.  Because we
+ * check "no man's land" in 8-byte chunks, VMM_STKTOP_HW_LEN must be
+ * divisible by 8.
+ */
+#define VMM_STKTOP_HW_LEN          168
+#define VMM_STKTOP_HW_LRET_LEN     (VMM_STKTOP_HW_LEN + sizeof(LretFrame64))
+#define VMM_STKTOP_LRET_OFFSET     (PAGE_SIZE - VMM_STKTOP_HW_LRET_LEN)
+
+#define MON_STACK_BASE            VPN_2_VA(MON_STACK_PAGES_START)
+#define MON_STACK_TOP             VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+#define DF_STACK_BASE              VPN_2_VA(DF_STACK_PAGES_START)
+#define DF_STACK_TOP               VPN_2_VA(DF_STACK_PAGES_START + \
+                                            DF_STACK_PAGES_LEN)
+#define MC_STACK_BASE              VPN_2_VA(MC_STACK_PAGES_START)
+#define MC_STACK_TOP               VPN_2_VA(MC_STACK_PAGES_START + \
+                                            MC_STACK_PAGES_LEN)
+#define NMI_STACK_BASE             VPN_2_VA(NMI_STACK_PAGES_START)
+#define NMI_STACK_TOP              VPN_2_VA(NMI_STACK_PAGES_START + \
+                                            NMI_STACK_PAGES_LEN)
+
+#define NUM_MON_PTABS   MON_PAGE_TABLE_L1_LEN
+
+#define MON_START_L5OFF PT_LA_2_L5OFF(MONITOR_LINEAR_START)
+#define MON_START_L4OFF PT_LA_2_L4OFF(MONITOR_LINEAR_START)
+#define MON_START_L3OFF PT_LA_2_L3OFF(MONITOR_LINEAR_START)
+#define MON_START_L2OFF PT_LA_2_L2OFF(MONITOR_LINEAR_START)
+
+/*
+ * Allocated wired pages for the monitor.
+ * Currently we allocate wired pages for the:
+ *  0) The shared area between user and monitor;
+ *  a) GDT of the monitor
+ *  a1) BT (CP1) stack
+ *  b) The monitor's page table with one page directory under 4 GB
+ *      and two page table pages.
+ *  c) Pages for the monitor's stack.
+ *  d) The BT area pages
+ *  e) The cross driver/monitor page.
+ *  f) The physical memory array for the machine.
+ *  g) The SMRAM memory
+ *
+ * We allocate the shared area separately since it must be read/write
+ * in the monitor.   See SharedArea_PowerOn().
+ */
+
+#define MON_BOOTSTRAP_PAGE_TABLE_PAGES (BOOTSTRAP_REGION_LEN     /  \
+                                        PAE_PTES_PER_PGTBL)
+#ifdef VMX86_SERVER
+#define NUM_MONWIRED_PAGES     0
+
+#define MON_PAGE_TABLE_ALLOC_PAGES 0
+#define NUM_MONWIRED_BOOTSTRAP_PAGES 0
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN  + \
+                                    DF_STACK_PAGES_LEN   + \
+                                    MC_STACK_PAGES_LEN   + \
+                                    NMI_STACK_PAGES_LEN  + \
+                                    HV_CURRENT_VMCB_LEN  + \
+                                    IDT_NORMAL_LEN       + \
+                                    GDT_AND_TASK_LEN)
+
+#else
+#define NUM_MONWIRED_PAGES    (CROSS_PAGE_CODE_LEN   + \
+                               CROSS_PAGE_DATA_LEN   + \
+                               MON_PAGE_TABLE_L4_LEN + \
+                               MON_PAGE_TABLE_L3_LEN + \
+                               MON_PAGE_TABLE_L2_LEN)
+
+#define MON_PAGE_TABLE_ALLOC_PAGES ((MONITOR_READONLY_REGION_LEN + \
+                                     MONITOR_DATA_REGION_LEN     + \
+                                     MONITOR_MISC_REGION_LEN)    / \
+                                     PAE_PTES_PER_PGTBL)
+
+#define NUM_MONWIRED_BOOTSTRAP_PAGES MON_BOOTSTRAP_PAGE_TABLE_PAGES
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN + \
+                                    DF_STACK_PAGES_LEN  + \
+                                    MC_STACK_PAGES_LEN  + \
+                                    NMI_STACK_PAGES_LEN + \
+                                    GDT_AND_TASK_LEN    + \
+                                    IDT_NORMAL_LEN      + \
+                                    HV_CURRENT_VMCB_LEN + \
+                                    SWITCH_PAGE_TABLE_LEN)
+#endif
+
+#define MON_PAGE_TABLE_L5        VPN_2_VA(MON_PAGE_TABLE_L5_START)
+#define MON_PAGE_TABLE_L4        VPN_2_VA(MON_PAGE_TABLE_L4_START)
+#define VMM_STACK_TOP            (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LEN)
+#define VMM_LRET_STACK_TOP       (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LRET_LEN)
+/*
+ * Mark out a guard page for the VMM stack, this is not in the addrlayout_table
+ * explicitly because it overlaps the last page of the MONITOR_READONLY region.
+ */
+#define VMM_STACK_GUARD_START    MON_STACK_PAGES_START
+#define VMM_STACK_GUARD_LEN      PAGE_SIZE
+
+/*
+ * Following functions return true if the range [va, va+len) is within
+ * range of a particular monitor stack.  (Don't adjust without
+ * considering uint64 overflow when va is very high.)
+ */
+static INLINE Bool
+AddrLayout_InMonStack(VA64 va, size_t len)
+{
+   return VPN_2_VA(MON_STACK_PAGES_START) <= va &&
+          va <= VMM_STACK_TOP_HW - len;
+}
+
+static INLINE Bool
+AddrLayout_InNMIStack(VA64 va, size_t len)
+{
+   return va >= NMI_STACK_BASE && va <= NMI_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InDFStack(VA64 va, size_t len)
+{
+   return va >= DF_STACK_BASE && va <= DF_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InMCStack(VA64 va, size_t len)
+{
+   return va >= MC_STACK_BASE && va <= MC_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InAMonitorStack(VA va, size_t len)
+{
+   return AddrLayout_InMonStack(va, len)  ||
+          AddrLayout_InNMIStack(va, len)  ||
+          AddrLayout_InDFStack(va, len)   ||
+          AddrLayout_InMCStack(va, len);
+}
+
+#endif
@@ -0,0 +1,144 @@
+/*********************************************************
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * addrlayout_table.h --
+ *
+ * Named elements in the monitor's address space of fixed size and location.
+ * All units are 4 kilobyte pages.  Also used for vmmlayout output.
+ *
+ * The monitor's address space is populated three different ways:
+ * 1) For elements of a fixed size and location, at compile-time.
+ * 2) For well-known program sections and the shared area, at link-time.
+ * 3) Dynamically, by the allocator, during bootstrapping and monitor power-on.
+ *
+ * This file describes all of the monitor's main 64MB address space.  All of
+ * (1) is described specifically.  Ranges for (2) and (3) are accounted for,
+ * but their contents are described generally here.
+ *
+ * The monitor's address space is organized into container regions which are an
+ * integer number of 2MB "large" pages.  Within a container region, items
+ * comprised of one or more 4KB pages may be described.
+ *
+ * All container regions and items are described by a name and a size in pages.
+ * No two items may have the same name nor may two regions.
+ *
+ * REGION(region-name, number-of-pages)
+ *   ITEM(item-name,   number-of-pages)     // first item in region
+ *   ITEM(item-name,   number-of-pages)     // second item in region
+ * <...>
+ */
+
+
+/*
+ * Monitor .rodata/.text.  Mapped large (reducing TLB pressure) and read-only.
+ */
+REGION(MONITOR_READONLY,        1024)
+  ITEM(MONITOR_READONLY_LINKER, 1024) // Used by linker.
+
+/*
+ * Monitor .data/.bss.  Mapped large (reducing TLB pressure) and read-write.
+ */
+REGION(MONITOR_DATA,            512)
+  ITEM(MON_STACK_PAGES,           8) // Monitor stack.
+  ITEM(IDT_NORMAL,                1) // Normal IDT (used for all but SVM).
+  ITEM(MONITOR_DATA,            503) // Used by linker and TC.
+
+/*
+ * The monitor's translation cache.  This object starts at the page boundary
+ * following the end of the used space in MONITOR_DATA and extends through
+ * the TC_REGION.
+ */
+REGION(TC_REGION,              1024)
+  ITEM(TC_BLOCK,               1024)
+
+/*
+ * The monitor's pages for architectural state, stacks, page tables, shared
+ * pages, and dynamic allocation.  This region also contains the linked shared
+ * area and other contents.
+ * Mapped small (allowing non-present mappings and sharing) and read-write.
+ */
+REGION(MONITOR_MISC,           5632)
+#ifdef VMX86_SERVER
+  ITEM(GUARD_PAGE,                2) // Reserved (for symmetry with hosted).
+#else
+  ITEM(CROSS_PAGE_DATA,           1) // Cross page data (RW).
+  ITEM(CROSS_PAGE_CODE,           1) // Cross page code (RX).
+#endif
+  ITEM(GDT_AND_TASK,              1) // GDT and Task State Segment.
+
+  ITEM(MON_PAGE_TABLE_L5,         1) // Monitor page root, if 5-Level PT used.
+  ITEM(MON_PAGE_TABLE_L4,         1) // Monitor page root, if 4-Level PT used.
+  ITEM(MON_PAGE_TABLE_L3,         1) // Monitor L3 page table.
+  ITEM(MON_PAGE_TABLE_L2,         1) // Monitor L2 page table.
+  ITEM(MON_PAGE_TABLE_L1,        32) // Monitor L1 page tables.
+
+  ITEM(HOST_APIC,                 1) // Physical APIC.
+  ITEM(GUEST_APIC,                1) // Guest APIC.
+
+  ITEM(DF_GUARD_PAGE,             1) // Double Fault stack guard page.
+  ITEM(DF_STACK_PAGES,            1) // Double Fault stack; need ~768 bytes.
+
+  ITEM(MC_GUARD_PAGE,             1) // Machine Check stack guard page.
+  ITEM(MC_STACK_PAGES,            1) // Machine Check stack.
+
+  ITEM(NMI_GUARD_PAGE,            1) // NMI stack guard page.
+  ITEM(NMI_STACK_PAGES,           4) // NMI stack.  Profiling requires a larger
+                                     // stack than MC/DF.
+
+  ITEM(HV_SWITCH,                 1) // SVM switch page
+  ITEM(HV_CURRENT_VMCB,           1) // SVM current VMCB
+  ITEM(HV_STD_NATIVE_VMCB,        1) // SVM/VT standard VMCB/VMCS
+  ITEM(HV_AUX_NATIVE_VMCB,        1) // SVM/VT auxiliary VMCB/VMCS
+  ITEM(VHV_GUEST_VMCB,            1) // VSVM/VVT guest VMCB/VMCS
+  ITEM(HV_SEV_VMSA,               1) // SVM SEV-ES state save area
+  ITEM(VPROBE_MON_RELOC,          1) // VProbe monitor reloc page.
+  ITEM(GART_LIST_PAGES,          48) // PhysMem vmm gart list pages.
+  ITEM(GART_ALT_LIST_PAGES,      48) // Alternate vmm gart pages
+                                     // (for invalidation).
+  ITEM(GART_BF_PAGES,             8) // Gart Bloom filter memory pages.
+  ITEM(VVT_GUEST_VIRT_APIC,       1) // Inner guest virtual APIC page.
+  ITEM(HT_STATE_MAP,              8) // Information used by the SecureHT module
+  ITEM(SHARED_RW_DATA,         5462) // R/W shared data, including the shared
+                                     // area and stat vars. Any remaining pages
+                                     // are left to the dynamic allocator, which
+                                     // extends throughout the entire reclaimed
+                                     // bootstrap region.
+/*
+ * Bootstrap-used space.  This space is reclaimed when bootstrap is done, and
+ * is used by the allocator thereafter.  The allocator starts at the first
+ * unused page in the SHARED_RW_DATA item from the MONITOR_MISC region and
+ * extends through to the end of the VMM address space once the bootstrap region
+ * is reclaimed.
+ */
+REGION(BOOTSTRAP,              8192)
+  ITEM(BS_TXT,                   17) // bootstrap's .text
+  ITEM(BS_RODATA,                10) // bootstrap's .rodata
+  ITEM(BS_DATA,                  10) // bootstrap's .data
+  ITEM(BS_BSS,                   10) // bootstrap's .bss
+  ITEM(BS_HEADER,                 1) // MonLoaderHeader mapping
+  ITEM(BS_VCPU_L1PT_RANGE,       32) // VCPU L1PT mappings
+  ITEM(BS_VCPU_L1PT_PT,           1) // L1PT that maps a VCPU's L1 page tables
+  ITEM(BS_INIT_POOL,              1) // pool for initializing VMM pages
+  ITEM(BS_DYNAMIC_ALLOC,       1880) // bs_alloc's dynamic allocator pages
+  ITEM(IDT_BOOTSTRAP,             1) // bootstrap IDT
+  ITEM(IDT_BOOTSTRAP_STUBS,       2) // bootstrap IDT gate stubs
+  ITEM(BS_PER_VM_VMX,           300) // vmm64's shared_per_vm_vmx
+  ITEM(BS_PER_VCPU,               8) // vmm64's shared_per_vcpu for VCPU 0
+  ITEM(BS_PER_VCPU_VMX,         136) // vmm64's shared_per_vcpu_vmx for VCPU 0
+  ITEM(VMM_MODULES,            5783) // ~22.6MB for unlinked VMM modules
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -345,6 +345,34 @@ CircList_Push(ListItem *p,               // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_InsertAfter --
+ *
+ *      Adds a new member to the list after the provided item.  Assumes p
+ *      is not a member of a list already.
+ *
+ * Result:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE void
+CircList_InsertAfter(ListItem *p,      // IN:
+                     ListItem *after)  // IN:
+{
+   p->prev = after;
+   p->next = after->next;
+   p->next->prev = p;
+   after->next = p;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,77 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _CPUID_INFO_H
-#define _CPUID_INFO_H
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-
-#include "includeCheck.h"
-
-#include "vm_basic_asm.h"
-#include "x86cpuid_asm.h"
-
-#if defined __cplusplus
-extern "C" {
-#endif
-
-
-typedef struct CPUID0 {
-   int numEntries;
-   char name[16];      // 4 extra bytes to null terminate
-} CPUID0;
-
-typedef struct CPUID1 {
-   uint32 version;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID1;
-
-typedef struct CPUID80 {
-   uint32 numEntries;
-   uint32 ebx;
-   uint32 ecx;
-   uint32 edx;
-} CPUID80;
-
-typedef struct CPUID81 {
-   uint32 eax;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID81;
-
-typedef struct CPUIDSummary {
-   CPUID0  id0;
-   CPUID1  id1;
-   CPUIDRegs ida;
-   CPUID80 id80;
-   CPUID81 id81;
-   CPUIDRegs id88, id8a;
-} CPUIDSummary;
-
-
-#if defined __cplusplus
-} // extern "C"
-#endif
-
-#endif // _CPUID_INFO_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,7 +37,6 @@
 #error iocontrols.h is for hosted vmmon, do not use on visor
 #endif

-#include "basic_initblock.h"
 #include "x86segdescrs.h"
 #include "rateconv.h"
 #include "overheadmem_types.h"
@@ -45,6 +44,8 @@
 #include "numa_defs.h"
 #include "bootstrap_vmm.h"
 #include "contextinfo.h"
+#include "vcpuid.h"
+#include "sharedAreaType.h"

 #if defined __cplusplus
 extern "C" {
@@ -142,9 +143,12 @@ PtrToVA64(void const *ptr) // IN
 * does not break vmcore compatibility.
 *
 * See bora/doc/vmcore details.
+ *
+ * If you increment VMMON_VERSION, you must also increment
+ * the NT specific VMX86_DRIVER_VERSION.
 */

-#define VMMON_VERSION           (360 << 16 | 0)
+#define VMMON_VERSION           (401 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -153,11 +157,8 @@ PtrToVA64(void const *ptr) // IN
 * ENOMEM returned after MAX_VMS virtual machines created
 */

-#ifdef VMX86_SERVER
-#define MAX_VMS 128
-#else
 #define MAX_VMS 64
-#endif
+
 /*
 * MsgWaitForMultipleObjects doesn't scale well enough on Win32.
 * Allocate with MAX_VMS so static buffers are large, but do
@@ -209,11 +210,13 @@ enum IOCTLCmd {
   IOCTLCMD(VERSION) = IOCTLCMD(FIRST),
   IOCTLCMD(CREATE_VM),
   IOCTLCMD(PROCESS_BOOTSTRAP),
+   IOCTLCMD(REGISTER_SHARED),
+   IOCTLCMD(REGISTER_STATVARS),
   IOCTLCMD(RELEASE_VM),
   IOCTLCMD(GET_NUM_VMS),
-   IOCTLCMD(INIT_VM),
   IOCTLCMD(RUN_VM),
   IOCTLCMD(LOOK_UP_MPN),
+   IOCTLCMD(GET_VMM_PAGE_ROOT),
   IOCTLCMD(LOCK_PAGE),
   IOCTLCMD(UNLOCK_PAGE),
   IOCTLCMD(APIC_INIT),
@@ -222,7 +225,6 @@ enum IOCTLCmd {
   IOCTLCMD(ADMIT),
   IOCTLCMD(UPDATE_MEM_INFO),
   IOCTLCMD(READMIT),
-   IOCTLCMD(GET_TOTAL_MEM_USAGE),
   IOCTLCMD(GET_KHZ_ESTIMATE),
   IOCTLCMD(SET_HOST_CLOCK_RATE),
   IOCTLCMD(READ_PAGE),
@@ -232,6 +234,7 @@ enum IOCTLCmd {
    /* AWE calls */
   IOCTLCMD(ALLOC_LOCKED_PAGES),
   IOCTLCMD(GET_NEXT_ANON_PAGE),
+   IOCTLCMD(GET_NUM_ANON_PAGES),

   IOCTLCMD(GET_ALL_MSRS),

@@ -245,6 +248,7 @@ enum IOCTLCmd {
   IOCTLCMD(GET_IPI_VECTORS),
   IOCTLCMD(SEND_IPI),
   IOCTLCMD(SEND_ONE_IPI),
+   IOCTLCMD(GET_SWITCH_ERROR_ADDR),

   /*
    * Keep host-specific calls at the end so they can be undefined
@@ -253,18 +257,10 @@ enum IOCTLCmd {

 #if defined __linux__
   IOCTLCMD(SET_UID),		// VMX86_DEVEL only
-#endif
-
-#if defined __linux__ || defined __APPLE__
   IOCTLCMD(GET_ALL_CPUID),
 #endif

-#if defined _WIN32 || defined __APPLE__
-   IOCTLCMD(ALLOC_CONTIG_PAGES),
-#endif
-
 #if defined _WIN32
-   IOCTLCMD(FREE_CONTIG_PAGES),
   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows
   IOCTLCMD(CHANGE_HARD_LIMIT),         // used by vmauthd on Windows
   IOCTLCMD(READ_DISASM_PROC_BINARY),
@@ -272,19 +268,16 @@ enum IOCTLCmd {
   IOCTLCMD(SET_MEMORY_PARAMS),
   IOCTLCMD(REMEMBER_KHZ_ESTIMATE),
   IOCTLCMD(REMAP_SCATTER_LIST),
+   IOCTLCMD(REMAP_SCATTER_LIST_RO),     // map the list as read-only
   IOCTLCMD(UNMAP_SCATTER_LIST),
 #endif

 #if defined __APPLE__
-   IOCTLCMD(ALLOC_LOW_PAGES),
+   IOCTLCMD(GET_NUM_RESPONDING_CPUS),
   IOCTLCMD(INIT_DRIVER),
   IOCTLCMD(BLUEPILL),
 #endif

-#if defined __linux__
-   IOCTLCMD(SET_HOST_SWAP_SIZE),
-#endif
-
   IOCTLCMD(GET_UNAVAIL_PERF_CTRS),
   IOCTLCMD(GET_MONITOR_CONTEXT),
   // Must be last.
@@ -317,13 +310,16 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_CREATE_VM           VMIOCTL_BUFFERED(CREATE_VM)
 #define IOCTL_VMX86_RELEASE_VM          VMIOCTL_BUFFERED(RELEASE_VM)
 #define IOCTL_VMX86_PROCESS_BOOTSTRAP   VMIOCTL_BUFFERED(PROCESS_BOOTSTRAP)
+#define IOCTL_VMX86_REGISTER_SHARED     VMIOCTL_BUFFERED(REGISTER_SHARED)
+#define IOCTL_VMX86_REGISTER_STATVARS   VMIOCTL_BUFFERED(REGISTER_STATVARS)
 #define IOCTL_VMX86_GET_NUM_VMS         VMIOCTL_BUFFERED(GET_NUM_VMS)
-#define IOCTL_VMX86_INIT_VM             VMIOCTL_BUFFERED(INIT_VM)
 #define IOCTL_VMX86_RUN_VM              VMIOCTL_NEITHER(RUN_VM)
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
+#define IOCTL_VMX86_GET_SWITCH_ERROR_ADDR VMIOCTL_BUFFERED(GET_SWITCH_ERROR_ADDR)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
+#define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
 #define IOCTL_VMX86_UNLOCK_PAGE         VMIOCTL_BUFFERED(UNLOCK_PAGE)
 #define IOCTL_VMX86_APIC_INIT           VMIOCTL_BUFFERED(APIC_INIT)
@@ -334,8 +330,6 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UPDATE_MEM_INFO     VMIOCTL_BUFFERED(UPDATE_MEM_INFO)
 #define IOCTL_VMX86_HARD_LIMIT_MONITOR_STATUS   VMIOCTL_BUFFERED(HARD_LIMIT_MONITOR_STATUS)
 #define IOCTL_VMX86_CHANGE_HARD_LIMIT   VMIOCTL_BUFFERED(CHANGE_HARD_LIMIT)
-#define IOCTL_VMX86_ALLOC_CONTIG_PAGES  VMIOCTL_BUFFERED(ALLOC_CONTIG_PAGES)
-#define IOCTL_VMX86_FREE_CONTIG_PAGES   VMIOCTL_BUFFERED(FREE_CONTIG_PAGES)

 #define IOCTL_VMX86_GET_TOTAL_MEM_USAGE	VMIOCTL_BUFFERED(GET_TOTAL_MEM_USAGE)
 #define IOCTL_VMX86_GET_KHZ_ESTIMATE    VMIOCTL_BUFFERED(GET_KHZ_ESTIMATE)
@@ -347,6 +341,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UNLOCK_PAGE_BY_MPN  VMIOCTL_BUFFERED(UNLOCK_PAGE_BY_MPN)
 #define IOCTL_VMX86_ALLOC_LOCKED_PAGES  VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES)
 #define IOCTL_VMX86_GET_NEXT_ANON_PAGE  VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE)
+#define IOCTL_VMX86_GET_NUM_ANON_PAGES  VMIOCTL_BUFFERED(GET_NUM_ANON_PAGES)

 #define IOCTL_VMX86_READ_DISASM_PROC_BINARY \
                                      VMIOCTL_BUFFERED(READ_DISASM_PROC_BINARY)
@@ -364,18 +359,14 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_GET_PSEUDO_TSC           VMIOCTL_NEITHER(GET_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS    VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS)
 #define IOCTL_VMX86_GET_MONITOR_CONTEXT      VMIOCTL_BUFFERED(GET_MONITOR_CONTEXT)
-#define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
-#define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST    VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST_RO VMIOCTL_BUFFERED(REMAP_SCATTER_LIST_RO)
+#define IOCTL_VMX86_UNMAP_SCATTER_LIST    VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
 #endif


-/*
- * Flags sent into APICBASE ioctl
- */
+#define INIT_BLOCK_MAGIC     (0x1789 + 14)

-#define APIC_FLAG_DISABLE_NMI       0x00000001
-#define APIC_FLAG_PROBE             0x00000002
-#define APIC_FLAG_FORCE_ENABLE      0x00000004

 typedef
 #include "vmware_pack_begin.h"
@@ -395,10 +386,14 @@ union {
 #include "vmware_pack_end.h"
 VMLockPage;

-
-typedef struct VMAPICInfo {
-   uint32 flags;
-} VMAPICInfo;
+typedef
+#include "vmware_pack_begin.h"
+union {
+   Vcpuid vcpuid; // IN: VCPU
+   MPN pageRoot;  // OUT: MPN of the VCPU's page root
+}
+#include "vmware_pack_end.h"
+VcpuPageRoot;

 #define VMX86_DRIVER_VCPUID_OFFSET 1000

@@ -413,9 +408,9 @@ typedef struct VMAPICInfo {
 * all these values at any given time.
 */
 typedef struct LockedPageLimit {
-   uint32 host;        // driver calculated maximum for this host
-   uint32 configured;  // user defined maximum pages to lock
-   uint32 dynamic;     // authd hardLimitMonitor pages to lock
+   PageCnt host;        // driver calculated maximum for this host
+   PageCnt configured;  // user defined maximum pages to lock
+   PageCnt dynamic;     // authd hardLimitMonitor pages to lock
 } LockedPageLimit;

 /*
@@ -436,15 +431,15 @@ typedef struct LockedPageLimit {
 */

 typedef struct VMMemMgmtInfo {
-   uint32          minAllocation;   // minimum pages for vm
-   uint32          maxAllocation;   // maximum pages the vm could lock
+   PageCnt         minAllocation;   // minimum pages for vm
+   PageCnt         maxAllocation;   // maximum pages the vm could lock
+   PageCnt         nonpaged;        // overhead memory (guest, mmap)
+   PageCnt         paged;           // vmx memory (malloc, statics)
+   PageCnt         anonymous;       // vmm memory
+   PageCnt         mainMemSize;     // guest main memory size
+   PageCnt         locked;          // number of pages locked by this vm
+   PageCnt         perVMOverhead;   // memory for vmx/vmmon overheads
   uint32          shares;          // proportional sharing weight
-   uint32          nonpaged;        // overhead memory (guest, mmap)
-   uint32          paged;           // vmx memory (malloc, statics)
-   uint32          anonymous;       // vmm memory
-   uint32          mainMemSize;     // guest main memory size
-   uint32          locked;          // number of pages locked by this vm
-   uint32          perVMOverhead;   // memory for vmx/vmmon overheads
   Percent         touchedPct;      // % of guest memory being touched
   Percent         dirtiedPct;      // % of guest memory being dirtied
   Bool            admitted;        // admission control
@@ -467,14 +462,14 @@ typedef struct VMMemMgmtInfoPatch {

 typedef struct VMMemInfoArgs {
   uint64          currentTime;        // Host time in secs of the call.
-   uint32          minVmMemPct;        // % of vm that must fit in memory
-   uint32          globalMinAllocation;// pages that must fit in maxLockedPages
-   uint32          numLockedPages;     // total locked pages by all vms
+   PageCnt         globalMinAllocation;// pages that must fit in maxLockedPages
+   PageCnt         numLockedPages;     // total locked pages by all vms
   LockedPageLimit lockedPageLimit;    // set of locked page limits
-   uint32          maxLockedPages;     // effective limit on locked pages
+   PageCnt         maxLockedPages;     // effective limit on locked pages
   uint32          callerIndex;        // this vm's index memInfo array
   uint32          numVMs;             // number of running VMs
-   uint8           _pad[4];
+   Percent         minVmMemPct;        // % of vm that must fit in memory
+   uint8           _pad[7];
   VMMemMgmtInfo   memInfo[1];
 } VMMemInfoArgs;

@@ -487,9 +482,9 @@ typedef struct VMMPNNext {
 } VMMPNNext;

 typedef struct VMMPNList {
-   uint32    mpnCount;   // IN (and OUT on Mac OS)
+   PageCnt   mpnCount;   // IN (and OUT on Mac OS)
   Bool      ignoreLimits;
-   uint8     _pad[3];
+   uint8     _pad[7];
   VA64      mpnList;    // IN: User VA of an array of 64-bit MPNs.
 } VMMPNList;

@@ -525,8 +520,6 @@ typedef struct PTSCCheckParams {
   uint8  _pad[7];
 } PTSCCheckParams;

-#ifndef VMX86_SERVER
-
 typedef struct IPIVectors {
   /*
    * Vectors we have allocated or stolen for the monitor interrupts.
@@ -535,13 +528,13 @@ typedef struct IPIVectors {
   uint8 hvIPIVector;
 } IPIVectors;

-#endif
-
 /*
 * Arguments and return value for VM creation.
 */
 typedef struct VMCreateBlock {
   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
+   VA64               vmmonData;     // IN: User VA of a userlevel scratch area
+                                     //     required by the Linux vmmon
   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
   uint32             numVCPUs;      // IN: Number of VCPUs.
   uint16             vmid;          // OUT: VM ID for the created VM.
@@ -551,11 +544,32 @@ typedef struct VMCreateBlock {
 * Information about a shared region.
 */
 typedef struct VMSharedRegion {
-   uint32 index;
-   VPN    baseVpn;
-   uint32 numPages;
+   SharedAreaType index;
+   VPN            baseVpn;
+   uint32         numPages;
 } VMSharedRegion;

+/*
+ * Arguments for VMM shared area registration.
+ */
+typedef struct VMSharedAreaRegistrationBlock {
+   Vcpuid vcpu;            // IN: VCPU being registered.
+   VMSharedRegion region;  // IN: Shared region being registered.
+} VMSharedAreaRegistrationBlock;
+
+/*
+ * Information about a VM's statvars.
+ */
+typedef struct VMStatVarsRegistrationBlock {
+   VPN     baseVpn;
+   PageCnt numPages;
+   Vcpuid  vcpu;
+} VMStatVarsRegistrationBlock;
+
+typedef struct {
+   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
+} PerVcpuPages;
+
 /*
 * Arguments for VMM bootstrap processing.
 */
@@ -564,10 +578,26 @@ typedef struct VMProcessBootstrapBlock {
   uint32         numBytes;      // IN: Size of VMM bootstrap blob.
   uint32         headerOffset;  // IN: Offset of header in blob.
   uint16         numVCPUs;      // IN: Number of VCPUs.
-   VA64           ptRootVAs[MAX_VCPUS];  // IN: User VA of PT roots.
   VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions.
+   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

+/*
+ * Arguments for retrieving the switch error address.
+ */
+typedef struct VMSwitchErrorArgs {
+   Vcpuid vcpuid;  // IN: The VCPU of interest.
+   uint64 addr;    // OUT: The code address that a failure was detected at, or 0
+                   //      if no failure has occurred.
+} VMSwitchErrorArgs;
+
+static INLINE size_t
+GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
+{
+   VMProcessBootstrapBlock *args = NULL;
+   return sizeof *args + numVCPUs * sizeof args->perVcpuPages[0];
+}
+
 /*
 * Arguments for VMM context retrieval.
 */
@@ -576,49 +606,7 @@ typedef union {
   Context64 context; // OUT
 } VMMonContext;

-#if defined __linux__
-
-/*
- * Linux uses mmap(2) to allocate contiguous locked pages, and uses these
- * macros to marshall real arguments to mmap's made-up 'offset' argument.
- */
-
-#define VMMON_MAP_MT_LOW4GB     0
-#define VMMON_MAP_MT_LOW16MB    1
-#define VMMON_MAP_MT_ANY        2
-
-#define VMMON_MAP_OFFSET_SHIFT  0
-#define VMMON_MAP_OFFSET_MASK   0x00000FFF
-#define VMMON_MAP_ORDER_SHIFT   12
-#define VMMON_MAP_ORDER_MASK    0xF
-#define VMMON_MAP_MT_SHIFT      16
-#define VMMON_MAP_MT_MASK       0x7
-#define VMMON_MAP_RSVD_SHIFT    19
-
-#define VMMON_MAP_RSVD(base)    \
-                ((base) >> VMMON_MAP_RSVD_SHIFT)
-#define VMMON_MAP_MT(base)      \
-                (((base) >> VMMON_MAP_MT_SHIFT) & VMMON_MAP_MT_MASK)
-#define VMMON_MAP_ORDER(base)   \
-                (((base) >> VMMON_MAP_ORDER_SHIFT) & VMMON_MAP_ORDER_MASK)
-#define VMMON_MAP_OFFSET(base)  \
-                (((base) >> VMMON_MAP_OFFSET_SHIFT) & VMMON_MAP_OFFSET_MASK)
-
-#define VMMON_MAP_BASE(mt, order)   (((mt) << VMMON_MAP_MT_SHIFT) | \
-                                     ((order) << VMMON_MAP_ORDER_SHIFT))
-
-#elif defined _WIN32
-/*
- * Windows uses an ioctl to allocate contiguous locked pages.
- */
-
-typedef struct VMAllocContiguousMem {
-   VA64   mpnList;  // IN: User VA of an array of 64-bit MPNs.
-   uint32 mpnCount; // IN
-   uint32 order;    // IN
-   MPN    maxMPN;   // IN
-} VMAllocContiguousMem;
-#elif defined __APPLE__
+#if defined __APPLE__
 #   include "iocontrolsMacos.h"
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,8 +42,6 @@
 #include "mon_assert.h"

 #define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
-#define CP_STUB_SIZE 16     /* A relative jmp instruction (5 bytes)
-                               padded to the next 16 byte boundary. */

 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
@@ -57,6 +55,7 @@
   MC(RELEASE_ANON_PAGES)                                                     \
   MC(LOOKUP_MPN)                                                             \
   MC(COSCHED)                                                                \
+   MC(ALLOC_CONTIG_PAGES)                                                     \
   MC(ALLOC_VMX_PAGE)                                                         \
   MC(ALLOC_TMP_GDT)                                                          \
   MC(PIN_MPN)                                                                \
@@ -65,8 +64,14 @@
   MC(ALLOC_ANON_LOW_PAGE)                                                    \
   MC(GET_MON_IPI_VECTOR)                                                     \
   MC(GET_HV_IPI_VECTOR)                                                      \
+   MC(GET_PERF_CTR_VECTOR)                                                    \
   MC(GET_HOST_TIMER_VECTORS)                                                 \
-   MC(BOOTSTRAP_CLEANUP)
+   MC(BOOTSTRAP_CLEANUP)                                                      \
+   MC(GET_SHARED_AREA)                                                        \
+   MC(GET_STAT_VARS)                                                          \
+   MC(GET_NUM_PTP_PAGES)                                                      \
+   MC(GET_HV_IO_BITMAP)                                                       \
+   MC(GET_MSR)                                                                \

 /*
 *----------------------------------------------------------------------
@@ -104,8 +109,8 @@ typedef enum ModuleCallType {
 */

 #if defined(VMX86_UCCOST) && !defined(VMX86_SERVER)
-#define UCTIMESTAMP(cp, stamp) \
-             do { (cp)->ucTimeStamps[UCCOST_ ## stamp] = RDTSC(); } while (0)
+#define UCTIMESTAMP(ptr, stamp) \
+             do { (ptr)[UCCOST_ ## stamp] = RDTSC(); } while (0)
 #else
 #define UCTIMESTAMP(cp, stamp)
 #endif
@@ -125,7 +130,7 @@ typedef struct UCCostResults {
 } UCCostResults;

 typedef enum UCCostStamp {
-#define UC(x) UCCOST_ ## x,
+#define UC(x, y) UCCOST_ ## x,
 #include "uccostTable.h"
   UCCOST_MAX
 } UCCostStamp;
@@ -139,12 +144,12 @@ typedef enum UCCostStamp {
 * MAX_SWITCH_PT_PATCHES
 *
 *   This is the maximum number of patches that must be placed into
- *   the monitor page tables so that the host GDT and the crosspage
- *   can be accessed during worldswitch.
+ *   the monitor page tables so that the host GDT, crosspage data, and
+ *   crosspage code can be accessed during worldswitch.
 *
 *----------------------------------------------------------------------
 */
-#define MAX_SWITCH_PT_PATCHES 2
+#define MAX_SWITCH_PT_PATCHES 3

 /*----------------------------------------------------------------------
 *
@@ -231,16 +236,23 @@ VMMPageTablePatch;
 * - the largest stack use instantaneously possible by #DB handling code
 * - one high-water uint32 used to detect stack overflows when debugging
 *
- * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+ * A breakdown of the worst-case exception handler stack usage (SwitchUDHandler)
+ * is: 5 * uint64 (Hardware) + 1 * uint64 (RAX) + 1 * uint64 (RBX) +
+ *     1 * uint64 (RCX) + 1 * uint64 (call) + 2 * uint64 (sidt) = 11 * uint64
+ * This is a slight over-estimate of the possible usage at any time but there
+ * is plenty of space available in the cross page data area.
+ *
+ * 264 (11 * sizeof(uint64) * 3) bytes is slightly more than enough as of
+ * 2020/06/14.
 */
-#define TINY_STACK_SIZE      184
+#define TINY_STACK_SIZE      264

 /*
 *----------------------------------------------------------------------
 *
 * VMCrossPageData --
 *
- *      Data which is stored on the VMCrossPage.
+ *      Data which is stored on the cross page.
 *
 *----------------------------------------------------------------------
 */
@@ -289,10 +301,13 @@ struct VMCrossPageData {
   uint16   monES;  /* Not saved/restored in assembly switch */
   uint16   monPad;

-   uint64   crosspageMA;
+   uint64   crosspageDataMA;

   uint64   hostDR[8];
-   LA64     hostCrossPageLA;   // where host has crosspage mapped
+   LA64     crosspageDataLA;   // where host/PTP map the cross data page
+   LA64     crossGDTLA;        // where host/PTP map the cross GDT
+   LA64     crosspageCodeLA;   // where host/PTP map the cross code page
+   LA64     vmmToHostLA;       // where host has placed the VmmToHost function
   uint16   hostInitial64CS;
   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
@@ -310,8 +325,6 @@ struct VMCrossPageData {
   Task64   monTask;          /* vmm's task */

   VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
-   LA64              vmmCrossPageLA;
-   LA64              vmmCrossGDTLA;   // where crossGDT is mapped by PT patch

   /*
    * The monitor may requests up to two actions when returning to the
@@ -333,6 +346,7 @@ struct VMCrossPageData {

 #if !defined(VMX86_SERVER)
   uint64 ucTimeStamps[UCCOST_MAX];
+   uint8  _ucPad[8];
 #endif

   SwitchedMSRState switchedMSRState;
@@ -395,74 +409,19 @@ struct VMCrossPageData {
 #include "vmware_pack_end.h"
 VMCrossPageData;

-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPageCode --
- *
- *      Code which is stored on the VMCrossPage.
- *
- *----------------------------------------------------------------------
- */
-
-#define CODE_BLOCK_SIZE (PAGE_SIZE - sizeof(VMCrossPageData) - \
-                         (2 + NUM_EXCEPTIONS) * CP_STUB_SIZE)
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPageCode {
-   uint8         toVmmFunc[CP_STUB_SIZE];    // Fixed-position stubs to jump
-   uint8         toHostFunc[CP_STUB_SIZE];   // to world switch functions
-   uint8         gateStubs[NUM_EXCEPTIONS *
-                           CP_STUB_SIZE];    // Stubs for crossIDT.
-   uint8         codeBlock[CODE_BLOCK_SIZE]; // Code for worldswitch and
-                                             // fault handling.
-}
-#include "vmware_pack_end.h"
-VMCrossPageCode;
-
-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPage --
- *
- *      Data structure shared between the monitor and the module
- *      that is used for crossing between the two.
- *      Accessible as vm->crosspage (kernel module) and CROSS_PAGE
- *      (monitor)
- *
- *      Must be exactly one page long.
- *
- *----------------------------------------------------------------------
- */
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPage {
-   VMCrossPageData crosspageData;
-   VMCrossPageCode crosspageCode;
-}
-#include "vmware_pack_end.h"
-VMCrossPage;
-
-#define CROSSPAGE_VERSION_BASE 0xc09 /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc12 /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE  ((VMCrossPage * const) VPN_2_VA(CROSS_PAGE_START))
-#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
+#define CROSS_PAGE             ((VMCrossPageData *) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
+#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData * const) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
 #endif

-#define NULLPAGE_LINEAR_START  (MONITOR_LINEAR_START + \
-                                PAGE_SIZE * CPL0_GUARD_PAGE_START)
-
 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
-#define MX_WAITNORMAL          1  // Must equal one; see linux module code.
-#define MX_WAITERROR           0  // Use MX_ISWAITERROR() to test for error.
+#define MX_WAITNORMAL          1
+#define MX_WAITERROR           0

-// Any zero or negative value denotes error.
-#define MX_ISWAITERROR(e)      ((e) <= MX_WAITERROR)
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006,2009-2011,2013-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006,2009-2011,2013-2015,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,8 +66,7 @@
   SWMSR(MSR_CSTAR)         \
   SWMSR(MSR_SFMASK)        \
   SWMSR(MSR_TSC_AUX)       \
-   SWMSR(MSR_BD_TSC_RATIO)  \
-   SWMSR(MSR_BNDCFGS)
+   SWMSR(MSR_BD_TSC_RATIO)

 /*
 *      Data structures for dealing with the context-switched MSRs that need
@@ -94,7 +93,6 @@ typedef struct SwitchedMSRValues {
 typedef struct SwitchedMSRState {
   SwitchedMSRValues smv;
   uint8             flags[NUM_SWITCHED_MSRS];
-   uint32            _pad;
 } SwitchedMSRState;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,6 +25,8 @@
 #ifndef _MON_ADDR_LAYOUT_H
 #define _MON_ADDR_LAYOUT_H

+#include "vm_pagetable.h"
+
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMKERNEL
@@ -37,6 +39,8 @@ typedef struct VMM64_AddrLayout {
    *All address are VPNs and all lengths are numPages
    */
   uint64 monBase;            // MONITOR_BASE_VPN
+   uint64 monL5Start;         // MON_PAGE_TABLE_L5_START
+   uint32 monL5Len;           // MON_PAGE_TABLE_L5_LEN
   uint64 monL4Start;         // MON_PAGE_TABLE_L4_START
   uint32 monL4Len;           // MON_PAGE_TABLE_L4_LEN
   uint64 monL3Start;         // MON_PAGE_TABLE_L3_START
@@ -51,7 +55,21 @@ typedef struct VMM64_AddrLayout {
 } VMM64_AddrLayout;
 #pragma pack(pop)

-#define VMM_MONAS_FIRST_L4OFF   1
-#define VMM_MONAS_LAST_L4OFF  127
+#define VMM_MONAS_4LP_FIRST_L4OFF   1
+#define VMM_MONAS_4LP_LAST_L4OFF  130

+#define VMM_MONAS_5LP_FIRST_L5OFF   1
+#define VMM_MONAS_5LP_LAST_L5OFF   34
+
+static INLINE PT_Level
+MonAS_GetPagingLevel(void)
+{
+   return PT_LEVEL_4;
+}
+
+static INLINE Bool
+MonAS_Uses5LevelPaging(void)
+{
+   return MonAS_GetPagingLevel() == PT_LEVEL_5;
+}
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,8 +31,8 @@
 * contains code and data statically built, empty but allocated space, shared
 * and run-time initialized content.  The monitor loader header regularizes
 * encoding of address space information, allowing a common representation and
- * common code to be re-used for different contexts (vmmon for hosted, vmkernel
- * for ESX, and the VMX while deprivileging of bootstrap is ongoing).
+ * common code to be re-used for different contexts (vmmon for hosted
+ * and vmkernel for ESX).
 *
 * The Header
 * ==========
@@ -87,7 +87,6 @@
 * MonLoaderCallout_Init(): Initialize MonLoader callouts.
 * MonLoaderCallout_GetSharedHostPage(subIdx, page): Get shared host page MPN.
 * MonLoaderCallout_GetSharedUserPage(subIdx, page): Get shared user page MPN.
- * MonLoaderCallout_SetEntrypoint(): Sets the entry code/stack for execution.
 *
 * Building vs Importing
 * =====================
@@ -119,21 +118,54 @@
 #define _MON_LOADER

 #include "vm_basic_types.h"
-#include "x86paging_common.h"
-#include "x86types.h" /* PT_L1E */
+#include "vm_pagetable.h"
 #include "vcpuid.h"   /* Vcpuid */

+#if defined VM_X86_64
+#include "x86paging_64.h"
+#elif defined VM_ARM_64
+#include "arm64_vmsa.h"
+#endif
+
 #define ML_NAME_MAX 16

 /* ML perms are simple and abbreviated. */

+#if defined VM_X86_64
+
 #define ML_PERM_RWX (PTE_P | PTE_RW)
 #define ML_PERM_RW  (PTE_P | PTE_RW | PTE_NX)
 #define ML_PERM_RO  (PTE_P |          PTE_NX)
 #define ML_PERM_RX   PTE_P

+#define ML_PERM_TBL   ML_PERM_RWX
 #define ML_PERM_MASK (PTE_P| PTE_RW | PTE_NX | PTE_US)

+#define ML_PERM_PRESENT(_flags)     (((_flags) & PTE_P) != 0)
+#define ML_PERM_WRITEABLE(_flags)   (((_flags) & PTE_RW) != 0)
+
+#define ML_PTE_2_PFN(_pte)          LM_PTE_2_PFN(_pte)
+
+#elif defined VM_ARM_64
+
+#define _ML_PERM_COMMON (ARM_PTE_BLOCK_AP(ARM_AP_PL0)   | ARM_PTE_BLOCK_AF | \
+                         ARM_PTE_BLOCK_SH(ARM_SH_OUTER) | ARM_PTE_BLOCK_L3_TYPE)
+#define ARM_PTE_BLOCK_AP_RO ARM_PTE_BLOCK_AP(ARM_AP_RO)
+
+#define ML_PERM_RW   (_ML_PERM_COMMON |                       ARM_PTE_BLOCK_XN)
+#define ML_PERM_RO   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+#define ML_PERM_RX   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO                   )
+
+#define ML_PERM_TBL   ML_PERM_RW
+#define ML_PERM_MASK (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+
+#define ML_PERM_PRESENT(_flags)   (((_flags) & ARM_PTE_VALID) != 0)
+#define ML_PERM_WRITEABLE(_flags) (((_flags) & ARM_PTE_BLOCK_AP_RO) == 0)
+
+#define ML_PTE_2_PFN(_pte)    (((_pte) & ARM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
+
+#endif
+
 #define ML_PERMS_MATCH(x,p) (((x) & ML_PERM_MASK) == ((p) & ML_PERM_MASK))

 #define LOADER_HEADER_MAGIC 0x8675309E98675309
@@ -150,10 +182,10 @@ typedef enum {
   ML_CONTENT_SHARE,         /* Share data from external source. */
 } MonLoaderContentType;

-#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? 4 : \
-                               x == ML_CONTENT_PAGETABLE_L3 ? 3 : \
-                               x == ML_CONTENT_PAGETABLE_L2 ? 2 : \
-                               x == ML_CONTENT_PAGETABLE_L1 ? 1 : \
+#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? PT_LEVEL_4 : \
+                               x == ML_CONTENT_PAGETABLE_L3 ? PT_LEVEL_3 : \
+                               x == ML_CONTENT_PAGETABLE_L2 ? PT_LEVEL_2 : \
+                               x == ML_CONTENT_PAGETABLE_L1 ? PT_LEVEL_1 : \
                               0)

 /* Sources of content, for pages not zeroed, unmapped or otherwise special. */
@@ -183,11 +215,14 @@ typedef struct {
   } blobSrc;
   uint64               bspOnly;           /* Process only on BSP. */
   uint64               subIndex;          /* Region ID for ML_CONTENT_COPY. */
-   uint64               procVmmon;         /* Processed by the vmmon on hosted. */
 } MonLoaderEntry;


-/* Packed for easy consumption by bootstrap-offsets.pl. */
+/*
+ * Packed for easy consumption by bootstrap-offsets.pl. If the contents of the
+ * MonLoaderHeader struct are changed then $HEADER_SIZE must be updated
+ * accordingly in bootstrap-offsets.pl.
+ */
 #pragma pack(push, 1)
 typedef struct MonLoaderHeader {
   uint64         magic;
@@ -199,6 +234,8 @@ typedef struct MonLoaderHeader {
   /* ss:rip */
   uint16         stackSelector;
   VA64           stackEntrypoint;
+   LPN64          monStartLPN;
+   LPN64          monEndLPN;
   MonLoaderEntry entries[];
 } MonLoaderHeader;
 #pragma pack(pop)
@@ -217,16 +254,13 @@ MPN  MonLoaderCallout_GetPageRoot(struct MonLoaderEnvContext *, Vcpuid);
 Bool MonLoaderCallout_GetPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                             Vcpuid, PT_L1E *);
 Bool MonLoaderCallout_ImportPage(struct MonLoaderEnvContext *, MPN, Vcpuid);
-Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **);
+Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **, unsigned);
 Bool MonLoaderCallout_MapMPNInPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                                  uint64, MPN, Vcpuid);
 MPN  MonLoaderCallout_GetSharedUserPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
 MPN  MonLoaderCallout_GetSharedHostPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
-Bool MonLoaderCallout_IsPrivileged(struct MonLoaderEnvContext *);
-Bool MonLoaderCallout_SetEntrypoint(struct MonLoaderEnvContext *, uint16, VA64,
-                                    uint16, VA64);
 MPN  MonLoaderCallout_GetBlobMpn(struct MonLoaderEnvContext *, uint64);

 typedef enum MonLoaderError {
@@ -237,7 +271,6 @@ typedef enum MonLoaderError {
   ML_ERROR_ARGS,
   ML_ERROR_CALLOUT_INIT,
   ML_ERROR_CALLOUT_COPY,
-   ML_ERROR_CALLOUT_ENTRYPOINT,
   ML_ERROR_CALLOUT_GETPTE,
   ML_ERROR_CALLOUT_MAPINPTE,
   ML_ERROR_CALLOUT_PAGEROOT_GET,
@@ -259,9 +292,14 @@ typedef enum MonLoaderError {
 } MonLoaderError;


-/* A subindex above shared area subindices for sharing of MonLoaderHeader. */
-#define MONLOADER_HEADER_IDX 6
-
+/*
+ * Values above shared area subindices for sharing of MonLoaderHeader
+ * and htSchedStateMap.
+ */
+#define MONLOADER_HEADER_IDX          6
+#define MONLOADER_HT_MAP_IDX          7
+#define MONLOADER_CROSS_PAGE_CODE_IDX 8
+#define MONLOADER_CROSS_PAGE_DATA_IDX 9

 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
                                 void *args, unsigned *line, Vcpuid *vcpu);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,23 +25,7 @@
 #ifndef _MONLOADER_LOG_H_
 #define _MONLOADER_LOG_H_

-#ifdef VMX86_VMX
-
-#define LOGLEVEL_MODULE main
-#include "loglevel_vmcore.h"
-/*
- * Redefine LOG() to match the vmkernel calling convention.
- */
-#undef LOG_BYNAME
-#undef LOG
-
-#define LOG_BYNAME(_mod, _min, _fmt, ...) \
-        (DOLOG_BYNAME(_mod, _min) ? Log(_fmt "\n", ## __VA_ARGS__) : (void) 0)
-
-#define LOG(_min, _fmt, ...) \
-        LOG_BYNAME(LOGLEVEL_MODULE, _min, _fmt, ## __VA_ARGS__)
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #define LOGLEVEL_MODULE MonLoader
 #include "log.h"
@@ -59,7 +43,7 @@ do {                                                 \
   }                                                 \
 } while (0)

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2007-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2007-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -60,20 +60,21 @@
 *     lengths, and this causes the 'monitor-modular-size' script to
 *     report differences in '.rodata'.
 *
- *   o To avoid differences in '.rodata', each __FILE__ is put into
- *     its own section.  The monitor's linker (not ld) will use the
- *     name of the section to recover the name of the source file.
+ *   o Each __FILE__ is put into its own .assert_pathname_##__FILE
+ *     section.  The monitor's runtime linker (not ld) then uses the name
+ *     of this section to recover the name of the corresponding source file.
 *
 *   o At run time, prior to loading, when our linker is creating an
 *     executable image of the monitor and extensions, all the file
 *     names are extracted from these sections, the '${VMTREE}' prefix
 *     is removed, and the resulting table of shortened file names is
- *     added to '.rodata'.
+ *     added to a non-loadable section called '.assert_pathnames' in the
+ *     VMX linker (not the bootstrap linker which does not need them).
 *
 *     Further, during linkage, each relocation to the original
 *     section containing the path name is modified so that the low
- *     16-bits contain an offset from '__vmm_pathnames_start' rather
- *     than the base of the original containing section.
+ *     16-bits contain an offset from the start of the .assert_pathnames
+ *     section rather than the base of the original containing section.
 *
 *     Only three types of relocations to the assertion strings are
 *     supported (32-bit PC-relative and 32-bit/64-bit absolute) because that
@@ -111,42 +112,12 @@ typedef struct Assert_Info {
   Assert_MonSrcLoc loc;
 } Assert_Info;

-/*
- * The portion of the __attribute__ line after __FILE__ is there so that
- * the .assert_pathname_* sections are not marked as ALLOC, since we only
- * need them in the vmx and do not need them loaded.
- */
-#define __VMM__FILE__SECTION \
-      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
-#define __VMM__FILE__ ({                                                \
-         static __VMM__FILE__SECTION const char file[] = "";            \
-         file;                                                          \
-      })
-
 #define ASSERT_MONSRCFILEOFFSET(loc)    LOWORD(loc)
 #define ASSERT_MONSRCLINE(loc)          HIWORD(loc)

 #define ASSERT_NULL_MONSRCLOC     0             // there is never line 0

-#ifdef VMM // {
-#ifdef MONITOR_APP // {
-
-#define ASSERT_MONSRCLOC() ASSERT_NULL_MONSRCLOC
-
-#else // } {
-
-#define ASSERT_MONSRCLOC() ({                                           \
-   const uintptr_t offset = ((__LINE__ << 16) +                         \
-                             (uintptr_t)__VMM__FILE__);                 \
-   const Assert_MonSrcLoc loc = offset;                                 \
-   loc;                                                                 \
-})
-
-extern const char __vmm_pathnames_start;
-#define ASSERT_MONSRCFILE(loc) \
-   (&__vmm_pathnames_start + ASSERT_MONSRCFILEOFFSET(loc))
-
-
+#if defined(VMM) && !defined(MONITOR_APP) // {
 /*
 * Assertion information is collected in a non-loadable section
 * named .assert_info.  Each record in this section contains
@@ -176,6 +147,18 @@ extern const char __vmm_pathnames_start;
 */

 #ifndef VMM_BOOTSTRAP
+/*
+ * The portion of the __attribute__ line after __FILE__ is there so that
+ * the .assert_pathname_* sections are not marked as ALLOC, since we only
+ * need them in the vmx and do not need them loaded.
+ */
+#define __VMM__FILE__SECTION \
+      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
+#define __VMM__FILE__ ({                                                \
+         static __VMM__FILE__SECTION const char file[] = "";            \
+         file;                                                          \
+      })
+
 #define ASSERT_RECORDINFO(assembly, assertType, bugNr)                   \
   __asm__ __volatile__(".pushsection .assert_info;"                     \
                        ".quad 0f;"                                      \
@@ -215,8 +198,6 @@ extern uint64 bsAssertRIP;
   ({COMPILER_MEM_BARRIER();                                             \
     ASSERT_RECORDINFO("", AssertType_##name##Bug, bug);                 \
     __builtin_trap();})
+#endif // VMM && !MONITOR_APP}

-#endif // MONITOR_APP }
-#endif // VMM }
-
-#endif
+#endif // _MON_ASSERT_H_
@@ -52,7 +52,7 @@ extern "C" {
 typedef struct PageListEntry {
   CompressedBPN cbpn;
   Bool voided;
-   uint8 _pad[2];
+   uint8 _pad[1];
 } PageListEntry;
 #pragma pack(pop) 

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -55,14 +55,17 @@ typedef enum PCIPassthruVectorIndex {
 } PCIPassthruVectorIndex;

 typedef enum PCIPassthru_IntrType {
-   PCIPASSTHRU_INTR_NONE   = 0x00,
-   PCIPASSTHRU_INTR_IOAPIC = 0x01,
-   PCIPASSTHRU_INTR_MSI    = 0x02,
-   PCIPASSTHRU_INTR_MSIX   = 0x04,
+   PCIPASSTHRU_INTR_NONE    = 0x00,
+   PCIPASSTHRU_INTR_IOAPIC  = 0x01,
+   PCIPASSTHRU_INTR_MSI     = 0x02,
+   PCIPASSTHRU_INTR_MSIX    = 0x04,
+   PCIPASSTHRU_INTR_MSI_INT = 0x08,
 } PCIPassthru_IntrType;

+#define PCIPASSTHRU_INTR_MSI_COMBINED                   \
+   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)
+
 typedef struct FPTIntrProxyInfo {
-   uint32            adapterIndex;
   uint32            vectorIndex;
   MonitorIdemAction actionID;
 } FPTIntrProxyInfo;
@@ -72,9 +75,15 @@ typedef struct UPTIntrProxyInfo {
   MonitorIdemAction actionID;
 } UPTIntrProxyInfo;

+typedef struct MPTIntrProxyInfo {
+   uint32            adapterIndex;
+   MonitorIdemAction actionID;
+} MPTIntrProxyInfo;
+
 typedef union PCIPassthru_IntrProxyInfo {
   FPTIntrProxyInfo fpt;
   UPTIntrProxyInfo upt;
+   MPTIntrProxyInfo mpt;
 } PCIPassthru_IntrProxyInfo;

 typedef enum PCIPassthru_ErrorType {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,12 @@
 *********************************************************/

 /*
- * perfctr_generic.h --
+ * perfctr.h --
 *
 */

-#ifndef _PERFCTR_GENERIC_H_
-#define _PERFCTR_GENERIC_H_
+#ifndef _PERFCTR_H_
+#define _PERFCTR_H_

 #define INCLUDE_ALLOW_USERLEVEL

@@ -33,7 +33,9 @@

 #include "includeCheck.h"
 #include "vm_basic_types.h"
+#include "perfctr_arch.h"

+#define PERF_EVENT_NAME_LEN                      64
 /*
 * nmiNo      -- vmm peer is not attempting to do nmi profiling this run
 * nmiYes     -- vmm peer is doing nmi profiling and nmis are currently enabled
@@ -69,4 +71,51 @@ typedef struct {
   uint8  _pad[3];
 } CrossProfShared;

-#endif // ifndef _PERFCTR_GENERIC_H_
+/*
+ * PerfCtr_Config --
+ *      Describes configuration for a single hardware performance counter
+ *
+ *      Since this is only used to record general performance counters, we
+ *      made the assumption in nmiProfiler.c that the type is GENERAL and
+ *      index is counter number of type GENERAL.
+ *
+ *      **** x86 ****
+ *      On AMD K8 and GH:
+ *      index:        Which perf ctr, 0 to 3.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On AMD with PerfCtrExtCore support:
+ *      index:        Which perf ctr, 0 to 5.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On Intel Core architecture:
+ *      <to be documented>
+ *
+ *      **** ARM ****
+ *      escrVal:     Value placed in PMEVTYPER<n>_EL0 to configure event counter
+ *      index:       Index of the event counter.
+ */
+typedef struct PerfCtr_Config {
+   uint64 escrVal;
+   uint32 index;
+   uint32 periodMean;
+
+   /*
+    * Random number (whose absolute value is capped at
+    * periodJitterMask) is used to randomize sampling interval.
+    */
+   uint32  periodJitterMask;
+   uint32  seed;    // seed is used to compute next random number
+   uint16  config;
+   Bool    valid;
+   PERFCTR_CONFIG_ARCH_FIELDS
+} PerfCtr_Config;
+
+
+#endif // ifndef _PERFCTR_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,17 @@
 *********************************************************/

 /*
- * x86perfctr.h --
+ * perfctr_arch.h --
 *
+ *      Performance counters (x64 specific).
 */

-#ifndef _X86PERFCTR_H_
-#define _X86PERFCTR_H_
+#ifndef _PERFCTR_ARCH_H
+#define _PERFCTR_ARCH_H
+
+#ifndef _PERFCTR_H_
+#error "This file can only be included by perfctr.h"
+#endif

 #define INCLUDE_ALLOW_USERLEVEL

@@ -30,25 +35,20 @@
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
-
 #include "includeCheck.h"
+
 #include "vm_asm.h"
 #include "x86cpuid_asm.h"
-#include "perfctr_generic.h"

-#define PERFCTR_PENTIUM4_NUM_COUNTERS            18
-#define PERFCTR_PENTIUM4_NUM_COUNTERS_WITH_L3    26
-#define PERFCTR_AMD_NUM_COUNTERS                 4 
+#define PERFCTR_AMD_NUM_COUNTERS                 4
 #define PERFCTR_AMD_EXT_NUM_COUNTERS             6
 #define PERFCTR_P6_NUM_COUNTERS                  2
 #define PERFCTR_NEHALEM_NUM_GEN_COUNTERS         4
 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS       3
 #define PERFCTR_SANDYBRIDGE_NUM_GEN_COUNTERS     8 /* When HT is disabled */
-#define PERFCTR_CORE_NUM_ARCH_EVENTS             7
-#define PERFCTR_CORE_NUM_FIXED_COUNTERS          3
-#define PERFCTR_PENTIUM4_VAL_MASK                0xffffffffffLL
+#define PERFCTR_CORE_NUM_ARCH_EVENTS             8
+#define PERFCTR_CORE_NUM_FIXED_COUNTERS          4
 #define PERFCTR_AMD_VAL_MASK                     0xffffffffffffLL
-#define PERF_EVENT_NAME_LEN                      64
 /*
 * Even though the performance counters in P6 are 40 bits,
 * we can only write to the lower 32 bits.  Bit 31 is
@@ -59,7 +59,7 @@

 /*
 * Performance counter width is determined at runtime in CORE.
- * But the writables bits are fixed. we can only write to the 
+ * But the writables bits are fixed. we can only write to the
 * lower 32 bits.  Bit 31 is used to sign-extend the upper 8 bits.
 */
 #define PERFCTR_CORE_WRITE_MASK                  0xffffffff
@@ -124,7 +124,7 @@
 #define PERFCTR_AMD_CPU_CLK_UNHALTED                           0x76

 /* AMD Load/Store unit events */
-#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20 
+#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20
 #define PERFCTR_AMD_LS_BUFFER2_FULL                            0x23

 /*
@@ -133,13 +133,19 @@
 */
 #define PERFCTR_AMD_SMI_COUNT                                  0x2b

-/* AMD Data Cache Events */
+/*
+ * AMD Data Cache Events
+ * Family <  17H: 0x45 - Unified TLB hit
+ *                0x46 - Unified TLB miss
+ * Family >= 17H: 0x45 - L1 DTLB miss (L2 DTLB hit or miss)
+ *                0x46 - Tablewalker
+ */
 #define PERFCTR_AMD_DATA_CACHE_ACCESSES                        0x40
 #define PERFCTR_AMD_DATA_CACHE_MISSES                          0x41
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_L2_OR_SYSTEM       0x42
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_SYSTEM             0x43
 #define PERFCTR_AMD_DATA_CACHE_LINES_EVICTED                   0x44
-#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT               0x45
+#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT_OR_MISS       0x45
 #define PERFCTR_AMD_L1_DTLB_AND_L2_DTLB_MISS                   0x46
 #define PERFCTR_AMD_MISALIGNED_ACCESSES                        0x47
 #define PERFCTR_AMD_PREFETCH_INSTRS_DISPATCHED                 0x4b
@@ -198,8 +204,8 @@

 #define PERFCTR_AMD_MEM_CTRL_PAGE_TABLE_OVERFLOWS                   0xe1
 #define PERFCTR_AMD_CPU_IO_REQUESTS_TO_MEMORY_IO                    0xe9
-#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec 
- 
+#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec
+
 /* AMD HyperTransport Interface Events */

 #define PERFCTR_AMD_HT_L0_TX_BW                                     0xf6
@@ -219,9 +225,9 @@
 */

 /* P6 Data Cache Unit (DCU) */
-#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043  
+#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043
 #define PERFCTR_P6_DCU_LINES_IN                  0x00000045
-#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046 
+#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046
 #define PERFCTR_P6_DCU_MISS_OUTSTANDING          0x00000048

 /* P6 Instruction Fetch Unit (IFU) */
@@ -379,7 +385,7 @@
 #define PERFCTR_P6_APIC_INTR                     0x00100000
 #define PERFCTR_P6_ENABLE                        0x00400000
 #define PERFCTR_P6_INVERT_COUNTER_MASK           0x00800000
-#define PERFCTR_P6_COUNTER_MASK                  0xff000000        
+#define PERFCTR_P6_COUNTER_MASK                  0xff000000
 #define PERFCTR_P6_COUNTER_MASK_SHIFT            24
 #define PERFCTR_P6_SHIFT_BY_UNITMASK(e)          (e << 8)

@@ -436,6 +442,7 @@
 #define PERFCTR_CORE_KERNEL_MODE                    PERFCTR_CPU_KERNEL_MODE
 #define PERFCTR_CORE_APIC_INTR                      PERFCTR_CPU_APIC_INTR
 #define PERFCTR_CORE_ENABLE                         PERFCTR_CPU_ENABLE
+/* AnyThread Counting deprecated since PMU v5. */
 #define PERFCTR_CORE_ANYTHREAD                      0x00200000
 #define PERFCTR_CORE_IN_TX                          (CONST64U(1) << 32)
 #define PERFCTR_CORE_IN_TXCP                        (CONST64U(1) << 33)
@@ -463,6 +470,7 @@
 #define PERFCTR_CORE_INST_RETIRED                0xc0
 /* bus cycles */
 #define PERFCTR_CORE_UNHALTED_REF_CYCLES         (0x3c | (0x01 << 8))
+#define PERFCTR_CORE_TOPDOWN_SLOTS               (0xa4 | (0x01 << 8))

 /*
 * See Tables 30-2, 30-4 of the
@@ -558,132 +566,20 @@
 #define PERFCTR_NEHALEM_UNCORE_ENABLE            0x400000
 #define PERFCTR_NEHALEM_UNCORE_L3_LINES_IN       (0x0a | (0x0f << 8))

-
-/*
- * ----------------------------------------------------------------------
- *
- * Pentium 4
- *
- * ----------------------------------------------------------------------
- */
-
-/*
- * Pentium 4 Counter Configuration Control Register flags and fields
- */
-#define PERFCTR_PENTIUM4_CCCR_ENABLE                   0x00001000
-#define PERFCTR_PENTIUM4_CCCR_SET_ESCR(c, e)           ((c) |= (((e) & 0x7) << 13))
-#define PERFCTR_PENTIUM4_CCCR_REQRSVD                  0x00030000 // always set these bits
-#define PERFCTR_PENTIUM4_CCCR_COMPARE                  0x00040000
-#define PERFCTR_PENTIUM4_CCCR_COMPLEMENT               0x00080000
-#define PERFCTR_PENTIUM4_CCCR_SET_THRESHOLD(c, e)      ((c) |= (((e) & 0xf) << 20))
-#define PERFCTR_PENTIUM4_CCCR_EDGE                     0x01000000
-#define PERFCTR_PENTIUM4_CCCR_FORCE_OVF                0x02000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T0               0x04000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T1               0x08000000
-#define PERFCTR_PENTIUM4_CCCR_CASCADE                  0x40000000
-#define PERFCTR_PENTIUM4_CCCR_OVF                      0x80000000
-
-#define PERFCTR_PENTIUM4_COUNTER_BASEADDR              0x00000300
-#define PERFCTR_PENTIUM4_CCCR_BASE_ADDR                0x00000360
-
-#define PERFCTR_PENTIUM4_CCCR_THRESHOLD(e)             ((e & 0xf) << 20)
-#define PERFCTR_PENTIUM4_OPT_EDGE_DETECT (PERFCTR_PENTIUM4_CCCR_COMPARE | PERFCTR_PENTIUM4_CCCR_EDGE)
-
-/*
- * Pentium 4 Event Selection Control Register flags
- */
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T0             0x00000004
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T0           0x00000008
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T1             0x00000001
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T1           0x00000002
-#define PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK(e)          (e << 9)
-#define PENTIUM4_EVTSEL(e)                             (e << 25)
-
-/*
- * Event definitions for Pentium4 Family Processors. 
- * There are many more of these possible.
- * See Appendix A.1 in Volume 3 of the IA32 manual.
- */
-
-#define PERFCTR_PENTIUM4_EVT_INSTR_RETIRED              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_INSTR_COMPLETED            PENTIUM4_EVTSEL(0x07)
-#define PERFCTR_PENTIUM4_EVT_BSQ_CACHE_REFERENCE        PENTIUM4_EVTSEL(0x0c)
-#define PERFCTR_PENTIUM4_EVT_ITLB_REFERENCE             PENTIUM4_EVTSEL(0x18)
-#define PERFCTR_PENTIUM4_EVT_PAGE_WALK_TYPE             PENTIUM4_EVTSEL(0x01)   
-#define PERFCTR_PENTIUM4_EVT_BPU_FETCH_REQUEST          PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_BRANCH_RETIRED             PENTIUM4_EVTSEL(0x06)  
-#define PERFCTR_PENTIUM4_EVT_MISPRED_BRANCH_RETIRED     PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_MACHINE_CLEAR              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_MEMORY_CANCEL              PENTIUM4_EVTSEL(0x02) 
-#define PERFCTR_PENTIUM4_GLOBAL_POWER_EVENTS            PENTIUM4_EVTSEL(0x13)
-
-/*
- * Pentium4 doesn't have an event for clock cycles, but Intel
- * outlines a method in the IA32 manual, vol 3, sec 14.9.9,
- * for measuring cycles which involves this event strategy.
- */
-
-#define PERFCTR_PENTIUM4_EVT_CLK_CYCLES \
-   PENTIUM4_EVTSEL(0x2) | \
-   PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK (0x01 | 0x02 | 0x04 | 0x08)
-
-
-/*
- * PerfCtr_Config --
- *      Describes configuration for a single hardware performance counter
- *
- *      Since this is only used to record general performance counters, we
- *      made the assumption in nmiProfiler.c  that the type is GENERAL and
- *      index is counter number of type GENERAL.
- *      
- *      
- *      On AMD K8 and GH:
- *      index:        Which perf ctr, 0 to 3.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- *
- *      On AMD with PerfCtrExtCore support:
- *      index:        Which perf ctr, 0 to 5.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- * 
- *      No field should be greater than 32-bit as it is shared with monitor.
- *
- *      On Intel Core architecture:
- *      <to be documented>
- */
-typedef struct PerfCtr_Config {
-   uint64 escrVal;
-   uint32 index;
-   uint32 addr;
-   uint32 escrAddr;
-   uint32 resetHi;
-   uint32 periodMean;
-
-   /*
-    * Random number (whose absolute value is capped at
-    * periodJitterMask) is used to randomize sampling interval.
-    */
-   uint32  periodJitterMask;
-   uint32  seed;    // seed is used to compute next random number
-   uint16  config;
-   Bool    valid;
-   Bool    pebsEnabled;
-} PerfCtr_Config;
-
+#define PERFCTR_CONFIG_ARCH_FIELDS \
+   uint32 addr;      \
+   uint32 escrAddr;  \
+   uint32 resetHi;   \
+   Bool pebsEnabled; \

 /*
 * Program/reprogram event reg(s) associated w/perfctrs & start or stop perfctrs
 */
 static INLINE void
-PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
-                   uint32 escrVal)       // IN: event register value
+PerfCtr_WriteEvtSel(uint32 addr,       // IN: Event register to write
+                    uint32 escrVal)    // IN: event register value
 {
-   __SET_MSR(ctr->escrAddr, escrVal);
+   X86MSR_SetMSR(addr, escrVal);
 }


@@ -691,11 +587,10 @@ PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
 * Set/reset performance counters to engender desired period before overflow
 */
 static INLINE void
-PerfCtrWriteCounter(PerfCtr_Config *ctr,   // IN: counter to write
-                    uint32 valueLo,        // IN: low 32 bits of value to write
-                    uint32 valueHi)        // IN: high 32 bits of value to write
+PerfCtr_WriteCounter(uint32 addr,   // IN: counter to write
+                     uint64 value)  // IN: value to write
 {
-   WRMSR(ctr->addr, valueLo, valueHi);
+   X86MSR_SetMSR(addr, value);
 }


@@ -814,8 +709,9 @@ PerfCtr_PEBSAvailable(void)
   CPUIDRegs regs;
   __GET_CPUID(0, &regs);
   if (CPUID_IsVendorIntel(&regs) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_EMON_AVAILABLE) != 0) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_PEBS_UNAVAILABLE) == 0)) {
+       (X86MSR_GetMSR(MSR_MISC_ENABLE) &
+        (MSR_MISC_ENABLE_EMON_AVAILABLE | MSR_MISC_ENABLE_PEBS_UNAVAILABLE))
+       == MSR_MISC_ENABLE_EMON_AVAILABLE) {
      CPUIDRegs hvendor;

      /*
@@ -827,7 +723,7 @@ PerfCtr_PEBSAvailable(void)
             !CPUID_IsRawVendor(&hvendor,
                                CPUID_HYPERV_HYPERVISOR_VENDOR_STRING);
   }
-   return FALSE; 
+   return FALSE;
 }

 /*
@@ -858,162 +754,4 @@ PerfCtr_PTAvailable(void)
   return FALSE;
 }

-/* The following are taken from the Intel Architecture Manual,
- * Book 3, Table 14-4. 
- */ 
-
-/* --------- BEGIN INTEL DEFINES ------------------------ */
-
-#define PENTIUM4_MSR_BPU_COUNTER0_IDX           0
-#define PENTIUM4_MSR_BPU_COUNTER0_ADDR          0x300
-#define PENTIUM4_MSR_BPU_CCCR0                  0x360
-#define PENTIUM4_MSR_BPU_COUNTER1_IDX           1
-#define PENTIUM4_MSR_BPU_COUNTER1_ADDR          0x301
-#define PENTIUM4_MSR_BPU_CCCR1                  0x361
-#define PENTIUM4_MSR_BPU_COUNTER2_IDX           2
-#define PENTIUM4_MSR_BPU_COUNTER2_ADDR          0x302
-#define PENTIUM4_MSR_BPU_CCCR2                  0x362
-#define PENTIUM4_MSR_BPU_COUNTER3_IDX           3
-#define PENTIUM4_MSR_BPU_COUNTER3_ADDR          0x303
-#define PENTIUM4_MSR_BPU_CCCR3                  0x363
-#define PENTIUM4_MSR_MS_COUNTER0_IDX            4
-#define PENTIUM4_MSR_MS_COUNTER0_ADDR           0x304
-#define PENTIUM4_MSR_MS_CCCR0                   0x364
-#define PENTIUM4_MSR_MS_COUNTER1_IDX            5
-#define PENTIUM4_MSR_MS_COUNTER1_ADDR           0x305
-#define PENTIUM4_MSR_MS_CCCR1                   0x365
-#define PENTIUM4_MSR_MS_COUNTER2_IDX            6
-#define PENTIUM4_MSR_MS_COUNTER2_ADDR           0x306
-#define PENTIUM4_MSR_MS_CCCR2                   0x366
-#define PENTIUM4_MSR_MS_COUNTER3_IDX            7
-#define PENTIUM4_MSR_MS_COUNTER3_ADDR           0x307
-#define PENTIUM4_MSR_MS_CCCR3                   0x367
-#define PENTIUM4_MSR_FLAME_COUNTER0_IDX         8
-#define PENTIUM4_MSR_FLAME_COUNTER0_ADDR        0x308
-#define PENTIUM4_MSR_FLAME_CCCR0                0x368
-#define PENTIUM4_MSR_FLAME_COUNTER1_IDX         9
-#define PENTIUM4_MSR_FLAME_COUNTER1_ADDR        0x309
-#define PENTIUM4_MSR_FLAME_CCCR1                0x369
-#define PENTIUM4_MSR_FLAME_COUNTER2_IDX         10
-#define PENTIUM4_MSR_FLAME_COUNTER2_ADDR        0x30A
-#define PENTIUM4_MSR_FLAME_CCCR2                0x36A
-#define PENTIUM4_MSR_FLAME_COUNTER3_IDX         11
-#define PENTIUM4_MSR_FLAME_COUNTER3_ADDR        0x30B
-#define PENTIUM4_MSR_FLAME_CCCR3                0x36B
-#define PENTIUM4_MSR_IQ_COUNTER0_IDX            12
-#define PENTIUM4_MSR_IQ_COUNTER0_ADDR           0x30C
-#define PENTIUM4_MSR_IQ_CCCR0                   0x36C
-#define PENTIUM4_MSR_IQ_COUNTER1_IDX            13
-#define PENTIUM4_MSR_IQ_COUNTER1_ADDR           0x30D
-#define PENTIUM4_MSR_IQ_CCCR1                   0x36D
-#define PENTIUM4_MSR_IQ_COUNTER2_IDX            14
-#define PENTIUM4_MSR_IQ_COUNTER2_ADDR           0x30E
-#define PENTIUM4_MSR_IQ_CCCR2                   0x36E
-#define PENTIUM4_MSR_IQ_COUNTER3_IDX            15
-#define PENTIUM4_MSR_IQ_COUNTER3_ADDR           0x30F
-#define PENTIUM4_MSR_IQ_CCCR3                   0x36F
-#define PENTIUM4_MSR_IQ_COUNTER4_IDX            16
-#define PENTIUM4_MSR_IQ_COUNTER4_ADDR           0x310
-#define PENTIUM4_MSR_IQ_CCCR4                   0x370
-#define PENTIUM4_MSR_IQ_COUNTER5_IDX            17
-#define PENTIUM4_MSR_IQ_COUNTER5_ADDR           0x311
-#define PENTIUM4_MSR_IQ_CCCR5                   0x371
-
-#define PENTIUM4_MSR_ALF_ESCR0_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR0_ADDR             0x3CA
-#define PENTIUM4_MSR_ALF_ESCR1_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR1_ADDR             0x3CB
-#define PENTIUM4_MSR_BPU_ESCR0_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR0_ADDR             0x3B2
-#define PENTIUM4_MSR_BPU_ESCR1_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR1_ADDR             0x3B3
-#define PENTIUM4_MSR_BSU_ESCR0_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR0_ADDR             0x3A0
-#define PENTIUM4_MSR_BSU_ESCR1_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR1_ADDR             0x3A1
-#define PENTIUM4_MSR_CRU_ESCR0_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR0_ADDR             0x3B8
-#define PENTIUM4_MSR_CRU_ESCR1_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR1_ADDR             0x3B9
-#define PENTIUM4_MSR_CRU_ESCR2_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR2_ADDR             0x3CC
-#define PENTIUM4_MSR_CRU_ESCR3_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR3_ADDR             0x3CD
-#define PENTIUM4_MSR_CRU_ESCR4_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR4_ADDR             0x3E0
-#define PENTIUM4_MSR_CRU_ESCR5_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR5_ADDR             0x3E1
-#define PENTIUM4_MSR_DAC_ESCR0_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR0_ADDR             0x3A8
-#define PENTIUM4_MSR_DAC_ESCR1_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR1_ADDR             0x3A9
-#define PENTIUM4_MSR_FIRM_ESCR0_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR0_ADDR            0x3A4
-#define PENTIUM4_MSR_FIRM_ESCR1_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR1_ADDR            0x3A5
-#define PENTIUM4_MSR_FLAME_ESCR0_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR0_ADDR           0x3A6
-#define PENTIUM4_MSR_FLAME_ESCR1_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR1_ADDR           0x3A7
-#define PENTIUM4_MSR_FSB_ESCR0_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR0_ADDR             0x3A2
-#define PENTIUM4_MSR_FSB_ESCR1_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR1_ADDR             0x3A3
-#define PENTIUM4_MSR_IQ_ESCR0_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR0_ADDR              0x3BA
-#define PENTIUM4_MSR_IQ_ESCR1_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR1_ADDR              0x3BB
-#define PENTIUM4_MSR_IS_ESCR0_IDX               1
-#define PENTIUM4_MSR_IS_ESCR0_ADDR              0x3B4
-#define PENTIUM4_MSR_IS_ESCR1_IDX               1
-#define PENTIUM4_MSR_IS_ESCR1_ADDR              0x3B5
-#define PENTIUM4_MSR_ITLB_ESCR0_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR0_ADDR            0x3B6
-#define PENTIUM4_MSR_ITLB_ESCR1_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR1_ADDR            0x3B7
-#define PENTIUM4_MSR_IX_ESCR0_IDX               5
-#define PENTIUM4_MSR_IX_ESCR0_ADDR              0x3C8
-#define PENTIUM4_MSR_IX_ESCR1_IDX               5
-#define PENTIUM4_MSR_IX_ESCR1_ADDR              0x3C9
-#define PENTIUM4_MSR_MOB_ESCR0_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR0_ADDR             0x3AA
-#define PENTIUM4_MSR_MOB_ESCR1_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR1_ADDR             0x3AB
-#define PENTIUM4_MSR_MS_ESCR0_IDX               0
-#define PENTIUM4_MSR_MS_ESCR0_ADDR              0x3C0
-#define PENTIUM4_MSR_MS_ESCR1_IDX               0
-#define PENTIUM4_MSR_MS_ESCR1_ADDR              0x3C1
-#define PENTIUM4_MSR_PMH_ESCR0_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR0_ADDR             0x3AC
-#define PENTIUM4_MSR_PMH_ESCR1_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR1_ADDR             0x3AD
-#define PENTIUM4_MSR_RAT_ESCR0_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR0_ADDR             0x3BC
-#define PENTIUM4_MSR_RAT_ESCR1_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR1_ADDR             0x3BD
-#define PENTIUM4_MSR_SAAT_ESCR0_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR0_ADDR            0x3AE
-#define PENTIUM4_MSR_SAAT_ESCR1_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR1_ADDR            0x3AF
-#define PENTIUM4_MSR_SSU_ESCR0_IDX              3
-#define PENTIUM4_MSR_SSU_ESCR0_ADDR             0x3BE
-#define PENTIUM4_MSR_TBPU_ESCR0_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR0_ADDR            0x3C2
-#define PENTIUM4_MSR_TBPU_ESCR1_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR1_ADDR            0x3C3
-#define PENTIUM4_MSR_TC_ESCR0_IDX               1
-#define PENTIUM4_MSR_TC_ESCR0_ADDR              0x3C4
-#define PENTIUM4_MSR_TC_ESCR1_IDX               1
-#define PENTIUM4_MSR_TC_ESCR1_ADDR              0x3C5
-#define PENTIUM4_MSR_U2L_ESCR0_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR0_ADDR             0x3B0
-#define PENTIUM4_MSR_U2L_ESCR1_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR1_ADDR             0x3B1
-
-#define PENTIUM4_MIN_ESCR_ADDR                  0x3A0
-#define PENTIUM4_MAX_ESCR_ADDR                  0x3E1
-#define PENTIUM4_NUM_ESCR_ADDRS                 (PENTIUM4_MAX_ESCR_ADDR - PENTIUM4_MIN_ESCR_ADDR + 1)
-
-/* -------- END INTEL DEFINES ------------------------- */
-
-#endif // ifndef _X86PERFCTR_H_ 
+#endif // _PERFCTR_ARCH_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -125,7 +125,7 @@ PTSC_MHz(void)

 #if defined(VM_X86_64) || defined(VM_ARM_64)

-/* 
+/*
 * Conversions to/from cycles.  Note that the conversions operate on
 * signed values, so be careful when taking the difference of two
 * VmAbsoluteTS (which is unsigned) that that value is not out of range
@@ -173,35 +173,6 @@ int64 PTSC_CyclesToUS(VmRelativeTS ts);
 */
 #include "user_layout.h"

-#ifdef VM_ARM_64
-static INLINE VmAbsoluteTS
-PTSC_Get(void)
-{
-   extern __thread User_ThreadData vmkUserTdata;
-
-   register RateConv_Params params;
-   register uint64 pseudoTSC;
-
-   /*
-    * On ARM64 the generic timer guarantees that the counters are synchronous
-    * and independent of CPU frequency.
-    */
-   params = vmkUserTdata.pseudoTSCConv;
-   pseudoTSC = RDTSC();
-
-   /*
-    *  assert that conversion rate is `identity'.
-    * we'll have to revisit that when/if this assumption changes
-    * on some platform
-    */
-   ASSERT((params.mult == 1 && params.shift == 0) ||
-          (params.mult == 0x80000000 && params.shift == 31));
-
-   pseudoTSC += params.add;
-
-   return pseudoTSC;
-}
-#else
 static INLINE VmAbsoluteTS
 PTSC_Get(void)
 {
@@ -215,7 +186,7 @@ PTSC_Get(void)
   ASSERT((int64)ptsc >= 0);
   return ptsc;
 }
-#endif
+
 #else

 /*
@@ -226,13 +197,13 @@ VmAbsoluteTS PTSC_Get(void);

 #endif

-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HasSynchronizedTSCs --
 *
 *      Returns TRUE iff the platform TSCs are known to be synchronized.
- * 
+ *
 *-----------------------------------------------------------------------------
 */

@@ -243,7 +214,7 @@ PTSC_HasSynchronizedTSCs(void)
 }


-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HostAdjustedTSCs --
@@ -260,4 +231,37 @@ PTSC_HostAdjustedTSCs(void)
   return ptscInfo.hwTSCsAdjusted;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PTSC_AdvanceTimer --
+ *
+ *      Advance '*deadline' in 'period' increments such that it is
+ *      greater than 'now'.  Return the number of ticks incremented.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE uint64
+PTSC_AdvanceTimer(VmAbsoluteTS now,
+                  VmIntervalTS period,
+                  VmAbsoluteTS *deadline)
+{
+   VmAbsoluteTS d = *deadline;
+   if (now >= d) {
+      uint64 count = 1;
+      d += period;
+      if (UNLIKELY(now >= d)) {
+         uint64 t = (now - d) / period + 1;
+         VmIntervalTS diff = t * period;
+         count += t;
+         d += diff;
+      }
+      *deadline = d;
+      return count;
+   }
+   return 0;
+}
+
 #endif /* ifndef _PTSC_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,7 +97,7 @@ void RateConv_LogRatio(const char *prefix,
 static INLINE uint64
 RateConv_Unsigned(const RateConv_Params *conv, uint64 x)
 {
-   return Mul64x3264(x, conv->mult, conv->shift) + conv->add;
+   return Mul64x3264(x, conv->mult, conv->shift) + (uint64)conv->add;
 }


@@ -0,0 +1,176 @@
+/*********************************************************
+ * Copyright (C) 2018-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * segs.h --
+ *
+ *      Describes segment descriptors and structures containing them for both
+ *      the monitor (in hosted and ESX) and the vmkernel.
+ */
+
+#ifndef _SEGS_H_
+#define _SEGS_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "x86types.h"
+#include "x86segdescrs.h"
+#include "x86sel.h"
+#include "addrlayout.h"
+
+
+/*
+ * Segment map of the monitor.
+ *
+ * The GDT and Task State Segment reside consecutively on one page.  The
+ * monitor segments are placed at the end of the GDT.  The vmkernel can use all
+ * lower-numbered segments for user-mode as well as higher-numbered segments
+ * (though the vmkernel should not use monitor-private segments).  The high
+ * segment placement ensures that there is no selector-overlap with hosted
+ * kernel segments (the hosted world switch code can be a bit faster then, as
+ * it can use a single cross GDT).
+ */
+
+#define VMMON_GDT_SIZE        (sizeof(Descriptor) * NUM_VALID_SEGMENTS)
+#define VMMON_GDT_LIMIT       (VMMON_GDT_SIZE - 1)
+
+#define IRB_SIZE              32 /* Interrupt redirection bitmap. */
+#define TSS_SIZE              (sizeof(Task64) + IRB_SIZE)
+
+#define GDT_START_VA          (VPN_2_VA(GDT_AND_TASK_START))
+#define TASK_START_VA         (VPN_2_VA(GDT_AND_TASK_START) + VMMON_GDT_SIZE)
+
+/*
+ * vmkBoot uses some of the lower-numbered segments, as do host kernels on
+ * hosted.  User segments could start earlier than 32; see bug #1904257.
+ * Descriptor is used to describe boot, user and kernel segments.
+ * Descriptor64 (twice as large) is used for task segments.
+ */
+#define NUM_BOOT_SEGMENTS     32
+#define NUM_USER_SEGMENTS     (FIRST_SYSTEM_SEGMENT - FIRST_USER_SEGMENT)
+#define NUM_SYSTEM_SEGMENTS   2
+#define NUM_TASK_SEGMENTS     2
+
+#define FIRST_USER_SEGMENT    NUM_BOOT_SEGMENTS
+#define FIRST_SYSTEM_SEGMENT  (PAGE_SIZE / sizeof(Descriptor) - \
+                               NUM_SYSTEM_SEGMENTS            - \
+                               NUM_TASK_SEGMENTS * 2          - \
+                               TSS_SIZE / sizeof(Descriptor))
+
+#define GDT_USER_TLS_MIN      USER_TLS_1_SEGMENT
+#define GDT_USER_TLS_MAX      USER_TLS_3_SEGMENT
+#define USER_TLS_COUNT        ((USER_TLS_3_SEGMENT - USER_TLS_1_SEGMENT) + 1)
+
+#define FOREACH_USER_TLS_INDEX(_i)              \
+   {                                            \
+      unsigned _i;                              \
+      for (_i = 0; _i < USER_TLS_COUNT; _i++) { \
+
+#define FOREACH_USER_TLS_INDEX_DONE             \
+      }                                         \
+   }
+
+#define NULL_LDTR             0
+
+/*
+ * The descriptor after SYSTEM_CODE_SEGMENT (loaded into %cs) must be
+ * appropriate for %ss because of the syscall instruction for 64-bit
+ * user worlds.  Thus SYSTEM_DATA_SEGMENT is directly after it.
+ */
+typedef enum VmwSegs {
+   NULL_SEGMENT             = 0,
+   /* (... reserved for host operating system or vmkBoot segments). */
+
+   USER32_CODE_SEGMENT      = FIRST_USER_SEGMENT,
+   USER_DATA_SEGMENT,
+   USER64_SYSRET_SEGMENT,
+   USER64_STACK_SEGMENT,
+   USER64_CODE_SEGMENT,
+   USER_TLS_1_SEGMENT,
+   USER_TLS_2_SEGMENT,
+   USER_TLS_3_SEGMENT,
+
+   AFTER_LAST_USER_SEGMENT,
+
+   SYSTEM_CODE_SEGMENT      = FIRST_SYSTEM_SEGMENT,
+   SYSTEM_DATA_SEGMENT,
+   VMKERNEL_TASK_SEGMENT,
+   VMKERNEL_TASK_SEGMENT_HI,
+   MONITOR_TASK_SEGMENT,
+   MONITOR_TASK_SEGMENT_HI,
+
+   NUM_VALID_SEGMENTS
+} VmwSegs;
+
+#define GDT_SYSTEM_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_SYSTEM_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_USER_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 3)
+#define GDT_USER_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 3)
+
+/* Selectors used statically in code or in assembly must be unchecked. */
+#define SYSTEM_NULL_SELECTOR    GDT_SYSTEM_SEL(NULL)
+#define USER32_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER32_CODE)
+#define USER_DATA_SELECTOR      GDT_USER_SEL_UNCHECKED(USER_DATA)
+#define USER64_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER64_CODE)
+#define USER64_SYSRET_SELECTOR  GDT_USER_SEL(USER64_SYSRET)
+#define SYSTEM_CODE_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_CODE)
+#define SYSTEM_DATA_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_DATA)
+#define MONITOR_TASK_SELECTOR   GDT_SYSTEM_SEL(MONITOR_TASK)
+#define VMKERNEL_TASK_SELECTOR  GDT_SYSTEM_SEL(VMKERNEL_TASK)
+
+/*
+ * The VMM GDT is comprised of many segment descriptors with one initial
+ * Task State Segment system descriptor.  The VMM Task State Segment is
+ * on the same page sequentially after its GDT.
+ */
+#pragma pack(push, 1)
+typedef struct StaticGDTAndTSS {
+   Descriptor   empty[NUM_BOOT_SEGMENTS + NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS];
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+   Task64       monTSS;
+   uint8        TSSIRBitmap[IRB_SIZE];
+} StaticGDTAndTSS;
+#pragma pack(pop)
+
+#pragma pack(push, 1)
+typedef struct VmkernelGDT {
+   Descriptor   bootSegs[NUM_BOOT_SEGMENTS];
+   Descriptor   userSegs[NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS]; /* VMM/VMK-shared. */
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+} VmkernelGDT;
+#pragma pack(pop)
+
+MY_ASSERTS(segs,
+           ASSERT_ON_COMPILE(SYSTEM_CODE_SEGMENT + 1 == SYSTEM_DATA_SEGMENT);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT - FIRST_USER_SEGMENT <=
+                             NUM_USER_SEGMENTS);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT <= FIRST_SYSTEM_SEGMENT);
+)
+
+#endif /* _SEGS_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,28 +16,32 @@
 *
 *********************************************************/

-#ifndef _INITBLOCK_H
-#define _INITBLOCK_H
+/*
+ * sharedAreaType.h --
+ *
+ *      This file contains shared area type definitions.
+ */

+#ifndef _SHAREDAREATYPE_H_
+#define _SHAREDAREATYPE_H_

-#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+
 #include "includeCheck.h"

+typedef enum {
+   SHARED_AREA_PER_VM_VMX = 0,
+   SHARED_AREA_INTER_VCPU_VMX,
+   SHARED_AREA_PER_VCPU_VMX,
+   SHARED_AREA_PER_VM,
+   SHARED_AREA_INTER_VCPU,
+   SHARED_AREA_PER_VCPU,
+   NUM_SHARED_AREAS
+} SharedAreaType;

-#include "basic_initblock.h"
-
-
-#define MAX_LOGFILE_NAME_LENGTH 256
-#define MAX_MONITOR_REGIONS     8
-#define INIT_BLOCK_MAGIC     (0x1789+14)
-
-
-/*
- * Option flags 
- */
-/* None yet */
-
-
-#endif
+#endif // _SHAREDAREATYPE_H_
@@ -0,0 +1,62 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.h --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifndef SHARED_AREA_VMMON_H
+#define SHARED_AREA_VMMON_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#include "includeCheck.h"
+
+#include "iocontrols.h"
+
+struct VMDriver;
+
+/* A collection of pages backing a shared area region. */
+typedef struct SharedAreaVmmonRegion {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} SharedAreaVmmonRegion;
+
+/* The VMMon Driver shared area. */
+typedef struct SharedAreaVmmon {
+   SharedAreaVmmonRegion regions[NUM_SHARED_AREAS];
+} SharedAreaVmmon;
+
+/* Request for a backing MPN for a shared area region at a given VCPU/offset. */
+typedef struct SharedAreaVmmonRequest {
+   SharedAreaType type;
+   Vcpuid vcpu;
+   PageCnt offset;
+} SharedAreaVmmonRequest;
+
+SharedAreaVmmon *SharedAreaVmmon_Init(struct VMDriver *vm);
+void SharedAreaVmmon_Cleanup(SharedAreaVmmon *area);
+Bool SharedAreaVmmon_RegisterRegion(struct VMDriver *driver,
+                                    VMSharedAreaRegistrationBlock *block);
+Bool SharedAreaVmmon_ValidateRegionArgs(struct VMDriver *driver,
+                                        VMSharedAreaRegistrationBlock *block);
+MPN SharedAreaVmmon_GetRegionMPN(struct VMDriver *vm,
+                                 SharedAreaVmmonRequest *request);
+#endif /* SHARED_AREA_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,45 +17,32 @@
 *********************************************************/

 /*
- * basic_initblock.h --
+ * statVarsVmmon.h --
 *
- *    VM initialization block.
+ *     VMMon stat vars management.
 */

-#ifndef _BASIC_INITBLOCK_H_
-#define _BASIC_INITBLOCK_H_
+#ifndef STAT_VARS_VMMON_H
+#define STAT_VARS_VMMON_H

-
-#define INCLUDE_ALLOW_USERLEVEL
-
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_DISTRIBUTE
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

+#include "iocontrols.h"

-#include "vcpuid.h"
+struct VMDriver;

+/* The VMMon Driver stat vars area. */
+typedef struct StatVarsVmmon {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} StatVarsVmmon;

-#define MAX_INITBLOCK_CPUS     MAX_VCPUS
-
-
-typedef
-#include "vmware_pack_begin.h"
-struct InitBlock {
-   uint32 magicNumber;     /* Magic number (INIT_BLOCK_MAGIC) */
-   uint32 vmInitFailurePeriod;
-   VA64   crosspage[MAX_INITBLOCK_CPUS];
-   LPN64  monStartLPN;
-   LPN64  monEndLPN;
-   MPN    crossGDTMPN;
-   uint16 numPTPPages;
-}
-#include "vmware_pack_end.h"
-InitBlock;
-
-
-#endif // _BASIC_INITBLOCK_H_
+StatVarsVmmon *StatVarsVmmon_Init(struct VMDriver *vm);
+void StatVarsVmmon_Cleanup(StatVarsVmmon *statVars);
+Bool StatVarsVmmon_RegisterVCPU(struct VMDriver *driver,
+                                VMStatVarsRegistrationBlock *block);
+MPN StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpuid,
+                               PageCnt offset);
+#endif /* STAT_VARS_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -20,15 +20,12 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-UC(CALL_START)
-UC(BEGIN_BACK_TO_HOST)
-UC(SWITCHED_TO_MODULE)
-UC(VMX_SELECT_RETURN)
-UC(VMX_HANDLER_START)
-UC(VMX_SIGNAL)
-UC(MODULE_SIGNAL)
-UC(SWITCHING_TO_MONITOR)
-UC(DONE_BACK_TO_HOST)
-UC(CALL_END)
+UC(CALL_START,             FALSE)
+UC(BEGIN_BACK_TO_HOST,     FALSE)
+UC(SWITCHED_TO_MODULE,     TRUE)
+UC(VMX_HANDLER_START,      FALSE)
+UC(SWITCHING_TO_MONITOR,   TRUE)
+UC(DONE_BACK_TO_HOST,      FALSE)
+UC(CALL_END,               FALSE)

 #undef UC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 #ifndef USERCALLDEFS_H
 #define USERCALLDEFS_H

-#ifdef linux
+#ifdef __linux__
 #include <linux/errno.h>
 #define USERCALL_RESTART (-ERESTARTNOINTR)
 #else
@@ -31,5 +31,6 @@
 * failing DeviceIoControl on Windows.
 */
 #define USERCALL_VMX86ALLOCERR (-8192)
+#define USERCALL_SWITCHERR     (-8193)

-#endif 
+#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,6 @@
 extern "C" {
 #endif

-
 typedef uint32 Vcpuid;                 // VCPU number

 #define VCPUID_INVALID  (~0U)
@@ -54,17 +53,14 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define MAX_CORES_PER_SOCKET   64
 #define MAX_VCPU_SOCKETS      128

-#ifdef VMM
-#include "vcpuset.h"
+/*
+ * There are several properties of the VM which change at the 128 VCPU
+ * boundary.  EFI firmware, x2APIC, and IOMMU are required among others.
+ */
+#define MAX_SMALL_VM_VCPUS 128

-/* In VMM, CurVcpuid() is available everywhere. */
-extern const Vcpuid      curVcpuid;
-extern const VCPUSet     curVcpuidSet;
-#define CurVcpuid()         (curVcpuid)
-#define CurVcpuidSet()      (&curVcpuidSet)
-#define IS_BOOT_VCPU()      IS_BOOT_VCPUID(CurVcpuid())
-
-#endif  /* VMM */
+/* Supported limit. */
+#define MAX_SUPPORTED_VCPUS   (FEATURE_STATE_IS_ENABLED(MonsterVM) ? 768 : 256)

 #if defined __cplusplus
 } // extern "C"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,10 +44,11 @@
 #   include "str.h"     /* Str_Snprintf */
 #   define VCS_SNPRINTF Str_Snprintf
 #elif defined MONITOR_APP
-#   include <stdio.h>   /* libc snprintf */
 #   if defined WIN32
-#      define VCS_SNPRINTF _snprintf
+#      include "str.h"     /* Str_Snprintf */
+#      define VCS_SNPRINTF Str_Snprintf
 #   else
+#      include <stdio.h>   /* libc snprintf */
 #      define VCS_SNPRINTF snprintf
 #   endif
 #elif defined VMM || defined VMKERNEL
@@ -112,7 +113,7 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET(_setIndex)                                     \
   do {                                                                       \
-      int _setIndex;                                                          \
+      unsigned _setIndex;                                                     \
      for (_setIndex = 0; _setIndex < VCS_SUBSET_COUNT; _setIndex++) {

 #define ROF_EACH_SUBSET_IN_SET()                                              \
@@ -122,8 +123,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET_COUNTDOWN(_setIndex)                           \
   do {                                                                       \
-      int _setIndex;                                                          \
-      for (_setIndex = VCS_SUBSET_COUNT - 1; _setIndex >= 0; _setIndex--) {
+      unsigned _setIndex = VCS_SUBSET_COUNT;                                  \
+      while (_setIndex-- > 0) {

 #define ROF_EACH_SUBSET_IN_SET_COUNTDOWN()                                    \
      }                                                                       \
@@ -132,8 +133,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex, _numVcpus)                \
   do {                                                                       \
-      int _setIndex;                                                          \
-      int _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);                 \
+      unsigned _setIndex;                                                     \
+      unsigned _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);            \
      for (_setIndex = 0; _setIndex <= _maxSubsets; _setIndex++) {

 #define ROF_EACH_POPULATED_SUBSET_IN_SET()                                    \
@@ -307,7 +308,7 @@ VCPUSet_FindFirst(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return lssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return lssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET();
   return VCPUID_INVALID;
@@ -319,7 +320,7 @@ VCPUSet_FindLast(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET_COUNTDOWN(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return mssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return mssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET_COUNTDOWN();
   return VCPUID_INVALID;
@@ -356,8 +357,7 @@ VCPUSet_FindFirstInSubset(const VCPUSet *vcs, uint64 *subset,
   ASSERT(*subsetIdx < maxSubsets && maxSubsets <= VCS_SUBSET_COUNT);
   do {
      if (*subset != 0) {
-         int bit;
-         bit = lssb64_0(*subset);
+         unsigned bit = (unsigned)lssb64_0(*subset);
         *subset &= ~(CONST64U(1) << bit);
         return bit + (*subsetIdx << VCS_SUBSET_SHIFT);
      }
@@ -672,7 +672,7 @@ static INLINE Vcpuid
 VCPUSet_FindSingleton(const VCPUSet *vcs)
 {
   uint64 foundSub = 0;
-   uint32 foundIdx;
+   uint32 foundIdx = 0;
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 sub = vcs->subset[idx];
      if (sub != 0) {
@@ -683,8 +683,11 @@ VCPUSet_FindSingleton(const VCPUSet *vcs)
         foundIdx = idx;
      }
   } ROF_EACH_SUBSET_IN_SET();
-   return foundSub != 0 ? lssb64_0(foundSub) + (foundIdx << VCS_SUBSET_SHIFT) :
-                          VCPUID_INVALID;
+   if (foundSub != 0) {
+      return lssb64(foundSub) - 1 + (foundIdx << VCS_SUBSET_SHIFT);
+   } else {
+      return VCPUID_INVALID;
+   }
 }


@@ -863,27 +866,27 @@ VCPUSet_Intersection(VCPUSet *dest, const VCPUSet *src)

 #ifdef VCS_SNPRINTF
 static INLINE char *
-VCPUSet_LogFormat(char *buf, const int size, const VCPUSet *vcs)
+VCPUSet_LogFormat(char *buf, size_t size, const VCPUSet *vcs)
 {
-   int offset = 0;
-   Vcpuid highest = VCPUSet_FindLast(vcs);
-   int idx = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+   unsigned offset  = 0;
+   Vcpuid   highest = VCPUSet_FindLast(vcs);
+   unsigned idx     = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+
   ASSERT(size >= VCS_BUF_SIZE);
 #define VCS_LOGF(...)                                                   \
   {                                                                    \
      int ret = VCS_SNPRINTF(buf + offset, size - offset, __VA_ARGS__); \
-      ASSERT(0 <= ret && ret < size - offset);                          \
-      offset += ret;                                                    \
+      ASSERT(0 <= ret && size >= offset && ret < (int)(size - offset)); \
+      offset += (unsigned)ret;                                          \
   }
   /* Print the leading value with no zero-extension. */
-   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx--]);
+   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx]);

-   while (idx >= 0) {
+   while (idx-- > 0) {
      if ((idx + 1) % (VCS_SUBSET_WIDTH / 8) == 0) {
         VCS_LOGF(".");
      }
      VCS_LOGF("%02x", ((unsigned char *)vcs)[idx]);
-      idx--;
   }
   return buf;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,52 +41,47 @@

 #include "vm_basic_asm.h"
 #include "x86msr.h"
-
-#ifdef VM_X86_64
-#include "vm_asm_x86_64.h"
-#else
 #include "vm_asm_x86.h"
-#endif


 static INLINE void
 SET_FS64(uint64 fs64)
 {
-   __SET_MSR(MSR_FSBASE, fs64);
+   X86MSR_SetMSR(MSR_FSBASE, fs64);
 }


 static INLINE void
 SET_GS64(uint64 gs64)
 {
-   __SET_MSR(MSR_GSBASE, gs64);
+   X86MSR_SetMSR(MSR_GSBASE, gs64);
 }

 static INLINE void
 SET_KernelGS64(uint64 kgs64)
 {
-   __SET_MSR(MSR_KERNELGSBASE, kgs64);
+   X86MSR_SetMSR(MSR_KERNELGSBASE, kgs64);
 }


 static INLINE uint64
 GET_FS64(void)
 {
-   return __GET_MSR(MSR_FSBASE);
+   return X86MSR_GetMSR(MSR_FSBASE);
 }


 static INLINE uint64
 GET_GS64(void)
 {
-   return __GET_MSR(MSR_GSBASE);
+   return X86MSR_GetMSR(MSR_GSBASE);
 }


 static INLINE uint64
 GET_KernelGS64(void)
 {
-   return __GET_MSR(MSR_KERNELGSBASE);
+   return X86MSR_GetMSR(MSR_KERNELGSBASE);
 }

 #endif // VM_ARM_64
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * vm_asm_x86.h
 *
- *	IA32 asm macros
+ *      IA32 asm macros
 */

 #ifndef _VM_ASM_X86_H_
@@ -49,10 +49,6 @@

 #ifdef __GNUC__

-#if defined __APPLE__
-/* PR 352418: GCC produces error if the non-Apple version is used */
-#define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
-#else
 /* ASSERT_ON_COMPILE_SELECTOR_SIZE:
 *
 *  - Selector must be 16-bits.
@@ -66,26 +62,16 @@
 *                    ((__builtin_constant_p(expr) ? ((expr) >> 16) == 0) \
 *                                                 : sizeof(expr) <= 2)
 */
-#if (__GNUC__ >= 4) && (__GNUC_MINOR__ >= 1) && !defined(USE_UBSAN)
+#ifndef USE_UBSAN
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)                                \
   ASSERT_ON_COMPILE(sizeof(Selector) == 2 &&                                \
                     ((__builtin_constant_p(expr) && ((expr) >> 16) == 0) || \
                      sizeof(expr) <= 2))
 #else
-/* gcc 3.3.3 is not able to produce a constant expression (PR 356383) */
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
 #endif
-#endif


-/*
- * [GS]ET_[GI]DT() are defined as macros wrapping a function
- * so we can pass the argument implicitly by reference (requires
- * a macro) and get type checking too (requires a function).
- */
-
-#define SET_GDT(var) _Set_GDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
@@ -97,8 +83,6 @@ _Set_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define SET_IDT(var) _Set_IDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_IDT(_GETSET_DTR_TYPE *dtr)
@@ -110,8 +94,6 @@ _Set_IDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_GDT(var) _Get_GDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because there's a hidden input (the [IG]DTR) that can change
@@ -126,8 +108,6 @@ _Get_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_IDT(var) _Get_IDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because the [IG]DT can change without the compiler knowing it
@@ -175,7 +155,6 @@ _GET_LDT(Selector * const result)
   } while (0)


-/* Checked against the Intel manual and GCC --thutt */
 #define _BUILD_SET_R(func, reg)        \
   static INLINE void                  \
   func(uintptr_t r)                   \
@@ -186,24 +165,20 @@ _GET_LDT(Selector * const result)
              : "memory");             \
   }

-/* Not yet checked against the Intel manual and GCC --slava
- *
- * 'volatile' because CRs and DRs can change without the compiler
- * knowing it (when there is a page fault, when a breakpoint occurs,
- * and moreover it seems there is no way to teach gcc that smsw
- * clobbers cr0 for example).
- *
- * The parameter is a 'uintptr_t *' so that the size of the actual
- * parameter must exactly match the size of the hardware register.
- * This prevents the use of 32-bit variables when building 64-bit
- * code.
+/*
+ * The inline asm is marked 'volatile' because CRs and DRs can change
+ * without the compiler knowing it (when there is a page fault, when a
+ * breakpoint occurs, and moreover it seems there is no way to teach
+ * gcc that smsw clobbers cr0 for example).
 */
 #define _BUILD_GET_R(func, reg)                         \
-   static INLINE void                                   \
-   func(uintptr_t *result)                              \
+   static INLINE uintptr_t                              \
+   func(void)                                           \
   {                                                    \
+      uintptr_t result;                                 \
      __asm__ __volatile__("mov %%" #reg ", %0"         \
-                           : "=r" (*result));           \
+                           : "=r" (result));            \
+      return result;                                    \
   }

 _BUILD_SET_R(_SET_CR0, cr0)
@@ -218,22 +193,6 @@ _BUILD_GET_R(_GET_CR3, cr3)
 _BUILD_GET_R(_GET_CR4, cr4)
 _BUILD_GET_R(_GET_CR8, cr8)

-#if defined __APPLE__
-/* Mac OS gcc 4 uses DBx instead of DRx register names. */
-_BUILD_SET_R(_SET_DR0, db0)
-_BUILD_SET_R(_SET_DR1, db1)
-_BUILD_SET_R(_SET_DR2, db2)
-_BUILD_SET_R(_SET_DR3, db3)
-_BUILD_SET_R(_SET_DR6, db6)
-_BUILD_SET_R(_SET_DR7, db7)
-
-_BUILD_GET_R(_GET_DR0, db0)
-_BUILD_GET_R(_GET_DR1, db1)
-_BUILD_GET_R(_GET_DR2, db2)
-_BUILD_GET_R(_GET_DR3, db3)
-_BUILD_GET_R(_GET_DR6, db6)
-_BUILD_GET_R(_GET_DR7, db7)
-#else
 _BUILD_SET_R(_SET_DR0, dr0)
 _BUILD_SET_R(_SET_DR1, dr1)
 _BUILD_SET_R(_SET_DR2, dr2)
@@ -247,47 +206,8 @@ _BUILD_GET_R(_GET_DR2, dr2)
 _BUILD_GET_R(_GET_DR3, dr3)
 _BUILD_GET_R(_GET_DR6, dr6)
 _BUILD_GET_R(_GET_DR7, dr7)
-#endif

-#define SET_CR_DR(regType, regNum, expr)                     \
-   do {                                                      \
-      /* Ensure no implicit truncation of 'expr' */          \
-      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
-      _SET_##regType##regNum(expr);                          \
-   } while (0)
-
-#define GET_CR_DR(regType, regNum, var) \
-   do {                                 \
-      _GET_##regType##regNum(&(var));   \
-   } while (0)
-
-#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
 #define SET_CR2(expr) SET_CR_DR(CR, 2, expr)
-#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
-#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
-#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
-
-/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
-#undef GET_CR0
-#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
-#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
-#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
-#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
-#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
-
-#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
-#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
-#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
-#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
-#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
-#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
-
-#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
-#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
-#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
-#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
-#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
-#define GET_DR7(var)  GET_CR_DR(DR, 7, var)

 #define SET_SEGREG(reg, expr)                                          \
   do {                                                                \
@@ -426,440 +346,200 @@ CLTS(void)
 /* Beginning of the section whose correctness has NOT been checked */
 #define FNCLEX()         __asm__("fnclex" ::);

-/* TLB_INVALIDATE_xxx are not checked yet */
+/* TLB_INVALIDATE_PAGE is not checked yet */
 #define TLB_INVALIDATE_PAGE(_addr) do { \
     __asm__ __volatile__("invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
 } while (0)

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) do { \
-     __asm__ __volatile__("fs; invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
-} while (0)
-
 #define RESTORE_FLAGS _Set_flags
 #define ENABLE_INTERRUPTS() __asm__ __volatile__ ("sti": : :"memory")
 #define CLEAR_INTERRUPTS()  __asm__ __volatile__ ("cli": : :"memory")
 #define RAISE_INTERRUPT(_x)  __asm__ __volatile__("int %0" :: "g" (_x))
 #define RETURN_FROM_INT()   __asm__ __volatile__("iret" :: )

-#if ! defined(VMKERNEL)
-#define NO_INTERRUPTS_BEGIN()	do { \
-                                   uintptr_t _flags; \
-                                   SAVE_FLAGS(_flags); \
-                                   CLEAR_INTERRUPTS();
-
-#define NO_INTERRUPTS_END()	   RESTORE_FLAGS(_flags); \
-                                } while(0)
-#endif
-
 /* End of the section whose correctness has NOT been checked */

 #elif defined _MSC_VER  /* !__GNUC__ */

-#define SET_IDT(_idt)    _Set_IDT(&(_idt))
-#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
-#define SET_TR(_tr)      _Set_TR(_tr)
-#define SET_LDT(_tr)     _Set_LDT(_tr)
+#define _BUILD_SET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writedr(reg, r);                              \
+   }

-#define GET_IDT(_idt)    _Get_IDT(&(_idt))
-#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
-#define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
-#define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)
+#define _BUILD_GET_DR(func, reg)                      \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
+   {                                                  \
+      return __readdr(reg);                           \
+   }

-#define GET_CR0(_reg)    __asm mov eax, cr0 __asm mov _reg, eax
-#define SET_CR0(_reg)    __asm mov eax, _reg __asm mov cr0, eax
-#define GET_CR2(_reg)    __asm mov eax, cr2 __asm mov _reg, eax
-#define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
-#define GET_CR3(_reg)    __asm mov eax, cr3 __asm mov _reg, eax
-#define SET_CR3(_reg)    __asm mov eax, _reg __asm mov cr3, eax
-/*
- * MSC doesn't seem to like CR4 in __asm statements. We emit
- * the opcode for MOV EAX,CR4 = 0xf020e0 and MOV CR4,EAX = 0xf022e0
- */
-#define GET_CR4(_reg) { \
- __asm _emit 0x0f __asm _emit 0x20 __asm _emit 0xe0 \
- __asm mov _reg, eax \
-}
-#define SET_CR4(_reg)    { \
-  __asm mov eax, _reg \
-  __asm _emit 0x0f __asm _emit 0x22 __asm _emit 0xe0 \
-}
+#define _BUILD_SET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writecr##reg(r);                              \
+   }

+#define _BUILD_GET_CR(func, reg)                      \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
+   {                                                  \
+      return __readcr##reg();                         \
+   }

-#define GET_DR0(_reg) do { __asm mov eax,dr0 __asm mov _reg,eax } while (0)
-#define SET_DR0(_reg) do { __asm mov eax,_reg __asm mov dr0,eax } while (0)
-#define GET_DR1(_reg) do { __asm mov eax,dr1 __asm mov _reg,eax } while (0)
-#define SET_DR1(_reg) do { __asm mov eax,_reg __asm mov dr1,eax } while (0)
-#define GET_DR2(_reg) do { __asm mov eax,dr2 __asm mov _reg,eax } while (0)
-#define SET_DR2(_reg) do { __asm mov eax,_reg __asm mov dr2,eax } while (0)
-#define GET_DR3(_reg) do { __asm mov eax,dr3 __asm mov _reg,eax } while (0)
-#define SET_DR3(_reg) do { __asm mov eax,_reg __asm mov dr3,eax } while (0)
-#define GET_DR6(_reg) do { __asm mov eax,dr6 __asm mov _reg,eax } while (0)
-#define SET_DR6(_reg) do { __asm mov eax,_reg __asm mov dr6,eax } while (0)
-#define GET_DR7(_reg) do { __asm mov eax,dr7 __asm mov _reg,eax } while (0)
-#define SET_DR7(_reg) do { __asm mov eax,_reg __asm mov dr7,eax } while (0)
+_BUILD_SET_DR(_SET_DR0, 0)
+_BUILD_SET_DR(_SET_DR1, 1)
+_BUILD_SET_DR(_SET_DR2, 2)
+_BUILD_SET_DR(_SET_DR3, 3)
+_BUILD_SET_DR(_SET_DR6, 6)
+_BUILD_SET_DR(_SET_DR7, 7)

+_BUILD_GET_DR(_GET_DR0, 0)
+_BUILD_GET_DR(_GET_DR1, 1)
+_BUILD_GET_DR(_GET_DR2, 2)
+_BUILD_GET_DR(_GET_DR3, 3)
+_BUILD_GET_DR(_GET_DR6, 6)
+_BUILD_GET_DR(_GET_DR7, 7)

-#define CLTS()           __asm clts
-
-#define FNCLEX()         __asm fnclex
-
-#define TLB_INVALIDATE_PAGE(_addr) {  \
-	 void *_a = (_addr); \
-     __asm mov eax, _a __asm invlpg [eax] \
-}
-
-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) { \
-	uint32 __a = (uint32) (_addr); \
-	__asm mov eax, __a _asm invlpg fs:[eax] \
-}
-
-
-#define ENABLE_INTERRUPTS() { __asm sti }
-#define CLEAR_INTERRUPTS()  { __asm cli }
-
-#define RAISE_INTERRUPT(_x)  {__asm int _x }
-#define RETURN_FROM_INT()   {__asm iretd }
-
-
-#define SAVE_FLAGS(x) { \
-     __asm pushfd __asm pop eax __asm mov x, eax \
-}
-
-#define RESTORE_FLAGS(x) { \
-     __asm push x __asm popfd\
-}
-
-
-
-static INLINE void SET_DS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ds, ax
-}
-
-static INLINE void SET_ES(Selector val)
-{
-   __asm mov ax, val
-   __asm mov es, ax
-}
-
-static INLINE void SET_FS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov fs, ax
-}
-
-static INLINE void SET_GS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov gs, ax
-}
-
-static INLINE void SET_SS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ss, ax
-}
-
-static INLINE Selector GET_FS(void)
-{
-	Selector _v;
-	__asm mov _v,fs
-	return _v;
-}
-
-static INLINE Selector GET_GS(void)
-{
-	Selector _v;
-	__asm mov _v,gs
-	return _v;
-}
-
-
-static INLINE Selector GET_DS(void)
-{
-	Selector _v;
-	__asm mov _v,ds
-	return _v;
-}
-
-static INLINE Selector GET_ES(void)
-{
-	Selector _v;
-	__asm mov _v,es
-	return _v;
-}
-
-static INLINE Selector GET_SS(void)
-{
-	Selector _v;
-	__asm mov _v,ss
-	return _v;
-}
-
-static INLINE Selector GET_CS(void)
-{
-	Selector _v;
-	__asm mov _v,cs
-	return _v;
-}
-
-#pragma warning( disable : 4035)
-
-static INLINE uint32  GET_WORD_FROM_FS(uint32 *_addr) {
-	__asm mov eax, _addr
-    __asm mov eax, fs:[eax]
-}
-
-static INLINE uint16  GET_SHORT_FROM_FS(uint16 *_addr) {
-	__asm mov eax, _addr
-    __asm mov ax, fs:[eax]
-}
-
-static INLINE uint8  GET_BYTE_FROM_FS(uint8 *_addr) {
-	__asm mov eax, _addr
-     __asm mov al, fs:[eax]
-}
-
-#pragma warning (default: 4035)
-
-static INLINE void  SET_WORD_FS(uint32 *_addr, uint32 _val) {
-    __asm mov eax, _addr
-    __asm mov ebx, _val
-    __asm mov fs:[eax], ebx
-}
-
-static INLINE void  SET_SHORT_FS(uint32 *_addr, uint16 _val) {
-    __asm mov eax, _addr
-    __asm mov bx, _val
-    __asm mov fs:[eax], bx
-}
-
-static INLINE void  SET_BYTE_FS(uint32 *_addr, uint8 _val) {
-    __asm mov eax, _addr
-    __asm mov bl, _val
-    __asm mov fs:[eax], bl
-}
-
-static INLINE void _Set_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lgdt [eax]
-}
-
-static INLINE void _Set_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lidt [eax]
-}
-
-static INLINE void _Set_LDT(Selector val)
-{
-   __asm lldt val
-}
-
-static INLINE void _Set_TR(Selector val)
-{
-   __asm ltr val
-}
-
-static INLINE void _Get_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sgdt [eax]
-}
-
-static INLINE void _Get_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sidt [eax]
-}
-
-static INLINE Selector _Get_LDT(void) {
-   Selector sel;
-   __asm sldt sel
-   return sel;
-}
-
-static INLINE Selector _Get_TR(void) {
-   Selector sel;
-   __asm str sel
-   return sel;
-}
+_BUILD_SET_CR(_SET_CR0, 0);
+_BUILD_SET_CR(_SET_CR3, 3);
+_BUILD_SET_CR(_SET_CR4, 4);
+_BUILD_SET_CR(_SET_CR8, 8);

+_BUILD_GET_CR(_GET_CR0, 0);
+_BUILD_GET_CR(_GET_CR2, 2);
+_BUILD_GET_CR(_GET_CR3, 3);
+_BUILD_GET_CR(_GET_CR4, 4);
+_BUILD_GET_CR(_GET_CR8, 8);

 static INLINE void
-MEMCOPY_TO_FS(VA to,
-              char * from,
-              unsigned long n)
+_Set_GDT(_GETSET_DTR_TYPE *dtr)
 {
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x = *(uint32*) (from + i);
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, x
-	  __asm mov fs:[eax], ebx
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x = from[i];
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov bl, x
-	  __asm mov fs:[eax], bl
-      i++;
-   }
+   _lgdt(dtr);
 }

-
-
 static INLINE void
-MEMCOPY_FROM_FS(char * to,
-                VA from,
-                unsigned long n)
+_Get_GDT(_GETSET_DTR_TYPE *dtr)
 {
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x;
-	  uint32 _faddr = (uint32)(from+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, fs:[eax]
-	  __asm mov x,ebx
-      *(uint32*)(to+i)=x;
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x;
-	  uint32 _faddr = (uint32) (from+i);
-	  __asm mov eax, _faddr;
-      __asm mov bl, fs:[eax]
-	  __asm mov x, bl
-      *(uint8*)(to+i)=x;
-      i++;
-   }
+   _sgdt(dtr);
 }

-#else
-#error No compiler defined for get/set
+static INLINE void
+_Set_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __lidt(dtr);
+}
+
+static INLINE void
+_Get_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __sidt(dtr);
+}
+
+
+#define ENABLE_INTERRUPTS() _enable()
+#define CLEAR_INTERRUPTS()  _disable()
+
+#define SAVE_FLAGS(x) do { \
+   (x) = __readeflags();   \
+} while (0)
+
+#define RESTORE_FLAGS(x) __writeeflags(x)
+
 #endif /* !__GNUC__ && !_MSC_VER */


-#ifdef __GNUC__
-static INLINE unsigned CURRENT_CPL(void)
+#define SET_CR_DR(regType, regNum, expr)                     \
+   do {                                                      \
+      /* Ensure no implicit truncation of 'expr' */          \
+      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
+      _SET_##regType##regNum(expr);                          \
+   } while (0)
+
+#define GET_CR_DR(regType, regNum, var) \
+   do {                                 \
+      var = _GET_##regType##regNum();   \
+   } while (0)
+
+#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
+#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
+#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
+#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
+#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
+#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
+
+#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
+#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
+#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
+#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
+#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
+#define GET_DR7(var)  GET_CR_DR(DR, 7, var)
+
+/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
+#undef GET_CR0
+#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
+#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
+#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
+#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
+#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
+
+#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
+#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
+#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
+#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
+
+/*
+ * When the ULM macro-defines INTERRUPTS_ENABLED, attempting to define
+ * it as a function produces hard-to-diagnose compile-time errors.
+ */
+#if !defined(ULM) && !defined(INTERRUPTS_ENABLED)
+static INLINE Bool
+INTERRUPTS_ENABLED(void)
+{
+   uintptr_t flags;
+   SAVE_FLAGS(flags);
+   return ((flags & EFLAGS_IF) != 0);
+}
+#endif
+
+/*
+ * [GS]ET_[GI]DT() are defined as macros wrapping a function
+ * so we can pass the argument implicitly by reference (requires
+ * a macro) and get type checking too (requires a function).
+ */
+
+#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
+
+#define SET_IDT(_idt)    _Set_IDT(&(_idt))
+#define GET_IDT(_idt)    _Get_IDT(&(_idt))
+
+#if !defined(VMKERNEL)
+#define NO_INTERRUPTS_BEGIN() do { \
+                                   uintptr_t _flags; \
+                                   SAVE_FLAGS(_flags); \
+                                   CLEAR_INTERRUPTS();
+
+#define NO_INTERRUPTS_END()        RESTORE_FLAGS(_flags); \
+                                 } while(0)
+#endif
+
+
+#if defined (__GNUC__)
+static INLINE unsigned
+CURRENT_CPL(void)
 {
   return SELECTOR_RPL(GET_CS());
 }
-#elif defined _MSC_VER
-static INLINE unsigned CURRENT_CPL(void) {
-   volatile Selector _v;
-   __asm mov ax, cs _asm mov _v, ax
-   return SELECTOR_RPL(_v);
-}
-#else
-#error No compiler defined for CURRENT_CPL
 #endif


+static INLINE uint64
+RDPMC(int counter)
+{
 #ifdef __GNUC__
-/* Checked against the Intel manual and GCC --hpreg
-
-   volatile because the msr can change without the compiler knowing it
-   (when we use wrmsr). */
-static INLINE uint64 __GET_MSR(int cx)
-{
-   uint64 msr;
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
-      : "=a" (msr)
-      : "c" (cx)
-      : "%rdx"
-   );
-#else
-   __asm__ __volatile__(
-      "rdmsr"
-      : "=A" (msr)
-      : "c" (cx)
-   );
-#endif
-   return msr;
-}
-#elif defined _MSC_VER
-#pragma warning( disable : 4035)
-static INLINE uint64 __GET_MSR(int input)
-{
-   __asm push ecx
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x32
-   __asm pop ecx
-}
-
-static INLINE void __SET_MSR(int input, uint64 val)
-{
-   uint32 hival = (uint32)(val >> 32);
-   uint32 loval = (uint32)val;
-   __asm push edx
-   __asm push ecx
-   __asm push eax
-   __asm mov  eax, loval
-   __asm mov  edx, hival
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x30
-   __asm pop eax
-   __asm pop ecx
-   __asm pop edx
-}
-#pragma warning (default: 4035)
-#else
-#error
-#endif
-
-#ifdef __GNUC__
-static INLINE void __SET_MSR(int cx, uint64 val)
-{
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "a" ((uint32) val), "d" ((uint32)(val >> 32)), "c" (cx)
-    );
-#else
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "A" (val),
-        "c" (cx)
-    );
-#endif
-}
-#endif
-
-
-/*
- * RDMSR/WRMSR access the 64bit MSRs as two
- * 32 bit quantities, whereas GET_MSR/SET_MSR
- * above access the MSRs as one 64bit quantity.
- */
-#ifdef __GNUC__
-#undef RDMSR
-#undef WRMSR
-#define RDMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "rdmsr"                         \
-      : "=a" (low), "=d" (high)       \
-      : "c" (msrNum)                  \
-   );                                 \
-} while (0)
-
-#define WRMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "wrmsr"                         \
-      : /* no outputs */              \
-      : "c" (msrNum),                 \
-        "a" (low),                    \
-        "d" (high)                    \
-    );                                \
-} while (0)
-
-static INLINE uint64 RDPMC(int cx)
-{
 #ifdef VM_X86_64
   uint64 pmcLow;
   uint64 pmcHigh;
@@ -867,7 +547,7 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=a" (pmcLow), "=d" (pmcHigh)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmcHigh << 32 | pmcLow;
@@ -877,43 +557,26 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=A" (pmc)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmc;
 #endif
-}
 #elif defined _MSC_VER
-#ifndef VM_X86_64 // XXX Switch to intrinsics with the new 32 and 64-bit compilers.
-
-static INLINE uint64 RDPMC(int counter)
-{
-   __asm mov ecx, counter
-   __asm rdpmc
+   return __readpmc(counter);
+#endif
 }

-static INLINE void WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   __asm mov ecx, msrNum
-   __asm mov eax, lo
-   __asm mov edx, hi
-   __asm wrmsr
-}
-#endif
-#endif

-
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
-static INLINE uint64 __XGETBV(int cx)
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined (ULM)
+static INLINE uint64
+__XGETBV(int cx)
 {
+#ifdef __GNUC__
 #ifdef VM_X86_64
   uint64 lowval, hival;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=a" (lowval), "=d" (hival)
      : "c" (cx)
   );
@@ -921,39 +584,41 @@ static INLINE uint64 __XGETBV(int cx)
 #else
   uint64 val;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=A" (val)
      : "c" (cx)
   );
   return val;
 #endif
+#elif defined _MSC_VER
+   return _xgetbv((unsigned)cx);
+#endif
 }

-static INLINE void __XSETBV(int cx, uint64 val)
+static INLINE void
+__XSETBV(int cx, uint64 val)
 {
+#ifdef __GNUC__
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd1"
-#else
      "xsetbv"
-#endif
      : /* no outputs */
      : "a" ((uint32)val), "d" ((uint32)(val >> 32)), "c" (cx)
    );
+#elif defined _MSC_VER
+   _xsetbv((unsigned)cx, val);
+#endif
 }

-static INLINE uint64 GET_XCR0(void)
+static INLINE uint64
+GET_XCR0(void)
 {
   return __XGETBV(0);
 }

 #define SET_XCR0(val) __XSETBV(0, val)

-static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
+static INLINE void
+SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 {
 #ifndef VMM_BOOTSTRAP
   ASSERT(oldVal == GET_XCR0());
@@ -965,33 +630,45 @@ static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 #endif


-#define START_TRACING() { \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags |= EFLAGS_TF;    \
-   RESTORE_FLAGS(flags);  \
-}
-
-#define STOP_TRACING() {  \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags &= ~EFLAGS_TF;   \
-   RESTORE_FLAGS(flags);  \
-}
-
-
-static INLINE Bool
-INTERRUPTS_ENABLED(void)
+static INLINE uint32
+RDTSCP_AuxOnly(void)
 {
-   uintptr_t flags;
-   SAVE_FLAGS(flags);
-   return ((flags & EFLAGS_IF) != 0);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return tscAux;
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   __rdtscp(&tscAux);
+
+   return tscAux;
+#endif
 }

-static INLINE void
-SET_KERNEL_PER_CORE(uint64 val)
+
+static INLINE uint64
+RDTSCP(void)
 {
-   __SET_MSR(MSR_GSBASE, val);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      :"=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return QWORD(tscHigh, tscLow);
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   return __rdtscp(&tscAux);
+#endif
 }

 #endif
@@ -1,137 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * vm_asm_x86_64.h
- *
- *	x86-64 asm macros
- */
-
-#ifndef _VM_ASM_X86_64_H_
-#define _VM_ASM_X86_64_H_
-
-
-#define INCLUDE_ALLOW_VMX
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_DISTRIBUTE
-#include "includeCheck.h"
-
-#ifdef __GNUC__
-#include "vm_asm_x86.h"
-#endif
-
-/*
- * This file contains inline assembly routines used by x86_64 code.
- */
-
-#ifdef __GNUC__
-
-  /* nop; prevents #error for no compiler definition from firing */
-
-#elif defined _MSC_VER  /* !__GNUC__ */
-
-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-
-#ifdef _WIN64
-#ifdef __cplusplus
-extern "C" {
-#endif
-unsigned __int64  __readmsr(unsigned long);
-void              __writemsr(unsigned long, unsigned __int64);
-#pragma intrinsic(__readmsr, __writemsr)
-#ifdef __cplusplus
-}
-#endif
-
-
-static INLINE uint64
-RDPMC(int counter)
-{
-   return __readpmc(counter);
-}
-
-
-static INLINE void
-WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   uint64 value = QWORD(hi, lo);
-   __writemsr(msrNum, value);
-}
-
-
-static INLINE uint64
-__GET_MSR(int input)
-{
-   return __readmsr((unsigned long)input);
-}
-
-
-static INLINE void
-__SET_MSR(int cx, uint64 val)
-{
-  __writemsr((unsigned long)cx, (unsigned __int64)val);
-}
-
-#endif
-
-#else
-#error No compiler defined for get/set
-#endif /* !__GNUC__ && !_MSC_VER */
-
-
-#ifdef __GNUC__
-static INLINE void
-SWAPGS(void)
-{
-   __asm__ __volatile__("swapgs");
-}
-
-
-static INLINE uint64
-RDTSCP_AuxOnly(void)
-{
-   uint64 tscLow, tscHigh, tscAux;
-
-   __asm__ __volatile__(
-      "rdtscp"
-      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
-   );
-
-   return tscAux;
-}
-#endif
-
-#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -67,6 +67,7 @@ extern "C" {
 #if defined (VMKPANIC) 
 #include "vmk_assert.h"
 #else /* !VMKPANIC */
+#include <linux/kernel.h>
 #define _ASSERT_PANIC(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
 #define _ASSERT_PANIC_BUG(bug, name) \
@@ -107,7 +108,7 @@ NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
 } while(0)

 #else
-NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+#define Panic panic
 #endif

 void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
@@ -202,12 +203,12 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +315,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -122,87 +129,29 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -2303,7 +2195,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2216,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3581,7 +3476,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3604,7 +3499,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -3618,7 +3513,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -48,7 +55,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -221,8 +227,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +269,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +382,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +398,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -497,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -550,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -586,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -630,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -653,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -768,27 +760,21 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -836,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -849,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -888,7 +868,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +882,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1078,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1136,113 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+#endif
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,24 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -113,40 +95,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +169,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +272,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +316,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +398,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +483,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +507,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -244,9 +145,127 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_sfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +297,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +412,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +430,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +510,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,12 +44,44 @@
 #endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -58,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -159,18 +157,6 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

-/*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-

 /*
 * Token concatenation
@@ -209,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -255,6 +241,10 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif
@@ -286,6 +276,14 @@ Max(int a, int b)
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -440,10 +438,6 @@ void *_ReturnAddress(void);

 #define strtok_r  strtok_s

-#if (_MSC_VER < 1500)
-#define	vsnprintf _vsnprintf
-#endif
-
 typedef int uid_t;
 typedef int gid_t;

@@ -467,20 +461,9 @@ typedef int pid_t;
 #define       W_OK          2
 #define       R_OK          4

-#endif // }
+#endif // } _WIN32

-/*
- * Macro for username comparison.
- */
-
-#ifdef _WIN32 // {
-#define USERCMP(x,y)  Str_Strcasecmp(x,y)
-#else
-#define USERCMP(x,y)  strcmp(x,y)
-#endif // }
-
-
-#endif // }
+#endif // } USERLEVEL

 #ifndef va_copy

@@ -556,10 +539,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -659,11 +642,27 @@ typedef int pid_t;
 #endif

 #ifdef VMM
+#define vmx86_vmm 1
 #define VMM_ONLY(x) x
 #else
+#define vmx86_vmm 0
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
@@ -687,7 +686,6 @@ typedef int pid_t;
 * display/printer drivers only.
 */
 #ifdef _WIN32
-#ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
 #ifdef _WIN64
 #define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
@@ -699,15 +697,8 @@ typedef int pid_t;
 #define WinDrvPrint(arg, ...)
 #define WinDrvEngPrint(arg, ...)
 #endif
-#endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -718,7 +709,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -795,4 +786,30 @@ typedef int pid_t;
 #define END_PTR_TO_ALIGNED_VAR \
   } while (0)

+
+/*
+ * -Wswitch means that when you pass switch an enum that it's looking for
+ * all values from that enum, and only that enum, to be accounted for.
+ * "default:;" is fine for catching values you don't care about. But today
+ * we have a bunch of code that uses internal and external enum values, or
+ * in other words combines two enums into a single variable. This cast is
+ * the workaround, but we really need to fix this mess.
+ */
+#define UNCHECKED_SWITCH__FIXME(x) switch ((uint64)(x))
+
+
+/*
+ * When clang static analyzer parses source files, it implicitly defines
+ * __clang_analyzer__ macro. We use this to define our custom macro to stop
+ * its execution for the current path of analysis by calling a function that
+ * doesn't return, making it think that it hit a failed assertion.
+ *
+ * DO NOT use to silence the analyzer! See PR2447238.
+ */
+#ifdef __clang_analyzer__
+#define VMW_CLANG_ANALYZER_NORETURN() Panic("Disable Clang static analyzer")
+#else
+#define VMW_CLANG_ANALYZER_NORETURN() ((void)0)
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,18 +63,9 @@
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/*
- * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -83,7 +74,6 @@
 #ifdef __x86_64__
 #define VM_X86_64
 #define vm_x86_64 1
-#define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
 #else
@@ -111,6 +101,7 @@

 #pragma warning (3 :4505) // unreferenced local function
 #pragma warning (disable :4018) // signed/unsigned mismatch
+#pragma warning (suppress:4619) // suppress warning next line (C4761 was removed in vs2019u4)
 #pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
 #pragma warning (disable :4305) // truncation from 'const int' to 'short'
 #pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
@@ -120,14 +111,91 @@

 #endif

-#if defined(__cplusplus) && __cplusplus >= 201103L || \
-    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
-    defined(__APPLE__) || defined(HAVE_STDINT_H)
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif
+
 /*
- * We're using <stdint.h> instead of <cstdint> below because some C++ code
- * deliberately compiles without C++ include paths.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif
+

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -138,40 +206,6 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-#else /* !HAVE_STDINT_H */
-
-/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
-#ifdef _MSC_VER
-
-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
-
-#elif __GNUC__
-#   if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#   else
-/*
- * Only strict c90 (without extensions) lacks a 'long long' type.
- * If this declaration fails ... use -std=c99 or -std=gnu90.
- */
-typedef unsigned long long uint64;
-typedef long long int64;
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-

 /*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
@@ -202,14 +236,6 @@ typedef char           Bool;
 #define IS_BOOL(x)     (((x) & ~1) == 0)


-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */
-
 /*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
@@ -218,9 +244,6 @@ typedef char           Bool;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -234,14 +257,11 @@ typedef char           Bool;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -249,46 +269,10 @@ typedef char           Bool;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -337,10 +321,11 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* macOS hosts use the same formatters for 32- and 64-bit. */
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
   #define FMT64         "ll"
-   #if KERNEL
+   #if defined(__APPLE__) && KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
@@ -452,7 +437,6 @@ typedef uintptr_t VPN;

 typedef uint64    PA;
 typedef uint64    PPN;
-typedef uint32    PPNTMP;

 typedef uint64    TPA;
 typedef uint64    TPPN;
@@ -470,9 +454,7 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint64    PgCnt64;
 typedef uint64    PageCnt;
-typedef uint64    PgNum64;
 typedef uint64    PageNum;
 typedef unsigned  MemHandle;
 typedef unsigned  IoHandle;
@@ -515,7 +497,7 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
@@ -598,15 +580,14 @@ typedef void * UserVA;


 /* Maximal observable PPN value. */
-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
 #define INVALID_PPN32     ((PPN32)0xffffffff)
-#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
-#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -624,7 +605,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((uint32)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -706,22 +688,28 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-3.3 (circa 2003).  Enforce this here, various things below
+ * gcc-4.1 (circa 2006).  Enforce this here, various things below
 * this line depend upon it.
 *
- * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
- * The various linux kernel modules may use older (gcc-3.3) compilers.
+ * Current oldest compilers:
+ * - guest tools: 4.1.2 (freebsd/solaris)
+ * - buildhost compiler: 4.4.3
+ * - hosted kernel modules: 4.5
+ *
+ * SWIG's preprocessor is exempt.
 */
-#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
-#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
+#ifndef SWIG
+#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
+#error "gcc version is too old, need gcc-4.1 or better"
+#endif
 #endif

 /*
- * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Similarly, we require a compiler that is at least vs2012.
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1400
-#error "cl.exe version is too old, need vc80 or better"
+#if defined _MSC_VER && _MSC_VER < 1700
+#error "cl.exe version is too old, need vs2012 or better"
 #endif


@@ -837,7 +825,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -883,12 +871,10 @@ typedef void * UserVA;
 #endif

 #ifndef UNUSED_TYPE
-// XXX _Pragma would better but doesn't always work right now.
 #  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
 #endif

 #ifndef UNUSED_VARIABLE
-// XXX is there a better way?
 #  define UNUSED_VARIABLE(_var) (void)_var
 #endif

@@ -903,29 +889,21 @@ typedef void * UserVA;

 /*
 * ALIGNED specifies minimum alignment in "n" bytes.
+ *
+ * NOTE: __declspec(align) has limited syntax; it must essentially be
+ *       an integer literal.  Expressions, such as sizeof(), do not
+ *       work.
 */

 #ifdef __GNUC__
 #define ALIGNED(n) __attribute__((__aligned__(n)))
+#elif defined(_MSC_VER)
+#define ALIGNED(n) __declspec(align(n))
 #else
 #define ALIGNED(n)
 #endif


-/*
- * Encapsulate the syntactic differences between gcc and msvc alignment control.
- * BOUNDARY must match in the prefix and suffix.
- */
-
-#ifdef _WIN32
-#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
-#define ALIGN_SUFFIX(BOUNDRY)
-#else
-#define ALIGN_PREFIX(BOUNDRY)
-#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
-#endif
-
-
 /*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
@@ -934,82 +912,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -0,0 +1,88 @@
+/*********************************************************
+ * Copyright (C) 2012,2017-2018,2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vm_idt_x86.h --
+ *
+ *	Definitions for IDT use in x86/x86-64 ESX and hosted VMware Products.
+ */
+
+#ifndef _VM_IDT_X86_H_
+#define _VM_IDT_X86_H_
+
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMCORE
+#include "includeCheck.h"
+
+#if defined __cplusplus
+extern "C" {
+#endif
+
+/*
+ * User worlds are the only worlds that expect to receive
+ * interrupts on a stack specified in the %rsp entries of
+ * the TSS.
+ */
+#define TSS_RSP0 0
+#define TSS_RSP1 1
+#define TSS_RSP2 2
+
+/*
+ * The monitor and the vmkernel use gate stub handlers of a single size in
+ * their IDTs.  The only exception is the #PF gate (#14) which is twice as
+ * long in the monitor IDT, to copy CR2.
+ */
+#define IDT_STUB_SIZE              16
+
+#define IST_NONE                    0
+/* IST entries for the monitor. */
+#define IST_VMM_DF                  1
+#define IST_VMM_NMI                 2
+#define IST_VMM_MCE                 3
+#define MAX_VMM_IST                 3
+/* IST entries for the vmkernel. */
+#define IST_VMK_MCE                 5
+#define IST_VMK_DF                  6
+#define IST_VMK_NMI                 7
+
+
+static INLINE int
+IDT_MonitorISTForVector(int v)
+{
+   /*
+    * For non-SVM execution, the monitor uses the IST for #DF, #NMI, and #MC.
+    */
+   if (v == EXC_DF) {
+      return IST_VMM_DF;
+   } else if (v == EXC_NMI) {
+      return IST_VMM_NMI;
+   } else if (v == EXC_MC) {
+      return IST_VMM_MCE;
+   }
+   return IST_NONE;
+}
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
+#endif /* _VM_IDT_X86_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,22 +40,25 @@
 /*
 * These definitions suit both the x64 and arm64 architectures. In particular
 * - At all levels, a page table is a 4KB page containing 512 8-byte entries.
- * - Each entry maps 39, 30, 21 and 12 bits at the respective page table
+ * - Each entry maps 48, 39, 30, 21 and 12 bits at the respective page table
 *   levels (with standard page sizes).
 */

-typedef uint64 PT_L4E;
-typedef uint64 PT_L3E;
-typedef uint64 PT_L2E;
-typedef uint64 PT_L1E;
+typedef uint64 PT_Entry;
+typedef PT_Entry PT_L5E;
+typedef PT_Entry PT_L4E;
+typedef PT_Entry PT_L3E;
+typedef PT_Entry PT_L2E;
+typedef PT_Entry PT_L1E;

 typedef enum {
   PT_LEVEL_1 = 1,
   PT_LEVEL_2,
   PT_LEVEL_3,
   PT_LEVEL_4,
+   PT_LEVEL_5,
   PT_LEVEL_STOP = PT_LEVEL_1,
-   PT_MAX_LEVELS = PT_LEVEL_4
+   PT_MAX_LEVELS = PT_LEVEL_5
 } PT_Level;

 #define PT_PTE_SIZE           8
@@ -76,6 +79,7 @@ typedef enum {

 /* Address space size (in 4KB pages) covered by a level-_l page table entry. */
 #define PT_PAGES_PER_LE(_l)   ((uint64)1 << PT_LE_PG_SHIFT(_l))
+#define PT_PAGES_PER_L5E      PT_PAGES_PER_LE(PT_LEVEL_5)
 #define PT_PAGES_PER_L4E      PT_PAGES_PER_LE(PT_LEVEL_4)
 #define PT_PAGES_PER_L3E      PT_PAGES_PER_LE(PT_LEVEL_3)
 #define PT_PAGES_PER_L2E      PT_PAGES_PER_LE(PT_LEVEL_2)
@@ -86,6 +90,7 @@ typedef enum {
 * address space size (in bytes) covered by a level-_l page table entry.
 */
 #define PT_LE_SHIFT(_l)       (PAGE_SHIFT + PT_LE_PG_SHIFT(_l))
+#define PT_L5E_SHIFT          PT_LE_SHIFT(PT_LEVEL_5)
 #define PT_L4E_SHIFT          PT_LE_SHIFT(PT_LEVEL_4)
 #define PT_L3E_SHIFT          PT_LE_SHIFT(PT_LEVEL_3)
 #define PT_L2E_SHIFT          PT_LE_SHIFT(PT_LEVEL_2)
@@ -93,6 +98,7 @@ typedef enum {

 /* Address space size (in bytes) covered by a level-_l page table entry. */
 #define PT_LE_SIZE(_l)        ((uint64)1 << PT_LE_SHIFT(_l))
+#define PT_L5E_SIZE           PT_LE_SIZE(PT_LEVEL_5)
 #define PT_L4E_SIZE           PT_LE_SIZE(PT_LEVEL_4)
 #define PT_L3E_SIZE           PT_LE_SIZE(PT_LEVEL_3)
 #define PT_L2E_SIZE           PT_LE_SIZE(PT_LEVEL_2)
@@ -103,6 +109,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LA_2_LOFF(_a, _l)  (((_a) >> PT_LE_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LA_2_L5OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LA_2_L4OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LA_2_L3OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LA_2_L2OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_2)
@@ -113,6 +120,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LPN_2_LOFF(_a, _l) (((_a) >> PT_LE_PG_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LPN_2_L5OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LPN_2_L4OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LPN_2_L3OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LPN_2_L2OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_2)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -30,9 +30,6 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-#define INTERRUPT_REDIRECTION_BITMAP_SIZE 32
-#define IO_PERMISSION_BITMAP_SIZE 0
-

 #define VMM_PANIC_NONE         0
 #define VMM_PANIC_MSG_COPYING  1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,6 +32,7 @@
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMX
 #include "includeCheck.h"
+#include "community_source.h"

 /*
 * Page remapping definitions.
@@ -46,6 +47,7 @@
 #define VMMEM_ANON_IOABLE_PAGE    VMMEM_FLAG_BIT(4)
 #define VMMEM_ANON_ALL_FLAGS      MASK(5)

+#define VMMEM_GUEST_NO_FLAGS      0
 #define VMMEM_GUEST_WRITEABLE     VMMEM_FLAG_BIT(0)
 #define VMMEM_GUEST_BREAKCOW      VMMEM_FLAG_BIT(1)
 #define VMMEM_GUEST_2M_PAGE       VMMEM_FLAG_BIT(2)
@@ -83,7 +85,7 @@
 * Structure used to query platform about the page state.
 */
 typedef struct PlatformPageInfoList {
-   PgCnt64 numPages;
+   PageCnt numPages;
   BPN     bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
   MPN     mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
   uint8   flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
@@ -96,9 +98,12 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

-#define VMMEM_SERVICES_DEFS                               \
-   MDEF(VMMEM_SERVICES_TYPE_P2M,  P2MUpdate_FilterPages)  \
-   MDEF(VMMEM_SERVICES_TYPE_SWAP, BusMemSwap_FilterPages)
+#define MDEF_DRAINER
+
+#define VMMEM_SERVICES_DEFS                                       \
+   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
+   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
+   MDEF_DRAINER

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,8 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#include <asm/processor-flags.h>
+
 #define X86_MAX_INSTR_LEN  15   /* Max byte length of an x86 instruction. */

 #define NUM_IDT_VECTORS 256
@@ -75,7 +77,9 @@
 #define CR3_PDB_MASK   0xfffff000
 #define CR3_IGNORE     0xFFF
 #define PAE_CR3_IGNORE 0x1F
+#ifndef CR3_PCID_MASK
 #define CR3_PCID_MASK  0xFFF
+#endif
 #define CR3_NO_FLUSH   (1ULL << 63)

 #define CR4_VME        0x00000001
@@ -89,6 +93,8 @@
 #define CR4_PCE        0x00000100
 #define CR4_OSFXSR     0x00000200 // CPU/OS supports SIMD insts
 #define CR4_OSXMMEXCPT 0x00000400 // #XF exception enable PIII only
+#define CR4_UMIP       0x00000800
+#define CR4_LA57       0x00001000
 #define CR4_VMXE       0x00002000
 #define CR4_SMXE       0x00004000
 #define CR4_FSGSBASE   0x00010000
@@ -97,7 +103,8 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
-#define CR4_RESERVED   CONST64U(0xffffffffff889800)
+#define CR4_CET        0x00800000
+#define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

 /*
@@ -201,8 +208,16 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_CP           21  // Control Protection exception.
+#define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

+/* Bitmap of the exception vectors that have associated error codes. */
+#define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
+                                (1u << EXC_NP) | (1u << EXC_SS) | \
+                                (1u << EXC_GP) | (1u << EXC_PF) | \
+                                (1u << EXC_AC) | (1u << EXC_CP))
+
 /*
 * eflag/rflag definitions.
 */
@@ -211,24 +226,24 @@

 typedef enum x86_FLAGS {
   EFLAGS_NONE         = 0,
-   EFLAGS_CF           = (1 << 0),     /* User */ 
-   EFLAGS_SET          = (1 << 1),                                  
-   EFLAGS_PF           = (1 << 2),     /* User */ 
-   EFLAGS_AF           = (1 << 4),     /* User */ 
-   EFLAGS_ZF           = (1 << 6),     /* User */ 
-   EFLAGS_SF           = (1 << 7),     /* User */ 
-   EFLAGS_TF           = (1 << 8),     /* Priv */ 
-   EFLAGS_IF           = (1 << 9),     /* Priv */ 
-   EFLAGS_DF           = (1 << 10),    /* User */ 
-   EFLAGS_OF           = (1 << 11),    /* User */ 
-   EFLAGS_NT           = (1 << 14),    /* Priv */ 
-   EFLAGS_RF           = (1 << 16),    /* Priv */ 
-   EFLAGS_VM           = (1 << 17),    /* Priv */ 
-   EFLAGS_AC           = (1 << 18),    /* Priv */ 
-   EFLAGS_VIF          = (1 << 19),    /* Priv */ 
-   EFLAGS_VIP          = (1 << 20),    /* Priv */ 
+   EFLAGS_CF           = (1 << 0),     /* User */
+   EFLAGS_SET          = (1 << 1),
+   EFLAGS_PF           = (1 << 2),     /* User */
+   EFLAGS_AF           = (1 << 4),     /* User */
+   EFLAGS_ZF           = (1 << 6),     /* User */
+   EFLAGS_SF           = (1 << 7),     /* User */
+   EFLAGS_TF           = (1 << 8),     /* Priv */
+   EFLAGS_IF           = (1 << 9),     /* Priv */
+   EFLAGS_DF           = (1 << 10),    /* User */
+   EFLAGS_OF           = (1 << 11),    /* User */
+   EFLAGS_NT           = (1 << 14),    /* Priv */
+   EFLAGS_RF           = (1 << 16),    /* Priv */
+   EFLAGS_VM           = (1 << 17),    /* Priv */
+   EFLAGS_AC           = (1 << 18),    /* Priv */
+   EFLAGS_VIF          = (1 << 19),    /* Priv */
+   EFLAGS_VIP          = (1 << 20),    /* Priv */
   EFLAGS_ID           = (1 << 21),    /* Priv */
-   
+
   EFLAGS_IOPL         = 3 << EFLAGS_IOPL_SHIFT,
   EFLAGS_ARITH        = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_OF),
@@ -240,7 +255,7 @@ typedef enum x86_FLAGS {
   EFLAGS_ALL          = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_DF | EFLAGS_OF | EFLAGS_TF |
                          EFLAGS_IF | EFLAGS_IOPL | EFLAGS_NT | EFLAGS_RF |
-                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP | 
+                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP |
                          EFLAGS_ID),
   EFLAGS_ALL_16       = EFLAGS_ALL & 0xffff,
   EFLAGS_REAL_32      = (EFLAGS_ALL & ~(EFLAGS_VIP | EFLAGS_VIF | EFLAGS_VM)),
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,46 +39,6 @@
 #include "x86cpuid.h"


-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#ifdef _MSC_VER
-#ifdef __cplusplus
-extern "C" {
-#endif
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-void __cpuid(int regs[4], int eax);
-#pragma intrinsic(__cpuid)
-
-/*
- * __cpuidex has been supported since VS2008
- */
-#if _MSC_VER >= 1500
-void __cpuidex(int regs[4], int eax, int ecx);
-#pragma intrinsic(__cpuidex)
-#endif /* _MSC_VER >= 1500 */
-#endif /* VM_X86_64 */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
-
 #ifdef __GNUC__ // {

 /*
@@ -89,68 +49,57 @@ void __cpuidex(int regs[4], int eax, int ecx);
 *
 */

-/*
- * %ebx is reserved on i386 PIC.  Apple's gcc-5493 (gcc 4.0) compiling
- * for x86_64 incorrectly errors out saying %ebx is reserved.  This is
- * Apple bug 7304232.
- */
-#if vm_x86_64 ? (defined __APPLE_CC__ && __APPLE_CC__ == 5493) : defined __PIC__
-#if vm_x86_64
-/*
- * Note that this generates movq %rbx,%rbx; cpuid; xchgq %rbx,%rbx ...
- * Unfortunately Apple's assembler does not have .ifnes, and I cannot
- * figure out how to do that with .if.   If we ever enable this code
- * on other 64bit systems, both movq & xchgq should be surrounded by
- * .ifnes \"%%rbx\", \"%q1\" & .endif
- */
-#define VM_CPUID_BLOCK  "movq %%rbx, %q1\n\t" \
-                        "cpuid\n\t"           \
-                        "xchgq %%rbx, %q1\n\t"
-#define VM_EBX_OUT(reg) "=&r"(reg)
+#if defined VM_X86_64 || !defined __PIC__
+#define VM_CPUID_BLOCK  "cpuid"
+#define VM_EBX_OUT(reg) "=b"(reg)
 #else
+/* %ebx is reserved on i386 PIC. */
 #define VM_CPUID_BLOCK  "movl %%ebx, %1\n\t" \
                        "cpuid\n\t"          \
                        "xchgl %%ebx, %1\n\t"
 #define VM_EBX_OUT(reg) "=&rm"(reg)
 #endif
-#else
-#define VM_CPUID_BLOCK  "cpuid"
-#define VM_EBX_OUT(reg) "=b"(reg)
-#endif

 static INLINE void
-__GET_CPUID(int eax,         // IN
+__GET_CPUID(uint32     eax,  // IN
            CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax)
      : "memory"
   );
 }

 static INLINE void
-__GET_CPUID2(int eax,         // IN
-             int ecx,         // IN
+__GET_CPUID2(uint32 eax,      // IN
+             uint32 ecx,      // IN
             CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax), "c" (ecx)
      : "memory"
   );
 }

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int eax) // IN
+__GET_EAX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx)
      : "a" (eax)
      : "memory", "%ecx", "%edx"
   );
@@ -159,7 +108,7 @@ __GET_EAX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int eax) // IN
+__GET_EBX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

@@ -174,14 +123,16 @@ __GET_EBX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int eax) // IN
+__GET_ECX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ecx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=c" (ecx)
      : "a" (eax)
      : "memory", "%edx"
   );
@@ -190,14 +141,16 @@ __GET_ECX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int eax) // IN
+__GET_EDX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 edx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=d" (edx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=d" (edx)
      : "a" (eax)
      : "memory", "%ecx"
   );
@@ -206,271 +159,59 @@ __GET_EDX_FROM_CPUID(int eax) // IN
 }


-static INLINE uint32
-__GET_EAX_FROM_CPUID4(int ecx) // IN
-{
-   uint32 eax;
-   uint32 ebx;
-
-   __asm__ __volatile__(
-      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
-      : "a" (4), "c" (ecx)
-      : "memory", "%edx"
-   );
-
-   return eax;
-}
-
 #undef VM_CPUID_BLOCK
 #undef VM_EBX_OUT

 #elif defined(_MSC_VER) // } {

 static INLINE void
-__GET_CPUID(int input, CPUIDRegs *regs)
+__GET_CPUID(uint32 input, CPUIDRegs *regs)
 {
-#ifdef VM_X86_64
   __cpuid((int *)regs, input);
-#else
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-#endif
 }

-#ifdef VM_X86_64
-
-#if _MSC_VER >= 1500
-
-/*
- * __cpuidex has been supported since VS2008
- */
-
 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __cpuidex((int *)regs, inputEax, inputEcx);
 }

-#else // _MSC_VER >= 1500
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs);
-#endif // _MSC_VER >= 1500
-
-#else // VM_X86_64
-
-static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
-{
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, inputEax
-   __asm mov  ecx, inputEcx
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-}
-#endif
-
 static INLINE uint32
-__GET_EAX_FROM_CPUID(int input)
+__GET_EAX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.eax;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int input)
+__GET_EBX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ebx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ebx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int input)
+__GET_ECX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ecx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ecx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int input)
+__GET_EDX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.edx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

-#ifdef VM_X86_64
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern uint32
-__GET_EAX_FROM_CPUID4(int inputEcx);
-
-#else // VM_X86_64
-
-static INLINE uint32
-__GET_EAX_FROM_CPUID4(int inputEcx)
-{
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, 4
-   __asm mov  ecx, inputEcx
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-}
-
-#endif // VM_X86_64
-
 #else // }
 #error
 #endif

 #define CPUID_FOR_SIDE_EFFECTS() ((void)__GET_EAX_FROM_CPUID(0))

-/* The first parameter is used as an rvalue and then as an lvalue. */
-#define GET_CPUID(_ax, _bx, _cx, _dx) { \
-   CPUIDRegs regs;                      \
-   __GET_CPUID(_ax, &regs);             \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
-#define GET_CPUID2(_ax, _bx, _cx, _dx) {\
-   CPUIDRegs regs;                      \
-   __GET_CPUID2(_ax, _cx, &regs);       \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,43 +97,78 @@ static INLINE uint32 Desc_LongMode(const Descriptor *d) { return d->longmode; }
 static INLINE uint32 Desc_DB(const Descriptor *d)       { return d->DB; }
 static INLINE uint32 Desc_Gran(const Descriptor *d)     { return d->gran; }

-static INLINE uint32 Desc64_Type(const Descriptor64 *d)     { return d->type; }
-static INLINE uint32 Desc64_S(const Descriptor64 *d)        { return d->S; }
-static INLINE uint32 Desc64_DPL(const Descriptor64 *d)      { return d->DPL; }
-static INLINE uint32 Desc64_Present(const Descriptor64 *d)  { return d->present; }
-static INLINE uint32 Desc64_AVL(const Descriptor64 *d)      { return d->AVL; }
-static INLINE uint32 Desc64_Gran(const Descriptor64 *d)     { return d->gran; }
-static INLINE uint32 Desc64_ExtAttrs(const Descriptor64 *d) { return d->ext_attrs; }
+static INLINE uint32
+Desc64_Type(const Descriptor64 *d)
+{
+   return (uint32)d->type;
+}
+
+static INLINE uint32
+Desc64_S(const Descriptor64 *d)
+{
+   return (uint32)d->S;
+}
+
+static INLINE uint32
+Desc64_DPL(const Descriptor64 *d)
+{
+   return (uint32)d->DPL;
+}
+
+static INLINE uint32
+Desc64_Present(const Descriptor64 *d)
+{
+   return (uint32)d->present;
+}
+
+static INLINE uint32
+Desc64_AVL(const Descriptor64 *d)
+{
+   return (uint32)d->AVL;
+}
+
+static INLINE uint32
+Desc64_Gran(const Descriptor64 *d)
+{
+   return (uint32)d->gran;
+}
+
+static INLINE uint32
+Desc64_ExtAttrs(const Descriptor64 *d)
+{
+   return (uint32)d->ext_attrs;
+}

 static INLINE LA32
 Desc_GetBase(const Descriptor *d)
 {
-   return (d->base_hi << 24) | (d->base_mid << 16) | d->base_lo;
+   return ((unsigned)d->base_hi  << 24) |
+          ((unsigned)d->base_mid << 16) | d->base_lo;
 }

 static INLINE LA64
 Desc64_GetBase(const Descriptor64 *d)
 {
-   return ((uint64)d->base_hi << 32)  | 
-          ((uint64)d->base_mid << 24) | (uint32)d->base_lo;
+   return ((uint64)d->base_hi << 32)  |
+          ((uint64)d->base_mid << 24) | (uint64)d->base_lo;
 }

 static INLINE LA32
 Desc64_GetBaseHi(const Descriptor64 *d)
 {
-   return d->base_hi;
+   return (LA32)d->base_hi;
 }

 static INLINE VA32
 Desc_GetLimit(const Descriptor *d)
 {
-   return (d->limit_hi << 16) | d->limit_lo;
+   return ((unsigned)d->limit_hi << 16) | d->limit_lo;
 }

 static INLINE VA32
 Desc64_GetLimit(const Descriptor64 *d)
 {
-   return ((uint32)d->limit_hi << 16) | (uint32)d->limit_lo;
+   return (VA32)(((unsigned)d->limit_hi << 16) | d->limit_lo);
 }

 static INLINE Bool
@@ -310,12 +345,13 @@ static INLINE uint32 DESC_PRESENT(Descriptor d)    { return d.present; }
 *-----------------------------------------------------------------------------
 */

-static INLINE void 
-Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type, 
-                           uint32 DPL, uint32 present, uint32 DB, uint32 gran)
+static INLINE void
+Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit,
+                           uint32 type, uint32 DPL, uint32 present,
+                           uint32 DB, uint32 gran)
 {
-   uint32 baseHi = (uint32) (base >> 32);
-   uint32 baseLo = (uint32) base;
+   uint32 baseHi = (uint32)(base >> 32);
+   uint32 baseLo = (uint32)base;

   /*
    * The first half of a 16-byte descriptor is a valid 8-byte descriptor
@@ -336,15 +372,17 @@ Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type
 *   Given descriptor, return the code/stack size that it specifies.
 *----------------------------------------------------------------------
 */
-static INLINE int
+static INLINE unsigned
 Desc_DBSize(const Descriptor *desc)
 {
   /*
    * Code/stack size is determined by the D/B bit; bit 22 of the 2nd word.
-    * Shift the bit to position 1, mask it out, add 2. Result: 2 or 4.
+    * Shift the bit to position 1, mask it out, add 2.
+    *
+    * Result: 2 or 4.
    */
-   int cSz = ((((const uint32*)desc)[1] >> 21) & 2) + 2;
-   ASSERT(cSz == (Desc_DB(desc) ? 4 : 2));
+   unsigned cSz = ((((const uint32 *)desc)[1] >> 21) & 2) + 2;
+   ASSERT(cSz == (Desc_DB(desc) ? 4U : 2U));
   return cSz;
 }

@@ -526,7 +564,7 @@ typedef struct Gate {
 static INLINE VA
 CallGate_GetOffset(const Gate *cg)
 {
-   return (cg->offset_hi << 16) | cg->offset_lo;
+   return ((unsigned)cg->offset_hi << 16) | cg->offset_lo;
 }

 static INLINE void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-/* 
+/*
 * x86msr.h --
 *
 *      MSR number definitions.
@@ -24,6 +24,7 @@

 #ifndef _X86MSR_H_
 #define _X86MSR_H_
+#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX

@@ -59,11 +60,8 @@ struct MSRReply {
    * we use it to correlate the replies of multiple queries.
    */
   uint64 tag;              // OUT
-
   uint64 msrVal;           // OUT
-
   uint8  implemented;      // OUT
-
   uint8  _pad[7];
 }
 #include "vmware_pack_end.h"
@@ -116,40 +114,63 @@ MSRQuery;
 #define MSR_TSC_AUX           0xc0000103
 #define MSR_BD_TSC_RATIO      0xc0000104

+/* CET MSRs */
+#define MSR_U_CET                            0x6a0
+#define MSR_S_CET                            0x6a2
+#define MSR_CET_SH_STK_EN                         (1ULL << 0)
+#define MSR_CET_WR_SHSTK_EN                       (1ULL << 1)
+#define MSR_PL0_SSP                          0x6a4
+#define MSR_PL1_SSP                          0x6a5
+#define MSR_PL2_SSP                          0x6a6
+#define MSR_PL3_SSP                          0x6a7
+#define MSR_ISST_ADDR                        0x6a8
+
+#define MSR_TEST_CTRL                        0x33
+#define MSR_TEST_CTRL_SPLIT_LOCK_DETECT           (1ULL << 29)
+
 #define IA32_MSR_ARCH_CAPABILITIES           0x10a
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
 #define MSR_ARCH_CAPABILITIES_RSBA                (1ULL << 2)
+#define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
+#define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)
+#define MSR_ARCH_CAPABILITIES_MDS_NO              (1ULL << 5)
+#define MSR_ARCH_CAPABILITIES_IF_PSCHANGE_MC_NO   (1ULL << 6)
+#define MSR_ARCH_CAPABILITIES_TSX_CTRL            (1ULL << 7)
+#define MSR_ARCH_CAPABILITIES_TAA_NO              (1ULL << 8)

+#define MSR_FLUSH_CMD                        0x10b
+#define MSR_FLUSH_CMD_FLUSH_L1D                   (1ULL << 0)

 #define MSR_SPEC_CTRL_IBRS                        (1UL << 0)
 #define MSR_SPEC_CTRL_STIBP                       (1UL << 1)
+#define MSR_SPEC_CTRL_SSBD                        (1UL << 2)

 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

+#define MSR_TSX_CTRL                         0x122
+#define MSR_TSX_CTRL_RTM_DISABLE                  (1ULL << 0)
+#define MSR_TSX_CTRL_CPUID_CLEAR                  (1ULL << 1)
+
+#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
+#endif

 /* Intel Core Architecture and later: use only architected counters. */
-#define IA32_MSR_PERF_CAPABILITIES                0x345
-#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT        0
-#define MSR_PERF_CAPABILITIES_LBRFMT_MASK         0x3f
-#define MSR_PERF_CAPABILITIES_PEBSTRAP            (1u << 6)
-#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS    (1u << 7)
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT 8
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK  0xf
-#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM    (1u << 12)
-#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES   (1u << 13)
+#define IA32_MSR_PERF_CAPABILITIES                   0x345
+#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT           0
+#define MSR_PERF_CAPABILITIES_LBRFMT_MASK            0x3f
+#define MSR_PERF_CAPABILITIES_PEBSTRAP               (1u << 6)
+#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS       (1u << 7)
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT    8
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK     0xf
+#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM       (1u << 12)
+#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES      (1u << 13)
+#define MSR_PERF_CAPABILITIES_PEBS_ADAPTIVE_DATA     (1u << 14)
+#define MSR_PERF_CAPABILITIES_PERF_METRICS_AVAILABLE (1u << 15)

 #define IA32_MSR_PEBS_ENABLE                      0x3f1

-typedef enum {
-   SL_PMC_FLAGS_NONE             = 0x00, /* No flags.                      */
-   SL_PMC_FLAGS_LBR_VA32         = 0x01, /* LBR format: 32-bit VA.         */
-   SL_PMC_FLAGS_LBR_LA64         = 0x02, /* LBR format: 64-bit LA.         */
-   SL_PMC_FLAGS_LBR_VA64         = 0x04, /* LBR format: 64-bit VA.         */
-   SL_PMC_FLAGS_LBR_PACKED_VA32  = 0x08, /* LBR format: 2x32-bit VAs.      */
-} StateLoggerPMCFlags;
-
 #define MSR_MTRR_BASE0        0x00000200
 #define MSR_MTRR_MASK0        0x00000201
 #define MSR_MTRR_BASE1        0x00000202
@@ -265,6 +286,8 @@ typedef enum {

 #define MSR_XSS              0x00000da0  // Extended Supervisor State Mask

+#define MSR_MPX_LAX          0x00001000  // MPX Linear Address Extension
+
 /* RTIT MSRs */
 #define MSR_RTIT_CTL              0x00000570
 #define MSR_RTIT_STATUS           0x00000571
@@ -444,7 +467,6 @@ typedef enum {

 /* MSR_EFER bits. */
 #define MSR_EFER_SCE         0x0000000000000001ULL  /* Sys call ext'ns:  r/w */
-#define MSR_EFER_RAZ         0x00000000000000feULL  /* Read as zero          */
 #define MSR_EFER_LME         0x0000000000000100ULL  /* Long mode enable: r/w */
 #define MSR_EFER_LMA         0x0000000000000400ULL  /* Long mode active: r/o */
 #define MSR_EFER_NXE         0x0000000000000800ULL  /* No-exec enable:   r/w */
@@ -452,7 +474,11 @@ typedef enum {
 #define MSR_EFER_LMSLE       0x0000000000002000ULL  /* LM seg lim enable:r/w */
 #define MSR_EFER_FFXSR       0x0000000000004000ULL  /* Fast FXSAVE:      r/w */
 #define MSR_EFER_TCE         0x0000000000008000ULL  /* Trans. cache ext. r/w */
-#define MSR_EFER_MBZ         0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+/* Vendor specific EFER bits */
+#define MSR_EFER_INTEL_MBZ   0xffffffffffff02feULL  /* Must be zero (resrvd) */
+#define MSR_EFER_INTEL_RAZ   0x0000000000000000ULL  /* Read as zero          */
+#define MSR_EFER_AMD_MBZ     0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+#define MSR_EFER_AMD_RAZ     0x00000000000000feULL  /* Read as zero          */

 #define MSR_AMD_PATCH_LOADER 0xc0010020

@@ -463,13 +489,19 @@ typedef enum {
 #define MSR_K7_HWCR_SSEDIS         0x00008000ULL // Disable SSE bit
 #define MSR_K7_HWCR_MONMWAITUSEREN 0x00000400ULL // Enable MONITOR/MWAIT CPL>0
 #define MSR_K7_HWCR_TLBFFDIS       0x00000040ULL // Disable TLB Flush Filter
+#ifndef MSR_K7_HWCR_SMMLOCK
 #define MSR_K7_HWCR_SMMLOCK        0x00000001ULL // Lock SMM environment
+#endif

 #ifndef MSR_K8_SYSCFG
 #define MSR_K8_SYSCFG        0xc0010010
 #endif
 #define MSR_K8_SYSCFG_MTRRTOM2EN         (1ULL<<21)
 #define MSR_K8_SYSCFG_TOM2FORCEMEMTYPEWB (1ULL<<22)
+#define MSR_K8_SYSCFG_SMEE               (1ULL<<23)
+#define MSR_K8_SYSCFG_SNPE               (1ULL<<24)
+#define MSR_K8_SYSCFG_VMPLE              (1ULL<<25)
+
 #define MSR_K8_TOPMEM2       0xc001001d

 /* AMD "Greyhound" MSRs */
@@ -507,9 +539,37 @@ typedef enum {
 #define MSR_VM_CR_SVME_DISABLE     0x0000000000000010ULL
 #define MSR_VM_CR_RESERVED         0xffffffffffffffe0ULL

+/* SEV related MSRs. */
+#define MSR_VMPAGE_FLUSH           0xc001011e
+#define MSR_GHCB_PA                0xc0010130
+#define MSR_GHCB_PA_FUNCTION_MASK       0xfff
+#define MSR_GHCB_PA_SEVINFO_HV              1
+#define MSR_GHCB_PA_SEVINFO_REQ             2
+#define MSR_GHCB_PA_AP_JUMP_TABLE           3
+#define MSR_GHCB_PA_CPUID_REQ               4
+#define MSR_GHCB_PA_CPUID_RESP              5
+#define MSR_GHCB_PA_TERMINATE             256
+#define MSR_SEV_STATUS             0xc0010131
+
+#define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
+#define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
+#define MSR_SEV_STATUS_SEV_SNP_EN  0x0000000000000004ULL // SEV-SNP is enabled
+
+/* SEV-SNP (Secure Nested Paging) MSRs. */
+#define MSR_RMP_BASE              0xc0010132 // Address of first byte of RMP
+#define MSR_RMP_END               0xc0010133 // Address of last byte of RMP
+
 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)

+#define MSR_AMD_LS_CFG           0xc0011020  // load-store configuration
+#define MSR_AMD_LS_CFG_SSBD_BULLDOZER (1ULL<<54)  // BD family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_KYOTO     (1ULL<<33)  // Kyoto family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_ZEN       (1ULL<<10)  // Zen family non-arch SSBD
+
+#define MSR_AMD_VIRT_SPEC_CTRL   0xc001011f  // Virtual speculation control
+#define MSR_AMD_VIRT_SPEC_CTRL_SSBD   (1ULL<<2)   // Virtual SSBD
+
 /* Syscall/Sysret related MSRs (x86_64) */
 #define MSR_STAR             0xc0000081 // Also present on Athlons.
 #define MSR_LSTAR            0xc0000082
@@ -591,6 +651,8 @@ typedef enum {
 #define MSR_HYPERV_HYPERCALL_EN                  1ULL
 #define MSR_HYPERV_REFERENCE_TSC_EN              1ULL
 #define MSR_HYPERV_VP_ASSIST_EN                  1ULL
+#define MSR_HYPERV_SIEF_EN                       1ULL
+#define MSR_HYPERV_SIM_EN                        1ULL

 #define MSR_HYPERV_GUESTOSID_VENDOR_SHIFT        48
 #define MSR_HYPERV_GUESTOSID_VENDOR_MASK         0xfULL
@@ -600,6 +662,11 @@ typedef enum {
 #define MSR_HYPERV_GUESTOSID_OS_MASK             0xfULL
 #define MSR_HYPERV_GUESTOSID_OS_WINNT_DERIVATIVE 4ULL

+/* MSR for forcing RTM abort to recover PMC3 (see PR 2333817) */
+#ifndef MSR_TSX_FORCE_ABORT
+#define MSR_TSX_FORCE_ABORT                      0x0000010f
+#endif
+#define MSR_TSX_FORCE_ABORT_RTM_BIT_INDEX        0

 /*
 * MTRR bit description
@@ -652,19 +719,28 @@ typedef unsigned char MTRRType;
 /*
 * MISC_FEATURES_ENABLES bits
 */
+#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
+#endif




+// Platform Quality of Service (PQM) MSRs
+#define MSR_INTEL_PQM_EVTSEL    0xc8d
+#define MSR_INTEL_PQM_CTR       0xc8e
+#define MSR_INTEL_PQM_ASSOC     0xc8f

-/*
- * Platform Quality of Service
- */
+ // Platform Quality Enforcement (PQE) MSRs
+#define MSR_INTEL_PQE_CLOS_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_MASK_MAX      0xd8f

-#define MSR_INTEL_QOS_EVTSEL    0xc8d
-#define MSR_INTEL_QOS_CTR       0xc8e
-#define MSR_INTEL_QOS_ASSOC     0xc8f
+#define MSR_INTEL_PQE_CLOS_L3_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_L3_MASK_MAX      0xd0f
+#define MSR_INTEL_PQE_CLOS_L2_MASK_BASE     0xd10
+#define MSR_INTEL_PQE_CLOS_L2_MASK_MAX      0xd4f

 static INLINE uint32
 X86MSR_SysCallEIP(uint64 star)
@@ -687,6 +763,94 @@ X86MSR_SysRetCS(uint64 star)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * X86MSR_{G,S}etMSR --
+ *
+ *      Get or set an MSR in hardware. The interface should remain
+ *      agnostic to the underlying compiler and architecture.
+ *
+ *----------------------------------------------------------------------
+ */
+
+#ifdef __GNUC__
+/*
+ * Checked against the Intel manual and GCC --hpreg
+ * volatile because the msr can change without the compiler knowing it
+ * (when we use wrmsr).
+ */
+#ifdef VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
+      : "=a" (msr)
+      : "c" (cx)
+      : "%rdx"
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "a" ((uint32) value),
+        "d" ((uint32)(value >> 32)),
+        "c" (cx)
+    );
+}
+#else // __GNUC__ && !VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr"
+      : "=A" (msr)
+      : "c"  (cx)
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "A" (value),
+        "c" (cx)
+    );
+}
+#endif
+#elif defined _MSC_VER // !__GNUC__ && _MSC_VER
+unsigned __int64  __readmsr(unsigned long);
+void              __writemsr(unsigned long, unsigned __int64);
+#pragma intrinsic(__readmsr, __writemsr)
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   return __readmsr((unsigned long)cx);
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
+}
+#else // !__GNUC__ && !_MSC_VER
+#error No compiler defined for RDMSR/WRMSR.
+#endif
+
+
 #if defined __cplusplus
 }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,6 +40,7 @@
 #include "x86types.h"
 #include "vm_pagetable.h"
 #include "x86paging_common.h"
+#include "vm_assert.h"

 #define LM_PTE_PFN_MASK      CONST64U(0xffffffffff000)
 #define LM_PTE_2_PFN(_pte)   (((_pte) & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
@@ -52,7 +53,6 @@
 #define LM_AVAIL_MASK         (CONST64(0x7) << LM_AVAIL_SHIFT)
 #define LM_FLAGS_MASK         CONST64(0x80000000000001ff)
 #define LM_CR3_FLAGS_MASK     CONST64(0x18)
-#define LM_L4_L3_RSVD_MASK    CONST64(0x180)
 #define LM_L3_1G_RSVD_MASK    CONST64(0x3fffe000)

 #define LM_MAKE_CR3(_mpfn, _flags) \
@@ -65,12 +65,62 @@
                    ((uint64)(_flags) & LM_FLAGS_MASK))

 #define LM_MAKE_PDE(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L5E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L4E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L3E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L2E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L1E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)


+/*
+ *----------------------------------------------------------------------
+ *
+ * LMPTEIsSafe --
+ *
+ *    A shadow PTE is considered "safe" if any of the following conditions are
+ *    met:
+ *
+ *      a) It is not terminal (i.e., present with no reserved bits set)
+ *      b) Terminal, but with MPN and PS fields set to zero
+ *      c) Terminal, but with MPN field specifying an uncachable page
+ *
+ *    In practice, for condition c), we require that bits 45:43 of the EPTE are
+ *    set to b'111.  The position of these bits is undocumented and not
+ *    architectural; they are truly magic.
+ *
+ *----------------------------------------------------------------------
+ */
+#ifdef VMX86_DEBUG
+
+#define LM_SAFE_BITS_MASK (MASK64(3) << 43)
+#define LM_SAFE_BITS(_v)  ((_v) & LM_SAFE_BITS_MASK)
+
+static INLINE Bool
+LMPTEIsTerminal(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   uint64 rsvd2m = (MASK(PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvd1g = (MASK(2 * PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvdl4 = PTE_PS;
+   uint64 rsvd = physMask & MASK64(52);
+
+   return (!PTE_PRESENT(pte) ||
+           (pte & LM_PTE_PFN_MASK) & rsvd) != 0 ||
+           ((pte & PTE_PS) != 0 &&
+            ((level == PT_LEVEL_2 && (pte & rsvd2m) != 0) ||
+             (level == PT_LEVEL_3 && (pte & rsvd1g) != 0))) ||
+           (level == PT_LEVEL_4 && (pte & rsvdl4) != 0);
+}
+
+static INLINE Bool
+LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   MPN mpn = (pte & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT;
+   Bool safe = !LMPTEIsTerminal(pte, level, physMask) ||
+               (mpn == 0 && !PTE_LARGEPAGE(pte)) ||
+               LM_SAFE_BITS(pte) == LM_SAFE_BITS_MASK;
+   return safe;
+}
+#endif /* VMX86_DEBUG */


 /*
@@ -87,4 +137,95 @@
 #define VA64_CANONICAL_HOLE_START   (CONST64U(1) << (VA64_IMPL_BITS - 1))
 #define VA64_CANONICAL_HOLE_LEN  VA64_CANONICAL_MASK - VA64_CANONICAL_HOLE_START

+static INLINE Bool
+x86IsCanonicalC(VA64 va)
+{
+   return (va & VA64_CANONICAL_MASK) == 0 ||
+          (va & VA64_CANONICAL_MASK) == VA64_CANONICAL_MASK;
+}
+
+#if defined(VM_X86_64) && defined(__GNUC__)
+static INLINE Bool
+x86IsCanonicalAsm(VA64 va)
+{
+   Bool out;
+   /*
+    * sarq $48, %0: Move 17 bits from position 63:47 into 15:0 and CF,
+    *  sign-extending as we do so.
+    * adcl $0, %0: Then add (0 + CF) to the shifted value.  The result is zero
+    *  iff (CF == 1 && bits 31:0 == 0xffffffff) or (CF == 0 && bits 31:0 == 0).
+    *  That is, if original bits 47 and higher were all 1s or all 0s.
+    * setz %1: Create the boolean.
+    *
+    * The chain of AdcLong macros will emit one adc instruction; gcc
+    * has no obvious way to force the size of a quad %0 to a long.
+    */
+#define AdcLong(reg64, reg32) \
+   ".ifc  %0, %%" #reg64 "\n" \
+   " adcl $0, %%" #reg32 "\n" \
+   ".endif \n"
+
+   asm ("sarq $48, %0 \n"
+        AdcLong(rax, eax)
+        AdcLong(rcx, ecx)
+        AdcLong(rdx, edx)
+        AdcLong(rbx, ebx)
+        AdcLong(rsp, esp)
+        AdcLong(rbp, ebp)
+        AdcLong(rsi, esi)
+        AdcLong(rdi, edi)
+        AdcLong(r8,  r8d)
+        AdcLong(r9,  r9d)
+        AdcLong(r10, r10d)
+        AdcLong(r11, r11d)
+        AdcLong(r12, r12d)
+        AdcLong(r13, r13d)
+        AdcLong(r14, r14d)
+        AdcLong(r15, r15d)
+        "setz %1" : "+r"(va), "=r"(out));
+   return out;
+#undef AdcLong
+}
+#endif
+
+static INLINE Bool
+x86_IsCanonical(VA64 va)
+{
+#if defined(VM_X86_64) && defined(__GNUC__)
+   if (__builtin_constant_p(va)) {
+      return x86IsCanonicalC(va);
+   } else {
+      return x86IsCanonicalAsm(va);
+   }
+#else
+   return x86IsCanonicalC(va);
+#endif
+}
+
+static INLINE Bool
+x86_IsCanonicalRange(VA64 va, unsigned size)
+{
+   /*
+    * The check is simple as long as the size is less
+    * than the number of implemented bits.
+    *
+    * The only case we don't handle is one where the VA goes from a
+    * high canonical address and wraps to a non-canonical address
+    * (e.g. 0x00008000_00000000) or higher.  Our test would falsely
+    * consider this canonical.
+    */
+   ASSERT_ON_COMPILE(sizeof(size) * 8 < VA64_IMPL_BITS);
+
+   /*
+    * VA64_CANONICAL_MASK is the lowest canonical address with the
+    * upper bits all set.
+    *
+    * VA64_CANONICAL_HOLE_START is one higher than the highest valid
+    * canonical address with the upper bits all cleared.  Note that we
+    * access up to (va + size - 1), not (va + size), so <= is correct.
+    */
+   return va >= VA64_CANONICAL_MASK ||
+          va + size <= VA64_CANONICAL_HOLE_START;
+}
+
 #endif /* _X86PAGING_64_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,7 @@
 #define PF_RSVD         0x0008
 #define PF_ID           0x0010
 #define PF_PK           0x0020
+#define PF_SS           0x0040
 #define PF_SGX          0x8000

 /*
@@ -109,4 +110,12 @@ typedef struct {
   uint64   la;      /* Used for single address invalidation.  */
 } InvpcidArg;

+typedef enum {
+   PM_FLAT,        /* non paging mode                                      */
+   PM_2_LEVEL,     /* classic 32 bit non-pae style paging mode.            */
+   PM_3_LEVEL,     /* classic 32 bit pae style paging mode.                */
+   PM_4_LEVEL,     /* long mode style paging mode (including compat mode). */
+   PM_NUM     = 4, /* number of different paging modes.                    */
+} PagingMode;
+
 #endif /* _X86PAGING_COMMON_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,21 +65,21 @@ typedef struct Descriptor {
 typedef 
 #include "vmware_pack_begin.h"
 struct Descriptor64 {
-   unsigned   limit_lo  : 16;   // Limit bits 15-0.
-   unsigned   base_lo   : 24;   // Base bits  23-0.
-   unsigned   type      : 4;
-   unsigned   S         : 1;
-   unsigned   DPL       : 2;
-   unsigned   present   : 1;
-   unsigned   limit_hi  : 4;    // Limit bits 19-16.
-   unsigned   AVL       : 1;
-   unsigned   reserved0 : 2;
-   unsigned   gran      : 1;
-   unsigned   base_mid  : 8;    // Base bits 31-24.
-   unsigned   base_hi   : 32;   // Base bits 63-32.
-   unsigned   reserved1 : 8;
-   unsigned   ext_attrs : 5;
-   unsigned   reserved2 : 19;
+   uint64   limit_lo  : 16;   // Limit bits 15-0.
+   uint64   base_lo   : 24;   // Base bits  23-0.
+   uint64   type      : 4;
+   uint64   S         : 1;
+   uint64   DPL       : 2;
+   uint64   present   : 1;
+   uint64   limit_hi  : 4;    // Limit bits 19-16.
+   uint64   AVL       : 1;
+   uint64   reserved0 : 2;
+   uint64   gran      : 1;
+   uint64   base_mid  : 8;    // Base bits 31-24.
+   uint64   base_hi   : 32;   // Base bits 63-32.
+   uint64   reserved1 : 8;
+   uint64   ext_attrs : 5;
+   uint64   reserved2 : 19;
 }
 #include "vmware_pack_end.h"
 Descriptor64;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,6 +39,7 @@
 extern "C" {
 #endif

+#include "vm_assert.h"

 #define SELECTOR_GDT             0
 #define SELECTOR_LDT             1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -47,8 +47,10 @@
 #endif
 #include "x86cpuid_asm.h"

-#define SVM_VMCB_IO_BITMAP_SIZE    (3 * PAGE_SIZE)
-#define SVM_VMCB_MSR_BITMAP_SIZE   (2 * PAGE_SIZE)
+#define SVM_VMCB_IO_BITMAP_PAGES   (3)
+#define SVM_VMCB_IO_BITMAP_SIZE    PAGES_2_BYTES(SVM_VMCB_IO_BITMAP_PAGES)
+#define SVM_VMCB_MSR_BITMAP_PAGES  (2)
+#define SVM_VMCB_MSR_BITMAP_SIZE   PAGES_2_BYTES(SVM_VMCB_MSR_BITMAP_PAGES)

 /* Exit controls for the CR/DR access and hardware exceptions */
 #define SVM_CR_RD_CTL(num)         (0x1 << (num))
@@ -63,53 +65,56 @@
 #define SVM_XCP_CTL_ALL            0xffffffff

 /* Execution intercept controls */
-/* VMCB.exitCtl */
-#define SVM_VMCB_EXEC_CTL_INTR        0x0000000000000001ULL
-#define SVM_VMCB_EXEC_CTL_NMI         0x0000000000000002ULL
-#define SVM_VMCB_EXEC_CTL_SMI         0x0000000000000004ULL
-#define SVM_VMCB_EXEC_CTL_INIT        0x0000000000000008ULL
-#define SVM_VMCB_EXEC_CTL_VINTR       0x0000000000000010ULL
-#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR  0x0000000000000020ULL
-#define SVM_VMCB_EXEC_CTL_SIDT        0x0000000000000040ULL
-#define SVM_VMCB_EXEC_CTL_SGDT        0x0000000000000080ULL
-#define SVM_VMCB_EXEC_CTL_SLDT        0x0000000000000100ULL
-#define SVM_VMCB_EXEC_CTL_STR         0x0000000000000200ULL
-#define SVM_VMCB_EXEC_CTL_LIDT        0x0000000000000400ULL
-#define SVM_VMCB_EXEC_CTL_LGDT        0x0000000000000800ULL
-#define SVM_VMCB_EXEC_CTL_LLDT        0x0000000000001000ULL
-#define SVM_VMCB_EXEC_CTL_LTR         0x0000000000002000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSC       0x0000000000004000ULL
-#define SVM_VMCB_EXEC_CTL_RDPMC       0x0000000000008000ULL
-#define SVM_VMCB_EXEC_CTL_PUSHF       0x0000000000010000ULL
-#define SVM_VMCB_EXEC_CTL_POPF        0x0000000000020000ULL
-#define SVM_VMCB_EXEC_CTL_CPUID       0x0000000000040000ULL
-#define SVM_VMCB_EXEC_CTL_RSM         0x0000000000080000ULL
-#define SVM_VMCB_EXEC_CTL_IRET        0x0000000000100000ULL
-#define SVM_VMCB_EXEC_CTL_SWINT       0x0000000000200000ULL
-#define SVM_VMCB_EXEC_CTL_INVD        0x0000000000400000ULL
-#define SVM_VMCB_EXEC_CTL_PAUSE       0x0000000000800000ULL
-#define SVM_VMCB_EXEC_CTL_HLT         0x0000000001000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPG      0x0000000002000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPGA     0x0000000004000000ULL
-#define SVM_VMCB_EXEC_CTL_IOIO        0x0000000008000000ULL
-#define SVM_VMCB_EXEC_CTL_MSR         0x0000000010000000ULL
-#define SVM_VMCB_EXEC_CTL_TS          0x0000000020000000ULL
-#define SVM_VMCB_EXEC_CTL_FERR_FRZ    0x0000000040000000ULL
-#define SVM_VMCB_EXEC_CTL_SHUTDOWN    0x0000000080000000ULL
-#define SVM_VMCB_EXEC_CTL_VMRUN       0x0000000100000000ULL
-#define SVM_VMCB_EXEC_CTL_VMMCALL     0x0000000200000000ULL
-#define SVM_VMCB_EXEC_CTL_VMLOAD      0x0000000400000000ULL
-#define SVM_VMCB_EXEC_CTL_VMSAVE      0x0000000800000000ULL
-#define SVM_VMCB_EXEC_CTL_STGI        0x0000001000000000ULL
-#define SVM_VMCB_EXEC_CTL_CLGI        0x0000002000000000ULL
-#define SVM_VMCB_EXEC_CTL_SKINIT      0x0000004000000000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSCP      0x0000008000000000ULL
-#define SVM_VMCB_EXEC_CTL_ICEBP       0x0000010000000000ULL
-#define SVM_VMCB_EXEC_CTL_WBINVD      0x0000020000000000ULL
-#define SVM_VMCB_EXEC_CTL_MONITOR     0x0000040000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT       0x0000080000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT_COND  0x0000100000000000ULL
-#define SVM_VMCB_EXEC_CTL_XSETBV      0x0000200000000000ULL
+/* VMCB.execCtl */
+#define SVM_VMCB_EXEC_CTL_INTR           0x0000000000000001ULL
+#define SVM_VMCB_EXEC_CTL_NMI            0x0000000000000002ULL
+#define SVM_VMCB_EXEC_CTL_SMI            0x0000000000000004ULL
+#define SVM_VMCB_EXEC_CTL_INIT           0x0000000000000008ULL
+#define SVM_VMCB_EXEC_CTL_VINTR          0x0000000000000010ULL
+#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR     0x0000000000000020ULL
+#define SVM_VMCB_EXEC_CTL_SIDT           0x0000000000000040ULL
+#define SVM_VMCB_EXEC_CTL_SGDT           0x0000000000000080ULL
+#define SVM_VMCB_EXEC_CTL_SLDT           0x0000000000000100ULL
+#define SVM_VMCB_EXEC_CTL_STR            0x0000000000000200ULL
+#define SVM_VMCB_EXEC_CTL_LIDT           0x0000000000000400ULL
+#define SVM_VMCB_EXEC_CTL_LGDT           0x0000000000000800ULL
+#define SVM_VMCB_EXEC_CTL_LLDT           0x0000000000001000ULL
+#define SVM_VMCB_EXEC_CTL_LTR            0x0000000000002000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSC          0x0000000000004000ULL
+#define SVM_VMCB_EXEC_CTL_RDPMC          0x0000000000008000ULL
+#define SVM_VMCB_EXEC_CTL_PUSHF          0x0000000000010000ULL
+#define SVM_VMCB_EXEC_CTL_POPF           0x0000000000020000ULL
+#define SVM_VMCB_EXEC_CTL_CPUID          0x0000000000040000ULL
+#define SVM_VMCB_EXEC_CTL_RSM            0x0000000000080000ULL
+#define SVM_VMCB_EXEC_CTL_IRET           0x0000000000100000ULL
+#define SVM_VMCB_EXEC_CTL_SWINT          0x0000000000200000ULL
+#define SVM_VMCB_EXEC_CTL_INVD           0x0000000000400000ULL
+#define SVM_VMCB_EXEC_CTL_PAUSE          0x0000000000800000ULL
+#define SVM_VMCB_EXEC_CTL_HLT            0x0000000001000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPG         0x0000000002000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPGA        0x0000000004000000ULL
+#define SVM_VMCB_EXEC_CTL_IOIO           0x0000000008000000ULL
+#define SVM_VMCB_EXEC_CTL_MSR            0x0000000010000000ULL
+#define SVM_VMCB_EXEC_CTL_TS             0x0000000020000000ULL
+#define SVM_VMCB_EXEC_CTL_FERR_FRZ       0x0000000040000000ULL
+#define SVM_VMCB_EXEC_CTL_SHUTDOWN       0x0000000080000000ULL
+#define SVM_VMCB_EXEC_CTL_VMRUN          0x0000000100000000ULL
+#define SVM_VMCB_EXEC_CTL_VMMCALL        0x0000000200000000ULL
+#define SVM_VMCB_EXEC_CTL_VMLOAD         0x0000000400000000ULL
+#define SVM_VMCB_EXEC_CTL_VMSAVE         0x0000000800000000ULL
+#define SVM_VMCB_EXEC_CTL_STGI           0x0000001000000000ULL
+#define SVM_VMCB_EXEC_CTL_CLGI           0x0000002000000000ULL
+#define SVM_VMCB_EXEC_CTL_SKINIT         0x0000004000000000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSCP         0x0000008000000000ULL
+#define SVM_VMCB_EXEC_CTL_ICEBP          0x0000010000000000ULL
+#define SVM_VMCB_EXEC_CTL_WBINVD         0x0000020000000000ULL
+#define SVM_VMCB_EXEC_CTL_MONITOR        0x0000040000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT          0x0000080000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT_COND     0x0000100000000000ULL
+#define SVM_VMCB_EXEC_CTL_XSETBV         0x0000200000000000ULL
+#define SVM_VMCB_EXEC_CTL_EFER_WR_TRAP   0x0000800000000000ULL
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP(n) (0x0001000000000000ULL << (n))
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP_ALL 0xffff000000000000ULL

 /* VMCB.tlbCtl */
 #define SVM_VMCB_TLB_CTL_FLUSH              0x01
@@ -130,9 +135,13 @@

 /* VMCB.intrState */
 #define SVM_VMCB_INTRSTATE_INTR_SHADOW     (1 << 0)
+#define SVM_VMCB_INTRSTATE_GUEST_IF        (1 << 1)

 /* VMCB.nptCtl */
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)
+#define SVM_VMCB_NPTCTL_SEV_ENABLE         (1 << 1)
+#define SVM_VMCB_NPTCTL_SEV_ES_ENABLE      (1 << 2)
+#define SVM_VMCB_NPTCTL_GMET_ENABLE        (1 << 3)

 /* VMCB.virtExt */
 #define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
@@ -234,9 +243,17 @@
 #define SVM_EXITCODE_MWAIT                139
 #define SVM_EXITCODE_MWAIT_COND           140
 #define SVM_EXITCODE_XSETBV               141
+#define SVM_EXITCODE_EFER_WRITE_TRAP      143
+#define SVM_EXITCODE_CR_WRITE_TRAP(n)    (144 + (n))
 #define SVM_EXITCODE_NPF                 1024
 #define SVM_EXITCODE_AVIC_INCOMPLETE_IPI 1025
 #define SVM_EXITCODE_AVIC_NOACCEL        1026
+#define SVM_EXITCODE_VMGEXIT             1027
+#define SVM_EXITCODE_MMIO_READ           0x80000001   // SW only
+#define SVM_EXITCODE_MMIO_WRITE          0x80000002   // SW only
+#define SVM_EXITCODE_NMI_COMPLETE        0x80000003   // SW only
+#define SVM_EXITCODE_AP_JUMP_TABLE       0x80000005   // SW only
+#define SVM_EXITCODE_UNSUPPORTED         0x8000FFFF   // SW only
 #define SVM_EXITCODE_INVALID             (-1ULL)

 /* ExitInfo1 for I/O exits */
@@ -260,6 +277,9 @@
 #define SVM_IOEXIT_MBZ           0x00000002
 #define SVM_IOEXIT_SBZ           0x0000e000

+/* ExitInfo1 for MSR exits */
+#define SVM_MSREXIT_WRMSR        0x00000001
+
 /* ExitInfo1 for CR exits */
 #define SVM_CREXIT_MOVCR         (1ull << 63)
 #define SVM_CREXIT_GPR_MASK      0xf
@@ -295,6 +315,10 @@
 #define SVM_NPFEXIT_FINAL_ADDR     (1ULL << 32)    /* Rev. C */
 #define SVM_NPFEXIT_PTE_ACCESS     (1ULL << 33)    /* Rev. C */

+/* ExitInfo1 for AP jump table exits */
+#define SVM_APEXIT_SET            0x0
+#define SVM_APEXIT_GET            0x1
+
 /* Event Injection */
 #define SVM_INTINFO_VECTOR_MASK   0x000000ff
 #define SVM_INTINFO_TYPE_SHIFT    8
@@ -430,7 +454,7 @@ SVM_LockedFromFeatures(uint64 vmCR)
 static INLINE Bool
 SVM_EnabledCPU(void)
 {
-   return SVM_EnabledFromFeatures(__GET_MSR(MSR_VM_CR));
+   return SVM_EnabledFromFeatures(X86MSR_GetMSR(MSR_VM_CR));
 }
 #endif /* } !defined(USERLEVEL) */

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -62,13 +62,6 @@ typedef uint64 VM_EPTE;
 * Registers
 */

-typedef union SharedReg64 {
-   Reg8  reg8[2];
-   Reg16 reg16;
-   Reg32 reg32;
-   Reg64 reg64;
-} SharedReg64;
-
 typedef union SharedUReg64 {
   UReg8  ureg8[2];
   UReg16 ureg16;
@@ -178,11 +171,7 @@ typedef Task32 Task;

 #pragma pack(push, 1)
 typedef struct {
-#if defined(VMM) || defined(COREQUERY)
-   uint64 va;
-#else
-   uint32 va;
-#endif
+   uint64   va;
   Selector seg;
 } FarPtr;
 #pragma pack(pop)
@@ -468,4 +457,12 @@ typedef struct ExcFrame64ForCopy {
 } ExcFrame64ForCopy;
 #pragma pack(pop)

+/*
+ * Layout of the stack for performing a 64 bit lret instruction.
+ */
+typedef struct LretFrame64 {
+   uint64 rip;
+   uint64 cs;
+} LretFrame64;
+
 #endif // ifndef _X86TYPES_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,12 @@ VT_EXIT(RDSEED,               61)
 VT_EXIT(PML_LOGFULL,          62)
 VT_EXIT(XSAVES,               63)
 VT_EXIT(XRSTORS,              64)
-
+VT_EXIT(VMEXIT65,             65)
+VT_EXIT(VMEXIT66,             66)
+VT_EXIT(VMEXIT67,             67)
+VT_EXIT(VMEXIT68,             68)
+VT_EXIT(VMEXIT69,             69)
+VT_EXIT(ENCLV,                70)
+VT_EXIT(SGX_CONFLICT,         71)
 /* Bump this up if you add an exit reason. */
-#define VT_NUM_EXIT_REASONS   65
+#define VT_NUM_EXIT_REASONS   72
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -95,9 +95,10 @@
 *       _V  -- virtualized
 *       _NV -- not virtualized
 *   _access is one of:
- *       _NA -- no access
- *       _RW -- read/write access
- *       _RO -- read-only access
+ *       _NA  -- no access
+ *       _RW  -- read/write access
+ *       _RO  -- read-only access
+ *       _URW -- read/write acess for the ULM only
 *   _sticky is one of:
 *       _S  -- sticky
 *       _NS -- non-sticky
@@ -109,9 +110,9 @@
 VMCS_SET_START(16)
 /* 16-bit control fields. */
 VMCS_GROUP_START(16, CTL)
-VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA,  _S)
-VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA, _NS)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV,  _NA,  _S)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _URW, _NS)
 VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
 VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
 VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
@@ -283,7 +284,7 @@ VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2030, _S64, _TC, 24)
 VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA,  _S)
 VMCS_UNUSED(                        0x2034, _S64, _TC, 26)
-VMCS_UNUSED(                        0x2036, _S64, _TC, 27)
+VMCS_FIELD(ENCLV_EXITING_BITMAP,    0x2036, _S64, _TC, 27, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2038, _S64, _TC, 28)
 VMCS_UNUSED(                        0x203A, _S64, _TC, 29)
 VMCS_UNUSED(                        0x203C, _S64, _TC, 30)
@@ -328,16 +329,16 @@ VMCS_GROUP_END(64, DATA)

 /* 64-bit guest state. */
 VMCS_GROUP_START(64, GUEST)
-VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA, _NS)
-VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA, _NS)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _URW, _NS)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V,  _NA, _NS)
 VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
 VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
 VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
@@ -403,24 +404,24 @@ VMCS_SET_END(64)
 VMCS_SET_START(32)
 /* 32-bit control fields. */
 VMCS_GROUP_START(32, CTL)
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA, _NS)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA,  _S)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA,  _S)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _URW,  _S)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _URW, _NS)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _URW,  _S)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V,  _NA,  _S)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
 VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
 VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
@@ -551,14 +552,14 @@ VMCS_SET_END(32)
 VMCS_SET_START(NAT)
 /* natural-width control fields. */
 VMCS_GROUP_START(NAT, CTL)
-VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
 VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
 VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
@@ -622,26 +623,26 @@ VMCS_GROUP_END(NAT, DATA)

 /* natural-width guest state. */
 VMCS_GROUP_START(NAT, GUEST)
-VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW, _NS)
-VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW, _NS)
-VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW, _NS)
-VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA, _NS)
-VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _URW, _NS)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _URW, _NS)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V,  _RW, _NS)
 VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
 VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
 VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
@@ -658,18 +659,18 @@ VMCS_GROUP_END(NAT, GUEST)

 /* natural-width host state. */
 VMCS_GROUP_START(NAT, HOST)
-VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
 VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
 VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2004-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2004-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -159,10 +159,10 @@
 #define VT_ENCODING_RSVD               0xffff9000

 /*
- * The highest index of any currently defined field is 25, for
- * TSC_MULTIPLIER.
+ * The highest index of any currently defined field is 27, for
+ * ENCLV_EXITING_BITMAP.
 */
-#define VT_ENCODING_MAX_INDEX                  25
+#define VT_ENCODING_MAX_INDEX                  27

 enum {
 #define VMCS_FIELD(_name, _val, ...) VT_VMCS_##_name = _val,
@@ -173,8 +173,11 @@ enum {
 /*
 * Sizes of referenced fields
 */
-#define VT_VMCS_IO_BITMAP_SIZE    (2 * PAGE_SIZE)
-#define VT_VMCS_MSR_BITMAP_SIZE   (1 * PAGE_SIZE)
+#define VT_VMCS_IO_BITMAP_PAGES   (2)
+#define VT_VMCS_IO_BITMAP_SIZE    PAGES_2_BYTES(VT_VMCS_IO_BITMAP_PAGES)
+#define VT_VMCS_MSR_BITMAP_PAGES  (1)
+#define VT_VMCS_MSR_BITMAP_SIZE   PAGES_2_BYTES(VT_VMCS_MSR_BITMAP_PAGES)
+

 /*
 * Execution control bit capabilities come in pairs: a "required" bit in
@@ -293,7 +296,9 @@ enum {
   VMX_CPU2(PT_SUPPRESS_NR_BIT, 19)                      \
   VMX_CPU2(XSAVES,             20)                      \
   VMX_CPU2(EPT_MBX,            22)                      \
-   VMX_CPU2(TSC_SCALING,        25)
+   VMX_CPU2(TSC_SCALING,        25)                      \
+   VMX_CPU2(ENCLV,              28)                      \
+   VMX_CPU2(EPC_VIRT_EXT,       29)

 #define VMX_PROCBASED_CTLS2_CAP                          \
        VMX_PROCBASED_CTLS2_CAP_NDA                      \
@@ -550,7 +555,7 @@ enum {
 #define VT_VMCS_PENDDBG_MBZ        0xfffeaff0

 /* Exception error must-be-zero bits for VMEntry */
-#define VT_XCP_ERR_MBZ             0xffff8000
+#define VT_XCP_ERR_MBZ             0xffff0000


 /* Exit reasons. */
@@ -687,6 +692,12 @@ enum {
 #define VT_GUESTFAIL_QUAL_NMI          3
 #define VT_GUESTFAIL_QUAL_LINK         4

+/* SGX conflict VM-exit Qualification Codes */
+#define VT_SGX_TRACKING_RESOURCE_CONFLICT     0
+#define VT_SGX_TRACKING_REFERENCE_CONFLICT    1
+#define VT_SGX_EPC_PAGE_CONFLICT_EXCEPTION    2
+#define VT_SGX_EPC_PAGE_CONFLICT_ERROR        3
+
 /* VMX abort indicators. */

 #define VT_VMX_ABORT_GUEST_MSRS        1
@@ -710,7 +721,8 @@ enum {
 #define VT_REQUIRED_PINBASED_CTLS                      \
   (VT_PINBASED_CTLS_DEFAULT1                        | \
    VT_VMCS_PIN_VMEXEC_CTL_EXTINT_EXIT               | \
-    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT)
+    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT                  | \
+    VT_VMCS_PIN_VMEXEC_CTL_VNMI)

 #define VT_REQUIRED_PROCBASED_CTLS                     \
   (VT_PROCBASED_CTLS_DEFAULT1                       | \
@@ -726,6 +738,7 @@ enum {
    VT_VMCS_CPU_VMEXEC_CTL_LDCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_STCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_TPR_SHADOW                | \
+    VT_VMCS_CPU_VMEXEC_CTL_VNMI_WINDOW               | \
    VT_VMCS_CPU_VMEXEC_CTL_MONITOR)

 #define VT_REQUIRED_EXIT_CTLS                          \
@@ -1075,7 +1088,7 @@ VT_MBXSupportedFromFeatures(uint64 secondary)
 static INLINE Bool
 VT_EnabledCPU(void)
 {
-   return VT_EnabledFromFeatures(__GET_MSR(MSR_FEATCTL));
+   return VT_EnabledFromFeatures(X86MSR_GetMSR(MSR_FEATCTL));
 }


@@ -1096,16 +1109,14 @@ VT_EnabledCPU(void)
 static INLINE Bool
 VT_SupportedCPU(void)
 {
-   if (__GET_MSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) {
-      return VT_SupportedFromFeatures(__GET_MSR(MSR_VMX_TRUE_PINBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_PROCBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_ENTRY_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_EXIT_CTLS),
-                                      __GET_MSR(MSR_VMX_BASIC));
-   } else {
-      /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
-      return FALSE;
-   }
+
+   /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
+   return (X86MSR_GetMSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) != 0 &&
+          VT_SupportedFromFeatures(X86MSR_GetMSR(MSR_VMX_TRUE_PINBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_PROCBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_ENTRY_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_EXIT_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_BASIC));
 }

 #endif /* } !defined(USERLEVEL) */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -23,6 +23,7 @@

 #include <linux/file.h>
 #include <linux/highmem.h>
+#include <linux/miscdevice.h>
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/slab.h>
@@ -36,8 +37,8 @@

 #include "usercalldefs.h"

-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
-#error Linux kernels before 2.6.32 are not supported
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0)
+#error Linux kernels before 3.10 are not supported
 #endif

 #include <asm/io.h>
@@ -48,12 +49,10 @@
 #include "modulecall.h"
 #include "vm_asm.h"
 #include "vmx86.h"
-#include "initblock.h"
 #include "task.h"
 #include "memtrack.h"
 #include "task.h"
 #include "cpuid.h"
-#include "cpuid_info.h"
 #include "circList.h"
 #include "x86msr.h"

@@ -64,13 +63,12 @@
 #include "hostif.h"
 #include "hostif_priv.h"
 #include "vmhost.h"
+#include "sharedAreaVmmon.h"

 static void LinuxDriverQueue(Device *device);
 static void LinuxDriverDequeue(Device *device);
 static Bool LinuxDriverCheckPadding(void);

-#define VMMON_UNKNOWN_SWAP_SIZE -1ULL
-
 struct VMXLinuxState linuxState;


@@ -96,20 +94,25 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd,
                       unsigned long ioarg);

 static int LinuxDriver_Close(struct inode *inode, struct file *filp);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-static int LinuxDriverFault(struct vm_fault *fault);
-#else
-static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault);
-#endif
-static int LinuxDriverMmap(struct file *filp, struct vm_area_struct *vma);

 static unsigned int LinuxDriverEstimateTSCkHz(void);

-static struct vm_operations_struct vmuser_mops = {
-        .fault  = LinuxDriverFault
+static const struct file_operations vmuser_fops = {
+   .owner = THIS_MODULE,
+   .open = LinuxDriver_Open,
+   .release = LinuxDriver_Close,
+   .unlocked_ioctl = LinuxDriver_Ioctl,
+   .compat_ioctl = LinuxDriver_Ioctl,
 };

-static struct file_operations vmuser_fops;
+#ifndef VMX86_DEVEL
+static struct miscdevice vmmon_miscdev = {
+   .name = "vmmon",
+   .minor = MISC_DYNAMIC_MINOR,
+   .fops = &vmuser_fops,
+};
+#endif
+
 static struct timer_list tscTimer;
 static Atomic_uint32 tsckHz;
 static VmTimeStart tsckHzStartTime;
@@ -264,9 +267,8 @@ LinuxDriverInitTSCkHz(void)
 *      linux module entry point. Called by /sbin/insmod command
 *
 * Results:
- *      registers a device driver for a major # that depends
- *      on the uid. Add yourself to that list.  List is now in
- *      private/driver-private.c.
+ *      Release: registers a device driver with a misc minor node.
+ *      Devel: registers for a major number with user-created node.
 *
 *----------------------------------------------------------------------
 */
@@ -290,48 +292,46 @@ init_module(void)
      return -ENOEXEC;
   }

+   if (!Vmx86_CreateHVIOBitmap()) {
+      return -ENOMEM;
+   }
+
+   if (!Vmx86_CheckMSRUniformity()) {
+      return -EPERM;
+   }
+
   linuxState.fastClockThread = NULL;
   linuxState.fastClockRate = 0;
-   linuxState.swapSize = VMMON_UNKNOWN_SWAP_SIZE;
-
-   /*
-    * Initialize the file_operations structure. Because this code is always
-    * compiled as a module, this is fine to do it here and not in a static
-    * initializer.
-    */
-
-   memset(&vmuser_fops, 0, sizeof vmuser_fops);
-   vmuser_fops.owner = THIS_MODULE;
-   vmuser_fops.unlocked_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.compat_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.open = LinuxDriver_Open;
-   vmuser_fops.release = LinuxDriver_Close;
-   vmuser_fops.mmap = LinuxDriverMmap;

 #ifdef VMX86_DEVEL
   devel_init_module();
-   linuxState.minor = 0;
   retval = register_chrdev(linuxState.major, linuxState.deviceName,
                            &vmuser_fops);
+   if (retval) {
+      Warning("Module %s: error registering with major=%d\n",
+              linuxState.deviceName, linuxState.major);
+   } else {
+      Log("Module %s: registered with major=%d\n",
+          linuxState.deviceName, linuxState.major);
+   }
 #else
   sprintf(linuxState.deviceName, "vmmon");
   linuxState.major = 10;
-   linuxState.minor = 165;
-   linuxState.misc.minor = linuxState.minor;
-   linuxState.misc.name = linuxState.deviceName;
-   linuxState.misc.fops = &vmuser_fops;

-   retval = misc_register(&linuxState.misc);
+   retval = misc_register(&vmmon_miscdev);
+   if (retval) {
+      Warning("Module %s: error registering misc device %s\n",
+              linuxState.deviceName, vmmon_miscdev.name);
+   } else {
+      Log("Module %s: registered as misc device %s\n", linuxState.deviceName,
+          vmmon_miscdev.name);
+   }
 #endif

   if (retval) {
-      Warning("Module %s: error registering with major=%d minor=%d\n",
-              linuxState.deviceName, linuxState.major, linuxState.minor);
-
+      Vmx86_CleanupHVIOBitmap();
      return -ENOENT;
   }
-   Log("Module %s: registered with major=%d minor=%d\n",
-       linuxState.deviceName, linuxState.major, linuxState.minor);

   HostIF_InitUptime();
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
@@ -369,13 +369,14 @@ cleanup_module(void)
 #ifdef VMX86_DEVEL
   unregister_chrdev(linuxState.major, linuxState.deviceName);
 #else
-   misc_deregister(&linuxState.misc);
+   misc_deregister(&vmmon_miscdev);
 #endif

   Log("Module %s: unloaded\n", linuxState.deviceName);

   del_timer_sync(&tscTimer);

+   Vmx86_CleanupHVIOBitmap();
   Task_Terminate();
   // Make sure fastClockThread is dead
   HostIF_FastClockLock(1);
@@ -413,7 +414,7 @@ LinuxDriver_Open(struct inode *inode, // IN
   }
   memset(device, 0, sizeof *device);

-   sema_init(&device->lock4Gb, 1);
+   init_rwsem(&device->vmDriverRWSema);

   filp->private_data = device;
   LinuxDriverQueue(device);
@@ -424,114 +425,6 @@ LinuxDriver_Open(struct inode *inode, // IN
 }


-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocPages --
- *
- *    Allocate physically contiguous block of memory with specified order.
- *    Pages in the allocated block are configured so that caller can pass
- *    independent pages to the VM.
- *
- * Results:
- *    Zero on success, non-zero (error code) on failure.
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverAllocPages(unsigned int gfpFlag, // IN
-                      unsigned int order,   // IN
-                      struct page **pg,     // OUT
-                      unsigned int size)    // IN
-{
-   struct page* page;
-
-   page = alloc_pages(gfpFlag, order);
-   if (page) {
-      unsigned int i;
-
-      /*
-       * Grab an extra reference on all pages except first one - first
-       * one was already refcounted by alloc_pages.
-       *
-       * Under normal situation all pages except first one in the block
-       * have refcount zero.  As we pass these pages to the VM, we must
-       * bump their count, otherwise VM will release these pages every
-       * time they would be unmapped from user's process, causing crash.
-       *
-       * Note that this depends on Linux VM internals.  It works on all
-       * kernels we care about.
-       */
-
-      order = 1 << order;
-      for (i = 0; i < order; i++) {
-         if (i) {
-            /*
-             * Debug kernels assert that page->_count is not zero when
-             * calling get_page. We use init_page_count as a temporary
-             * workaround. PR 894174
-             */
-#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 16)
-            ASSERT(page_count(page) == 0);
-            init_page_count(page);
-#else
-            get_page(page);
-#endif
-         }
-         if (i >= size) {
-            put_page(page);
-         } else {
-            void *addr = kmap(page);
-
-            memset(addr, 0, PAGE_SIZE);
-            kunmap(page);
-            *pg++ = page;
-         }
-         page++;
-      }
-
-      return 0;
-   }
-
-   return -ENOMEM;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverDestructor4Gb --
- *
- *    Deallocate all directly mappable memory.
- *
- * Results:
- *    None
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static void
-LinuxDriverDestructor4Gb(Device *device) // IN
-{
-   unsigned int pg;
-
-   if (!device->size4Gb) {
-      return;
-   }
-   for (pg = 0; pg < device->size4Gb; pg++) {
-      put_page(device->pages4Gb[pg]);
-   }
-   device->size4Gb = 0;
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -554,220 +447,21 @@ LinuxDriver_Close(struct inode *inode, // IN
   ASSERT(device);

   LinuxDriverDequeue(device);
+   down_write(&device->vmDriverRWSema);
   if (device->vm != NULL) {
      Vmx86_ReleaseVM(device->vm);
      device->vm = NULL;
   }
+   up_write(&device->vmDriverRWSema);

   Vmx86_Close();

-   /*
-    * Destroy all low memory allocations.
-    * We are closing the struct file here, so clearly no other process
-    * uses it anymore, and we do not need to hold the semaphore.
-    */
-
-   LinuxDriverDestructor4Gb(device);
-
   kfree(device);
   filp->private_data = NULL;

   return 0;
 }

-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverFault --
- *
- *      Callback for returning allocated page for memory mapping
- *
- * Results:
- *    NoPage:
- *      Page or page address on success, NULL or 0 on failure.
- *    Fault:
- *      Error code; 0, minor page fault.
- *
- * Side effects:
- *      None.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-LinuxDriverFault(struct vm_fault *fault)     //IN/OUT
-#else
-LinuxDriverFault(struct vm_area_struct *vma, //IN
-                 struct vm_fault *fault)     //IN/OUT
-#endif
-{
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-   struct vm_area_struct *vma = fault->vma;
-#endif
-   Device *device = (Device *) vma->vm_file->private_data;
-   unsigned long pg;
-   struct page* page;
-
-   pg = fault->pgoff;
-   pg = VMMON_MAP_OFFSET(pg);
-   if (pg >= device->size4Gb) {
-      return VM_FAULT_SIGBUS;
-   }
-   page = device->pages4Gb[pg];
-   get_page(page);
-   fault->page = page;
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocContig --
- *
- *      Create mapping for contiguous memory areas.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int LinuxDriverAllocContig(Device *device,
-                                  struct vm_area_struct *vma,
-                                  unsigned long off,
-                                  unsigned long size)
-{
-   unsigned long vmaOrder = VMMON_MAP_ORDER(off);
-   unsigned long vmaAllocSize;
-   unsigned int gfpFlag;
-   unsigned long i;
-
-   if (VMMON_MAP_RSVD(off)) {
-      /* Reserved bits set... */
-      return -EINVAL;
-   }
-   if (VMMON_MAP_OFFSET(off)) {
-      /* We do not need non-zero offsets... */
-      return -EINVAL;
-   }
-   switch (VMMON_MAP_MT(off)) {
-      case VMMON_MAP_MT_LOW4GB:
-#ifdef GFP_DMA32
-         gfpFlag = GFP_USER | GFP_DMA32;
-#else
-         gfpFlag = GFP_USER | GFP_DMA;
-#endif
-         break;
-      case VMMON_MAP_MT_LOW16MB:
-         gfpFlag = GFP_USER | GFP_DMA;
-         break;
-      case VMMON_MAP_MT_ANY:
-         gfpFlag = GFP_HIGHUSER;
-         break;
-      default:
-         /* Invalid memory type */
-         return -EINVAL;
-   }
-   if (size > VMMON_MAP_OFFSET_MASK + 1) {
-      /* Size is too big to fit to our window. */
-      return -ENOMEM;
-   }
-
-   /* 16 pages looks like a good limit... */
-   if (size > VMMON_MAX_LOWMEM_PAGES) {
-      return -ENOMEM;
-   }
-   /* Sorry. Only one mmap per one open. */
-   down(&device->lock4Gb);
-   if (device->size4Gb) {
-      up(&device->lock4Gb);
-      return -EINVAL;
-   }
-   vmaAllocSize = 1 << vmaOrder;
-   for (i = 0; i < size; i += vmaAllocSize) {
-      int err;
-
-      err = LinuxDriverAllocPages(gfpFlag, vmaOrder,
-                                  device->pages4Gb + i, size - i);
-      if (err) {
-         while (i > 0) {
-            put_page(device->pages4Gb[--i]);
-         }
-         up(&device->lock4Gb);
-
-         return err;
-      }
-   }
-   device->size4Gb = size;
-   up(&device->lock4Gb);
-   vma->vm_ops = &vmuser_mops;
-
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverMmap --
- *
- *      Create mapping for lowmem or locked memory.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverMmap(struct file *filp,
-                struct vm_area_struct *vma)
-{
-   Device *device = (Device *) filp->private_data;
-   unsigned long size;
-   int err;
-
-   /* Only shared mappings */
-   if (!(vma->vm_flags & VM_SHARED)) {
-      return -EINVAL;
-   }
-   if ((vma->vm_end | vma->vm_start) & (PAGE_SIZE - 1)) {
-      return -EINVAL;
-   }
-   size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
-   if (size < 1) {
-      return -EINVAL;
-   }
-   if (device->vm) {
-      err = -EINVAL;
-   } else {
-      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
-   }
-   if (err) {
-      return err;
-   }
-   /* Clear VM_IO, otherwise SuSE's kernels refuse to do get_user_pages */
-   vma->vm_flags &= ~VM_IO;
-
-   return 0;
-}
-
-
 typedef Bool (*SyncFunc)(void *data, unsigned cpu);

 typedef struct {
@@ -982,7 +676,7 @@ LinuxDriverReadTSC(void *data,   // OUT: TSC values
 *-----------------------------------------------------------------------------
 */

-__attribute__((always_inline)) static Bool
+__always_inline static Bool
 LinuxDriverSyncReadTSCs(uint64 *delta) // OUT: TSC max - TSC min
 {
   TSCDelta tscDelta;
@@ -1034,26 +728,30 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
                  unsigned long ioarg)  // IN:
 {
   Device *device = (Device *) filp->private_data;
-   int retval = 0;
+   long retval = 0;
   Vcpuid vcpuid;
   VMDriver *vm;
+   Bool needsWriteLock = iocmd == IOCTL_VMX86_CREATE_VM ||
+                         iocmd == IOCTL_VMX86_RELEASE_VM;

+   ASSERT_ON_COMPILE(sizeof(long) == sizeof(int64));
   if (device == NULL) {
      return -EINVAL;
   }

+   if (needsWriteLock) {
+      down_write(&device->vmDriverRWSema);
+   } else {
+      down_read(&device->vmDriverRWSema);
+   }
+
   vm = device->vm;
-
-   /*
-    * Validate the VM pointer for those IOCTLs that require it.
-    */
-
+   /* Validate the VM pointer for those IOCTLs that require it. */
   switch (iocmd) {
   case IOCTL_VMX86_VERSION:
   case IOCTL_VMX86_CREATE_VM:
   case IOCTL_VMX86_SET_UID:
   case IOCTL_VMX86_GET_NUM_VMS:
-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
   case IOCTL_VMX86_SET_HARD_LIMIT:
   case IOCTL_VMX86_GET_IPI_VECTORS:
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
@@ -1101,6 +799,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      } else {
         device->vm = vm;
         args.vmid = vm->userID;
+         vm->vmhost->vmmonData = (void *__user)args.vmmonData;
         retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args);
      }
      break;
@@ -1108,67 +807,73 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:

   case IOCTL_VMX86_PROCESS_BOOTSTRAP: {
      VMProcessBootstrapBlock *args;
+      size_t argsSz = GetVMProcessBootstrapBlockSize(vm->numVCPUs);
      VA64 uAddr;
-      Bool res;

      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
      if (retval != 0) {
         break;
      }
-      args = HostIF_AllocKernelMem(sizeof *args, TRUE);
+      args = HostIF_AllocKernelMem(argsSz, TRUE);
      if (args == NULL) {
         retval = -ENOMEM;
         break;
      }
-      retval = HostIF_CopyFromUser(args, uAddr, sizeof *args);
+      retval = HostIF_CopyFromUser(args, uAddr, argsSz);
      if (retval != 0) {
         HostIF_FreeKernelMem(args);
         break;
      }
-      res = Vmx86_ProcessBootstrap(vm,
-                                   args->bsBlobAddr,
-                                   args->numBytes,
-                                   args->headerOffset,
-                                   args->numVCPUs,
-                                   args->ptRootVAs,
-                                   args->shRegions);
-      if (!res) {
+      if (args->numVCPUs != vm->numVCPUs) {
+         retval = -EINVAL;
+      } else if (!Vmx86_ProcessBootstrap(vm,
+                                         args->bsBlobAddr,
+                                         args->numBytes,
+                                         args->headerOffset,
+                                         args->numVCPUs,
+                                         args->perVcpuPages,
+                                         args->shRegions)) {
         retval = -ENOMEM;
      }
      HostIF_FreeKernelMem(args);
      break;
   }

+   case IOCTL_VMX86_REGISTER_SHARED: {
+      VMSharedAreaRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!SharedAreaVmmon_ValidateRegionArgs(vm, &args)) {
+         retval = -EINVAL;
+         break;
+      }
+      if (!SharedAreaVmmon_RegisterRegion(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
+   case IOCTL_VMX86_REGISTER_STATVARS: {
+      VMStatVarsRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!StatVarsVmmon_RegisterVCPU(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
   case IOCTL_VMX86_RELEASE_VM:
      device->vm = NULL;
      Vmx86_ReleaseVM(vm);
      break;

-   case IOCTL_VMX86_INIT_VM: {
-      InitBlock *initParams;
-      VA64 uAddr;
-
-      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
-      if (retval != 0) {
-         break;
-      }
-      initParams = HostIF_AllocKernelMem(sizeof *initParams, TRUE);
-      if (initParams == NULL) {
-         retval = -ENOMEM;
-         break;
-      }
-      retval = HostIF_CopyFromUser(initParams, uAddr, sizeof *initParams);
-      if (retval == 0) {
-         if (Vmx86_InitVM(vm, initParams) == 0) {
-            retval = HostIF_CopyToUser(uAddr, initParams, sizeof *initParams);
-         } else {
-            retval = -EINVAL;
-         }
-      }
-      HostIF_FreeKernelMem(initParams);
-      break;
-   }
-
   case IOCTL_VMX86_RUN_VM:
      vcpuid = ioarg;

@@ -1236,17 +941,28 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_VMM_PAGE_ROOT: {
+      VcpuPageRoot args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->ptRootMpns == NULL) {
+         retval = -EINVAL;
+      } else {
+         args.pageRoot = vm->ptRootMpns[args.vcpuid];
+         retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      }
+      break;
+   }
+
   case IOCTL_VMX86_GET_NUM_VMS:
      retval = Vmx86_GetNumVMs();
      break;

-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
-      retval = Vmx86_GetTotalMemUsage();
-      break;
-
   case IOCTL_VMX86_SET_HARD_LIMIT: {
-      int32 limit;
-
+      PageCnt limit;
      retval = HostIF_CopyFromUser(&limit, ioarg, sizeof limit);
      if (retval != 0) {
         break;
@@ -1332,25 +1048,12 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

   case IOCTL_VMX86_APIC_INIT: {
-      VMAPICInfo info;
-      Bool setVMPtr;
-      Bool probe;
-
-      retval = HostIF_CopyFromUser(&info, ioarg, sizeof info);
-      if (retval != 0) {
-         break;
-      }
-      setVMPtr = ((info.flags & APIC_FLAG_DISABLE_NMI) != 0);
-      probe = ((info.flags & APIC_FLAG_PROBE) != 0);
-
      /*
-       * Kernel uses NMIs for deadlock detection - set APIC VMptr so that
-       * NMIs get disabled in the monitor.
+       * Kernel uses NMIs for deadlock detection - so we always have to find
+       * APIC so that NMIs get disabled in the monitor.
       */
-
-      setVMPtr = TRUE;
-
-      retval = HostIF_APICInit(vm, setVMPtr, probe) ? 0 : -ENODEV;
+      HostIF_APICInit(vm);
+      retval = 0;
      break;
   }

@@ -1390,6 +1093,23 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_SWITCH_ERROR_ADDR: {
+      VMSwitchErrorArgs args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->crosspage == NULL ||
+          vm->crosspage[args.vcpuid] == NULL) {
+         retval = -EINVAL;
+         break;
+      }
+      args.addr = vm->crosspage[args.vcpuid]->wsUD2;
+      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      break;
+   }
+
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
      retval = LinuxDriverEstimateTSCkHz();
      break;
@@ -1516,6 +1236,13 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_NUM_ANON_PAGES: {
+      PageCnt numAnonPages;
+      numAnonPages = Vmx86_GetNumAnonPages(vm);
+      retval = HostIF_CopyToUser(ioarg, &numAnonPages, sizeof numAnonPages);
+      break;
+   }
+
   case IOCTL_VMX86_READ_PAGE: {
         VMMReadWritePage req;

@@ -1530,6 +1257,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
+#ifdef VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1538,6 +1266,9 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
         }
         retval = HostIF_WritePhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE,
                                       PAGE_SIZE);
+#else
+         retval = -EINVAL;
+#endif
         break;
      }

@@ -1592,17 +1323,6 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

-   case IOCTL_VMX86_SET_HOST_SWAP_SIZE: {
-      uint64 swapSize;
-      retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize);
-      if (retval != 0) {
-         Warning("Could not copy swap size from user, status %d\n", retval);
-         break;
-      }
-      linuxState.swapSize = swapSize;
-      break;
-   }
-
   case IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS: {
      uint64 ctrs = Vmx86_GetUnavailablePerfCtrs();
      retval = HostIF_CopyToUser(ioarg, &ctrs, sizeof ctrs);
@@ -1629,6 +1349,11 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

 exit:
+   if (needsWriteLock) {
+      up_write(&device->vmDriverRWSema);
+   } else {
+      up_read(&device->vmDriverRWSema);
+   }
   return retval;
 }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2011,2014-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -23,49 +23,31 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
+#include <linux/rwsem.h>
 #include <linux/wait.h>

 #include "vmx86.h"
 #include "driver_vmcore.h"


-/*
- * Used to track a piece of memory that's been mapped into the kernel from
- * userlevel.
- */
-typedef struct VMMappedUserMem {
-   void *addr;
-   size_t numPages;
-   struct page *pages[0];
-} VMMappedUserMem;
-
-
-/*
- * Per-instance driver state
- */
-
+/* Per-instance driver state */
 struct VMDriver;

-/* 16 pages (64KB) looks as a good limit for one allocation */
-#define VMMON_MAX_LOWMEM_PAGES  16
-
 typedef struct Device {
   struct Device   *next;
   struct VMDriver *vm;
-
   /*
-    * The semaphore protect accesses to size4Gb and pages4Gb
-    * in mmap(). mmap() may happen only once, and all other
-    * accesses except cleanup are read-only, and may happen
-    * only after successful mmap.
+    * This RW semaphore protects accesses to the VMDriver to
+    * avoid racing between various ioctls, and the creation
+    * and removal of the VM in question. The lock is read-acquired
+    * by ioctls that reference the VMDriver, and write-acquired by
+    * ioctls or device callbacks that allocate or destroy the
+    * VMDriver.
    */
-   struct semaphore lock4Gb;
-   unsigned int size4Gb;
-   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
+   struct rw_semaphore vmDriverRWSema;
 } Device;


@@ -78,19 +60,14 @@ typedef struct Device {

 typedef struct VMXLinuxState {
   int major;
-   int minor;
-   struct miscdevice misc;
   char deviceName[VM_DEVICE_NAME_SIZE];
   char buf[LINUXLOG_BUFFER_SIZE];
   Device *head;

   struct task_struct *fastClockThread;
   unsigned fastClockRate;
-   uint64 swapSize;
 } VMXLinuxState;

 extern VMXLinuxState linuxState;
-extern uint8 monitorIPIVector;
-extern uint8 hvIPIVector;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,12 +25,6 @@
 #ifndef _HOSTIF_PRIV_H_
 #define _HOSTIF_PRIV_H_

-/* Functions for mapping and unmapping userspace memory. */
-struct VMMappedUserMem;
-EXTERN void  *HostIF_MapUserMem(VA addr, size_t size,
-                                struct VMMappedUserMem **handle);
-EXTERN void   HostIF_UnmapUserMem(struct VMMappedUserMem *handle);
-
 /* Uptime-related functions. */
 EXTERN void   HostIF_InitUptime(void);
 EXTERN void   HostIF_CleanupUptime(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -79,8 +79,6 @@ typedef struct VMHost {
    */
   Mutex vmMutex;

-   unsigned int         crosspagePagesCount;
-   struct page        **crosspagePages;      /* ptr to numVCPUs-sized array */
   struct task_struct **vcpuSemaTask;        /* ptr to numVCPUs-sized array */

   /*
@@ -94,8 +92,11 @@ typedef struct VMHost {
    * as pages for "AWE" guest memory.
    */
   struct PhysTracker  *AWEPages;
-   /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */
-   Bool                 hostAPICIsMapped;
+   /*
+    * Pointer to a userlevel 64-bit area containing the value 1.
+    * This is used for HostIF_SemaphoreSignal.
+    */
+   void *__user vmmonData;
 } VMHost;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,7 +24,7 @@
 *
 */

-#if defined(linux)
+#if defined(__linux__)
 /* Must come before any kernel header file */
 #   include "driver-config.h"
 #   include <linux/kernel.h>
@@ -34,7 +34,6 @@
 #include "modulecall.h"
 #include "vmx86.h"
 #include "task.h"
-#include "initblock.h"
 #include "vm_basic_asm.h"
 #include "iocontrols.h"
 #include "hostif.h"
@@ -43,6 +42,7 @@
 #include "usercalldefs.h"
 #include "cpuid.h"
 #include "vmmblob.h"
+#include "sharedAreaVmmon.h"

 /*
 *----------------------------------------------------------------------
@@ -72,13 +72,16 @@ int
 Vmx86_RunVM(VMDriver *vm,   // IN:
            Vcpuid vcpuid)  // IN:
 {
-   uint32           retval    = MODULECALL_USERRETURN;
+   uint64           retval    = MODULECALL_USERRETURN;
   VMCrossPageData *crosspage;
   int              bailValue = 0;
+   Bool             switchOk = TRUE;

-   ASSERT(vcpuid < vm->numVCPUs && CPUID_HostSupportsHV());
-   crosspage = &vm->crosspage[vcpuid]->crosspageData;
-   ASSERT(crosspage);
+   ASSERT(vcpuid < vm->numVCPUs);
+   if (vm->crosspage[vcpuid] == NULL) {
+      return USERCALL_VMX86ALLOCERR;
+   }
+   crosspage = vm->crosspage[vcpuid];

   /*
    * Check if we were interrupted by signal.
@@ -93,9 +96,9 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
       * Task_Switch changes the world to the monitor.
       * The monitor is waiting in the BackToHost routine.
       */
-      UCTIMESTAMP(crosspage, SWITCHING_TO_MONITOR);
-      Task_Switch(vm, vcpuid);
-      UCTIMESTAMP(crosspage, SWITCHED_TO_MODULE);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHING_TO_MONITOR);
+      switchOk = Task_Switch(vm, vcpuid);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHED_TO_MODULE);

      /*
       * Wake up anything that was waiting for this vcpu to run
@@ -117,13 +120,19 @@ skipTaskSwitch:;

      retval = MODULECALL_USERRETURN;

+      if (UNLIKELY(!switchOk)) {
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
+
      if (crosspage->userCallType != MODULECALL_USERCALL_NONE) {
         /*
          * This is the main user call path.  Handle by returning
          * from the ioctl (back to the userlevel side of a VCPU thread).
          */
         bailValue = crosspage->userCallType;
-         crosspage->retval = retval;
+         ASSERT(retval == (uint64)((uint32)retval));
+         crosspage->retval = (uint32)retval;
         goto bailOut;
      }

@@ -136,18 +145,19 @@ skipTaskSwitch:;

      case MODULECALL_GET_RECYCLED_PAGES: {
         MPN mpns[MODULECALL_NUM_ARGS];
-         int nPages = MIN((int)crosspage->args[0], MODULECALL_NUM_ARGS);
-
-         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages, TRUE,
-                                         FALSE);
+         PageCnt nPages = MIN(crosspage->args[0], MODULECALL_NUM_ARGS);
+         ASSERT((int64)crosspage->args[0] >= 0);
+         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages,
+                                         TRUE, FALSE);
         if (retval <= nPages) {
-            int i;
+            PageCnt i;
            for (i = 0; i < retval; i++) {
               crosspage->args[i] = mpns[i];
            }
         } else {
-            // retval is holding an error code
-            Warning("Failed to alloc %u pages: %d\n", nPages, (int)retval);
+            /* retval is holding an error code */
+            Warning("Failed to alloc %"FMT64"u pages: %"FMT64"d\n", nPages,
+                    (int64)retval);
            retval = 0;
         }
         break;
@@ -161,7 +171,6 @@ skipTaskSwitch:;

      case MODULECALL_SEMAWAIT: {
         retval = HostIF_SemaphoreWait(vm, vcpuid, crosspage->args);
-
         if (retval == MX_WAITINTERRUPTED) {
            crosspage->moduleCallInterrupted = TRUE;
            bailValue = USERCALL_RESTART;
@@ -171,7 +180,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_SEMASIGNAL: {
-         retval = HostIF_SemaphoreSignal(crosspage->args);
+         retval = HostIF_SemaphoreSignal(vm, crosspage->args);

         if (retval == MX_WAITINTERRUPTED) {
             crosspage->moduleCallInterrupted = TRUE;
@@ -197,7 +206,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_RELEASE_ANON_PAGES: {
-         unsigned count;
+         PageCnt count;
         MPN mpns[MODULECALL_NUM_ARGS];
         for (count = 0; count < MODULECALL_NUM_ARGS; count++) {
            mpns[count] = (MPN)crosspage->args[count];
@@ -206,15 +215,14 @@ skipTaskSwitch:;
            }
         }
         ASSERT(count > 0);
-         retval = Vmx86_FreeLockedPages(vm, PtrToVA64(mpns), count, TRUE);
+         retval = Vmx86_FreeLockedPages(vm, mpns, count);
         break;
      }

      case MODULECALL_LOOKUP_MPN: {
-         int i;
-         VPN64  vpn    = (VPN64)crosspage->args[0];
-         uint32 nPages = (uint32)crosspage->args[1];
-         VA64   uAddr  = (VA64)VPN_2_VA(vpn);
+         VPN64   vpn        = (VPN64)crosspage->args[0];
+         PageCnt i, nPages  = crosspage->args[1];
+         VA64    uAddr      = (VA64)VPN_2_VA(vpn);
         ASSERT(nPages <= MODULECALL_NUM_ARGS);
         HostIF_VMLock(vm, 38);
         for (i = 0; i < nPages; i++) {
@@ -280,6 +288,10 @@ skipTaskSwitch:;
         retval = HostIF_GetHVIPIVector();
      } break;

+      case MODULECALL_GET_PERF_CTR_VECTOR: {
+         retval = HostIF_GetPerfCtrVector();
+      } break;
+
      case MODULECALL_GET_HOST_TIMER_VECTORS: {
         uint8 v0, v1;
         HostIF_GetTimerVectors(&v0, &v1);
@@ -292,17 +304,76 @@ skipTaskSwitch:;
         vm->blobInfo = NULL;
      } break;

+      case MODULECALL_GET_SHARED_AREA: {
+         SharedAreaVmmonRequest request;
+         request.type       = (SharedAreaType)crosspage->args[0];
+         request.vcpu       = (Vcpuid)crosspage->args[1];
+         request.offset     = (PageCnt)crosspage->args[2];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[3] = SharedAreaVmmon_GetRegionMPN(vm, &request);
+      } break;
+
+      case MODULECALL_GET_STAT_VARS: {
+         Vcpuid  vcpu   = (Vcpuid)crosspage->args[0];
+         PageCnt offset = (PageCnt)crosspage->args[1];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[2] = StatVarsVmmon_GetRegionMPN(vm, vcpu, offset);
+      } break;
+
+      case MODULECALL_GET_NUM_PTP_PAGES: {
+         /* Store PageCnt in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[1] = vm->numPTPPages;
+      } break;
+
+      case MODULECALL_GET_HV_IO_BITMAP: {
+         crosspage->args[0] = hvIOBitmap == NULL ? INVALID_MPN :
+                                                   hvIOBitmap->mpn;
+      } break;
+
+      case MODULECALL_GET_MSR: {
+         /*
+          * Allocate a MSRQuery request on stack with only one MSRReply slot
+          * since this MSR will be queried from a MSR cache or on a single pcpu.
+          */
+         uint8 req[sizeof(MSRQuery) + sizeof(MSRReply)];
+         MSRQuery *query = (MSRQuery *)&req[0];
+         query->msrNum = (uint32)crosspage->args[0];
+         query->numLogicalCPUs = 1;
+         retval = Vmx86_GetAllMSRs(query);
+         crosspage->args[0] = query->logicalCPUs[0].msrVal;
+      } break;
+
+      case MODULECALL_ALLOC_CONTIG_PAGES: {
+         PageCnt pages = crosspage->args[0];
+         HostIFContigMemMap *alloc;
+
+         HostIF_VMLock(vm, 47);
+         alloc = HostIF_AllocContigPages(vm, pages);
+         if (alloc != NULL) {
+            alloc->next = vm->contigMappings;
+            vm->contigMappings = alloc;
+            crosspage->args[1] = alloc->mpn;
+         } else {
+            crosspage->args[1] = INVALID_MPN;
+         }
+         HostIF_VMUnlock(vm, 47);
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }

-      crosspage->retval = retval;
-
-#if defined(linux)
-      cond_resched(); // Other kernels are preemptable
+      if (retval != (uint64)((uint32)retval)) {
+         Warning("Unexpected error in modulecall %u (%llu)\n",
+                 crosspage->moduleCallType, retval);
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
+      crosspage->retval = (uint32)retval;
+#if defined(__linux__)
+      cond_resched(); // Other kernels are preemptible
 #endif
   }
-
 bailOut:
   return bailValue;
 }
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -129,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -32,6 +32,6 @@ $(DRIVER)-y := driver.o hub.o userif.o netif.o bridge.o procfs.o smac_compat.o \
 ####

 clean:
-	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko \
+	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko .cache.mk \
 	       .tmp_versions Module.symvers Modules.symvers Module.markers \
 	       modules.order $($(DRIVER)-y) .*.cmd .*.o.flags
@@ -66,7 +66,7 @@
 #endif

 #if LOGLEVEL >= 4
-static struct timeval vnetTime;
+static u64 vnetTime;
 #endif

 typedef struct VNetBridge VNetBridge;
@@ -693,7 +693,7 @@ VNetBridgeReceiveFromVNet(VNetJack        *this, // IN: jack

 	 netif_rx_ni(clone);
 #	 if LOGLEVEL >= 4
-	 do_gettimeofday(&vnetTime);
+	 vnetTime = ktime_get_ns();
 #	 endif
      }
   }
@@ -1490,12 +1490,11 @@ VNetBridgeReceiveFromDev(struct sk_buff *skb,         // IN: packet to receive

 #  if LOGLEVEL >= 4
   {
-      struct timeval now;
-      do_gettimeofday(&now);
+      u64 now;
+
+      now = ktime_get_ns();
      LOG(3, (KERN_DEBUG "bridge-%s: time %d\n",
-	      bridge->name,
-	      (int)((now.tv_sec * 1000000 + now.tv_usec)
-                    - (vnetTime.tv_sec * 1000000 + vnetTime.tv_usec))));
+	      bridge->name, (int)((now - vnetTime) / NSEC_PER_USEC)));
   }
 #  endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,8 @@

 #define ETHER_MAX_QUEUED_PACKET 1600

+/* Most Ethernet equipment can support jumbo frames up to 9216 bytes. */
+#define ETHER_MAX_JUMBO_FRAME_LEN 9216

 /*
 * State's that a NIC can be in currently we only use this
@@ -198,7 +200,7 @@ Net_GetNumAdapters(Net_AdapterCount *counts)
      } else if (Str_Strcasecmp(adapterStr, "e1000e") == 0) {
         counts->e1000e++;
      } else {
-         LOG_ONCE(("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr));
+         LOG_ONCE("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr);
      }
      free(adapterStr);
   }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -28,6 +28,7 @@

 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
 #include <linux/mm.h>
 #include "compat_skbuff.h"
 #include <linux/sockios.h>
@@ -44,6 +45,22 @@
 #include "vmnetInt.h"


+/*
+ * Default min MTU value as defined by kernel versions >= 4.10.0.
+ * Use the same value for earlier versions of the kernel which do not
+ * enforce a minimum MTU size.
+ */
+#define VMNET_MIN_MTU 68
+#define VMNET_MAX_MTU (ETHER_MAX_JUMBO_FRAME_LEN - \
+                      (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
+
+#ifdef RHEL_RELEASE_CODE
+#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 5) && \
+     RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8, 0))
+#define ndo_change_mtu ndo_change_mtu_rh74
+#endif
+#endif /* RHEL_RELEASE_CODE */
+
 typedef struct VNetNetIF {
   VNetPort                port;
   struct net_device      *dev;
@@ -64,6 +81,11 @@ static int  VNetNetifSetMAC(struct net_device *dev, void *addr);
 static void VNetNetifSetMulticast(struct net_device *dev);
 static int  VNetNetIfProcRead(char *page, char **start, off_t off,
                              int count, int *eof, void *data);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+static int VNetNetifChangeMtu(struct net_device *dev, int new_mtu);
+#endif
+

 /*
 *----------------------------------------------------------------------
@@ -84,7 +106,7 @@ static int  VNetNetIfProcRead(char *page, char **start, off_t off,
 static void
 VNetNetIfSetup(struct net_device *dev)  // IN:
 {
-#if !COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) || defined(HAVE_NET_DEVICE_OPS)
+#ifdef HAVE_NET_DEVICE_OPS
   static const struct net_device_ops vnetNetifOps = {
      .ndo_init = VNetNetifProbe,
      .ndo_open = VNetNetifOpen,
@@ -97,12 +119,23 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
 #else
      .ndo_set_rx_mode = VNetNetifSetMulticast,
 #endif
-
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
+      .ndo_change_mtu = VNetNetifChangeMtu,
+#endif
   };
+#endif /* HAVE_NET_DEVICE_OPS */
+
+   /* Turns on IFF_BROADCAST, IFF_MULTICAST. */
+   ether_setup(dev);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+   /* Set the maximum allowed MTU value. */
+   dev->max_mtu = VMNET_MAX_MTU;
 #endif

-   ether_setup(dev); // turns on IFF_BROADCAST, IFF_MULTICAST
-#if COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) && !defined(HAVE_NET_DEVICE_OPS) 
+#ifdef HAVE_NET_DEVICE_OPS
+   dev->netdev_ops = &vnetNetifOps;
+#else
   dev->init = VNetNetifProbe;
   dev->open = VNetNetifOpen;
   dev->hard_start_xmit = VNetNetifStartXmit;
@@ -110,9 +143,10 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
   dev->get_stats = VNetNetifGetStats;
   dev->set_mac_address = VNetNetifSetMAC;
   dev->set_multicast_list = VNetNetifSetMulticast;
-#else
-   dev->netdev_ops = &vnetNetifOps;
-   #endif /* HAVE_NET_DEVICE_OPS */
+#ifdef HAVE_CHANGE_MTU
+   dev->change_mtu = VNetNetifChangeMtu;
+#endif /* HAVE_CHANGE_MTU */
+#endif /* HAVE_NET_DEVICE_OPS */

 }

@@ -530,6 +564,38 @@ VNetNetifSetMulticast(struct net_device *dev) // IN: unused
 }


+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+/*
+ *----------------------------------------------------------------------
+ *
+ * VNetNetifChangeMtu --
+ *
+ *      Changes the current MTU value of a given vmnet interface.
+ *
+ * Results:
+ *      Returns zero on success, or invalid argument error if an incorrect
+ *      MTU size is being set.
+ *
+ * Side effects:
+ *      None.
+ *----------------------------------------------------------------------
+ */
+
+int
+VNetNetifChangeMtu(struct net_device *dev, // IN/OUT: assigning the mtu member.
+                   int new_mtu)            // IN:     new mtu value.
+{
+   if (new_mtu < VMNET_MIN_MTU || new_mtu > VMNET_MAX_MTU) {
+      return -EINVAL;
+   }
+
+   dev->mtu = new_mtu;
+   return 0;
+}
+#endif
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013, 2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -137,6 +137,7 @@ VNetProcShow(struct seq_file *p, // IN:
 }


+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0)
 /*
 *----------------------------------------------------------------------
 *
@@ -168,6 +169,7 @@ static struct file_operations fops = {
   .release = single_release,
 };
 #endif
+#endif


 /*
@@ -203,7 +205,12 @@ VNetProcMakeEntryInt(VNetProcEntry   *parent,   // IN:
      } else {
         ent->data   = data;
         ent->fn     = fn;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+         ent->pde    = proc_create_single_data(name, mode, parent->pde,
+                                               VNetProcShow, ent);
+#else
         ent->pde    = proc_create_data(name, mode, parent->pde, &fops, ent);
+#endif
      }
      if (ent->pde != NULL) {
         *ret = ent;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,8 +41,6 @@

 #else /* _WIN32 */

-#undef __KERNEL__ //To prevent including any kernel specific stuff 
-
 #ifdef VMX86_DEVEL
 #define DBG 1
 #else
@@ -4329,7 +4327,7 @@ LookupTypeClass(unsigned short typeValue) // IN: ethernet type
 *----------------------------------------------------------------------
 */

-#if _WIN32
+#ifdef _WIN32
 _Use_decl_annotations_
 #endif
 void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -78,11 +78,13 @@ static int  VNetUserIfSetUplinkState(VNetPort *port, uint8 linkUp);
 extern unsigned int  vnet_max_qlen;

 #if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
-#   define compat_kmap(page) kmap(page)
-#   define compat_kunmap(page) kunmap(page)
-#else
-#   define compat_kmap(page) kmap((page).p)
-#   define compat_kunmap(page) kunmap((page).p)
+#   define skb_frag_page(frag) (frag)->page
+#   define skb_frag_size(frag) (frag)->size
+#endif
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0) && \
+    !(defined(CONFIG_SUSE_VERSION) && CONFIG_SUSE_VERSION == 15 && \
+      defined(CONFIG_SUSE_PATCHLEVEL) && CONFIG_SUSE_PATCHLEVEL >= 2)
+#   define skb_frag_off(frag) (frag)->page_offset
 #endif

 /*
@@ -137,16 +139,21 @@ UserifLockPage(VA addr) // IN
 */

 static INLINE int
-VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapPtr(VA64 uAddr,      // IN: pointer to user memory
                 size_t size,     // IN: size of data
                 struct page **p, // OUT: locked page
                 void **ptr)      // OUT: kernel mapped pointer
 {
-   if (!access_ok(VERIFY_WRITE, (void *)uAddr, size) ||
-       (((uAddr + size - 1) & ~(PAGE_SIZE - 1)) !=
-        (uAddr & ~(PAGE_SIZE - 1)))) {
+   uint8 v;
+
+   /* Check area does not straddle two pages. */
+   if ((uAddr & (PAGE_SIZE - 1)) + size > PAGE_SIZE) {
      return -EINVAL;
   }
+   /* Check if it is user's area.  UserifLockPage() checks writability. */
+   if (copy_from_user(&v, (void *)(unsigned long)uAddr, sizeof v) != 0) {
+      return -EFAULT;
+   }

   *p = UserifLockPage(uAddr);
   if (*p == NULL) {
@@ -158,7 +165,7 @@ VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
 }

 static INLINE int
-VNetUserIfMapUint32Ptr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapUint32Ptr(VA64 uAddr,      // IN: pointer to user memory
                       struct page **p, // OUT: locked page
                       uint32 **ptr)    // OUT: kernel mapped pointer
 {
@@ -201,7 +208,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      return -EBUSY;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->pollPtr, &pollPage,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->pollPtr, &pollPage,
                                        &pollPtr)) < 0) {
      return retval;
   }
@@ -213,7 +220,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      goto error_free;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->recvClusterPtr,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->recvClusterPtr,
                                        &recvClusterPage,
                                        &recvClusterCount)) < 0) {
      goto error_free;
@@ -362,12 +369,12 @@ VNetUserIfReceive(VNetJack       *this, // IN
   VNetUserIF *userIf = (VNetUserIF*)this->private;
   uint8 *dest = SKB_2_DESTMAC(skb);
   unsigned long flags;
-   
+
   if (!UP_AND_RUNNING(userIf->port.flags)) {
      userIf->stats.droppedDown++;
      goto drop_packet;
   }
-   
+
   if (!VNetPacketMatch(dest,
                        userIf->port.paddr,
                        (const uint8 *)userIf->port.exactFilter,
@@ -377,13 +384,17 @@ VNetUserIfReceive(VNetJack       *this, // IN
      userIf->stats.droppedMismatch++;
      goto drop_packet;
   }
-   
+
   if (skb_queue_len(&userIf->packetQueue) >= vnet_max_qlen) {
      userIf->stats.droppedOverflow++;
      goto drop_packet;
   }
-   
-   if (skb->len > ETHER_MAX_QUEUED_PACKET) {
+
+   /*
+    * The check would be more accurate if based on the current MTU value
+    * of the corresponding vmnet interface. PR 2267716 is filed to track this.
+    */
+   if (skb->len > ETHER_MAX_JUMBO_FRAME_LEN) {
      userIf->stats.droppedLargePacket++;
      goto drop_packet;
   }
@@ -550,7 +561,12 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
      return -EINVAL;
   }

+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
   csum = csum_and_copy_to_user(skb->data + offset, curr, len, 0, &err);
+#else
+   csum = csum_and_copy_to_user(skb->data + offset, curr, len);
+   err = (csum == 0) ? -EFAULT : 0;
+#endif
   if (err) {
      return err;
   }
@@ -559,20 +575,26 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
   for (frag = skb_shinfo(skb)->frags;
 	frag != skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
 	frag++) {
-      if (frag->size > 0) {
+      if (skb_frag_size(frag) > 0) {
 	 unsigned int tmpCsum;
 	 const void *vaddr;

-	 vaddr = compat_kmap(frag->page);
-	 tmpCsum = csum_and_copy_to_user(vaddr + frag->page_offset,
-					 curr, frag->size, 0, &err);
-	 compat_kunmap(frag->page);
+	 vaddr = kmap(skb_frag_page(frag));
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
+	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
+					 curr, skb_frag_size(frag), 0, &err);
+#else
+	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
+					 curr, skb_frag_size(frag));
+         err = (tmpCsum == 0) ? -EFAULT : 0;
+#endif
+	 kunmap(skb_frag_page(frag));

 	 if (err) {
 	    return err;
 	 }
 	 csum = csum_block_add(csum, tmpCsum, curr - buf);
-	 curr += frag->size;
+	 curr += skb_frag_size(frag);
      }
   }

@@ -747,7 +769,7 @@ VNetUserIfRead(VNetPort    *port, // IN
 *----------------------------------------------------------------------
 */

-static int 
+static int
 VNetUserIfWrite(VNetPort    *port, // IN
                struct file *filp, // IN
                const char  *buf,  // IN
@@ -757,11 +779,14 @@ VNetUserIfWrite(VNetPort    *port, // IN
   struct sk_buff *skb;

   /*
-    * Check size
+    * Check size.
+    *
+    * Regarding the upper count limit the check would be more accurate
+    * if based on the current MTU value of the corresponding vmnet interface.
+    * PR 2267716 is filed to track this.
    */
-   
-   if (count < sizeof (struct ethhdr) || 
-       count > ETHER_MAX_QUEUED_PACKET) {
+   if (count < sizeof (struct ethhdr) ||
+       count > ETHER_MAX_JUMBO_FRAME_LEN) {
      return -EINVAL;
   }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -202,12 +202,12 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +314,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -122,87 +129,29 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -2303,7 +2195,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2216,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3581,7 +3476,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3604,7 +3499,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -3618,7 +3513,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -48,7 +55,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -221,8 +227,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +269,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +382,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +398,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -497,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -550,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -586,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -630,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -653,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -768,27 +760,21 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -836,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -849,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -888,7 +868,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +882,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1078,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1136,113 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+#endif
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,24 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -113,40 +95,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +169,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +272,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +316,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +398,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +483,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +507,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -244,9 +145,127 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_sfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +297,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +412,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +430,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +510,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
Author	SHA1	Message	Date
Michal Kubecek	aa241f9c4e	vmmon: use standard definition of MSR_TSX_FORCE_ABORT if available The MSR_TSX_FORCE_ABORT macro is defined in mainline since commit 52f64909409c ("x86: Add TSX Force Abort CPUID/MSR") in v5.1-rc1.	2020-11-20 10:42:58 +01:00
Michal Kubecek	3b8b45db97	vmnet: convert VNetBridge debugging code to ktime_get_ns() The do_gettimeofday() helper was removed by commit e4b92b108c6c ("timekeeping: remove obsolete time accessors") in v5.0-rc1 and since commit c766d1472c70 ("y2038: hide timeval/timespec/itimerval/itimerspec types") in v5.6-rc3, struct timeval should no longer be used in kernel code. Convert the do_gettimeofday() relics in VNetBridge (which are only compiled with LOGLEVEL >= 4) completely to ktime based interface.	2020-11-20 10:42:22 +01:00
Michal Kubecek	118a62923f	vmmon: fix always_inline attribute usage Function declared with __attribute__((always_inline)) should also be declared as inline, otherwise gcc issues a warning "always_inline function might not be inlinable". It's just cosmetic but getting rid of known harmless warnings makes it easier to spot actual problems. Use the __always_inline macro for LinuxDriverSyncReadTSCs() as this is how always inline functions should be declared in kernel code.	2020-11-20 10:33:49 +01:00
Michal Kubecek	6e4a029ff4	vmmon: use standard definition of MSR_K7_HWCR_SMMLOCK if available The MSR_K7_HWCR_SMMLOCK macro is defined in mainline since commit 18c71ce9c882 ("x86/CPU/AMD: Add the Secure Encrypted Virtualization CPU feature") in v4.16-rc1.	2020-11-20 10:33:34 +01:00
Theo Bertacchini	c009b6c030	modules: remove .cache.mk on make clean Remove .cache.mk inside vmmon-only and vmnet-only when executing "make clean". This file can cause issues when upgrading gcc as make will still look for includes inside older gcc includes directory. File .cache.mk with cache of generated variables was created by build since kernel v4.15-rc1, commit 3298b690b21c ("kbuild: Add a cache for generated variables") until the feature was removed in v4.18-rc1, commit e08d6de4e532 ("kbuild: remove kbuild cache").	2020-11-20 10:32:57 +01:00
Michal Kubecek	8d95f4a9eb	vmmon: quick workaround for objtool warnings As discussed in https://bugzilla.suse.com/show_bug.cgi?id=1059674 the reason for multiple objtool warnings is the fact that vmmon module defines its own Panic() function which never returns. While it is marked as such which is used by the compiler for optimization, there is no way to find this from object file. While this seems innocuous, it might result in problems with unwinder later. The quickest way around is to replace vmmon's own Panic() with standard kernel panic() until a cleaner solution is found.	2020-11-20 10:32:42 +01:00
Michal Kubecek	367fd2f9bb	vmmon: use standard definition of CR3_PCID_MASK if available The CR3_PCID_MASK is defined in mainline asm/processor-flags.h since commit 6c690ee1039b ("x86/mm: Split read_cr3() into read_cr3_pa() and __read_cr3()") in v4.13-rc1.	2020-11-20 10:32:08 +01:00
Michal Kubecek	a1cbe0a517	vmmon: use standard definition of MSR_MISC_FEATURES_ENABLES if available The MSR_MISC_FEATURES_ENABLES macro is defined in mainline since commit ab6d9468631a ("x86/msr: Rename MISC_FEATURE_ENABLES to MISC_FEATURES_ENABLES") in v4.12-rc1. The MSR_MISC_FEATURES_ENABLES_CPUID_FAULT macro is defined in mainline since commit e9ea1e7f53b8 ("x86/arch_prctl: Add ARCH_[GET\|SET]_CPUID") in v4.12-rc1.	2020-11-20 10:31:40 +01:00
Michal Kubecek	81199fdcc2	vmmon: rename GDT_SIZE macro to avoid collision When building against kernel 4.12 and newer, macro name GDT_SIZE used in (vmmon) include/segs.h collides with macro defined in (kernel source) arch/x86/include/asm/segment.h, resulting in its redefinition. To prevent potential problems, rename vmmon's GDT_SIZE to VMMON_GDT_SIZE and GDT_LIMIT to VMMON_GDT_LIMIT. (There is no GDT_LIMIT in mainline kernel source but let's be consistent.)	2020-11-20 10:30:39 +01:00
Michal Kubecek	1ed98fa816	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE_VMXNET3 if available The PCI_DEVICE_ID_VMWARE_VMXNET3 macro is defined in mainline pci_ids.h since commit b1226c7db1d9 ("vmxnet3: Move PCI Id to pci_ids.h") in v4.10-rc1.	2020-11-20 10:29:05 +01:00
Michal Kubecek	0a3c167687	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE if available The PCI_VENDOR_ID_VMWARE macro is defined in mainline pci_ids.h since commit 94e57fea6202 ("PCI: Move PCI_VENDOR_ID_VMWARE to pci_ids.h") in v3.18-rc1.	2020-11-20 10:27:21 +01:00
Michal Kubecek	087aeb56ea	vmmon: fix wrong VMX86_DEVEL test If VMX86_DEVEL macro is not defined, preprocessor warns when it is tested with "#if" in vmmon-only/linux/driver.c. As VMX86_DEVEL is tested with "#ifdef" everywhere else, change this one test to match the rest.	2020-11-20 10:21:49 +01:00
Michal Kubecek	a4d51adffe	import Workstation 16.1.0 module sources	2020-11-20 09:10:32 +01:00
Michal Kubecek	d67ec2e11c	import Workstation 16.0.0 module sources	2020-09-15 17:05:51 +02:00
Michal Kubecek	dec5bab6a0	import Workstation 15.5.5 module sources	2020-05-30 02:35:30 +02:00
Michal Kubecek	02fd1f0829	import Workstation 15.5.2 module sources	2020-03-14 09:56:53 +01:00
Michal Kubecek	bf5aeaca48	import Workstation 15.5.1 module sources	2019-11-14 08:00:40 +01:00
Michal Kubecek	51949a3872	import Workstation 15.5.0 module sources	2019-09-21 00:18:33 +02:00
Michal Kubecek	4d36d6a1d2	Merge branch 'master' into workstation	2019-09-21 00:10:22 +02:00
Michal Kubecek	85ede6aae9	misc: fix wrong directory name in INSTALL (fixes #36 ) Fix wrong directory name in one of the examples.	2019-09-09 08:38:27 +02:00
Michal Kubecek	a93f44ee44	import Workstation 15.1.0 module sources	2019-05-15 07:10:51 +02:00