import Player 16.1.0 module sources

import Player 16.0.0 module sources
import Player 15.5.5 module sources
2020-11-20 09:13:14 +01:00 · 2020-09-15 20:45:48 +02:00 · 2020-05-30 02:30:43 +02:00 · 2020-03-14 09:59:24 +01:00 · 2019-11-14 07:56:57 +01:00 · 2019-09-21 00:11:26 +02:00
100 changed files with 8367 additions and 6651 deletions
@@ -150,7 +150,7 @@ test build).
 When using a downloaded tarball, simply run

  tar -cf vmmon.tar vmmon-only
-  tar -cf vmnet.tar vmmon-only
+  tar -cf vmnet.tar vmnet-only

 to create the tarballs. In this case, there is no protection against
 unwanted local changes.
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -129,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998,2015,2017 VMware, Inc. All rights reserved.
+# Copyright (C) 1998,2015,2017,2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -72,7 +72,7 @@ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')

 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
-C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o
+C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o crosspage.o
 C_TARGETS_VMCORE := moduleloop.o
 C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,14 +24,7 @@
 *      See monLoader.h for a full description.
 */

-#ifdef VMX86_VMX
-
-#include <string.h>
-#include <limits.h>
-#include "vmx.h"
-#include "vm_assert.h"
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #include "vmkernel.h"
 #include "libc.h"
@@ -57,15 +50,15 @@

 #include "vm_assert.h"

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

 #include "vm_basic_types.h"
 #include "monLoader.h"
 #include "vcpuid.h"
-#include "x86types.h"
-#include "x86paging_64.h"
+#include "vm_pagetable.h"
+#include "address_defs.h"
 #include "monLoaderLog.h"

 #define CANONICAL_MASK MASK64(36)
@@ -75,12 +68,6 @@
 #define L2EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 18))
 #define L1EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 9))

-/*
- * Import address space from VMX rather than building it.  Once monitor loading
- * is removed from VMX entirely, code to import will be deleted.
- */
-#define IMPORT_AS_FROM_VMX (TRUE)
-
 /* The maximum (canonical-address) VPN */
 #define VPN_MAX MASK64(52)

@@ -111,15 +98,11 @@ typedef struct MonLoaderContext {
      VPN          ASLastVPN;   /* last VPN in the address space (inclusive) */
      uint64       ASPTEFlags;  /* PTE flags for L4->L1 connection. */
      Vcpuid       currentVCPU;
-      Bool         ASImported;  /* Was this AS imported or created anew? */
      Bool         hasAddrSpace;
   } vcpu;
 } MonLoaderContext;


-static const Bool importFromVMX = IMPORT_AS_FROM_VMX;
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -192,7 +175,7 @@ MonLoaderIsMapped(MonLoaderContext *ctx,    // IN/OUT
   if (ret != ML_OK) {
      return ret;
   }
-   *mapped = PTE_PRESENT(pte);
+   *mapped = ML_PERM_PRESENT(pte);
   return ML_OK;
 }

@@ -280,15 +263,40 @@ MonLoaderMapMPN(MonLoaderContext *ctx,    // IN/OUT
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderBuildsPtLevel --
+ *
+ *      Determines whether MonLoader allocates and maps the page table(s) for
+ *      the monitor at the given level.
+ *
+ * Result:
+ *      TRUE if MonLoader creates the page table at the given level, FALSE
+ *      otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+static Bool
+MonLoaderCreatesPtLevel(PT_Level level)
+{
+   ASSERT(level >= PT_LEVEL_STOP && level <= PT_MAX_LEVELS);
+   return FALSE;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
 * MonLoaderCreateAddressSpace --
 *
- *      Creates or (if importAS is set) verifies an address space.  The VPN
- *      range specified by firstVPN and size is used to determine page counts
- *      at each page table level to map every page.  Sufficient pages are then
- *      allocated or verified.
+ *      Creates or verifies an address space.  The VPN range specified by
+ *      firstVPN and size is used to determine page counts at each page table
+ *      level to map every page.  Sufficient pages are then allocated or
+ *      verified.
 *
 * Result:
 *      ML_OK if successful, some other value on error.
@@ -302,8 +310,7 @@ static MonLoaderError
 MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
                            VPN               firstVPN, // IN
                            uint64            flags,    // IN
-                            uint64            monPages, // IN
-                            Bool              importAS) // IN
+                            uint64            monPages) // IN
 {
   Vcpuid vcpu = ctx->vcpu.currentVCPU;
   VPN lastVPN = firstVPN + monPages - 1;
@@ -329,11 +336,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
   ctx->vcpu.ASFirstVPN = firstVPN;
   ctx->vcpu.ASLastVPN = lastVPN;
   ctx->vcpu.ASPTEFlags = flags;
-   ctx->vcpu.ASImported = importAS;

   ptMPNs = &ctx->vcpu.ptMPNs;

-   if (importAS) {
+   if (MonLoaderCreatesPtLevel(PT_LEVEL_4)) {
+      NOT_IMPLEMENTED();
+   } else {
+      /* Verify the VMX's allocation. */
      mpn = MonLoaderCallout_GetPageRoot(ctx->envCtx, vcpu);
      if (mpn == INVALID_MPN) {
         return ML_ERROR_CALLOUT_PAGEROOT_GET;
@@ -341,13 +350,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
      LOG(5, "%s: vcpu %u page root=0x%"FMT64"x\n", __FUNCTION__, vcpu, mpn);
      ptMPNs->L4MPNs[0] = mpn;
      MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-   } else {
-      NOT_IMPLEMENTED();
   }
   ptMPNs->L4MPNCount = 1;

   for (i = 0; i < L3MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_3)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L4E;
         unsigned L4Off = PT_LPN_2_L4OFF(monVPN);
         PT_L1E pte;
@@ -357,20 +366,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L4E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L3MPNs[i] = mpn;
   }
   ptMPNs->L3MPNCount = L3MPNsNeeded;
      
   for (i = 0; i < L2MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_2)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L3E;
         unsigned L3Off = PT_LPN_2_L3OFF(monVPN);
         unsigned L3Page = L3EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -381,20 +390,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L3E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L2MPNs[i] = mpn;
   }
   ptMPNs->L2MPNCount = L2MPNsNeeded;

   for (i = 0; i < L1MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_1)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L2E;
         unsigned L2Off = PT_LPN_2_L2OFF(monVPN);
         unsigned L2Page = L2EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -405,17 +414,15 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         if (!PTE_PRESENT(pte)) {
+         if (!ML_PERM_PRESENT(pte)) {
            ptMPNs->L1MPNs[i] = INVALID_MPN;
            continue;
         }
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L1MPNs[i] = mpn;
   }
@@ -446,12 +453,12 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
 */
 static MonLoaderError
 MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
-                       unsigned          level,    // IN
+                       PT_Level          level,    // IN
                       uint64            flags,    // IN
                       VPN               monVPN,   // IN
                       uint64            monPages) // IN
 {
-   Bool verify = ctx->vcpu.ASImported;
+   Bool verify = !MonLoaderCreatesPtLevel(level);
   uint64    i;
   unsigned  count;
   MPN      *ptMPNs;
@@ -460,25 +467,25 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
      if (!ctx->vcpu.hasAddrSpace) {
         return ML_ERROR_NO_ADDRSPACE;
      }
-      if (level < 1 || level > 4) {
+      if (level < PT_LEVEL_STOP || level > PT_MAX_LEVELS) {
         return ML_ERROR_ARGS;
      }
   }

   switch (level) {
-      case 4:
+      case PT_LEVEL_4:
         count = ctx->vcpu.ptMPNs.L4MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L4MPNs;
         break;
-      case 3:
+      case PT_LEVEL_3:
         count = ctx->vcpu.ptMPNs.L3MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L3MPNs;
         break;
-      case 2:
+      case PT_LEVEL_2:
         count = ctx->vcpu.ptMPNs.L2MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L2MPNs;
         break;
-      case 1:
+      case PT_LEVEL_1:
         count = ctx->vcpu.ptMPNs.L1MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L1MPNs;
         break;
@@ -494,11 +501,11 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
         VPN vpn = monVPN + i;
         if (verify) {
            PT_L1E l1e;
-            if (ptMPNs[i] == INVALID_MPN && level == 1) {
+            if (ptMPNs[i] == INVALID_MPN && level == PT_LEVEL_1) {
               continue;
            }
            if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK ||
-                LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
+                ML_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
               return ML_ERROR_PAGE_TABLE_VERIFY;
            }
         } else {
@@ -629,7 +636,7 @@ MonLoaderCopyFromBlob(MonLoaderContext *ctx,        // IN/OUT
 *      ML_OK if successful, some other value on error.
 *
 * Side effects:
- *      User pages may be mapped in.
+ *      User or host pages may be mapped in.
 *
 *----------------------------------------------------------------------
 */
@@ -660,6 +667,14 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
                                                  vcpu);
      }
      if (mpn == INVALID_MPN) {
+         if (subIndex == MONLOADER_HT_MAP_IDX) {
+            /*
+             * This item is tied to a vmkernel feature.
+             * When the feature is disabled, there is nothing to share.
+             */
+            ASSERT(vmx86_server);
+            return ML_OK;
+         }
         /* Partial sharing is allowed.  Return success if any occurred. */
         return i != 0 ? ML_OK : ML_ERROR_SHARE;
      }
@@ -672,45 +687,6 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
 }


-#ifndef VMMON
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderSetEntrypoint --
- *
- *      Calls out to set the code and stack entrypoint.
- *
- * Result:
- *      ML_OK if successful, some other value on error.
- *
- * Side effects:
- *      The monitor entrypoint for the current VCPU may be set.
- *
- *----------------------------------------------------------------------
- */
-static MonLoaderError
-MonLoaderSetEntrypoint(MonLoaderContext *ctx,           // IN
-                       uint16            codeSelector,  // IN
-                       VA64              code,          // IN
-                       uint16            stackSelector, // IN
-                       VA64              stack)         // IN
-{
-   if (VA_2_VPN(code) < ctx->vcpu.ASFirstVPN  ||
-       VA_2_VPN(code) > ctx->vcpu.ASLastVPN   ||
-       VA_2_VPN(stack) < ctx->vcpu.ASFirstVPN ||
-       VA_2_VPN(stack) > ctx->vcpu.ASLastVPN) {
-      return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
-   }
-   if (MonLoaderCallout_SetEntrypoint(ctx->envCtx, codeSelector, code,
-                                      stackSelector, stack)) {
-      return ML_OK;
-   } else {
-      return ML_ERROR_CALLOUT_ENTRYPOINT;
-   }
-}
-#endif
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -795,7 +771,7 @@ MonLoaderShareFromBlob(MonLoaderContext *ctx,        // IN/OUT

   if ((blobOffset & (PAGE_SIZE - 1)) != 0 ||
       (blobSize & (PAGE_SIZE - 1)) != 0 ||
-       (flags & PTE_RW) != 0) {
+       ML_PERM_WRITEABLE(flags)) {
      return ML_ERROR_SHARE;
   }
   if (blobSize > monBytes || blobSize == 0) {
@@ -866,7 +842,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
      return ML_ERROR_TABLE_MISSING;
   }

-   if (!MonLoaderCallout_Init(args, &ctx.envCtx)) {
+   if (!MonLoaderCallout_Init(args, &ctx.envCtx, numVCPUs)) {
      return ML_ERROR_CALLOUT_INIT;
   }
   ret = ML_OK;
@@ -887,7 +863,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         uint64               monBytes   = PAGES_2_BYTES(monPages);
         uint64               subIndex   = entry->subIndex;
         uint64               bspOnly    = entry->bspOnly;
-         uint64               isPriv     = vmx86_server || entry->procVmmon;
         *line = i;

         /*
@@ -896,17 +871,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         if (bspOnly && !IS_BOOT_VCPUID(*vcpu)) {
            continue;
         }
-         /*
-          * Entries marked as privileged (isPriv == TRUE) are processed
-          * by the privileged layer (VMKernel/vmmon). Others are
-          * processed by the VMX. One exception is the entry of type
-          * ML_CONTENT_ADDRSPACE: it is always processed by the VMX and,
-          * if marked as privileged, also by the privileged layer.
-          */
-         if (isPriv != MonLoaderCallout_IsPrivileged(ctx.envCtx) &&
-             (content != ML_CONTENT_ADDRSPACE || isPriv == 0)) {
-            continue;
-         }
         if (monPages == 0 || monVPN + monPages - 1 > VPN_MAX) {
            ret = ML_ERROR_SIZE;
            continue;
@@ -919,8 +883,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
                  continue;
               }
               /* Create or verify the address space and page table MPNs. */
-               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages,
-                                                 importFromVMX);
+               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages);
               break;
            case ML_CONTENT_PAGETABLE_L4:
            case ML_CONTENT_PAGETABLE_L3:
@@ -988,15 +951,10 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         }
      }
   }
-#ifndef VMMON
+#ifdef VMKERNEL
   if (ret == ML_OK) {
      if (ctx.vcpu.ASLastVPN == 0) {
         ret = ML_ERROR_NO_ADDRSPACE;
-      } else {
-         ret = MonLoaderSetEntrypoint(&ctx, header->codeSelector,
-                                      header->codeEntrypoint,
-                                      header->stackSelector,
-                                      header->stackEntrypoint);
      }
   }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,7 +65,7 @@ typedef struct MonLoaderEnvContext {
 */

 Bool
-MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx)
+MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx, unsigned numVCPUs)
 {
   MonLoaderEnvContext *c;
   MonLoaderArgs *mlArgs = (MonLoaderArgs *)args;
@@ -215,7 +215,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
   vpn = Vmx86_MapPage(mpn);
   if (vpn == 0) {
      Log("Failed to map MPN 0x%"FMT64"x\n", mpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   HostIF_VMLock(vm, 41);
@@ -225,7 +225,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
      Log("Failed to track mapping from VPN 0x%"FMTVPN"x to MPN 0x%"FMT64"x\n",
          vpn, mpn);
      Vmx86_UnmapPage(vpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   return mpn;
@@ -386,10 +386,8 @@ MonLoaderFindSharedRegion(MonLoaderEnvContext *ctx,
   int i;

   for (i = 0; i < ML_SHARED_REGIONS_MAX; i++) {
-      if (ctx->shRegions[i].baseVpn == INVALID_VPN) {
-         /* Not found. */
-         return NULL;
-      } else if (ctx->shRegions[i].index == index) {
+      if (ctx->shRegions[i].index == index &&
+          ctx->shRegions[i].baseVpn != INVALID_VPN) {
         return &ctx->shRegions[i];
      }
   }
@@ -425,10 +423,6 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,

   ASSERT(IS_BOOT_VCPUID(vcpuid));

-   if (index >= ML_SHARED_REGIONS_MAX) {
-      Log("Invalid shared region %"FMT64"x", index);
-      return INVALID_MPN;
-   }
   s = MonLoaderFindSharedRegion(ctx, index);
   if (s == NULL) {
      return INVALID_MPN;
@@ -453,7 +447,7 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 *
 * MonLoaderCallout_GetSharedUserPage --
 *
- *      Gets a user-provided shared page's MPN for a VCPU.
+ *      Gets a shared page's MPN for a VCPU.
 *
 * Returns:
 *      An MPN or INVALID_MPN.
@@ -465,10 +459,10 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 */

 MPN
-MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx,
-                                   uint64 subIndex, // IN
-                                   unsigned page,   // IN
-                                   Vcpuid vcpu)     // IN
+MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx, // IN
+                                   uint64 subIndex,          // IN
+                                   unsigned page,            // IN
+                                   Vcpuid vcpu)              // IN
 {
   if (subIndex == MONLOADER_HEADER_IDX) {
      return VmmBlob_GetHeaderMpn(ctx->vm);
@@ -484,30 +478,14 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   NOT_IMPLEMENTED();
-}
-
-
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderCallout_IsPrivileged --
- *
- *      Returns whether or not this is a privileged environment.
- *
- * Returns:
- *      TRUE if privileged, FALSE if not.
- *
- * Side effects:
- *      None.
- *
- *----------------------------------------------------------------------
- */
-
-Bool
-MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN
-{
-   return TRUE;
+   switch (subIndex) {
+   case MONLOADER_CROSS_PAGE_DATA_IDX:
+      return HostIF_GetCrossPageDataMPN(ctx->vm->crosspage[vcpu]);
+   case MONLOADER_CROSS_PAGE_CODE_IDX:
+      return HostIF_GetCrossPageCodeMPN();
+   default:
+      return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   }
 }


@@ -534,13 +512,3 @@ MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx,    // IN
   ASSERT((offset & (PAGE_SIZE - 1)) == 0);
   return VmmBlob_GetMpn(ctx->vm, BYTES_2_PAGES(offset));
 }
-
-Bool
-MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *ctx,           // IN
-                               uint16               codeSelector,  // IN
-                               VA64                 code,          // IN
-                               uint16               stackSelector, // IN
-                               VA64                 stack)         // IN
-{
-   NOT_IMPLEMENTED();
-}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,7 +31,8 @@
 #include "vmmblob.h"
 #include "monLoader.h"

-#define VMMBLOB_SIZE_MAX (12 * 1024 * 1024)
+#define VMMBLOB_SIZE_MAX (22 * 1024 * 1024) /* Ensure enough space for
+                                             * obj build with * GCOV_VMM=1. */

 /*
 *----------------------------------------------------------------------
@@ -65,7 +65,7 @@ APIC_GetMA(void)
    * Check if X2 APIC mode is enabled.
    */

-   if ((__GET_MSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
      return (MA)-1;
   }

@@ -75,7 +75,7 @@ APIC_GetMA(void)
    */

   // Mask out goo in the low 12 bits, which is unrelated to the address.
-   result = __GET_MSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);
+   result = X86MSR_GetMSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);

   /*
    * On Intel, the high bits are reserved so we mask.
@@ -122,7 +122,7 @@ APIC_Read(const APICDescriptor *desc, // IN
          int regNum)                 // IN
 {
   if (desc->isX2) {
-      return (uint32 )__GET_MSR(MSR_X2APIC_BASE + regNum);
+      return (uint32)X86MSR_GetMSR(MSR_X2APIC_BASE + regNum);
   } else {
      return desc->base[regNum][0];
   }
@@ -182,7 +182,7 @@ APIC_Write(const APICDescriptor *desc, // IN
           uint32 val)                 // IN
 {
   if (desc->isX2) {
-      __SET_MSR(MSR_X2APIC_BASE + regNum, val);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + regNum, val);
   } else {
      desc->base[regNum][0] = val;
   }
@@ -210,7 +210,7 @@ uint64
 APIC_ReadICR(const APICDescriptor *desc) // IN
 {
   if (desc->isX2) {
-      return __GET_MSR(MSR_X2APIC_BASE + APICR_ICRLO);
+      return X86MSR_GetMSR(MSR_X2APIC_BASE + APICR_ICRLO);
   } else {
      uint32 icrHi = desc->base[APICR_ICRHI][0];
      uint32 icrLo = desc->base[APICR_ICRLO][0];
@@ -244,7 +244,7 @@ APIC_WriteICR(const APICDescriptor *desc, // IN
 {
   if (desc->isX2) {
      uint64 icr = (uint64) id << 32 | icrLo;
-      __SET_MSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
   } else {
      ASSERT(!(id & ~(APIC_ICRHI_DEST_MASK >> APIC_ICRHI_DEST_OFFSET)));
      desc->base[APICR_ICRHI][0] = id << APIC_ICRHI_DEST_OFFSET;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -40,6 +40,7 @@ uint32      cpuidVersion;
 Bool        hostSupportsVT;
 Bool        hostSupportsSVM;
 Bool        hostHasSpecCtrl;
+Bool        hostSupportsXSave;


 /*
@@ -84,6 +85,7 @@ CPUID_Init(void)
   __GET_CPUID(1, &regs);
   cpuidVersion = regs.eax;
   cpuidFeatures = regs.edx;
+   hostSupportsXSave = CPUID_ISSET(1, ECX, XSAVE, regs.ecx);

   __GET_CPUID(0, &regs);
   ptr = (uint32 *)name;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,6 +37,7 @@ extern CpuidVendor cpuidVendor;
 extern Bool        hostSupportsVT;
 extern Bool        hostSupportsSVM;
 extern Bool        hostHasSpecCtrl;
+extern Bool        hostSupportsXSave;

 void CPUID_Init(void);
 Bool CPUID_AddressSizeSupported(void);
@@ -84,6 +85,12 @@ CPUID_HostSupportsSpecCtrl(void)
   return hostHasSpecCtrl;
 }

+static INLINE Bool
+CPUID_HostSupportsXSave(void)
+{
+   return hostSupportsXSave;
+}
+
 static INLINE Bool
 CPUID_SSE2Supported(void)
 {
@@ -0,0 +1,609 @@
+/*********************************************************
+ * Copyright (C) 2016-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * crosspage.c --
+ *
+ *    Cross page code and data.
+ *
+ *    The cross code page contains world switch code and interrupt/exception
+ *    handlers in support of world switch.  A separate data page is also mapped
+ *    in both the vmm and vmmon address spaces to describe the host and monitor
+ *    state.
+ *
+ *    Both the monitor and the vmmon driver call into the cross page to switch
+ *    worlds.  The world switch from host->vmm is able to refer directly to the
+ *    various symbols defined in CrossPage_CodePage.  On the vmm->host path
+ *    the VmmToHost entry point is stored in the data page.  Data references
+ *    must be handled as offsets from the base of the data page.
+ *
+ *    Use of inline assembly in C code is delicate as the compiler has many
+ *    possible reasons to emit unexpected instructions surrounding any assembly
+ *    in a C function.  Frame pointers are inconvenient to disable on a per-file
+ *    basis given structure of the Linux Makefile so they are being emitted
+ *    in the C wrapper function.  NORETURN and NOT_REACHED_MINIMAL are used to
+ *    omit a function epilogue.  The output/clobber lists of the assembly blocks
+ *    are not used as the compiler needs not compensate for the register use of
+ *    these hand-written assembly functions.
+ *    Each assembly function begins with a label denoting the proper entry point
+ *    for that function.  Only a handful of the functions need to be exported
+ *    out of this file, see crosspage.h.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#      include <linux/objtool.h>
+#   elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
+#      include <linux/frame.h>
+#   endif
+#endif
+
+#include "modulecall.h"
+#include "iocontrols.h"
+#include "segs.h"
+#include "x86_basic_defs.h"
+#include "vm_basic_defs.h"
+#include "vm_idt_x86.h"
+#include "crosspage.h"
+
+#ifdef __APPLE__
+/*
+ * The Mac compiler expects symbols to have a _ prefix.
+ * This arranges for any symbol being exported out of the assembly code
+ * to be accessible.
+ */
+#define ASM_PREFIX "_"
+/*
+ * OSX uses a segment,section style syntax for specifying output sections.
+ * https://developer.apple.com/library/archive/documentation/Performance/
+ * Conceptual/CodeFootprint/Articles/MachOOverview.html
+ */
+#define ASM_SECTION "__TEXT,cross"
+#else
+/*
+ * This is a COFF grouped section identifier.  This ensures the crosspage code
+ * is merged with the normal .text area but as a separate unit so it can have
+ * its own alignment properties.
+ * https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
+ *
+ * The Linux build will handle the section based on its contents (code).
+ */
+#define ASM_SECTION ".text$cross"
+#define ASM_PREFIX
+#endif
+
+#define EXPORTED_ASM_SYMBOL(fn) ".global " ASM_PREFIX #fn "\n"   \
+                                ASM_PREFIX #fn ":\n"
+
+/*
+ * Tag the crosspage code C wrapper with the crosspage section and page
+ * alignment.
+ */
+#define CPCODE __attribute__((section(ASM_SECTION)))   \
+               __attribute__((aligned(PAGE_SIZE)))
+
+/* No special handling is needed for the data. */
+#define CPDATA
+
+#define VMMDATALA(off) (LPN_2_LA(CROSS_PAGE_DATA_START) + (off))
+
+#define NOT_REACHED_MINIMAL __builtin_unreachable
+
+#ifdef ANNOTATE_INTRA_FUNCTION_CALL
+#define ANNOTATE_ASM_CALL_STR2(x) #x
+#define ANNOTATE_ASM_CALL_STR(x) ANNOTATE_ASM_CALL_STR2(x) "\n"
+#define ANNOTATE_ASM_CALL ANNOTATE_ASM_CALL_STR(ANNOTATE_INTRA_FUNCTION_CALL)
+#else
+#define ANNOTATE_ASM_CALL
+#endif
+
+void VmmToHost(void);
+
+CPDATA const VMCrossPageData cpDataTemplate = {
+   .version        = CROSSPAGE_VERSION,
+   .vmmonVersion   = VMMON_VERSION,
+
+   .monRSP         = VMM_LRET_STACK_TOP,
+   .monSS          = SYSTEM_DATA_SELECTOR,
+   .monDS          = SYSTEM_DATA_SELECTOR,
+   .monES          = SYSTEM_DATA_SELECTOR,
+
+   .vmmToHostLA    = (VA)VmmToHost,
+   .wsCR0          = CR0_PE | CR0_MP | CR0_EM | CR0_NE | CR0_WP | CR0_PG,
+   .wsCR4          = CR4_PAE | CR4_OSFXSR,
+
+   .monTask.rsp[0] = VMM_STACK_TOP,                   /* Monitor stack. */
+   .monTask.rsp[1] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL1 is not used. */
+   .monTask.rsp[2] = VPN_2_VA(VMM_STACK_GUARD_START), /* CPL2 is not used. */
+
+   .monTask.ist[IST_NONE]                 = 0, /* No stack switch. */
+   .monTask.ist[IST_VMM_DF]               = DF_STACK_TOP,
+   .monTask.ist[IST_VMM_NMI]              = NMI_STACK_TOP,
+   .monTask.ist[IST_VMM_MCE]              = MC_STACK_TOP,
+
+   .monTask.IOMapBase = sizeof(Task64),
+
+   .monGDTR.limit  = GDT_LIMIT,
+   .monGDTR.offset = GDT_START_VA,
+
+   .shadowDR[6].ureg64 = DR6_DEFAULT,
+   .shadowDR[7].ureg64 = DR7_ONES,
+
+   /*
+    * switchHostIDT and switchHostIDTR are initialized in vmmon.
+    * switchMonIDTR is static and can be fully initialized at compile-time.
+    * switchMonIDT is mostly static with all non-present entries except for
+    * the 4 handled exceptions which are populated by the crosspage setup
+    * code in task.c
+    */
+   .switchMonIDTR  = { sizeof(Gate64) * NUM_EXCEPTIONS - 1,
+                       VMMDATALA(offsetof(VMCrossPageData, switchMonIDT)) },
+};
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * CrossPage_CodePage --
+ *
+ *      This function only serves as an anchor and wrapper for the code that
+ *      runs in the world switch's cross page.  The function must not be called
+ *      directly.  Its address may be used to locate the cross page code.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+CPCODE void
+CrossPage_CodePage(void)
+{
+   __asm__ __volatile__ (
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchDBHandler --
+    *
+    *      DB handler that operates during worldswitch (in both directions).
+    *
+    *      This handler is expected to be used in two cases:
+    *      1) If the host kernel has set the DR7_GD (guard) bit, when switch
+    *         code attempts to save debug registers, a #DB will be thrown.  In
+    *         this case, the handler sets wsException[EXC_DB] and returns.
+    *      2) When stress-testing the worldswitch code, RFLAGS.TF is set across
+    *         most switch paths.  This causes a #DB to be thrown before each
+    *         instruction is executed.  In this case, the handler must set
+    *         RFLAGS.RF in the return frame to guarantee progress while
+    *         RFLAGS.TF is set.  For added stress, the handler induces a
+    *         simulated #NMI.
+    *
+    *      In case (1), returning is accomplished via a simulated iret in order
+    *      to preserve any NMI-blocking.  In case (2), the inherent
+    *      incompatibility of RFLAGS.RF for single-stepping and simulating an
+    *      iret via popf and lretq necessitates the use of an actual iret.  As
+    *      a result, there is no strong NMI-blocking guarantee when
+    *      stress-testing the switch path.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_DB] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchDBHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionDB], %%rax\n"
+   "movb         $1,               (%%rax)\n" /* log EXC_DB */
+   "popq         %%rax\n"
+   "testl        %[EFLAGS_TF],     16(%%rsp)\n" /* check trap flag */
+   "jz           return_without_enabling_nmi\n"
+   "orl          %[EFLAGS_RF],     16(%%rsp)\n"
+   "int          $2\n"
+   "iretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchUDHandler --
+    *
+    *      UD handler that operates during worldswitch (in both directions).
+    *
+    *      It sets the wsException[EXC_UD] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_UD] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchUDHandler)
+   "pushq        %%rax\n"
+   "pushq        %%rbx\n"
+   "pushq        %%rcx\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "movl         %[wsExceptionUD],      %%ecx\n"    /* log EXC_UD */
+   "movb         $1,                    (%%rax, %%rcx)\n"
+   /* Check if the exception came from a monitor RIP. */
+   "cmpq         %[MONITOR_MINIMUM_VA], 24(%%rsp)\n"
+   "jae          monitor_context\n"
+   /* Check if the exception came from the crosspage code. */
+   "movq         24(%%rsp),              %%rbx\n"
+   "andq         %[PageAlignMask],       %%rbx\n"
+   "movl         %[crosspageCodeLA],     %%ecx\n"
+   "cmpq         (%%rax, %%rcx),         %%rbx\n"
+   "je           monitor_context\n"
+   /*
+    * Linux halts the processor on host #UD2.  Act similarly.  PR 1281662.
+    * Note that this can escape due to an #NMI, #MC, or #DB.
+    */
+   "cli\n"
+   "hlt\n"
+   /* Advance 2 bytes to skip past the #UD2 instruction. */
+   "monitor_context:\n"
+   "movq         24(%%rsp), %%rbx\n"
+   "movl         %[wsUD2],  %%ecx\n"
+   "movq         %%rbx,     (%%rax, %%rcx)\n"
+   "addq         $2,        24(%%rsp)\n"
+   "popq         %%rcx\n"
+   "popq         %%rbx\n"
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchNMIHandler --
+    *
+    *      NMI handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It 'simply' sets the wsException[EXC_NMI] flag then returns,
+    *      leaving further NMI delivery inhibited.
+    *
+    *      As long as we don't execute 'iret', the CPU will not allow further
+    *      NMIs to be delivered; this is how the CPU protects itself from
+    *      runaway NMIs eating up the stack and triple-faulting.
+    *
+    *      See Vol 3, 5.5.1, Handling multiple NMIs.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_NMI] = TRUE
+    *      further NMI delivery inhibited
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchNMIHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionNMI], %%rax\n"
+   "movb         $1,                (%%rax)\n" /* log EXC_NMI */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchMCEHandler --
+    *
+    *      MCE handler that operates during worldswitch (in both
+    *      directions).
+    *
+    *      It simply sets the wsException[EXC_MC] flag then returns.
+    *
+    * Input:
+    *
+    *       0(%rsp) = previous RIP
+    *       8(%rsp) = previous CS
+    *      16(%rsp) = previous RFLAGS
+    *      24(%rsp) = previous RSP
+    *      32(%rsp) = previous SS
+    *
+    * Output:
+    *
+    *      wsException[EXC_MC] = TRUE
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(SwitchMCEHandler)
+   "pushq        %%rax\n"
+   ANNOTATE_ASM_CALL
+   "call         SwitchExcGetCrossPageData\n"
+   "addq         %[wsExceptionMC], %%rax\n"
+   "movb         $1,              (%%rax)\n" /* log EXC_MC */
+   "popq         %%rax\n"
+   "jmp          return_without_enabling_nmi\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchHandlerReturns --
+    *
+    *      Shared code to return without enabling NMIs.  Switch handlers call
+    *      directly to these labels as needed.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "return_without_enabling_nmi:\n"
+   "pushq        %%rbp\n"
+   "pushq        %%rax\n"
+   "movq         40(%%rsp), %%rbp\n"
+   "subq         $32, %%rbp\n"
+   "movq         24(%%rsp), %%rax\n"
+   "movq         %%rax, 24(%%rbp)\n"
+   "movq         16(%%rsp), %%rax\n"
+   "movq         %%rax, 16(%%rbp)\n"
+   "movq         32(%%rsp), %%rax\n"
+   "movq         %%rax, 8(%%rbp)\n"
+   "movq         8(%%rsp),  %%rax\n"
+   "movq         %%rax, (%%rbp)\n"
+   "popq         %%rax\n"
+   "movq         %%rbp, %%rsp\n"
+   "popq         %%rbp\n"
+   "popfq\n"
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * HostToVmm --
+    *
+    *      Switch from host to monitor.
+    *
+    * Entered with:
+    *      RCX = crosspage host address
+    *      (RSP) = return to host address
+    *
+    * Must preserve:
+    *      RBX, RSI, RDI, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(HostToVmm)
+   /* Create an lret frame on the host stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold host %rsp */
+
+   /* Start from the "empty" host context and save. */
+   "leaq            %c[hostContextEmpty](%%rcx), %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* host %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rdi\n"
+   "pushq           %%rsi\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" monitor context and load. */
+   "leaq            %c[monContextFull](%%rcx), %%rsp\n"
+   "popq            %%rsi\n" /* monitor %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+   "popq            %%rax\n" /* monitor %rsp */
+   "popw            %%dx\n"  /* monitor %ss */
+   /* Load the cross GDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%rsi, %%cr3\n"
+   /* Reload the monitor's %ss into %ds/%es. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   "lretq\n"
+
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * VmmToHost --
+    *
+    *      Switch from monitor to host.
+    *
+    * Must preserve:
+    *      RBX, RBP, RSP, R12..R15
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   EXPORTED_ASM_SYMBOL(VmmToHost)
+   "movq            %c[VMMCROSSPAGE] + %c[crosspageDataLA], %%rcx\n"
+   /* Create an lret frame on the monitor stack. */
+   "pushq           (%%rsp)\n"
+   "mov             %%cs,  8(%%rsp)\n"
+   "movq            %%rsp, %%rax\n" /* Temporarily hold monitor %rsp */
+
+   /* Start from the "empty" monitor context and save. */
+   "leaq            %c[VMMCROSSPAGE] + %c[monContextEmpty], %%rsp\n"
+   /* Note that only %ss is stored, not %ds/%es. */
+   "movw            %%ss,  %%dx\n"
+   "pushw           %%dx\n"
+   "pushq           %%rax\n" /* monitor %rsp */
+   "pushq           %%r15\n"
+   "pushq           %%r14\n"
+   "pushq           %%r13\n"
+   "pushq           %%r12\n"
+   "pushq           %%rbp\n"
+   "pushq           %%rbx\n"
+   "movq            %%cr3, %%rax\n"
+   "pushq           %%rax\n"
+
+   /* Start from the "full" host context and load. */
+   "leaq            %c[hostContextFull](%%rcx), %%rsp\n"
+   "popq            %%r9\n" /* host %cr3 */
+   "popq            %%rbx\n"
+   "popq            %%rbp\n"
+   "popq            %%rsi\n" /* Not ABI-required. */
+   "popq            %%rdi\n" /* Not ABI-required. */
+   "popq            %%r12\n"
+   "popq            %%r13\n"
+   "popq            %%r14\n"
+   "popq            %%r15\n"
+
+   "popq            %%rax\n" /* host %rsp */
+   "popw            %%dx\n"  /* host %ss */
+   /* Load the cross GDT and IDT before reloading segments, %cr3 */
+   "lgdtq           %c[crossGDTHKLADesc](%%rcx)\n"
+   "lidtq           %c[switchHostIDTR](%%rcx)\n"
+   /* Load %cr3 last to flush the TLB after all stack ops. */
+   "movq            %%r9,  %%cr3\n"
+   /* Reload the host's %ss into %ds/%es.  Technically wrong. */
+   "movw            %%dx,  %%ds\n"
+   "movw            %%dx,  %%es\n"
+   "movw            %%dx,  %%ss\n"
+   "movq            %%rax, %%rsp\n"
+
+   /* Microsoft RTL/codegen assumes EFLAGS<DF> = 0. */
+   "cld\n"
+   "lretq\n"
+
+   /*
+    *---------------------------------------------------------------------------
+    *
+    * SwitchExcGetCrossPageData --
+    *
+    *      Common function for the exception handlers to locate the data
+    *      crosspage so they can record their respective events.  In order to
+    *      reach this code an exception had to vector through the IDT.  The IDT
+    *      is known to be in the data page.  Therefore, the data page can be
+    *      found by accessing IDTR and rounding down to page alignment.
+    *
+    * Input:
+    *     None
+    *
+    * Output:
+    *     %rax = Page aligned address of the current crosspage data area.
+    *
+    * Note:
+    *     %rax (return value) and %rflags are destroyed, all other registers
+    *     are preserved.  Since this is only called by exception handlers, the
+    *     CPU has already saved %rflags so no additional handling is required.
+    *
+    *---------------------------------------------------------------------------
+    */
+
+   ".p2align 4\n"
+   "SwitchExcGetCrossPageData:\n"
+   "subq            $0x10,            %%rsp\n"
+   "sidt            0(%%rsp)\n"
+   "movq            2(%%rsp),         %%rax\n" /* DTR.offset */
+   "addq            $0x10,            %%rsp\n"
+   "andq            %[PageAlignMask], %%rax\n"
+   "ret\n"
+
+   EXPORTED_ASM_SYMBOL(CrossPage_CodeEnd)
+
+   : /* No output list, this is not really C code. */
+   : [MONITOR_MINIMUM_VA] "i" (MONITOR_LINEAR_START),
+     [PageAlignMask]      "i" (~PAGE_MASK),
+     [EFLAGS_TF]          "i" (EFLAGS_TF),
+     [EFLAGS_RF]          "i" (EFLAGS_RF),
+     [VMMCROSSPAGE]       "i" (CROSS_PAGE_DATA_START * PAGE_SIZE),
+     [wsUD2]              "i" (offsetof(VMCrossPageData, wsUD2)),
+
+     [wsExceptionDB]      "i" (offsetof(VMCrossPageData, wsException[EXC_DB])),
+     [wsExceptionUD]      "i" (offsetof(VMCrossPageData, wsException[EXC_UD])),
+     [wsExceptionNMI]     "i" (offsetof(VMCrossPageData, wsException[EXC_NMI])),
+     [wsExceptionMC]      "i" (offsetof(VMCrossPageData, wsException[EXC_MC])),
+
+     [hostContextEmpty]   "i" (offsetof(VMCrossPageData, hostDS)),
+     [monContextEmpty]    "i" (offsetof(VMCrossPageData, monDS)),
+
+     [hostContextFull]    "i" (offsetof(VMCrossPageData, hostCR3)),
+     [monContextFull]     "i" (offsetof(VMCrossPageData, monCR3)),
+
+     [crosspageDataLA]    "i" (offsetof(VMCrossPageData, crosspageDataLA)),
+     [crosspageCodeLA]    "i" (offsetof(VMCrossPageData, crosspageCodeLA)),
+     [crossGDTHKLADesc]   "i" (offsetof(VMCrossPageData, crossGDTHKLADesc)),
+     [switchHostIDTR]     "i" (offsetof(VMCrossPageData, switchHostIDTR))
+   );
+   NOT_REACHED_MINIMAL();
+}
+
+#ifdef STACK_FRAME_NON_STANDARD
+STACK_FRAME_NON_STANDARD(CrossPage_CodePage);
+#endif
+
+/*
+ * The crosspage data must not exceed a single page.
+ * The code area also must fit in a single page but that size isn't available
+ * until link time.
+ */
+MY_ASSERTS(CROSSPAGE_DATA,
+   ASSERT_ON_COMPILE(sizeof(VMCrossPageData) <= PAGE_SIZE);
+)
@@ -0,0 +1,43 @@
+/*********************************************************
+ * Copyright (C) 2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _CROSSPAGE_H_
+#define _CROSSPAGE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct VMCrossPageData;
+
+extern void HostToVmm(struct VMCrossPageData *crosspageWIN, void *unused1,
+                      void *unused2, struct VMCrossPageData *crosspageSYSV);
+extern const VMCrossPageData cpDataTemplate;
+
+extern void SwitchDBHandler(void);
+extern void SwitchUDHandler(void);
+extern void SwitchNMIHandler(void);
+extern void SwitchMCEHandler(void);
+extern void CrossPage_CodePage(void);
+extern void CrossPage_CodeEnd(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,7 +27,7 @@
 #   define __HOST_KERNEL_H__


-#   ifdef linux
+#   ifdef __linux__
 /*
 * In some cases, this files needs to include Linux kernel header file
 * asm/page.h.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,9 +59,18 @@

 #define CROSSCALL_SLEEP_US 1000

-EXTERN int   HostIF_Init(VMDriver *vm, uint32 numVCPUs);
+typedef struct HostIFContigMemMap {
+   MPN mpn;
+   void *addr;
+   PageCnt pages;
+   struct HostIFContigMemMap *next;
+} HostIFContigMemMap;
+
+EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
-EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);
+
+EXTERN MPN   HostIF_GetCrossPageDataMPN(VMCrossPageData *crosspageData);
+EXTERN MPN   HostIF_GetCrossPageCodeMPN(void);

 EXTERN void *HostIF_AllocPage(void);
 EXTERN void  HostIF_FreePage(void *ptr);
@@ -76,28 +85,32 @@ EXTERN Bool  HostIF_IsLockedByMPN(VMDriver *vm, MPN mpn);
 EXTERN void  HostIF_FreeAllResources(VMDriver *vm);
 EXTERN uint64 HostIF_ReadUptime(void);
 EXTERN uint64 HostIF_UptimeFrequency(void);
-EXTERN unsigned int HostIF_EstimateLockedPageLimit(const VMDriver *vm,
-                                                   unsigned int lockedPages);
+EXTERN PageCnt HostIF_EstimateLockedPageLimit(const VMDriver *vm,
+                                              PageCnt lockedPages);
 EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
-EXTERN void *HostIF_AllocKernelPages(unsigned numPages, MPN *mpns);
-EXTERN void  HostIF_FreeKernelPages(unsigned numPages, void *ptr);
+EXTERN void *HostIF_AllocKernelPages(PageCnt numPages, MPN *mpns);
+EXTERN void  HostIF_FreeKernelPages(PageCnt numPages, void *ptr);
+EXTERN HostIFContigMemMap *HostIF_AllocContigPages(VMDriver *vm,
+                                                   PageCnt numPages);
+EXTERN void  HostIF_FreeContigPages(VMDriver *vm, HostIFContigMemMap *mapping);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
 EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 #endif

-EXTERN Bool  HostIF_APICInit(VMDriver *vm, Bool setVMPtr, Bool probe);
+EXTERN void  HostIF_APICInit(VMDriver *vm);
 EXTERN uint8 HostIF_GetMonitorIPIVector(void);
 EXTERN uint8 HostIF_GetHVIPIVector(void);
+EXTERN uint8 HostIF_GetPerfCtrVector(void);
 EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);

 EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
                                  Vcpuid vcpuid,
                                  uint64 *args);

-EXTERN int   HostIF_SemaphoreSignal(uint64 *args);
+EXTERN int   HostIF_SemaphoreSignal(VMDriver *vm, uint64 *args);

 EXTERN void  HostIF_SemaphoreForceWakeup(VMDriver *vm, const VCPUSet *vcs);
 EXTERN void  HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
@@ -111,11 +124,11 @@ EXTERN void HostIF_WaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_CancelWaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_WakeUpYielders(VMDriver *vm, Vcpuid currVcpu);

-EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                   unsigned int numPages, Bool kernelMPNBuffer);
-EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned int numPages, Bool kernelMPNBuffer);
+EXTERN int64 HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                     PageCnt numPages, Bool kernelMPNBuffer);
+EXTERN int HostIF_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn);
+EXTERN PageCnt HostIF_GetNumAnonPages(VMDriver *vm);
 EXTERN MPN HostIF_AllocLowPage(VMDriver *vm);

 EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr,
@@ -29,9 +29,10 @@
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
+#include "vm_basic_types.h"


-void *HostIF_AllocKernelMem(size_t size, int nonPageable);
+void *HostIF_AllocKernelMem(size_t size, Bool nonPageable);
 void  HostIF_FreeKernelMem(void *ptr);

 #endif // ifdef _HOSTIFMEM_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -96,7 +96,7 @@
 * Modify this value to increase the maximum number of tracked pages
 * per MemTrack instance.
 */
-#define MEMTRACK_MAX_TRACKED        MBYTES_2_PAGES(4096)
+#define MEMTRACK_MAX_TRACKED        GBYTES_2_PAGES(119)

 /*
 * Linux uses a 3-level directory, because we want to keep allocations
@@ -148,7 +148,7 @@ typedef uint64 MemTrackHTKey;

 typedef struct MemTrack {
   VMDriver         *vm;            /* The VM instance. */
-   unsigned          numPages;      /* Number of pages tracked. */
+   PageCnt           numPages;      /* Number of pages tracked. */
   MemTrackDir1      dir1;          /* First level directory. */
   MemTrackHT        vpnHashTable;  /* VPN to entry hashtable. */
   MemTrackHT       *mpnHashTable;  /* MPN to entry hashtable. */
@@ -197,7 +197,7 @@ MemTrackAllocPage(void)

 #define MEMTRACK_ALLOCDFN(_name, _itype, _otype)   \
   static INLINE _otype *                          \
-   _name(_itype *arg, unsigned pos)                \
+   _name(_itype *arg, uint64 pos)                  \
   {                                               \
      if (arg->dir[pos] == NULL) {                 \
         arg->dir[pos] = MemTrackAllocPage();      \
@@ -273,8 +273,8 @@ MemTrackHTInsert(MemTrackHT *ht,          // IN
 static void
 MemTrackCleanup(MemTrack *mt)    // IN
 {
-   unsigned idx;
-   unsigned p1;
+   PageCnt idx;
+   uint64 p1;
   MemTrackDir1 *dir1;
   Bool freeBackMap = mt != NULL && mt->mpnHashTable != NULL;

@@ -284,7 +284,7 @@ MemTrackCleanup(MemTrack *mt)    // IN
   dir1 = &mt->dir1;

   for (p1 = 0; p1 < MEMTRACK_DIR1_ENTRIES; p1++) {
-      unsigned p2;
+      uint64 p2;
      MemTrackDir2 *dir2 = MEMTRACK_GETDIR2(dir1, p1);

      if (dir2 == NULL) {
@@ -339,7 +339,7 @@ MemTrack *
 MemTrack_Init(VMDriver *vm) // IN:
 {
   MemTrack *mt;
-   unsigned idx;
+   PageCnt idx;

 #if defined(MEMTRACK_3LEVEL)
   ASSERT_ON_COMPILE(sizeof *mt <= PAGE_SIZE);
@@ -359,7 +359,8 @@ MemTrack_Init(VMDriver *vm) // IN:
      MemTrackHTPage *htPage = MemTrackAllocPage();

      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate VPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate VPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->vpnHashTable.pages[idx] = htPage;
@@ -374,7 +375,8 @@ MemTrack_Init(VMDriver *vm) // IN:
   for (idx = 0; idx < MEMTRACK_HT_PAGES; idx++) {
      MemTrackHTPage *htPage = MemTrackAllocPage();
      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate MPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate MPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->mpnHashTable->pages[idx] = htPage;
@@ -409,8 +411,8 @@ MemTrack_Add(MemTrack *mt,    // IN
             VPN64 vpn,       // IN
             MPN mpn)         // IN
 {
-   unsigned idx = mt->numPages;
-   unsigned p1, p2, p3;
+   PageCnt idx = mt->numPages;
+   uint64 p1, p2, p3;
   MemTrackEntry *ent;
   MemTrackDir1 *dir1 = &mt->dir1;
   MemTrackDir2 *dir2;
@@ -535,16 +537,16 @@ MemTrack_LookupMPN(MemTrack *mt, // IN
 *----------------------------------------------------------------------
 */

-unsigned
+PageCnt
 MemTrack_Cleanup(MemTrack *mt,            // IN
                 MemTrackCleanupCb *cb,   // IN
                 void *cData)             // IN
 {
-   unsigned idx;
-   unsigned count = 0;
+   PageCnt idx;
+   PageCnt count = 0;

   for (idx = 0; idx < mt->numPages; idx++) {
-      unsigned p1, p2, p3;
+      uint64 p1, p2, p3;
      MemTrackEntry *ent;
      MemTrackDir1 *dir1 = &mt->dir1;
      MEMTRACK_IDX2DIR(idx, p1, p2, p3);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -43,8 +43,8 @@ typedef struct MemTrackEntry {
 typedef void (MemTrackCleanupCb)(void *cData, MemTrackEntry *entry);

 extern struct MemTrack *MemTrack_Init(VMDriver *vm);
-extern unsigned MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
-                                 void *cbData);
+extern PageCnt MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
+                                void *cbData);
 extern MemTrackEntry *MemTrack_Add(struct MemTrack *mt, VPN64 vpn, MPN mpn);
 extern MemTrackEntry *MemTrack_LookupVPN(struct MemTrack *mt, VPN64 vpn);
 extern MemTrackEntry *MemTrack_LookupMPN(struct MemTrack *mt, MPN mpn);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,16 +24,15 @@
 *    Depending on configuration phystracker provides either 2-level or
 *    3-level structure to track whether page (specified by its MPN) is
 *    locked or no.  Linux uses 3-level structures with top limit of
- *    1TB (32bit) or 16TB (64bit).  Windows use 2-level structures
- *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.  On Mac
- *    limit is 2-level 128GB.
+ *    1TB (32bit) or 16TB (64bit).  Windows and Mac use 2-level structures
+ *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.
 *
 *    2-level phystracker is built on top of 3-level one by collapsing
 *    middle level.
 */


-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -50,9 +49,7 @@
 #define BYTES_PER_ENTRY      (PAGE_SIZE)
 #define PHYSTRACK_L3_ENTRIES (8 * BYTES_PER_ENTRY) /* 128MB */

-#if defined(WINNT_DDK)
-#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
-#elif defined(__linux__)
+#if defined(__linux__)
 #define PHYSTRACK_L2_ENTRIES (BYTES_PER_ENTRY / sizeof(void *)) /* 64GB or 128GB */
 /*
 * Currently MPN is 32 bits.  15 bits are in L3, 9 bits are in L2,
@@ -60,7 +57,7 @@
 */
 #define PHYSTRACK_L1_ENTRIES (256) /* 16TB. */
 #else
-#define PHYSTRACK_L1_ENTRIES ((128 + 4) * 8) /* 128 GB */
+#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
 #endif

 #ifndef PHYSTRACK_L2_ENTRIES
@@ -493,3 +490,32 @@ PhysTrack_GetNext(const PhysTracker *tracker, // IN
   return INVALID_MPN;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PhysTrack_GetNumTrackedPages --
+ *
+ *      Returns the total number of tracked pages
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+PageCnt
+PhysTrack_GetNumTrackedPages(const PhysTracker *tracker)
+{
+   PageCnt numTrackedMPNs = 0;
+   MPN nextMPN;
+   ASSERT(tracker);
+   ASSERT(HostIF_VMLockIsHeld(tracker->vm));
+   nextMPN = PhysTrack_GetNext(tracker, INVALID_MPN);
+   while (nextMPN != INVALID_MPN) {
+      numTrackedMPNs++;
+      nextMPN = PhysTrack_GetNext(tracker, nextMPN);
+   }
+   return numTrackedMPNs;
+}
+
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined(WINNT_DDK)
+#if !defined (__linux__)
 /* XXX: can be more efficient based on server vs. desktop and version of Windows */
 #define PHYSTRACK_MAX_SUPPORTED_GB (2048 + 4) /* 2 TB 64-bit W2k8 + 4 GB PCI */
 #endif
@@ -44,6 +44,7 @@ EXTERN void PhysTrack_Add(struct PhysTracker *, MPN);
 EXTERN void PhysTrack_Remove(struct PhysTracker *, MPN);
 EXTERN Bool PhysTrack_Test(const struct PhysTracker *, MPN);
 EXTERN MPN PhysTrack_GetNext(const struct PhysTracker *, MPN);
+EXTERN PageCnt PhysTrack_GetNumTrackedPages(const struct PhysTracker *);

 #endif

@@ -0,0 +1,299 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.c --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "sharedAreaVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmonIsMultiVCPU --
+ *
+ *      Helper function for determining if a shared area type represents
+ *      a region that has VCPU-specific memory.
+ *
+ * Results:
+ *      TRUE if type has VCPU-specific memory, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+SharedAreaVmmonIsMultiVCPU(SharedAreaType type)
+{
+   switch (type) {
+   case SHARED_AREA_PER_VM:
+   case SHARED_AREA_PER_VM_VMX:
+      return FALSE;
+   default:
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Init --
+ *
+ *      Initializes the shared area component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon shared area.
+ *
+ *----------------------------------------------------------------------
+ */
+
+SharedAreaVmmon *
+SharedAreaVmmon_Init(VMDriver *vm)
+{
+   SharedAreaVmmon *sa;
+
+   /*
+    * Allocate the shared area pointer here. The pages in each region are
+    * lazily allocated when a region is registered.
+    */
+   sa = HostIF_AllocKernelMem(sizeof *sa, FALSE);
+   if (sa == NULL) {
+      Warning("SharedArea failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(sa, 0, sizeof *sa);
+   return sa;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Cleanup --
+ *
+ *      Cleans up the shared area component of the VM Driver by freeing all
+ *      previously allocated VMMon shared area memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+SharedAreaVmmon_Cleanup(SharedAreaVmmon *area)
+{
+   SharedAreaType t;
+
+   if (area == NULL) {
+      return;
+   }
+
+   for (t = 0; t < NUM_SHARED_AREAS; t++) {
+      SharedAreaVmmonRegion *region = &area->regions[t];
+
+      if (region->pages != NULL) {
+         HostIF_FreeKernelMem(region->pages);
+      }
+   }
+   HostIF_FreeKernelMem(area);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_ValidateRegionArgs --
+ *
+ *      Validate the given VMMon shared area region registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register a shared
+ *      area region for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_ValidateRegionArgs(VMDriver *vm,
+                                   VMSharedAreaRegistrationBlock *block)
+{
+   SharedAreaVmmonRegion *region;
+   Vcpuid vcpu         = block->vcpu;
+   SharedAreaType type = block->region.index;
+   PageCnt numPages    = block->region.numPages;
+
+   if (vm == NULL               ||
+       vm->sharedArea == NULL   ||
+       vcpu >= vm->numVCPUs     ||
+       numPages == 0            ||
+       type >= NUM_SHARED_AREAS ||
+       (!SharedAreaVmmonIsMultiVCPU(type) && vcpu > 0)) {
+      return FALSE;
+   }
+
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu != 0) {
+
+      /*
+       * A region that was previously registered for a given VCPU should have
+       * the same number of pages as originally specified. Also, a region can
+       * only be reserved once for a given VCPU throughout the runtime of a VM.
+       */
+      return region->pagesPerVcpu == numPages &&
+             region->pages[vcpu * region->pagesPerVcpu] == INVALID_MPN;
+   } else {
+      /* First registration for this region. */
+      ASSERT(region->pages == NULL);
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_RegisterRegion --
+ *
+ *      Register a VMMon shared area region for a given VCPU with the VM Driver.
+ *      A successful return value indicates that the backing pages of the
+ *      shared area region are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the region was successfully registered for the given VCPU,
+ *      FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_RegisterRegion(VMDriver *vm,
+                               VMSharedAreaRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   SharedAreaType type = block->region.index;
+   SharedAreaVmmonRegion *region;
+   MPN *pages;
+   PageCnt page;
+
+   ASSERT(SharedAreaVmmon_ValidateRegionArgs(vm, block));
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu == 0) {
+      PageCnt pagesInRegion = block->region.numPages;
+
+      if (SharedAreaVmmonIsMultiVCPU(type)) {
+         pagesInRegion *= vm->numVCPUs;
+      }
+      region->pages = HostIF_AllocKernelMem(sizeof *pages * pagesInRegion,
+                                            FALSE);
+      if (region->pages == NULL) {
+         Warning("Failed to allocate pages array for region %u\n", type);
+         return FALSE;
+      }
+      for (page = 0; page < pagesInRegion; page++) {
+         region->pages[page] = INVALID_MPN;
+      }
+      region->pagesPerVcpu = block->region.numPages;
+   }
+
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+
+   /*
+    * Lock all shared area backing pages throughout the runtime of the monitor.
+    * The pages remain locked until they are freed by HostIF during VM Driver
+    * tear-down.
+    */
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      VPN vmxVPN = block->region.baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, pages + page);
+
+      if (status != PAGE_LOCK_SUCCESS) {
+         PageCnt resetPage;
+
+         /* Reset the region pages for this VCPU to their original state. */
+         for (resetPage = 0; resetPage < page; resetPage++) {
+            status = Vmx86_UnlockPage(vm, uAddr);
+            ASSERT(status == PAGE_UNLOCK_SUCCESS);
+            pages[resetPage] = INVALID_MPN;
+         }
+         return FALSE;
+      }
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_GetRegionMPN --
+ *
+ *      For the shared area region corresponding to the given type and VCPU,
+ *      get the MPN backing the region at the given offset.
+ *
+ * Results:
+ *      The MPN backing the region, or INVALID_MPN if no MPN is found.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+SharedAreaVmmon_GetRegionMPN(VMDriver *vm, SharedAreaVmmonRequest *request)
+{
+   SharedAreaType type = request->type;
+   Vcpuid vcpu = request->vcpu;
+   SharedAreaVmmonRegion *region;
+   PageCnt pgOffset = request->offset;
+   MPN *pages;
+
+   ASSERT(vcpu < vm->numVCPUs);
+   ASSERT(type < NUM_SHARED_AREAS);
+   ASSERT(IMPLIES(!SharedAreaVmmonIsMultiVCPU(type), vcpu == 0));
+   region = &vm->sharedArea->regions[type];
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   if (region->pages == NULL || region->pagesPerVcpu == 0) {
+      Warning("(%s) Requested unregistered region %u, VCPU %u\n", __FUNCTION__,
+              type, vcpu);
+      return INVALID_MPN;
+   }
+   if (pgOffset >= region->pagesPerVcpu) {
+      Warning("(%s) Offset %"FMT64"u (type %u, per-VCPU size %"FMT64"u)\n",
+              __FUNCTION__, pgOffset, type, region->pagesPerVcpu);
+      return INVALID_MPN;
+   }
+
+   return pages[pgOffset];
+}
@@ -0,0 +1,282 @@
+/*********************************************************
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * statVarsVmmon.c --
+ *
+ *     VMMon stat variables management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "statVarsVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Init --
+ *
+ *      Initializes the stat vars component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon stat vars struct.
+ *
+ *----------------------------------------------------------------------
+ */
+
+StatVarsVmmon *
+StatVarsVmmon_Init(VMDriver *vm)
+{
+   StatVarsVmmon *statVars;
+
+   statVars = HostIF_AllocKernelMem(sizeof *statVars, FALSE);
+   if (statVars == NULL) {
+      Warning("StatVars failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(statVars, 0, sizeof *statVars);
+   return statVars;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Cleanup --
+ *
+ *      Cleans up the stat vars component of the VM Driver by freeing all
+ *      previously allocated VMMon stat vars memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+StatVarsVmmon_Cleanup(StatVarsVmmon *statVars)
+{
+   if (statVars == NULL) {
+      return;
+   }
+
+   if (statVars->pages != NULL) {
+      HostIF_FreeKernelMem(statVars->pages);
+   }
+   HostIF_FreeKernelMem(statVars);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmonValidateRegistration --
+ *
+ *      Validate the given VMMon stat vars registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register stat vars
+ *      backing pages for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+StatVarsVmmonValidateRegistration(VMDriver *vm,
+                                   VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu      = block->vcpu;
+   PageCnt numPages = block->numPages;
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+
+   if (vm == NULL           ||
+       vm->statVars == NULL ||
+       numPages == 0        ||
+       vcpu >= vm->numVCPUs) {
+      return FALSE;
+   }
+
+   pagesPerVcpu = vm->statVars->pagesPerVcpu;
+   pages = vm->statVars->pages;
+   if (pagesPerVcpu == 0) {
+      ASSERT(pages == NULL);
+      return TRUE;
+   } else {
+      return pagesPerVcpu == numPages &&
+             pages[vcpu * pagesPerVcpu] == INVALID_MPN;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_RegisterVCPU --
+ *
+ *      Register stat vars backing pages for the given VCPU with VMMon.
+ *      A successful return value indicates that stat vars backing pages
+ *      for the given VCPU are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the backing pages were successfully registered for the
+ *      given VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+StatVarsVmmon_RegisterVCPU(VMDriver *vm,
+                           VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   PageCnt pagesPerVcpu = block->numPages;
+   StatVarsVmmon *statVars = vm->statVars;
+   MPN *pages;
+   PageCnt page;
+
+   HostIF_VMLock(vm, 45);
+   if (!StatVarsVmmonValidateRegistration(vm, block)) {
+      HostIF_VMUnlock(vm, 45);
+      return FALSE;
+   }
+   if (statVars->pagesPerVcpu == 0) {
+      PageCnt totalPages = pagesPerVcpu * vm->numVCPUs;
+      statVars->pages = HostIF_AllocKernelMem(totalPages * sizeof *pages,
+                                              FALSE);
+      if (statVars->pages == NULL) {
+         HostIF_VMUnlock(vm, 45);
+         return FALSE;
+      }
+      for (page = 0; page < totalPages; page++) {
+         statVars->pages[page] = INVALID_MPN;
+      }
+      statVars->pagesPerVcpu = pagesPerVcpu;
+   }
+
+   /*
+    * Validate that all pages in this VCPU's region are still invalid. Before
+    * unlocking, temporarily set the first page to 0 to prevent a malicious user
+    * from triggering an ASSERT by firing parallel ioctls for the same VCPU.
+    */
+   pages = statVars->pages + vcpu * pagesPerVcpu;
+   for (page = 0; page < pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+   ASSERT_ON_COMPILE(INVALID_MPN != 0);
+   pages[0] = 0;
+   HostIF_VMUnlock(vm, 45);
+
+   /*
+    * Verify that all stat vars backing pages are locked.  The pages are
+    * assumed to remain locked either until they are unlocked by the VMX
+    * following a NUMA migration, or by HostIF during VM Driver tear-down.
+    */
+   for (page = 0; page < pagesPerVcpu; page++) {
+      VPN vmxVPN = block->baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      PageCnt resetPage;
+      MPN *currPage = pages + page;
+      Bool verifiedLocked = FALSE;
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, currPage);
+
+      /*
+       * Backing pages are expected to be locked by the VMX before being
+       * registered.
+       */
+      if (status == PAGE_LOCK_ALREADY_LOCKED) {
+         verifiedLocked = TRUE;
+         status = Vmx86_LookupUserMPN(vm, uAddr, currPage);
+         /*
+          * PR 2260615: Some platforms return PAGE_LOCK_SUCCESS for a successful
+          * user MPN lookup, others return PAGE_LOOKUP_SUCCESS, but they both
+          * happen to equal the same value.
+          */
+         ASSERT_ON_COMPILE(PAGE_LOOKUP_SUCCESS == PAGE_LOCK_SUCCESS);
+         if (status == PAGE_LOOKUP_SUCCESS) {
+            /*
+             * The backing page was verified as previously locked, and was
+             * successfully recorded.
+             */
+            continue;
+         }
+      }
+      /*
+       * The backing page was either not previously locked, or was not recorded.
+       * In either case, reset all previously recorded pages to INVALID_MPN.
+       */
+      if (!verifiedLocked && status == PAGE_LOCK_SUCCESS) {
+         status = Vmx86_UnlockPage(vm, uAddr);
+         ASSERT(status == PAGE_UNLOCK_SUCCESS);
+      }
+      for (resetPage = 0; resetPage <= page; resetPage++) {
+         pages[resetPage] = INVALID_MPN;
+      }
+      return FALSE;
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_GetRegionMPN --
+ *
+ *      Obtain the backing MPN for stat vars for the given VCPU at
+ *      the given offset.
+ *
+ * Results:
+ *      The backing stat vars MPN for the given VCPU at the given offset
+ *      if it has been registered, or INVALID_MPN otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpu, PageCnt offset)
+{
+   MPN *pages;
+   MPN backingPage;
+   StatVarsVmmon *statVars = vm->statVars;
+
+   ASSERT(vcpu < vm->numVCPUs);
+
+   HostIF_VMLock(vm, 46);
+   if (offset >= statVars->pagesPerVcpu) {
+      HostIF_VMUnlock(vm, 46);
+      Warning("(%s) Requested MPN at invalid offset %"FMT64"u for VCPU %u\n",
+              __FUNCTION__, offset, vcpu);
+      return INVALID_MPN;
+   }
+
+   pages = statVars->pages + vcpu * statVars->pagesPerVcpu;
+   backingPage = pages[offset];
+   HostIF_VMUnlock(vm, 46);
+   return backingPage;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,11 +24,12 @@
 #include "includeCheck.h"
 #include "bootstrap_vmm.h"

-struct InitBlock;
+struct MonLoaderHeader;

 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
-extern int Task_InitCrosspage(VMDriver *vm, struct InitBlock *params);
-extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
+extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
+                               PerVcpuPages *perVcpuPages);
+extern Bool Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
 extern MPN Task_GetHVRootPageForPCPU(uint32 pCPU);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,13 +32,14 @@
 #include "x86msr.h"
 #include "modulecall.h"
 #include "vcpuid.h"
-#include "initblock.h"
 #include "iocontrols.h"
 #include "numa_defs.h"
 #include "rateconv.h"
 #include "vmmem_shared.h"
 #include "apic.h"
 #include "bootstrap_vmm.h"
+#include "sharedAreaVmmon.h"
+#include "statVarsVmmon.h"

 typedef struct TSCDelta {
   Atomic_uint64 min;
@@ -47,6 +48,10 @@ typedef struct TSCDelta {

 struct VmmBlobInfo;

+struct HostIFContigMemMap;
+
+extern struct HostIFContigMemMap *hvIOBitmap;
+
 /*
 * VMDriver - the main data structure for the driver side of a
 *            virtual machine.
@@ -55,23 +60,19 @@ struct VmmBlobInfo;
 typedef struct VMDriver {
   /* Unique (in the driver), strictly positive, VM ID used by userland. */
   int                     userID;
-
   Vcpuid                  numVCPUs;         /* Number of vcpus in VM. */
-
   struct VMDriver        *nextDriver;       /* Next on list of all VMDrivers */
-
   struct VMHost          *vmhost;           /* Host-specific fields. */
-
   MPN                    *ptRootMpns;       /* numVCPUs-sized array. */
   struct VmmBlobInfo     *blobInfo;         /* VMM bootstrap blob info. */
-
+   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
+   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
-   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
+   struct VMCrossPageData **crosspage;        /* numVCPUs-sized array. */
   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
   APICDescriptor          hostAPIC;
-
   struct MemTrack        *memtracker;       /* Memory tracker pointer */
   Bool                    checkFuncFailed;
   struct PerfCounter     *perfCounter;
@@ -81,6 +82,13 @@ typedef struct VMDriver {
   Atomic_uint64           ptscLatest;
   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
+   PageCnt                 numPTPPages;      /* Num PTP pages allocated. */
+   /*
+    * List of physically contiguous allocations associated with this VM.
+    * Access is protected by the VM lock.
+    */
+   struct HostIFContigMemMap     *contigMappings;
+
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -110,7 +118,7 @@ typedef struct PseudoTSC {

 extern PseudoTSC pseudoTSC;

-#define MAX_LOCKED_PAGES (-1)
+#define MAX_LOCKED_PAGES MAX_PPN

 extern void Vmx86_CacheNXState(void);
 extern VMDriver *Vmx86_CreateVM(VA64 bsBlob,
@@ -121,11 +129,10 @@ extern Bool Vmx86_ProcessBootstrap(VMDriver *vm,
                                   uint32 numBytes,
                                   uint32 headerOffset,
                                   uint16 numVCPUs,
-                                   UserVA64 *ptRootVAs,
+                                   PerVcpuPages *perVcpuPages,
                                   VMSharedRegion *shRegions);
 extern int Vmx86_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 extern int Vmx86_ReleaseVM(VMDriver *vm);
-extern int Vmx86_InitVM(VMDriver *vm, InitBlock *initParams);
 extern int Vmx86_LateInitVM(VMDriver *vm);
 extern int Vmx86_RunVM(VMDriver *vm, Vcpuid vcpuid);
 extern void Vmx86_YieldToSet(VMDriver *vm, Vcpuid currVcpu, const VCPUSet *req,
@@ -142,22 +149,21 @@ extern int Vmx86_UnlockPage(VMDriver *vm, VA64 uAddr);
 extern int Vmx86_UnlockPageByMPN(VMDriver *vm, MPN mpn, VA64 uAddr);
 extern MPN Vmx86_GetRecycledPage(VMDriver *vm);
 extern int Vmx86_ReleaseAnonPage(VMDriver *vm, MPN mpn);
-extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned numPages, Bool kernelMPNBuffer,
-                                  Bool ignoreLimits);
-extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                 unsigned numPages, Bool kernelMPNBuffer);
+extern int64 Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                    PageCnt numPages, Bool kernelMPNBuffer,
+                                    Bool ignoreLimits);
+extern int Vmx86_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn);
+extern MPN Vmx86_GetNumAnonPages(VMDriver *vm);
 extern MPN Vmx86_AllocLowPage(VMDriver *vm, Bool ignoreLimits);
 extern void *Vmx86_Calloc(size_t numElements,
                          size_t elementSize,
-                          int nonPageable);
+                          Bool nonPageable);
 extern void  Vmx86_Free(void *ptr);

 extern int32 Vmx86_GetNumVMs(void);
-extern int32 Vmx86_GetTotalMemUsage(void);
-extern Bool Vmx86_SetConfiguredLockedPagesLimit(unsigned limit);
-extern void Vmx86_SetDynamicLockedPagesLimit(unsigned limit);
+extern Bool Vmx86_SetConfiguredLockedPagesLimit(PageCnt limit);
+extern void Vmx86_SetDynamicLockedPagesLimit(PageCnt limit);
 extern Bool Vmx86_GetMemInfo(VMDriver *curVM,
                             Bool curVMOnly,
                             VMMemInfoArgs *args,
@@ -169,9 +175,12 @@ extern void Vmx86_UpdateMemInfo(VMDriver *curVM,
 extern void Vmx86_Add2MonPageTable(VMDriver *vm, VPN vpn, MPN mpn,
                                   Bool readOnly);
 extern Bool Vmx86_GetAllMSRs(MSRQuery *query);
+extern Bool Vmx86_CheckMSRUniformity(void);
 extern void Vmx86_FlushVMCSAllCPUs(MA vmcs);
 extern void Vmx86_MonTimerIPI(void);
 extern void Vmx86_InitIDList(void);
+extern Bool Vmx86_CreateHVIOBitmap(void);
+extern void Vmx86_CleanupHVIOBitmap(void);
 extern Bool Vmx86_GetPageRoot(VMDriver *vm, Vcpuid vcpuid, MPN *mpn);
 extern void Vmx86_Open(void);
 extern void Vmx86_Close(void);
@@ -0,0 +1,319 @@
+/*********************************************************
+ * Copyright (C) 2007-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+#ifndef _ADDRLAYOUT_H_
+#define _ADDRLAYOUT_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "address_defs.h"
+#include "x86desc.h"
+#include "x86types.h"
+#include "vmm_constants.h"
+
+#define DIRECT_EXEC_USER_RPL    3
+#define BINARY_TRANSLATION_RPL  1
+typedef char x86_PAGE[PAGE_SIZE];
+
+#define MAX_VADDR                    CONST64U(0xffffffffffffffff)
+#define MONITOR_SIZE                 (64 * 1024 * 1024)
+#define MONITOR_LINEAR_START         (MAX_VADDR - MONITOR_SIZE + 1)
+#define MONITOR_LINEAR_END           (MAX_VADDR)
+#define NUM_MONITOR_PAGES            (MONITOR_SIZE / PAGE_SIZE)
+#define MONITOR_BASE_VPN             (MONITOR_LINEAR_START >> PAGE_SHIFT)
+
+#define MONITOR_AS_START             MONITOR_BASE_VPN
+#define MONITOR_AS_LEN               NUM_MONITOR_PAGES
+
+/*
+ * Regions and items necessarily overlap.  Build parallel structures for each.
+ * These 'field' structures are used to determine page numbers and offsets.
+ * AddrLayout contains zero-length regions used for alignment-checking.  (These
+ * are used by gcc as Windows prohibits zero-length arrays.)
+ */
+
+/* Item layout. */
+typedef struct {
+#if defined __GNUC__
+#define REGION(name, length) x86_PAGE name##_REGION_MARKER_FIELD[0];
+#else
+#define REGION(name, length)
+#endif
+#define ITEM(name, length) x86_PAGE name##_FIELD[length];
+#include "addrlayout_table.h"
+} AddrLayout;
+#undef REGION
+#undef ITEM
+#define FIELD_PAGE_NUMBER(name) PAGE_NUMBER(&(((AddrLayout *)0)->name##_FIELD))
+
+
+/* Region layout. */
+typedef struct {
+#define REGION(name, length) x86_PAGE name##_REGION_FIELD[length];
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+} RegionAddrLayout;
+#undef REGION
+#undef ITEM
+#define REGION_FIELD_PAGE_NUMBER(name) \
+   PAGE_NUMBER(&(((RegionAddrLayout *)0)->name##_REGION_FIELD))
+
+#if defined __GNUC__
+/*
+ * The sum of sizes of items in a region must add up to the region's size.
+ * AddrLayout contains zero-sized region markers at their correct offsets.
+ * RegionAddrLayout contains full-sized regions (but no items).  Comparing the
+ * marker and region offsets ensures that all items sizes in the previous
+ * region sum to the size of the previous region.  To check the final region
+ * (as it has no successor) we assert the sizes of the tables match.
+ *
+ * We also assert 2MB region granularity.
+ *
+ * This checking is only performed when using gcc.
+ */
+#define REGION(name, length) static INLINE void name(void) { \
+   ASSERT_ON_COMPILE(FIELD_PAGE_NUMBER(name##_REGION_MARKER) == \
+                     REGION_FIELD_PAGE_NUMBER(name)); \
+   ASSERT_ON_COMPILE((length % 512) == 0); \
+   ASSERT_ON_COMPILE(sizeof(AddrLayout) == sizeof(RegionAddrLayout)); \
+   ASSERT_ON_COMPILE(sizeof(RegionAddrLayout) == MONITOR_SIZE); \
+}
+#define ITEM(name, length)
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+/*
+ * Sizes.
+ */
+
+enum {
+#define REGION(name, length) name##_REGION_LEN = length,
+#define ITEM(name, length) name##_LEN = length,
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+/*
+ * Starts.
+ */
+
+#if defined __GNUC__
+/* Windows compiler and gdb do not accept 64-bit enumerations, but gcc
+ * does.  These are used by geninfo.c.
+ */
+enum {
+#define REGION(name, length) _asm_##name##_REGION_START = \
+    0ULL + MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) _asm_##name##_START = \
+    0ULL + MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+enum {
+#define REGION(name, length) name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+#else
+#define REGION(name, length) \
+   static const uint64 name##_REGION_START = \
+   MONITOR_BASE_VPN + REGION_FIELD_PAGE_NUMBER(name);
+#define ITEM(name, length) \
+   static const uint64 name##_START = \
+   MONITOR_BASE_VPN + FIELD_PAGE_NUMBER(name);
+#include "addrlayout_table.h"
+#undef REGION
+#undef ITEM
+#endif
+
+enum {
+#define REGION(name, length) name##_REGION_start_page = \
+   REGION_FIELD_PAGE_NUMBER(name),
+#define ITEM(name, length) name##_start_page = \
+   FIELD_PAGE_NUMBER(name),
+#include "addrlayout_table.h"
+};
+#undef REGION
+#undef ITEM
+
+
+/* Derive the bootstrap start address from table contents. */
+#define MONITOR_BOOTSTRAP_START_LA   VPN_2_VA(BS_TXT_START)
+
+#ifndef VMX86_SERVER
+#define SWITCH_PAGE_TABLE_LEN          12
+#endif
+
+#define VMM_STACK_TOP_HW          VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+/*
+ * VMM_STKTOP_HW_LEN needs to contain an ExcFrame64ForCopy (72 bytes) and
+ * some more bytes for a "no man's land" region.  We make sure the "no
+ * man's land" region is 0 on BT_Resume and HV_Resume.  Because we
+ * check "no man's land" in 8-byte chunks, VMM_STKTOP_HW_LEN must be
+ * divisible by 8.
+ */
+#define VMM_STKTOP_HW_LEN          168
+#define VMM_STKTOP_HW_LRET_LEN     (VMM_STKTOP_HW_LEN + sizeof(LretFrame64))
+#define VMM_STKTOP_LRET_OFFSET     (PAGE_SIZE - VMM_STKTOP_HW_LRET_LEN)
+
+#define MON_STACK_BASE            VPN_2_VA(MON_STACK_PAGES_START)
+#define MON_STACK_TOP             VPN_2_VA(MON_STACK_PAGES_START + \
+                                           MON_STACK_PAGES_LEN)
+#define DF_STACK_BASE              VPN_2_VA(DF_STACK_PAGES_START)
+#define DF_STACK_TOP               VPN_2_VA(DF_STACK_PAGES_START + \
+                                            DF_STACK_PAGES_LEN)
+#define MC_STACK_BASE              VPN_2_VA(MC_STACK_PAGES_START)
+#define MC_STACK_TOP               VPN_2_VA(MC_STACK_PAGES_START + \
+                                            MC_STACK_PAGES_LEN)
+#define NMI_STACK_BASE             VPN_2_VA(NMI_STACK_PAGES_START)
+#define NMI_STACK_TOP              VPN_2_VA(NMI_STACK_PAGES_START + \
+                                            NMI_STACK_PAGES_LEN)
+
+#define NUM_MON_PTABS   MON_PAGE_TABLE_L1_LEN
+
+#define MON_START_L5OFF PT_LA_2_L5OFF(MONITOR_LINEAR_START)
+#define MON_START_L4OFF PT_LA_2_L4OFF(MONITOR_LINEAR_START)
+#define MON_START_L3OFF PT_LA_2_L3OFF(MONITOR_LINEAR_START)
+#define MON_START_L2OFF PT_LA_2_L2OFF(MONITOR_LINEAR_START)
+
+/*
+ * Allocated wired pages for the monitor.
+ * Currently we allocate wired pages for the:
+ *  0) The shared area between user and monitor;
+ *  a) GDT of the monitor
+ *  a1) BT (CP1) stack
+ *  b) The monitor's page table with one page directory under 4 GB
+ *      and two page table pages.
+ *  c) Pages for the monitor's stack.
+ *  d) The BT area pages
+ *  e) The cross driver/monitor page.
+ *  f) The physical memory array for the machine.
+ *  g) The SMRAM memory
+ *
+ * We allocate the shared area separately since it must be read/write
+ * in the monitor.   See SharedArea_PowerOn().
+ */
+
+#define MON_BOOTSTRAP_PAGE_TABLE_PAGES (BOOTSTRAP_REGION_LEN     /  \
+                                        PAE_PTES_PER_PGTBL)
+#ifdef VMX86_SERVER
+#define NUM_MONWIRED_PAGES     0
+
+#define MON_PAGE_TABLE_ALLOC_PAGES 0
+#define NUM_MONWIRED_BOOTSTRAP_PAGES 0
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN  + \
+                                    DF_STACK_PAGES_LEN   + \
+                                    MC_STACK_PAGES_LEN   + \
+                                    NMI_STACK_PAGES_LEN  + \
+                                    HV_CURRENT_VMCB_LEN  + \
+                                    IDT_NORMAL_LEN       + \
+                                    GDT_AND_TASK_LEN)
+
+#else
+#define NUM_MONWIRED_PAGES    (CROSS_PAGE_CODE_LEN   + \
+                               CROSS_PAGE_DATA_LEN   + \
+                               MON_PAGE_TABLE_L4_LEN + \
+                               MON_PAGE_TABLE_L3_LEN + \
+                               MON_PAGE_TABLE_L2_LEN)
+
+#define MON_PAGE_TABLE_ALLOC_PAGES ((MONITOR_READONLY_REGION_LEN + \
+                                     MONITOR_DATA_REGION_LEN     + \
+                                     MONITOR_MISC_REGION_LEN)    / \
+                                     PAE_PTES_PER_PGTBL)
+
+#define NUM_MONWIRED_BOOTSTRAP_PAGES MON_BOOTSTRAP_PAGE_TABLE_PAGES
+
+#define NUM_MONWIRED_NUMA_PAGES_ML (MON_STACK_PAGES_LEN + \
+                                    DF_STACK_PAGES_LEN  + \
+                                    MC_STACK_PAGES_LEN  + \
+                                    NMI_STACK_PAGES_LEN + \
+                                    GDT_AND_TASK_LEN    + \
+                                    IDT_NORMAL_LEN      + \
+                                    HV_CURRENT_VMCB_LEN + \
+                                    SWITCH_PAGE_TABLE_LEN)
+#endif
+
+#define MON_PAGE_TABLE_L5        VPN_2_VA(MON_PAGE_TABLE_L5_START)
+#define MON_PAGE_TABLE_L4        VPN_2_VA(MON_PAGE_TABLE_L4_START)
+#define VMM_STACK_TOP            (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LEN)
+#define VMM_LRET_STACK_TOP       (VMM_STACK_TOP_HW - VMM_STKTOP_HW_LRET_LEN)
+/*
+ * Mark out a guard page for the VMM stack, this is not in the addrlayout_table
+ * explicitly because it overlaps the last page of the MONITOR_READONLY region.
+ */
+#define VMM_STACK_GUARD_START    MON_STACK_PAGES_START
+#define VMM_STACK_GUARD_LEN      PAGE_SIZE
+
+/*
+ * Following functions return true if the range [va, va+len) is within
+ * range of a particular monitor stack.  (Don't adjust without
+ * considering uint64 overflow when va is very high.)
+ */
+static INLINE Bool
+AddrLayout_InMonStack(VA64 va, size_t len)
+{
+   return VPN_2_VA(MON_STACK_PAGES_START) <= va &&
+          va <= VMM_STACK_TOP_HW - len;
+}
+
+static INLINE Bool
+AddrLayout_InNMIStack(VA64 va, size_t len)
+{
+   return va >= NMI_STACK_BASE && va <= NMI_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InDFStack(VA64 va, size_t len)
+{
+   return va >= DF_STACK_BASE && va <= DF_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InMCStack(VA64 va, size_t len)
+{
+   return va >= MC_STACK_BASE && va <= MC_STACK_TOP - len;
+}
+
+static INLINE Bool
+AddrLayout_InAMonitorStack(VA va, size_t len)
+{
+   return AddrLayout_InMonStack(va, len)  ||
+          AddrLayout_InNMIStack(va, len)  ||
+          AddrLayout_InDFStack(va, len)   ||
+          AddrLayout_InMCStack(va, len);
+}
+
+#endif
@@ -0,0 +1,144 @@
+/*********************************************************
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * addrlayout_table.h --
+ *
+ * Named elements in the monitor's address space of fixed size and location.
+ * All units are 4 kilobyte pages.  Also used for vmmlayout output.
+ *
+ * The monitor's address space is populated three different ways:
+ * 1) For elements of a fixed size and location, at compile-time.
+ * 2) For well-known program sections and the shared area, at link-time.
+ * 3) Dynamically, by the allocator, during bootstrapping and monitor power-on.
+ *
+ * This file describes all of the monitor's main 64MB address space.  All of
+ * (1) is described specifically.  Ranges for (2) and (3) are accounted for,
+ * but their contents are described generally here.
+ *
+ * The monitor's address space is organized into container regions which are an
+ * integer number of 2MB "large" pages.  Within a container region, items
+ * comprised of one or more 4KB pages may be described.
+ *
+ * All container regions and items are described by a name and a size in pages.
+ * No two items may have the same name nor may two regions.
+ *
+ * REGION(region-name, number-of-pages)
+ *   ITEM(item-name,   number-of-pages)     // first item in region
+ *   ITEM(item-name,   number-of-pages)     // second item in region
+ * <...>
+ */
+
+
+/*
+ * Monitor .rodata/.text.  Mapped large (reducing TLB pressure) and read-only.
+ */
+REGION(MONITOR_READONLY,        1024)
+  ITEM(MONITOR_READONLY_LINKER, 1024) // Used by linker.
+
+/*
+ * Monitor .data/.bss.  Mapped large (reducing TLB pressure) and read-write.
+ */
+REGION(MONITOR_DATA,            512)
+  ITEM(MON_STACK_PAGES,           8) // Monitor stack.
+  ITEM(IDT_NORMAL,                1) // Normal IDT (used for all but SVM).
+  ITEM(MONITOR_DATA,            503) // Used by linker and TC.
+
+/*
+ * The monitor's translation cache.  This object starts at the page boundary
+ * following the end of the used space in MONITOR_DATA and extends through
+ * the TC_REGION.
+ */
+REGION(TC_REGION,              1024)
+  ITEM(TC_BLOCK,               1024)
+
+/*
+ * The monitor's pages for architectural state, stacks, page tables, shared
+ * pages, and dynamic allocation.  This region also contains the linked shared
+ * area and other contents.
+ * Mapped small (allowing non-present mappings and sharing) and read-write.
+ */
+REGION(MONITOR_MISC,           5632)
+#ifdef VMX86_SERVER
+  ITEM(GUARD_PAGE,                2) // Reserved (for symmetry with hosted).
+#else
+  ITEM(CROSS_PAGE_DATA,           1) // Cross page data (RW).
+  ITEM(CROSS_PAGE_CODE,           1) // Cross page code (RX).
+#endif
+  ITEM(GDT_AND_TASK,              1) // GDT and Task State Segment.
+
+  ITEM(MON_PAGE_TABLE_L5,         1) // Monitor page root, if 5-Level PT used.
+  ITEM(MON_PAGE_TABLE_L4,         1) // Monitor page root, if 4-Level PT used.
+  ITEM(MON_PAGE_TABLE_L3,         1) // Monitor L3 page table.
+  ITEM(MON_PAGE_TABLE_L2,         1) // Monitor L2 page table.
+  ITEM(MON_PAGE_TABLE_L1,        32) // Monitor L1 page tables.
+
+  ITEM(HOST_APIC,                 1) // Physical APIC.
+  ITEM(GUEST_APIC,                1) // Guest APIC.
+
+  ITEM(DF_GUARD_PAGE,             1) // Double Fault stack guard page.
+  ITEM(DF_STACK_PAGES,            1) // Double Fault stack; need ~768 bytes.
+
+  ITEM(MC_GUARD_PAGE,             1) // Machine Check stack guard page.
+  ITEM(MC_STACK_PAGES,            1) // Machine Check stack.
+
+  ITEM(NMI_GUARD_PAGE,            1) // NMI stack guard page.
+  ITEM(NMI_STACK_PAGES,           4) // NMI stack.  Profiling requires a larger
+                                     // stack than MC/DF.
+
+  ITEM(HV_SWITCH,                 1) // SVM switch page
+  ITEM(HV_CURRENT_VMCB,           1) // SVM current VMCB
+  ITEM(HV_STD_NATIVE_VMCB,        1) // SVM/VT standard VMCB/VMCS
+  ITEM(HV_AUX_NATIVE_VMCB,        1) // SVM/VT auxiliary VMCB/VMCS
+  ITEM(VHV_GUEST_VMCB,            1) // VSVM/VVT guest VMCB/VMCS
+  ITEM(HV_SEV_VMSA,               1) // SVM SEV-ES state save area
+  ITEM(VPROBE_MON_RELOC,          1) // VProbe monitor reloc page.
+  ITEM(GART_LIST_PAGES,          48) // PhysMem vmm gart list pages.
+  ITEM(GART_ALT_LIST_PAGES,      48) // Alternate vmm gart pages
+                                     // (for invalidation).
+  ITEM(GART_BF_PAGES,             8) // Gart Bloom filter memory pages.
+  ITEM(VVT_GUEST_VIRT_APIC,       1) // Inner guest virtual APIC page.
+  ITEM(HT_STATE_MAP,              8) // Information used by the SecureHT module
+  ITEM(SHARED_RW_DATA,         5462) // R/W shared data, including the shared
+                                     // area and stat vars. Any remaining pages
+                                     // are left to the dynamic allocator, which
+                                     // extends throughout the entire reclaimed
+                                     // bootstrap region.
+/*
+ * Bootstrap-used space.  This space is reclaimed when bootstrap is done, and
+ * is used by the allocator thereafter.  The allocator starts at the first
+ * unused page in the SHARED_RW_DATA item from the MONITOR_MISC region and
+ * extends through to the end of the VMM address space once the bootstrap region
+ * is reclaimed.
+ */
+REGION(BOOTSTRAP,              8192)
+  ITEM(BS_TXT,                   17) // bootstrap's .text
+  ITEM(BS_RODATA,                10) // bootstrap's .rodata
+  ITEM(BS_DATA,                  10) // bootstrap's .data
+  ITEM(BS_BSS,                   10) // bootstrap's .bss
+  ITEM(BS_HEADER,                 1) // MonLoaderHeader mapping
+  ITEM(BS_VCPU_L1PT_RANGE,       32) // VCPU L1PT mappings
+  ITEM(BS_VCPU_L1PT_PT,           1) // L1PT that maps a VCPU's L1 page tables
+  ITEM(BS_INIT_POOL,              1) // pool for initializing VMM pages
+  ITEM(BS_DYNAMIC_ALLOC,       1880) // bs_alloc's dynamic allocator pages
+  ITEM(IDT_BOOTSTRAP,             1) // bootstrap IDT
+  ITEM(IDT_BOOTSTRAP_STUBS,       2) // bootstrap IDT gate stubs
+  ITEM(BS_PER_VM_VMX,           300) // vmm64's shared_per_vm_vmx
+  ITEM(BS_PER_VCPU,               8) // vmm64's shared_per_vcpu for VCPU 0
+  ITEM(BS_PER_VCPU_VMX,         136) // vmm64's shared_per_vcpu_vmx for VCPU 0
+  ITEM(VMM_MODULES,            5783) // ~22.6MB for unlinked VMM modules
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -345,6 +345,34 @@ CircList_Push(ListItem *p,               // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_InsertAfter --
+ *
+ *      Adds a new member to the list after the provided item.  Assumes p
+ *      is not a member of a list already.
+ *
+ * Result:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE void
+CircList_InsertAfter(ListItem *p,      // IN:
+                     ListItem *after)  // IN:
+{
+   p->prev = after;
+   p->next = after->next;
+   p->next->prev = p;
+   after->next = p;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,77 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _CPUID_INFO_H
-#define _CPUID_INFO_H
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-
-#include "includeCheck.h"
-
-#include "vm_basic_asm.h"
-#include "x86cpuid_asm.h"
-
-#if defined __cplusplus
-extern "C" {
-#endif
-
-
-typedef struct CPUID0 {
-   int numEntries;
-   char name[16];      // 4 extra bytes to null terminate
-} CPUID0;
-
-typedef struct CPUID1 {
-   uint32 version;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID1;
-
-typedef struct CPUID80 {
-   uint32 numEntries;
-   uint32 ebx;
-   uint32 ecx;
-   uint32 edx;
-} CPUID80;
-
-typedef struct CPUID81 {
-   uint32 eax;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID81;
-
-typedef struct CPUIDSummary {
-   CPUID0  id0;
-   CPUID1  id1;
-   CPUIDRegs ida;
-   CPUID80 id80;
-   CPUID81 id81;
-   CPUIDRegs id88, id8a;
-} CPUIDSummary;
-
-
-#if defined __cplusplus
-} // extern "C"
-#endif
-
-#endif // _CPUID_INFO_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,7 +37,6 @@
 #error iocontrols.h is for hosted vmmon, do not use on visor
 #endif

-#include "basic_initblock.h"
 #include "x86segdescrs.h"
 #include "rateconv.h"
 #include "overheadmem_types.h"
@@ -45,6 +44,8 @@
 #include "numa_defs.h"
 #include "bootstrap_vmm.h"
 #include "contextinfo.h"
+#include "vcpuid.h"
+#include "sharedAreaType.h"

 #if defined __cplusplus
 extern "C" {
@@ -142,9 +143,12 @@ PtrToVA64(void const *ptr) // IN
 * does not break vmcore compatibility.
 *
 * See bora/doc/vmcore details.
+ *
+ * If you increment VMMON_VERSION, you must also increment
+ * the NT specific VMX86_DRIVER_VERSION.
 */

-#define VMMON_VERSION           (360 << 16 | 0)
+#define VMMON_VERSION           (401 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -153,11 +157,8 @@ PtrToVA64(void const *ptr) // IN
 * ENOMEM returned after MAX_VMS virtual machines created
 */

-#ifdef VMX86_SERVER
-#define MAX_VMS 128
-#else
 #define MAX_VMS 64
-#endif
+
 /*
 * MsgWaitForMultipleObjects doesn't scale well enough on Win32.
 * Allocate with MAX_VMS so static buffers are large, but do
@@ -209,11 +210,13 @@ enum IOCTLCmd {
   IOCTLCMD(VERSION) = IOCTLCMD(FIRST),
   IOCTLCMD(CREATE_VM),
   IOCTLCMD(PROCESS_BOOTSTRAP),
+   IOCTLCMD(REGISTER_SHARED),
+   IOCTLCMD(REGISTER_STATVARS),
   IOCTLCMD(RELEASE_VM),
   IOCTLCMD(GET_NUM_VMS),
-   IOCTLCMD(INIT_VM),
   IOCTLCMD(RUN_VM),
   IOCTLCMD(LOOK_UP_MPN),
+   IOCTLCMD(GET_VMM_PAGE_ROOT),
   IOCTLCMD(LOCK_PAGE),
   IOCTLCMD(UNLOCK_PAGE),
   IOCTLCMD(APIC_INIT),
@@ -222,7 +225,6 @@ enum IOCTLCmd {
   IOCTLCMD(ADMIT),
   IOCTLCMD(UPDATE_MEM_INFO),
   IOCTLCMD(READMIT),
-   IOCTLCMD(GET_TOTAL_MEM_USAGE),
   IOCTLCMD(GET_KHZ_ESTIMATE),
   IOCTLCMD(SET_HOST_CLOCK_RATE),
   IOCTLCMD(READ_PAGE),
@@ -232,6 +234,7 @@ enum IOCTLCmd {
    /* AWE calls */
   IOCTLCMD(ALLOC_LOCKED_PAGES),
   IOCTLCMD(GET_NEXT_ANON_PAGE),
+   IOCTLCMD(GET_NUM_ANON_PAGES),

   IOCTLCMD(GET_ALL_MSRS),

@@ -245,6 +248,7 @@ enum IOCTLCmd {
   IOCTLCMD(GET_IPI_VECTORS),
   IOCTLCMD(SEND_IPI),
   IOCTLCMD(SEND_ONE_IPI),
+   IOCTLCMD(GET_SWITCH_ERROR_ADDR),

   /*
    * Keep host-specific calls at the end so they can be undefined
@@ -253,18 +257,10 @@ enum IOCTLCmd {

 #if defined __linux__
   IOCTLCMD(SET_UID),		// VMX86_DEVEL only
-#endif
-
-#if defined __linux__ || defined __APPLE__
   IOCTLCMD(GET_ALL_CPUID),
 #endif

-#if defined _WIN32 || defined __APPLE__
-   IOCTLCMD(ALLOC_CONTIG_PAGES),
-#endif
-
 #if defined _WIN32
-   IOCTLCMD(FREE_CONTIG_PAGES),
   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows
   IOCTLCMD(CHANGE_HARD_LIMIT),         // used by vmauthd on Windows
   IOCTLCMD(READ_DISASM_PROC_BINARY),
@@ -272,19 +268,16 @@ enum IOCTLCmd {
   IOCTLCMD(SET_MEMORY_PARAMS),
   IOCTLCMD(REMEMBER_KHZ_ESTIMATE),
   IOCTLCMD(REMAP_SCATTER_LIST),
+   IOCTLCMD(REMAP_SCATTER_LIST_RO),     // map the list as read-only
   IOCTLCMD(UNMAP_SCATTER_LIST),
 #endif

 #if defined __APPLE__
-   IOCTLCMD(ALLOC_LOW_PAGES),
+   IOCTLCMD(GET_NUM_RESPONDING_CPUS),
   IOCTLCMD(INIT_DRIVER),
   IOCTLCMD(BLUEPILL),
 #endif

-#if defined __linux__
-   IOCTLCMD(SET_HOST_SWAP_SIZE),
-#endif
-
   IOCTLCMD(GET_UNAVAIL_PERF_CTRS),
   IOCTLCMD(GET_MONITOR_CONTEXT),
   // Must be last.
@@ -317,13 +310,16 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_CREATE_VM           VMIOCTL_BUFFERED(CREATE_VM)
 #define IOCTL_VMX86_RELEASE_VM          VMIOCTL_BUFFERED(RELEASE_VM)
 #define IOCTL_VMX86_PROCESS_BOOTSTRAP   VMIOCTL_BUFFERED(PROCESS_BOOTSTRAP)
+#define IOCTL_VMX86_REGISTER_SHARED     VMIOCTL_BUFFERED(REGISTER_SHARED)
+#define IOCTL_VMX86_REGISTER_STATVARS   VMIOCTL_BUFFERED(REGISTER_STATVARS)
 #define IOCTL_VMX86_GET_NUM_VMS         VMIOCTL_BUFFERED(GET_NUM_VMS)
-#define IOCTL_VMX86_INIT_VM             VMIOCTL_BUFFERED(INIT_VM)
 #define IOCTL_VMX86_RUN_VM              VMIOCTL_NEITHER(RUN_VM)
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
+#define IOCTL_VMX86_GET_SWITCH_ERROR_ADDR VMIOCTL_BUFFERED(GET_SWITCH_ERROR_ADDR)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
+#define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
 #define IOCTL_VMX86_UNLOCK_PAGE         VMIOCTL_BUFFERED(UNLOCK_PAGE)
 #define IOCTL_VMX86_APIC_INIT           VMIOCTL_BUFFERED(APIC_INIT)
@@ -334,8 +330,6 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UPDATE_MEM_INFO     VMIOCTL_BUFFERED(UPDATE_MEM_INFO)
 #define IOCTL_VMX86_HARD_LIMIT_MONITOR_STATUS   VMIOCTL_BUFFERED(HARD_LIMIT_MONITOR_STATUS)
 #define IOCTL_VMX86_CHANGE_HARD_LIMIT   VMIOCTL_BUFFERED(CHANGE_HARD_LIMIT)
-#define IOCTL_VMX86_ALLOC_CONTIG_PAGES  VMIOCTL_BUFFERED(ALLOC_CONTIG_PAGES)
-#define IOCTL_VMX86_FREE_CONTIG_PAGES   VMIOCTL_BUFFERED(FREE_CONTIG_PAGES)

 #define IOCTL_VMX86_GET_TOTAL_MEM_USAGE	VMIOCTL_BUFFERED(GET_TOTAL_MEM_USAGE)
 #define IOCTL_VMX86_GET_KHZ_ESTIMATE    VMIOCTL_BUFFERED(GET_KHZ_ESTIMATE)
@@ -347,6 +341,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UNLOCK_PAGE_BY_MPN  VMIOCTL_BUFFERED(UNLOCK_PAGE_BY_MPN)
 #define IOCTL_VMX86_ALLOC_LOCKED_PAGES  VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES)
 #define IOCTL_VMX86_GET_NEXT_ANON_PAGE  VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE)
+#define IOCTL_VMX86_GET_NUM_ANON_PAGES  VMIOCTL_BUFFERED(GET_NUM_ANON_PAGES)

 #define IOCTL_VMX86_READ_DISASM_PROC_BINARY \
                                      VMIOCTL_BUFFERED(READ_DISASM_PROC_BINARY)
@@ -364,18 +359,14 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_GET_PSEUDO_TSC           VMIOCTL_NEITHER(GET_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS    VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS)
 #define IOCTL_VMX86_GET_MONITOR_CONTEXT      VMIOCTL_BUFFERED(GET_MONITOR_CONTEXT)
-#define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
-#define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST    VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST_RO VMIOCTL_BUFFERED(REMAP_SCATTER_LIST_RO)
+#define IOCTL_VMX86_UNMAP_SCATTER_LIST    VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
 #endif


-/*
- * Flags sent into APICBASE ioctl
- */
+#define INIT_BLOCK_MAGIC     (0x1789 + 14)

-#define APIC_FLAG_DISABLE_NMI       0x00000001
-#define APIC_FLAG_PROBE             0x00000002
-#define APIC_FLAG_FORCE_ENABLE      0x00000004

 typedef
 #include "vmware_pack_begin.h"
@@ -395,10 +386,14 @@ union {
 #include "vmware_pack_end.h"
 VMLockPage;

-
-typedef struct VMAPICInfo {
-   uint32 flags;
-} VMAPICInfo;
+typedef
+#include "vmware_pack_begin.h"
+union {
+   Vcpuid vcpuid; // IN: VCPU
+   MPN pageRoot;  // OUT: MPN of the VCPU's page root
+}
+#include "vmware_pack_end.h"
+VcpuPageRoot;

 #define VMX86_DRIVER_VCPUID_OFFSET 1000

@@ -413,9 +408,9 @@ typedef struct VMAPICInfo {
 * all these values at any given time.
 */
 typedef struct LockedPageLimit {
-   uint32 host;        // driver calculated maximum for this host
-   uint32 configured;  // user defined maximum pages to lock
-   uint32 dynamic;     // authd hardLimitMonitor pages to lock
+   PageCnt host;        // driver calculated maximum for this host
+   PageCnt configured;  // user defined maximum pages to lock
+   PageCnt dynamic;     // authd hardLimitMonitor pages to lock
 } LockedPageLimit;

 /*
@@ -436,15 +431,15 @@ typedef struct LockedPageLimit {
 */

 typedef struct VMMemMgmtInfo {
-   uint32          minAllocation;   // minimum pages for vm
-   uint32          maxAllocation;   // maximum pages the vm could lock
+   PageCnt         minAllocation;   // minimum pages for vm
+   PageCnt         maxAllocation;   // maximum pages the vm could lock
+   PageCnt         nonpaged;        // overhead memory (guest, mmap)
+   PageCnt         paged;           // vmx memory (malloc, statics)
+   PageCnt         anonymous;       // vmm memory
+   PageCnt         mainMemSize;     // guest main memory size
+   PageCnt         locked;          // number of pages locked by this vm
+   PageCnt         perVMOverhead;   // memory for vmx/vmmon overheads
   uint32          shares;          // proportional sharing weight
-   uint32          nonpaged;        // overhead memory (guest, mmap)
-   uint32          paged;           // vmx memory (malloc, statics)
-   uint32          anonymous;       // vmm memory
-   uint32          mainMemSize;     // guest main memory size
-   uint32          locked;          // number of pages locked by this vm
-   uint32          perVMOverhead;   // memory for vmx/vmmon overheads
   Percent         touchedPct;      // % of guest memory being touched
   Percent         dirtiedPct;      // % of guest memory being dirtied
   Bool            admitted;        // admission control
@@ -467,14 +462,14 @@ typedef struct VMMemMgmtInfoPatch {

 typedef struct VMMemInfoArgs {
   uint64          currentTime;        // Host time in secs of the call.
-   uint32          minVmMemPct;        // % of vm that must fit in memory
-   uint32          globalMinAllocation;// pages that must fit in maxLockedPages
-   uint32          numLockedPages;     // total locked pages by all vms
+   PageCnt         globalMinAllocation;// pages that must fit in maxLockedPages
+   PageCnt         numLockedPages;     // total locked pages by all vms
   LockedPageLimit lockedPageLimit;    // set of locked page limits
-   uint32          maxLockedPages;     // effective limit on locked pages
+   PageCnt         maxLockedPages;     // effective limit on locked pages
   uint32          callerIndex;        // this vm's index memInfo array
   uint32          numVMs;             // number of running VMs
-   uint8           _pad[4];
+   Percent         minVmMemPct;        // % of vm that must fit in memory
+   uint8           _pad[7];
   VMMemMgmtInfo   memInfo[1];
 } VMMemInfoArgs;

@@ -487,9 +482,9 @@ typedef struct VMMPNNext {
 } VMMPNNext;

 typedef struct VMMPNList {
-   uint32    mpnCount;   // IN (and OUT on Mac OS)
+   PageCnt   mpnCount;   // IN (and OUT on Mac OS)
   Bool      ignoreLimits;
-   uint8     _pad[3];
+   uint8     _pad[7];
   VA64      mpnList;    // IN: User VA of an array of 64-bit MPNs.
 } VMMPNList;

@@ -525,8 +520,6 @@ typedef struct PTSCCheckParams {
   uint8  _pad[7];
 } PTSCCheckParams;

-#ifndef VMX86_SERVER
-
 typedef struct IPIVectors {
   /*
    * Vectors we have allocated or stolen for the monitor interrupts.
@@ -535,13 +528,13 @@ typedef struct IPIVectors {
   uint8 hvIPIVector;
 } IPIVectors;

-#endif
-
 /*
 * Arguments and return value for VM creation.
 */
 typedef struct VMCreateBlock {
   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
+   VA64               vmmonData;     // IN: User VA of a userlevel scratch area
+                                     //     required by the Linux vmmon
   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
   uint32             numVCPUs;      // IN: Number of VCPUs.
   uint16             vmid;          // OUT: VM ID for the created VM.
@@ -551,11 +544,32 @@ typedef struct VMCreateBlock {
 * Information about a shared region.
 */
 typedef struct VMSharedRegion {
-   uint32 index;
-   VPN    baseVpn;
-   uint32 numPages;
+   SharedAreaType index;
+   VPN            baseVpn;
+   uint32         numPages;
 } VMSharedRegion;

+/*
+ * Arguments for VMM shared area registration.
+ */
+typedef struct VMSharedAreaRegistrationBlock {
+   Vcpuid vcpu;            // IN: VCPU being registered.
+   VMSharedRegion region;  // IN: Shared region being registered.
+} VMSharedAreaRegistrationBlock;
+
+/*
+ * Information about a VM's statvars.
+ */
+typedef struct VMStatVarsRegistrationBlock {
+   VPN     baseVpn;
+   PageCnt numPages;
+   Vcpuid  vcpu;
+} VMStatVarsRegistrationBlock;
+
+typedef struct {
+   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
+} PerVcpuPages;
+
 /*
 * Arguments for VMM bootstrap processing.
 */
@@ -564,10 +578,26 @@ typedef struct VMProcessBootstrapBlock {
   uint32         numBytes;      // IN: Size of VMM bootstrap blob.
   uint32         headerOffset;  // IN: Offset of header in blob.
   uint16         numVCPUs;      // IN: Number of VCPUs.
-   VA64           ptRootVAs[MAX_VCPUS];  // IN: User VA of PT roots.
   VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions.
+   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

+/*
+ * Arguments for retrieving the switch error address.
+ */
+typedef struct VMSwitchErrorArgs {
+   Vcpuid vcpuid;  // IN: The VCPU of interest.
+   uint64 addr;    // OUT: The code address that a failure was detected at, or 0
+                   //      if no failure has occurred.
+} VMSwitchErrorArgs;
+
+static INLINE size_t
+GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
+{
+   VMProcessBootstrapBlock *args = NULL;
+   return sizeof *args + numVCPUs * sizeof args->perVcpuPages[0];
+}
+
 /*
 * Arguments for VMM context retrieval.
 */
@@ -576,49 +606,7 @@ typedef union {
   Context64 context; // OUT
 } VMMonContext;

-#if defined __linux__
-
-/*
- * Linux uses mmap(2) to allocate contiguous locked pages, and uses these
- * macros to marshall real arguments to mmap's made-up 'offset' argument.
- */
-
-#define VMMON_MAP_MT_LOW4GB     0
-#define VMMON_MAP_MT_LOW16MB    1
-#define VMMON_MAP_MT_ANY        2
-
-#define VMMON_MAP_OFFSET_SHIFT  0
-#define VMMON_MAP_OFFSET_MASK   0x00000FFF
-#define VMMON_MAP_ORDER_SHIFT   12
-#define VMMON_MAP_ORDER_MASK    0xF
-#define VMMON_MAP_MT_SHIFT      16
-#define VMMON_MAP_MT_MASK       0x7
-#define VMMON_MAP_RSVD_SHIFT    19
-
-#define VMMON_MAP_RSVD(base)    \
-                ((base) >> VMMON_MAP_RSVD_SHIFT)
-#define VMMON_MAP_MT(base)      \
-                (((base) >> VMMON_MAP_MT_SHIFT) & VMMON_MAP_MT_MASK)
-#define VMMON_MAP_ORDER(base)   \
-                (((base) >> VMMON_MAP_ORDER_SHIFT) & VMMON_MAP_ORDER_MASK)
-#define VMMON_MAP_OFFSET(base)  \
-                (((base) >> VMMON_MAP_OFFSET_SHIFT) & VMMON_MAP_OFFSET_MASK)
-
-#define VMMON_MAP_BASE(mt, order)   (((mt) << VMMON_MAP_MT_SHIFT) | \
-                                     ((order) << VMMON_MAP_ORDER_SHIFT))
-
-#elif defined _WIN32
-/*
- * Windows uses an ioctl to allocate contiguous locked pages.
- */
-
-typedef struct VMAllocContiguousMem {
-   VA64   mpnList;  // IN: User VA of an array of 64-bit MPNs.
-   uint32 mpnCount; // IN
-   uint32 order;    // IN
-   MPN    maxMPN;   // IN
-} VMAllocContiguousMem;
-#elif defined __APPLE__
+#if defined __APPLE__
 #   include "iocontrolsMacos.h"
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,8 +42,6 @@
 #include "mon_assert.h"

 #define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
-#define CP_STUB_SIZE 16     /* A relative jmp instruction (5 bytes)
-                               padded to the next 16 byte boundary. */

 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
@@ -57,6 +55,7 @@
   MC(RELEASE_ANON_PAGES)                                                     \
   MC(LOOKUP_MPN)                                                             \
   MC(COSCHED)                                                                \
+   MC(ALLOC_CONTIG_PAGES)                                                     \
   MC(ALLOC_VMX_PAGE)                                                         \
   MC(ALLOC_TMP_GDT)                                                          \
   MC(PIN_MPN)                                                                \
@@ -65,8 +64,14 @@
   MC(ALLOC_ANON_LOW_PAGE)                                                    \
   MC(GET_MON_IPI_VECTOR)                                                     \
   MC(GET_HV_IPI_VECTOR)                                                      \
+   MC(GET_PERF_CTR_VECTOR)                                                    \
   MC(GET_HOST_TIMER_VECTORS)                                                 \
-   MC(BOOTSTRAP_CLEANUP)
+   MC(BOOTSTRAP_CLEANUP)                                                      \
+   MC(GET_SHARED_AREA)                                                        \
+   MC(GET_STAT_VARS)                                                          \
+   MC(GET_NUM_PTP_PAGES)                                                      \
+   MC(GET_HV_IO_BITMAP)                                                       \
+   MC(GET_MSR)                                                                \

 /*
 *----------------------------------------------------------------------
@@ -104,8 +109,8 @@ typedef enum ModuleCallType {
 */

 #if defined(VMX86_UCCOST) && !defined(VMX86_SERVER)
-#define UCTIMESTAMP(cp, stamp) \
-             do { (cp)->ucTimeStamps[UCCOST_ ## stamp] = RDTSC(); } while (0)
+#define UCTIMESTAMP(ptr, stamp) \
+             do { (ptr)[UCCOST_ ## stamp] = RDTSC(); } while (0)
 #else
 #define UCTIMESTAMP(cp, stamp)
 #endif
@@ -125,7 +130,7 @@ typedef struct UCCostResults {
 } UCCostResults;

 typedef enum UCCostStamp {
-#define UC(x) UCCOST_ ## x,
+#define UC(x, y) UCCOST_ ## x,
 #include "uccostTable.h"
   UCCOST_MAX
 } UCCostStamp;
@@ -139,12 +144,12 @@ typedef enum UCCostStamp {
 * MAX_SWITCH_PT_PATCHES
 *
 *   This is the maximum number of patches that must be placed into
- *   the monitor page tables so that the host GDT and the crosspage
- *   can be accessed during worldswitch.
+ *   the monitor page tables so that the host GDT, crosspage data, and
+ *   crosspage code can be accessed during worldswitch.
 *
 *----------------------------------------------------------------------
 */
-#define MAX_SWITCH_PT_PATCHES 2
+#define MAX_SWITCH_PT_PATCHES 3

 /*----------------------------------------------------------------------
 *
@@ -231,16 +236,23 @@ VMMPageTablePatch;
 * - the largest stack use instantaneously possible by #DB handling code
 * - one high-water uint32 used to detect stack overflows when debugging
 *
- * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+ * A breakdown of the worst-case exception handler stack usage (SwitchUDHandler)
+ * is: 5 * uint64 (Hardware) + 1 * uint64 (RAX) + 1 * uint64 (RBX) +
+ *     1 * uint64 (RCX) + 1 * uint64 (call) + 2 * uint64 (sidt) = 11 * uint64
+ * This is a slight over-estimate of the possible usage at any time but there
+ * is plenty of space available in the cross page data area.
+ *
+ * 264 (11 * sizeof(uint64) * 3) bytes is slightly more than enough as of
+ * 2020/06/14.
 */
-#define TINY_STACK_SIZE      184
+#define TINY_STACK_SIZE      264

 /*
 *----------------------------------------------------------------------
 *
 * VMCrossPageData --
 *
- *      Data which is stored on the VMCrossPage.
+ *      Data which is stored on the cross page.
 *
 *----------------------------------------------------------------------
 */
@@ -289,10 +301,13 @@ struct VMCrossPageData {
   uint16   monES;  /* Not saved/restored in assembly switch */
   uint16   monPad;

-   uint64   crosspageMA;
+   uint64   crosspageDataMA;

   uint64   hostDR[8];
-   LA64     hostCrossPageLA;   // where host has crosspage mapped
+   LA64     crosspageDataLA;   // where host/PTP map the cross data page
+   LA64     crossGDTLA;        // where host/PTP map the cross GDT
+   LA64     crosspageCodeLA;   // where host/PTP map the cross code page
+   LA64     vmmToHostLA;       // where host has placed the VmmToHost function
   uint16   hostInitial64CS;
   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
@@ -310,8 +325,6 @@ struct VMCrossPageData {
   Task64   monTask;          /* vmm's task */

   VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
-   LA64              vmmCrossPageLA;
-   LA64              vmmCrossGDTLA;   // where crossGDT is mapped by PT patch

   /*
    * The monitor may requests up to two actions when returning to the
@@ -333,6 +346,7 @@ struct VMCrossPageData {

 #if !defined(VMX86_SERVER)
   uint64 ucTimeStamps[UCCOST_MAX];
+   uint8  _ucPad[8];
 #endif

   SwitchedMSRState switchedMSRState;
@@ -395,74 +409,19 @@ struct VMCrossPageData {
 #include "vmware_pack_end.h"
 VMCrossPageData;

-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPageCode --
- *
- *      Code which is stored on the VMCrossPage.
- *
- *----------------------------------------------------------------------
- */
-
-#define CODE_BLOCK_SIZE (PAGE_SIZE - sizeof(VMCrossPageData) - \
-                         (2 + NUM_EXCEPTIONS) * CP_STUB_SIZE)
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPageCode {
-   uint8         toVmmFunc[CP_STUB_SIZE];    // Fixed-position stubs to jump
-   uint8         toHostFunc[CP_STUB_SIZE];   // to world switch functions
-   uint8         gateStubs[NUM_EXCEPTIONS *
-                           CP_STUB_SIZE];    // Stubs for crossIDT.
-   uint8         codeBlock[CODE_BLOCK_SIZE]; // Code for worldswitch and
-                                             // fault handling.
-}
-#include "vmware_pack_end.h"
-VMCrossPageCode;
-
-
-/*
- *----------------------------------------------------------------------
- *
- * VMCrossPage --
- *
- *      Data structure shared between the monitor and the module
- *      that is used for crossing between the two.
- *      Accessible as vm->crosspage (kernel module) and CROSS_PAGE
- *      (monitor)
- *
- *      Must be exactly one page long.
- *
- *----------------------------------------------------------------------
- */
-
-typedef
-#include "vmware_pack_begin.h"
-struct VMCrossPage {
-   VMCrossPageData crosspageData;
-   VMCrossPageCode crosspageCode;
-}
-#include "vmware_pack_end.h"
-VMCrossPage;
-
-#define CROSSPAGE_VERSION_BASE 0xc09 /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc12 /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE  ((VMCrossPage * const) VPN_2_VA(CROSS_PAGE_START))
-#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
+#define CROSS_PAGE             ((VMCrossPageData *) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
+#define VMM_SWITCH_SHARED_DATA ((VMCrossPageData * const) \
+                                VPN_2_VA(CROSS_PAGE_DATA_START))
 #endif

-#define NULLPAGE_LINEAR_START  (MONITOR_LINEAR_START + \
-                                PAGE_SIZE * CPL0_GUARD_PAGE_START)
-
 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
-#define MX_WAITNORMAL          1  // Must equal one; see linux module code.
-#define MX_WAITERROR           0  // Use MX_ISWAITERROR() to test for error.
+#define MX_WAITNORMAL          1
+#define MX_WAITERROR           0

-// Any zero or negative value denotes error.
-#define MX_ISWAITERROR(e)      ((e) <= MX_WAITERROR)
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006,2009-2011,2013-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006,2009-2011,2013-2015,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,8 +66,7 @@
   SWMSR(MSR_CSTAR)         \
   SWMSR(MSR_SFMASK)        \
   SWMSR(MSR_TSC_AUX)       \
-   SWMSR(MSR_BD_TSC_RATIO)  \
-   SWMSR(MSR_BNDCFGS)
+   SWMSR(MSR_BD_TSC_RATIO)

 /*
 *      Data structures for dealing with the context-switched MSRs that need
@@ -94,7 +93,6 @@ typedef struct SwitchedMSRValues {
 typedef struct SwitchedMSRState {
   SwitchedMSRValues smv;
   uint8             flags[NUM_SWITCHED_MSRS];
-   uint32            _pad;
 } SwitchedMSRState;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,6 +25,8 @@
 #ifndef _MON_ADDR_LAYOUT_H
 #define _MON_ADDR_LAYOUT_H

+#include "vm_pagetable.h"
+
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMKERNEL
@@ -37,6 +39,8 @@ typedef struct VMM64_AddrLayout {
    *All address are VPNs and all lengths are numPages
    */
   uint64 monBase;            // MONITOR_BASE_VPN
+   uint64 monL5Start;         // MON_PAGE_TABLE_L5_START
+   uint32 monL5Len;           // MON_PAGE_TABLE_L5_LEN
   uint64 monL4Start;         // MON_PAGE_TABLE_L4_START
   uint32 monL4Len;           // MON_PAGE_TABLE_L4_LEN
   uint64 monL3Start;         // MON_PAGE_TABLE_L3_START
@@ -51,7 +55,21 @@ typedef struct VMM64_AddrLayout {
 } VMM64_AddrLayout;
 #pragma pack(pop)

-#define VMM_MONAS_FIRST_L4OFF   1
-#define VMM_MONAS_LAST_L4OFF  127
+#define VMM_MONAS_4LP_FIRST_L4OFF   1
+#define VMM_MONAS_4LP_LAST_L4OFF  130

+#define VMM_MONAS_5LP_FIRST_L5OFF   1
+#define VMM_MONAS_5LP_LAST_L5OFF   34
+
+static INLINE PT_Level
+MonAS_GetPagingLevel(void)
+{
+   return PT_LEVEL_4;
+}
+
+static INLINE Bool
+MonAS_Uses5LevelPaging(void)
+{
+   return MonAS_GetPagingLevel() == PT_LEVEL_5;
+}
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,8 +31,8 @@
 * contains code and data statically built, empty but allocated space, shared
 * and run-time initialized content.  The monitor loader header regularizes
 * encoding of address space information, allowing a common representation and
- * common code to be re-used for different contexts (vmmon for hosted, vmkernel
- * for ESX, and the VMX while deprivileging of bootstrap is ongoing).
+ * common code to be re-used for different contexts (vmmon for hosted
+ * and vmkernel for ESX).
 *
 * The Header
 * ==========
@@ -87,7 +87,6 @@
 * MonLoaderCallout_Init(): Initialize MonLoader callouts.
 * MonLoaderCallout_GetSharedHostPage(subIdx, page): Get shared host page MPN.
 * MonLoaderCallout_GetSharedUserPage(subIdx, page): Get shared user page MPN.
- * MonLoaderCallout_SetEntrypoint(): Sets the entry code/stack for execution.
 *
 * Building vs Importing
 * =====================
@@ -119,21 +118,54 @@
 #define _MON_LOADER

 #include "vm_basic_types.h"
-#include "x86paging_common.h"
-#include "x86types.h" /* PT_L1E */
+#include "vm_pagetable.h"
 #include "vcpuid.h"   /* Vcpuid */

+#if defined VM_X86_64
+#include "x86paging_64.h"
+#elif defined VM_ARM_64
+#include "arm64_vmsa.h"
+#endif
+
 #define ML_NAME_MAX 16

 /* ML perms are simple and abbreviated. */

+#if defined VM_X86_64
+
 #define ML_PERM_RWX (PTE_P | PTE_RW)
 #define ML_PERM_RW  (PTE_P | PTE_RW | PTE_NX)
 #define ML_PERM_RO  (PTE_P |          PTE_NX)
 #define ML_PERM_RX   PTE_P

+#define ML_PERM_TBL   ML_PERM_RWX
 #define ML_PERM_MASK (PTE_P| PTE_RW | PTE_NX | PTE_US)

+#define ML_PERM_PRESENT(_flags)     (((_flags) & PTE_P) != 0)
+#define ML_PERM_WRITEABLE(_flags)   (((_flags) & PTE_RW) != 0)
+
+#define ML_PTE_2_PFN(_pte)          LM_PTE_2_PFN(_pte)
+
+#elif defined VM_ARM_64
+
+#define _ML_PERM_COMMON (ARM_PTE_BLOCK_AP(ARM_AP_PL0)   | ARM_PTE_BLOCK_AF | \
+                         ARM_PTE_BLOCK_SH(ARM_SH_OUTER) | ARM_PTE_BLOCK_L3_TYPE)
+#define ARM_PTE_BLOCK_AP_RO ARM_PTE_BLOCK_AP(ARM_AP_RO)
+
+#define ML_PERM_RW   (_ML_PERM_COMMON |                       ARM_PTE_BLOCK_XN)
+#define ML_PERM_RO   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+#define ML_PERM_RX   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO                   )
+
+#define ML_PERM_TBL   ML_PERM_RW
+#define ML_PERM_MASK (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+
+#define ML_PERM_PRESENT(_flags)   (((_flags) & ARM_PTE_VALID) != 0)
+#define ML_PERM_WRITEABLE(_flags) (((_flags) & ARM_PTE_BLOCK_AP_RO) == 0)
+
+#define ML_PTE_2_PFN(_pte)    (((_pte) & ARM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
+
+#endif
+
 #define ML_PERMS_MATCH(x,p) (((x) & ML_PERM_MASK) == ((p) & ML_PERM_MASK))

 #define LOADER_HEADER_MAGIC 0x8675309E98675309
@@ -150,10 +182,10 @@ typedef enum {
   ML_CONTENT_SHARE,         /* Share data from external source. */
 } MonLoaderContentType;

-#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? 4 : \
-                               x == ML_CONTENT_PAGETABLE_L3 ? 3 : \
-                               x == ML_CONTENT_PAGETABLE_L2 ? 2 : \
-                               x == ML_CONTENT_PAGETABLE_L1 ? 1 : \
+#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? PT_LEVEL_4 : \
+                               x == ML_CONTENT_PAGETABLE_L3 ? PT_LEVEL_3 : \
+                               x == ML_CONTENT_PAGETABLE_L2 ? PT_LEVEL_2 : \
+                               x == ML_CONTENT_PAGETABLE_L1 ? PT_LEVEL_1 : \
                               0)

 /* Sources of content, for pages not zeroed, unmapped or otherwise special. */
@@ -183,11 +215,14 @@ typedef struct {
   } blobSrc;
   uint64               bspOnly;           /* Process only on BSP. */
   uint64               subIndex;          /* Region ID for ML_CONTENT_COPY. */
-   uint64               procVmmon;         /* Processed by the vmmon on hosted. */
 } MonLoaderEntry;


-/* Packed for easy consumption by bootstrap-offsets.pl. */
+/*
+ * Packed for easy consumption by bootstrap-offsets.pl. If the contents of the
+ * MonLoaderHeader struct are changed then $HEADER_SIZE must be updated
+ * accordingly in bootstrap-offsets.pl.
+ */
 #pragma pack(push, 1)
 typedef struct MonLoaderHeader {
   uint64         magic;
@@ -199,6 +234,8 @@ typedef struct MonLoaderHeader {
   /* ss:rip */
   uint16         stackSelector;
   VA64           stackEntrypoint;
+   LPN64          monStartLPN;
+   LPN64          monEndLPN;
   MonLoaderEntry entries[];
 } MonLoaderHeader;
 #pragma pack(pop)
@@ -217,16 +254,13 @@ MPN  MonLoaderCallout_GetPageRoot(struct MonLoaderEnvContext *, Vcpuid);
 Bool MonLoaderCallout_GetPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                             Vcpuid, PT_L1E *);
 Bool MonLoaderCallout_ImportPage(struct MonLoaderEnvContext *, MPN, Vcpuid);
-Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **);
+Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **, unsigned);
 Bool MonLoaderCallout_MapMPNInPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                                  uint64, MPN, Vcpuid);
 MPN  MonLoaderCallout_GetSharedUserPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
 MPN  MonLoaderCallout_GetSharedHostPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
-Bool MonLoaderCallout_IsPrivileged(struct MonLoaderEnvContext *);
-Bool MonLoaderCallout_SetEntrypoint(struct MonLoaderEnvContext *, uint16, VA64,
-                                    uint16, VA64);
 MPN  MonLoaderCallout_GetBlobMpn(struct MonLoaderEnvContext *, uint64);

 typedef enum MonLoaderError {
@@ -237,7 +271,6 @@ typedef enum MonLoaderError {
   ML_ERROR_ARGS,
   ML_ERROR_CALLOUT_INIT,
   ML_ERROR_CALLOUT_COPY,
-   ML_ERROR_CALLOUT_ENTRYPOINT,
   ML_ERROR_CALLOUT_GETPTE,
   ML_ERROR_CALLOUT_MAPINPTE,
   ML_ERROR_CALLOUT_PAGEROOT_GET,
@@ -259,9 +292,14 @@ typedef enum MonLoaderError {
 } MonLoaderError;


-/* A subindex above shared area subindices for sharing of MonLoaderHeader. */
-#define MONLOADER_HEADER_IDX 6
-
+/*
+ * Values above shared area subindices for sharing of MonLoaderHeader
+ * and htSchedStateMap.
+ */
+#define MONLOADER_HEADER_IDX          6
+#define MONLOADER_HT_MAP_IDX          7
+#define MONLOADER_CROSS_PAGE_CODE_IDX 8
+#define MONLOADER_CROSS_PAGE_DATA_IDX 9

 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
                                 void *args, unsigned *line, Vcpuid *vcpu);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,23 +25,7 @@
 #ifndef _MONLOADER_LOG_H_
 #define _MONLOADER_LOG_H_

-#ifdef VMX86_VMX
-
-#define LOGLEVEL_MODULE main
-#include "loglevel_vmcore.h"
-/*
- * Redefine LOG() to match the vmkernel calling convention.
- */
-#undef LOG_BYNAME
-#undef LOG
-
-#define LOG_BYNAME(_mod, _min, _fmt, ...) \
-        (DOLOG_BYNAME(_mod, _min) ? Log(_fmt "\n", ## __VA_ARGS__) : (void) 0)
-
-#define LOG(_min, _fmt, ...) \
-        LOG_BYNAME(LOGLEVEL_MODULE, _min, _fmt, ## __VA_ARGS__)
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #define LOGLEVEL_MODULE MonLoader
 #include "log.h"
@@ -59,7 +43,7 @@ do {                                                 \
   }                                                 \
 } while (0)

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2007-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2007-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -60,20 +60,21 @@
 *     lengths, and this causes the 'monitor-modular-size' script to
 *     report differences in '.rodata'.
 *
- *   o To avoid differences in '.rodata', each __FILE__ is put into
- *     its own section.  The monitor's linker (not ld) will use the
- *     name of the section to recover the name of the source file.
+ *   o Each __FILE__ is put into its own .assert_pathname_##__FILE
+ *     section.  The monitor's runtime linker (not ld) then uses the name
+ *     of this section to recover the name of the corresponding source file.
 *
 *   o At run time, prior to loading, when our linker is creating an
 *     executable image of the monitor and extensions, all the file
 *     names are extracted from these sections, the '${VMTREE}' prefix
 *     is removed, and the resulting table of shortened file names is
- *     added to '.rodata'.
+ *     added to a non-loadable section called '.assert_pathnames' in the
+ *     VMX linker (not the bootstrap linker which does not need them).
 *
 *     Further, during linkage, each relocation to the original
 *     section containing the path name is modified so that the low
- *     16-bits contain an offset from '__vmm_pathnames_start' rather
- *     than the base of the original containing section.
+ *     16-bits contain an offset from the start of the .assert_pathnames
+ *     section rather than the base of the original containing section.
 *
 *     Only three types of relocations to the assertion strings are
 *     supported (32-bit PC-relative and 32-bit/64-bit absolute) because that
@@ -111,42 +112,12 @@ typedef struct Assert_Info {
   Assert_MonSrcLoc loc;
 } Assert_Info;

-/*
- * The portion of the __attribute__ line after __FILE__ is there so that
- * the .assert_pathname_* sections are not marked as ALLOC, since we only
- * need them in the vmx and do not need them loaded.
- */
-#define __VMM__FILE__SECTION \
-      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
-#define __VMM__FILE__ ({                                                \
-         static __VMM__FILE__SECTION const char file[] = "";            \
-         file;                                                          \
-      })
-
 #define ASSERT_MONSRCFILEOFFSET(loc)    LOWORD(loc)
 #define ASSERT_MONSRCLINE(loc)          HIWORD(loc)

 #define ASSERT_NULL_MONSRCLOC     0             // there is never line 0

-#ifdef VMM // {
-#ifdef MONITOR_APP // {
-
-#define ASSERT_MONSRCLOC() ASSERT_NULL_MONSRCLOC
-
-#else // } {
-
-#define ASSERT_MONSRCLOC() ({                                           \
-   const uintptr_t offset = ((__LINE__ << 16) +                         \
-                             (uintptr_t)__VMM__FILE__);                 \
-   const Assert_MonSrcLoc loc = offset;                                 \
-   loc;                                                                 \
-})
-
-extern const char __vmm_pathnames_start;
-#define ASSERT_MONSRCFILE(loc) \
-   (&__vmm_pathnames_start + ASSERT_MONSRCFILEOFFSET(loc))
-
-
+#if defined(VMM) && !defined(MONITOR_APP) // {
 /*
 * Assertion information is collected in a non-loadable section
 * named .assert_info.  Each record in this section contains
@@ -176,6 +147,18 @@ extern const char __vmm_pathnames_start;
 */

 #ifndef VMM_BOOTSTRAP
+/*
+ * The portion of the __attribute__ line after __FILE__ is there so that
+ * the .assert_pathname_* sections are not marked as ALLOC, since we only
+ * need them in the vmx and do not need them loaded.
+ */
+#define __VMM__FILE__SECTION \
+      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
+#define __VMM__FILE__ ({                                                \
+         static __VMM__FILE__SECTION const char file[] = "";            \
+         file;                                                          \
+      })
+
 #define ASSERT_RECORDINFO(assembly, assertType, bugNr)                   \
   __asm__ __volatile__(".pushsection .assert_info;"                     \
                        ".quad 0f;"                                      \
@@ -215,8 +198,6 @@ extern uint64 bsAssertRIP;
   ({COMPILER_MEM_BARRIER();                                             \
     ASSERT_RECORDINFO("", AssertType_##name##Bug, bug);                 \
     __builtin_trap();})
+#endif // VMM && !MONITOR_APP}

-#endif // MONITOR_APP }
-#endif // VMM }
-
-#endif
+#endif // _MON_ASSERT_H_
@@ -52,7 +52,7 @@ extern "C" {
 typedef struct PageListEntry {
   CompressedBPN cbpn;
   Bool voided;
-   uint8 _pad[2];
+   uint8 _pad[1];
 } PageListEntry;
 #pragma pack(pop) 

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -55,14 +55,17 @@ typedef enum PCIPassthruVectorIndex {
 } PCIPassthruVectorIndex;

 typedef enum PCIPassthru_IntrType {
-   PCIPASSTHRU_INTR_NONE   = 0x00,
-   PCIPASSTHRU_INTR_IOAPIC = 0x01,
-   PCIPASSTHRU_INTR_MSI    = 0x02,
-   PCIPASSTHRU_INTR_MSIX   = 0x04,
+   PCIPASSTHRU_INTR_NONE    = 0x00,
+   PCIPASSTHRU_INTR_IOAPIC  = 0x01,
+   PCIPASSTHRU_INTR_MSI     = 0x02,
+   PCIPASSTHRU_INTR_MSIX    = 0x04,
+   PCIPASSTHRU_INTR_MSI_INT = 0x08,
 } PCIPassthru_IntrType;

+#define PCIPASSTHRU_INTR_MSI_COMBINED                   \
+   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)
+
 typedef struct FPTIntrProxyInfo {
-   uint32            adapterIndex;
   uint32            vectorIndex;
   MonitorIdemAction actionID;
 } FPTIntrProxyInfo;
@@ -72,9 +75,15 @@ typedef struct UPTIntrProxyInfo {
   MonitorIdemAction actionID;
 } UPTIntrProxyInfo;

+typedef struct MPTIntrProxyInfo {
+   uint32            adapterIndex;
+   MonitorIdemAction actionID;
+} MPTIntrProxyInfo;
+
 typedef union PCIPassthru_IntrProxyInfo {
   FPTIntrProxyInfo fpt;
   UPTIntrProxyInfo upt;
+   MPTIntrProxyInfo mpt;
 } PCIPassthru_IntrProxyInfo;

 typedef enum PCIPassthru_ErrorType {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,12 @@
 *********************************************************/

 /*
- * perfctr_generic.h --
+ * perfctr.h --
 *
 */

-#ifndef _PERFCTR_GENERIC_H_
-#define _PERFCTR_GENERIC_H_
+#ifndef _PERFCTR_H_
+#define _PERFCTR_H_

 #define INCLUDE_ALLOW_USERLEVEL

@@ -33,7 +33,9 @@

 #include "includeCheck.h"
 #include "vm_basic_types.h"
+#include "perfctr_arch.h"

+#define PERF_EVENT_NAME_LEN                      64
 /*
 * nmiNo      -- vmm peer is not attempting to do nmi profiling this run
 * nmiYes     -- vmm peer is doing nmi profiling and nmis are currently enabled
@@ -69,4 +71,51 @@ typedef struct {
   uint8  _pad[3];
 } CrossProfShared;

-#endif // ifndef _PERFCTR_GENERIC_H_
+/*
+ * PerfCtr_Config --
+ *      Describes configuration for a single hardware performance counter
+ *
+ *      Since this is only used to record general performance counters, we
+ *      made the assumption in nmiProfiler.c that the type is GENERAL and
+ *      index is counter number of type GENERAL.
+ *
+ *      **** x86 ****
+ *      On AMD K8 and GH:
+ *      index:        Which perf ctr, 0 to 3.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On AMD with PerfCtrExtCore support:
+ *      index:        Which perf ctr, 0 to 5.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On Intel Core architecture:
+ *      <to be documented>
+ *
+ *      **** ARM ****
+ *      escrVal:     Value placed in PMEVTYPER<n>_EL0 to configure event counter
+ *      index:       Index of the event counter.
+ */
+typedef struct PerfCtr_Config {
+   uint64 escrVal;
+   uint32 index;
+   uint32 periodMean;
+
+   /*
+    * Random number (whose absolute value is capped at
+    * periodJitterMask) is used to randomize sampling interval.
+    */
+   uint32  periodJitterMask;
+   uint32  seed;    // seed is used to compute next random number
+   uint16  config;
+   Bool    valid;
+   PERFCTR_CONFIG_ARCH_FIELDS
+} PerfCtr_Config;
+
+
+#endif // ifndef _PERFCTR_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,17 @@
 *********************************************************/

 /*
- * x86perfctr.h --
+ * perfctr_arch.h --
 *
+ *      Performance counters (x64 specific).
 */

-#ifndef _X86PERFCTR_H_
-#define _X86PERFCTR_H_
+#ifndef _PERFCTR_ARCH_H
+#define _PERFCTR_ARCH_H
+
+#ifndef _PERFCTR_H_
+#error "This file can only be included by perfctr.h"
+#endif

 #define INCLUDE_ALLOW_USERLEVEL

@@ -30,25 +35,20 @@
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
-
 #include "includeCheck.h"
+
 #include "vm_asm.h"
 #include "x86cpuid_asm.h"
-#include "perfctr_generic.h"

-#define PERFCTR_PENTIUM4_NUM_COUNTERS            18
-#define PERFCTR_PENTIUM4_NUM_COUNTERS_WITH_L3    26
-#define PERFCTR_AMD_NUM_COUNTERS                 4 
+#define PERFCTR_AMD_NUM_COUNTERS                 4
 #define PERFCTR_AMD_EXT_NUM_COUNTERS             6
 #define PERFCTR_P6_NUM_COUNTERS                  2
 #define PERFCTR_NEHALEM_NUM_GEN_COUNTERS         4
 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS       3
 #define PERFCTR_SANDYBRIDGE_NUM_GEN_COUNTERS     8 /* When HT is disabled */
-#define PERFCTR_CORE_NUM_ARCH_EVENTS             7
-#define PERFCTR_CORE_NUM_FIXED_COUNTERS          3
-#define PERFCTR_PENTIUM4_VAL_MASK                0xffffffffffLL
+#define PERFCTR_CORE_NUM_ARCH_EVENTS             8
+#define PERFCTR_CORE_NUM_FIXED_COUNTERS          4
 #define PERFCTR_AMD_VAL_MASK                     0xffffffffffffLL
-#define PERF_EVENT_NAME_LEN                      64
 /*
 * Even though the performance counters in P6 are 40 bits,
 * we can only write to the lower 32 bits.  Bit 31 is
@@ -59,7 +59,7 @@

 /*
 * Performance counter width is determined at runtime in CORE.
- * But the writables bits are fixed. we can only write to the 
+ * But the writables bits are fixed. we can only write to the
 * lower 32 bits.  Bit 31 is used to sign-extend the upper 8 bits.
 */
 #define PERFCTR_CORE_WRITE_MASK                  0xffffffff
@@ -124,7 +124,7 @@
 #define PERFCTR_AMD_CPU_CLK_UNHALTED                           0x76

 /* AMD Load/Store unit events */
-#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20 
+#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20
 #define PERFCTR_AMD_LS_BUFFER2_FULL                            0x23

 /*
@@ -133,13 +133,19 @@
 */
 #define PERFCTR_AMD_SMI_COUNT                                  0x2b

-/* AMD Data Cache Events */
+/*
+ * AMD Data Cache Events
+ * Family <  17H: 0x45 - Unified TLB hit
+ *                0x46 - Unified TLB miss
+ * Family >= 17H: 0x45 - L1 DTLB miss (L2 DTLB hit or miss)
+ *                0x46 - Tablewalker
+ */
 #define PERFCTR_AMD_DATA_CACHE_ACCESSES                        0x40
 #define PERFCTR_AMD_DATA_CACHE_MISSES                          0x41
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_L2_OR_SYSTEM       0x42
 #define PERFCTR_AMD_DATA_CACHE_REFILLS_FROM_SYSTEM             0x43
 #define PERFCTR_AMD_DATA_CACHE_LINES_EVICTED                   0x44
-#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT               0x45
+#define PERFCTR_AMD_L1_DTLB_MISS_AND_L2_DTLB_HIT_OR_MISS       0x45
 #define PERFCTR_AMD_L1_DTLB_AND_L2_DTLB_MISS                   0x46
 #define PERFCTR_AMD_MISALIGNED_ACCESSES                        0x47
 #define PERFCTR_AMD_PREFETCH_INSTRS_DISPATCHED                 0x4b
@@ -198,8 +204,8 @@

 #define PERFCTR_AMD_MEM_CTRL_PAGE_TABLE_OVERFLOWS                   0xe1
 #define PERFCTR_AMD_CPU_IO_REQUESTS_TO_MEMORY_IO                    0xe9
-#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec 
- 
+#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec
+
 /* AMD HyperTransport Interface Events */

 #define PERFCTR_AMD_HT_L0_TX_BW                                     0xf6
@@ -219,9 +225,9 @@
 */

 /* P6 Data Cache Unit (DCU) */
-#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043  
+#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043
 #define PERFCTR_P6_DCU_LINES_IN                  0x00000045
-#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046 
+#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046
 #define PERFCTR_P6_DCU_MISS_OUTSTANDING          0x00000048

 /* P6 Instruction Fetch Unit (IFU) */
@@ -379,7 +385,7 @@
 #define PERFCTR_P6_APIC_INTR                     0x00100000
 #define PERFCTR_P6_ENABLE                        0x00400000
 #define PERFCTR_P6_INVERT_COUNTER_MASK           0x00800000
-#define PERFCTR_P6_COUNTER_MASK                  0xff000000        
+#define PERFCTR_P6_COUNTER_MASK                  0xff000000
 #define PERFCTR_P6_COUNTER_MASK_SHIFT            24
 #define PERFCTR_P6_SHIFT_BY_UNITMASK(e)          (e << 8)

@@ -436,6 +442,7 @@
 #define PERFCTR_CORE_KERNEL_MODE                    PERFCTR_CPU_KERNEL_MODE
 #define PERFCTR_CORE_APIC_INTR                      PERFCTR_CPU_APIC_INTR
 #define PERFCTR_CORE_ENABLE                         PERFCTR_CPU_ENABLE
+/* AnyThread Counting deprecated since PMU v5. */
 #define PERFCTR_CORE_ANYTHREAD                      0x00200000
 #define PERFCTR_CORE_IN_TX                          (CONST64U(1) << 32)
 #define PERFCTR_CORE_IN_TXCP                        (CONST64U(1) << 33)
@@ -463,6 +470,7 @@
 #define PERFCTR_CORE_INST_RETIRED                0xc0
 /* bus cycles */
 #define PERFCTR_CORE_UNHALTED_REF_CYCLES         (0x3c | (0x01 << 8))
+#define PERFCTR_CORE_TOPDOWN_SLOTS               (0xa4 | (0x01 << 8))

 /*
 * See Tables 30-2, 30-4 of the
@@ -558,132 +566,20 @@
 #define PERFCTR_NEHALEM_UNCORE_ENABLE            0x400000
 #define PERFCTR_NEHALEM_UNCORE_L3_LINES_IN       (0x0a | (0x0f << 8))

-
-/*
- * ----------------------------------------------------------------------
- *
- * Pentium 4
- *
- * ----------------------------------------------------------------------
- */
-
-/*
- * Pentium 4 Counter Configuration Control Register flags and fields
- */
-#define PERFCTR_PENTIUM4_CCCR_ENABLE                   0x00001000
-#define PERFCTR_PENTIUM4_CCCR_SET_ESCR(c, e)           ((c) |= (((e) & 0x7) << 13))
-#define PERFCTR_PENTIUM4_CCCR_REQRSVD                  0x00030000 // always set these bits
-#define PERFCTR_PENTIUM4_CCCR_COMPARE                  0x00040000
-#define PERFCTR_PENTIUM4_CCCR_COMPLEMENT               0x00080000
-#define PERFCTR_PENTIUM4_CCCR_SET_THRESHOLD(c, e)      ((c) |= (((e) & 0xf) << 20))
-#define PERFCTR_PENTIUM4_CCCR_EDGE                     0x01000000
-#define PERFCTR_PENTIUM4_CCCR_FORCE_OVF                0x02000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T0               0x04000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T1               0x08000000
-#define PERFCTR_PENTIUM4_CCCR_CASCADE                  0x40000000
-#define PERFCTR_PENTIUM4_CCCR_OVF                      0x80000000
-
-#define PERFCTR_PENTIUM4_COUNTER_BASEADDR              0x00000300
-#define PERFCTR_PENTIUM4_CCCR_BASE_ADDR                0x00000360
-
-#define PERFCTR_PENTIUM4_CCCR_THRESHOLD(e)             ((e & 0xf) << 20)
-#define PERFCTR_PENTIUM4_OPT_EDGE_DETECT (PERFCTR_PENTIUM4_CCCR_COMPARE | PERFCTR_PENTIUM4_CCCR_EDGE)
-
-/*
- * Pentium 4 Event Selection Control Register flags
- */
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T0             0x00000004
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T0           0x00000008
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T1             0x00000001
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T1           0x00000002
-#define PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK(e)          (e << 9)
-#define PENTIUM4_EVTSEL(e)                             (e << 25)
-
-/*
- * Event definitions for Pentium4 Family Processors. 
- * There are many more of these possible.
- * See Appendix A.1 in Volume 3 of the IA32 manual.
- */
-
-#define PERFCTR_PENTIUM4_EVT_INSTR_RETIRED              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_INSTR_COMPLETED            PENTIUM4_EVTSEL(0x07)
-#define PERFCTR_PENTIUM4_EVT_BSQ_CACHE_REFERENCE        PENTIUM4_EVTSEL(0x0c)
-#define PERFCTR_PENTIUM4_EVT_ITLB_REFERENCE             PENTIUM4_EVTSEL(0x18)
-#define PERFCTR_PENTIUM4_EVT_PAGE_WALK_TYPE             PENTIUM4_EVTSEL(0x01)   
-#define PERFCTR_PENTIUM4_EVT_BPU_FETCH_REQUEST          PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_BRANCH_RETIRED             PENTIUM4_EVTSEL(0x06)  
-#define PERFCTR_PENTIUM4_EVT_MISPRED_BRANCH_RETIRED     PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_MACHINE_CLEAR              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_MEMORY_CANCEL              PENTIUM4_EVTSEL(0x02) 
-#define PERFCTR_PENTIUM4_GLOBAL_POWER_EVENTS            PENTIUM4_EVTSEL(0x13)
-
-/*
- * Pentium4 doesn't have an event for clock cycles, but Intel
- * outlines a method in the IA32 manual, vol 3, sec 14.9.9,
- * for measuring cycles which involves this event strategy.
- */
-
-#define PERFCTR_PENTIUM4_EVT_CLK_CYCLES \
-   PENTIUM4_EVTSEL(0x2) | \
-   PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK (0x01 | 0x02 | 0x04 | 0x08)
-
-
-/*
- * PerfCtr_Config --
- *      Describes configuration for a single hardware performance counter
- *
- *      Since this is only used to record general performance counters, we
- *      made the assumption in nmiProfiler.c  that the type is GENERAL and
- *      index is counter number of type GENERAL.
- *      
- *      
- *      On AMD K8 and GH:
- *      index:        Which perf ctr, 0 to 3.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- *
- *      On AMD with PerfCtrExtCore support:
- *      index:        Which perf ctr, 0 to 5.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- * 
- *      No field should be greater than 32-bit as it is shared with monitor.
- *
- *      On Intel Core architecture:
- *      <to be documented>
- */
-typedef struct PerfCtr_Config {
-   uint64 escrVal;
-   uint32 index;
-   uint32 addr;
-   uint32 escrAddr;
-   uint32 resetHi;
-   uint32 periodMean;
-
-   /*
-    * Random number (whose absolute value is capped at
-    * periodJitterMask) is used to randomize sampling interval.
-    */
-   uint32  periodJitterMask;
-   uint32  seed;    // seed is used to compute next random number
-   uint16  config;
-   Bool    valid;
-   Bool    pebsEnabled;
-} PerfCtr_Config;
-
+#define PERFCTR_CONFIG_ARCH_FIELDS \
+   uint32 addr;      \
+   uint32 escrAddr;  \
+   uint32 resetHi;   \
+   Bool pebsEnabled; \

 /*
 * Program/reprogram event reg(s) associated w/perfctrs & start or stop perfctrs
 */
 static INLINE void
-PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
-                   uint32 escrVal)       // IN: event register value
+PerfCtr_WriteEvtSel(uint32 addr,       // IN: Event register to write
+                    uint32 escrVal)    // IN: event register value
 {
-   __SET_MSR(ctr->escrAddr, escrVal);
+   X86MSR_SetMSR(addr, escrVal);
 }


@@ -691,11 +587,10 @@ PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
 * Set/reset performance counters to engender desired period before overflow
 */
 static INLINE void
-PerfCtrWriteCounter(PerfCtr_Config *ctr,   // IN: counter to write
-                    uint32 valueLo,        // IN: low 32 bits of value to write
-                    uint32 valueHi)        // IN: high 32 bits of value to write
+PerfCtr_WriteCounter(uint32 addr,   // IN: counter to write
+                     uint64 value)  // IN: value to write
 {
-   WRMSR(ctr->addr, valueLo, valueHi);
+   X86MSR_SetMSR(addr, value);
 }


@@ -814,8 +709,9 @@ PerfCtr_PEBSAvailable(void)
   CPUIDRegs regs;
   __GET_CPUID(0, &regs);
   if (CPUID_IsVendorIntel(&regs) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_EMON_AVAILABLE) != 0) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_PEBS_UNAVAILABLE) == 0)) {
+       (X86MSR_GetMSR(MSR_MISC_ENABLE) &
+        (MSR_MISC_ENABLE_EMON_AVAILABLE | MSR_MISC_ENABLE_PEBS_UNAVAILABLE))
+       == MSR_MISC_ENABLE_EMON_AVAILABLE) {
      CPUIDRegs hvendor;

      /*
@@ -827,7 +723,7 @@ PerfCtr_PEBSAvailable(void)
             !CPUID_IsRawVendor(&hvendor,
                                CPUID_HYPERV_HYPERVISOR_VENDOR_STRING);
   }
-   return FALSE; 
+   return FALSE;
 }

 /*
@@ -858,162 +754,4 @@ PerfCtr_PTAvailable(void)
   return FALSE;
 }

-/* The following are taken from the Intel Architecture Manual,
- * Book 3, Table 14-4. 
- */ 
-
-/* --------- BEGIN INTEL DEFINES ------------------------ */
-
-#define PENTIUM4_MSR_BPU_COUNTER0_IDX           0
-#define PENTIUM4_MSR_BPU_COUNTER0_ADDR          0x300
-#define PENTIUM4_MSR_BPU_CCCR0                  0x360
-#define PENTIUM4_MSR_BPU_COUNTER1_IDX           1
-#define PENTIUM4_MSR_BPU_COUNTER1_ADDR          0x301
-#define PENTIUM4_MSR_BPU_CCCR1                  0x361
-#define PENTIUM4_MSR_BPU_COUNTER2_IDX           2
-#define PENTIUM4_MSR_BPU_COUNTER2_ADDR          0x302
-#define PENTIUM4_MSR_BPU_CCCR2                  0x362
-#define PENTIUM4_MSR_BPU_COUNTER3_IDX           3
-#define PENTIUM4_MSR_BPU_COUNTER3_ADDR          0x303
-#define PENTIUM4_MSR_BPU_CCCR3                  0x363
-#define PENTIUM4_MSR_MS_COUNTER0_IDX            4
-#define PENTIUM4_MSR_MS_COUNTER0_ADDR           0x304
-#define PENTIUM4_MSR_MS_CCCR0                   0x364
-#define PENTIUM4_MSR_MS_COUNTER1_IDX            5
-#define PENTIUM4_MSR_MS_COUNTER1_ADDR           0x305
-#define PENTIUM4_MSR_MS_CCCR1                   0x365
-#define PENTIUM4_MSR_MS_COUNTER2_IDX            6
-#define PENTIUM4_MSR_MS_COUNTER2_ADDR           0x306
-#define PENTIUM4_MSR_MS_CCCR2                   0x366
-#define PENTIUM4_MSR_MS_COUNTER3_IDX            7
-#define PENTIUM4_MSR_MS_COUNTER3_ADDR           0x307
-#define PENTIUM4_MSR_MS_CCCR3                   0x367
-#define PENTIUM4_MSR_FLAME_COUNTER0_IDX         8
-#define PENTIUM4_MSR_FLAME_COUNTER0_ADDR        0x308
-#define PENTIUM4_MSR_FLAME_CCCR0                0x368
-#define PENTIUM4_MSR_FLAME_COUNTER1_IDX         9
-#define PENTIUM4_MSR_FLAME_COUNTER1_ADDR        0x309
-#define PENTIUM4_MSR_FLAME_CCCR1                0x369
-#define PENTIUM4_MSR_FLAME_COUNTER2_IDX         10
-#define PENTIUM4_MSR_FLAME_COUNTER2_ADDR        0x30A
-#define PENTIUM4_MSR_FLAME_CCCR2                0x36A
-#define PENTIUM4_MSR_FLAME_COUNTER3_IDX         11
-#define PENTIUM4_MSR_FLAME_COUNTER3_ADDR        0x30B
-#define PENTIUM4_MSR_FLAME_CCCR3                0x36B
-#define PENTIUM4_MSR_IQ_COUNTER0_IDX            12
-#define PENTIUM4_MSR_IQ_COUNTER0_ADDR           0x30C
-#define PENTIUM4_MSR_IQ_CCCR0                   0x36C
-#define PENTIUM4_MSR_IQ_COUNTER1_IDX            13
-#define PENTIUM4_MSR_IQ_COUNTER1_ADDR           0x30D
-#define PENTIUM4_MSR_IQ_CCCR1                   0x36D
-#define PENTIUM4_MSR_IQ_COUNTER2_IDX            14
-#define PENTIUM4_MSR_IQ_COUNTER2_ADDR           0x30E
-#define PENTIUM4_MSR_IQ_CCCR2                   0x36E
-#define PENTIUM4_MSR_IQ_COUNTER3_IDX            15
-#define PENTIUM4_MSR_IQ_COUNTER3_ADDR           0x30F
-#define PENTIUM4_MSR_IQ_CCCR3                   0x36F
-#define PENTIUM4_MSR_IQ_COUNTER4_IDX            16
-#define PENTIUM4_MSR_IQ_COUNTER4_ADDR           0x310
-#define PENTIUM4_MSR_IQ_CCCR4                   0x370
-#define PENTIUM4_MSR_IQ_COUNTER5_IDX            17
-#define PENTIUM4_MSR_IQ_COUNTER5_ADDR           0x311
-#define PENTIUM4_MSR_IQ_CCCR5                   0x371
-
-#define PENTIUM4_MSR_ALF_ESCR0_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR0_ADDR             0x3CA
-#define PENTIUM4_MSR_ALF_ESCR1_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR1_ADDR             0x3CB
-#define PENTIUM4_MSR_BPU_ESCR0_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR0_ADDR             0x3B2
-#define PENTIUM4_MSR_BPU_ESCR1_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR1_ADDR             0x3B3
-#define PENTIUM4_MSR_BSU_ESCR0_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR0_ADDR             0x3A0
-#define PENTIUM4_MSR_BSU_ESCR1_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR1_ADDR             0x3A1
-#define PENTIUM4_MSR_CRU_ESCR0_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR0_ADDR             0x3B8
-#define PENTIUM4_MSR_CRU_ESCR1_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR1_ADDR             0x3B9
-#define PENTIUM4_MSR_CRU_ESCR2_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR2_ADDR             0x3CC
-#define PENTIUM4_MSR_CRU_ESCR3_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR3_ADDR             0x3CD
-#define PENTIUM4_MSR_CRU_ESCR4_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR4_ADDR             0x3E0
-#define PENTIUM4_MSR_CRU_ESCR5_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR5_ADDR             0x3E1
-#define PENTIUM4_MSR_DAC_ESCR0_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR0_ADDR             0x3A8
-#define PENTIUM4_MSR_DAC_ESCR1_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR1_ADDR             0x3A9
-#define PENTIUM4_MSR_FIRM_ESCR0_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR0_ADDR            0x3A4
-#define PENTIUM4_MSR_FIRM_ESCR1_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR1_ADDR            0x3A5
-#define PENTIUM4_MSR_FLAME_ESCR0_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR0_ADDR           0x3A6
-#define PENTIUM4_MSR_FLAME_ESCR1_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR1_ADDR           0x3A7
-#define PENTIUM4_MSR_FSB_ESCR0_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR0_ADDR             0x3A2
-#define PENTIUM4_MSR_FSB_ESCR1_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR1_ADDR             0x3A3
-#define PENTIUM4_MSR_IQ_ESCR0_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR0_ADDR              0x3BA
-#define PENTIUM4_MSR_IQ_ESCR1_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR1_ADDR              0x3BB
-#define PENTIUM4_MSR_IS_ESCR0_IDX               1
-#define PENTIUM4_MSR_IS_ESCR0_ADDR              0x3B4
-#define PENTIUM4_MSR_IS_ESCR1_IDX               1
-#define PENTIUM4_MSR_IS_ESCR1_ADDR              0x3B5
-#define PENTIUM4_MSR_ITLB_ESCR0_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR0_ADDR            0x3B6
-#define PENTIUM4_MSR_ITLB_ESCR1_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR1_ADDR            0x3B7
-#define PENTIUM4_MSR_IX_ESCR0_IDX               5
-#define PENTIUM4_MSR_IX_ESCR0_ADDR              0x3C8
-#define PENTIUM4_MSR_IX_ESCR1_IDX               5
-#define PENTIUM4_MSR_IX_ESCR1_ADDR              0x3C9
-#define PENTIUM4_MSR_MOB_ESCR0_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR0_ADDR             0x3AA
-#define PENTIUM4_MSR_MOB_ESCR1_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR1_ADDR             0x3AB
-#define PENTIUM4_MSR_MS_ESCR0_IDX               0
-#define PENTIUM4_MSR_MS_ESCR0_ADDR              0x3C0
-#define PENTIUM4_MSR_MS_ESCR1_IDX               0
-#define PENTIUM4_MSR_MS_ESCR1_ADDR              0x3C1
-#define PENTIUM4_MSR_PMH_ESCR0_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR0_ADDR             0x3AC
-#define PENTIUM4_MSR_PMH_ESCR1_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR1_ADDR             0x3AD
-#define PENTIUM4_MSR_RAT_ESCR0_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR0_ADDR             0x3BC
-#define PENTIUM4_MSR_RAT_ESCR1_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR1_ADDR             0x3BD
-#define PENTIUM4_MSR_SAAT_ESCR0_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR0_ADDR            0x3AE
-#define PENTIUM4_MSR_SAAT_ESCR1_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR1_ADDR            0x3AF
-#define PENTIUM4_MSR_SSU_ESCR0_IDX              3
-#define PENTIUM4_MSR_SSU_ESCR0_ADDR             0x3BE
-#define PENTIUM4_MSR_TBPU_ESCR0_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR0_ADDR            0x3C2
-#define PENTIUM4_MSR_TBPU_ESCR1_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR1_ADDR            0x3C3
-#define PENTIUM4_MSR_TC_ESCR0_IDX               1
-#define PENTIUM4_MSR_TC_ESCR0_ADDR              0x3C4
-#define PENTIUM4_MSR_TC_ESCR1_IDX               1
-#define PENTIUM4_MSR_TC_ESCR1_ADDR              0x3C5
-#define PENTIUM4_MSR_U2L_ESCR0_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR0_ADDR             0x3B0
-#define PENTIUM4_MSR_U2L_ESCR1_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR1_ADDR             0x3B1
-
-#define PENTIUM4_MIN_ESCR_ADDR                  0x3A0
-#define PENTIUM4_MAX_ESCR_ADDR                  0x3E1
-#define PENTIUM4_NUM_ESCR_ADDRS                 (PENTIUM4_MAX_ESCR_ADDR - PENTIUM4_MIN_ESCR_ADDR + 1)
-
-/* -------- END INTEL DEFINES ------------------------- */
-
-#endif // ifndef _X86PERFCTR_H_ 
+#endif // _PERFCTR_ARCH_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -125,7 +125,7 @@ PTSC_MHz(void)

 #if defined(VM_X86_64) || defined(VM_ARM_64)

-/* 
+/*
 * Conversions to/from cycles.  Note that the conversions operate on
 * signed values, so be careful when taking the difference of two
 * VmAbsoluteTS (which is unsigned) that that value is not out of range
@@ -173,35 +173,6 @@ int64 PTSC_CyclesToUS(VmRelativeTS ts);
 */
 #include "user_layout.h"

-#ifdef VM_ARM_64
-static INLINE VmAbsoluteTS
-PTSC_Get(void)
-{
-   extern __thread User_ThreadData vmkUserTdata;
-
-   register RateConv_Params params;
-   register uint64 pseudoTSC;
-
-   /*
-    * On ARM64 the generic timer guarantees that the counters are synchronous
-    * and independent of CPU frequency.
-    */
-   params = vmkUserTdata.pseudoTSCConv;
-   pseudoTSC = RDTSC();
-
-   /*
-    *  assert that conversion rate is `identity'.
-    * we'll have to revisit that when/if this assumption changes
-    * on some platform
-    */
-   ASSERT((params.mult == 1 && params.shift == 0) ||
-          (params.mult == 0x80000000 && params.shift == 31));
-
-   pseudoTSC += params.add;
-
-   return pseudoTSC;
-}
-#else
 static INLINE VmAbsoluteTS
 PTSC_Get(void)
 {
@@ -215,7 +186,7 @@ PTSC_Get(void)
   ASSERT((int64)ptsc >= 0);
   return ptsc;
 }
-#endif
+
 #else

 /*
@@ -226,13 +197,13 @@ VmAbsoluteTS PTSC_Get(void);

 #endif

-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HasSynchronizedTSCs --
 *
 *      Returns TRUE iff the platform TSCs are known to be synchronized.
- * 
+ *
 *-----------------------------------------------------------------------------
 */

@@ -243,7 +214,7 @@ PTSC_HasSynchronizedTSCs(void)
 }


-/* 
+/*
 *-----------------------------------------------------------------------------
 *
 * PTSC_HostAdjustedTSCs --
@@ -260,4 +231,37 @@ PTSC_HostAdjustedTSCs(void)
   return ptscInfo.hwTSCsAdjusted;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PTSC_AdvanceTimer --
+ *
+ *      Advance '*deadline' in 'period' increments such that it is
+ *      greater than 'now'.  Return the number of ticks incremented.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE uint64
+PTSC_AdvanceTimer(VmAbsoluteTS now,
+                  VmIntervalTS period,
+                  VmAbsoluteTS *deadline)
+{
+   VmAbsoluteTS d = *deadline;
+   if (now >= d) {
+      uint64 count = 1;
+      d += period;
+      if (UNLIKELY(now >= d)) {
+         uint64 t = (now - d) / period + 1;
+         VmIntervalTS diff = t * period;
+         count += t;
+         d += diff;
+      }
+      *deadline = d;
+      return count;
+   }
+   return 0;
+}
+
 #endif /* ifndef _PTSC_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,7 +97,7 @@ void RateConv_LogRatio(const char *prefix,
 static INLINE uint64
 RateConv_Unsigned(const RateConv_Params *conv, uint64 x)
 {
-   return Mul64x3264(x, conv->mult, conv->shift) + conv->add;
+   return Mul64x3264(x, conv->mult, conv->shift) + (uint64)conv->add;
 }


@@ -0,0 +1,176 @@
+/*********************************************************
+ * Copyright (C) 2018-2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * segs.h --
+ *
+ *      Describes segment descriptors and structures containing them for both
+ *      the monitor (in hosted and ESX) and the vmkernel.
+ */
+
+#ifndef _SEGS_H_
+#define _SEGS_H_
+
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "x86types.h"
+#include "x86segdescrs.h"
+#include "x86sel.h"
+#include "addrlayout.h"
+
+
+/*
+ * Segment map of the monitor.
+ *
+ * The GDT and Task State Segment reside consecutively on one page.  The
+ * monitor segments are placed at the end of the GDT.  The vmkernel can use all
+ * lower-numbered segments for user-mode as well as higher-numbered segments
+ * (though the vmkernel should not use monitor-private segments).  The high
+ * segment placement ensures that there is no selector-overlap with hosted
+ * kernel segments (the hosted world switch code can be a bit faster then, as
+ * it can use a single cross GDT).
+ */
+
+#define GDT_SIZE              (sizeof(Descriptor) * NUM_VALID_SEGMENTS)
+#define GDT_LIMIT             (GDT_SIZE - 1)
+
+#define IRB_SIZE              32 /* Interrupt redirection bitmap. */
+#define TSS_SIZE              (sizeof(Task64) + IRB_SIZE)
+
+#define GDT_START_VA          (VPN_2_VA(GDT_AND_TASK_START))
+#define TASK_START_VA         (VPN_2_VA(GDT_AND_TASK_START) + GDT_SIZE)
+
+/*
+ * vmkBoot uses some of the lower-numbered segments, as do host kernels on
+ * hosted.  User segments could start earlier than 32; see bug #1904257.
+ * Descriptor is used to describe boot, user and kernel segments.
+ * Descriptor64 (twice as large) is used for task segments.
+ */
+#define NUM_BOOT_SEGMENTS     32
+#define NUM_USER_SEGMENTS     (FIRST_SYSTEM_SEGMENT - FIRST_USER_SEGMENT)
+#define NUM_SYSTEM_SEGMENTS   2
+#define NUM_TASK_SEGMENTS     2
+
+#define FIRST_USER_SEGMENT    NUM_BOOT_SEGMENTS
+#define FIRST_SYSTEM_SEGMENT  (PAGE_SIZE / sizeof(Descriptor) - \
+                               NUM_SYSTEM_SEGMENTS            - \
+                               NUM_TASK_SEGMENTS * 2          - \
+                               TSS_SIZE / sizeof(Descriptor))
+
+#define GDT_USER_TLS_MIN      USER_TLS_1_SEGMENT
+#define GDT_USER_TLS_MAX      USER_TLS_3_SEGMENT
+#define USER_TLS_COUNT        ((USER_TLS_3_SEGMENT - USER_TLS_1_SEGMENT) + 1)
+
+#define FOREACH_USER_TLS_INDEX(_i)              \
+   {                                            \
+      unsigned _i;                              \
+      for (_i = 0; _i < USER_TLS_COUNT; _i++) { \
+
+#define FOREACH_USER_TLS_INDEX_DONE             \
+      }                                         \
+   }
+
+#define NULL_LDTR             0
+
+/*
+ * The descriptor after SYSTEM_CODE_SEGMENT (loaded into %cs) must be
+ * appropriate for %ss because of the syscall instruction for 64-bit
+ * user worlds.  Thus SYSTEM_DATA_SEGMENT is directly after it.
+ */
+typedef enum VmwSegs {
+   NULL_SEGMENT             = 0,
+   /* (... reserved for host operating system or vmkBoot segments). */
+
+   USER32_CODE_SEGMENT      = FIRST_USER_SEGMENT,
+   USER_DATA_SEGMENT,
+   USER64_SYSRET_SEGMENT,
+   USER64_STACK_SEGMENT,
+   USER64_CODE_SEGMENT,
+   USER_TLS_1_SEGMENT,
+   USER_TLS_2_SEGMENT,
+   USER_TLS_3_SEGMENT,
+
+   AFTER_LAST_USER_SEGMENT,
+
+   SYSTEM_CODE_SEGMENT      = FIRST_SYSTEM_SEGMENT,
+   SYSTEM_DATA_SEGMENT,
+   VMKERNEL_TASK_SEGMENT,
+   VMKERNEL_TASK_SEGMENT_HI,
+   MONITOR_TASK_SEGMENT,
+   MONITOR_TASK_SEGMENT_HI,
+
+   NUM_VALID_SEGMENTS
+} VmwSegs;
+
+#define GDT_SYSTEM_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_SYSTEM_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 0)
+#define GDT_USER_SEL(x) MAKE_SELECTOR(x##_SEGMENT, SELECTOR_GDT, 3)
+#define GDT_USER_SEL_UNCHECKED(x) \
+   MAKE_SELECTOR_UNCHECKED(x##_SEGMENT, SELECTOR_GDT, 3)
+
+/* Selectors used statically in code or in assembly must be unchecked. */
+#define SYSTEM_NULL_SELECTOR    GDT_SYSTEM_SEL(NULL)
+#define USER32_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER32_CODE)
+#define USER_DATA_SELECTOR      GDT_USER_SEL_UNCHECKED(USER_DATA)
+#define USER64_CODE_SELECTOR    GDT_USER_SEL_UNCHECKED(USER64_CODE)
+#define USER64_SYSRET_SELECTOR  GDT_USER_SEL(USER64_SYSRET)
+#define SYSTEM_CODE_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_CODE)
+#define SYSTEM_DATA_SELECTOR    GDT_SYSTEM_SEL_UNCHECKED(SYSTEM_DATA)
+#define MONITOR_TASK_SELECTOR   GDT_SYSTEM_SEL(MONITOR_TASK)
+#define VMKERNEL_TASK_SELECTOR  GDT_SYSTEM_SEL(VMKERNEL_TASK)
+
+/*
+ * The VMM GDT is comprised of many segment descriptors with one initial
+ * Task State Segment system descriptor.  The VMM Task State Segment is
+ * on the same page sequentially after its GDT.
+ */
+#pragma pack(push, 1)
+typedef struct StaticGDTAndTSS {
+   Descriptor   empty[NUM_BOOT_SEGMENTS + NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS];
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+   Task64       monTSS;
+   uint8        TSSIRBitmap[IRB_SIZE];
+} StaticGDTAndTSS;
+#pragma pack(pop)
+
+#pragma pack(push, 1)
+typedef struct VmkernelGDT {
+   Descriptor   bootSegs[NUM_BOOT_SEGMENTS];
+   Descriptor   userSegs[NUM_USER_SEGMENTS];
+   Descriptor   systemSegs[NUM_SYSTEM_SEGMENTS]; /* VMM/VMK-shared. */
+   Descriptor64 vmkTask;
+   Descriptor64 monTask;
+} VmkernelGDT;
+#pragma pack(pop)
+
+MY_ASSERTS(segs,
+           ASSERT_ON_COMPILE(SYSTEM_CODE_SEGMENT + 1 == SYSTEM_DATA_SEGMENT);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT - FIRST_USER_SEGMENT <=
+                             NUM_USER_SEGMENTS);
+           ASSERT_ON_COMPILE(AFTER_LAST_USER_SEGMENT <= FIRST_SYSTEM_SEGMENT);
+)
+
+#endif /* _SEGS_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,28 +16,32 @@
 *
 *********************************************************/

-#ifndef _INITBLOCK_H
-#define _INITBLOCK_H
+/*
+ * sharedAreaType.h --
+ *
+ *      This file contains shared area type definitions.
+ */

+#ifndef _SHAREDAREATYPE_H_
+#define _SHAREDAREATYPE_H_

-#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+
 #include "includeCheck.h"

+typedef enum {
+   SHARED_AREA_PER_VM_VMX = 0,
+   SHARED_AREA_INTER_VCPU_VMX,
+   SHARED_AREA_PER_VCPU_VMX,
+   SHARED_AREA_PER_VM,
+   SHARED_AREA_INTER_VCPU,
+   SHARED_AREA_PER_VCPU,
+   NUM_SHARED_AREAS
+} SharedAreaType;

-#include "basic_initblock.h"
-
-
-#define MAX_LOGFILE_NAME_LENGTH 256
-#define MAX_MONITOR_REGIONS     8
-#define INIT_BLOCK_MAGIC     (0x1789+14)
-
-
-/*
- * Option flags 
- */
-/* None yet */
-
-
-#endif
+#endif // _SHAREDAREATYPE_H_
@@ -0,0 +1,62 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.h --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifndef SHARED_AREA_VMMON_H
+#define SHARED_AREA_VMMON_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#include "includeCheck.h"
+
+#include "iocontrols.h"
+
+struct VMDriver;
+
+/* A collection of pages backing a shared area region. */
+typedef struct SharedAreaVmmonRegion {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} SharedAreaVmmonRegion;
+
+/* The VMMon Driver shared area. */
+typedef struct SharedAreaVmmon {
+   SharedAreaVmmonRegion regions[NUM_SHARED_AREAS];
+} SharedAreaVmmon;
+
+/* Request for a backing MPN for a shared area region at a given VCPU/offset. */
+typedef struct SharedAreaVmmonRequest {
+   SharedAreaType type;
+   Vcpuid vcpu;
+   PageCnt offset;
+} SharedAreaVmmonRequest;
+
+SharedAreaVmmon *SharedAreaVmmon_Init(struct VMDriver *vm);
+void SharedAreaVmmon_Cleanup(SharedAreaVmmon *area);
+Bool SharedAreaVmmon_RegisterRegion(struct VMDriver *driver,
+                                    VMSharedAreaRegistrationBlock *block);
+Bool SharedAreaVmmon_ValidateRegionArgs(struct VMDriver *driver,
+                                        VMSharedAreaRegistrationBlock *block);
+MPN SharedAreaVmmon_GetRegionMPN(struct VMDriver *vm,
+                                 SharedAreaVmmonRequest *request);
+#endif /* SHARED_AREA_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,45 +17,32 @@
 *********************************************************/

 /*
- * basic_initblock.h --
+ * statVarsVmmon.h --
 *
- *    VM initialization block.
+ *     VMMon stat vars management.
 */

-#ifndef _BASIC_INITBLOCK_H_
-#define _BASIC_INITBLOCK_H_
+#ifndef STAT_VARS_VMMON_H
+#define STAT_VARS_VMMON_H

-
-#define INCLUDE_ALLOW_USERLEVEL
-
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_DISTRIBUTE
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

+#include "iocontrols.h"

-#include "vcpuid.h"
+struct VMDriver;

+/* The VMMon Driver stat vars area. */
+typedef struct StatVarsVmmon {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} StatVarsVmmon;

-#define MAX_INITBLOCK_CPUS     MAX_VCPUS
-
-
-typedef
-#include "vmware_pack_begin.h"
-struct InitBlock {
-   uint32 magicNumber;     /* Magic number (INIT_BLOCK_MAGIC) */
-   uint32 vmInitFailurePeriod;
-   VA64   crosspage[MAX_INITBLOCK_CPUS];
-   LPN64  monStartLPN;
-   LPN64  monEndLPN;
-   MPN    crossGDTMPN;
-   uint16 numPTPPages;
-}
-#include "vmware_pack_end.h"
-InitBlock;
-
-
-#endif // _BASIC_INITBLOCK_H_
+StatVarsVmmon *StatVarsVmmon_Init(struct VMDriver *vm);
+void StatVarsVmmon_Cleanup(StatVarsVmmon *statVars);
+Bool StatVarsVmmon_RegisterVCPU(struct VMDriver *driver,
+                                VMStatVarsRegistrationBlock *block);
+MPN StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpuid,
+                               PageCnt offset);
+#endif /* STAT_VARS_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -20,15 +20,12 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-UC(CALL_START)
-UC(BEGIN_BACK_TO_HOST)
-UC(SWITCHED_TO_MODULE)
-UC(VMX_SELECT_RETURN)
-UC(VMX_HANDLER_START)
-UC(VMX_SIGNAL)
-UC(MODULE_SIGNAL)
-UC(SWITCHING_TO_MONITOR)
-UC(DONE_BACK_TO_HOST)
-UC(CALL_END)
+UC(CALL_START,             FALSE)
+UC(BEGIN_BACK_TO_HOST,     FALSE)
+UC(SWITCHED_TO_MODULE,     TRUE)
+UC(VMX_HANDLER_START,      FALSE)
+UC(SWITCHING_TO_MONITOR,   TRUE)
+UC(DONE_BACK_TO_HOST,      FALSE)
+UC(CALL_END,               FALSE)

 #undef UC
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 #ifndef USERCALLDEFS_H
 #define USERCALLDEFS_H

-#ifdef linux
+#ifdef __linux__
 #include <linux/errno.h>
 #define USERCALL_RESTART (-ERESTARTNOINTR)
 #else
@@ -31,5 +31,6 @@
 * failing DeviceIoControl on Windows.
 */
 #define USERCALL_VMX86ALLOCERR (-8192)
+#define USERCALL_SWITCHERR     (-8193)

-#endif 
+#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,6 @@
 extern "C" {
 #endif

-
 typedef uint32 Vcpuid;                 // VCPU number

 #define VCPUID_INVALID  (~0U)
@@ -54,17 +53,14 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define MAX_CORES_PER_SOCKET   64
 #define MAX_VCPU_SOCKETS      128

-#ifdef VMM
-#include "vcpuset.h"
+/*
+ * There are several properties of the VM which change at the 128 VCPU
+ * boundary.  EFI firmware, x2APIC, and IOMMU are required among others.
+ */
+#define MAX_SMALL_VM_VCPUS 128

-/* In VMM, CurVcpuid() is available everywhere. */
-extern const Vcpuid      curVcpuid;
-extern const VCPUSet     curVcpuidSet;
-#define CurVcpuid()         (curVcpuid)
-#define CurVcpuidSet()      (&curVcpuidSet)
-#define IS_BOOT_VCPU()      IS_BOOT_VCPUID(CurVcpuid())
-
-#endif  /* VMM */
+/* Supported limit. */
+#define MAX_SUPPORTED_VCPUS   (FEATURE_STATE_IS_ENABLED(MonsterVM) ? 768 : 256)

 #if defined __cplusplus
 } // extern "C"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,10 +44,11 @@
 #   include "str.h"     /* Str_Snprintf */
 #   define VCS_SNPRINTF Str_Snprintf
 #elif defined MONITOR_APP
-#   include <stdio.h>   /* libc snprintf */
 #   if defined WIN32
-#      define VCS_SNPRINTF _snprintf
+#      include "str.h"     /* Str_Snprintf */
+#      define VCS_SNPRINTF Str_Snprintf
 #   else
+#      include <stdio.h>   /* libc snprintf */
 #      define VCS_SNPRINTF snprintf
 #   endif
 #elif defined VMM || defined VMKERNEL
@@ -112,7 +113,7 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET(_setIndex)                                     \
   do {                                                                       \
-      int _setIndex;                                                          \
+      unsigned _setIndex;                                                     \
      for (_setIndex = 0; _setIndex < VCS_SUBSET_COUNT; _setIndex++) {

 #define ROF_EACH_SUBSET_IN_SET()                                              \
@@ -122,8 +123,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET_COUNTDOWN(_setIndex)                           \
   do {                                                                       \
-      int _setIndex;                                                          \
-      for (_setIndex = VCS_SUBSET_COUNT - 1; _setIndex >= 0; _setIndex--) {
+      unsigned _setIndex = VCS_SUBSET_COUNT;                                  \
+      while (_setIndex-- > 0) {

 #define ROF_EACH_SUBSET_IN_SET_COUNTDOWN()                                    \
      }                                                                       \
@@ -132,8 +133,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex, _numVcpus)                \
   do {                                                                       \
-      int _setIndex;                                                          \
-      int _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);                 \
+      unsigned _setIndex;                                                     \
+      unsigned _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);            \
      for (_setIndex = 0; _setIndex <= _maxSubsets; _setIndex++) {

 #define ROF_EACH_POPULATED_SUBSET_IN_SET()                                    \
@@ -307,7 +308,7 @@ VCPUSet_FindFirst(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return lssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return lssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET();
   return VCPUID_INVALID;
@@ -319,7 +320,7 @@ VCPUSet_FindLast(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET_COUNTDOWN(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return mssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return mssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET_COUNTDOWN();
   return VCPUID_INVALID;
@@ -356,8 +357,7 @@ VCPUSet_FindFirstInSubset(const VCPUSet *vcs, uint64 *subset,
   ASSERT(*subsetIdx < maxSubsets && maxSubsets <= VCS_SUBSET_COUNT);
   do {
      if (*subset != 0) {
-         int bit;
-         bit = lssb64_0(*subset);
+         unsigned bit = (unsigned)lssb64_0(*subset);
         *subset &= ~(CONST64U(1) << bit);
         return bit + (*subsetIdx << VCS_SUBSET_SHIFT);
      }
@@ -672,7 +672,7 @@ static INLINE Vcpuid
 VCPUSet_FindSingleton(const VCPUSet *vcs)
 {
   uint64 foundSub = 0;
-   uint32 foundIdx;
+   uint32 foundIdx = 0;
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 sub = vcs->subset[idx];
      if (sub != 0) {
@@ -683,8 +683,11 @@ VCPUSet_FindSingleton(const VCPUSet *vcs)
         foundIdx = idx;
      }
   } ROF_EACH_SUBSET_IN_SET();
-   return foundSub != 0 ? lssb64_0(foundSub) + (foundIdx << VCS_SUBSET_SHIFT) :
-                          VCPUID_INVALID;
+   if (foundSub != 0) {
+      return lssb64(foundSub) - 1 + (foundIdx << VCS_SUBSET_SHIFT);
+   } else {
+      return VCPUID_INVALID;
+   }
 }


@@ -863,27 +866,27 @@ VCPUSet_Intersection(VCPUSet *dest, const VCPUSet *src)

 #ifdef VCS_SNPRINTF
 static INLINE char *
-VCPUSet_LogFormat(char *buf, const int size, const VCPUSet *vcs)
+VCPUSet_LogFormat(char *buf, size_t size, const VCPUSet *vcs)
 {
-   int offset = 0;
-   Vcpuid highest = VCPUSet_FindLast(vcs);
-   int idx = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+   unsigned offset  = 0;
+   Vcpuid   highest = VCPUSet_FindLast(vcs);
+   unsigned idx     = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+
   ASSERT(size >= VCS_BUF_SIZE);
 #define VCS_LOGF(...)                                                   \
   {                                                                    \
      int ret = VCS_SNPRINTF(buf + offset, size - offset, __VA_ARGS__); \
-      ASSERT(0 <= ret && ret < size - offset);                          \
-      offset += ret;                                                    \
+      ASSERT(0 <= ret && size >= offset && ret < (int)(size - offset)); \
+      offset += (unsigned)ret;                                          \
   }
   /* Print the leading value with no zero-extension. */
-   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx--]);
+   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx]);

-   while (idx >= 0) {
+   while (idx-- > 0) {
      if ((idx + 1) % (VCS_SUBSET_WIDTH / 8) == 0) {
         VCS_LOGF(".");
      }
      VCS_LOGF("%02x", ((unsigned char *)vcs)[idx]);
-      idx--;
   }
   return buf;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,52 +41,47 @@

 #include "vm_basic_asm.h"
 #include "x86msr.h"
-
-#ifdef VM_X86_64
-#include "vm_asm_x86_64.h"
-#else
 #include "vm_asm_x86.h"
-#endif


 static INLINE void
 SET_FS64(uint64 fs64)
 {
-   __SET_MSR(MSR_FSBASE, fs64);
+   X86MSR_SetMSR(MSR_FSBASE, fs64);
 }


 static INLINE void
 SET_GS64(uint64 gs64)
 {
-   __SET_MSR(MSR_GSBASE, gs64);
+   X86MSR_SetMSR(MSR_GSBASE, gs64);
 }

 static INLINE void
 SET_KernelGS64(uint64 kgs64)
 {
-   __SET_MSR(MSR_KERNELGSBASE, kgs64);
+   X86MSR_SetMSR(MSR_KERNELGSBASE, kgs64);
 }


 static INLINE uint64
 GET_FS64(void)
 {
-   return __GET_MSR(MSR_FSBASE);
+   return X86MSR_GetMSR(MSR_FSBASE);
 }


 static INLINE uint64
 GET_GS64(void)
 {
-   return __GET_MSR(MSR_GSBASE);
+   return X86MSR_GetMSR(MSR_GSBASE);
 }


 static INLINE uint64
 GET_KernelGS64(void)
 {
-   return __GET_MSR(MSR_KERNELGSBASE);
+   return X86MSR_GetMSR(MSR_KERNELGSBASE);
 }

 #endif // VM_ARM_64
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * vm_asm_x86.h
 *
- *	IA32 asm macros
+ *      IA32 asm macros
 */

 #ifndef _VM_ASM_X86_H_
@@ -49,10 +49,6 @@

 #ifdef __GNUC__

-#if defined __APPLE__
-/* PR 352418: GCC produces error if the non-Apple version is used */
-#define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
-#else
 /* ASSERT_ON_COMPILE_SELECTOR_SIZE:
 *
 *  - Selector must be 16-bits.
@@ -66,26 +62,16 @@
 *                    ((__builtin_constant_p(expr) ? ((expr) >> 16) == 0) \
 *                                                 : sizeof(expr) <= 2)
 */
-#if (__GNUC__ >= 4) && (__GNUC_MINOR__ >= 1) && !defined(USE_UBSAN)
+#ifndef USE_UBSAN
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)                                \
   ASSERT_ON_COMPILE(sizeof(Selector) == 2 &&                                \
                     ((__builtin_constant_p(expr) && ((expr) >> 16) == 0) || \
                      sizeof(expr) <= 2))
 #else
-/* gcc 3.3.3 is not able to produce a constant expression (PR 356383) */
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
 #endif
-#endif


-/*
- * [GS]ET_[GI]DT() are defined as macros wrapping a function
- * so we can pass the argument implicitly by reference (requires
- * a macro) and get type checking too (requires a function).
- */
-
-#define SET_GDT(var) _Set_GDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
@@ -97,8 +83,6 @@ _Set_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define SET_IDT(var) _Set_IDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_IDT(_GETSET_DTR_TYPE *dtr)
@@ -110,8 +94,6 @@ _Set_IDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_GDT(var) _Get_GDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because there's a hidden input (the [IG]DTR) that can change
@@ -126,8 +108,6 @@ _Get_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_IDT(var) _Get_IDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because the [IG]DT can change without the compiler knowing it
@@ -175,7 +155,6 @@ _GET_LDT(Selector * const result)
   } while (0)


-/* Checked against the Intel manual and GCC --thutt */
 #define _BUILD_SET_R(func, reg)        \
   static INLINE void                  \
   func(uintptr_t r)                   \
@@ -186,24 +165,20 @@ _GET_LDT(Selector * const result)
              : "memory");             \
   }

-/* Not yet checked against the Intel manual and GCC --slava
- *
- * 'volatile' because CRs and DRs can change without the compiler
- * knowing it (when there is a page fault, when a breakpoint occurs,
- * and moreover it seems there is no way to teach gcc that smsw
- * clobbers cr0 for example).
- *
- * The parameter is a 'uintptr_t *' so that the size of the actual
- * parameter must exactly match the size of the hardware register.
- * This prevents the use of 32-bit variables when building 64-bit
- * code.
+/*
+ * The inline asm is marked 'volatile' because CRs and DRs can change
+ * without the compiler knowing it (when there is a page fault, when a
+ * breakpoint occurs, and moreover it seems there is no way to teach
+ * gcc that smsw clobbers cr0 for example).
 */
 #define _BUILD_GET_R(func, reg)                         \
-   static INLINE void                                   \
-   func(uintptr_t *result)                              \
+   static INLINE uintptr_t                              \
+   func(void)                                           \
   {                                                    \
+      uintptr_t result;                                 \
      __asm__ __volatile__("mov %%" #reg ", %0"         \
-                           : "=r" (*result));           \
+                           : "=r" (result));            \
+      return result;                                    \
   }

 _BUILD_SET_R(_SET_CR0, cr0)
@@ -218,22 +193,6 @@ _BUILD_GET_R(_GET_CR3, cr3)
 _BUILD_GET_R(_GET_CR4, cr4)
 _BUILD_GET_R(_GET_CR8, cr8)

-#if defined __APPLE__
-/* Mac OS gcc 4 uses DBx instead of DRx register names. */
-_BUILD_SET_R(_SET_DR0, db0)
-_BUILD_SET_R(_SET_DR1, db1)
-_BUILD_SET_R(_SET_DR2, db2)
-_BUILD_SET_R(_SET_DR3, db3)
-_BUILD_SET_R(_SET_DR6, db6)
-_BUILD_SET_R(_SET_DR7, db7)
-
-_BUILD_GET_R(_GET_DR0, db0)
-_BUILD_GET_R(_GET_DR1, db1)
-_BUILD_GET_R(_GET_DR2, db2)
-_BUILD_GET_R(_GET_DR3, db3)
-_BUILD_GET_R(_GET_DR6, db6)
-_BUILD_GET_R(_GET_DR7, db7)
-#else
 _BUILD_SET_R(_SET_DR0, dr0)
 _BUILD_SET_R(_SET_DR1, dr1)
 _BUILD_SET_R(_SET_DR2, dr2)
@@ -247,47 +206,8 @@ _BUILD_GET_R(_GET_DR2, dr2)
 _BUILD_GET_R(_GET_DR3, dr3)
 _BUILD_GET_R(_GET_DR6, dr6)
 _BUILD_GET_R(_GET_DR7, dr7)
-#endif

-#define SET_CR_DR(regType, regNum, expr)                     \
-   do {                                                      \
-      /* Ensure no implicit truncation of 'expr' */          \
-      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
-      _SET_##regType##regNum(expr);                          \
-   } while (0)
-
-#define GET_CR_DR(regType, regNum, var) \
-   do {                                 \
-      _GET_##regType##regNum(&(var));   \
-   } while (0)
-
-#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
 #define SET_CR2(expr) SET_CR_DR(CR, 2, expr)
-#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
-#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
-#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
-
-/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
-#undef GET_CR0
-#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
-#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
-#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
-#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
-#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
-
-#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
-#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
-#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
-#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
-#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
-#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
-
-#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
-#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
-#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
-#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
-#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
-#define GET_DR7(var)  GET_CR_DR(DR, 7, var)

 #define SET_SEGREG(reg, expr)                                          \
   do {                                                                \
@@ -426,440 +346,200 @@ CLTS(void)
 /* Beginning of the section whose correctness has NOT been checked */
 #define FNCLEX()         __asm__("fnclex" ::);

-/* TLB_INVALIDATE_xxx are not checked yet */
+/* TLB_INVALIDATE_PAGE is not checked yet */
 #define TLB_INVALIDATE_PAGE(_addr) do { \
     __asm__ __volatile__("invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
 } while (0)

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) do { \
-     __asm__ __volatile__("fs; invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
-} while (0)
-
 #define RESTORE_FLAGS _Set_flags
 #define ENABLE_INTERRUPTS() __asm__ __volatile__ ("sti": : :"memory")
 #define CLEAR_INTERRUPTS()  __asm__ __volatile__ ("cli": : :"memory")
 #define RAISE_INTERRUPT(_x)  __asm__ __volatile__("int %0" :: "g" (_x))
 #define RETURN_FROM_INT()   __asm__ __volatile__("iret" :: )

-#if ! defined(VMKERNEL)
-#define NO_INTERRUPTS_BEGIN()	do { \
-                                   uintptr_t _flags; \
-                                   SAVE_FLAGS(_flags); \
-                                   CLEAR_INTERRUPTS();
-
-#define NO_INTERRUPTS_END()	   RESTORE_FLAGS(_flags); \
-                                } while(0)
-#endif
-
 /* End of the section whose correctness has NOT been checked */

 #elif defined _MSC_VER  /* !__GNUC__ */

-#define SET_IDT(_idt)    _Set_IDT(&(_idt))
-#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
-#define SET_TR(_tr)      _Set_TR(_tr)
-#define SET_LDT(_tr)     _Set_LDT(_tr)
+#define _BUILD_SET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writedr(reg, r);                              \
+   }

-#define GET_IDT(_idt)    _Get_IDT(&(_idt))
-#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
-#define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
-#define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)
+#define _BUILD_GET_DR(func, reg)                      \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
+   {                                                  \
+      return __readdr(reg);                           \
+   }

-#define GET_CR0(_reg)    __asm mov eax, cr0 __asm mov _reg, eax
-#define SET_CR0(_reg)    __asm mov eax, _reg __asm mov cr0, eax
-#define GET_CR2(_reg)    __asm mov eax, cr2 __asm mov _reg, eax
-#define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
-#define GET_CR3(_reg)    __asm mov eax, cr3 __asm mov _reg, eax
-#define SET_CR3(_reg)    __asm mov eax, _reg __asm mov cr3, eax
-/*
- * MSC doesn't seem to like CR4 in __asm statements. We emit
- * the opcode for MOV EAX,CR4 = 0xf020e0 and MOV CR4,EAX = 0xf022e0
- */
-#define GET_CR4(_reg) { \
- __asm _emit 0x0f __asm _emit 0x20 __asm _emit 0xe0 \
- __asm mov _reg, eax \
-}
-#define SET_CR4(_reg)    { \
-  __asm mov eax, _reg \
-  __asm _emit 0x0f __asm _emit 0x22 __asm _emit 0xe0 \
-}
+#define _BUILD_SET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writecr##reg(r);                              \
+   }

+#define _BUILD_GET_CR(func, reg)                      \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
+   {                                                  \
+      return __readcr##reg();                         \
+   }

-#define GET_DR0(_reg) do { __asm mov eax,dr0 __asm mov _reg,eax } while (0)
-#define SET_DR0(_reg) do { __asm mov eax,_reg __asm mov dr0,eax } while (0)
-#define GET_DR1(_reg) do { __asm mov eax,dr1 __asm mov _reg,eax } while (0)
-#define SET_DR1(_reg) do { __asm mov eax,_reg __asm mov dr1,eax } while (0)
-#define GET_DR2(_reg) do { __asm mov eax,dr2 __asm mov _reg,eax } while (0)
-#define SET_DR2(_reg) do { __asm mov eax,_reg __asm mov dr2,eax } while (0)
-#define GET_DR3(_reg) do { __asm mov eax,dr3 __asm mov _reg,eax } while (0)
-#define SET_DR3(_reg) do { __asm mov eax,_reg __asm mov dr3,eax } while (0)
-#define GET_DR6(_reg) do { __asm mov eax,dr6 __asm mov _reg,eax } while (0)
-#define SET_DR6(_reg) do { __asm mov eax,_reg __asm mov dr6,eax } while (0)
-#define GET_DR7(_reg) do { __asm mov eax,dr7 __asm mov _reg,eax } while (0)
-#define SET_DR7(_reg) do { __asm mov eax,_reg __asm mov dr7,eax } while (0)
+_BUILD_SET_DR(_SET_DR0, 0)
+_BUILD_SET_DR(_SET_DR1, 1)
+_BUILD_SET_DR(_SET_DR2, 2)
+_BUILD_SET_DR(_SET_DR3, 3)
+_BUILD_SET_DR(_SET_DR6, 6)
+_BUILD_SET_DR(_SET_DR7, 7)

+_BUILD_GET_DR(_GET_DR0, 0)
+_BUILD_GET_DR(_GET_DR1, 1)
+_BUILD_GET_DR(_GET_DR2, 2)
+_BUILD_GET_DR(_GET_DR3, 3)
+_BUILD_GET_DR(_GET_DR6, 6)
+_BUILD_GET_DR(_GET_DR7, 7)

-#define CLTS()           __asm clts
-
-#define FNCLEX()         __asm fnclex
-
-#define TLB_INVALIDATE_PAGE(_addr) {  \
-	 void *_a = (_addr); \
-     __asm mov eax, _a __asm invlpg [eax] \
-}
-
-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) { \
-	uint32 __a = (uint32) (_addr); \
-	__asm mov eax, __a _asm invlpg fs:[eax] \
-}
-
-
-#define ENABLE_INTERRUPTS() { __asm sti }
-#define CLEAR_INTERRUPTS()  { __asm cli }
-
-#define RAISE_INTERRUPT(_x)  {__asm int _x }
-#define RETURN_FROM_INT()   {__asm iretd }
-
-
-#define SAVE_FLAGS(x) { \
-     __asm pushfd __asm pop eax __asm mov x, eax \
-}
-
-#define RESTORE_FLAGS(x) { \
-     __asm push x __asm popfd\
-}
-
-
-
-static INLINE void SET_DS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ds, ax
-}
-
-static INLINE void SET_ES(Selector val)
-{
-   __asm mov ax, val
-   __asm mov es, ax
-}
-
-static INLINE void SET_FS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov fs, ax
-}
-
-static INLINE void SET_GS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov gs, ax
-}
-
-static INLINE void SET_SS(Selector val)
-{
-   __asm mov ax, val
-   __asm mov ss, ax
-}
-
-static INLINE Selector GET_FS(void)
-{
-	Selector _v;
-	__asm mov _v,fs
-	return _v;
-}
-
-static INLINE Selector GET_GS(void)
-{
-	Selector _v;
-	__asm mov _v,gs
-	return _v;
-}
-
-
-static INLINE Selector GET_DS(void)
-{
-	Selector _v;
-	__asm mov _v,ds
-	return _v;
-}
-
-static INLINE Selector GET_ES(void)
-{
-	Selector _v;
-	__asm mov _v,es
-	return _v;
-}
-
-static INLINE Selector GET_SS(void)
-{
-	Selector _v;
-	__asm mov _v,ss
-	return _v;
-}
-
-static INLINE Selector GET_CS(void)
-{
-	Selector _v;
-	__asm mov _v,cs
-	return _v;
-}
-
-#pragma warning( disable : 4035)
-
-static INLINE uint32  GET_WORD_FROM_FS(uint32 *_addr) {
-	__asm mov eax, _addr
-    __asm mov eax, fs:[eax]
-}
-
-static INLINE uint16  GET_SHORT_FROM_FS(uint16 *_addr) {
-	__asm mov eax, _addr
-    __asm mov ax, fs:[eax]
-}
-
-static INLINE uint8  GET_BYTE_FROM_FS(uint8 *_addr) {
-	__asm mov eax, _addr
-     __asm mov al, fs:[eax]
-}
-
-#pragma warning (default: 4035)
-
-static INLINE void  SET_WORD_FS(uint32 *_addr, uint32 _val) {
-    __asm mov eax, _addr
-    __asm mov ebx, _val
-    __asm mov fs:[eax], ebx
-}
-
-static INLINE void  SET_SHORT_FS(uint32 *_addr, uint16 _val) {
-    __asm mov eax, _addr
-    __asm mov bx, _val
-    __asm mov fs:[eax], bx
-}
-
-static INLINE void  SET_BYTE_FS(uint32 *_addr, uint8 _val) {
-    __asm mov eax, _addr
-    __asm mov bl, _val
-    __asm mov fs:[eax], bl
-}
-
-static INLINE void _Set_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lgdt [eax]
-}
-
-static INLINE void _Set_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lidt [eax]
-}
-
-static INLINE void _Set_LDT(Selector val)
-{
-   __asm lldt val
-}
-
-static INLINE void _Set_TR(Selector val)
-{
-   __asm ltr val
-}
-
-static INLINE void _Get_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sgdt [eax]
-}
-
-static INLINE void _Get_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sidt [eax]
-}
-
-static INLINE Selector _Get_LDT(void) {
-   Selector sel;
-   __asm sldt sel
-   return sel;
-}
-
-static INLINE Selector _Get_TR(void) {
-   Selector sel;
-   __asm str sel
-   return sel;
-}
+_BUILD_SET_CR(_SET_CR0, 0);
+_BUILD_SET_CR(_SET_CR3, 3);
+_BUILD_SET_CR(_SET_CR4, 4);
+_BUILD_SET_CR(_SET_CR8, 8);

+_BUILD_GET_CR(_GET_CR0, 0);
+_BUILD_GET_CR(_GET_CR2, 2);
+_BUILD_GET_CR(_GET_CR3, 3);
+_BUILD_GET_CR(_GET_CR4, 4);
+_BUILD_GET_CR(_GET_CR8, 8);

 static INLINE void
-MEMCOPY_TO_FS(VA to,
-              char * from,
-              unsigned long n)
+_Set_GDT(_GETSET_DTR_TYPE *dtr)
 {
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x = *(uint32*) (from + i);
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, x
-	  __asm mov fs:[eax], ebx
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x = from[i];
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov bl, x
-	  __asm mov fs:[eax], bl
-      i++;
-   }
+   _lgdt(dtr);
 }

-
-
 static INLINE void
-MEMCOPY_FROM_FS(char * to,
-                VA from,
-                unsigned long n)
+_Get_GDT(_GETSET_DTR_TYPE *dtr)
 {
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x;
-	  uint32 _faddr = (uint32)(from+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, fs:[eax]
-	  __asm mov x,ebx
-      *(uint32*)(to+i)=x;
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x;
-	  uint32 _faddr = (uint32) (from+i);
-	  __asm mov eax, _faddr;
-      __asm mov bl, fs:[eax]
-	  __asm mov x, bl
-      *(uint8*)(to+i)=x;
-      i++;
-   }
+   _sgdt(dtr);
 }

-#else
-#error No compiler defined for get/set
+static INLINE void
+_Set_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __lidt(dtr);
+}
+
+static INLINE void
+_Get_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __sidt(dtr);
+}
+
+
+#define ENABLE_INTERRUPTS() _enable()
+#define CLEAR_INTERRUPTS()  _disable()
+
+#define SAVE_FLAGS(x) do { \
+   (x) = __readeflags();   \
+} while (0)
+
+#define RESTORE_FLAGS(x) __writeeflags(x)
+
 #endif /* !__GNUC__ && !_MSC_VER */


-#ifdef __GNUC__
-static INLINE unsigned CURRENT_CPL(void)
+#define SET_CR_DR(regType, regNum, expr)                     \
+   do {                                                      \
+      /* Ensure no implicit truncation of 'expr' */          \
+      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
+      _SET_##regType##regNum(expr);                          \
+   } while (0)
+
+#define GET_CR_DR(regType, regNum, var) \
+   do {                                 \
+      var = _GET_##regType##regNum();   \
+   } while (0)
+
+#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
+#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
+#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
+#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
+#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
+#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
+
+#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
+#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
+#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
+#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
+#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
+#define GET_DR7(var)  GET_CR_DR(DR, 7, var)
+
+/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
+#undef GET_CR0
+#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
+#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
+#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
+#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
+#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
+
+#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
+#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
+#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
+#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
+
+/*
+ * When the ULM macro-defines INTERRUPTS_ENABLED, attempting to define
+ * it as a function produces hard-to-diagnose compile-time errors.
+ */
+#if !defined(ULM) && !defined(INTERRUPTS_ENABLED)
+static INLINE Bool
+INTERRUPTS_ENABLED(void)
+{
+   uintptr_t flags;
+   SAVE_FLAGS(flags);
+   return ((flags & EFLAGS_IF) != 0);
+}
+#endif
+
+/*
+ * [GS]ET_[GI]DT() are defined as macros wrapping a function
+ * so we can pass the argument implicitly by reference (requires
+ * a macro) and get type checking too (requires a function).
+ */
+
+#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
+
+#define SET_IDT(_idt)    _Set_IDT(&(_idt))
+#define GET_IDT(_idt)    _Get_IDT(&(_idt))
+
+#if !defined(VMKERNEL)
+#define NO_INTERRUPTS_BEGIN() do { \
+                                   uintptr_t _flags; \
+                                   SAVE_FLAGS(_flags); \
+                                   CLEAR_INTERRUPTS();
+
+#define NO_INTERRUPTS_END()        RESTORE_FLAGS(_flags); \
+                                 } while(0)
+#endif
+
+
+#if defined (__GNUC__)
+static INLINE unsigned
+CURRENT_CPL(void)
 {
   return SELECTOR_RPL(GET_CS());
 }
-#elif defined _MSC_VER
-static INLINE unsigned CURRENT_CPL(void) {
-   volatile Selector _v;
-   __asm mov ax, cs _asm mov _v, ax
-   return SELECTOR_RPL(_v);
-}
-#else
-#error No compiler defined for CURRENT_CPL
 #endif


+static INLINE uint64
+RDPMC(int counter)
+{
 #ifdef __GNUC__
-/* Checked against the Intel manual and GCC --hpreg
-
-   volatile because the msr can change without the compiler knowing it
-   (when we use wrmsr). */
-static INLINE uint64 __GET_MSR(int cx)
-{
-   uint64 msr;
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
-      : "=a" (msr)
-      : "c" (cx)
-      : "%rdx"
-   );
-#else
-   __asm__ __volatile__(
-      "rdmsr"
-      : "=A" (msr)
-      : "c" (cx)
-   );
-#endif
-   return msr;
-}
-#elif defined _MSC_VER
-#pragma warning( disable : 4035)
-static INLINE uint64 __GET_MSR(int input)
-{
-   __asm push ecx
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x32
-   __asm pop ecx
-}
-
-static INLINE void __SET_MSR(int input, uint64 val)
-{
-   uint32 hival = (uint32)(val >> 32);
-   uint32 loval = (uint32)val;
-   __asm push edx
-   __asm push ecx
-   __asm push eax
-   __asm mov  eax, loval
-   __asm mov  edx, hival
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x30
-   __asm pop eax
-   __asm pop ecx
-   __asm pop edx
-}
-#pragma warning (default: 4035)
-#else
-#error
-#endif
-
-#ifdef __GNUC__
-static INLINE void __SET_MSR(int cx, uint64 val)
-{
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "a" ((uint32) val), "d" ((uint32)(val >> 32)), "c" (cx)
-    );
-#else
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "A" (val),
-        "c" (cx)
-    );
-#endif
-}
-#endif
-
-
-/*
- * RDMSR/WRMSR access the 64bit MSRs as two
- * 32 bit quantities, whereas GET_MSR/SET_MSR
- * above access the MSRs as one 64bit quantity.
- */
-#ifdef __GNUC__
-#undef RDMSR
-#undef WRMSR
-#define RDMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "rdmsr"                         \
-      : "=a" (low), "=d" (high)       \
-      : "c" (msrNum)                  \
-   );                                 \
-} while (0)
-
-#define WRMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "wrmsr"                         \
-      : /* no outputs */              \
-      : "c" (msrNum),                 \
-        "a" (low),                    \
-        "d" (high)                    \
-    );                                \
-} while (0)
-
-static INLINE uint64 RDPMC(int cx)
-{
 #ifdef VM_X86_64
   uint64 pmcLow;
   uint64 pmcHigh;
@@ -867,7 +547,7 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=a" (pmcLow), "=d" (pmcHigh)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmcHigh << 32 | pmcLow;
@@ -877,43 +557,26 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=A" (pmc)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmc;
 #endif
-}
 #elif defined _MSC_VER
-#ifndef VM_X86_64 // XXX Switch to intrinsics with the new 32 and 64-bit compilers.
-
-static INLINE uint64 RDPMC(int counter)
-{
-   __asm mov ecx, counter
-   __asm rdpmc
+   return __readpmc(counter);
+#endif
 }

-static INLINE void WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   __asm mov ecx, msrNum
-   __asm mov eax, lo
-   __asm mov edx, hi
-   __asm wrmsr
-}
-#endif
-#endif

-
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
-static INLINE uint64 __XGETBV(int cx)
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined (ULM)
+static INLINE uint64
+__XGETBV(int cx)
 {
+#ifdef __GNUC__
 #ifdef VM_X86_64
   uint64 lowval, hival;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=a" (lowval), "=d" (hival)
      : "c" (cx)
   );
@@ -921,39 +584,41 @@ static INLINE uint64 __XGETBV(int cx)
 #else
   uint64 val;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=A" (val)
      : "c" (cx)
   );
   return val;
 #endif
+#elif defined _MSC_VER
+   return _xgetbv((unsigned)cx);
+#endif
 }

-static INLINE void __XSETBV(int cx, uint64 val)
+static INLINE void
+__XSETBV(int cx, uint64 val)
 {
+#ifdef __GNUC__
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd1"
-#else
      "xsetbv"
-#endif
      : /* no outputs */
      : "a" ((uint32)val), "d" ((uint32)(val >> 32)), "c" (cx)
    );
+#elif defined _MSC_VER
+   _xsetbv((unsigned)cx, val);
+#endif
 }

-static INLINE uint64 GET_XCR0(void)
+static INLINE uint64
+GET_XCR0(void)
 {
   return __XGETBV(0);
 }

 #define SET_XCR0(val) __XSETBV(0, val)

-static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
+static INLINE void
+SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 {
 #ifndef VMM_BOOTSTRAP
   ASSERT(oldVal == GET_XCR0());
@@ -965,33 +630,45 @@ static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 #endif


-#define START_TRACING() { \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags |= EFLAGS_TF;    \
-   RESTORE_FLAGS(flags);  \
-}
-
-#define STOP_TRACING() {  \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags &= ~EFLAGS_TF;   \
-   RESTORE_FLAGS(flags);  \
-}
-
-
-static INLINE Bool
-INTERRUPTS_ENABLED(void)
+static INLINE uint32
+RDTSCP_AuxOnly(void)
 {
-   uintptr_t flags;
-   SAVE_FLAGS(flags);
-   return ((flags & EFLAGS_IF) != 0);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return tscAux;
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   __rdtscp(&tscAux);
+
+   return tscAux;
+#endif
 }

-static INLINE void
-SET_KERNEL_PER_CORE(uint64 val)
+
+static INLINE uint64
+RDTSCP(void)
 {
-   __SET_MSR(MSR_GSBASE, val);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      :"=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return QWORD(tscHigh, tscLow);
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   return __rdtscp(&tscAux);
+#endif
 }

 #endif
@@ -1,137 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * vm_asm_x86_64.h
- *
- *	x86-64 asm macros
- */
-
-#ifndef _VM_ASM_X86_64_H_
-#define _VM_ASM_X86_64_H_
-
-
-#define INCLUDE_ALLOW_VMX
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_DISTRIBUTE
-#include "includeCheck.h"
-
-#ifdef __GNUC__
-#include "vm_asm_x86.h"
-#endif
-
-/*
- * This file contains inline assembly routines used by x86_64 code.
- */
-
-#ifdef __GNUC__
-
-  /* nop; prevents #error for no compiler definition from firing */
-
-#elif defined _MSC_VER  /* !__GNUC__ */
-
-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-
-#ifdef _WIN64
-#ifdef __cplusplus
-extern "C" {
-#endif
-unsigned __int64  __readmsr(unsigned long);
-void              __writemsr(unsigned long, unsigned __int64);
-#pragma intrinsic(__readmsr, __writemsr)
-#ifdef __cplusplus
-}
-#endif
-
-
-static INLINE uint64
-RDPMC(int counter)
-{
-   return __readpmc(counter);
-}
-
-
-static INLINE void
-WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   uint64 value = QWORD(hi, lo);
-   __writemsr(msrNum, value);
-}
-
-
-static INLINE uint64
-__GET_MSR(int input)
-{
-   return __readmsr((unsigned long)input);
-}
-
-
-static INLINE void
-__SET_MSR(int cx, uint64 val)
-{
-  __writemsr((unsigned long)cx, (unsigned __int64)val);
-}
-
-#endif
-
-#else
-#error No compiler defined for get/set
-#endif /* !__GNUC__ && !_MSC_VER */
-
-
-#ifdef __GNUC__
-static INLINE void
-SWAPGS(void)
-{
-   __asm__ __volatile__("swapgs");
-}
-
-
-static INLINE uint64
-RDTSCP_AuxOnly(void)
-{
-   uint64 tscLow, tscHigh, tscAux;
-
-   __asm__ __volatile__(
-      "rdtscp"
-      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
-   );
-
-   return tscAux;
-}
-#endif
-
-#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -202,12 +202,12 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +314,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -122,87 +129,29 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -2303,7 +2195,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2216,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3581,7 +3476,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3604,7 +3499,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -3618,7 +3513,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -48,7 +55,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -221,8 +227,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +269,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +382,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +398,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -497,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -550,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -586,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -630,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -653,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -768,27 +760,21 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -836,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -849,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -888,7 +868,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +882,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1078,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1136,113 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+#endif
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,24 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -113,40 +95,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +169,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +272,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +316,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +398,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +483,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +507,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -244,9 +145,127 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_sfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +297,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +412,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +430,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +510,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,12 +44,44 @@
 #endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -58,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -159,18 +157,6 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

-/*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-

 /*
 * Token concatenation
@@ -209,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -255,6 +241,10 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif
@@ -286,6 +276,14 @@ Max(int a, int b)
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -440,10 +438,6 @@ void *_ReturnAddress(void);

 #define strtok_r  strtok_s

-#if (_MSC_VER < 1500)
-#define	vsnprintf _vsnprintf
-#endif
-
 typedef int uid_t;
 typedef int gid_t;

@@ -467,20 +461,9 @@ typedef int pid_t;
 #define       W_OK          2
 #define       R_OK          4

-#endif // }
+#endif // } _WIN32

-/*
- * Macro for username comparison.
- */
-
-#ifdef _WIN32 // {
-#define USERCMP(x,y)  Str_Strcasecmp(x,y)
-#else
-#define USERCMP(x,y)  strcmp(x,y)
-#endif // }
-
-
-#endif // }
+#endif // } USERLEVEL

 #ifndef va_copy

@@ -556,10 +539,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -659,11 +642,27 @@ typedef int pid_t;
 #endif

 #ifdef VMM
+#define vmx86_vmm 1
 #define VMM_ONLY(x) x
 #else
+#define vmx86_vmm 0
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
@@ -687,7 +686,6 @@ typedef int pid_t;
 * display/printer drivers only.
 */
 #ifdef _WIN32
-#ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
 #ifdef _WIN64
 #define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
@@ -699,15 +697,8 @@ typedef int pid_t;
 #define WinDrvPrint(arg, ...)
 #define WinDrvEngPrint(arg, ...)
 #endif
-#endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -718,7 +709,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -795,4 +786,30 @@ typedef int pid_t;
 #define END_PTR_TO_ALIGNED_VAR \
   } while (0)

+
+/*
+ * -Wswitch means that when you pass switch an enum that it's looking for
+ * all values from that enum, and only that enum, to be accounted for.
+ * "default:;" is fine for catching values you don't care about. But today
+ * we have a bunch of code that uses internal and external enum values, or
+ * in other words combines two enums into a single variable. This cast is
+ * the workaround, but we really need to fix this mess.
+ */
+#define UNCHECKED_SWITCH__FIXME(x) switch ((uint64)(x))
+
+
+/*
+ * When clang static analyzer parses source files, it implicitly defines
+ * __clang_analyzer__ macro. We use this to define our custom macro to stop
+ * its execution for the current path of analysis by calling a function that
+ * doesn't return, making it think that it hit a failed assertion.
+ *
+ * DO NOT use to silence the analyzer! See PR2447238.
+ */
+#ifdef __clang_analyzer__
+#define VMW_CLANG_ANALYZER_NORETURN() Panic("Disable Clang static analyzer")
+#else
+#define VMW_CLANG_ANALYZER_NORETURN() ((void)0)
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,18 +63,9 @@
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/*
- * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -83,7 +74,6 @@
 #ifdef __x86_64__
 #define VM_X86_64
 #define vm_x86_64 1
-#define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
 #else
@@ -111,6 +101,7 @@

 #pragma warning (3 :4505) // unreferenced local function
 #pragma warning (disable :4018) // signed/unsigned mismatch
+#pragma warning (suppress:4619) // suppress warning next line (C4761 was removed in vs2019u4)
 #pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
 #pragma warning (disable :4305) // truncation from 'const int' to 'short'
 #pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
@@ -120,14 +111,91 @@

 #endif

-#if defined(__cplusplus) && __cplusplus >= 201103L || \
-    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
-    defined(__APPLE__) || defined(HAVE_STDINT_H)
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif
+
 /*
- * We're using <stdint.h> instead of <cstdint> below because some C++ code
- * deliberately compiles without C++ include paths.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif
+

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -138,40 +206,6 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-#else /* !HAVE_STDINT_H */
-
-/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
-#ifdef _MSC_VER
-
-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
-
-#elif __GNUC__
-#   if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#   else
-/*
- * Only strict c90 (without extensions) lacks a 'long long' type.
- * If this declaration fails ... use -std=c99 or -std=gnu90.
- */
-typedef unsigned long long uint64;
-typedef long long int64;
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-

 /*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
@@ -202,14 +236,6 @@ typedef char           Bool;
 #define IS_BOOL(x)     (((x) & ~1) == 0)


-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */
-
 /*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
@@ -218,9 +244,6 @@ typedef char           Bool;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -234,14 +257,11 @@ typedef char           Bool;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -249,46 +269,10 @@ typedef char           Bool;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -337,10 +321,11 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* macOS hosts use the same formatters for 32- and 64-bit. */
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
   #define FMT64         "ll"
-   #if KERNEL
+   #if defined(__APPLE__) && KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
@@ -452,7 +437,6 @@ typedef uintptr_t VPN;

 typedef uint64    PA;
 typedef uint64    PPN;
-typedef uint32    PPNTMP;

 typedef uint64    TPA;
 typedef uint64    TPPN;
@@ -470,9 +454,7 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint64    PgCnt64;
 typedef uint64    PageCnt;
-typedef uint64    PgNum64;
 typedef uint64    PageNum;
 typedef unsigned  MemHandle;
 typedef unsigned  IoHandle;
@@ -515,7 +497,7 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
@@ -598,15 +580,14 @@ typedef void * UserVA;


 /* Maximal observable PPN value. */
-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
 #define INVALID_PPN32     ((PPN32)0xffffffff)
-#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
-#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -624,7 +605,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((uint32)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -706,22 +688,28 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-3.3 (circa 2003).  Enforce this here, various things below
+ * gcc-4.1 (circa 2006).  Enforce this here, various things below
 * this line depend upon it.
 *
- * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
- * The various linux kernel modules may use older (gcc-3.3) compilers.
+ * Current oldest compilers:
+ * - guest tools: 4.1.2 (freebsd/solaris)
+ * - buildhost compiler: 4.4.3
+ * - hosted kernel modules: 4.5
+ *
+ * SWIG's preprocessor is exempt.
 */
-#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
-#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
+#ifndef SWIG
+#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
+#error "gcc version is too old, need gcc-4.1 or better"
+#endif
 #endif

 /*
- * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Similarly, we require a compiler that is at least vs2012.
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1400
-#error "cl.exe version is too old, need vc80 or better"
+#if defined _MSC_VER && _MSC_VER < 1700
+#error "cl.exe version is too old, need vs2012 or better"
 #endif


@@ -837,7 +825,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -883,12 +871,10 @@ typedef void * UserVA;
 #endif

 #ifndef UNUSED_TYPE
-// XXX _Pragma would better but doesn't always work right now.
 #  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
 #endif

 #ifndef UNUSED_VARIABLE
-// XXX is there a better way?
 #  define UNUSED_VARIABLE(_var) (void)_var
 #endif

@@ -903,29 +889,21 @@ typedef void * UserVA;

 /*
 * ALIGNED specifies minimum alignment in "n" bytes.
+ *
+ * NOTE: __declspec(align) has limited syntax; it must essentially be
+ *       an integer literal.  Expressions, such as sizeof(), do not
+ *       work.
 */

 #ifdef __GNUC__
 #define ALIGNED(n) __attribute__((__aligned__(n)))
+#elif defined(_MSC_VER)
+#define ALIGNED(n) __declspec(align(n))
 #else
 #define ALIGNED(n)
 #endif


-/*
- * Encapsulate the syntactic differences between gcc and msvc alignment control.
- * BOUNDARY must match in the prefix and suffix.
- */
-
-#ifdef _WIN32
-#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
-#define ALIGN_SUFFIX(BOUNDRY)
-#else
-#define ALIGN_PREFIX(BOUNDRY)
-#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
-#endif
-
-
 /*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
@@ -934,82 +912,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -0,0 +1,88 @@
+/*********************************************************
+ * Copyright (C) 2012,2017-2018,2020 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vm_idt_x86.h --
+ *
+ *	Definitions for IDT use in x86/x86-64 ESX and hosted VMware Products.
+ */
+
+#ifndef _VM_IDT_X86_H_
+#define _VM_IDT_X86_H_
+
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMCORE
+#include "includeCheck.h"
+
+#if defined __cplusplus
+extern "C" {
+#endif
+
+/*
+ * User worlds are the only worlds that expect to receive
+ * interrupts on a stack specified in the %rsp entries of
+ * the TSS.
+ */
+#define TSS_RSP0 0
+#define TSS_RSP1 1
+#define TSS_RSP2 2
+
+/*
+ * The monitor and the vmkernel use gate stub handlers of a single size in
+ * their IDTs.  The only exception is the #PF gate (#14) which is twice as
+ * long in the monitor IDT, to copy CR2.
+ */
+#define IDT_STUB_SIZE              16
+
+#define IST_NONE                    0
+/* IST entries for the monitor. */
+#define IST_VMM_DF                  1
+#define IST_VMM_NMI                 2
+#define IST_VMM_MCE                 3
+#define MAX_VMM_IST                 3
+/* IST entries for the vmkernel. */
+#define IST_VMK_MCE                 5
+#define IST_VMK_DF                  6
+#define IST_VMK_NMI                 7
+
+
+static INLINE int
+IDT_MonitorISTForVector(int v)
+{
+   /*
+    * For non-SVM execution, the monitor uses the IST for #DF, #NMI, and #MC.
+    */
+   if (v == EXC_DF) {
+      return IST_VMM_DF;
+   } else if (v == EXC_NMI) {
+      return IST_VMM_NMI;
+   } else if (v == EXC_MC) {
+      return IST_VMM_MCE;
+   }
+   return IST_NONE;
+}
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
+#endif /* _VM_IDT_X86_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,22 +40,25 @@
 /*
 * These definitions suit both the x64 and arm64 architectures. In particular
 * - At all levels, a page table is a 4KB page containing 512 8-byte entries.
- * - Each entry maps 39, 30, 21 and 12 bits at the respective page table
+ * - Each entry maps 48, 39, 30, 21 and 12 bits at the respective page table
 *   levels (with standard page sizes).
 */

-typedef uint64 PT_L4E;
-typedef uint64 PT_L3E;
-typedef uint64 PT_L2E;
-typedef uint64 PT_L1E;
+typedef uint64 PT_Entry;
+typedef PT_Entry PT_L5E;
+typedef PT_Entry PT_L4E;
+typedef PT_Entry PT_L3E;
+typedef PT_Entry PT_L2E;
+typedef PT_Entry PT_L1E;

 typedef enum {
   PT_LEVEL_1 = 1,
   PT_LEVEL_2,
   PT_LEVEL_3,
   PT_LEVEL_4,
+   PT_LEVEL_5,
   PT_LEVEL_STOP = PT_LEVEL_1,
-   PT_MAX_LEVELS = PT_LEVEL_4
+   PT_MAX_LEVELS = PT_LEVEL_5
 } PT_Level;

 #define PT_PTE_SIZE           8
@@ -76,6 +79,7 @@ typedef enum {

 /* Address space size (in 4KB pages) covered by a level-_l page table entry. */
 #define PT_PAGES_PER_LE(_l)   ((uint64)1 << PT_LE_PG_SHIFT(_l))
+#define PT_PAGES_PER_L5E      PT_PAGES_PER_LE(PT_LEVEL_5)
 #define PT_PAGES_PER_L4E      PT_PAGES_PER_LE(PT_LEVEL_4)
 #define PT_PAGES_PER_L3E      PT_PAGES_PER_LE(PT_LEVEL_3)
 #define PT_PAGES_PER_L2E      PT_PAGES_PER_LE(PT_LEVEL_2)
@@ -86,6 +90,7 @@ typedef enum {
 * address space size (in bytes) covered by a level-_l page table entry.
 */
 #define PT_LE_SHIFT(_l)       (PAGE_SHIFT + PT_LE_PG_SHIFT(_l))
+#define PT_L5E_SHIFT          PT_LE_SHIFT(PT_LEVEL_5)
 #define PT_L4E_SHIFT          PT_LE_SHIFT(PT_LEVEL_4)
 #define PT_L3E_SHIFT          PT_LE_SHIFT(PT_LEVEL_3)
 #define PT_L2E_SHIFT          PT_LE_SHIFT(PT_LEVEL_2)
@@ -93,6 +98,7 @@ typedef enum {

 /* Address space size (in bytes) covered by a level-_l page table entry. */
 #define PT_LE_SIZE(_l)        ((uint64)1 << PT_LE_SHIFT(_l))
+#define PT_L5E_SIZE           PT_LE_SIZE(PT_LEVEL_5)
 #define PT_L4E_SIZE           PT_LE_SIZE(PT_LEVEL_4)
 #define PT_L3E_SIZE           PT_LE_SIZE(PT_LEVEL_3)
 #define PT_L2E_SIZE           PT_LE_SIZE(PT_LEVEL_2)
@@ -103,6 +109,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LA_2_LOFF(_a, _l)  (((_a) >> PT_LE_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LA_2_L5OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LA_2_L4OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LA_2_L3OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LA_2_L2OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_2)
@@ -113,6 +120,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LPN_2_LOFF(_a, _l) (((_a) >> PT_LE_PG_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LPN_2_L5OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LPN_2_L4OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LPN_2_L3OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LPN_2_L2OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_2)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -30,9 +30,6 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-#define INTERRUPT_REDIRECTION_BITMAP_SIZE 32
-#define IO_PERMISSION_BITMAP_SIZE 0
-

 #define VMM_PANIC_NONE         0
 #define VMM_PANIC_MSG_COPYING  1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,6 +32,7 @@
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMX
 #include "includeCheck.h"
+#include "community_source.h"

 /*
 * Page remapping definitions.
@@ -46,6 +47,7 @@
 #define VMMEM_ANON_IOABLE_PAGE    VMMEM_FLAG_BIT(4)
 #define VMMEM_ANON_ALL_FLAGS      MASK(5)

+#define VMMEM_GUEST_NO_FLAGS      0
 #define VMMEM_GUEST_WRITEABLE     VMMEM_FLAG_BIT(0)
 #define VMMEM_GUEST_BREAKCOW      VMMEM_FLAG_BIT(1)
 #define VMMEM_GUEST_2M_PAGE       VMMEM_FLAG_BIT(2)
@@ -83,7 +85,7 @@
 * Structure used to query platform about the page state.
 */
 typedef struct PlatformPageInfoList {
-   PgCnt64 numPages;
+   PageCnt numPages;
   BPN     bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
   MPN     mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
   uint8   flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
@@ -96,9 +98,12 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

-#define VMMEM_SERVICES_DEFS                               \
-   MDEF(VMMEM_SERVICES_TYPE_P2M,  P2MUpdate_FilterPages)  \
-   MDEF(VMMEM_SERVICES_TYPE_SWAP, BusMemSwap_FilterPages)
+#define MDEF_DRAINER
+
+#define VMMEM_SERVICES_DEFS                                       \
+   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
+   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
+   MDEF_DRAINER

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -89,6 +89,8 @@
 #define CR4_PCE        0x00000100
 #define CR4_OSFXSR     0x00000200 // CPU/OS supports SIMD insts
 #define CR4_OSXMMEXCPT 0x00000400 // #XF exception enable PIII only
+#define CR4_UMIP       0x00000800
+#define CR4_LA57       0x00001000
 #define CR4_VMXE       0x00002000
 #define CR4_SMXE       0x00004000
 #define CR4_FSGSBASE   0x00010000
@@ -97,7 +99,8 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
-#define CR4_RESERVED   CONST64U(0xffffffffff889800)
+#define CR4_CET        0x00800000
+#define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

 /*
@@ -201,8 +204,16 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_CP           21  // Control Protection exception.
+#define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

+/* Bitmap of the exception vectors that have associated error codes. */
+#define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
+                                (1u << EXC_NP) | (1u << EXC_SS) | \
+                                (1u << EXC_GP) | (1u << EXC_PF) | \
+                                (1u << EXC_AC) | (1u << EXC_CP))
+
 /*
 * eflag/rflag definitions.
 */
@@ -211,24 +222,24 @@

 typedef enum x86_FLAGS {
   EFLAGS_NONE         = 0,
-   EFLAGS_CF           = (1 << 0),     /* User */ 
-   EFLAGS_SET          = (1 << 1),                                  
-   EFLAGS_PF           = (1 << 2),     /* User */ 
-   EFLAGS_AF           = (1 << 4),     /* User */ 
-   EFLAGS_ZF           = (1 << 6),     /* User */ 
-   EFLAGS_SF           = (1 << 7),     /* User */ 
-   EFLAGS_TF           = (1 << 8),     /* Priv */ 
-   EFLAGS_IF           = (1 << 9),     /* Priv */ 
-   EFLAGS_DF           = (1 << 10),    /* User */ 
-   EFLAGS_OF           = (1 << 11),    /* User */ 
-   EFLAGS_NT           = (1 << 14),    /* Priv */ 
-   EFLAGS_RF           = (1 << 16),    /* Priv */ 
-   EFLAGS_VM           = (1 << 17),    /* Priv */ 
-   EFLAGS_AC           = (1 << 18),    /* Priv */ 
-   EFLAGS_VIF          = (1 << 19),    /* Priv */ 
-   EFLAGS_VIP          = (1 << 20),    /* Priv */ 
+   EFLAGS_CF           = (1 << 0),     /* User */
+   EFLAGS_SET          = (1 << 1),
+   EFLAGS_PF           = (1 << 2),     /* User */
+   EFLAGS_AF           = (1 << 4),     /* User */
+   EFLAGS_ZF           = (1 << 6),     /* User */
+   EFLAGS_SF           = (1 << 7),     /* User */
+   EFLAGS_TF           = (1 << 8),     /* Priv */
+   EFLAGS_IF           = (1 << 9),     /* Priv */
+   EFLAGS_DF           = (1 << 10),    /* User */
+   EFLAGS_OF           = (1 << 11),    /* User */
+   EFLAGS_NT           = (1 << 14),    /* Priv */
+   EFLAGS_RF           = (1 << 16),    /* Priv */
+   EFLAGS_VM           = (1 << 17),    /* Priv */
+   EFLAGS_AC           = (1 << 18),    /* Priv */
+   EFLAGS_VIF          = (1 << 19),    /* Priv */
+   EFLAGS_VIP          = (1 << 20),    /* Priv */
   EFLAGS_ID           = (1 << 21),    /* Priv */
-   
+
   EFLAGS_IOPL         = 3 << EFLAGS_IOPL_SHIFT,
   EFLAGS_ARITH        = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_OF),
@@ -240,7 +251,7 @@ typedef enum x86_FLAGS {
   EFLAGS_ALL          = (EFLAGS_CF | EFLAGS_PF | EFLAGS_AF | EFLAGS_ZF |
                          EFLAGS_SF | EFLAGS_DF | EFLAGS_OF | EFLAGS_TF |
                          EFLAGS_IF | EFLAGS_IOPL | EFLAGS_NT | EFLAGS_RF |
-                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP | 
+                          EFLAGS_VM | EFLAGS_AC | EFLAGS_VIF | EFLAGS_VIP |
                          EFLAGS_ID),
   EFLAGS_ALL_16       = EFLAGS_ALL & 0xffff,
   EFLAGS_REAL_32      = (EFLAGS_ALL & ~(EFLAGS_VIP | EFLAGS_VIF | EFLAGS_VM)),
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,46 +39,6 @@
 #include "x86cpuid.h"


-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#ifdef _MSC_VER
-#ifdef __cplusplus
-extern "C" {
-#endif
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-void __cpuid(int regs[4], int eax);
-#pragma intrinsic(__cpuid)
-
-/*
- * __cpuidex has been supported since VS2008
- */
-#if _MSC_VER >= 1500
-void __cpuidex(int regs[4], int eax, int ecx);
-#pragma intrinsic(__cpuidex)
-#endif /* _MSC_VER >= 1500 */
-#endif /* VM_X86_64 */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
-
 #ifdef __GNUC__ // {

 /*
@@ -89,68 +49,57 @@ void __cpuidex(int regs[4], int eax, int ecx);
 *
 */

-/*
- * %ebx is reserved on i386 PIC.  Apple's gcc-5493 (gcc 4.0) compiling
- * for x86_64 incorrectly errors out saying %ebx is reserved.  This is
- * Apple bug 7304232.
- */
-#if vm_x86_64 ? (defined __APPLE_CC__ && __APPLE_CC__ == 5493) : defined __PIC__
-#if vm_x86_64
-/*
- * Note that this generates movq %rbx,%rbx; cpuid; xchgq %rbx,%rbx ...
- * Unfortunately Apple's assembler does not have .ifnes, and I cannot
- * figure out how to do that with .if.   If we ever enable this code
- * on other 64bit systems, both movq & xchgq should be surrounded by
- * .ifnes \"%%rbx\", \"%q1\" & .endif
- */
-#define VM_CPUID_BLOCK  "movq %%rbx, %q1\n\t" \
-                        "cpuid\n\t"           \
-                        "xchgq %%rbx, %q1\n\t"
-#define VM_EBX_OUT(reg) "=&r"(reg)
+#if defined VM_X86_64 || !defined __PIC__
+#define VM_CPUID_BLOCK  "cpuid"
+#define VM_EBX_OUT(reg) "=b"(reg)
 #else
+/* %ebx is reserved on i386 PIC. */
 #define VM_CPUID_BLOCK  "movl %%ebx, %1\n\t" \
                        "cpuid\n\t"          \
                        "xchgl %%ebx, %1\n\t"
 #define VM_EBX_OUT(reg) "=&rm"(reg)
 #endif
-#else
-#define VM_CPUID_BLOCK  "cpuid"
-#define VM_EBX_OUT(reg) "=b"(reg)
-#endif

 static INLINE void
-__GET_CPUID(int eax,         // IN
+__GET_CPUID(uint32     eax,  // IN
            CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax)
      : "memory"
   );
 }

 static INLINE void
-__GET_CPUID2(int eax,         // IN
-             int ecx,         // IN
+__GET_CPUID2(uint32 eax,      // IN
+             uint32 ecx,      // IN
             CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax), "c" (ecx)
      : "memory"
   );
 }

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int eax) // IN
+__GET_EAX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx)
      : "a" (eax)
      : "memory", "%ecx", "%edx"
   );
@@ -159,7 +108,7 @@ __GET_EAX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int eax) // IN
+__GET_EBX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

@@ -174,14 +123,16 @@ __GET_EBX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int eax) // IN
+__GET_ECX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ecx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=c" (ecx)
      : "a" (eax)
      : "memory", "%edx"
   );
@@ -190,14 +141,16 @@ __GET_ECX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int eax) // IN
+__GET_EDX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 edx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=d" (edx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=d" (edx)
      : "a" (eax)
      : "memory", "%ecx"
   );
@@ -206,271 +159,59 @@ __GET_EDX_FROM_CPUID(int eax) // IN
 }


-static INLINE uint32
-__GET_EAX_FROM_CPUID4(int ecx) // IN
-{
-   uint32 eax;
-   uint32 ebx;
-
-   __asm__ __volatile__(
-      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
-      : "a" (4), "c" (ecx)
-      : "memory", "%edx"
-   );
-
-   return eax;
-}
-
 #undef VM_CPUID_BLOCK
 #undef VM_EBX_OUT

 #elif defined(_MSC_VER) // } {

 static INLINE void
-__GET_CPUID(int input, CPUIDRegs *regs)
+__GET_CPUID(uint32 input, CPUIDRegs *regs)
 {
-#ifdef VM_X86_64
   __cpuid((int *)regs, input);
-#else
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-#endif
 }

-#ifdef VM_X86_64
-
-#if _MSC_VER >= 1500
-
-/*
- * __cpuidex has been supported since VS2008
- */
-
 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __cpuidex((int *)regs, inputEax, inputEcx);
 }

-#else // _MSC_VER >= 1500
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs);
-#endif // _MSC_VER >= 1500
-
-#else // VM_X86_64
-
-static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
-{
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, inputEax
-   __asm mov  ecx, inputEcx
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-}
-#endif
-
 static INLINE uint32
-__GET_EAX_FROM_CPUID(int input)
+__GET_EAX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.eax;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int input)
+__GET_EBX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ebx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ebx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int input)
+__GET_ECX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ecx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ecx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int input)
+__GET_EDX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.edx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

-#ifdef VM_X86_64
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern uint32
-__GET_EAX_FROM_CPUID4(int inputEcx);
-
-#else // VM_X86_64
-
-static INLINE uint32
-__GET_EAX_FROM_CPUID4(int inputEcx)
-{
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, 4
-   __asm mov  ecx, inputEcx
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-}
-
-#endif // VM_X86_64
-
 #else // }
 #error
 #endif

 #define CPUID_FOR_SIDE_EFFECTS() ((void)__GET_EAX_FROM_CPUID(0))

-/* The first parameter is used as an rvalue and then as an lvalue. */
-#define GET_CPUID(_ax, _bx, _cx, _dx) { \
-   CPUIDRegs regs;                      \
-   __GET_CPUID(_ax, &regs);             \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
-#define GET_CPUID2(_ax, _bx, _cx, _dx) {\
-   CPUIDRegs regs;                      \
-   __GET_CPUID2(_ax, _cx, &regs);       \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,43 +97,78 @@ static INLINE uint32 Desc_LongMode(const Descriptor *d) { return d->longmode; }
 static INLINE uint32 Desc_DB(const Descriptor *d)       { return d->DB; }
 static INLINE uint32 Desc_Gran(const Descriptor *d)     { return d->gran; }

-static INLINE uint32 Desc64_Type(const Descriptor64 *d)     { return d->type; }
-static INLINE uint32 Desc64_S(const Descriptor64 *d)        { return d->S; }
-static INLINE uint32 Desc64_DPL(const Descriptor64 *d)      { return d->DPL; }
-static INLINE uint32 Desc64_Present(const Descriptor64 *d)  { return d->present; }
-static INLINE uint32 Desc64_AVL(const Descriptor64 *d)      { return d->AVL; }
-static INLINE uint32 Desc64_Gran(const Descriptor64 *d)     { return d->gran; }
-static INLINE uint32 Desc64_ExtAttrs(const Descriptor64 *d) { return d->ext_attrs; }
+static INLINE uint32
+Desc64_Type(const Descriptor64 *d)
+{
+   return (uint32)d->type;
+}
+
+static INLINE uint32
+Desc64_S(const Descriptor64 *d)
+{
+   return (uint32)d->S;
+}
+
+static INLINE uint32
+Desc64_DPL(const Descriptor64 *d)
+{
+   return (uint32)d->DPL;
+}
+
+static INLINE uint32
+Desc64_Present(const Descriptor64 *d)
+{
+   return (uint32)d->present;
+}
+
+static INLINE uint32
+Desc64_AVL(const Descriptor64 *d)
+{
+   return (uint32)d->AVL;
+}
+
+static INLINE uint32
+Desc64_Gran(const Descriptor64 *d)
+{
+   return (uint32)d->gran;
+}
+
+static INLINE uint32
+Desc64_ExtAttrs(const Descriptor64 *d)
+{
+   return (uint32)d->ext_attrs;
+}

 static INLINE LA32
 Desc_GetBase(const Descriptor *d)
 {
-   return (d->base_hi << 24) | (d->base_mid << 16) | d->base_lo;
+   return ((unsigned)d->base_hi  << 24) |
+          ((unsigned)d->base_mid << 16) | d->base_lo;
 }

 static INLINE LA64
 Desc64_GetBase(const Descriptor64 *d)
 {
-   return ((uint64)d->base_hi << 32)  | 
-          ((uint64)d->base_mid << 24) | (uint32)d->base_lo;
+   return ((uint64)d->base_hi << 32)  |
+          ((uint64)d->base_mid << 24) | (uint64)d->base_lo;
 }

 static INLINE LA32
 Desc64_GetBaseHi(const Descriptor64 *d)
 {
-   return d->base_hi;
+   return (LA32)d->base_hi;
 }

 static INLINE VA32
 Desc_GetLimit(const Descriptor *d)
 {
-   return (d->limit_hi << 16) | d->limit_lo;
+   return ((unsigned)d->limit_hi << 16) | d->limit_lo;
 }

 static INLINE VA32
 Desc64_GetLimit(const Descriptor64 *d)
 {
-   return ((uint32)d->limit_hi << 16) | (uint32)d->limit_lo;
+   return (VA32)(((unsigned)d->limit_hi << 16) | d->limit_lo);
 }

 static INLINE Bool
@@ -310,12 +345,13 @@ static INLINE uint32 DESC_PRESENT(Descriptor d)    { return d.present; }
 *-----------------------------------------------------------------------------
 */

-static INLINE void 
-Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type, 
-                           uint32 DPL, uint32 present, uint32 DB, uint32 gran)
+static INLINE void
+Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit,
+                           uint32 type, uint32 DPL, uint32 present,
+                           uint32 DB, uint32 gran)
 {
-   uint32 baseHi = (uint32) (base >> 32);
-   uint32 baseLo = (uint32) base;
+   uint32 baseHi = (uint32)(base >> 32);
+   uint32 baseLo = (uint32)base;

   /*
    * The first half of a 16-byte descriptor is a valid 8-byte descriptor
@@ -336,15 +372,17 @@ Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type
 *   Given descriptor, return the code/stack size that it specifies.
 *----------------------------------------------------------------------
 */
-static INLINE int
+static INLINE unsigned
 Desc_DBSize(const Descriptor *desc)
 {
   /*
    * Code/stack size is determined by the D/B bit; bit 22 of the 2nd word.
-    * Shift the bit to position 1, mask it out, add 2. Result: 2 or 4.
+    * Shift the bit to position 1, mask it out, add 2.
+    *
+    * Result: 2 or 4.
    */
-   int cSz = ((((const uint32*)desc)[1] >> 21) & 2) + 2;
-   ASSERT(cSz == (Desc_DB(desc) ? 4 : 2));
+   unsigned cSz = ((((const uint32 *)desc)[1] >> 21) & 2) + 2;
+   ASSERT(cSz == (Desc_DB(desc) ? 4U : 2U));
   return cSz;
 }

@@ -526,7 +564,7 @@ typedef struct Gate {
 static INLINE VA
 CallGate_GetOffset(const Gate *cg)
 {
-   return (cg->offset_hi << 16) | cg->offset_lo;
+   return ((unsigned)cg->offset_hi << 16) | cg->offset_lo;
 }

 static INLINE void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-/* 
+/*
 * x86msr.h --
 *
 *      MSR number definitions.
@@ -59,11 +59,8 @@ struct MSRReply {
    * we use it to correlate the replies of multiple queries.
    */
   uint64 tag;              // OUT
-
   uint64 msrVal;           // OUT
-
   uint8  implemented;      // OUT
-
   uint8  _pad[7];
 }
 #include "vmware_pack_end.h"
@@ -116,40 +113,61 @@ MSRQuery;
 #define MSR_TSC_AUX           0xc0000103
 #define MSR_BD_TSC_RATIO      0xc0000104

+/* CET MSRs */
+#define MSR_U_CET                            0x6a0
+#define MSR_S_CET                            0x6a2
+#define MSR_CET_SH_STK_EN                         (1ULL << 0)
+#define MSR_CET_WR_SHSTK_EN                       (1ULL << 1)
+#define MSR_PL0_SSP                          0x6a4
+#define MSR_PL1_SSP                          0x6a5
+#define MSR_PL2_SSP                          0x6a6
+#define MSR_PL3_SSP                          0x6a7
+#define MSR_ISST_ADDR                        0x6a8
+
+#define MSR_TEST_CTRL                        0x33
+#define MSR_TEST_CTRL_SPLIT_LOCK_DETECT           (1ULL << 29)
+
 #define IA32_MSR_ARCH_CAPABILITIES           0x10a
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
 #define MSR_ARCH_CAPABILITIES_RSBA                (1ULL << 2)
+#define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
+#define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)
+#define MSR_ARCH_CAPABILITIES_MDS_NO              (1ULL << 5)
+#define MSR_ARCH_CAPABILITIES_IF_PSCHANGE_MC_NO   (1ULL << 6)
+#define MSR_ARCH_CAPABILITIES_TSX_CTRL            (1ULL << 7)
+#define MSR_ARCH_CAPABILITIES_TAA_NO              (1ULL << 8)

+#define MSR_FLUSH_CMD                        0x10b
+#define MSR_FLUSH_CMD_FLUSH_L1D                   (1ULL << 0)

 #define MSR_SPEC_CTRL_IBRS                        (1UL << 0)
 #define MSR_SPEC_CTRL_STIBP                       (1UL << 1)
+#define MSR_SPEC_CTRL_SSBD                        (1UL << 2)

 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

+#define MSR_TSX_CTRL                         0x122
+#define MSR_TSX_CTRL_RTM_DISABLE                  (1ULL << 0)
+#define MSR_TSX_CTRL_CPUID_CLEAR                  (1ULL << 1)
+
 #define MSR_MISC_FEATURES_ENABLES            0x140

 /* Intel Core Architecture and later: use only architected counters. */
-#define IA32_MSR_PERF_CAPABILITIES                0x345
-#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT        0
-#define MSR_PERF_CAPABILITIES_LBRFMT_MASK         0x3f
-#define MSR_PERF_CAPABILITIES_PEBSTRAP            (1u << 6)
-#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS    (1u << 7)
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT 8
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK  0xf
-#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM    (1u << 12)
-#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES   (1u << 13)
+#define IA32_MSR_PERF_CAPABILITIES                   0x345
+#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT           0
+#define MSR_PERF_CAPABILITIES_LBRFMT_MASK            0x3f
+#define MSR_PERF_CAPABILITIES_PEBSTRAP               (1u << 6)
+#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS       (1u << 7)
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT    8
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK     0xf
+#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM       (1u << 12)
+#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES      (1u << 13)
+#define MSR_PERF_CAPABILITIES_PEBS_ADAPTIVE_DATA     (1u << 14)
+#define MSR_PERF_CAPABILITIES_PERF_METRICS_AVAILABLE (1u << 15)

 #define IA32_MSR_PEBS_ENABLE                      0x3f1

-typedef enum {
-   SL_PMC_FLAGS_NONE             = 0x00, /* No flags.                      */
-   SL_PMC_FLAGS_LBR_VA32         = 0x01, /* LBR format: 32-bit VA.         */
-   SL_PMC_FLAGS_LBR_LA64         = 0x02, /* LBR format: 64-bit LA.         */
-   SL_PMC_FLAGS_LBR_VA64         = 0x04, /* LBR format: 64-bit VA.         */
-   SL_PMC_FLAGS_LBR_PACKED_VA32  = 0x08, /* LBR format: 2x32-bit VAs.      */
-} StateLoggerPMCFlags;
-
 #define MSR_MTRR_BASE0        0x00000200
 #define MSR_MTRR_MASK0        0x00000201
 #define MSR_MTRR_BASE1        0x00000202
@@ -265,6 +283,8 @@ typedef enum {

 #define MSR_XSS              0x00000da0  // Extended Supervisor State Mask

+#define MSR_MPX_LAX          0x00001000  // MPX Linear Address Extension
+
 /* RTIT MSRs */
 #define MSR_RTIT_CTL              0x00000570
 #define MSR_RTIT_STATUS           0x00000571
@@ -444,7 +464,6 @@ typedef enum {

 /* MSR_EFER bits. */
 #define MSR_EFER_SCE         0x0000000000000001ULL  /* Sys call ext'ns:  r/w */
-#define MSR_EFER_RAZ         0x00000000000000feULL  /* Read as zero          */
 #define MSR_EFER_LME         0x0000000000000100ULL  /* Long mode enable: r/w */
 #define MSR_EFER_LMA         0x0000000000000400ULL  /* Long mode active: r/o */
 #define MSR_EFER_NXE         0x0000000000000800ULL  /* No-exec enable:   r/w */
@@ -452,7 +471,11 @@ typedef enum {
 #define MSR_EFER_LMSLE       0x0000000000002000ULL  /* LM seg lim enable:r/w */
 #define MSR_EFER_FFXSR       0x0000000000004000ULL  /* Fast FXSAVE:      r/w */
 #define MSR_EFER_TCE         0x0000000000008000ULL  /* Trans. cache ext. r/w */
-#define MSR_EFER_MBZ         0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+/* Vendor specific EFER bits */
+#define MSR_EFER_INTEL_MBZ   0xffffffffffff02feULL  /* Must be zero (resrvd) */
+#define MSR_EFER_INTEL_RAZ   0x0000000000000000ULL  /* Read as zero          */
+#define MSR_EFER_AMD_MBZ     0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+#define MSR_EFER_AMD_RAZ     0x00000000000000feULL  /* Read as zero          */

 #define MSR_AMD_PATCH_LOADER 0xc0010020

@@ -470,6 +493,10 @@ typedef enum {
 #endif
 #define MSR_K8_SYSCFG_MTRRTOM2EN         (1ULL<<21)
 #define MSR_K8_SYSCFG_TOM2FORCEMEMTYPEWB (1ULL<<22)
+#define MSR_K8_SYSCFG_SMEE               (1ULL<<23)
+#define MSR_K8_SYSCFG_SNPE               (1ULL<<24)
+#define MSR_K8_SYSCFG_VMPLE              (1ULL<<25)
+
 #define MSR_K8_TOPMEM2       0xc001001d

 /* AMD "Greyhound" MSRs */
@@ -507,9 +534,37 @@ typedef enum {
 #define MSR_VM_CR_SVME_DISABLE     0x0000000000000010ULL
 #define MSR_VM_CR_RESERVED         0xffffffffffffffe0ULL

+/* SEV related MSRs. */
+#define MSR_VMPAGE_FLUSH           0xc001011e
+#define MSR_GHCB_PA                0xc0010130
+#define MSR_GHCB_PA_FUNCTION_MASK       0xfff
+#define MSR_GHCB_PA_SEVINFO_HV              1
+#define MSR_GHCB_PA_SEVINFO_REQ             2
+#define MSR_GHCB_PA_AP_JUMP_TABLE           3
+#define MSR_GHCB_PA_CPUID_REQ               4
+#define MSR_GHCB_PA_CPUID_RESP              5
+#define MSR_GHCB_PA_TERMINATE             256
+#define MSR_SEV_STATUS             0xc0010131
+
+#define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
+#define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
+#define MSR_SEV_STATUS_SEV_SNP_EN  0x0000000000000004ULL // SEV-SNP is enabled
+
+/* SEV-SNP (Secure Nested Paging) MSRs. */
+#define MSR_RMP_BASE              0xc0010132 // Address of first byte of RMP
+#define MSR_RMP_END               0xc0010133 // Address of last byte of RMP
+
 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)

+#define MSR_AMD_LS_CFG           0xc0011020  // load-store configuration
+#define MSR_AMD_LS_CFG_SSBD_BULLDOZER (1ULL<<54)  // BD family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_KYOTO     (1ULL<<33)  // Kyoto family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_ZEN       (1ULL<<10)  // Zen family non-arch SSBD
+
+#define MSR_AMD_VIRT_SPEC_CTRL   0xc001011f  // Virtual speculation control
+#define MSR_AMD_VIRT_SPEC_CTRL_SSBD   (1ULL<<2)   // Virtual SSBD
+
 /* Syscall/Sysret related MSRs (x86_64) */
 #define MSR_STAR             0xc0000081 // Also present on Athlons.
 #define MSR_LSTAR            0xc0000082
@@ -591,6 +646,8 @@ typedef enum {
 #define MSR_HYPERV_HYPERCALL_EN                  1ULL
 #define MSR_HYPERV_REFERENCE_TSC_EN              1ULL
 #define MSR_HYPERV_VP_ASSIST_EN                  1ULL
+#define MSR_HYPERV_SIEF_EN                       1ULL
+#define MSR_HYPERV_SIM_EN                        1ULL

 #define MSR_HYPERV_GUESTOSID_VENDOR_SHIFT        48
 #define MSR_HYPERV_GUESTOSID_VENDOR_MASK         0xfULL
@@ -600,6 +657,9 @@ typedef enum {
 #define MSR_HYPERV_GUESTOSID_OS_MASK             0xfULL
 #define MSR_HYPERV_GUESTOSID_OS_WINNT_DERIVATIVE 4ULL

+/* MSR for forcing RTM abort to recover PMC3 (see PR 2333817) */
+#define MSR_TSX_FORCE_ABORT                      0x0000010f
+#define MSR_TSX_FORCE_ABORT_RTM_BIT_INDEX        0

 /*
 * MTRR bit description
@@ -657,14 +717,19 @@ typedef unsigned char MTRRType;



+// Platform Quality of Service (PQM) MSRs
+#define MSR_INTEL_PQM_EVTSEL    0xc8d
+#define MSR_INTEL_PQM_CTR       0xc8e
+#define MSR_INTEL_PQM_ASSOC     0xc8f

-/*
- * Platform Quality of Service
- */
+ // Platform Quality Enforcement (PQE) MSRs
+#define MSR_INTEL_PQE_CLOS_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_MASK_MAX      0xd8f

-#define MSR_INTEL_QOS_EVTSEL    0xc8d
-#define MSR_INTEL_QOS_CTR       0xc8e
-#define MSR_INTEL_QOS_ASSOC     0xc8f
+#define MSR_INTEL_PQE_CLOS_L3_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_L3_MASK_MAX      0xd0f
+#define MSR_INTEL_PQE_CLOS_L2_MASK_BASE     0xd10
+#define MSR_INTEL_PQE_CLOS_L2_MASK_MAX      0xd4f

 static INLINE uint32
 X86MSR_SysCallEIP(uint64 star)
@@ -687,6 +752,94 @@ X86MSR_SysRetCS(uint64 star)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * X86MSR_{G,S}etMSR --
+ *
+ *      Get or set an MSR in hardware. The interface should remain
+ *      agnostic to the underlying compiler and architecture.
+ *
+ *----------------------------------------------------------------------
+ */
+
+#ifdef __GNUC__
+/*
+ * Checked against the Intel manual and GCC --hpreg
+ * volatile because the msr can change without the compiler knowing it
+ * (when we use wrmsr).
+ */
+#ifdef VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
+      : "=a" (msr)
+      : "c" (cx)
+      : "%rdx"
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "a" ((uint32) value),
+        "d" ((uint32)(value >> 32)),
+        "c" (cx)
+    );
+}
+#else // __GNUC__ && !VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr"
+      : "=A" (msr)
+      : "c"  (cx)
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "A" (value),
+        "c" (cx)
+    );
+}
+#endif
+#elif defined _MSC_VER // !__GNUC__ && _MSC_VER
+unsigned __int64  __readmsr(unsigned long);
+void              __writemsr(unsigned long, unsigned __int64);
+#pragma intrinsic(__readmsr, __writemsr)
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   return __readmsr((unsigned long)cx);
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
+}
+#else // !__GNUC__ && !_MSC_VER
+#error No compiler defined for RDMSR/WRMSR.
+#endif
+
+
 #if defined __cplusplus
 }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,6 +40,7 @@
 #include "x86types.h"
 #include "vm_pagetable.h"
 #include "x86paging_common.h"
+#include "vm_assert.h"

 #define LM_PTE_PFN_MASK      CONST64U(0xffffffffff000)
 #define LM_PTE_2_PFN(_pte)   (((_pte) & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
@@ -52,7 +53,6 @@
 #define LM_AVAIL_MASK         (CONST64(0x7) << LM_AVAIL_SHIFT)
 #define LM_FLAGS_MASK         CONST64(0x80000000000001ff)
 #define LM_CR3_FLAGS_MASK     CONST64(0x18)
-#define LM_L4_L3_RSVD_MASK    CONST64(0x180)
 #define LM_L3_1G_RSVD_MASK    CONST64(0x3fffe000)

 #define LM_MAKE_CR3(_mpfn, _flags) \
@@ -65,12 +65,62 @@
                    ((uint64)(_flags) & LM_FLAGS_MASK))

 #define LM_MAKE_PDE(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L5E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L4E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L3E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L2E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L1E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)


+/*
+ *----------------------------------------------------------------------
+ *
+ * LMPTEIsSafe --
+ *
+ *    A shadow PTE is considered "safe" if any of the following conditions are
+ *    met:
+ *
+ *      a) It is not terminal (i.e., present with no reserved bits set)
+ *      b) Terminal, but with MPN and PS fields set to zero
+ *      c) Terminal, but with MPN field specifying an uncachable page
+ *
+ *    In practice, for condition c), we require that bits 45:43 of the EPTE are
+ *    set to b'111.  The position of these bits is undocumented and not
+ *    architectural; they are truly magic.
+ *
+ *----------------------------------------------------------------------
+ */
+#ifdef VMX86_DEBUG
+
+#define LM_SAFE_BITS_MASK (MASK64(3) << 43)
+#define LM_SAFE_BITS(_v)  ((_v) & LM_SAFE_BITS_MASK)
+
+static INLINE Bool
+LMPTEIsTerminal(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   uint64 rsvd2m = (MASK(PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvd1g = (MASK(2 * PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvdl4 = PTE_PS;
+   uint64 rsvd = physMask & MASK64(52);
+
+   return (!PTE_PRESENT(pte) ||
+           (pte & LM_PTE_PFN_MASK) & rsvd) != 0 ||
+           ((pte & PTE_PS) != 0 &&
+            ((level == PT_LEVEL_2 && (pte & rsvd2m) != 0) ||
+             (level == PT_LEVEL_3 && (pte & rsvd1g) != 0))) ||
+           (level == PT_LEVEL_4 && (pte & rsvdl4) != 0);
+}
+
+static INLINE Bool
+LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   MPN mpn = (pte & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT;
+   Bool safe = !LMPTEIsTerminal(pte, level, physMask) ||
+               (mpn == 0 && !PTE_LARGEPAGE(pte)) ||
+               LM_SAFE_BITS(pte) == LM_SAFE_BITS_MASK;
+   return safe;
+}
+#endif /* VMX86_DEBUG */


 /*
@@ -87,4 +137,95 @@
 #define VA64_CANONICAL_HOLE_START   (CONST64U(1) << (VA64_IMPL_BITS - 1))
 #define VA64_CANONICAL_HOLE_LEN  VA64_CANONICAL_MASK - VA64_CANONICAL_HOLE_START

+static INLINE Bool
+x86IsCanonicalC(VA64 va)
+{
+   return (va & VA64_CANONICAL_MASK) == 0 ||
+          (va & VA64_CANONICAL_MASK) == VA64_CANONICAL_MASK;
+}
+
+#if defined(VM_X86_64) && defined(__GNUC__)
+static INLINE Bool
+x86IsCanonicalAsm(VA64 va)
+{
+   Bool out;
+   /*
+    * sarq $48, %0: Move 17 bits from position 63:47 into 15:0 and CF,
+    *  sign-extending as we do so.
+    * adcl $0, %0: Then add (0 + CF) to the shifted value.  The result is zero
+    *  iff (CF == 1 && bits 31:0 == 0xffffffff) or (CF == 0 && bits 31:0 == 0).
+    *  That is, if original bits 47 and higher were all 1s or all 0s.
+    * setz %1: Create the boolean.
+    *
+    * The chain of AdcLong macros will emit one adc instruction; gcc
+    * has no obvious way to force the size of a quad %0 to a long.
+    */
+#define AdcLong(reg64, reg32) \
+   ".ifc  %0, %%" #reg64 "\n" \
+   " adcl $0, %%" #reg32 "\n" \
+   ".endif \n"
+
+   asm ("sarq $48, %0 \n"
+        AdcLong(rax, eax)
+        AdcLong(rcx, ecx)
+        AdcLong(rdx, edx)
+        AdcLong(rbx, ebx)
+        AdcLong(rsp, esp)
+        AdcLong(rbp, ebp)
+        AdcLong(rsi, esi)
+        AdcLong(rdi, edi)
+        AdcLong(r8,  r8d)
+        AdcLong(r9,  r9d)
+        AdcLong(r10, r10d)
+        AdcLong(r11, r11d)
+        AdcLong(r12, r12d)
+        AdcLong(r13, r13d)
+        AdcLong(r14, r14d)
+        AdcLong(r15, r15d)
+        "setz %1" : "+r"(va), "=r"(out));
+   return out;
+#undef AdcLong
+}
+#endif
+
+static INLINE Bool
+x86_IsCanonical(VA64 va)
+{
+#if defined(VM_X86_64) && defined(__GNUC__)
+   if (__builtin_constant_p(va)) {
+      return x86IsCanonicalC(va);
+   } else {
+      return x86IsCanonicalAsm(va);
+   }
+#else
+   return x86IsCanonicalC(va);
+#endif
+}
+
+static INLINE Bool
+x86_IsCanonicalRange(VA64 va, unsigned size)
+{
+   /*
+    * The check is simple as long as the size is less
+    * than the number of implemented bits.
+    *
+    * The only case we don't handle is one where the VA goes from a
+    * high canonical address and wraps to a non-canonical address
+    * (e.g. 0x00008000_00000000) or higher.  Our test would falsely
+    * consider this canonical.
+    */
+   ASSERT_ON_COMPILE(sizeof(size) * 8 < VA64_IMPL_BITS);
+
+   /*
+    * VA64_CANONICAL_MASK is the lowest canonical address with the
+    * upper bits all set.
+    *
+    * VA64_CANONICAL_HOLE_START is one higher than the highest valid
+    * canonical address with the upper bits all cleared.  Note that we
+    * access up to (va + size - 1), not (va + size), so <= is correct.
+    */
+   return va >= VA64_CANONICAL_MASK ||
+          va + size <= VA64_CANONICAL_HOLE_START;
+}
+
 #endif /* _X86PAGING_64_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,7 @@
 #define PF_RSVD         0x0008
 #define PF_ID           0x0010
 #define PF_PK           0x0020
+#define PF_SS           0x0040
 #define PF_SGX          0x8000

 /*
@@ -109,4 +110,12 @@ typedef struct {
   uint64   la;      /* Used for single address invalidation.  */
 } InvpcidArg;

+typedef enum {
+   PM_FLAT,        /* non paging mode                                      */
+   PM_2_LEVEL,     /* classic 32 bit non-pae style paging mode.            */
+   PM_3_LEVEL,     /* classic 32 bit pae style paging mode.                */
+   PM_4_LEVEL,     /* long mode style paging mode (including compat mode). */
+   PM_NUM     = 4, /* number of different paging modes.                    */
+} PagingMode;
+
 #endif /* _X86PAGING_COMMON_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,21 +65,21 @@ typedef struct Descriptor {
 typedef 
 #include "vmware_pack_begin.h"
 struct Descriptor64 {
-   unsigned   limit_lo  : 16;   // Limit bits 15-0.
-   unsigned   base_lo   : 24;   // Base bits  23-0.
-   unsigned   type      : 4;
-   unsigned   S         : 1;
-   unsigned   DPL       : 2;
-   unsigned   present   : 1;
-   unsigned   limit_hi  : 4;    // Limit bits 19-16.
-   unsigned   AVL       : 1;
-   unsigned   reserved0 : 2;
-   unsigned   gran      : 1;
-   unsigned   base_mid  : 8;    // Base bits 31-24.
-   unsigned   base_hi   : 32;   // Base bits 63-32.
-   unsigned   reserved1 : 8;
-   unsigned   ext_attrs : 5;
-   unsigned   reserved2 : 19;
+   uint64   limit_lo  : 16;   // Limit bits 15-0.
+   uint64   base_lo   : 24;   // Base bits  23-0.
+   uint64   type      : 4;
+   uint64   S         : 1;
+   uint64   DPL       : 2;
+   uint64   present   : 1;
+   uint64   limit_hi  : 4;    // Limit bits 19-16.
+   uint64   AVL       : 1;
+   uint64   reserved0 : 2;
+   uint64   gran      : 1;
+   uint64   base_mid  : 8;    // Base bits 31-24.
+   uint64   base_hi   : 32;   // Base bits 63-32.
+   uint64   reserved1 : 8;
+   uint64   ext_attrs : 5;
+   uint64   reserved2 : 19;
 }
 #include "vmware_pack_end.h"
 Descriptor64;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2014,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,6 +39,7 @@
 extern "C" {
 #endif

+#include "vm_assert.h"

 #define SELECTOR_GDT             0
 #define SELECTOR_LDT             1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -47,8 +47,10 @@
 #endif
 #include "x86cpuid_asm.h"

-#define SVM_VMCB_IO_BITMAP_SIZE    (3 * PAGE_SIZE)
-#define SVM_VMCB_MSR_BITMAP_SIZE   (2 * PAGE_SIZE)
+#define SVM_VMCB_IO_BITMAP_PAGES   (3)
+#define SVM_VMCB_IO_BITMAP_SIZE    PAGES_2_BYTES(SVM_VMCB_IO_BITMAP_PAGES)
+#define SVM_VMCB_MSR_BITMAP_PAGES  (2)
+#define SVM_VMCB_MSR_BITMAP_SIZE   PAGES_2_BYTES(SVM_VMCB_MSR_BITMAP_PAGES)

 /* Exit controls for the CR/DR access and hardware exceptions */
 #define SVM_CR_RD_CTL(num)         (0x1 << (num))
@@ -63,53 +65,56 @@
 #define SVM_XCP_CTL_ALL            0xffffffff

 /* Execution intercept controls */
-/* VMCB.exitCtl */
-#define SVM_VMCB_EXEC_CTL_INTR        0x0000000000000001ULL
-#define SVM_VMCB_EXEC_CTL_NMI         0x0000000000000002ULL
-#define SVM_VMCB_EXEC_CTL_SMI         0x0000000000000004ULL
-#define SVM_VMCB_EXEC_CTL_INIT        0x0000000000000008ULL
-#define SVM_VMCB_EXEC_CTL_VINTR       0x0000000000000010ULL
-#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR  0x0000000000000020ULL
-#define SVM_VMCB_EXEC_CTL_SIDT        0x0000000000000040ULL
-#define SVM_VMCB_EXEC_CTL_SGDT        0x0000000000000080ULL
-#define SVM_VMCB_EXEC_CTL_SLDT        0x0000000000000100ULL
-#define SVM_VMCB_EXEC_CTL_STR         0x0000000000000200ULL
-#define SVM_VMCB_EXEC_CTL_LIDT        0x0000000000000400ULL
-#define SVM_VMCB_EXEC_CTL_LGDT        0x0000000000000800ULL
-#define SVM_VMCB_EXEC_CTL_LLDT        0x0000000000001000ULL
-#define SVM_VMCB_EXEC_CTL_LTR         0x0000000000002000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSC       0x0000000000004000ULL
-#define SVM_VMCB_EXEC_CTL_RDPMC       0x0000000000008000ULL
-#define SVM_VMCB_EXEC_CTL_PUSHF       0x0000000000010000ULL
-#define SVM_VMCB_EXEC_CTL_POPF        0x0000000000020000ULL
-#define SVM_VMCB_EXEC_CTL_CPUID       0x0000000000040000ULL
-#define SVM_VMCB_EXEC_CTL_RSM         0x0000000000080000ULL
-#define SVM_VMCB_EXEC_CTL_IRET        0x0000000000100000ULL
-#define SVM_VMCB_EXEC_CTL_SWINT       0x0000000000200000ULL
-#define SVM_VMCB_EXEC_CTL_INVD        0x0000000000400000ULL
-#define SVM_VMCB_EXEC_CTL_PAUSE       0x0000000000800000ULL
-#define SVM_VMCB_EXEC_CTL_HLT         0x0000000001000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPG      0x0000000002000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPGA     0x0000000004000000ULL
-#define SVM_VMCB_EXEC_CTL_IOIO        0x0000000008000000ULL
-#define SVM_VMCB_EXEC_CTL_MSR         0x0000000010000000ULL
-#define SVM_VMCB_EXEC_CTL_TS          0x0000000020000000ULL
-#define SVM_VMCB_EXEC_CTL_FERR_FRZ    0x0000000040000000ULL
-#define SVM_VMCB_EXEC_CTL_SHUTDOWN    0x0000000080000000ULL
-#define SVM_VMCB_EXEC_CTL_VMRUN       0x0000000100000000ULL
-#define SVM_VMCB_EXEC_CTL_VMMCALL     0x0000000200000000ULL
-#define SVM_VMCB_EXEC_CTL_VMLOAD      0x0000000400000000ULL
-#define SVM_VMCB_EXEC_CTL_VMSAVE      0x0000000800000000ULL
-#define SVM_VMCB_EXEC_CTL_STGI        0x0000001000000000ULL
-#define SVM_VMCB_EXEC_CTL_CLGI        0x0000002000000000ULL
-#define SVM_VMCB_EXEC_CTL_SKINIT      0x0000004000000000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSCP      0x0000008000000000ULL
-#define SVM_VMCB_EXEC_CTL_ICEBP       0x0000010000000000ULL
-#define SVM_VMCB_EXEC_CTL_WBINVD      0x0000020000000000ULL
-#define SVM_VMCB_EXEC_CTL_MONITOR     0x0000040000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT       0x0000080000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT_COND  0x0000100000000000ULL
-#define SVM_VMCB_EXEC_CTL_XSETBV      0x0000200000000000ULL
+/* VMCB.execCtl */
+#define SVM_VMCB_EXEC_CTL_INTR           0x0000000000000001ULL
+#define SVM_VMCB_EXEC_CTL_NMI            0x0000000000000002ULL
+#define SVM_VMCB_EXEC_CTL_SMI            0x0000000000000004ULL
+#define SVM_VMCB_EXEC_CTL_INIT           0x0000000000000008ULL
+#define SVM_VMCB_EXEC_CTL_VINTR          0x0000000000000010ULL
+#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR     0x0000000000000020ULL
+#define SVM_VMCB_EXEC_CTL_SIDT           0x0000000000000040ULL
+#define SVM_VMCB_EXEC_CTL_SGDT           0x0000000000000080ULL
+#define SVM_VMCB_EXEC_CTL_SLDT           0x0000000000000100ULL
+#define SVM_VMCB_EXEC_CTL_STR            0x0000000000000200ULL
+#define SVM_VMCB_EXEC_CTL_LIDT           0x0000000000000400ULL
+#define SVM_VMCB_EXEC_CTL_LGDT           0x0000000000000800ULL
+#define SVM_VMCB_EXEC_CTL_LLDT           0x0000000000001000ULL
+#define SVM_VMCB_EXEC_CTL_LTR            0x0000000000002000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSC          0x0000000000004000ULL
+#define SVM_VMCB_EXEC_CTL_RDPMC          0x0000000000008000ULL
+#define SVM_VMCB_EXEC_CTL_PUSHF          0x0000000000010000ULL
+#define SVM_VMCB_EXEC_CTL_POPF           0x0000000000020000ULL
+#define SVM_VMCB_EXEC_CTL_CPUID          0x0000000000040000ULL
+#define SVM_VMCB_EXEC_CTL_RSM            0x0000000000080000ULL
+#define SVM_VMCB_EXEC_CTL_IRET           0x0000000000100000ULL
+#define SVM_VMCB_EXEC_CTL_SWINT          0x0000000000200000ULL
+#define SVM_VMCB_EXEC_CTL_INVD           0x0000000000400000ULL
+#define SVM_VMCB_EXEC_CTL_PAUSE          0x0000000000800000ULL
+#define SVM_VMCB_EXEC_CTL_HLT            0x0000000001000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPG         0x0000000002000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPGA        0x0000000004000000ULL
+#define SVM_VMCB_EXEC_CTL_IOIO           0x0000000008000000ULL
+#define SVM_VMCB_EXEC_CTL_MSR            0x0000000010000000ULL
+#define SVM_VMCB_EXEC_CTL_TS             0x0000000020000000ULL
+#define SVM_VMCB_EXEC_CTL_FERR_FRZ       0x0000000040000000ULL
+#define SVM_VMCB_EXEC_CTL_SHUTDOWN       0x0000000080000000ULL
+#define SVM_VMCB_EXEC_CTL_VMRUN          0x0000000100000000ULL
+#define SVM_VMCB_EXEC_CTL_VMMCALL        0x0000000200000000ULL
+#define SVM_VMCB_EXEC_CTL_VMLOAD         0x0000000400000000ULL
+#define SVM_VMCB_EXEC_CTL_VMSAVE         0x0000000800000000ULL
+#define SVM_VMCB_EXEC_CTL_STGI           0x0000001000000000ULL
+#define SVM_VMCB_EXEC_CTL_CLGI           0x0000002000000000ULL
+#define SVM_VMCB_EXEC_CTL_SKINIT         0x0000004000000000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSCP         0x0000008000000000ULL
+#define SVM_VMCB_EXEC_CTL_ICEBP          0x0000010000000000ULL
+#define SVM_VMCB_EXEC_CTL_WBINVD         0x0000020000000000ULL
+#define SVM_VMCB_EXEC_CTL_MONITOR        0x0000040000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT          0x0000080000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT_COND     0x0000100000000000ULL
+#define SVM_VMCB_EXEC_CTL_XSETBV         0x0000200000000000ULL
+#define SVM_VMCB_EXEC_CTL_EFER_WR_TRAP   0x0000800000000000ULL
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP(n) (0x0001000000000000ULL << (n))
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP_ALL 0xffff000000000000ULL

 /* VMCB.tlbCtl */
 #define SVM_VMCB_TLB_CTL_FLUSH              0x01
@@ -130,9 +135,13 @@

 /* VMCB.intrState */
 #define SVM_VMCB_INTRSTATE_INTR_SHADOW     (1 << 0)
+#define SVM_VMCB_INTRSTATE_GUEST_IF        (1 << 1)

 /* VMCB.nptCtl */
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)
+#define SVM_VMCB_NPTCTL_SEV_ENABLE         (1 << 1)
+#define SVM_VMCB_NPTCTL_SEV_ES_ENABLE      (1 << 2)
+#define SVM_VMCB_NPTCTL_GMET_ENABLE        (1 << 3)

 /* VMCB.virtExt */
 #define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
@@ -234,9 +243,17 @@
 #define SVM_EXITCODE_MWAIT                139
 #define SVM_EXITCODE_MWAIT_COND           140
 #define SVM_EXITCODE_XSETBV               141
+#define SVM_EXITCODE_EFER_WRITE_TRAP      143
+#define SVM_EXITCODE_CR_WRITE_TRAP(n)    (144 + (n))
 #define SVM_EXITCODE_NPF                 1024
 #define SVM_EXITCODE_AVIC_INCOMPLETE_IPI 1025
 #define SVM_EXITCODE_AVIC_NOACCEL        1026
+#define SVM_EXITCODE_VMGEXIT             1027
+#define SVM_EXITCODE_MMIO_READ           0x80000001   // SW only
+#define SVM_EXITCODE_MMIO_WRITE          0x80000002   // SW only
+#define SVM_EXITCODE_NMI_COMPLETE        0x80000003   // SW only
+#define SVM_EXITCODE_AP_JUMP_TABLE       0x80000005   // SW only
+#define SVM_EXITCODE_UNSUPPORTED         0x8000FFFF   // SW only
 #define SVM_EXITCODE_INVALID             (-1ULL)

 /* ExitInfo1 for I/O exits */
@@ -260,6 +277,9 @@
 #define SVM_IOEXIT_MBZ           0x00000002
 #define SVM_IOEXIT_SBZ           0x0000e000

+/* ExitInfo1 for MSR exits */
+#define SVM_MSREXIT_WRMSR        0x00000001
+
 /* ExitInfo1 for CR exits */
 #define SVM_CREXIT_MOVCR         (1ull << 63)
 #define SVM_CREXIT_GPR_MASK      0xf
@@ -295,6 +315,10 @@
 #define SVM_NPFEXIT_FINAL_ADDR     (1ULL << 32)    /* Rev. C */
 #define SVM_NPFEXIT_PTE_ACCESS     (1ULL << 33)    /* Rev. C */

+/* ExitInfo1 for AP jump table exits */
+#define SVM_APEXIT_SET            0x0
+#define SVM_APEXIT_GET            0x1
+
 /* Event Injection */
 #define SVM_INTINFO_VECTOR_MASK   0x000000ff
 #define SVM_INTINFO_TYPE_SHIFT    8
@@ -430,7 +454,7 @@ SVM_LockedFromFeatures(uint64 vmCR)
 static INLINE Bool
 SVM_EnabledCPU(void)
 {
-   return SVM_EnabledFromFeatures(__GET_MSR(MSR_VM_CR));
+   return SVM_EnabledFromFeatures(X86MSR_GetMSR(MSR_VM_CR));
 }
 #endif /* } !defined(USERLEVEL) */

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -62,13 +62,6 @@ typedef uint64 VM_EPTE;
 * Registers
 */

-typedef union SharedReg64 {
-   Reg8  reg8[2];
-   Reg16 reg16;
-   Reg32 reg32;
-   Reg64 reg64;
-} SharedReg64;
-
 typedef union SharedUReg64 {
   UReg8  ureg8[2];
   UReg16 ureg16;
@@ -178,11 +171,7 @@ typedef Task32 Task;

 #pragma pack(push, 1)
 typedef struct {
-#if defined(VMM) || defined(COREQUERY)
-   uint64 va;
-#else
-   uint32 va;
-#endif
+   uint64   va;
   Selector seg;
 } FarPtr;
 #pragma pack(pop)
@@ -468,4 +457,12 @@ typedef struct ExcFrame64ForCopy {
 } ExcFrame64ForCopy;
 #pragma pack(pop)

+/*
+ * Layout of the stack for performing a 64 bit lret instruction.
+ */
+typedef struct LretFrame64 {
+   uint64 rip;
+   uint64 cs;
+} LretFrame64;
+
 #endif // ifndef _X86TYPES_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,12 @@ VT_EXIT(RDSEED,               61)
 VT_EXIT(PML_LOGFULL,          62)
 VT_EXIT(XSAVES,               63)
 VT_EXIT(XRSTORS,              64)
-
+VT_EXIT(VMEXIT65,             65)
+VT_EXIT(VMEXIT66,             66)
+VT_EXIT(VMEXIT67,             67)
+VT_EXIT(VMEXIT68,             68)
+VT_EXIT(VMEXIT69,             69)
+VT_EXIT(ENCLV,                70)
+VT_EXIT(SGX_CONFLICT,         71)
 /* Bump this up if you add an exit reason. */
-#define VT_NUM_EXIT_REASONS   65
+#define VT_NUM_EXIT_REASONS   72
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -95,9 +95,10 @@
 *       _V  -- virtualized
 *       _NV -- not virtualized
 *   _access is one of:
- *       _NA -- no access
- *       _RW -- read/write access
- *       _RO -- read-only access
+ *       _NA  -- no access
+ *       _RW  -- read/write access
+ *       _RO  -- read-only access
+ *       _URW -- read/write acess for the ULM only
 *   _sticky is one of:
 *       _S  -- sticky
 *       _NS -- non-sticky
@@ -109,9 +110,9 @@
 VMCS_SET_START(16)
 /* 16-bit control fields. */
 VMCS_GROUP_START(16, CTL)
-VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA,  _S)
-VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA, _NS)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV,  _NA,  _S)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _URW, _NS)
 VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
 VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
 VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
@@ -283,7 +284,7 @@ VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2030, _S64, _TC, 24)
 VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA,  _S)
 VMCS_UNUSED(                        0x2034, _S64, _TC, 26)
-VMCS_UNUSED(                        0x2036, _S64, _TC, 27)
+VMCS_FIELD(ENCLV_EXITING_BITMAP,    0x2036, _S64, _TC, 27, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2038, _S64, _TC, 28)
 VMCS_UNUSED(                        0x203A, _S64, _TC, 29)
 VMCS_UNUSED(                        0x203C, _S64, _TC, 30)
@@ -328,16 +329,16 @@ VMCS_GROUP_END(64, DATA)

 /* 64-bit guest state. */
 VMCS_GROUP_START(64, GUEST)
-VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA, _NS)
-VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA, _NS)
-VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA, _NS)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _URW, _NS)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V,  _NA, _NS)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V,  _NA, _NS)
 VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
 VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
 VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
@@ -403,24 +404,24 @@ VMCS_SET_END(64)
 VMCS_SET_START(32)
 /* 32-bit control fields. */
 VMCS_GROUP_START(32, CTL)
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA, _NS)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA,  _S)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA,  _S)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA,  _S)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA,  _S)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _URW,  _S)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _URW, _NS)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _URW,  _S)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _URW,  _S)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V,  _NA,  _S)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _URW,  _S)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
 VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
 VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
@@ -551,14 +552,14 @@ VMCS_SET_END(32)
 VMCS_SET_START(NAT)
 /* natural-width control fields. */
 VMCS_GROUP_START(NAT, CTL)
-VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA,  _S)
-VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _URW,  _S)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
 VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
 VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
@@ -622,26 +623,26 @@ VMCS_GROUP_END(NAT, DATA)

 /* natural-width guest state. */
 VMCS_GROUP_START(NAT, GUEST)
-VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA, _NS)
-VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW, _NS)
-VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW, _NS)
-VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW, _NS)
-VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW, _NS)
-VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW, _NS)
-VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW, _NS)
-VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW, _NS)
-VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW, _NS)
-VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA, _NS)
-VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW, _NS)
-VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW, _NS)
-VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW, _NS)
-VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _URW, _NS)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _URW, _NS)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V,  _RW, _NS)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V,  _RW, _NS)
 VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
 VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
 VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
@@ -658,18 +659,18 @@ VMCS_GROUP_END(NAT, GUEST)

 /* natural-width host state. */
 VMCS_GROUP_START(NAT, HOST)
-VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA,  _S)
-VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V,  _NA,  _S)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _URW,  _S)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V,  _NA,  _S)
 VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
 VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
 VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2004-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2004-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -159,10 +159,10 @@
 #define VT_ENCODING_RSVD               0xffff9000

 /*
- * The highest index of any currently defined field is 25, for
- * TSC_MULTIPLIER.
+ * The highest index of any currently defined field is 27, for
+ * ENCLV_EXITING_BITMAP.
 */
-#define VT_ENCODING_MAX_INDEX                  25
+#define VT_ENCODING_MAX_INDEX                  27

 enum {
 #define VMCS_FIELD(_name, _val, ...) VT_VMCS_##_name = _val,
@@ -173,8 +173,11 @@ enum {
 /*
 * Sizes of referenced fields
 */
-#define VT_VMCS_IO_BITMAP_SIZE    (2 * PAGE_SIZE)
-#define VT_VMCS_MSR_BITMAP_SIZE   (1 * PAGE_SIZE)
+#define VT_VMCS_IO_BITMAP_PAGES   (2)
+#define VT_VMCS_IO_BITMAP_SIZE    PAGES_2_BYTES(VT_VMCS_IO_BITMAP_PAGES)
+#define VT_VMCS_MSR_BITMAP_PAGES  (1)
+#define VT_VMCS_MSR_BITMAP_SIZE   PAGES_2_BYTES(VT_VMCS_MSR_BITMAP_PAGES)
+

 /*
 * Execution control bit capabilities come in pairs: a "required" bit in
@@ -293,7 +296,9 @@ enum {
   VMX_CPU2(PT_SUPPRESS_NR_BIT, 19)                      \
   VMX_CPU2(XSAVES,             20)                      \
   VMX_CPU2(EPT_MBX,            22)                      \
-   VMX_CPU2(TSC_SCALING,        25)
+   VMX_CPU2(TSC_SCALING,        25)                      \
+   VMX_CPU2(ENCLV,              28)                      \
+   VMX_CPU2(EPC_VIRT_EXT,       29)

 #define VMX_PROCBASED_CTLS2_CAP                          \
        VMX_PROCBASED_CTLS2_CAP_NDA                      \
@@ -550,7 +555,7 @@ enum {
 #define VT_VMCS_PENDDBG_MBZ        0xfffeaff0

 /* Exception error must-be-zero bits for VMEntry */
-#define VT_XCP_ERR_MBZ             0xffff8000
+#define VT_XCP_ERR_MBZ             0xffff0000


 /* Exit reasons. */
@@ -687,6 +692,12 @@ enum {
 #define VT_GUESTFAIL_QUAL_NMI          3
 #define VT_GUESTFAIL_QUAL_LINK         4

+/* SGX conflict VM-exit Qualification Codes */
+#define VT_SGX_TRACKING_RESOURCE_CONFLICT     0
+#define VT_SGX_TRACKING_REFERENCE_CONFLICT    1
+#define VT_SGX_EPC_PAGE_CONFLICT_EXCEPTION    2
+#define VT_SGX_EPC_PAGE_CONFLICT_ERROR        3
+
 /* VMX abort indicators. */

 #define VT_VMX_ABORT_GUEST_MSRS        1
@@ -710,7 +721,8 @@ enum {
 #define VT_REQUIRED_PINBASED_CTLS                      \
   (VT_PINBASED_CTLS_DEFAULT1                        | \
    VT_VMCS_PIN_VMEXEC_CTL_EXTINT_EXIT               | \
-    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT)
+    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT                  | \
+    VT_VMCS_PIN_VMEXEC_CTL_VNMI)

 #define VT_REQUIRED_PROCBASED_CTLS                     \
   (VT_PROCBASED_CTLS_DEFAULT1                       | \
@@ -726,6 +738,7 @@ enum {
    VT_VMCS_CPU_VMEXEC_CTL_LDCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_STCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_TPR_SHADOW                | \
+    VT_VMCS_CPU_VMEXEC_CTL_VNMI_WINDOW               | \
    VT_VMCS_CPU_VMEXEC_CTL_MONITOR)

 #define VT_REQUIRED_EXIT_CTLS                          \
@@ -1075,7 +1088,7 @@ VT_MBXSupportedFromFeatures(uint64 secondary)
 static INLINE Bool
 VT_EnabledCPU(void)
 {
-   return VT_EnabledFromFeatures(__GET_MSR(MSR_FEATCTL));
+   return VT_EnabledFromFeatures(X86MSR_GetMSR(MSR_FEATCTL));
 }


@@ -1096,16 +1109,14 @@ VT_EnabledCPU(void)
 static INLINE Bool
 VT_SupportedCPU(void)
 {
-   if (__GET_MSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) {
-      return VT_SupportedFromFeatures(__GET_MSR(MSR_VMX_TRUE_PINBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_PROCBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_ENTRY_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_EXIT_CTLS),
-                                      __GET_MSR(MSR_VMX_BASIC));
-   } else {
-      /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
-      return FALSE;
-   }
+
+   /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
+   return (X86MSR_GetMSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) != 0 &&
+          VT_SupportedFromFeatures(X86MSR_GetMSR(MSR_VMX_TRUE_PINBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_PROCBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_ENTRY_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_EXIT_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_BASIC));
 }

 #endif /* } !defined(USERLEVEL) */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -23,6 +23,7 @@

 #include <linux/file.h>
 #include <linux/highmem.h>
+#include <linux/miscdevice.h>
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/slab.h>
@@ -36,8 +37,8 @@

 #include "usercalldefs.h"

-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
-#error Linux kernels before 2.6.32 are not supported
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0)
+#error Linux kernels before 3.10 are not supported
 #endif

 #include <asm/io.h>
@@ -48,12 +49,10 @@
 #include "modulecall.h"
 #include "vm_asm.h"
 #include "vmx86.h"
-#include "initblock.h"
 #include "task.h"
 #include "memtrack.h"
 #include "task.h"
 #include "cpuid.h"
-#include "cpuid_info.h"
 #include "circList.h"
 #include "x86msr.h"

@@ -64,13 +63,12 @@
 #include "hostif.h"
 #include "hostif_priv.h"
 #include "vmhost.h"
+#include "sharedAreaVmmon.h"

 static void LinuxDriverQueue(Device *device);
 static void LinuxDriverDequeue(Device *device);
 static Bool LinuxDriverCheckPadding(void);

-#define VMMON_UNKNOWN_SWAP_SIZE -1ULL
-
 struct VMXLinuxState linuxState;


@@ -96,20 +94,25 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd,
                       unsigned long ioarg);

 static int LinuxDriver_Close(struct inode *inode, struct file *filp);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-static int LinuxDriverFault(struct vm_fault *fault);
-#else
-static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault);
-#endif
-static int LinuxDriverMmap(struct file *filp, struct vm_area_struct *vma);

 static unsigned int LinuxDriverEstimateTSCkHz(void);

-static struct vm_operations_struct vmuser_mops = {
-        .fault  = LinuxDriverFault
+static const struct file_operations vmuser_fops = {
+   .owner = THIS_MODULE,
+   .open = LinuxDriver_Open,
+   .release = LinuxDriver_Close,
+   .unlocked_ioctl = LinuxDriver_Ioctl,
+   .compat_ioctl = LinuxDriver_Ioctl,
 };

-static struct file_operations vmuser_fops;
+#ifndef VMX86_DEVEL
+static struct miscdevice vmmon_miscdev = {
+   .name = "vmmon",
+   .minor = MISC_DYNAMIC_MINOR,
+   .fops = &vmuser_fops,
+};
+#endif
+
 static struct timer_list tscTimer;
 static Atomic_uint32 tsckHz;
 static VmTimeStart tsckHzStartTime;
@@ -264,9 +267,8 @@ LinuxDriverInitTSCkHz(void)
 *      linux module entry point. Called by /sbin/insmod command
 *
 * Results:
- *      registers a device driver for a major # that depends
- *      on the uid. Add yourself to that list.  List is now in
- *      private/driver-private.c.
+ *      Release: registers a device driver with a misc minor node.
+ *      Devel: registers for a major number with user-created node.
 *
 *----------------------------------------------------------------------
 */
@@ -290,48 +292,46 @@ init_module(void)
      return -ENOEXEC;
   }

+   if (!Vmx86_CreateHVIOBitmap()) {
+      return -ENOMEM;
+   }
+
+   if (!Vmx86_CheckMSRUniformity()) {
+      return -EPERM;
+   }
+
   linuxState.fastClockThread = NULL;
   linuxState.fastClockRate = 0;
-   linuxState.swapSize = VMMON_UNKNOWN_SWAP_SIZE;
-
-   /*
-    * Initialize the file_operations structure. Because this code is always
-    * compiled as a module, this is fine to do it here and not in a static
-    * initializer.
-    */
-
-   memset(&vmuser_fops, 0, sizeof vmuser_fops);
-   vmuser_fops.owner = THIS_MODULE;
-   vmuser_fops.unlocked_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.compat_ioctl = LinuxDriver_Ioctl;
-   vmuser_fops.open = LinuxDriver_Open;
-   vmuser_fops.release = LinuxDriver_Close;
-   vmuser_fops.mmap = LinuxDriverMmap;

 #ifdef VMX86_DEVEL
   devel_init_module();
-   linuxState.minor = 0;
   retval = register_chrdev(linuxState.major, linuxState.deviceName,
                            &vmuser_fops);
+   if (retval) {
+      Warning("Module %s: error registering with major=%d\n",
+              linuxState.deviceName, linuxState.major);
+   } else {
+      Log("Module %s: registered with major=%d\n",
+          linuxState.deviceName, linuxState.major);
+   }
 #else
   sprintf(linuxState.deviceName, "vmmon");
   linuxState.major = 10;
-   linuxState.minor = 165;
-   linuxState.misc.minor = linuxState.minor;
-   linuxState.misc.name = linuxState.deviceName;
-   linuxState.misc.fops = &vmuser_fops;

-   retval = misc_register(&linuxState.misc);
+   retval = misc_register(&vmmon_miscdev);
+   if (retval) {
+      Warning("Module %s: error registering misc device %s\n",
+              linuxState.deviceName, vmmon_miscdev.name);
+   } else {
+      Log("Module %s: registered as misc device %s\n", linuxState.deviceName,
+          vmmon_miscdev.name);
+   }
 #endif

   if (retval) {
-      Warning("Module %s: error registering with major=%d minor=%d\n",
-              linuxState.deviceName, linuxState.major, linuxState.minor);
-
+      Vmx86_CleanupHVIOBitmap();
      return -ENOENT;
   }
-   Log("Module %s: registered with major=%d minor=%d\n",
-       linuxState.deviceName, linuxState.major, linuxState.minor);

   HostIF_InitUptime();
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
@@ -369,13 +369,14 @@ cleanup_module(void)
 #ifdef VMX86_DEVEL
   unregister_chrdev(linuxState.major, linuxState.deviceName);
 #else
-   misc_deregister(&linuxState.misc);
+   misc_deregister(&vmmon_miscdev);
 #endif

   Log("Module %s: unloaded\n", linuxState.deviceName);

   del_timer_sync(&tscTimer);

+   Vmx86_CleanupHVIOBitmap();
   Task_Terminate();
   // Make sure fastClockThread is dead
   HostIF_FastClockLock(1);
@@ -413,7 +414,7 @@ LinuxDriver_Open(struct inode *inode, // IN
   }
   memset(device, 0, sizeof *device);

-   sema_init(&device->lock4Gb, 1);
+   init_rwsem(&device->vmDriverRWSema);

   filp->private_data = device;
   LinuxDriverQueue(device);
@@ -424,114 +425,6 @@ LinuxDriver_Open(struct inode *inode, // IN
 }


-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocPages --
- *
- *    Allocate physically contiguous block of memory with specified order.
- *    Pages in the allocated block are configured so that caller can pass
- *    independent pages to the VM.
- *
- * Results:
- *    Zero on success, non-zero (error code) on failure.
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverAllocPages(unsigned int gfpFlag, // IN
-                      unsigned int order,   // IN
-                      struct page **pg,     // OUT
-                      unsigned int size)    // IN
-{
-   struct page* page;
-
-   page = alloc_pages(gfpFlag, order);
-   if (page) {
-      unsigned int i;
-
-      /*
-       * Grab an extra reference on all pages except first one - first
-       * one was already refcounted by alloc_pages.
-       *
-       * Under normal situation all pages except first one in the block
-       * have refcount zero.  As we pass these pages to the VM, we must
-       * bump their count, otherwise VM will release these pages every
-       * time they would be unmapped from user's process, causing crash.
-       *
-       * Note that this depends on Linux VM internals.  It works on all
-       * kernels we care about.
-       */
-
-      order = 1 << order;
-      for (i = 0; i < order; i++) {
-         if (i) {
-            /*
-             * Debug kernels assert that page->_count is not zero when
-             * calling get_page. We use init_page_count as a temporary
-             * workaround. PR 894174
-             */
-#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 16)
-            ASSERT(page_count(page) == 0);
-            init_page_count(page);
-#else
-            get_page(page);
-#endif
-         }
-         if (i >= size) {
-            put_page(page);
-         } else {
-            void *addr = kmap(page);
-
-            memset(addr, 0, PAGE_SIZE);
-            kunmap(page);
-            *pg++ = page;
-         }
-         page++;
-      }
-
-      return 0;
-   }
-
-   return -ENOMEM;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverDestructor4Gb --
- *
- *    Deallocate all directly mappable memory.
- *
- * Results:
- *    None
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static void
-LinuxDriverDestructor4Gb(Device *device) // IN
-{
-   unsigned int pg;
-
-   if (!device->size4Gb) {
-      return;
-   }
-   for (pg = 0; pg < device->size4Gb; pg++) {
-      put_page(device->pages4Gb[pg]);
-   }
-   device->size4Gb = 0;
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -554,220 +447,21 @@ LinuxDriver_Close(struct inode *inode, // IN
   ASSERT(device);

   LinuxDriverDequeue(device);
+   down_write(&device->vmDriverRWSema);
   if (device->vm != NULL) {
      Vmx86_ReleaseVM(device->vm);
      device->vm = NULL;
   }
+   up_write(&device->vmDriverRWSema);

   Vmx86_Close();

-   /*
-    * Destroy all low memory allocations.
-    * We are closing the struct file here, so clearly no other process
-    * uses it anymore, and we do not need to hold the semaphore.
-    */
-
-   LinuxDriverDestructor4Gb(device);
-
   kfree(device);
   filp->private_data = NULL;

   return 0;
 }

-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverFault --
- *
- *      Callback for returning allocated page for memory mapping
- *
- * Results:
- *    NoPage:
- *      Page or page address on success, NULL or 0 on failure.
- *    Fault:
- *      Error code; 0, minor page fault.
- *
- * Side effects:
- *      None.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-LinuxDriverFault(struct vm_fault *fault)     //IN/OUT
-#else
-LinuxDriverFault(struct vm_area_struct *vma, //IN
-                 struct vm_fault *fault)     //IN/OUT
-#endif
-{
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-   struct vm_area_struct *vma = fault->vma;
-#endif
-   Device *device = (Device *) vma->vm_file->private_data;
-   unsigned long pg;
-   struct page* page;
-
-   pg = fault->pgoff;
-   pg = VMMON_MAP_OFFSET(pg);
-   if (pg >= device->size4Gb) {
-      return VM_FAULT_SIGBUS;
-   }
-   page = device->pages4Gb[pg];
-   get_page(page);
-   fault->page = page;
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocContig --
- *
- *      Create mapping for contiguous memory areas.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int LinuxDriverAllocContig(Device *device,
-                                  struct vm_area_struct *vma,
-                                  unsigned long off,
-                                  unsigned long size)
-{
-   unsigned long vmaOrder = VMMON_MAP_ORDER(off);
-   unsigned long vmaAllocSize;
-   unsigned int gfpFlag;
-   unsigned long i;
-
-   if (VMMON_MAP_RSVD(off)) {
-      /* Reserved bits set... */
-      return -EINVAL;
-   }
-   if (VMMON_MAP_OFFSET(off)) {
-      /* We do not need non-zero offsets... */
-      return -EINVAL;
-   }
-   switch (VMMON_MAP_MT(off)) {
-      case VMMON_MAP_MT_LOW4GB:
-#ifdef GFP_DMA32
-         gfpFlag = GFP_USER | GFP_DMA32;
-#else
-         gfpFlag = GFP_USER | GFP_DMA;
-#endif
-         break;
-      case VMMON_MAP_MT_LOW16MB:
-         gfpFlag = GFP_USER | GFP_DMA;
-         break;
-      case VMMON_MAP_MT_ANY:
-         gfpFlag = GFP_HIGHUSER;
-         break;
-      default:
-         /* Invalid memory type */
-         return -EINVAL;
-   }
-   if (size > VMMON_MAP_OFFSET_MASK + 1) {
-      /* Size is too big to fit to our window. */
-      return -ENOMEM;
-   }
-
-   /* 16 pages looks like a good limit... */
-   if (size > VMMON_MAX_LOWMEM_PAGES) {
-      return -ENOMEM;
-   }
-   /* Sorry. Only one mmap per one open. */
-   down(&device->lock4Gb);
-   if (device->size4Gb) {
-      up(&device->lock4Gb);
-      return -EINVAL;
-   }
-   vmaAllocSize = 1 << vmaOrder;
-   for (i = 0; i < size; i += vmaAllocSize) {
-      int err;
-
-      err = LinuxDriverAllocPages(gfpFlag, vmaOrder,
-                                  device->pages4Gb + i, size - i);
-      if (err) {
-         while (i > 0) {
-            put_page(device->pages4Gb[--i]);
-         }
-         up(&device->lock4Gb);
-
-         return err;
-      }
-   }
-   device->size4Gb = size;
-   up(&device->lock4Gb);
-   vma->vm_ops = &vmuser_mops;
-
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverMmap --
- *
- *      Create mapping for lowmem or locked memory.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverMmap(struct file *filp,
-                struct vm_area_struct *vma)
-{
-   Device *device = (Device *) filp->private_data;
-   unsigned long size;
-   int err;
-
-   /* Only shared mappings */
-   if (!(vma->vm_flags & VM_SHARED)) {
-      return -EINVAL;
-   }
-   if ((vma->vm_end | vma->vm_start) & (PAGE_SIZE - 1)) {
-      return -EINVAL;
-   }
-   size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
-   if (size < 1) {
-      return -EINVAL;
-   }
-   if (device->vm) {
-      err = -EINVAL;
-   } else {
-      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
-   }
-   if (err) {
-      return err;
-   }
-   /* Clear VM_IO, otherwise SuSE's kernels refuse to do get_user_pages */
-   vma->vm_flags &= ~VM_IO;
-
-   return 0;
-}
-
-
 typedef Bool (*SyncFunc)(void *data, unsigned cpu);

 typedef struct {
@@ -1034,26 +728,30 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
                  unsigned long ioarg)  // IN:
 {
   Device *device = (Device *) filp->private_data;
-   int retval = 0;
+   long retval = 0;
   Vcpuid vcpuid;
   VMDriver *vm;
+   Bool needsWriteLock = iocmd == IOCTL_VMX86_CREATE_VM ||
+                         iocmd == IOCTL_VMX86_RELEASE_VM;

+   ASSERT_ON_COMPILE(sizeof(long) == sizeof(int64));
   if (device == NULL) {
      return -EINVAL;
   }

+   if (needsWriteLock) {
+      down_write(&device->vmDriverRWSema);
+   } else {
+      down_read(&device->vmDriverRWSema);
+   }
+
   vm = device->vm;
-
-   /*
-    * Validate the VM pointer for those IOCTLs that require it.
-    */
-
+   /* Validate the VM pointer for those IOCTLs that require it. */
   switch (iocmd) {
   case IOCTL_VMX86_VERSION:
   case IOCTL_VMX86_CREATE_VM:
   case IOCTL_VMX86_SET_UID:
   case IOCTL_VMX86_GET_NUM_VMS:
-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
   case IOCTL_VMX86_SET_HARD_LIMIT:
   case IOCTL_VMX86_GET_IPI_VECTORS:
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
@@ -1101,6 +799,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      } else {
         device->vm = vm;
         args.vmid = vm->userID;
+         vm->vmhost->vmmonData = (void *__user)args.vmmonData;
         retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args);
      }
      break;
@@ -1108,67 +807,73 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:

   case IOCTL_VMX86_PROCESS_BOOTSTRAP: {
      VMProcessBootstrapBlock *args;
+      size_t argsSz = GetVMProcessBootstrapBlockSize(vm->numVCPUs);
      VA64 uAddr;
-      Bool res;

      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
      if (retval != 0) {
         break;
      }
-      args = HostIF_AllocKernelMem(sizeof *args, TRUE);
+      args = HostIF_AllocKernelMem(argsSz, TRUE);
      if (args == NULL) {
         retval = -ENOMEM;
         break;
      }
-      retval = HostIF_CopyFromUser(args, uAddr, sizeof *args);
+      retval = HostIF_CopyFromUser(args, uAddr, argsSz);
      if (retval != 0) {
         HostIF_FreeKernelMem(args);
         break;
      }
-      res = Vmx86_ProcessBootstrap(vm,
-                                   args->bsBlobAddr,
-                                   args->numBytes,
-                                   args->headerOffset,
-                                   args->numVCPUs,
-                                   args->ptRootVAs,
-                                   args->shRegions);
-      if (!res) {
+      if (args->numVCPUs != vm->numVCPUs) {
+         retval = -EINVAL;
+      } else if (!Vmx86_ProcessBootstrap(vm,
+                                         args->bsBlobAddr,
+                                         args->numBytes,
+                                         args->headerOffset,
+                                         args->numVCPUs,
+                                         args->perVcpuPages,
+                                         args->shRegions)) {
         retval = -ENOMEM;
      }
      HostIF_FreeKernelMem(args);
      break;
   }

+   case IOCTL_VMX86_REGISTER_SHARED: {
+      VMSharedAreaRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!SharedAreaVmmon_ValidateRegionArgs(vm, &args)) {
+         retval = -EINVAL;
+         break;
+      }
+      if (!SharedAreaVmmon_RegisterRegion(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
+   case IOCTL_VMX86_REGISTER_STATVARS: {
+      VMStatVarsRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!StatVarsVmmon_RegisterVCPU(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
   case IOCTL_VMX86_RELEASE_VM:
      device->vm = NULL;
      Vmx86_ReleaseVM(vm);
      break;

-   case IOCTL_VMX86_INIT_VM: {
-      InitBlock *initParams;
-      VA64 uAddr;
-
-      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
-      if (retval != 0) {
-         break;
-      }
-      initParams = HostIF_AllocKernelMem(sizeof *initParams, TRUE);
-      if (initParams == NULL) {
-         retval = -ENOMEM;
-         break;
-      }
-      retval = HostIF_CopyFromUser(initParams, uAddr, sizeof *initParams);
-      if (retval == 0) {
-         if (Vmx86_InitVM(vm, initParams) == 0) {
-            retval = HostIF_CopyToUser(uAddr, initParams, sizeof *initParams);
-         } else {
-            retval = -EINVAL;
-         }
-      }
-      HostIF_FreeKernelMem(initParams);
-      break;
-   }
-
   case IOCTL_VMX86_RUN_VM:
      vcpuid = ioarg;

@@ -1236,17 +941,28 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_VMM_PAGE_ROOT: {
+      VcpuPageRoot args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->ptRootMpns == NULL) {
+         retval = -EINVAL;
+      } else {
+         args.pageRoot = vm->ptRootMpns[args.vcpuid];
+         retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      }
+      break;
+   }
+
   case IOCTL_VMX86_GET_NUM_VMS:
      retval = Vmx86_GetNumVMs();
      break;

-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
-      retval = Vmx86_GetTotalMemUsage();
-      break;
-
   case IOCTL_VMX86_SET_HARD_LIMIT: {
-      int32 limit;
-
+      PageCnt limit;
      retval = HostIF_CopyFromUser(&limit, ioarg, sizeof limit);
      if (retval != 0) {
         break;
@@ -1332,25 +1048,12 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

   case IOCTL_VMX86_APIC_INIT: {
-      VMAPICInfo info;
-      Bool setVMPtr;
-      Bool probe;
-
-      retval = HostIF_CopyFromUser(&info, ioarg, sizeof info);
-      if (retval != 0) {
-         break;
-      }
-      setVMPtr = ((info.flags & APIC_FLAG_DISABLE_NMI) != 0);
-      probe = ((info.flags & APIC_FLAG_PROBE) != 0);
-
      /*
-       * Kernel uses NMIs for deadlock detection - set APIC VMptr so that
-       * NMIs get disabled in the monitor.
+       * Kernel uses NMIs for deadlock detection - so we always have to find
+       * APIC so that NMIs get disabled in the monitor.
       */
-
-      setVMPtr = TRUE;
-
-      retval = HostIF_APICInit(vm, setVMPtr, probe) ? 0 : -ENODEV;
+      HostIF_APICInit(vm);
+      retval = 0;
      break;
   }

@@ -1390,6 +1093,23 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_SWITCH_ERROR_ADDR: {
+      VMSwitchErrorArgs args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->crosspage == NULL ||
+          vm->crosspage[args.vcpuid] == NULL) {
+         retval = -EINVAL;
+         break;
+      }
+      args.addr = vm->crosspage[args.vcpuid]->wsUD2;
+      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      break;
+   }
+
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
      retval = LinuxDriverEstimateTSCkHz();
      break;
@@ -1516,6 +1236,13 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_NUM_ANON_PAGES: {
+      PageCnt numAnonPages;
+      numAnonPages = Vmx86_GetNumAnonPages(vm);
+      retval = HostIF_CopyToUser(ioarg, &numAnonPages, sizeof numAnonPages);
+      break;
+   }
+
   case IOCTL_VMX86_READ_PAGE: {
         VMMReadWritePage req;

@@ -1530,6 +1257,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
+#if VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1538,6 +1266,9 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
         }
         retval = HostIF_WritePhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE,
                                       PAGE_SIZE);
+#else
+         retval = -EINVAL;
+#endif
         break;
      }

@@ -1592,17 +1323,6 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

-   case IOCTL_VMX86_SET_HOST_SWAP_SIZE: {
-      uint64 swapSize;
-      retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize);
-      if (retval != 0) {
-         Warning("Could not copy swap size from user, status %d\n", retval);
-         break;
-      }
-      linuxState.swapSize = swapSize;
-      break;
-   }
-
   case IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS: {
      uint64 ctrs = Vmx86_GetUnavailablePerfCtrs();
      retval = HostIF_CopyToUser(ioarg, &ctrs, sizeof ctrs);
@@ -1629,6 +1349,11 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

 exit:
+   if (needsWriteLock) {
+      up_write(&device->vmDriverRWSema);
+   } else {
+      up_read(&device->vmDriverRWSema);
+   }
   return retval;
 }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2011,2014-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -23,49 +23,31 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
+#include <linux/rwsem.h>
 #include <linux/wait.h>

 #include "vmx86.h"
 #include "driver_vmcore.h"


-/*
- * Used to track a piece of memory that's been mapped into the kernel from
- * userlevel.
- */
-typedef struct VMMappedUserMem {
-   void *addr;
-   size_t numPages;
-   struct page *pages[0];
-} VMMappedUserMem;
-
-
-/*
- * Per-instance driver state
- */
-
+/* Per-instance driver state */
 struct VMDriver;

-/* 16 pages (64KB) looks as a good limit for one allocation */
-#define VMMON_MAX_LOWMEM_PAGES  16
-
 typedef struct Device {
   struct Device   *next;
   struct VMDriver *vm;
-
   /*
-    * The semaphore protect accesses to size4Gb and pages4Gb
-    * in mmap(). mmap() may happen only once, and all other
-    * accesses except cleanup are read-only, and may happen
-    * only after successful mmap.
+    * This RW semaphore protects accesses to the VMDriver to
+    * avoid racing between various ioctls, and the creation
+    * and removal of the VM in question. The lock is read-acquired
+    * by ioctls that reference the VMDriver, and write-acquired by
+    * ioctls or device callbacks that allocate or destroy the
+    * VMDriver.
    */
-   struct semaphore lock4Gb;
-   unsigned int size4Gb;
-   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
+   struct rw_semaphore vmDriverRWSema;
 } Device;


@@ -78,19 +60,14 @@ typedef struct Device {

 typedef struct VMXLinuxState {
   int major;
-   int minor;
-   struct miscdevice misc;
   char deviceName[VM_DEVICE_NAME_SIZE];
   char buf[LINUXLOG_BUFFER_SIZE];
   Device *head;

   struct task_struct *fastClockThread;
   unsigned fastClockRate;
-   uint64 swapSize;
 } VMXLinuxState;

 extern VMXLinuxState linuxState;
-extern uint8 monitorIPIVector;
-extern uint8 hvIPIVector;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,12 +25,6 @@
 #ifndef _HOSTIF_PRIV_H_
 #define _HOSTIF_PRIV_H_

-/* Functions for mapping and unmapping userspace memory. */
-struct VMMappedUserMem;
-EXTERN void  *HostIF_MapUserMem(VA addr, size_t size,
-                                struct VMMappedUserMem **handle);
-EXTERN void   HostIF_UnmapUserMem(struct VMMappedUserMem *handle);
-
 /* Uptime-related functions. */
 EXTERN void   HostIF_InitUptime(void);
 EXTERN void   HostIF_CleanupUptime(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -79,8 +79,6 @@ typedef struct VMHost {
    */
   Mutex vmMutex;

-   unsigned int         crosspagePagesCount;
-   struct page        **crosspagePages;      /* ptr to numVCPUs-sized array */
   struct task_struct **vcpuSemaTask;        /* ptr to numVCPUs-sized array */

   /*
@@ -94,8 +92,11 @@ typedef struct VMHost {
    * as pages for "AWE" guest memory.
    */
   struct PhysTracker  *AWEPages;
-   /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */
-   Bool                 hostAPICIsMapped;
+   /*
+    * Pointer to a userlevel 64-bit area containing the value 1.
+    * This is used for HostIF_SemaphoreSignal.
+    */
+   void *__user vmmonData;
 } VMHost;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,7 +24,7 @@
 *
 */

-#if defined(linux)
+#if defined(__linux__)
 /* Must come before any kernel header file */
 #   include "driver-config.h"
 #   include <linux/kernel.h>
@@ -34,7 +34,6 @@
 #include "modulecall.h"
 #include "vmx86.h"
 #include "task.h"
-#include "initblock.h"
 #include "vm_basic_asm.h"
 #include "iocontrols.h"
 #include "hostif.h"
@@ -43,6 +42,7 @@
 #include "usercalldefs.h"
 #include "cpuid.h"
 #include "vmmblob.h"
+#include "sharedAreaVmmon.h"

 /*
 *----------------------------------------------------------------------
@@ -72,13 +72,16 @@ int
 Vmx86_RunVM(VMDriver *vm,   // IN:
            Vcpuid vcpuid)  // IN:
 {
-   uint32           retval    = MODULECALL_USERRETURN;
+   uint64           retval    = MODULECALL_USERRETURN;
   VMCrossPageData *crosspage;
   int              bailValue = 0;
+   Bool             switchOk = TRUE;

-   ASSERT(vcpuid < vm->numVCPUs && CPUID_HostSupportsHV());
-   crosspage = &vm->crosspage[vcpuid]->crosspageData;
-   ASSERT(crosspage);
+   ASSERT(vcpuid < vm->numVCPUs);
+   if (vm->crosspage[vcpuid] == NULL) {
+      return USERCALL_VMX86ALLOCERR;
+   }
+   crosspage = vm->crosspage[vcpuid];

   /*
    * Check if we were interrupted by signal.
@@ -93,9 +96,9 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
       * Task_Switch changes the world to the monitor.
       * The monitor is waiting in the BackToHost routine.
       */
-      UCTIMESTAMP(crosspage, SWITCHING_TO_MONITOR);
-      Task_Switch(vm, vcpuid);
-      UCTIMESTAMP(crosspage, SWITCHED_TO_MODULE);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHING_TO_MONITOR);
+      switchOk = Task_Switch(vm, vcpuid);
+      UCTIMESTAMP(crosspage->ucTimeStamps, SWITCHED_TO_MODULE);

      /*
       * Wake up anything that was waiting for this vcpu to run
@@ -117,13 +120,19 @@ skipTaskSwitch:;

      retval = MODULECALL_USERRETURN;

+      if (UNLIKELY(!switchOk)) {
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
+
      if (crosspage->userCallType != MODULECALL_USERCALL_NONE) {
         /*
          * This is the main user call path.  Handle by returning
          * from the ioctl (back to the userlevel side of a VCPU thread).
          */
         bailValue = crosspage->userCallType;
-         crosspage->retval = retval;
+         ASSERT(retval == (uint64)((uint32)retval));
+         crosspage->retval = (uint32)retval;
         goto bailOut;
      }

@@ -136,18 +145,19 @@ skipTaskSwitch:;

      case MODULECALL_GET_RECYCLED_PAGES: {
         MPN mpns[MODULECALL_NUM_ARGS];
-         int nPages = MIN((int)crosspage->args[0], MODULECALL_NUM_ARGS);
-
-         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages, TRUE,
-                                         FALSE);
+         PageCnt nPages = MIN(crosspage->args[0], MODULECALL_NUM_ARGS);
+         ASSERT((int64)crosspage->args[0] >= 0);
+         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages,
+                                         TRUE, FALSE);
         if (retval <= nPages) {
-            int i;
+            PageCnt i;
            for (i = 0; i < retval; i++) {
               crosspage->args[i] = mpns[i];
            }
         } else {
-            // retval is holding an error code
-            Warning("Failed to alloc %u pages: %d\n", nPages, (int)retval);
+            /* retval is holding an error code */
+            Warning("Failed to alloc %"FMT64"u pages: %"FMT64"d\n", nPages,
+                    (int64)retval);
            retval = 0;
         }
         break;
@@ -161,7 +171,6 @@ skipTaskSwitch:;

      case MODULECALL_SEMAWAIT: {
         retval = HostIF_SemaphoreWait(vm, vcpuid, crosspage->args);
-
         if (retval == MX_WAITINTERRUPTED) {
            crosspage->moduleCallInterrupted = TRUE;
            bailValue = USERCALL_RESTART;
@@ -171,7 +180,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_SEMASIGNAL: {
-         retval = HostIF_SemaphoreSignal(crosspage->args);
+         retval = HostIF_SemaphoreSignal(vm, crosspage->args);

         if (retval == MX_WAITINTERRUPTED) {
             crosspage->moduleCallInterrupted = TRUE;
@@ -197,7 +206,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_RELEASE_ANON_PAGES: {
-         unsigned count;
+         PageCnt count;
         MPN mpns[MODULECALL_NUM_ARGS];
         for (count = 0; count < MODULECALL_NUM_ARGS; count++) {
            mpns[count] = (MPN)crosspage->args[count];
@@ -206,15 +215,14 @@ skipTaskSwitch:;
            }
         }
         ASSERT(count > 0);
-         retval = Vmx86_FreeLockedPages(vm, PtrToVA64(mpns), count, TRUE);
+         retval = Vmx86_FreeLockedPages(vm, mpns, count);
         break;
      }

      case MODULECALL_LOOKUP_MPN: {
-         int i;
-         VPN64  vpn    = (VPN64)crosspage->args[0];
-         uint32 nPages = (uint32)crosspage->args[1];
-         VA64   uAddr  = (VA64)VPN_2_VA(vpn);
+         VPN64   vpn        = (VPN64)crosspage->args[0];
+         PageCnt i, nPages  = crosspage->args[1];
+         VA64    uAddr      = (VA64)VPN_2_VA(vpn);
         ASSERT(nPages <= MODULECALL_NUM_ARGS);
         HostIF_VMLock(vm, 38);
         for (i = 0; i < nPages; i++) {
@@ -280,6 +288,10 @@ skipTaskSwitch:;
         retval = HostIF_GetHVIPIVector();
      } break;

+      case MODULECALL_GET_PERF_CTR_VECTOR: {
+         retval = HostIF_GetPerfCtrVector();
+      } break;
+
      case MODULECALL_GET_HOST_TIMER_VECTORS: {
         uint8 v0, v1;
         HostIF_GetTimerVectors(&v0, &v1);
@@ -292,17 +304,76 @@ skipTaskSwitch:;
         vm->blobInfo = NULL;
      } break;

+      case MODULECALL_GET_SHARED_AREA: {
+         SharedAreaVmmonRequest request;
+         request.type       = (SharedAreaType)crosspage->args[0];
+         request.vcpu       = (Vcpuid)crosspage->args[1];
+         request.offset     = (PageCnt)crosspage->args[2];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[3] = SharedAreaVmmon_GetRegionMPN(vm, &request);
+      } break;
+
+      case MODULECALL_GET_STAT_VARS: {
+         Vcpuid  vcpu   = (Vcpuid)crosspage->args[0];
+         PageCnt offset = (PageCnt)crosspage->args[1];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[2] = StatVarsVmmon_GetRegionMPN(vm, vcpu, offset);
+      } break;
+
+      case MODULECALL_GET_NUM_PTP_PAGES: {
+         /* Store PageCnt in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[1] = vm->numPTPPages;
+      } break;
+
+      case MODULECALL_GET_HV_IO_BITMAP: {
+         crosspage->args[0] = hvIOBitmap == NULL ? INVALID_MPN :
+                                                   hvIOBitmap->mpn;
+      } break;
+
+      case MODULECALL_GET_MSR: {
+         /*
+          * Allocate a MSRQuery request on stack with only one MSRReply slot
+          * since this MSR will be queried from a MSR cache or on a single pcpu.
+          */
+         uint8 req[sizeof(MSRQuery) + sizeof(MSRReply)];
+         MSRQuery *query = (MSRQuery *)&req[0];
+         query->msrNum = (uint32)crosspage->args[0];
+         query->numLogicalCPUs = 1;
+         retval = Vmx86_GetAllMSRs(query);
+         crosspage->args[0] = query->logicalCPUs[0].msrVal;
+      } break;
+
+      case MODULECALL_ALLOC_CONTIG_PAGES: {
+         PageCnt pages = crosspage->args[0];
+         HostIFContigMemMap *alloc;
+
+         HostIF_VMLock(vm, 47);
+         alloc = HostIF_AllocContigPages(vm, pages);
+         if (alloc != NULL) {
+            alloc->next = vm->contigMappings;
+            vm->contigMappings = alloc;
+            crosspage->args[1] = alloc->mpn;
+         } else {
+            crosspage->args[1] = INVALID_MPN;
+         }
+         HostIF_VMUnlock(vm, 47);
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }

-      crosspage->retval = retval;
-
-#if defined(linux)
-      cond_resched(); // Other kernels are preemptable
+      if (retval != (uint64)((uint32)retval)) {
+         Warning("Unexpected error in modulecall %u (%llu)\n",
+                 crosspage->moduleCallType, retval);
+         bailValue = USERCALL_SWITCHERR;
+         goto bailOut;
+      }
+      crosspage->retval = (uint32)retval;
+#if defined(__linux__)
+      cond_resched(); // Other kernels are preemptible
 #endif
   }
-
 bailOut:
   return bailValue;
 }
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -129,6 +136,8 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,8 @@

 #define ETHER_MAX_QUEUED_PACKET 1600

+/* Most Ethernet equipment can support jumbo frames up to 9216 bytes. */
+#define ETHER_MAX_JUMBO_FRAME_LEN 9216

 /*
 * State's that a NIC can be in currently we only use this
@@ -198,7 +200,7 @@ Net_GetNumAdapters(Net_AdapterCount *counts)
      } else if (Str_Strcasecmp(adapterStr, "e1000e") == 0) {
         counts->e1000e++;
      } else {
-         LOG_ONCE(("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr));
+         LOG_ONCE("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr);
      }
      free(adapterStr);
   }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -28,6 +28,7 @@

 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
 #include <linux/mm.h>
 #include "compat_skbuff.h"
 #include <linux/sockios.h>
@@ -44,6 +45,22 @@
 #include "vmnetInt.h"


+/*
+ * Default min MTU value as defined by kernel versions >= 4.10.0.
+ * Use the same value for earlier versions of the kernel which do not
+ * enforce a minimum MTU size.
+ */
+#define VMNET_MIN_MTU 68
+#define VMNET_MAX_MTU (ETHER_MAX_JUMBO_FRAME_LEN - \
+                      (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
+
+#ifdef RHEL_RELEASE_CODE
+#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 5) && \
+     RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8, 0))
+#define ndo_change_mtu ndo_change_mtu_rh74
+#endif
+#endif /* RHEL_RELEASE_CODE */
+
 typedef struct VNetNetIF {
   VNetPort                port;
   struct net_device      *dev;
@@ -64,6 +81,11 @@ static int  VNetNetifSetMAC(struct net_device *dev, void *addr);
 static void VNetNetifSetMulticast(struct net_device *dev);
 static int  VNetNetIfProcRead(char *page, char **start, off_t off,
                              int count, int *eof, void *data);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+static int VNetNetifChangeMtu(struct net_device *dev, int new_mtu);
+#endif
+

 /*
 *----------------------------------------------------------------------
@@ -84,7 +106,7 @@ static int  VNetNetIfProcRead(char *page, char **start, off_t off,
 static void
 VNetNetIfSetup(struct net_device *dev)  // IN:
 {
-#if !COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) || defined(HAVE_NET_DEVICE_OPS)
+#ifdef HAVE_NET_DEVICE_OPS
   static const struct net_device_ops vnetNetifOps = {
      .ndo_init = VNetNetifProbe,
      .ndo_open = VNetNetifOpen,
@@ -97,12 +119,23 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
 #else
      .ndo_set_rx_mode = VNetNetifSetMulticast,
 #endif
-
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
+      .ndo_change_mtu = VNetNetifChangeMtu,
+#endif
   };
+#endif /* HAVE_NET_DEVICE_OPS */
+
+   /* Turns on IFF_BROADCAST, IFF_MULTICAST. */
+   ether_setup(dev);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+   /* Set the maximum allowed MTU value. */
+   dev->max_mtu = VMNET_MAX_MTU;
 #endif

-   ether_setup(dev); // turns on IFF_BROADCAST, IFF_MULTICAST
-#if COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) && !defined(HAVE_NET_DEVICE_OPS) 
+#ifdef HAVE_NET_DEVICE_OPS
+   dev->netdev_ops = &vnetNetifOps;
+#else
   dev->init = VNetNetifProbe;
   dev->open = VNetNetifOpen;
   dev->hard_start_xmit = VNetNetifStartXmit;
@@ -110,9 +143,10 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
   dev->get_stats = VNetNetifGetStats;
   dev->set_mac_address = VNetNetifSetMAC;
   dev->set_multicast_list = VNetNetifSetMulticast;
-#else
-   dev->netdev_ops = &vnetNetifOps;
-   #endif /* HAVE_NET_DEVICE_OPS */
+#ifdef HAVE_CHANGE_MTU
+   dev->change_mtu = VNetNetifChangeMtu;
+#endif /* HAVE_CHANGE_MTU */
+#endif /* HAVE_NET_DEVICE_OPS */

 }

@@ -530,6 +564,38 @@ VNetNetifSetMulticast(struct net_device *dev) // IN: unused
 }


+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+/*
+ *----------------------------------------------------------------------
+ *
+ * VNetNetifChangeMtu --
+ *
+ *      Changes the current MTU value of a given vmnet interface.
+ *
+ * Results:
+ *      Returns zero on success, or invalid argument error if an incorrect
+ *      MTU size is being set.
+ *
+ * Side effects:
+ *      None.
+ *----------------------------------------------------------------------
+ */
+
+int
+VNetNetifChangeMtu(struct net_device *dev, // IN/OUT: assigning the mtu member.
+                   int new_mtu)            // IN:     new mtu value.
+{
+   if (new_mtu < VMNET_MIN_MTU || new_mtu > VMNET_MAX_MTU) {
+      return -EINVAL;
+   }
+
+   dev->mtu = new_mtu;
+   return 0;
+}
+#endif
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013, 2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -137,6 +137,7 @@ VNetProcShow(struct seq_file *p, // IN:
 }


+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0)
 /*
 *----------------------------------------------------------------------
 *
@@ -168,6 +169,7 @@ static struct file_operations fops = {
   .release = single_release,
 };
 #endif
+#endif


 /*
@@ -203,7 +205,12 @@ VNetProcMakeEntryInt(VNetProcEntry   *parent,   // IN:
      } else {
         ent->data   = data;
         ent->fn     = fn;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+         ent->pde    = proc_create_single_data(name, mode, parent->pde,
+                                               VNetProcShow, ent);
+#else
         ent->pde    = proc_create_data(name, mode, parent->pde, &fops, ent);
+#endif
      }
      if (ent->pde != NULL) {
         *ret = ent;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,8 +41,6 @@

 #else /* _WIN32 */

-#undef __KERNEL__ //To prevent including any kernel specific stuff 
-
 #ifdef VMX86_DEVEL
 #define DBG 1
 #else
@@ -4329,7 +4327,7 @@ LookupTypeClass(unsigned short typeValue) // IN: ethernet type
 *----------------------------------------------------------------------
 */

-#if _WIN32
+#ifdef _WIN32
 _Use_decl_annotations_
 #endif
 void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -78,11 +78,13 @@ static int  VNetUserIfSetUplinkState(VNetPort *port, uint8 linkUp);
 extern unsigned int  vnet_max_qlen;

 #if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
-#   define compat_kmap(page) kmap(page)
-#   define compat_kunmap(page) kunmap(page)
-#else
-#   define compat_kmap(page) kmap((page).p)
-#   define compat_kunmap(page) kunmap((page).p)
+#   define skb_frag_page(frag) (frag)->page
+#   define skb_frag_size(frag) (frag)->size
+#endif
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0) && \
+    !(defined(CONFIG_SUSE_VERSION) && CONFIG_SUSE_VERSION == 15 && \
+      defined(CONFIG_SUSE_PATCHLEVEL) && CONFIG_SUSE_PATCHLEVEL >= 2)
+#   define skb_frag_off(frag) (frag)->page_offset
 #endif

 /*
@@ -137,16 +139,21 @@ UserifLockPage(VA addr) // IN
 */

 static INLINE int
-VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapPtr(VA64 uAddr,      // IN: pointer to user memory
                 size_t size,     // IN: size of data
                 struct page **p, // OUT: locked page
                 void **ptr)      // OUT: kernel mapped pointer
 {
-   if (!access_ok(VERIFY_WRITE, (void *)uAddr, size) ||
-       (((uAddr + size - 1) & ~(PAGE_SIZE - 1)) !=
-        (uAddr & ~(PAGE_SIZE - 1)))) {
+   uint8 v;
+
+   /* Check area does not straddle two pages. */
+   if ((uAddr & (PAGE_SIZE - 1)) + size > PAGE_SIZE) {
      return -EINVAL;
   }
+   /* Check if it is user's area.  UserifLockPage() checks writability. */
+   if (copy_from_user(&v, (void *)(unsigned long)uAddr, sizeof v) != 0) {
+      return -EFAULT;
+   }

   *p = UserifLockPage(uAddr);
   if (*p == NULL) {
@@ -158,7 +165,7 @@ VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
 }

 static INLINE int
-VNetUserIfMapUint32Ptr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapUint32Ptr(VA64 uAddr,      // IN: pointer to user memory
                       struct page **p, // OUT: locked page
                       uint32 **ptr)    // OUT: kernel mapped pointer
 {
@@ -201,7 +208,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      return -EBUSY;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->pollPtr, &pollPage,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->pollPtr, &pollPage,
                                        &pollPtr)) < 0) {
      return retval;
   }
@@ -213,7 +220,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      goto error_free;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->recvClusterPtr,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->recvClusterPtr,
                                        &recvClusterPage,
                                        &recvClusterCount)) < 0) {
      goto error_free;
@@ -362,12 +369,12 @@ VNetUserIfReceive(VNetJack       *this, // IN
   VNetUserIF *userIf = (VNetUserIF*)this->private;
   uint8 *dest = SKB_2_DESTMAC(skb);
   unsigned long flags;
-   
+
   if (!UP_AND_RUNNING(userIf->port.flags)) {
      userIf->stats.droppedDown++;
      goto drop_packet;
   }
-   
+
   if (!VNetPacketMatch(dest,
                        userIf->port.paddr,
                        (const uint8 *)userIf->port.exactFilter,
@@ -377,13 +384,17 @@ VNetUserIfReceive(VNetJack       *this, // IN
      userIf->stats.droppedMismatch++;
      goto drop_packet;
   }
-   
+
   if (skb_queue_len(&userIf->packetQueue) >= vnet_max_qlen) {
      userIf->stats.droppedOverflow++;
      goto drop_packet;
   }
-   
-   if (skb->len > ETHER_MAX_QUEUED_PACKET) {
+
+   /*
+    * The check would be more accurate if based on the current MTU value
+    * of the corresponding vmnet interface. PR 2267716 is filed to track this.
+    */
+   if (skb->len > ETHER_MAX_JUMBO_FRAME_LEN) {
      userIf->stats.droppedLargePacket++;
      goto drop_packet;
   }
@@ -550,7 +561,12 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
      return -EINVAL;
   }

+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
   csum = csum_and_copy_to_user(skb->data + offset, curr, len, 0, &err);
+#else
+   csum = csum_and_copy_to_user(skb->data + offset, curr, len);
+   err = (csum == 0) ? -EFAULT : 0;
+#endif
   if (err) {
      return err;
   }
@@ -559,20 +575,26 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
   for (frag = skb_shinfo(skb)->frags;
 	frag != skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
 	frag++) {
-      if (frag->size > 0) {
+      if (skb_frag_size(frag) > 0) {
 	 unsigned int tmpCsum;
 	 const void *vaddr;

-	 vaddr = compat_kmap(frag->page);
-	 tmpCsum = csum_and_copy_to_user(vaddr + frag->page_offset,
-					 curr, frag->size, 0, &err);
-	 compat_kunmap(frag->page);
+	 vaddr = kmap(skb_frag_page(frag));
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 10, 0)
+	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
+					 curr, skb_frag_size(frag), 0, &err);
+#else
+	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
+					 curr, skb_frag_size(frag));
+         err = (tmpCsum == 0) ? -EFAULT : 0;
+#endif
+	 kunmap(skb_frag_page(frag));

 	 if (err) {
 	    return err;
 	 }
 	 csum = csum_block_add(csum, tmpCsum, curr - buf);
-	 curr += frag->size;
+	 curr += skb_frag_size(frag);
      }
   }

@@ -747,7 +769,7 @@ VNetUserIfRead(VNetPort    *port, // IN
 *----------------------------------------------------------------------
 */

-static int 
+static int
 VNetUserIfWrite(VNetPort    *port, // IN
                struct file *filp, // IN
                const char  *buf,  // IN
@@ -757,11 +779,14 @@ VNetUserIfWrite(VNetPort    *port, // IN
   struct sk_buff *skb;

   /*
-    * Check size
+    * Check size.
+    *
+    * Regarding the upper count limit the check would be more accurate
+    * if based on the current MTU value of the corresponding vmnet interface.
+    * PR 2267716 is filed to track this.
    */
-   
-   if (count < sizeof (struct ethhdr) || 
-       count > ETHER_MAX_QUEUED_PACKET) {
+   if (count < sizeof (struct ethhdr) ||
+       count > ETHER_MAX_JUMBO_FRAME_LEN) {
      return -EINVAL;
   }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -202,12 +202,12 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +314,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,6 +41,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"
 #include "vm_assert.h"

@@ -122,87 +129,29 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

-/*
- * Prototypes for msft atomics.  These are defined & inlined by the
- * compiler so no function definition is needed.  The prototypes are
- * needed for C++.
- *
- * The declarations for the intrinsic functions were taken from ntddk.h
- * in the DDK. The declarations must match otherwise the 64-bit C++
- * compiler will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
-#ifdef __cplusplus
-extern "C" {
-#endif
-long  _InterlockedExchange(long volatile*, long);
-long  _InterlockedCompareExchange(long volatile*, long, long);
-long  _InterlockedExchangeAdd(long volatile*, long);
-long  _InterlockedDecrement(long volatile*);
-long  _InterlockedIncrement(long volatile*);
-__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
-#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedCompareExchange64)
-
-# if _MSC_VER >= 1600
-char     _InterlockedExchange8(char volatile *, char);
-char     _InterlockedCompareExchange8(char volatile *, char, char);
-#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
-#endif
-
-#if defined VM_X86_64
-long     _InterlockedAnd(long volatile*, long);
-__int64  _InterlockedAnd64(__int64 volatile*, __int64);
-long     _InterlockedOr(long volatile*, long);
-__int64  _InterlockedOr64(__int64 volatile*, __int64);
-long     _InterlockedXor(long volatile*, long);
-__int64  _InterlockedXor64(__int64 volatile*, __int64);
-__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
-__int64  _InterlockedIncrement64(__int64 volatile*);
-__int64  _InterlockedDecrement64(__int64 volatile*);
-__int64  _InterlockedExchange64(__int64 volatile*, __int64);
-#if !defined _WIN64
-#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
-#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
-#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
-#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
-#endif /* !_WIN64 */
-#endif /* __x86_64__ */
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* _MSC_VER */
-
 #if defined __arm__
 /*
 * LDREX without STREX or CLREX may cause problems in environments where the
@@ -255,15 +204,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
 * registers.
 */

-#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
-uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
-                              uint8 val);
-
-uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
-                                     uint8 newVal,
-                                     uint8 oldVal);
-#endif
-
 #if defined __GNUC__ && defined VM_ARM_32
 /* Force the link step to fail for unimplemented functions. */
 extern int AtomicUndefined(void const *);
@@ -405,19 +345,8 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "memory"
   );
   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedExchange8((volatile char *)&var->value, val);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm movzx eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint8.value, al
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedExchange8(&var->value, val);
 #else
 #error No compiler defined for Atomic_ReadWrite8
 #endif
@@ -500,23 +429,9 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
   );

   return val;
-#elif defined _MSC_VER && _MSC_VER >= 1600
+#elif defined _MSC_VER
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
-#elif defined _MSC_VER && defined __i386__
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov al, oldVal
-      __asm mov ebx, var
-      __asm mov cl, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
-      __asm movzx eax, al
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#elif defined _MSC_VER && defined __x86_64__
-   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite8
 #endif
@@ -1312,13 +1227,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_And
 #endif
@@ -1377,13 +1286,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Or
 #endif
@@ -1442,13 +1345,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Xor
 #endif
@@ -2187,18 +2084,13 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
    */
   return var->value;
 #elif defined _MSC_VER && defined VM_ARM_32
+   /* MSVC + 32-bit ARM has add64 but no cmpxchg64 */
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
-#   pragma warning(push)
-#   pragma warning(disable : 4035)      // disable no-return warning
-   {
-      __asm mov ecx, var
-      __asm mov edx, ecx
-      __asm mov eax, ebx
-      __asm lock cmpxchg8b [ecx]
-      // edx:eax is the return value; this is documented to work. --mann
-   }
-#   pragma warning(pop)
+   /* MSVC + 32-bit x86 has cmpxchg64 but no add64 */
+   return _InterlockedCompareExchange64((__int64 *)&var->value,
+                                        (__int64)255,  // Unlikely value to
+                                        (__int64)255); // not dirty cache
 #elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
@@ -2303,7 +2195,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2216,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3581,7 +3476,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3604,7 +3499,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -3618,7 +3513,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,13 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
+#pragma warning(push)
+#pragma warning(disable : 4255)      // disable no-prototype() to (void) warning
+#include <intrin.h>
+#pragma warning(pop)
+#endif
+
 #include "vm_basic_types.h"

 #if defined VM_X86_64
@@ -48,7 +55,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -221,8 +227,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +269,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +382,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +398,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -497,7 +506,7 @@ uint16set(void *dst, uint16 val, size_t count)
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
-      : "r" (val)
+      : "r" ((uint64)val)
      : "cc", "memory");
 #else
   size_t dummy0;
@@ -550,6 +559,7 @@ uint32set(void *dst, uint32 val, size_t count)
      : "cc", "memory");
 #elif defined(VM_ARM_64)
   void   *tmpDst = dst;
+   uint64 tmpVal = val;

   if (count == 0) {
      return dst;
@@ -586,7 +596,7 @@ uint32set(void *dst, uint32 val, size_t count)
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
-      : "+r" (tmpDst), "+r" (count), "+r" (val)
+      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      :
      : "cc", "memory");
 #else
@@ -630,22 +640,13 @@ uint32set(void *dst, uint32 val, size_t count)
 static INLINE void *
 uint16set(void *dst, uint16 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosw((uint16*)dst, val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov ax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosw;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -653,22 +654,13 @@ uint16set(void *dst, uint16 val, size_t count)
 static INLINE void *
 uint32set(void *dst, uint32 val, size_t count)
 {
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
   __stosd((unsigned long*)dst, (unsigned long)val, count);
-#elif defined(VM_ARM_32)
+#else
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
-#else
-   __asm { pushf;
-           mov eax, val;
-           mov ecx, count;
-           mov edi, dst;
-           cld;
-           rep stosd;
-           popf;
-   }
 #endif
   return dst;
 }
@@ -768,27 +760,21 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
 {
   _mm_pause();
 }
-#else /* VM_X86_64 */
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0xf3 __asm _emit 0x90
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
 #else  /* __GNUC__  */
 #error No compiler defined for PAUSE
 #endif
@@ -836,11 +822,11 @@ RDTSC(void)
 #endif
 }
 #elif defined(_MSC_VER)
-#ifdef VM_X86_64
+#ifdef VM_X86_ANY
 {
   return __rdtsc();
 }
-#elif defined(VM_ARM_32)
+#else
 {
   /*
    * We need to do more inverstagetion here to find
@@ -849,13 +835,7 @@ RDTSC(void)
   NOT_IMPLEMENTED();
   return 0;
 }
-#else
-#pragma warning( disable : 4035)
-{
-   __asm _emit 0x0f __asm _emit 0x31
-}
-#pragma warning (default: 4035)
-#endif /* VM_X86_64 */
+#endif /* VM_X86_ANY */
 #else  /* __GNUC__  */
 #error No compiler defined for RDTSC
 #endif /* __GNUC__  */
@@ -888,7 +868,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +882,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1078,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1136,113 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+#endif
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -39,24 +39,6 @@
 #error "This file is x86-64 only!"
 #endif

-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-uint64 _umul128(uint64 multiplier, uint64 multiplicand,
-                uint64 *highProduct);
-int64 _mul128(int64 multiplier, int64 multiplicand,
-              int64 *highProduct);
-uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
-#ifdef __cplusplus
-}
-#endif
-
-#pragma intrinsic(_umul128, _mul128, __shiftright128)
-
-#endif // _MSC_VER
-
 #if defined(__GNUC__)
 /*
 * GET_CURRENT_PC
@@ -113,40 +95,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +169,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +272,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +316,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +398,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +483,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +507,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,97 +38,6 @@
 #error "Should be included only in x86 builds"
 #endif

-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
-#ifdef __cplusplus
-extern "C" {
-#endif
-/*
- * It seems x86 & x86-64 windows still implements these intrinsic
- * functions.  The documentation for the x86-64 suggest the
- * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
- * __inbyte/__outbyte aren't supported on x86.
- */
-int            _inp(unsigned short);
-unsigned short _inpw(unsigned short);
-unsigned long  _inpd(unsigned short);
-
-int            _outp(unsigned short, int);
-unsigned short _outpw(unsigned short, unsigned short);
-unsigned long  _outpd(uint16, unsigned long);
-#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)
-
-/*
- * Prevents compiler from re-ordering reads, writes and reads&writes.
- * These functions do not add any instructions thus only affect
- * the compiler ordering.
- *
- * See:
- * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
- * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
- */
-void _ReadBarrier(void);
-void _WriteBarrier(void);
-void _ReadWriteBarrier(void);
-#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)
-
-void _mm_mfence(void);
-void _mm_lfence(void);
-#pragma intrinsic(_mm_mfence, _mm_lfence)
-
-long _InterlockedXor(long volatile *, long);
-#pragma intrinsic(_InterlockedXor)
-
-unsigned int __getcallerseflags(void);
-#pragma intrinsic(__getcallerseflags)
-
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-unsigned __int64 __rdtsc(void);
-void             __stosw(unsigned short *, unsigned short, size_t);
-void             __stosd(unsigned long *, unsigned long, size_t);
-void             _mm_pause(void);
-#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)
-
-unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
-unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
-#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
-#endif /* VM_X86_64 */
-
-unsigned char  _BitScanForward(unsigned long *, unsigned long);
-unsigned char  _BitScanReverse(unsigned long *, unsigned long);
-#pragma intrinsic(_BitScanForward, _BitScanReverse)
-
-unsigned char  _bittest(const long *, long);
-unsigned char  _bittestandset(long *, long);
-unsigned char  _bittestandreset(long *, long);
-unsigned char  _bittestandcomplement(long *, long);
-#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
-#ifdef VM_X86_64
-unsigned char  _bittestandset64(__int64 *, __int64);
-unsigned char  _bittestandreset64(__int64 *, __int64);
-#pragma intrinsic(_bittestandset64, _bittestandreset64)
-#endif // VM_X86_64
-#ifdef __cplusplus
-}
-#endif
-#endif // _MSC_VER
-
 #ifdef __GNUC__
 /*
 * Checked against the Intel manual and GCC --hpreg
@@ -179,9 +88,6 @@ __GCC_IN(l, uint32, IN32)
 #define OUTW(port, val) __GCC_OUT(w, w, port, val)
 #define OUT32(port, val) __GCC_OUT(l, , port, val)

-#define GET_CURRENT_EIP(_eip) \
-      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
-
 static INLINE unsigned int
 GetCallerEFlags(void)
 {
@@ -191,35 +97,35 @@ GetCallerEFlags(void)
 }

 #elif defined(_MSC_VER)
-static INLINE  uint8
+static INLINE uint8
 INB(uint16 port)
 {
-   return (uint8)_inp(port);
+   return __inbyte(port);
 }
 static INLINE void
 OUTB(uint16 port, uint8 value)
 {
-   _outp(port, value);
+   __outbyte(port, value);
 }
 static INLINE uint16
 INW(uint16 port)
 {
-   return _inpw(port);
+   return __inword(port);
 }
 static INLINE void
 OUTW(uint16 port, uint16 value)
 {
-   _outpw(port, value);
+   __outword(port, value);
 }
 static INLINE  uint32
 IN32(uint16 port)
 {
-   return _inpd(port);
+   return __indword(port);
 }
 static INLINE void
 OUT32(uint16 port, uint32 value)
 {
-   _outpd(port, value);
+   __outdword(port, value);
 }

 #ifndef VM_X86_64
@@ -227,11 +133,6 @@ OUT32(uint16 port, uint32 value)
 #undef NEAR
 #endif

-#define GET_CURRENT_EIP(_eip) do { \
-   __asm call NEAR PTR $+5 \
-   __asm pop eax \
-   __asm mov _eip, eax \
-} while (0)
 #endif // VM_X86_64

 static INLINE unsigned int
@@ -244,9 +145,127 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_sfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +297,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +412,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +430,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +510,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,12 +44,44 @@
 #endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -58,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -159,18 +157,6 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

-/*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-

 /*
 * Token concatenation
@@ -209,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -255,6 +241,10 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif
@@ -286,6 +276,14 @@ Max(int a, int b)
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -440,10 +438,6 @@ void *_ReturnAddress(void);

 #define strtok_r  strtok_s

-#if (_MSC_VER < 1500)
-#define	vsnprintf _vsnprintf
-#endif
-
 typedef int uid_t;
 typedef int gid_t;

@@ -467,20 +461,9 @@ typedef int pid_t;
 #define       W_OK          2
 #define       R_OK          4

-#endif // }
+#endif // } _WIN32

-/*
- * Macro for username comparison.
- */
-
-#ifdef _WIN32 // {
-#define USERCMP(x,y)  Str_Strcasecmp(x,y)
-#else
-#define USERCMP(x,y)  strcmp(x,y)
-#endif // }
-
-
-#endif // }
+#endif // } USERLEVEL

 #ifndef va_copy

@@ -556,10 +539,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -659,11 +642,27 @@ typedef int pid_t;
 #endif

 #ifdef VMM
+#define vmx86_vmm 1
 #define VMM_ONLY(x) x
 #else
+#define vmx86_vmm 0
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
@@ -687,7 +686,6 @@ typedef int pid_t;
 * display/printer drivers only.
 */
 #ifdef _WIN32
-#ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
 #ifdef _WIN64
 #define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
@@ -699,15 +697,8 @@ typedef int pid_t;
 #define WinDrvPrint(arg, ...)
 #define WinDrvEngPrint(arg, ...)
 #endif
-#endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -718,7 +709,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -795,4 +786,30 @@ typedef int pid_t;
 #define END_PTR_TO_ALIGNED_VAR \
   } while (0)

+
+/*
+ * -Wswitch means that when you pass switch an enum that it's looking for
+ * all values from that enum, and only that enum, to be accounted for.
+ * "default:;" is fine for catching values you don't care about. But today
+ * we have a bunch of code that uses internal and external enum values, or
+ * in other words combines two enums into a single variable. This cast is
+ * the workaround, but we really need to fix this mess.
+ */
+#define UNCHECKED_SWITCH__FIXME(x) switch ((uint64)(x))
+
+
+/*
+ * When clang static analyzer parses source files, it implicitly defines
+ * __clang_analyzer__ macro. We use this to define our custom macro to stop
+ * its execution for the current path of analysis by calling a function that
+ * doesn't return, making it think that it hit a failed assertion.
+ *
+ * DO NOT use to silence the analyzer! See PR2447238.
+ */
+#ifdef __clang_analyzer__
+#define VMW_CLANG_ANALYZER_NORETURN() Panic("Disable Clang static analyzer")
+#else
+#define VMW_CLANG_ANALYZER_NORETURN() ((void)0)
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,18 +63,9 @@
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/*
- * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -83,7 +74,6 @@
 #ifdef __x86_64__
 #define VM_X86_64
 #define vm_x86_64 1
-#define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
 #else
@@ -111,6 +101,7 @@

 #pragma warning (3 :4505) // unreferenced local function
 #pragma warning (disable :4018) // signed/unsigned mismatch
+#pragma warning (suppress:4619) // suppress warning next line (C4761 was removed in vs2019u4)
 #pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
 #pragma warning (disable :4305) // truncation from 'const int' to 'short'
 #pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
@@ -120,14 +111,91 @@

 #endif

-#if defined(__cplusplus) && __cplusplus >= 201103L || \
-    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
-    defined(__APPLE__) || defined(HAVE_STDINT_H)
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif
+
 /*
- * We're using <stdint.h> instead of <cstdint> below because some C++ code
- * deliberately compiles without C++ include paths.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif
+

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -138,40 +206,6 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-#else /* !HAVE_STDINT_H */
-
-/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
-#ifdef _MSC_VER
-
-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
-
-#elif __GNUC__
-#   if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#   else
-/*
- * Only strict c90 (without extensions) lacks a 'long long' type.
- * If this declaration fails ... use -std=c99 or -std=gnu90.
- */
-typedef unsigned long long uint64;
-typedef long long int64;
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-

 /*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
@@ -202,14 +236,6 @@ typedef char           Bool;
 #define IS_BOOL(x)     (((x) & ~1) == 0)


-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */
-
 /*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
@@ -218,9 +244,6 @@ typedef char           Bool;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -234,14 +257,11 @@ typedef char           Bool;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -249,46 +269,10 @@ typedef char           Bool;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -337,10 +321,11 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* macOS hosts use the same formatters for 32- and 64-bit. */
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
   #define FMT64         "ll"
-   #if KERNEL
+   #if defined(__APPLE__) && KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
@@ -452,7 +437,6 @@ typedef uintptr_t VPN;

 typedef uint64    PA;
 typedef uint64    PPN;
-typedef uint32    PPNTMP;

 typedef uint64    TPA;
 typedef uint64    TPPN;
@@ -470,9 +454,7 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint64    PgCnt64;
 typedef uint64    PageCnt;
-typedef uint64    PgNum64;
 typedef uint64    PageNum;
 typedef unsigned  MemHandle;
 typedef unsigned  IoHandle;
@@ -515,7 +497,7 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
@@ -598,15 +580,14 @@ typedef void * UserVA;


 /* Maximal observable PPN value. */
-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
 #define INVALID_PPN32     ((PPN32)0xffffffff)
-#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
-#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -624,7 +605,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((uint32)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -706,22 +688,28 @@ typedef void * UserVA;

 /*
 * At present, we effectively require a compiler that is at least
- * gcc-3.3 (circa 2003).  Enforce this here, various things below
+ * gcc-4.1 (circa 2006).  Enforce this here, various things below
 * this line depend upon it.
 *
- * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
- * The various linux kernel modules may use older (gcc-3.3) compilers.
+ * Current oldest compilers:
+ * - guest tools: 4.1.2 (freebsd/solaris)
+ * - buildhost compiler: 4.4.3
+ * - hosted kernel modules: 4.5
+ *
+ * SWIG's preprocessor is exempt.
 */
-#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
-#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
+#ifndef SWIG
+#if defined __GNUC__ && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
+#error "gcc version is too old, need gcc-4.1 or better"
+#endif
 #endif

 /*
- * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Similarly, we require a compiler that is at least vs2012.
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1400
-#error "cl.exe version is too old, need vc80 or better"
+#if defined _MSC_VER && _MSC_VER < 1700
+#error "cl.exe version is too old, need vs2012 or better"
 #endif


@@ -837,7 +825,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -883,12 +871,10 @@ typedef void * UserVA;
 #endif

 #ifndef UNUSED_TYPE
-// XXX _Pragma would better but doesn't always work right now.
 #  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
 #endif

 #ifndef UNUSED_VARIABLE
-// XXX is there a better way?
 #  define UNUSED_VARIABLE(_var) (void)_var
 #endif

@@ -903,29 +889,21 @@ typedef void * UserVA;

 /*
 * ALIGNED specifies minimum alignment in "n" bytes.
+ *
+ * NOTE: __declspec(align) has limited syntax; it must essentially be
+ *       an integer literal.  Expressions, such as sizeof(), do not
+ *       work.
 */

 #ifdef __GNUC__
 #define ALIGNED(n) __attribute__((__aligned__(n)))
+#elif defined(_MSC_VER)
+#define ALIGNED(n) __declspec(align(n))
 #else
 #define ALIGNED(n)
 #endif


-/*
- * Encapsulate the syntactic differences between gcc and msvc alignment control.
- * BOUNDARY must match in the prefix and suffix.
- */
-
-#ifdef _WIN32
-#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
-#define ALIGN_SUFFIX(BOUNDRY)
-#else
-#define ALIGN_PREFIX(BOUNDRY)
-#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
-#endif
-
-
 /*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
@@ -934,82 +912,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2005-2012,2014-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2005-2012,2014-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,7 @@
 *    VMware HD Audio controller
 */
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
+#define PCI_DEVICE_ID_VMWARE_SVGA3              0x0406
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
 #define PCI_DEVICE_ID_VMWARE_VGA                0x0711
@@ -138,23 +139,24 @@
 *    Intel 82545EM (e1000, server adapter, single port)
 *    Intel 82546EB (e1000, server adapter, dual port)
 *    Intel HECI (as embedded in ich9m)
- *    Intel XHCI (Panther Point / Intel 7 Series)
+ *    Intel XHCI (Panther Point / Intel 7 Series, 5Gbps)
+ *    Intel XHCI (Cannon Lake / Intel 300 Series, 10Gbps)
 */
-#define PCI_VENDOR_ID_INTEL             0x8086
-#define PCI_DEVICE_ID_INTEL_82439TX     0x7100
-#define PCI_DEVICE_ID_INTEL_82371AB_0   0x7110
-#define PCI_DEVICE_ID_INTEL_82371AB_2   0x7112
-#define PCI_DEVICE_ID_INTEL_82371AB_3   0x7113
-#define PCI_DEVICE_ID_INTEL_82371AB     0x7111
-#define PCI_DEVICE_ID_INTEL_82443BX     0x7190
-#define PCI_DEVICE_ID_INTEL_82443BX_1   0x7191
-#define PCI_DEVICE_ID_INTEL_82443BX_2   0x7192 /* Used when no AGP support */
-#define PCI_DEVICE_ID_INTEL_82545EM     0x100f
-#define PCI_DEVICE_ID_INTEL_82546EB     0x1010
-#define PCI_DEVICE_ID_INTEL_82574       0x10d3
-#define PCI_DEVICE_ID_INTEL_82574_APPLE 0x10f6
-#define PCI_DEVICE_ID_INTEL_HECI        0x2a74
+#define PCI_VENDOR_ID_INTEL                   0x8086
+#define PCI_DEVICE_ID_INTEL_82439TX           0x7100
+#define PCI_DEVICE_ID_INTEL_82371AB_0         0x7110
+#define PCI_DEVICE_ID_INTEL_82371AB_2         0x7112
+#define PCI_DEVICE_ID_INTEL_82371AB_3         0x7113
+#define PCI_DEVICE_ID_INTEL_82371AB           0x7111
+#define PCI_DEVICE_ID_INTEL_82443BX           0x7190
+#define PCI_DEVICE_ID_INTEL_82443BX_1         0x7191
+#define PCI_DEVICE_ID_INTEL_82443BX_2         0x7192 /* Used when no AGP support */
+#define PCI_DEVICE_ID_INTEL_82545EM           0x100f
+#define PCI_DEVICE_ID_INTEL_82546EB           0x1010
+#define PCI_DEVICE_ID_INTEL_82574             0x10d3
+#define PCI_DEVICE_ID_INTEL_82574_APPLE       0x10f6
 #define PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI 0x1e31
+#define PCI_DEVICE_ID_INTEL_CANNONLAKE_XHCI   0xa36d

 /*
 *  From drivers/usb/host/xhci-pci.c:
@@ -342,11 +344,14 @@
 #define MAX_NVDIMM 64

 /************* vRDMA implementation limits ******************************/
-#define MAX_VRDMA_DEVICES 1
+#define MAX_VRDMA_DEVICES 10

 /************* QAT implementation limits ********************/
 #define MAX_QAT_PCI_DEVICES 4

+/************* PrecisionClock implementation limits ********************/
+#define MAX_PRECISIONCLOCK_DEVICES 1
+
 /************* Strings for Host USB Driver *******************************/

 #ifdef _WIN32
Author	SHA1	Message	Date
Michal Kubecek	31c032e989	import Player 16.1.0 module sources	2020-11-20 09:13:14 +01:00
Michal Kubecek	a84c5005fe	import Player 16.0.0 module sources	2020-09-15 20:45:48 +02:00
Michal Kubecek	7fd1a58863	import Player 15.5.5 module sources	2020-05-30 02:30:43 +02:00
Michal Kubecek	cbd3374fd0	import Player 15.5.2 module sources	2020-03-14 09:59:24 +01:00
Michal Kubecek	87feab5f03	import Player 15.5.1 module sources	2019-11-14 07:56:57 +01:00
Michal Kubecek	6daa6921e7	import Player 15.5.0 module sources	2019-09-21 00:11:26 +02:00
Michal Kubecek	193db5638c	Merge branch 'master' into player	2019-09-21 00:10:31 +02:00
Michal Kubecek	85ede6aae9	misc: fix wrong directory name in INSTALL (fixes #36 ) Fix wrong directory name in one of the examples.	2019-09-09 08:38:27 +02:00
Michal Kubecek	1f8ca44e74	import Player 15.1.0 module sources	2019-05-15 07:06:02 +02:00