import Player 15.5.2 module sources

import Player 15.5.1 module sources
import Player 15.5.0 module sources
2020-03-14 09:59:24 +01:00 · 2019-11-14 07:56:57 +01:00 · 2019-09-21 00:11:26 +02:00 · 2019-09-21 00:10:31 +02:00 · 2019-09-09 08:38:27 +02:00 · 2019-05-15 07:06:02 +02:00
88 changed files with 5294 additions and 4335 deletions
@@ -150,7 +150,7 @@ test build).
 When using a downloaded tarball, simply run

  tar -cf vmmon.tar vmmon-only
-  tar -cf vmnet.tar vmmon-only
+  tar -cf vmnet.tar vmnet-only

 to create the tarballs. In this case, there is no protection against
 unwanted local changes.
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -129,6 +136,9 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+CC_OPTS += $(call vm_check_gcc,-mindirect-branch=thunk -mindirect-branch-register,)
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,14 +24,7 @@
 *      See monLoader.h for a full description.
 */

-#ifdef VMX86_VMX
-
-#include <string.h>
-#include <limits.h>
-#include "vmx.h"
-#include "vm_assert.h"
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #include "vmkernel.h"
 #include "libc.h"
@@ -57,15 +50,15 @@

 #include "vm_assert.h"

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

 #include "vm_basic_types.h"
 #include "monLoader.h"
 #include "vcpuid.h"
-#include "x86types.h"
-#include "x86paging_64.h"
+#include "vm_pagetable.h"
+#include "address_defs.h"
 #include "monLoaderLog.h"

 #define CANONICAL_MASK MASK64(36)
@@ -75,12 +68,6 @@
 #define L2EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 18))
 #define L1EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 9))

-/*
- * Import address space from VMX rather than building it.  Once monitor loading
- * is removed from VMX entirely, code to import will be deleted.
- */
-#define IMPORT_AS_FROM_VMX (TRUE)
-
 /* The maximum (canonical-address) VPN */
 #define VPN_MAX MASK64(52)

@@ -111,15 +98,11 @@ typedef struct MonLoaderContext {
      VPN          ASLastVPN;   /* last VPN in the address space (inclusive) */
      uint64       ASPTEFlags;  /* PTE flags for L4->L1 connection. */
      Vcpuid       currentVCPU;
-      Bool         ASImported;  /* Was this AS imported or created anew? */
      Bool         hasAddrSpace;
   } vcpu;
 } MonLoaderContext;


-static const Bool importFromVMX = IMPORT_AS_FROM_VMX;
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -192,7 +175,7 @@ MonLoaderIsMapped(MonLoaderContext *ctx,    // IN/OUT
   if (ret != ML_OK) {
      return ret;
   }
-   *mapped = PTE_PRESENT(pte);
+   *mapped = ML_PERM_PRESENT(pte);
   return ML_OK;
 }

@@ -280,15 +263,40 @@ MonLoaderMapMPN(MonLoaderContext *ctx,    // IN/OUT
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderBuildsPtLevel --
+ *
+ *      Determines whether MonLoader allocates and maps the page table(s) for
+ *      the monitor at the given level.
+ *
+ * Result:
+ *      TRUE if MonLoader creates the page table at the given level, FALSE
+ *      otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+static Bool
+MonLoaderCreatesPtLevel(PT_Level level)
+{
+   ASSERT(level >= PT_LEVEL_STOP && level <= PT_MAX_LEVELS);
+   return FALSE;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
 * MonLoaderCreateAddressSpace --
 *
- *      Creates or (if importAS is set) verifies an address space.  The VPN
- *      range specified by firstVPN and size is used to determine page counts
- *      at each page table level to map every page.  Sufficient pages are then
- *      allocated or verified.
+ *      Creates or verifies an address space.  The VPN range specified by
+ *      firstVPN and size is used to determine page counts at each page table
+ *      level to map every page.  Sufficient pages are then allocated or
+ *      verified.
 *
 * Result:
 *      ML_OK if successful, some other value on error.
@@ -302,8 +310,7 @@ static MonLoaderError
 MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
                            VPN               firstVPN, // IN
                            uint64            flags,    // IN
-                            uint64            monPages, // IN
-                            Bool              importAS) // IN
+                            uint64            monPages) // IN
 {
   Vcpuid vcpu = ctx->vcpu.currentVCPU;
   VPN lastVPN = firstVPN + monPages - 1;
@@ -329,11 +336,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
   ctx->vcpu.ASFirstVPN = firstVPN;
   ctx->vcpu.ASLastVPN = lastVPN;
   ctx->vcpu.ASPTEFlags = flags;
-   ctx->vcpu.ASImported = importAS;

   ptMPNs = &ctx->vcpu.ptMPNs;

-   if (importAS) {
+   if (MonLoaderCreatesPtLevel(PT_LEVEL_4)) {
+      NOT_IMPLEMENTED();
+   } else {
+      /* Verify the VMX's allocation. */
      mpn = MonLoaderCallout_GetPageRoot(ctx->envCtx, vcpu);
      if (mpn == INVALID_MPN) {
         return ML_ERROR_CALLOUT_PAGEROOT_GET;
@@ -341,13 +350,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
      LOG(5, "%s: vcpu %u page root=0x%"FMT64"x\n", __FUNCTION__, vcpu, mpn);
      ptMPNs->L4MPNs[0] = mpn;
      MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-   } else {
-      NOT_IMPLEMENTED();
   }
   ptMPNs->L4MPNCount = 1;

   for (i = 0; i < L3MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_3)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L4E;
         unsigned L4Off = PT_LPN_2_L4OFF(monVPN);
         PT_L1E pte;
@@ -357,20 +366,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L4E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L3MPNs[i] = mpn;
   }
   ptMPNs->L3MPNCount = L3MPNsNeeded;
      
   for (i = 0; i < L2MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_2)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L3E;
         unsigned L3Off = PT_LPN_2_L3OFF(monVPN);
         unsigned L3Page = L3EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -381,20 +390,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L3E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L2MPNs[i] = mpn;
   }
   ptMPNs->L2MPNCount = L2MPNsNeeded;

   for (i = 0; i < L1MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_1)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L2E;
         unsigned L2Off = PT_LPN_2_L2OFF(monVPN);
         unsigned L2Page = L2EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -405,17 +414,15 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         if (!PTE_PRESENT(pte)) {
+         if (!ML_PERM_PRESENT(pte)) {
            ptMPNs->L1MPNs[i] = INVALID_MPN;
            continue;
         }
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L1MPNs[i] = mpn;
   }
@@ -446,12 +453,12 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
 */
 static MonLoaderError
 MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
-                       unsigned          level,    // IN
+                       PT_Level          level,    // IN
                       uint64            flags,    // IN
                       VPN               monVPN,   // IN
                       uint64            monPages) // IN
 {
-   Bool verify = ctx->vcpu.ASImported;
+   Bool verify = !MonLoaderCreatesPtLevel(level);
   uint64    i;
   unsigned  count;
   MPN      *ptMPNs;
@@ -460,25 +467,25 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
      if (!ctx->vcpu.hasAddrSpace) {
         return ML_ERROR_NO_ADDRSPACE;
      }
-      if (level < 1 || level > 4) {
+      if (level < PT_LEVEL_STOP || level > PT_MAX_LEVELS) {
         return ML_ERROR_ARGS;
      }
   }

   switch (level) {
-      case 4:
+      case PT_LEVEL_4:
         count = ctx->vcpu.ptMPNs.L4MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L4MPNs;
         break;
-      case 3:
+      case PT_LEVEL_3:
         count = ctx->vcpu.ptMPNs.L3MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L3MPNs;
         break;
-      case 2:
+      case PT_LEVEL_2:
         count = ctx->vcpu.ptMPNs.L2MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L2MPNs;
         break;
-      case 1:
+      case PT_LEVEL_1:
         count = ctx->vcpu.ptMPNs.L1MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L1MPNs;
         break;
@@ -494,11 +501,11 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
         VPN vpn = monVPN + i;
         if (verify) {
            PT_L1E l1e;
-            if (ptMPNs[i] == INVALID_MPN && level == 1) {
+            if (ptMPNs[i] == INVALID_MPN && level == PT_LEVEL_1) {
               continue;
            }
            if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK ||
-                LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
+                ML_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
               return ML_ERROR_PAGE_TABLE_VERIFY;
            }
         } else {
@@ -629,7 +636,7 @@ MonLoaderCopyFromBlob(MonLoaderContext *ctx,        // IN/OUT
 *      ML_OK if successful, some other value on error.
 *
 * Side effects:
- *      User pages may be mapped in.
+ *      User or host pages may be mapped in.
 *
 *----------------------------------------------------------------------
 */
@@ -660,6 +667,14 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
                                                  vcpu);
      }
      if (mpn == INVALID_MPN) {
+         if (subIndex == MONLOADER_HT_MAP_IDX) {
+            /*
+             * This item is tied to a vmkernel feature.
+             * When the feature is disabled, there is nothing to share.
+             */
+            ASSERT(vmx86_server);
+            return ML_OK;
+         }
         /* Partial sharing is allowed.  Return success if any occurred. */
         return i != 0 ? ML_OK : ML_ERROR_SHARE;
      }
@@ -672,45 +687,6 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
 }


-#ifndef VMMON
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderSetEntrypoint --
- *
- *      Calls out to set the code and stack entrypoint.
- *
- * Result:
- *      ML_OK if successful, some other value on error.
- *
- * Side effects:
- *      The monitor entrypoint for the current VCPU may be set.
- *
- *----------------------------------------------------------------------
- */
-static MonLoaderError
-MonLoaderSetEntrypoint(MonLoaderContext *ctx,           // IN
-                       uint16            codeSelector,  // IN
-                       VA64              code,          // IN
-                       uint16            stackSelector, // IN
-                       VA64              stack)         // IN
-{
-   if (VA_2_VPN(code) < ctx->vcpu.ASFirstVPN  ||
-       VA_2_VPN(code) > ctx->vcpu.ASLastVPN   ||
-       VA_2_VPN(stack) < ctx->vcpu.ASFirstVPN ||
-       VA_2_VPN(stack) > ctx->vcpu.ASLastVPN) {
-      return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
-   }
-   if (MonLoaderCallout_SetEntrypoint(ctx->envCtx, codeSelector, code,
-                                      stackSelector, stack)) {
-      return ML_OK;
-   } else {
-      return ML_ERROR_CALLOUT_ENTRYPOINT;
-   }
-}
-#endif
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -795,7 +771,7 @@ MonLoaderShareFromBlob(MonLoaderContext *ctx,        // IN/OUT

   if ((blobOffset & (PAGE_SIZE - 1)) != 0 ||
       (blobSize & (PAGE_SIZE - 1)) != 0 ||
-       (flags & PTE_RW) != 0) {
+       ML_PERM_WRITEABLE(flags)) {
      return ML_ERROR_SHARE;
   }
   if (blobSize > monBytes || blobSize == 0) {
@@ -866,7 +842,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
      return ML_ERROR_TABLE_MISSING;
   }

-   if (!MonLoaderCallout_Init(args, &ctx.envCtx)) {
+   if (!MonLoaderCallout_Init(args, &ctx.envCtx, numVCPUs)) {
      return ML_ERROR_CALLOUT_INIT;
   }
   ret = ML_OK;
@@ -887,7 +863,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         uint64               monBytes   = PAGES_2_BYTES(monPages);
         uint64               subIndex   = entry->subIndex;
         uint64               bspOnly    = entry->bspOnly;
-         uint64               isPriv     = vmx86_server || entry->procVmmon;
         *line = i;

         /*
@@ -896,17 +871,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         if (bspOnly && !IS_BOOT_VCPUID(*vcpu)) {
            continue;
         }
-         /*
-          * Entries marked as privileged (isPriv == TRUE) are processed
-          * by the privileged layer (VMKernel/vmmon). Others are
-          * processed by the VMX. One exception is the entry of type
-          * ML_CONTENT_ADDRSPACE: it is always processed by the VMX and,
-          * if marked as privileged, also by the privileged layer.
-          */
-         if (isPriv != MonLoaderCallout_IsPrivileged(ctx.envCtx) &&
-             (content != ML_CONTENT_ADDRSPACE || isPriv == 0)) {
-            continue;
-         }
         if (monPages == 0 || monVPN + monPages - 1 > VPN_MAX) {
            ret = ML_ERROR_SIZE;
            continue;
@@ -919,8 +883,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
                  continue;
               }
               /* Create or verify the address space and page table MPNs. */
-               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages,
-                                                 importFromVMX);
+               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages);
               break;
            case ML_CONTENT_PAGETABLE_L4:
            case ML_CONTENT_PAGETABLE_L3:
@@ -988,15 +951,10 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         }
      }
   }
-#ifndef VMMON
+#ifdef VMKERNEL
   if (ret == ML_OK) {
      if (ctx.vcpu.ASLastVPN == 0) {
         ret = ML_ERROR_NO_ADDRSPACE;
-      } else {
-         ret = MonLoaderSetEntrypoint(&ctx, header->codeSelector,
-                                      header->codeEntrypoint,
-                                      header->stackSelector,
-                                      header->stackEntrypoint);
      }
   }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,7 +65,7 @@ typedef struct MonLoaderEnvContext {
 */

 Bool
-MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx)
+MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx, unsigned numVCPUs)
 {
   MonLoaderEnvContext *c;
   MonLoaderArgs *mlArgs = (MonLoaderArgs *)args;
@@ -215,7 +215,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
   vpn = Vmx86_MapPage(mpn);
   if (vpn == 0) {
      Log("Failed to map MPN 0x%"FMT64"x\n", mpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   HostIF_VMLock(vm, 41);
@@ -225,7 +225,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
      Log("Failed to track mapping from VPN 0x%"FMTVPN"x to MPN 0x%"FMT64"x\n",
          vpn, mpn);
      Vmx86_UnmapPage(vpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   return mpn;
@@ -386,10 +386,8 @@ MonLoaderFindSharedRegion(MonLoaderEnvContext *ctx,
   int i;

   for (i = 0; i < ML_SHARED_REGIONS_MAX; i++) {
-      if (ctx->shRegions[i].baseVpn == INVALID_VPN) {
-         /* Not found. */
-         return NULL;
-      } else if (ctx->shRegions[i].index == index) {
+      if (ctx->shRegions[i].index == index &&
+          ctx->shRegions[i].baseVpn != INVALID_VPN) {
         return &ctx->shRegions[i];
      }
   }
@@ -425,10 +423,6 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,

   ASSERT(IS_BOOT_VCPUID(vcpuid));

-   if (index >= ML_SHARED_REGIONS_MAX) {
-      Log("Invalid shared region %"FMT64"x", index);
-      return INVALID_MPN;
-   }
   s = MonLoaderFindSharedRegion(ctx, index);
   if (s == NULL) {
      return INVALID_MPN;
@@ -465,10 +459,10 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 */

 MPN
-MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx,
-                                   uint64 subIndex, // IN
-                                   unsigned page,   // IN
-                                   Vcpuid vcpu)     // IN
+MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx, // IN
+                                   uint64 subIndex,          // IN
+                                   unsigned page,            // IN
+                                   Vcpuid vcpu)              // IN
 {
   if (subIndex == MONLOADER_HEADER_IDX) {
      return VmmBlob_GetHeaderMpn(ctx->vm);
@@ -484,30 +478,7 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   NOT_IMPLEMENTED();
-}
-
-
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderCallout_IsPrivileged --
- *
- *      Returns whether or not this is a privileged environment.
- *
- * Returns:
- *      TRUE if privileged, FALSE if not.
- *
- * Side effects:
- *      None.
- *
- *----------------------------------------------------------------------
- */
-
-Bool
-MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN
-{
-   return TRUE;
+   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
 }


@@ -534,13 +505,3 @@ MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx,    // IN
   ASSERT((offset & (PAGE_SIZE - 1)) == 0);
   return VmmBlob_GetMpn(ctx->vm, BYTES_2_PAGES(offset));
 }
-
-Bool
-MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *ctx,           // IN
-                               uint16               codeSelector,  // IN
-                               VA64                 code,          // IN
-                               uint16               stackSelector, // IN
-                               VA64                 stack)         // IN
-{
-   NOT_IMPLEMENTED();
-}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,7 +31,8 @@
 #include "vmmblob.h"
 #include "monLoader.h"

-#define VMMBLOB_SIZE_MAX (12 * 1024 * 1024)
+#define VMMBLOB_SIZE_MAX (22 * 1024 * 1024) /* Ensure enough space for
+                                             * obj build with * GCOV_VMM=1. */

 /*
 *----------------------------------------------------------------------
@@ -65,7 +65,7 @@ APIC_GetMA(void)
    * Check if X2 APIC mode is enabled.
    */

-   if ((__GET_MSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
      return (MA)-1;
   }

@@ -75,7 +75,7 @@ APIC_GetMA(void)
    */

   // Mask out goo in the low 12 bits, which is unrelated to the address.
-   result = __GET_MSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);
+   result = X86MSR_GetMSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);

   /*
    * On Intel, the high bits are reserved so we mask.
@@ -122,7 +122,7 @@ APIC_Read(const APICDescriptor *desc, // IN
          int regNum)                 // IN
 {
   if (desc->isX2) {
-      return (uint32 )__GET_MSR(MSR_X2APIC_BASE + regNum);
+      return (uint32)X86MSR_GetMSR(MSR_X2APIC_BASE + regNum);
   } else {
      return desc->base[regNum][0];
   }
@@ -182,7 +182,7 @@ APIC_Write(const APICDescriptor *desc, // IN
           uint32 val)                 // IN
 {
   if (desc->isX2) {
-      __SET_MSR(MSR_X2APIC_BASE + regNum, val);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + regNum, val);
   } else {
      desc->base[regNum][0] = val;
   }
@@ -210,7 +210,7 @@ uint64
 APIC_ReadICR(const APICDescriptor *desc) // IN
 {
   if (desc->isX2) {
-      return __GET_MSR(MSR_X2APIC_BASE + APICR_ICRLO);
+      return X86MSR_GetMSR(MSR_X2APIC_BASE + APICR_ICRLO);
   } else {
      uint32 icrHi = desc->base[APICR_ICRHI][0];
      uint32 icrLo = desc->base[APICR_ICRLO][0];
@@ -244,7 +244,7 @@ APIC_WriteICR(const APICDescriptor *desc, // IN
 {
   if (desc->isX2) {
      uint64 icr = (uint64) id << 32 | icrLo;
-      __SET_MSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
   } else {
      ASSERT(!(id & ~(APIC_ICRHI_DEST_MASK >> APIC_ICRHI_DEST_OFFSET)));
      desc->base[APICR_ICRHI][0] = id << APIC_ICRHI_DEST_OFFSET;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -40,6 +40,7 @@ uint32      cpuidVersion;
 Bool        hostSupportsVT;
 Bool        hostSupportsSVM;
 Bool        hostHasSpecCtrl;
+Bool        hostSupportsXSave;


 /*
@@ -84,6 +85,7 @@ CPUID_Init(void)
   __GET_CPUID(1, &regs);
   cpuidVersion = regs.eax;
   cpuidFeatures = regs.edx;
+   hostSupportsXSave = CPUID_ISSET(1, ECX, XSAVE, regs.ecx);

   __GET_CPUID(0, &regs);
   ptr = (uint32 *)name;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,6 +37,7 @@ extern CpuidVendor cpuidVendor;
 extern Bool        hostSupportsVT;
 extern Bool        hostSupportsSVM;
 extern Bool        hostHasSpecCtrl;
+extern Bool        hostSupportsXSave;

 void CPUID_Init(void);
 Bool CPUID_AddressSizeSupported(void);
@@ -84,6 +85,12 @@ CPUID_HostSupportsSpecCtrl(void)
   return hostHasSpecCtrl;
 }

+static INLINE Bool
+CPUID_HostSupportsXSave(void)
+{
+   return hostSupportsXSave;
+}
+
 static INLINE Bool
 CPUID_SSE2Supported(void)
 {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,7 +27,7 @@
 #   define __HOST_KERNEL_H__


-#   ifdef linux
+#   ifdef __linux__
 /*
 * In some cases, this files needs to include Linux kernel header file
 * asm/page.h.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,7 +59,7 @@

 #define CROSSCALL_SLEEP_US 1000

-EXTERN int   HostIF_Init(VMDriver *vm, uint32 numVCPUs);
+EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);

@@ -76,12 +76,12 @@ EXTERN Bool  HostIF_IsLockedByMPN(VMDriver *vm, MPN mpn);
 EXTERN void  HostIF_FreeAllResources(VMDriver *vm);
 EXTERN uint64 HostIF_ReadUptime(void);
 EXTERN uint64 HostIF_UptimeFrequency(void);
-EXTERN unsigned int HostIF_EstimateLockedPageLimit(const VMDriver *vm,
-                                                   unsigned int lockedPages);
+EXTERN PageCnt HostIF_EstimateLockedPageLimit(const VMDriver *vm,
+                                              PageCnt lockedPages);
 EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
-EXTERN void *HostIF_AllocKernelPages(unsigned numPages, MPN *mpns);
-EXTERN void  HostIF_FreeKernelPages(unsigned numPages, void *ptr);
+EXTERN void *HostIF_AllocKernelPages(PageCnt numPages, MPN *mpns);
+EXTERN void  HostIF_FreeKernelPages(PageCnt numPages, void *ptr);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
@@ -111,11 +111,11 @@ EXTERN void HostIF_WaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_CancelWaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_WakeUpYielders(VMDriver *vm, Vcpuid currVcpu);

-EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                   unsigned int numPages, Bool kernelMPNBuffer);
-EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned int numPages, Bool kernelMPNBuffer);
+EXTERN int64 HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                     PageCnt numPages, Bool kernelMPNBuffer);
+EXTERN int HostIF_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn);
+EXTERN PageCnt HostIF_GetNumAnonPages(VMDriver *vm);
 EXTERN MPN HostIF_AllocLowPage(VMDriver *vm);

 EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr,
@@ -29,9 +29,10 @@
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
+#include "vm_basic_types.h"


-void *HostIF_AllocKernelMem(size_t size, int nonPageable);
+void *HostIF_AllocKernelMem(size_t size, Bool nonPageable);
 void  HostIF_FreeKernelMem(void *ptr);

 #endif // ifdef _HOSTIFMEM_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -96,7 +96,7 @@
 * Modify this value to increase the maximum number of tracked pages
 * per MemTrack instance.
 */
-#define MEMTRACK_MAX_TRACKED        MBYTES_2_PAGES(4096)
+#define MEMTRACK_MAX_TRACKED        GBYTES_2_PAGES(119)

 /*
 * Linux uses a 3-level directory, because we want to keep allocations
@@ -148,7 +148,7 @@ typedef uint64 MemTrackHTKey;

 typedef struct MemTrack {
   VMDriver         *vm;            /* The VM instance. */
-   unsigned          numPages;      /* Number of pages tracked. */
+   PageCnt           numPages;      /* Number of pages tracked. */
   MemTrackDir1      dir1;          /* First level directory. */
   MemTrackHT        vpnHashTable;  /* VPN to entry hashtable. */
   MemTrackHT       *mpnHashTable;  /* MPN to entry hashtable. */
@@ -197,7 +197,7 @@ MemTrackAllocPage(void)

 #define MEMTRACK_ALLOCDFN(_name, _itype, _otype)   \
   static INLINE _otype *                          \
-   _name(_itype *arg, unsigned pos)                \
+   _name(_itype *arg, uint64 pos)                  \
   {                                               \
      if (arg->dir[pos] == NULL) {                 \
         arg->dir[pos] = MemTrackAllocPage();      \
@@ -273,8 +273,8 @@ MemTrackHTInsert(MemTrackHT *ht,          // IN
 static void
 MemTrackCleanup(MemTrack *mt)    // IN
 {
-   unsigned idx;
-   unsigned p1;
+   PageCnt idx;
+   uint64 p1;
   MemTrackDir1 *dir1;
   Bool freeBackMap = mt != NULL && mt->mpnHashTable != NULL;

@@ -284,7 +284,7 @@ MemTrackCleanup(MemTrack *mt)    // IN
   dir1 = &mt->dir1;

   for (p1 = 0; p1 < MEMTRACK_DIR1_ENTRIES; p1++) {
-      unsigned p2;
+      uint64 p2;
      MemTrackDir2 *dir2 = MEMTRACK_GETDIR2(dir1, p1);

      if (dir2 == NULL) {
@@ -339,7 +339,7 @@ MemTrack *
 MemTrack_Init(VMDriver *vm) // IN:
 {
   MemTrack *mt;
-   unsigned idx;
+   PageCnt idx;

 #if defined(MEMTRACK_3LEVEL)
   ASSERT_ON_COMPILE(sizeof *mt <= PAGE_SIZE);
@@ -359,7 +359,8 @@ MemTrack_Init(VMDriver *vm) // IN:
      MemTrackHTPage *htPage = MemTrackAllocPage();

      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate VPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate VPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->vpnHashTable.pages[idx] = htPage;
@@ -374,7 +375,8 @@ MemTrack_Init(VMDriver *vm) // IN:
   for (idx = 0; idx < MEMTRACK_HT_PAGES; idx++) {
      MemTrackHTPage *htPage = MemTrackAllocPage();
      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate MPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate MPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->mpnHashTable->pages[idx] = htPage;
@@ -409,8 +411,8 @@ MemTrack_Add(MemTrack *mt,    // IN
             VPN64 vpn,       // IN
             MPN mpn)         // IN
 {
-   unsigned idx = mt->numPages;
-   unsigned p1, p2, p3;
+   PageCnt idx = mt->numPages;
+   uint64 p1, p2, p3;
   MemTrackEntry *ent;
   MemTrackDir1 *dir1 = &mt->dir1;
   MemTrackDir2 *dir2;
@@ -535,16 +537,16 @@ MemTrack_LookupMPN(MemTrack *mt, // IN
 *----------------------------------------------------------------------
 */

-unsigned
+PageCnt
 MemTrack_Cleanup(MemTrack *mt,            // IN
                 MemTrackCleanupCb *cb,   // IN
                 void *cData)             // IN
 {
-   unsigned idx;
-   unsigned count = 0;
+   PageCnt idx;
+   PageCnt count = 0;

   for (idx = 0; idx < mt->numPages; idx++) {
-      unsigned p1, p2, p3;
+      uint64 p1, p2, p3;
      MemTrackEntry *ent;
      MemTrackDir1 *dir1 = &mt->dir1;
      MEMTRACK_IDX2DIR(idx, p1, p2, p3);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -43,8 +43,8 @@ typedef struct MemTrackEntry {
 typedef void (MemTrackCleanupCb)(void *cData, MemTrackEntry *entry);

 extern struct MemTrack *MemTrack_Init(VMDriver *vm);
-extern unsigned MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
-                                 void *cbData);
+extern PageCnt MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
+                                void *cbData);
 extern MemTrackEntry *MemTrack_Add(struct MemTrack *mt, VPN64 vpn, MPN mpn);
 extern MemTrackEntry *MemTrack_LookupVPN(struct MemTrack *mt, VPN64 vpn);
 extern MemTrackEntry *MemTrack_LookupMPN(struct MemTrack *mt, MPN mpn);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,16 +24,15 @@
 *    Depending on configuration phystracker provides either 2-level or
 *    3-level structure to track whether page (specified by its MPN) is
 *    locked or no.  Linux uses 3-level structures with top limit of
- *    1TB (32bit) or 16TB (64bit).  Windows use 2-level structures
- *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.  On Mac
- *    limit is 2-level 128GB.
+ *    1TB (32bit) or 16TB (64bit).  Windows and Mac use 2-level structures
+ *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.
 *
 *    2-level phystracker is built on top of 3-level one by collapsing
 *    middle level.
 */


-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -50,9 +49,7 @@
 #define BYTES_PER_ENTRY      (PAGE_SIZE)
 #define PHYSTRACK_L3_ENTRIES (8 * BYTES_PER_ENTRY) /* 128MB */

-#if defined(WINNT_DDK)
-#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
-#elif defined(__linux__)
+#if defined(__linux__)
 #define PHYSTRACK_L2_ENTRIES (BYTES_PER_ENTRY / sizeof(void *)) /* 64GB or 128GB */
 /*
 * Currently MPN is 32 bits.  15 bits are in L3, 9 bits are in L2,
@@ -60,7 +57,7 @@
 */
 #define PHYSTRACK_L1_ENTRIES (256) /* 16TB. */
 #else
-#define PHYSTRACK_L1_ENTRIES ((128 + 4) * 8) /* 128 GB */
+#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
 #endif

 #ifndef PHYSTRACK_L2_ENTRIES
@@ -493,3 +490,32 @@ PhysTrack_GetNext(const PhysTracker *tracker, // IN
   return INVALID_MPN;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PhysTrack_GetNumTrackedPages --
+ *
+ *      Returns the total number of tracked pages
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+PageCnt
+PhysTrack_GetNumTrackedPages(const PhysTracker *tracker)
+{
+   PageCnt numTrackedMPNs = 0;
+   MPN nextMPN;
+   ASSERT(tracker);
+   ASSERT(HostIF_VMLockIsHeld(tracker->vm));
+   nextMPN = PhysTrack_GetNext(tracker, INVALID_MPN);
+   while (nextMPN != INVALID_MPN) {
+      numTrackedMPNs++;
+      nextMPN = PhysTrack_GetNext(tracker, nextMPN);
+   }
+   return numTrackedMPNs;
+}
+
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined(WINNT_DDK)
+#if !defined (__linux__)
 /* XXX: can be more efficient based on server vs. desktop and version of Windows */
 #define PHYSTRACK_MAX_SUPPORTED_GB (2048 + 4) /* 2 TB 64-bit W2k8 + 4 GB PCI */
 #endif
@@ -44,6 +44,7 @@ EXTERN void PhysTrack_Add(struct PhysTracker *, MPN);
 EXTERN void PhysTrack_Remove(struct PhysTracker *, MPN);
 EXTERN Bool PhysTrack_Test(const struct PhysTracker *, MPN);
 EXTERN MPN PhysTrack_GetNext(const struct PhysTracker *, MPN);
+EXTERN PageCnt PhysTrack_GetNumTrackedPages(const struct PhysTracker *);

 #endif

@@ -0,0 +1,299 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.c --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "sharedAreaVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmonIsMultiVCPU --
+ *
+ *      Helper function for determining if a shared area type represents
+ *      a region that has VCPU-specific memory.
+ *
+ * Results:
+ *      TRUE if type has VCPU-specific memory, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+SharedAreaVmmonIsMultiVCPU(SharedAreaType type)
+{
+   switch (type) {
+   case SHARED_AREA_PER_VM:
+   case SHARED_AREA_PER_VM_VMX:
+      return FALSE;
+   default:
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Init --
+ *
+ *      Initializes the shared area component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon shared area.
+ *
+ *----------------------------------------------------------------------
+ */
+
+SharedAreaVmmon *
+SharedAreaVmmon_Init(VMDriver *vm)
+{
+   SharedAreaVmmon *sa;
+
+   /*
+    * Allocate the shared area pointer here. The pages in each region are
+    * lazily allocated when a region is registered.
+    */
+   sa = HostIF_AllocKernelMem(sizeof *sa, FALSE);
+   if (sa == NULL) {
+      Warning("SharedArea failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(sa, 0, sizeof *sa);
+   return sa;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Cleanup --
+ *
+ *      Cleans up the shared area component of the VM Driver by freeing all
+ *      previously allocated VMMon shared area memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+SharedAreaVmmon_Cleanup(SharedAreaVmmon *area)
+{
+   SharedAreaType t;
+
+   if (area == NULL) {
+      return;
+   }
+
+   for (t = 0; t < NUM_SHARED_AREAS; t++) {
+      SharedAreaVmmonRegion *region = &area->regions[t];
+
+      if (region->pages != NULL) {
+         HostIF_FreeKernelMem(region->pages);
+      }
+   }
+   HostIF_FreeKernelMem(area);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_ValidateRegionArgs --
+ *
+ *      Validate the given VMMon shared area region registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register a shared
+ *      area region for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_ValidateRegionArgs(VMDriver *vm,
+                                   VMSharedAreaRegistrationBlock *block)
+{
+   SharedAreaVmmonRegion *region;
+   Vcpuid vcpu         = block->vcpu;
+   SharedAreaType type = block->region.index;
+   PageCnt numPages    = block->region.numPages;
+
+   if (vm == NULL               ||
+       vm->sharedArea == NULL   ||
+       vcpu >= vm->numVCPUs     ||
+       numPages == 0            ||
+       type >= NUM_SHARED_AREAS ||
+       (!SharedAreaVmmonIsMultiVCPU(type) && vcpu > 0)) {
+      return FALSE;
+   }
+
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu != 0) {
+
+      /*
+       * A region that was previously registered for a given VCPU should have
+       * the same number of pages as originally specified. Also, a region can
+       * only be reserved once for a given VCPU throughout the runtime of a VM.
+       */
+      return region->pagesPerVcpu == numPages &&
+             region->pages[vcpu * region->pagesPerVcpu] == INVALID_MPN;
+   } else {
+      /* First registration for this region. */
+      ASSERT(region->pages == NULL);
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_RegisterRegion --
+ *
+ *      Register a VMMon shared area region for a given VCPU with the VM Driver.
+ *      A successful return value indicates that the backing pages of the
+ *      shared area region are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the region was successfully registered for the given VCPU,
+ *      FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_RegisterRegion(VMDriver *vm,
+                               VMSharedAreaRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   SharedAreaType type = block->region.index;
+   SharedAreaVmmonRegion *region;
+   MPN *pages;
+   PageCnt page;
+
+   ASSERT(SharedAreaVmmon_ValidateRegionArgs(vm, block));
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu == 0) {
+      PageCnt pagesInRegion = block->region.numPages;
+
+      if (SharedAreaVmmonIsMultiVCPU(type)) {
+         pagesInRegion *= vm->numVCPUs;
+      }
+      region->pages = HostIF_AllocKernelMem(sizeof *pages * pagesInRegion,
+                                            FALSE);
+      if (region->pages == NULL) {
+         Warning("Failed to allocate pages array for region %u\n", type);
+         return FALSE;
+      }
+      for (page = 0; page < pagesInRegion; page++) {
+         region->pages[page] = INVALID_MPN;
+      }
+      region->pagesPerVcpu = block->region.numPages;
+   }
+
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+
+   /*
+    * Lock all shared area backing pages throughout the runtime of the monitor.
+    * The pages remain locked until they are freed by HostIF during VM Driver
+    * tear-down.
+    */
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      VPN vmxVPN = block->region.baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, pages + page);
+
+      if (status != PAGE_LOCK_SUCCESS) {
+         PageCnt resetPage;
+
+         /* Reset the region pages for this VCPU to their original state. */
+         for (resetPage = 0; resetPage < page; resetPage++) {
+            status = Vmx86_UnlockPage(vm, uAddr);
+            ASSERT(status == PAGE_UNLOCK_SUCCESS);
+            pages[resetPage] = INVALID_MPN;
+         }
+         return FALSE;
+      }
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_GetRegionMPN --
+ *
+ *      For the shared area region corresponding to the given type and VCPU,
+ *      get the MPN backing the region at the given offset.
+ *
+ * Results:
+ *      The MPN backing the region, or INVALID_MPN if no MPN is found.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+SharedAreaVmmon_GetRegionMPN(VMDriver *vm, SharedAreaVmmonRequest *request)
+{
+   SharedAreaType type = request->type;
+   Vcpuid vcpu = request->vcpu;
+   SharedAreaVmmonRegion *region;
+   PageCnt pgOffset = request->offset;
+   MPN *pages;
+
+   ASSERT(vcpu < vm->numVCPUs);
+   ASSERT(type < NUM_SHARED_AREAS);
+   ASSERT(IMPLIES(!SharedAreaVmmonIsMultiVCPU(type), vcpu == 0));
+   region = &vm->sharedArea->regions[type];
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   if (region->pages == NULL || region->pagesPerVcpu == 0) {
+      Warning("(%s) Requested unregistered region %u, VCPU %u\n", __FUNCTION__,
+              type, vcpu);
+      return INVALID_MPN;
+   }
+   if (pgOffset >= region->pagesPerVcpu) {
+      Warning("(%s) Offset %"FMT64"u (type %u, per-VCPU size %"FMT64"u)\n",
+              __FUNCTION__, pgOffset, type, region->pagesPerVcpu);
+      return INVALID_MPN;
+   }
+
+   return pages[pgOffset];
+}
@@ -0,0 +1,282 @@
+/*********************************************************
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * statVarsVmmon.c --
+ *
+ *     VMMon stat variables management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "statVarsVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Init --
+ *
+ *      Initializes the stat vars component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon stat vars struct.
+ *
+ *----------------------------------------------------------------------
+ */
+
+StatVarsVmmon *
+StatVarsVmmon_Init(VMDriver *vm)
+{
+   StatVarsVmmon *statVars;
+
+   statVars = HostIF_AllocKernelMem(sizeof *statVars, FALSE);
+   if (statVars == NULL) {
+      Warning("StatVars failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(statVars, 0, sizeof *statVars);
+   return statVars;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Cleanup --
+ *
+ *      Cleans up the stat vars component of the VM Driver by freeing all
+ *      previously allocated VMMon stat vars memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+StatVarsVmmon_Cleanup(StatVarsVmmon *statVars)
+{
+   if (statVars == NULL) {
+      return;
+   }
+
+   if (statVars->pages != NULL) {
+      HostIF_FreeKernelMem(statVars->pages);
+   }
+   HostIF_FreeKernelMem(statVars);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmonValidateRegistration --
+ *
+ *      Validate the given VMMon stat vars registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register stat vars
+ *      backing pages for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+StatVarsVmmonValidateRegistration(VMDriver *vm,
+                                   VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu      = block->vcpu;
+   PageCnt numPages = block->numPages;
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+
+   if (vm == NULL           ||
+       vm->statVars == NULL ||
+       numPages == 0        ||
+       vcpu >= vm->numVCPUs) {
+      return FALSE;
+   }
+
+   pagesPerVcpu = vm->statVars->pagesPerVcpu;
+   pages = vm->statVars->pages;
+   if (pagesPerVcpu == 0) {
+      ASSERT(pages == NULL);
+      return TRUE;
+   } else {
+      return pagesPerVcpu == numPages &&
+             pages[vcpu * pagesPerVcpu] == INVALID_MPN;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_RegisterVCPU --
+ *
+ *      Register stat vars backing pages for the given VCPU with VMMon.
+ *      A successful return value indicates that stat vars backing pages
+ *      for the given VCPU are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the backing pages were successfully registered for the
+ *      given VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+StatVarsVmmon_RegisterVCPU(VMDriver *vm,
+                           VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   PageCnt pagesPerVcpu = block->numPages;
+   StatVarsVmmon *statVars = vm->statVars;
+   MPN *pages;
+   PageCnt page;
+
+   HostIF_VMLock(vm, 45);
+   if (!StatVarsVmmonValidateRegistration(vm, block)) {
+      HostIF_VMUnlock(vm, 45);
+      return FALSE;
+   }
+   if (statVars->pagesPerVcpu == 0) {
+      PageCnt totalPages = pagesPerVcpu * vm->numVCPUs;
+      statVars->pages = HostIF_AllocKernelMem(totalPages * sizeof *pages,
+                                              FALSE);
+      if (statVars->pages == NULL) {
+         HostIF_VMUnlock(vm, 45);
+         return FALSE;
+      }
+      for (page = 0; page < totalPages; page++) {
+         statVars->pages[page] = INVALID_MPN;
+      }
+      statVars->pagesPerVcpu = pagesPerVcpu;
+   }
+
+   /*
+    * Validate that all pages in this VCPU's region are still invalid. Before
+    * unlocking, temporarily set the first page to 0 to prevent a malicious user
+    * from triggering an ASSERT by firing parallel ioctls for the same VCPU.
+    */
+   pages = statVars->pages + vcpu * pagesPerVcpu;
+   for (page = 0; page < pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+   ASSERT_ON_COMPILE(INVALID_MPN != 0);
+   pages[0] = 0;
+   HostIF_VMUnlock(vm, 45);
+
+   /*
+    * Verify that all stat vars backing pages are locked.  The pages are
+    * assumed to remain locked either until they are unlocked by the VMX
+    * following a NUMA migration, or by HostIF during VM Driver tear-down.
+    */
+   for (page = 0; page < pagesPerVcpu; page++) {
+      VPN vmxVPN = block->baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      PageCnt resetPage;
+      MPN *currPage = pages + page;
+      Bool verifiedLocked = FALSE;
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, currPage);
+
+      /*
+       * Backing pages are expected to be locked by the VMX before being
+       * registered.
+       */
+      if (status == PAGE_LOCK_ALREADY_LOCKED) {
+         verifiedLocked = TRUE;
+         status = Vmx86_LookupUserMPN(vm, uAddr, currPage);
+         /*
+          * PR 2260615: Some platforms return PAGE_LOCK_SUCCESS for a successful
+          * user MPN lookup, others return PAGE_LOOKUP_SUCCESS, but they both
+          * happen to equal the same value.
+          */
+         ASSERT_ON_COMPILE(PAGE_LOOKUP_SUCCESS == PAGE_LOCK_SUCCESS);
+         if (status == PAGE_LOOKUP_SUCCESS) {
+            /*
+             * The backing page was verified as previously locked, and was
+             * successfully recorded.
+             */
+            continue;
+         }
+      }
+      /*
+       * The backing page was either not previously locked, or was not recorded.
+       * In either case, reset all previously recorded pages to INVALID_MPN.
+       */
+      if (!verifiedLocked && status == PAGE_LOCK_SUCCESS) {
+         status = Vmx86_UnlockPage(vm, uAddr);
+         ASSERT(status == PAGE_UNLOCK_SUCCESS);
+      }
+      for (resetPage = 0; resetPage <= page; resetPage++) {
+         pages[resetPage] = INVALID_MPN;
+      }
+      return FALSE;
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_GetRegionMPN --
+ *
+ *      Obtain the backing MPN for stat vars for the given VCPU at
+ *      the given offset.
+ *
+ * Results:
+ *      The backing stat vars MPN for the given VCPU at the given offset
+ *      if it has been registered, or INVALID_MPN otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpu, PageCnt offset)
+{
+   MPN *pages;
+   MPN backingPage;
+   StatVarsVmmon *statVars = vm->statVars;
+
+   ASSERT(vcpu < vm->numVCPUs);
+
+   HostIF_VMLock(vm, 46);
+   if (offset >= statVars->pagesPerVcpu) {
+      HostIF_VMUnlock(vm, 46);
+      Warning("(%s) Requested MPN at invalid offset %"FMT64"u for VCPU %u\n",
+              __FUNCTION__, offset, vcpu);
+      return INVALID_MPN;
+   }
+
+   pages = statVars->pages + vcpu * statVars->pagesPerVcpu;
+   backingPage = pages[offset];
+   HostIF_VMUnlock(vm, 46);
+   return backingPage;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -33,7 +33,7 @@
 *
 */

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"
 #   include <linux/string.h> /* memset() in the kernel */
@@ -61,10 +61,11 @@
 #include "x86vt.h"
 #include "x86vtinstr.h"
 #include "apic.h"
-#include "x86perfctr.h"
+#include "perfctr.h"
 #include "x86paging_common.h"
 #include "x86paging_64.h"
 #include "memtrack.h"
+#include "monLoader.h"

 #ifdef LINUX_GDT_IS_RO
 #   include <asm/desc.h>
@@ -846,7 +847,7 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
      VMMPageTablePatch *patch = &cpData->vmmPTP[i];
      unsigned l4idx = PT_LPN_2_L4OFF(patch->lpn);
      unsigned l3idx = PT_LPN_2_L3OFF(patch->lpn);
-      PT_L4E pte;
+      PT_L4E pte = 0;
      MA ma = cpData->monCR3 + l4idx * sizeof pte; /* PTE machine address. */

      if (patch->level == PTP_EMPTY) {
@@ -943,8 +944,8 @@ static Bool
 TaskVerifyPTPatches(VMDriver *vm,
                    VMCrossPageData *cpData)
 {
-   LPN64 xgCPN = LA_2_LPN(cpData->vmmCrossGDTLA);
-   LPN64 xpCPN = LA_2_LPN(cpData->vmmCrossPageLA);
+   LPN64 xgCPN = LA_2_LPN(cpData->crossGDTLA);
+   LPN64 xpCPN = LA_2_LPN(cpData->crossPageLA);
   MPN   xpMPN = MA_2_MPN(cpData->crosspageMA);
   MA    cr3   = cpData->monCR3;
   return TaskVerifyPTMap(vm, cr3, xpCPN, xpMPN) &&
@@ -1092,9 +1093,8 @@ TaskSetCrossGDTVMM(BSVMM_GDTInit *gdt)
 * TaskSetCrossGDTHost --
 *
 *      Initializes the host portion of the crossGDT by copying it directly
- *      from the host kernel's GDT. We assume that all the host segments
- *      we will ever need come from the first page of the host's GDT and
- *      precede the lowest segment needed by the monitor.
+ *      from the host kernel's GDT. We assume that all the host segments we
+ *      will ever need come from the first page of the host's GDT.
 *
 * Results:
 *      TRUE on success, FALSE otherwise.
@@ -1106,30 +1106,17 @@ TaskSetCrossGDTVMM(BSVMM_GDTInit *gdt)
 */

 static void
-TaskSetCrossGDTHost(const BSVMM_GDTInit *gdt)
+TaskSetCrossGDTHost(void)
 {
   unsigned len;
   DTR64 hostGDT;
-   unsigned i;
-   /* Copy descriptors up to at most the size of the CrossGDT. */
-   unsigned minVMMIndex = sizeof(CrossGDT) / sizeof(Descriptor);
-
   /*
    * All copied host segment descriptors will come from the first page of
-    * the host kernel GDT and precede the first monitor segment descriptor.
+    * the host kernel GDT.
    */
-   ASSERT_ON_COMPILE(sizeof(CrossGDT) % sizeof(Descriptor) == 0);
   ASSERT(HostIF_GlobalLockIsHeld());
-   for (i = 0; i < ARRAYSIZE(gdt->entries); i++) {
-      const BSVMM_GDTInitEntry *entry = &gdt->entries[i];
-      if (entry->present == 1) {
-         minVMMIndex = MIN(entry->index, minVMMIndex);
-      }
-   }
   TaskSaveGDT(&hostGDT);
-   /* PR 2142795: Only copy up to at most the monitor's first descriptor. */
-   len = MIN((unsigned)hostGDT.limit + 1, minVMMIndex * sizeof(Descriptor));
-   ASSERT(len <= sizeof(CrossGDT));
+   len = MIN((unsigned)hostGDT.limit + 1, sizeof(CrossGDT));
   memcpy(crossGDT->gdtes, (void*)HOST_KERNEL_LA_2_VA((LA)hostGDT.offset), len);
 }

@@ -1171,7 +1158,7 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
         return FALSE;
      }
      memset(crossGDT, 0, sizeof *crossGDT);
-      TaskSetCrossGDTHost(gdt);
+      TaskSetCrossGDTHost();
   }

   populatedCrossGDT = TaskSetCrossGDTVMM(gdt);
@@ -1589,8 +1576,8 @@ TaskCreatePTPatches(VMDriver    *vm,
                    uint16      *numPages)
 {
   VMCrossPageData * const cpData = &crosspage->crosspageData;
-   LPN64                   xgCPN  = LA_2_LPN(cpData->vmmCrossGDTLA);
-   LPN64                   xpCPN  = LA_2_LPN(cpData->vmmCrossPageLA);
+   LPN64                   xgCPN  = LA_2_LPN(cpData->crossGDTLA);
+   LPN64                   xpCPN  = LA_2_LPN(cpData->crossPageLA);
   MPN                     xpMPN  = MA_2_MPN(cpData->crosspageMA);
   /*
    * Set up patches that the BackToHost code will use to map the
@@ -1616,8 +1603,7 @@ TaskCreatePTPatches(VMDriver    *vm,
 *    Initialize the crosspage used to switch to the monitor task.
 *
 * Results:
- *    0 on success
- *    != 0 on failure
+ *    TRUE on success, FALSE on failure.
 *
 * Side effects:
 *    None
@@ -1625,29 +1611,28 @@ TaskCreatePTPatches(VMDriver    *vm,
 *-----------------------------------------------------------------------------
 */

-int
-Task_InitCrosspage(VMDriver *vm,          // IN
-                   InitBlock *initParams) // IN/OUT: Initial params from the
-                                          //         VM
+Bool
+Task_InitCrosspage(VMDriver *vm,               // IN
+                   LPN monStartLPN,            // IN
+                   LPN monEndLPN,              // IN
+                   PerVcpuPages *perVcpuPages) // IN
 {
   Vcpuid vcpuid;
   uint16 numPTPPages = 0;

   if (crossGDT == NULL) {
-      return 1;
+      return FALSE;
   }

-   initParams->crossGDTMPN = crossGDTMPN;
-
   ASSERT(0 < vm->numVCPUs && vm->numVCPUs <= MAX_VCPUS);
   for (vcpuid = 0; vcpuid < vm->numVCPUs; vcpuid++) {
-      VA64             crossPageUserAddr = initParams->crosspage[vcpuid];
+      VA64             crossPageUserAddr = perVcpuPages[vcpuid].crosspage;
      VMCrossPage     *p = HostIF_MapCrossPage(vm, crossPageUserAddr);
      VMCrossPageData *cpData;
      MPN              crossPageMPN;

      if (p == NULL) {
-         return 1;
+         return FALSE;
      }
      cpData = &p->crosspageData;

@@ -1656,7 +1641,7 @@ Task_InitCrosspage(VMDriver *vm,          // IN
          PAGE_LOOKUP_SUCCESS ||
          crossPageMPN == 0) {
         HostIF_VMUnlock(vm, 38);
-         return 1;
+         return FALSE;
      }
      HostIF_VMUnlock(vm, 38);

@@ -1673,39 +1658,39 @@ Task_InitCrosspage(VMDriver *vm,          // IN
         Warning("%s: crosspage version mismatch: vmmon claims %#x, must "
                 "match vmx version of %#x.\n", __FUNCTION__,
                 (int)CROSSPAGE_VERSION, cpData->version);
-         return 1;
+         return FALSE;
      }
      if (!pseudoTSC.initialized) {
         Warning("%s*: PseudoTSC has not been initialized\n", __FUNCTION__);
-         return 1;
+         return FALSE;
      }
      cpData->crosspageMA = MPN_2_MA(crossPageMPN);
-      cpData->hostCrossPageLA = (LA64)(uintptr_t)p;
-      cpData->vmmCrossPageLA = cpData->hostCrossPageLA;
-      cpData->vmmCrossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT);
+      cpData->crossPageLA = (LA64)(uintptr_t)p;
+      cpData->crossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.offset = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.limit  = sizeof(CrossGDT) - 1;
+      if (CPUID_HostSupportsXSave()) {
+         cpData->wsCR4 |= CR4_OSXSAVE;
+      }
+      cpData->monCR3 = MPN_2_MA(vm->ptRootMpns[vcpuid]);

      HostIF_VMLock(vm, 39);
-      if (!TaskCreatePTPatches(vm, p,
-                               initParams->monStartLPN,
-                               initParams->monEndLPN,
-                               &numPTPPages)) {
+      if (!TaskCreatePTPatches(vm, p, monStartLPN, monEndLPN, &numPTPPages)) {
         HostIF_VMUnlock(vm, 39);
         Warning("%s: Could not create page table patches for VCPU %d\n",
                 __FUNCTION__, vcpuid);
-         return 1;
+         return FALSE;
      }
      HostIF_VMUnlock(vm, 39);
      if (!TaskApplyPTPatches(vm, cpData)) {
         Warning("%s: Could not apply page table patches for VCPU %d\n",
                 __FUNCTION__, vcpuid);
-         return 1;
+         return FALSE;
      }
      if (!TaskVerifyPTPatches(vm, cpData)) {
         Warning("%s: Page table patches for VCPU %d failed verification\n",
                 __FUNCTION__, vcpuid);
-         return 1;
+         return FALSE;
      }

      /*
@@ -1733,12 +1718,12 @@ Task_InitCrosspage(VMDriver *vm,          // IN
      TaskInitHostSwitchIDT(p);
   }
   /*
-    * Report back to the VMX the number of pages allocated for this VM's
-    * page table patches.
+    * Store the number of pages allocated for this VM's page table patches so
+    * the bootstrap can account for the memory overhead later in a module call.
    */
-   initParams->numPTPPages = numPTPPages;
+   vm->numPTPPages = numPTPPages;

-   return 0;
+   return TRUE;
 }


@@ -2520,7 +2505,7 @@ Task_Switch(VMDriver *vm,  // IN
    * world switch.  We must be careful not to overwrite the
    * crosspages arguments when doing this though, see bug 820257.
    */
-   if (hvRootMPN == INVALID_MPN) {
+   if (hvRootMPN == INVALID_MPN && CPUID_HostSupportsHV()) {
      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
      crosspage->crosspageData.pcpuNum = pCPU;
@@ -2545,9 +2530,9 @@ Task_Switch(VMDriver *vm,  // IN
          * hosts IDT - PR 848701.
          */
         if (pebsAvailable) {
-            pebsMSR = __GET_MSR(IA32_MSR_PEBS_ENABLE);
+            pebsMSR = X86MSR_GetMSR(IA32_MSR_PEBS_ENABLE);
            if (pebsMSR != 0) {
-               __SET_MSR(IA32_MSR_PEBS_ENABLE, 0);
+               X86MSR_SetMSR(IA32_MSR_PEBS_ENABLE, 0);
            }
         }

@@ -2556,9 +2541,9 @@ Task_Switch(VMDriver *vm,  // IN
          * enabled.
          */
         if (ptAvailable) {
-            ptMSR = __GET_MSR(MSR_RTIT_CTL);
+            ptMSR = X86MSR_GetMSR(MSR_RTIT_CTL);
            if ((ptMSR & MSR_RTIT_CTL_TRACE_EN) != 0) {
-               __SET_MSR(MSR_RTIT_CTL, ptMSR & ~MSR_RTIT_CTL_TRACE_EN);
+               X86MSR_SetMSR(MSR_RTIT_CTL, ptMSR & ~MSR_RTIT_CTL_TRACE_EN);
            }
         }

@@ -2578,12 +2563,12 @@ Task_Switch(VMDriver *vm,  // IN
             * so that we can set CR4.VMXE to activate VMX.
             */
            uint64 bits = MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE;
-            uint64 featCtl = __GET_MSR(MSR_FEATCTL);
+            uint64 featCtl = X86MSR_GetMSR(MSR_FEATCTL);
            if ((featCtl & bits) != bits) {
               if ((featCtl & MSR_FEATCTL_LOCK) != 0) {
                  Panic("Intel VT-x is disabled and locked on CPU %d\n", pCPU);
               }
-               __SET_MSR(MSR_FEATCTL, featCtl | bits);
+               X86MSR_SetMSR(MSR_FEATCTL, featCtl | bits);
            }
         }

@@ -2651,12 +2636,12 @@ Task_Switch(VMDriver *vm,  // IN
         }

         if (CPUID_HostSupportsSVM()) {
-            efer = __GET_MSR(MSR_EFER);
+            efer = X86MSR_GetMSR(MSR_EFER);
            if ((efer & MSR_EFER_SVME) == 0) {
-               __SET_MSR(MSR_EFER, efer | MSR_EFER_SVME);
+               X86MSR_SetMSR(MSR_EFER, efer | MSR_EFER_SVME);
            }
-            foreignHSAVE = __GET_MSR(MSR_VM_HSAVE_PA);
-            __SET_MSR(MSR_VM_HSAVE_PA, MPN_2_MA(hvRootMPN));
+            foreignHSAVE = X86MSR_GetMSR(MSR_VM_HSAVE_PA);
+            X86MSR_SetMSR(MSR_VM_HSAVE_PA, MPN_2_MA(hvRootMPN));
         }

         /*
@@ -2721,9 +2706,9 @@ Task_Switch(VMDriver *vm,  // IN
         if (CPUID_HostSupportsSpecCtrl()) {
 #ifdef CYCLE_SPEC_CTRL
            currentSpecCtrlValue = (currentSpecCtrlValue + 1) % 4;
-            __SET_MSR(MSR_SPEC_CTRL, currentSpecCtrlValue);
+            X86MSR_SetMSR(MSR_SPEC_CTRL, currentSpecCtrlValue);
 #endif
-            crosspage->crosspageData.specCtrl = __GET_MSR(MSR_SPEC_CTRL);
+            crosspage->crosspageData.specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
         }

         DEBUG_ONLY(crosspage->crosspageData.monTinyStack[0] = 0xDEADBEEF;)
@@ -2737,11 +2722,11 @@ Task_Switch(VMDriver *vm,  // IN

 #ifdef CYCLE_SPEC_CTRL
         if (CPUID_HostSupportsSpecCtrl()) {
-            readSpecCtrlValue = __GET_MSR(MSR_SPEC_CTRL);
+            readSpecCtrlValue = X86MSR_GetMSR(MSR_SPEC_CTRL);
            specCtrlEqual = readSpecCtrlValue ==
                            crosspage->crosspageData.specCtrl;
            /* Do not leak cycling SPEC_CTRL value back to host. */
-            __SET_MSR(MSR_SPEC_CTRL, 0);
+            X86MSR_SetMSR(MSR_SPEC_CTRL, 0);
         }
 #endif

@@ -2764,9 +2749,9 @@ Task_Switch(VMDriver *vm,  // IN
         }

         if (CPUID_HostSupportsSVM()) {
-            __SET_MSR(MSR_VM_HSAVE_PA, foreignHSAVE);
+            X86MSR_SetMSR(MSR_VM_HSAVE_PA, foreignHSAVE);
            if ((efer & MSR_EFER_SVME) == 0) {
-               __SET_MSR(MSR_EFER, efer);
+               X86MSR_SetMSR(MSR_EFER, efer);
            }
         }

@@ -2840,11 +2825,11 @@ Task_Switch(VMDriver *vm,  // IN
         TaskLoadIDT(&hostIDT);

         if (pebsMSR != 0) {
-            __SET_MSR(IA32_MSR_PEBS_ENABLE, pebsMSR);
+            X86MSR_SetMSR(IA32_MSR_PEBS_ENABLE, pebsMSR);
         }

         if ((ptMSR & MSR_RTIT_CTL_TRACE_EN) != 0) {
-            __SET_MSR(MSR_RTIT_CTL, ptMSR);
+            X86MSR_SetMSR(MSR_RTIT_CTL, ptMSR);
         }

         TaskUpdateLatestPTSC(vm, &crosspage->crosspageData);
@@ -2909,6 +2894,18 @@ Task_Switch(VMDriver *vm,  // IN
   }

   if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR) {
+
+      /*
+       * Newer versions of Window expect EFLAGS_AC to be set when handling an
+       * interrupt - PR 2248661.
+       */
+      if ((flags & EFLAGS_AC) != 0) {
+         uintptr_t curFlags;
+         SAVE_FLAGS(curFlags);
+         curFlags |= EFLAGS_AC;
+         RESTORE_FLAGS(curFlags);
+      }
+
      /*
       * Note we must do the RAISE_INTERRUPT before ever enabling
       * interrupts or bad things have happened (might want to know exactly
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,10 +24,11 @@
 #include "includeCheck.h"
 #include "bootstrap_vmm.h"

-struct InitBlock;
+struct MonLoaderHeader;

 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
-extern int Task_InitCrosspage(VMDriver *vm, struct InitBlock *params);
+extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
+                               PerVcpuPages *perVcpuPages);
 extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,6 @@
 #include "vm_basic_math.h"
 #include "vmx86.h"
 #include "task.h"
-#include "initblock.h"
 #include "vm_asm.h"
 #include "iocontrols.h"
 #include "hostif.h"
@@ -56,14 +55,16 @@
 #include "x86vt.h"
 #include "x86svm.h"
 #include "x86cpuid_asm.h"
-#if defined(linux)
+#if defined(__linux__)
 #include <asm/timex.h>
 #endif
-#include "x86perfctr.h"
+#include "perfctr.h"
 #include "x86vtinstr.h"
 #include "bootstrap_vmm.h"
 #include "monLoader.h"
 #include "vmmblob.h"
+#include "sharedAreaVmmon.h"
+#include "statVarsVmmon.h"

 PseudoTSC pseudoTSC;

@@ -77,14 +78,14 @@ static VMDriver *vmDriverList = NULL;
 static LockedPageLimit lockedPageLimit = {
   0,                        // host: does not need to be initialized.
   0,                        // configured: must be set by some VM as it is powered on.
-   (uint32)MAX_LOCKED_PAGES, // dynamic
+   MAX_LOCKED_PAGES,         // dynamic
 };

 /* Percentage of guest "paged" memory that must fit within the hard limit. */
-static unsigned minVmMemPct;
+static Percent minVmMemPct;

 /* Number of pages actually locked by all virtual machines */
-static unsigned numLockedPages;
+static PageCnt numLockedPages;

 /* Total virtual machines on this host */
 static unsigned vmCount;
@@ -141,13 +142,12 @@ typedef struct NXData {
 *----------------------------------------------------------------------
 */

-static INLINE unsigned
+static INLINE PageCnt
 Vmx86AdjustLimitForOverheads(const VMDriver* vm,
-                             const uint32 limit)
+                             const PageCnt limit)
 {
-   uint32 extraCost = (vm != NULL) ? vmCount * vm->memInfo.perVMOverhead : 0;
+   PageCnt extraCost = (vm != NULL) ? vmCount * vm->memInfo.perVMOverhead : 0;
   ASSERT(HostIF_GlobalLockIsHeld());
-
   return (extraCost < limit) ?  (limit - extraCost) : 0;
 }

@@ -176,12 +176,11 @@ Vmx86AdjustLimitForOverheads(const VMDriver* vm,
 *----------------------------------------------------------------------
 */

-static INLINE unsigned
+static INLINE PageCnt
 Vmx86LockedPageLimit(const VMDriver* vm)  // IN:
 {
-   uint32 overallLimit;
+   PageCnt overallLimit;
   ASSERT(HostIF_GlobalLockIsHeld());
-
   lockedPageLimit.host = HostIF_EstimateLockedPageLimit(vm, numLockedPages);
   overallLimit = MIN(MIN(lockedPageLimit.configured, lockedPageLimit.dynamic),
                      lockedPageLimit.host);
@@ -212,7 +211,7 @@ Vmx86LockedPageLimit(const VMDriver* vm)  // IN:

 static INLINE Bool
 Vmx86HasFreePages(VMDriver *vm,
-                  unsigned int numPages,
+                  PageCnt numPages,
                  Bool checkVM)
 {
   /*
@@ -226,10 +225,7 @@ Vmx86HasFreePages(VMDriver *vm,
          (!checkVM || HostIF_VMLockIsHeld(vm)));

   if (checkVM) {
-      /*
-       * Check the per-vm limit.
-       */
-
+      /* Check the per-vm limit. */
      ASSERT(HostIF_VMLockIsHeld(vm));
      if (vm->memInfo.admitted) {
         if (vm->memInfo.maxAllocation <= vm->memInfo.locked) {
@@ -239,12 +235,8 @@ Vmx86HasFreePages(VMDriver *vm,
         }
      }
   } else {
-      /*
-       * Check the global limit.
-       */
-
-      unsigned limit = Vmx86LockedPageLimit(vm);
-
+      /* Check the global limit. */
+      PageCnt limit = Vmx86LockedPageLimit(vm);
      if (limit <= numLockedPages) {
         return FALSE;
      } else if (limit - numLockedPages < numPages) {
@@ -555,7 +547,7 @@ Vmx86_Free(void *ptr)
 void *
 Vmx86_Calloc(size_t numElements, // IN
             size_t elementSize, // IN
-             int nonPageable)    // IN
+             Bool nonPageable)   // IN
 {
   size_t numBytes = numElements * elementSize;
   void *retval;
@@ -732,6 +724,14 @@ Vmx86FreeAllVMResources(VMDriver *vm)
         VmmBlob_Cleanup(vm->blobInfo);
         vm->blobInfo = NULL;
      }
+      if (vm->sharedArea != NULL) {
+         SharedAreaVmmon_Cleanup(vm->sharedArea);
+         vm->sharedArea = NULL;
+      }
+      if (vm->statVars != NULL) {
+         StatVarsVmmon_Cleanup(vm->statVars);
+         vm->statVars = NULL;
+      }

      HostIF_FreeAllResources(vm);

@@ -768,7 +768,7 @@ Vmx86FreeAllVMResources(VMDriver *vm)

 static Bool
 Vmx86ReserveFreePages(VMDriver *vm,
-                      unsigned int numPages,
+                      PageCnt numPages,
                      Bool ignoreLimits)
 {
   Bool retval = FALSE;
@@ -779,15 +779,14 @@ Vmx86ReserveFreePages(VMDriver *vm,
   for (retries = 3; !retval && (retries > 0); retries--) {
      HostIF_GlobalLock(17);
      HostIF_VMLock(vm, 0);
-
-      // Check VM's limit and don't wait.
+      /* Check VM's limit and don't wait. */
      retval = Vmx86HasFreePages(vm, numPages, TRUE);
      if (!retval) {
         HostIF_VMUnlock(vm, 0);
         HostIF_GlobalUnlock(17);
         break;
      } else {
-         // Wait to satisfy the global limit.
+         /* Wait to satisfy the global limit. */
         retval = Vmx86HasFreePages(vm, numPages, FALSE);
         if (retval) {
            numLockedPages += numPages;
@@ -841,7 +840,7 @@ Vmx86ReserveFreePages(VMDriver *vm,

 static void
 Vmx86UnreserveFreePages(VMDriver *vm,
-                        unsigned int numPages)
+                        PageCnt numPages)
 {
   ASSERT(vm);

@@ -879,7 +878,7 @@ static void
 Vmx86GetNX(void *clientData) // IN/OUT: A NXData *
 {
   NXData *nxData = (NXData *)clientData;
-   uint64 efer = __GET_MSR(MSR_EFER);
+   uint64 efer = X86MSR_GetMSR(MSR_EFER);

   Atomic_Inc32(&nxData->responded);
   if ((efer & MSR_EFER_NXE) == MSR_EFER_NXE) {
@@ -946,11 +945,6 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)
      return NULL;
   }

-   /* Disallow VM creation if HV is not available, as the VM cannot be run. */
-   if (!CPUID_HostSupportsHV()) {
-      return NULL;
-   }
-
   /* Disallow VM creation if NX is disabled on the host as VMM requires NX. */
   if (!hostUsesNX) {
      Log("NX/XD must be enabled.  Cannot create VM.\n");
@@ -970,25 +964,30 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)
      Atomic_Write32(&vm->currentHostCpu[v], INVALID_PCPU);
      vm->ptRootMpns[v] = INVALID_MPN;
   }
-   if (HostIF_Init(vm, numVCPUs)) {
-      goto cleanup;
-   }
-   bsBuf = HostIF_AllocKernelMem(bsBlobSize, FALSE);
-   if (bsBuf == NULL) {
-      goto cleanup;
-   }
-   if (HostIF_CopyFromUser(bsBuf, bsBlob, bsBlobSize) != 0) {
-      goto cleanup;
-   }
-   bsParams = BSVMM_Validate(bsBuf, bsBlobSize);
-   if (bsParams == NULL) {
-      Warning("Could not validate the VMM bootstrap blob");
+   if (!HostIF_Init(vm, numVCPUs)) {
      goto cleanup;
   }

-   if (!Task_CreateCrossGDT(&bsParams->gdtInit)) {
-      goto cleanup;
+   /* The ULM does not use the cross GDT. */
+   if (bsBlobSize != 0) {
+      bsBuf = HostIF_AllocKernelMem(bsBlobSize, FALSE);
+      if (bsBuf == NULL) {
+         goto cleanup;
+      }
+      if (HostIF_CopyFromUser(bsBuf, bsBlob, bsBlobSize) != 0) {
+         goto cleanup;
+      }
+      bsParams = BSVMM_Validate(bsBuf, bsBlobSize);
+      if (bsParams == NULL) {
+         Warning("Could not validate the VMM bootstrap blob");
+         goto cleanup;
+      }
+
+      if (!Task_CreateCrossGDT(&bsParams->gdtInit)) {
+         goto cleanup;
+      }
   }
+
   vm->ptpTracker = MemTrack_Init(vm);
   if (vm->ptpTracker == NULL) {
      goto cleanup;
@@ -997,6 +996,14 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)
   if (vm->vmmTracker == NULL) {
      goto cleanup;
   }
+   vm->sharedArea = SharedAreaVmmon_Init(vm);
+   if (vm->sharedArea == NULL) {
+      goto cleanup;
+   }
+   vm->statVars = StatVarsVmmon_Init(vm);
+   if (vm->statVars == NULL) {
+      goto cleanup;
+   }

   HostIF_GlobalLock(0);

@@ -1015,7 +1022,9 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)

   HostIF_GlobalUnlock(0);

-   HostIF_FreeKernelMem(bsBuf);
+   if (bsBuf != NULL) {
+      HostIF_FreeKernelMem(bsBuf);
+   }
   return vm;

 cleanup:
@@ -1052,7 +1061,8 @@ cleanup:
 */

 static Bool
-Vmx86SetPageTableRoots(VMDriver *vm, UserVA64 *ptRootVAs, uint16 numVCPUs)
+Vmx86SetPageTableRoots(VMDriver *vm, PerVcpuPages *perVcpuPages,
+                       uint16 numVCPUs)
 {
   uint16 vcpu;

@@ -1060,14 +1070,16 @@ Vmx86SetPageTableRoots(VMDriver *vm, UserVA64 *ptRootVAs, uint16 numVCPUs)
      return FALSE;
   }
   for (vcpu = 0; vcpu < numVCPUs; vcpu++) {
-      if ((ptRootVAs[vcpu] & (PAGE_SIZE - 1)) != 0) {
+      VA64 ptRoot = perVcpuPages[vcpu].ptRoot;
+
+      if ((ptRoot & (PAGE_SIZE - 1)) != 0) {
         Warning("Error: page table VA %"FMT64"x is not page-aligned\n",
-                 ptRootVAs[vcpu]);
+                 ptRoot);
         return FALSE;
      }
      ASSERT(vm->ptRootMpns[vcpu] == INVALID_MPN);
      HostIF_VMLock(vm, 38);
-      if (HostIF_LookupUserMPN(vm, ptRootVAs[vcpu], &vm->ptRootMpns[vcpu]) !=
+      if (HostIF_LookupUserMPN(vm, ptRoot, &vm->ptRootMpns[vcpu]) !=
          PAGE_LOOKUP_SUCCESS) {
         HostIF_VMUnlock(vm, 38);
         Warning("Failure looking up page table root MPN for VCPU %d\n", vcpu);
@@ -1128,7 +1140,7 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
                       uint32 numBytes,
                       uint32 headerOffset,
                       uint16 numVCPUs,
-                       UserVA64 *ptRootVAs,
+                       PerVcpuPages *perVcpuPages,
                       VMSharedRegion *shRegions)
 {
   VmmBlobInfo *bi = NULL;
@@ -1136,18 +1148,31 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
   Vcpuid errVcpu;
   MonLoaderError ret;
   MonLoaderArgs args;
+   MonLoaderHeader *header;

   if (!VmmBlob_Load(bsBlobAddr, numBytes, headerOffset, &bi)) {
      Warning("Error loading VMM bootstrap blob\n");
      goto error;
   }
   vm->blobInfo = bi;
-   if (!Vmx86SetPageTableRoots(vm, ptRootVAs, numVCPUs)) {
+   header = bi->header;
+   if (!Vmx86SetPageTableRoots(vm, perVcpuPages, numVCPUs)) {
      goto error;
   }
+
+   /*
+    * Initialize the driver's part of the cross-over page used to
+    * talk to the monitor.
+    */
+   if (!Task_InitCrosspage(vm, header->monStartLPN, header->monEndLPN,
+                           perVcpuPages)) {
+      Warning("Error initializing crosspage\n");
+      goto error;
+   }
+
   args.vm = vm;
   args.shRegions = shRegions;
-   ret = MonLoader_Process(bi->header, numVCPUs, &args, &errLine, &errVcpu);
+   ret = MonLoader_Process(header, numVCPUs, &args, &errLine, &errVcpu);
   if (ret != ML_OK) {
      Warning("Error processing bootstrap: error %d at line %u, vcpu %u\n",
               ret, errLine, errVcpu);
@@ -1268,78 +1293,6 @@ Vmx86_Close(void)
 }


-/*
- *-----------------------------------------------------------------------------
- *
- * Vmx86_InitVM --
- *
- *    Initializaiton of the VM.  Expects all initial arguments
- *    to be part of the InitBlock structure.
- *
- * Results:
- *    0 on success
- *    != 0 on failure
- *
- * Side effects:
- *    Many
- *
- *-----------------------------------------------------------------------------
- */
-
-int
-Vmx86_InitVM(VMDriver *vm,          // IN
-             InitBlock *initParams) // IN/OUT: Initial params from the VM
-{
-   int retval;
-
-   if (initParams->magicNumber != INIT_BLOCK_MAGIC) {
-      Warning("Bad magic number for init block 0x%x\n",
-              initParams->magicNumber);
-
-      return 1;
-   }
-   if (vm->numVCPUs == 0 || vm->numVCPUs > MAX_VCPUS) {
-      Warning("Invalid number of VCPUs %d\n", vm->numVCPUs);
-
-      return 1;
-   }
-
-   /*
-    * Initialize the driver's part of the cross-over page used to
-    * talk to the monitor
-    */
-
-   retval = Task_InitCrosspage(vm, initParams);
-   if (retval) {
-      Warning("Task crosspage init died with retval=%d\n", retval);
-      /*
-       *  Note that any clean-up of resources will be handled during
-       *  power-off when Vmx86_ReleaseVM() is called as part of
-       *  MonitorLoop_PowerOff().
-       */
-
-      return 1;
-   }
-
-   /*
-    *  Check if we want to arbitrarily fail every N VM initializations.
-    *  Useful in testing PR 72482.
-    */
-
-   if (initParams->vmInitFailurePeriod != 0) {
-      static uint32 counter = 0;
-
-      if ((++counter) % initParams->vmInitFailurePeriod == 0) {
-         Warning("VM initialization failed on %d iteration\n", counter);
-
-         return 1;
-      }
-   }
-
-   return 0;
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -1690,42 +1643,13 @@ Vmx86_GetNumVMs(void)
 }


-int32
-Vmx86_GetTotalMemUsage(void)
+static INLINE PageCnt
+Vmx86MinAllocationFunc(PageCnt nonpaged,
+                       PageCnt anonymous,
+                       PageCnt mainmem,
+                       Percent memPct)
 {
-   VMDriver *vm;
-   int totalmem = 0;
-
-   HostIF_GlobalLock(15);
-   vm = vmDriverList;
-
-   for (vm = vmDriverList; vm != NULL; vm = vm->nextDriver) {
-      /*
-       * The VM lock is not strictly necessary as the vm will
-       * stay on the list until we release the global lock and
-       * because of order in which "admitted" and "mainMemSize"
-       * are set when each VM is admitted.
-       */
-
-      if (vm->memInfo.admitted) {
-          totalmem += PAGES_2_MBYTES(ROUNDUP(vm->memInfo.mainMemSize,
-                                             MBYTES_2_PAGES(1)));
-      }
-   }
-
-   HostIF_GlobalUnlock(15);
-
-   return totalmem;
-}
-
-
-static INLINE unsigned
-Vmx86MinAllocationFunc(unsigned nonpaged,
-                       unsigned anonymous,
-                       unsigned mainmem,
-                       unsigned memPct)
-{
-   return RatioOf(memPct, mainmem, 100) + nonpaged + anonymous;
+   return (memPct * mainmem) / 100 + nonpaged + anonymous;
 }


@@ -1749,12 +1673,11 @@ Vmx86MinAllocationFunc(unsigned nonpaged,
 *----------------------------------------------------------------------
 */

-static INLINE unsigned
+static INLINE PageCnt
 Vmx86MinAllocation(VMDriver *vm,
-                   unsigned memPct)
+                   Percent memPct)
 {
   ASSERT(HostIF_VMLockIsHeld(vm));
-
   return Vmx86MinAllocationFunc(vm->memInfo.nonpaged, vm->memInfo.anonymous,
                                 vm->memInfo.mainMemSize, memPct);
 }
@@ -1780,12 +1703,11 @@ Vmx86MinAllocation(VMDriver *vm,
 *----------------------------------------------------------------------
 */

-static unsigned
-Vmx86CalculateGlobalMinAllocation(unsigned memPct)
+static PageCnt
+Vmx86CalculateGlobalMinAllocation(Percent memPct)
 {
   VMDriver *vm;
-   unsigned minAllocation = 0;
-
+   PageCnt minAllocation = 0;
   ASSERT(HostIF_GlobalLockIsHeld());
   /* Pages of other vms required to fit inside the hard limit. */
   for (vm = vmDriverList; vm; vm = vm->nextDriver) {
@@ -1819,10 +1741,9 @@ Vmx86CalculateGlobalMinAllocation(unsigned memPct)
 */

 static INLINE_SINGLE_CALLER void
-Vmx86UpdateMinAllocations(unsigned memPct)  // IN:
+Vmx86UpdateMinAllocations(Percent memPct)  // IN:
 {
   VMDriver *vm;
-
   ASSERT(HostIF_GlobalLockIsHeld());
   /* Pages of other vms required to fit inside the hard limit. */
   for (vm = vmDriverList; vm; vm = vm->nextDriver) {
@@ -1855,7 +1776,7 @@ Vmx86UpdateMinAllocations(unsigned memPct)  // IN:
 */

 Bool
-Vmx86_SetConfiguredLockedPagesLimit(unsigned limit)  // IN:
+Vmx86_SetConfiguredLockedPagesLimit(PageCnt limit)  // IN:
 {
   Bool retval = FALSE;

@@ -1889,7 +1810,7 @@ Vmx86_SetConfiguredLockedPagesLimit(unsigned limit)  // IN:
 */

 void
-Vmx86_SetDynamicLockedPagesLimit(unsigned limit)  // IN:
+Vmx86_SetDynamicLockedPagesLimit(PageCnt limit)  // IN:
 {
   HostIF_GlobalLock(11);
   lockedPageLimit.dynamic = limit;
@@ -2031,33 +1952,29 @@ Vmx86_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver
 *-----------------------------------------------------------------------------
 */

-int
+int64
 Vmx86_AllocLockedPages(VMDriver *vm,         // IN: VMDriver
                       VA64 addr,            // OUT: VA of an array for
                                             //      allocated MPNs.
-                       unsigned numPages,    // IN: number of pages to allocate
+                       PageCnt numPages,     // IN: number of pages to allocate
                       Bool kernelMPNBuffer, // IN: is the MPN buffer in kernel
                                             //     or user address space?
                       Bool ignoreLimits)    // IN: should limits be ignored?
 {
-   int allocatedPages;
-
+   int64 allocatedPages;
   if (!Vmx86ReserveFreePages(vm, numPages, ignoreLimits)) {
      // XXX What kind of system-specific error code is that? --hpreg
      return PAGE_LOCK_LIMIT_EXCEEDED;
   }
-
   HostIF_VMLock(vm, 7);
   allocatedPages = HostIF_AllocLockedPages(vm, addr, numPages,
                                            kernelMPNBuffer);
   HostIF_VMUnlock(vm, 7);
-
   if (allocatedPages < 0) {
      Vmx86UnreserveFreePages(vm, numPages);
   } else if (allocatedPages < numPages) {
      Vmx86UnreserveFreePages(vm, numPages - allocatedPages);
   }
-
   return allocatedPages;
 }

@@ -2082,14 +1999,13 @@ Vmx86_AllocLockedPages(VMDriver *vm,         // IN: VMDriver

 int
 Vmx86_FreeLockedPages(VMDriver *vm,         // IN: VM instance pointer
-                      VA64 addr,            // IN: user or kernel array of MPNs to free
-                      unsigned numPages,    // IN: number of pages to free
-                      Bool kernelMPNBuffer) // IN: is the MPN buffer in kernel or user address space?
+                      MPN *mpns,            // IN: MPNs to free
+                      PageCnt numPages)     // IN: number of pages to free
 {
   int ret;

   HostIF_VMLock(vm, 8);
-   ret = HostIF_FreeLockedPages(vm, addr, numPages, kernelMPNBuffer);
+   ret = HostIF_FreeLockedPages(vm, mpns, numPages);
   HostIF_VMUnlock(vm, 8);

   if (ret == 0) {
@@ -2170,6 +2086,33 @@ Vmx86_GetNextAnonPage(VMDriver *vm,       // IN: VM instance pointer
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_GetNumAnonPages --
+ *
+ *      Queries the driver for the total number of anonymous pages.
+ *
+ * Results:
+ *      Total number of anonymous pages
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+PageCnt
+Vmx86_GetNumAnonPages(VMDriver *vm)       // IN: VM instance pointer
+{
+   PageCnt ret;
+   HostIF_VMLock(vm, 45);
+   ret = HostIF_GetNumAnonPages(vm);
+   HostIF_VMUnlock(vm, 45);
+   return ret;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -2275,10 +2218,10 @@ Vmx86_GetMemInfo(VMDriver *curVM,

 static void
 Vmx86SetMemoryUsage(VMDriver *curVM,       // IN/OUT
-                    unsigned paged,        // IN
-                    unsigned nonpaged,     // IN
-                    unsigned anonymous,    // IN
-                    unsigned aminVmMemPct) // IN
+                    PageCnt  paged,        // IN
+                    PageCnt  nonpaged,     // IN
+                    PageCnt  anonymous,    // IN
+                    Percent  aminVmMemPct) // IN
 {
   ASSERT(HostIF_VMLockIsHeld(curVM));
   curVM->memInfo.paged         = paged;
@@ -2317,8 +2260,7 @@ Vmx86_Admit(VMDriver *curVM,     // IN
            VMMemInfoArgs *args) // IN/OUT
 {
   Bool allowAdmissionCheck = FALSE;
-   unsigned int globalMinAllocation;
-
+   PageCnt globalMinAllocation;
   HostIF_GlobalLock(9);

   /*
@@ -2360,8 +2302,8 @@ Vmx86_Admit(VMDriver *curVM,     // IN

 #if defined _WIN32
   if (curVM->memInfo.admitted) {
-      unsigned int allocatedPages, nonpaged;
-      signed int pages;
+      PageCnt allocatedPages, nonpaged;
+      int64 pages;
      MPN *mpns;

      /*
@@ -2397,10 +2339,7 @@ Vmx86_Admit(VMDriver *curVM,     // IN
       * XXX Do not free the pages but hand them directly to the admitted VM.
       */

-      for (pages = 0; pages < allocatedPages; pages += ALLOCATE_CHUNK_SIZE) {
-         Vmx86_FreeLockedPages(curVM, PtrToVA64(mpns + pages),
-                               MIN(ALLOCATE_CHUNK_SIZE, allocatedPages - pages), TRUE);
-      }
+      Vmx86_FreeLockedPages(curVM, mpns, allocatedPages);
      HostIF_FreeKernelMem(mpns);
 #undef ALLOCATE_CHUNK_SIZE

@@ -2429,11 +2368,11 @@ undoAdmission:
 Bool
 Vmx86_Readmit(VMDriver *curVM, OvhdMem_Deltas *delta)
 {
-   unsigned globalMinAllocation, newMinAllocation;
+   PageCnt globalMinAllocation, newMinAllocation;
   Bool retval = FALSE;
-   int paged;
-   int nonpaged;
-   int anonymous;
+   int64 paged;
+   int64 nonpaged;
+   int64 anonymous;

   HostIF_GlobalLock(31);
   globalMinAllocation = Vmx86CalculateGlobalMinAllocation(minVmMemPct);
@@ -2542,19 +2481,20 @@ static void
 Vmx86EnableHVOnCPU(void)
 {
   if (CPUID_HostSupportsSVM()) {
-      uint64 vmCR = __GET_MSR(MSR_VM_CR);
+      uint64 vmCR = X86MSR_GetMSR(MSR_VM_CR);
      if (!SVM_LockedFromFeatures(vmCR)) {
         CPUIDRegs regs;
         __GET_CPUID(0x8000000A, &regs);
         if (CPUID_GET(0x8000000A, EDX, SVM_LOCK, regs.edx) != 0) {
-            __SET_MSR(MSR_VM_CR, (vmCR & ~MSR_VM_CR_SVME_DISABLE) |
-                                  MSR_VM_CR_SVM_LOCK);
+            X86MSR_SetMSR(MSR_VM_CR, (vmCR & ~MSR_VM_CR_SVME_DISABLE) |
+                                      MSR_VM_CR_SVM_LOCK);
         }
      }
   } else if (CPUID_HostSupportsVT()) {
-      uint64 featCtl = __GET_MSR(MSR_FEATCTL);
+      uint64 featCtl = X86MSR_GetMSR(MSR_FEATCTL);
      if (!VT_LockedFromFeatures(featCtl)) {
-         __SET_MSR(MSR_FEATCTL, featCtl | MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE);
+         X86MSR_SetMSR(MSR_FEATCTL,
+                       featCtl | MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE);
      }
   }
 }
@@ -2925,9 +2865,9 @@ Vmx86CheckVMXStatus(const char *operation, // IN: Operation string
 #ifdef __GNUC__
 #if __GNUC__== 4 && __GNUC_MINOR__ > 3
      if (status == VMX_FailValid) {
-         uint64 errorCode;
+         size_t errorCode;
         VMREAD_2_STATUS(VT_VMCS_VMINSTR_ERR, &errorCode);
-         Log("VM-instruction error: Error %"FMT64"d\n", errorCode);
+         Log("VM-instruction error: Error %"FMTSZ"d\n", errorCode);
      }
 #endif
 #endif
@@ -3169,12 +3109,12 @@ Vmx86PerfCtrInUse(Bool isGen, unsigned pmcNum, unsigned ctrlMSR,
                  unsigned cntMSR, Bool hasPGC)
 {
   volatile unsigned delay;
-   uint64 origPGC = hasPGC ? __GET_MSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR) : 0;
+   uint64 origPGC = hasPGC ? X86MSR_GetMSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR) : 0;
   uint64 pmcCtrl;
   uint64 pmcCount, count;
   uint64 ctrlEna, pgcEna;

-   pmcCtrl = __GET_MSR(ctrlMSR);
+   pmcCtrl = X86MSR_GetMSR(ctrlMSR);
   if (isGen) {
      ASSERT(pmcNum < 32);
      if ((pmcCtrl & PERFCTR_CPU_ENABLE) != 0) {
@@ -3191,21 +3131,21 @@ Vmx86PerfCtrInUse(Bool isGen, unsigned pmcNum, unsigned ctrlMSR,
      ctrlEna = pmcCtrl | PERFCTR_CORE_FIXED_KERNEL_MASKn(pmcNum);
      pgcEna = CONST64U(1) << (pmcNum + 32);
   }
-   pmcCount = __GET_MSR(cntMSR);
+   pmcCount = X86MSR_GetMSR(cntMSR);
   /* Enable the counter. */
-   __SET_MSR(ctrlMSR, ctrlEna);
+   X86MSR_SetMSR(ctrlMSR, ctrlEna);
   if (hasPGC) {
-      __SET_MSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, pgcEna | origPGC);
+      X86MSR_SetMSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, pgcEna | origPGC);
   }
   /* Retire some instructions and wait a few cycles. */
   for (delay = 0; delay < 100; delay++) ;
   /* Disable the counter. */
   if (hasPGC) {
-      __SET_MSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, origPGC);
+      X86MSR_SetMSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, origPGC);
   }
-   count = __GET_MSR(cntMSR);
-   __SET_MSR(ctrlMSR, pmcCtrl);
-   __SET_MSR(cntMSR, pmcCount);
+   count = X86MSR_GetMSR(cntMSR);
+   X86MSR_SetMSR(ctrlMSR, pmcCtrl);
+   X86MSR_SetMSR(cntMSR, pmcCount);
   return count == pmcCount;
 }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,13 +32,14 @@
 #include "x86msr.h"
 #include "modulecall.h"
 #include "vcpuid.h"
-#include "initblock.h"
 #include "iocontrols.h"
 #include "numa_defs.h"
 #include "rateconv.h"
 #include "vmmem_shared.h"
 #include "apic.h"
 #include "bootstrap_vmm.h"
+#include "sharedAreaVmmon.h"
+#include "statVarsVmmon.h"

 typedef struct TSCDelta {
   Atomic_uint64 min;
@@ -55,23 +56,19 @@ struct VmmBlobInfo;
 typedef struct VMDriver {
   /* Unique (in the driver), strictly positive, VM ID used by userland. */
   int                     userID;
-
   Vcpuid                  numVCPUs;         /* Number of vcpus in VM. */
-
   struct VMDriver        *nextDriver;       /* Next on list of all VMDrivers */
-
   struct VMHost          *vmhost;           /* Host-specific fields. */
-
   MPN                    *ptRootMpns;       /* numVCPUs-sized array. */
   struct VmmBlobInfo     *blobInfo;         /* VMM bootstrap blob info. */
-
+   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
+   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
   APICDescriptor          hostAPIC;
-
   struct MemTrack        *memtracker;       /* Memory tracker pointer */
   Bool                    checkFuncFailed;
   struct PerfCounter     *perfCounter;
@@ -81,6 +78,7 @@ typedef struct VMDriver {
   Atomic_uint64           ptscLatest;
   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
+   PageCnt                 numPTPPages;      /* Num PTP pages allocated. */
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -110,7 +108,7 @@ typedef struct PseudoTSC {

 extern PseudoTSC pseudoTSC;

-#define MAX_LOCKED_PAGES (-1)
+#define MAX_LOCKED_PAGES MAX_PPN

 extern void Vmx86_CacheNXState(void);
 extern VMDriver *Vmx86_CreateVM(VA64 bsBlob,
@@ -121,11 +119,10 @@ extern Bool Vmx86_ProcessBootstrap(VMDriver *vm,
                                   uint32 numBytes,
                                   uint32 headerOffset,
                                   uint16 numVCPUs,
-                                   UserVA64 *ptRootVAs,
+                                   PerVcpuPages *perVcpuPages,
                                   VMSharedRegion *shRegions);
 extern int Vmx86_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 extern int Vmx86_ReleaseVM(VMDriver *vm);
-extern int Vmx86_InitVM(VMDriver *vm, InitBlock *initParams);
 extern int Vmx86_LateInitVM(VMDriver *vm);
 extern int Vmx86_RunVM(VMDriver *vm, Vcpuid vcpuid);
 extern void Vmx86_YieldToSet(VMDriver *vm, Vcpuid currVcpu, const VCPUSet *req,
@@ -142,22 +139,21 @@ extern int Vmx86_UnlockPage(VMDriver *vm, VA64 uAddr);
 extern int Vmx86_UnlockPageByMPN(VMDriver *vm, MPN mpn, VA64 uAddr);
 extern MPN Vmx86_GetRecycledPage(VMDriver *vm);
 extern int Vmx86_ReleaseAnonPage(VMDriver *vm, MPN mpn);
-extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned numPages, Bool kernelMPNBuffer,
-                                  Bool ignoreLimits);
-extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                 unsigned numPages, Bool kernelMPNBuffer);
+extern int64 Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                    PageCnt numPages, Bool kernelMPNBuffer,
+                                    Bool ignoreLimits);
+extern int Vmx86_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn);
+extern MPN Vmx86_GetNumAnonPages(VMDriver *vm);
 extern MPN Vmx86_AllocLowPage(VMDriver *vm, Bool ignoreLimits);
 extern void *Vmx86_Calloc(size_t numElements,
                          size_t elementSize,
-                          int nonPageable);
+                          Bool nonPageable);
 extern void  Vmx86_Free(void *ptr);

 extern int32 Vmx86_GetNumVMs(void);
-extern int32 Vmx86_GetTotalMemUsage(void);
-extern Bool Vmx86_SetConfiguredLockedPagesLimit(unsigned limit);
-extern void Vmx86_SetDynamicLockedPagesLimit(unsigned limit);
+extern Bool Vmx86_SetConfiguredLockedPagesLimit(PageCnt limit);
+extern void Vmx86_SetDynamicLockedPagesLimit(PageCnt limit);
 extern Bool Vmx86_GetMemInfo(VMDriver *curVM,
                             Bool curVMOnly,
                             VMMemInfoArgs *args,
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -345,6 +345,34 @@ CircList_Push(ListItem *p,               // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_InsertAfter --
+ *
+ *      Adds a new member to the list after the provided item.  Assumes p
+ *      is not a member of a list already.
+ *
+ * Result:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE void
+CircList_InsertAfter(ListItem *p,      // IN:
+                     ListItem *after)  // IN:
+{
+   p->prev = after;
+   p->next = after->next;
+   p->next->prev = p;
+   after->next = p;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,77 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _CPUID_INFO_H
-#define _CPUID_INFO_H
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-
-#include "includeCheck.h"
-
-#include "vm_basic_asm.h"
-#include "x86cpuid_asm.h"
-
-#if defined __cplusplus
-extern "C" {
-#endif
-
-
-typedef struct CPUID0 {
-   int numEntries;
-   char name[16];      // 4 extra bytes to null terminate
-} CPUID0;
-
-typedef struct CPUID1 {
-   uint32 version;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID1;
-
-typedef struct CPUID80 {
-   uint32 numEntries;
-   uint32 ebx;
-   uint32 ecx;
-   uint32 edx;
-} CPUID80;
-
-typedef struct CPUID81 {
-   uint32 eax;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID81;
-
-typedef struct CPUIDSummary {
-   CPUID0  id0;
-   CPUID1  id1;
-   CPUIDRegs ida;
-   CPUID80 id80;
-   CPUID81 id81;
-   CPUIDRegs id88, id8a;
-} CPUIDSummary;
-
-
-#if defined __cplusplus
-} // extern "C"
-#endif
-
-#endif // _CPUID_INFO_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,7 +37,6 @@
 #error iocontrols.h is for hosted vmmon, do not use on visor
 #endif

-#include "basic_initblock.h"
 #include "x86segdescrs.h"
 #include "rateconv.h"
 #include "overheadmem_types.h"
@@ -45,6 +44,8 @@
 #include "numa_defs.h"
 #include "bootstrap_vmm.h"
 #include "contextinfo.h"
+#include "vcpuid.h"
+#include "sharedAreaType.h"

 #if defined __cplusplus
 extern "C" {
@@ -144,7 +145,7 @@ PtrToVA64(void const *ptr) // IN
 * See bora/doc/vmcore details.
 */

-#define VMMON_VERSION           (360 << 16 | 0)
+#define VMMON_VERSION           (385 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -153,11 +154,8 @@ PtrToVA64(void const *ptr) // IN
 * ENOMEM returned after MAX_VMS virtual machines created
 */

-#ifdef VMX86_SERVER
-#define MAX_VMS 128
-#else
 #define MAX_VMS 64
-#endif
+
 /*
 * MsgWaitForMultipleObjects doesn't scale well enough on Win32.
 * Allocate with MAX_VMS so static buffers are large, but do
@@ -209,11 +207,13 @@ enum IOCTLCmd {
   IOCTLCMD(VERSION) = IOCTLCMD(FIRST),
   IOCTLCMD(CREATE_VM),
   IOCTLCMD(PROCESS_BOOTSTRAP),
+   IOCTLCMD(REGISTER_SHARED),
+   IOCTLCMD(REGISTER_STATVARS),
   IOCTLCMD(RELEASE_VM),
   IOCTLCMD(GET_NUM_VMS),
-   IOCTLCMD(INIT_VM),
   IOCTLCMD(RUN_VM),
   IOCTLCMD(LOOK_UP_MPN),
+   IOCTLCMD(GET_VMM_PAGE_ROOT),
   IOCTLCMD(LOCK_PAGE),
   IOCTLCMD(UNLOCK_PAGE),
   IOCTLCMD(APIC_INIT),
@@ -222,7 +222,6 @@ enum IOCTLCmd {
   IOCTLCMD(ADMIT),
   IOCTLCMD(UPDATE_MEM_INFO),
   IOCTLCMD(READMIT),
-   IOCTLCMD(GET_TOTAL_MEM_USAGE),
   IOCTLCMD(GET_KHZ_ESTIMATE),
   IOCTLCMD(SET_HOST_CLOCK_RATE),
   IOCTLCMD(READ_PAGE),
@@ -232,6 +231,7 @@ enum IOCTLCmd {
    /* AWE calls */
   IOCTLCMD(ALLOC_LOCKED_PAGES),
   IOCTLCMD(GET_NEXT_ANON_PAGE),
+   IOCTLCMD(GET_NUM_ANON_PAGES),

   IOCTLCMD(GET_ALL_MSRS),

@@ -276,7 +276,7 @@ enum IOCTLCmd {
 #endif

 #if defined __APPLE__
-   IOCTLCMD(ALLOC_LOW_PAGES),
+   IOCTLCMD(GET_NUM_RESPONDING_CPUS),
   IOCTLCMD(INIT_DRIVER),
   IOCTLCMD(BLUEPILL),
 #endif
@@ -317,13 +317,15 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_CREATE_VM           VMIOCTL_BUFFERED(CREATE_VM)
 #define IOCTL_VMX86_RELEASE_VM          VMIOCTL_BUFFERED(RELEASE_VM)
 #define IOCTL_VMX86_PROCESS_BOOTSTRAP   VMIOCTL_BUFFERED(PROCESS_BOOTSTRAP)
+#define IOCTL_VMX86_REGISTER_SHARED     VMIOCTL_BUFFERED(REGISTER_SHARED)
+#define IOCTL_VMX86_REGISTER_STATVARS   VMIOCTL_BUFFERED(REGISTER_STATVARS)
 #define IOCTL_VMX86_GET_NUM_VMS         VMIOCTL_BUFFERED(GET_NUM_VMS)
-#define IOCTL_VMX86_INIT_VM             VMIOCTL_BUFFERED(INIT_VM)
 #define IOCTL_VMX86_RUN_VM              VMIOCTL_NEITHER(RUN_VM)
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
+#define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
 #define IOCTL_VMX86_UNLOCK_PAGE         VMIOCTL_BUFFERED(UNLOCK_PAGE)
 #define IOCTL_VMX86_APIC_INIT           VMIOCTL_BUFFERED(APIC_INIT)
@@ -347,6 +349,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UNLOCK_PAGE_BY_MPN  VMIOCTL_BUFFERED(UNLOCK_PAGE_BY_MPN)
 #define IOCTL_VMX86_ALLOC_LOCKED_PAGES  VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES)
 #define IOCTL_VMX86_GET_NEXT_ANON_PAGE  VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE)
+#define IOCTL_VMX86_GET_NUM_ANON_PAGES  VMIOCTL_BUFFERED(GET_NUM_ANON_PAGES)

 #define IOCTL_VMX86_READ_DISASM_PROC_BINARY \
                                      VMIOCTL_BUFFERED(READ_DISASM_PROC_BINARY)
@@ -369,6 +372,8 @@ enum IOCTLCmd {
 #endif


+#define INIT_BLOCK_MAGIC     (0x1789 + 14)
+
 /*
 * Flags sent into APICBASE ioctl
 */
@@ -395,6 +400,14 @@ union {
 #include "vmware_pack_end.h"
 VMLockPage;

+typedef
+#include "vmware_pack_begin.h"
+union {
+   Vcpuid vcpuid; // IN: VCPU
+   MPN pageRoot;  // OUT: MPN of the VCPU's page root
+}
+#include "vmware_pack_end.h"
+VcpuPageRoot;

 typedef struct VMAPICInfo {
   uint32 flags;
@@ -413,9 +426,9 @@ typedef struct VMAPICInfo {
 * all these values at any given time.
 */
 typedef struct LockedPageLimit {
-   uint32 host;        // driver calculated maximum for this host
-   uint32 configured;  // user defined maximum pages to lock
-   uint32 dynamic;     // authd hardLimitMonitor pages to lock
+   PageCnt host;        // driver calculated maximum for this host
+   PageCnt configured;  // user defined maximum pages to lock
+   PageCnt dynamic;     // authd hardLimitMonitor pages to lock
 } LockedPageLimit;

 /*
@@ -436,15 +449,15 @@ typedef struct LockedPageLimit {
 */

 typedef struct VMMemMgmtInfo {
-   uint32          minAllocation;   // minimum pages for vm
-   uint32          maxAllocation;   // maximum pages the vm could lock
+   PageCnt         minAllocation;   // minimum pages for vm
+   PageCnt         maxAllocation;   // maximum pages the vm could lock
+   PageCnt         nonpaged;        // overhead memory (guest, mmap)
+   PageCnt         paged;           // vmx memory (malloc, statics)
+   PageCnt         anonymous;       // vmm memory
+   PageCnt         mainMemSize;     // guest main memory size
+   PageCnt         locked;          // number of pages locked by this vm
+   PageCnt         perVMOverhead;   // memory for vmx/vmmon overheads
   uint32          shares;          // proportional sharing weight
-   uint32          nonpaged;        // overhead memory (guest, mmap)
-   uint32          paged;           // vmx memory (malloc, statics)
-   uint32          anonymous;       // vmm memory
-   uint32          mainMemSize;     // guest main memory size
-   uint32          locked;          // number of pages locked by this vm
-   uint32          perVMOverhead;   // memory for vmx/vmmon overheads
   Percent         touchedPct;      // % of guest memory being touched
   Percent         dirtiedPct;      // % of guest memory being dirtied
   Bool            admitted;        // admission control
@@ -467,14 +480,14 @@ typedef struct VMMemMgmtInfoPatch {

 typedef struct VMMemInfoArgs {
   uint64          currentTime;        // Host time in secs of the call.
-   uint32          minVmMemPct;        // % of vm that must fit in memory
-   uint32          globalMinAllocation;// pages that must fit in maxLockedPages
-   uint32          numLockedPages;     // total locked pages by all vms
+   PageCnt         globalMinAllocation;// pages that must fit in maxLockedPages
+   PageCnt         numLockedPages;     // total locked pages by all vms
   LockedPageLimit lockedPageLimit;    // set of locked page limits
-   uint32          maxLockedPages;     // effective limit on locked pages
+   PageCnt         maxLockedPages;     // effective limit on locked pages
   uint32          callerIndex;        // this vm's index memInfo array
   uint32          numVMs;             // number of running VMs
-   uint8           _pad[4];
+   Percent         minVmMemPct;        // % of vm that must fit in memory
+   uint8           _pad[7];
   VMMemMgmtInfo   memInfo[1];
 } VMMemInfoArgs;

@@ -487,9 +500,9 @@ typedef struct VMMPNNext {
 } VMMPNNext;

 typedef struct VMMPNList {
-   uint32    mpnCount;   // IN (and OUT on Mac OS)
+   PageCnt   mpnCount;   // IN (and OUT on Mac OS)
   Bool      ignoreLimits;
-   uint8     _pad[3];
+   uint8     _pad[7];
   VA64      mpnList;    // IN: User VA of an array of 64-bit MPNs.
 } VMMPNList;

@@ -525,8 +538,6 @@ typedef struct PTSCCheckParams {
   uint8  _pad[7];
 } PTSCCheckParams;

-#ifndef VMX86_SERVER
-
 typedef struct IPIVectors {
   /*
    * Vectors we have allocated or stolen for the monitor interrupts.
@@ -535,8 +546,6 @@ typedef struct IPIVectors {
   uint8 hvIPIVector;
 } IPIVectors;

-#endif
-
 /*
 * Arguments and return value for VM creation.
 */
@@ -551,11 +560,33 @@ typedef struct VMCreateBlock {
 * Information about a shared region.
 */
 typedef struct VMSharedRegion {
-   uint32 index;
-   VPN    baseVpn;
-   uint32 numPages;
+   SharedAreaType index;
+   VPN            baseVpn;
+   uint32         numPages;
 } VMSharedRegion;

+/*
+ * Arguments for VMM shared area registration.
+ */
+typedef struct VMSharedAreaRegistrationBlock {
+   Vcpuid vcpu;            // IN: VCPU being registered.
+   VMSharedRegion region;  // IN: Shared region being registered.
+} VMSharedAreaRegistrationBlock;
+
+/*
+ * Information about a VM's statvars.
+ */
+typedef struct VMStatVarsRegistrationBlock {
+   VPN     baseVpn;
+   PageCnt numPages;
+   Vcpuid  vcpu;
+} VMStatVarsRegistrationBlock;
+
+typedef struct {
+   VA64   crosspage; // IN: User VA of VCPU crosspage.
+   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
+} PerVcpuPages;
+
 /*
 * Arguments for VMM bootstrap processing.
 */
@@ -564,10 +595,17 @@ typedef struct VMProcessBootstrapBlock {
   uint32         numBytes;      // IN: Size of VMM bootstrap blob.
   uint32         headerOffset;  // IN: Offset of header in blob.
   uint16         numVCPUs;      // IN: Number of VCPUs.
-   VA64           ptRootVAs[MAX_VCPUS];  // IN: User VA of PT roots.
   VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions.
+   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

+static INLINE size_t
+GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
+{
+   VMProcessBootstrapBlock *args = NULL;
+   return sizeof *args + numVCPUs * sizeof args->perVcpuPages[0];
+}
+
 /*
 * Arguments for VMM context retrieval.
 */
@@ -583,29 +621,20 @@ typedef union {
 * macros to marshall real arguments to mmap's made-up 'offset' argument.
 */

-#define VMMON_MAP_MT_LOW4GB     0
-#define VMMON_MAP_MT_LOW16MB    1
-#define VMMON_MAP_MT_ANY        2
-
 #define VMMON_MAP_OFFSET_SHIFT  0
 #define VMMON_MAP_OFFSET_MASK   0x00000FFF
 #define VMMON_MAP_ORDER_SHIFT   12
 #define VMMON_MAP_ORDER_MASK    0xF
-#define VMMON_MAP_MT_SHIFT      16
-#define VMMON_MAP_MT_MASK       0x7
-#define VMMON_MAP_RSVD_SHIFT    19
+#define VMMON_MAP_RSVD_SHIFT    16

 #define VMMON_MAP_RSVD(base)    \
                ((base) >> VMMON_MAP_RSVD_SHIFT)
-#define VMMON_MAP_MT(base)      \
-                (((base) >> VMMON_MAP_MT_SHIFT) & VMMON_MAP_MT_MASK)
 #define VMMON_MAP_ORDER(base)   \
                (((base) >> VMMON_MAP_ORDER_SHIFT) & VMMON_MAP_ORDER_MASK)
 #define VMMON_MAP_OFFSET(base)  \
                (((base) >> VMMON_MAP_OFFSET_SHIFT) & VMMON_MAP_OFFSET_MASK)

-#define VMMON_MAP_BASE(mt, order)   (((mt) << VMMON_MAP_MT_SHIFT) | \
-                                     ((order) << VMMON_MAP_ORDER_SHIFT))
+#define VMMON_MAP_BASE(order)   ((order) << VMMON_MAP_ORDER_SHIFT)

 #elif defined _WIN32
 /*
@@ -616,7 +645,6 @@ typedef struct VMAllocContiguousMem {
   VA64   mpnList;  // IN: User VA of an array of 64-bit MPNs.
   uint32 mpnCount; // IN
   uint32 order;    // IN
-   MPN    maxMPN;   // IN
 } VMAllocContiguousMem;
 #elif defined __APPLE__
 #   include "iocontrolsMacos.h"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,8 +42,8 @@
 #include "mon_assert.h"

 #define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
-#define CP_STUB_SIZE 16     /* A relative jmp instruction (5 bytes)
-                               padded to the next 16 byte boundary. */
+#define CP_STUB_SIZE     16     /* A relative jmp instruction (5 bytes)
+                                   padded to the next 16 byte boundary. */

 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
@@ -66,7 +66,10 @@
   MC(GET_MON_IPI_VECTOR)                                                     \
   MC(GET_HV_IPI_VECTOR)                                                      \
   MC(GET_HOST_TIMER_VECTORS)                                                 \
-   MC(BOOTSTRAP_CLEANUP)
+   MC(BOOTSTRAP_CLEANUP)                                                      \
+   MC(GET_SHARED_AREA)                                                        \
+   MC(GET_STAT_VARS)                                                          \
+   MC(GET_NUM_PTP_PAGES)

 /*
 *----------------------------------------------------------------------
@@ -292,7 +295,8 @@ struct VMCrossPageData {
   uint64   crosspageMA;

   uint64   hostDR[8];
-   LA64     hostCrossPageLA;   // where host has crosspage mapped
+   LA64     crossPageLA;       // where host/PTP map the cross page
+   LA64     crossGDTLA;        // where host/PTP map the cross GDT
   uint16   hostInitial64CS;
   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
@@ -310,8 +314,6 @@ struct VMCrossPageData {
   Task64   monTask;          /* vmm's task */

   VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
-   LA64              vmmCrossPageLA;
-   LA64              vmmCrossGDTLA;   // where crossGDT is mapped by PT patch

   /*
    * The monitor may requests up to two actions when returning to the
@@ -391,6 +393,7 @@ struct VMCrossPageData {
   uint8         _pad7[4];
   uint64        wsUD2;                       // IP of ud2 instr or 0 if unset.
   uint64        specCtrl; /* host MSR_SPEC_CTRL value before world switch. */
+   uint8         _pad8[8];
 }
 #include "vmware_pack_end.h"
 VMCrossPageData;
@@ -447,17 +450,14 @@ struct VMCrossPage {
 #include "vmware_pack_end.h"
 VMCrossPage;

-#define CROSSPAGE_VERSION_BASE 0xc09 /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc0c /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE  ((VMCrossPage * const) VPN_2_VA(CROSS_PAGE_START))
+#define CROSS_PAGE             ((VMCrossPage * const)VPN_2_VA(CROSS_PAGE_START))
 #define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
 #endif

-#define NULLPAGE_LINEAR_START  (MONITOR_LINEAR_START + \
-                                PAGE_SIZE * CPL0_GUARD_PAGE_START)
-
 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
 #define MX_WAITNORMAL          1  // Must equal one; see linux module code.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,6 +25,8 @@
 #ifndef _MON_ADDR_LAYOUT_H
 #define _MON_ADDR_LAYOUT_H

+#include "vm_pagetable.h"
+
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMKERNEL
@@ -37,6 +39,8 @@ typedef struct VMM64_AddrLayout {
    *All address are VPNs and all lengths are numPages
    */
   uint64 monBase;            // MONITOR_BASE_VPN
+   uint64 monL5Start;         // MON_PAGE_TABLE_L5_START
+   uint32 monL5Len;           // MON_PAGE_TABLE_L5_LEN
   uint64 monL4Start;         // MON_PAGE_TABLE_L4_START
   uint32 monL4Len;           // MON_PAGE_TABLE_L4_LEN
   uint64 monL3Start;         // MON_PAGE_TABLE_L3_START
@@ -51,7 +55,21 @@ typedef struct VMM64_AddrLayout {
 } VMM64_AddrLayout;
 #pragma pack(pop)

-#define VMM_MONAS_FIRST_L4OFF   1
-#define VMM_MONAS_LAST_L4OFF  127
+#define VMM_MONAS_4LP_FIRST_L4OFF   1
+#define VMM_MONAS_4LP_LAST_L4OFF  130

+#define VMM_MONAS_5LP_FIRST_L5OFF   1
+#define VMM_MONAS_5LP_LAST_L5OFF   34
+
+static INLINE PT_Level
+MonAS_GetPagingLevel(void)
+{
+   return PT_LEVEL_4;
+}
+
+static INLINE Bool
+MonAS_Uses5LevelPaging(void)
+{
+   return MonAS_GetPagingLevel() == PT_LEVEL_5;
+}
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,8 +31,8 @@
 * contains code and data statically built, empty but allocated space, shared
 * and run-time initialized content.  The monitor loader header regularizes
 * encoding of address space information, allowing a common representation and
- * common code to be re-used for different contexts (vmmon for hosted, vmkernel
- * for ESX, and the VMX while deprivileging of bootstrap is ongoing).
+ * common code to be re-used for different contexts (vmmon for hosted
+ * and vmkernel for ESX).
 *
 * The Header
 * ==========
@@ -87,7 +87,6 @@
 * MonLoaderCallout_Init(): Initialize MonLoader callouts.
 * MonLoaderCallout_GetSharedHostPage(subIdx, page): Get shared host page MPN.
 * MonLoaderCallout_GetSharedUserPage(subIdx, page): Get shared user page MPN.
- * MonLoaderCallout_SetEntrypoint(): Sets the entry code/stack for execution.
 *
 * Building vs Importing
 * =====================
@@ -119,21 +118,54 @@
 #define _MON_LOADER

 #include "vm_basic_types.h"
-#include "x86paging_common.h"
-#include "x86types.h" /* PT_L1E */
+#include "vm_pagetable.h"
 #include "vcpuid.h"   /* Vcpuid */

+#if defined VM_X86_64
+#include "x86paging_64.h"
+#elif defined VM_ARM_64
+#include "arm64_vmsa.h"
+#endif
+
 #define ML_NAME_MAX 16

 /* ML perms are simple and abbreviated. */

+#if defined VM_X86_64
+
 #define ML_PERM_RWX (PTE_P | PTE_RW)
 #define ML_PERM_RW  (PTE_P | PTE_RW | PTE_NX)
 #define ML_PERM_RO  (PTE_P |          PTE_NX)
 #define ML_PERM_RX   PTE_P

+#define ML_PERM_TBL   ML_PERM_RWX
 #define ML_PERM_MASK (PTE_P| PTE_RW | PTE_NX | PTE_US)

+#define ML_PERM_PRESENT(_flags)     (((_flags) & PTE_P) != 0)
+#define ML_PERM_WRITEABLE(_flags)   (((_flags) & PTE_RW) != 0)
+
+#define ML_PTE_2_PFN(_pte)          LM_PTE_2_PFN(_pte)
+
+#elif defined VM_ARM_64
+
+#define _ML_PERM_COMMON (ARM_PTE_BLOCK_AP(ARM_AP_PL0)   | ARM_PTE_BLOCK_AF | \
+                         ARM_PTE_BLOCK_SH(ARM_SH_OUTER) | ARM_PTE_BLOCK_L3_TYPE)
+#define ARM_PTE_BLOCK_AP_RO ARM_PTE_BLOCK_AP(ARM_AP_RO)
+
+#define ML_PERM_RW   (_ML_PERM_COMMON |                       ARM_PTE_BLOCK_XN)
+#define ML_PERM_RO   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+#define ML_PERM_RX   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO                   )
+
+#define ML_PERM_TBL   ML_PERM_RW
+#define ML_PERM_MASK (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+
+#define ML_PERM_PRESENT(_flags)   (((_flags) & ARM_PTE_VALID) != 0)
+#define ML_PERM_WRITEABLE(_flags) (((_flags) & ARM_PTE_BLOCK_AP_RO) == 0)
+
+#define ML_PTE_2_PFN(_pte)    (((_pte) & ARM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
+
+#endif
+
 #define ML_PERMS_MATCH(x,p) (((x) & ML_PERM_MASK) == ((p) & ML_PERM_MASK))

 #define LOADER_HEADER_MAGIC 0x8675309E98675309
@@ -150,10 +182,10 @@ typedef enum {
   ML_CONTENT_SHARE,         /* Share data from external source. */
 } MonLoaderContentType;

-#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? 4 : \
-                               x == ML_CONTENT_PAGETABLE_L3 ? 3 : \
-                               x == ML_CONTENT_PAGETABLE_L2 ? 2 : \
-                               x == ML_CONTENT_PAGETABLE_L1 ? 1 : \
+#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? PT_LEVEL_4 : \
+                               x == ML_CONTENT_PAGETABLE_L3 ? PT_LEVEL_3 : \
+                               x == ML_CONTENT_PAGETABLE_L2 ? PT_LEVEL_2 : \
+                               x == ML_CONTENT_PAGETABLE_L1 ? PT_LEVEL_1 : \
                               0)

 /* Sources of content, for pages not zeroed, unmapped or otherwise special. */
@@ -183,11 +215,14 @@ typedef struct {
   } blobSrc;
   uint64               bspOnly;           /* Process only on BSP. */
   uint64               subIndex;          /* Region ID for ML_CONTENT_COPY. */
-   uint64               procVmmon;         /* Processed by the vmmon on hosted. */
 } MonLoaderEntry;


-/* Packed for easy consumption by bootstrap-offsets.pl. */
+/*
+ * Packed for easy consumption by bootstrap-offsets.pl. If the contents of the
+ * MonLoaderHeader struct are changed then $HEADER_SIZE must be updated
+ * accordingly in bootstrap-offsets.pl.
+ */
 #pragma pack(push, 1)
 typedef struct MonLoaderHeader {
   uint64         magic;
@@ -199,6 +234,8 @@ typedef struct MonLoaderHeader {
   /* ss:rip */
   uint16         stackSelector;
   VA64           stackEntrypoint;
+   LPN64          monStartLPN;
+   LPN64          monEndLPN;
   MonLoaderEntry entries[];
 } MonLoaderHeader;
 #pragma pack(pop)
@@ -217,16 +254,13 @@ MPN  MonLoaderCallout_GetPageRoot(struct MonLoaderEnvContext *, Vcpuid);
 Bool MonLoaderCallout_GetPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                             Vcpuid, PT_L1E *);
 Bool MonLoaderCallout_ImportPage(struct MonLoaderEnvContext *, MPN, Vcpuid);
-Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **);
+Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **, unsigned);
 Bool MonLoaderCallout_MapMPNInPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                                  uint64, MPN, Vcpuid);
 MPN  MonLoaderCallout_GetSharedUserPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
 MPN  MonLoaderCallout_GetSharedHostPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
-Bool MonLoaderCallout_IsPrivileged(struct MonLoaderEnvContext *);
-Bool MonLoaderCallout_SetEntrypoint(struct MonLoaderEnvContext *, uint16, VA64,
-                                    uint16, VA64);
 MPN  MonLoaderCallout_GetBlobMpn(struct MonLoaderEnvContext *, uint64);

 typedef enum MonLoaderError {
@@ -237,7 +271,6 @@ typedef enum MonLoaderError {
   ML_ERROR_ARGS,
   ML_ERROR_CALLOUT_INIT,
   ML_ERROR_CALLOUT_COPY,
-   ML_ERROR_CALLOUT_ENTRYPOINT,
   ML_ERROR_CALLOUT_GETPTE,
   ML_ERROR_CALLOUT_MAPINPTE,
   ML_ERROR_CALLOUT_PAGEROOT_GET,
@@ -259,8 +292,12 @@ typedef enum MonLoaderError {
 } MonLoaderError;


-/* A subindex above shared area subindices for sharing of MonLoaderHeader. */
+/*
+ * Values above shared area subindices for sharing of MonLoaderHeader
+ * and htSchedStateMap.
+ */
 #define MONLOADER_HEADER_IDX 6
+#define MONLOADER_HT_MAP_IDX 7


 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
@@ -25,23 +25,7 @@
 #ifndef _MONLOADER_LOG_H_
 #define _MONLOADER_LOG_H_

-#ifdef VMX86_VMX
-
-#define LOGLEVEL_MODULE main
-#include "loglevel_vmcore.h"
-/*
- * Redefine LOG() to match the vmkernel calling convention.
- */
-#undef LOG_BYNAME
-#undef LOG
-
-#define LOG_BYNAME(_mod, _min, _fmt, ...) \
-        (DOLOG_BYNAME(_mod, _min) ? Log(_fmt "\n", ## __VA_ARGS__) : (void) 0)
-
-#define LOG(_min, _fmt, ...) \
-        LOG_BYNAME(LOGLEVEL_MODULE, _min, _fmt, ## __VA_ARGS__)
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #define LOGLEVEL_MODULE MonLoader
 #include "log.h"
@@ -59,7 +43,7 @@ do {                                                 \
   }                                                 \
 } while (0)

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2007-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2007-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -60,20 +60,21 @@
 *     lengths, and this causes the 'monitor-modular-size' script to
 *     report differences in '.rodata'.
 *
- *   o To avoid differences in '.rodata', each __FILE__ is put into
- *     its own section.  The monitor's linker (not ld) will use the
- *     name of the section to recover the name of the source file.
+ *   o Each __FILE__ is put into its own .assert_pathname_##__FILE
+ *     section.  The monitor's runtime linker (not ld) then uses the name
+ *     of this section to recover the name of the corresponding source file.
 *
 *   o At run time, prior to loading, when our linker is creating an
 *     executable image of the monitor and extensions, all the file
 *     names are extracted from these sections, the '${VMTREE}' prefix
 *     is removed, and the resulting table of shortened file names is
- *     added to '.rodata'.
+ *     added to a non-loadable section called '.assert_pathnames' in the
+ *     VMX linker (not the bootstrap linker which does not need them).
 *
 *     Further, during linkage, each relocation to the original
 *     section containing the path name is modified so that the low
- *     16-bits contain an offset from '__vmm_pathnames_start' rather
- *     than the base of the original containing section.
+ *     16-bits contain an offset from the start of the .assert_pathnames
+ *     section rather than the base of the original containing section.
 *
 *     Only three types of relocations to the assertion strings are
 *     supported (32-bit PC-relative and 32-bit/64-bit absolute) because that
@@ -111,42 +112,12 @@ typedef struct Assert_Info {
   Assert_MonSrcLoc loc;
 } Assert_Info;

-/*
- * The portion of the __attribute__ line after __FILE__ is there so that
- * the .assert_pathname_* sections are not marked as ALLOC, since we only
- * need them in the vmx and do not need them loaded.
- */
-#define __VMM__FILE__SECTION \
-      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
-#define __VMM__FILE__ ({                                                \
-         static __VMM__FILE__SECTION const char file[] = "";            \
-         file;                                                          \
-      })
-
 #define ASSERT_MONSRCFILEOFFSET(loc)    LOWORD(loc)
 #define ASSERT_MONSRCLINE(loc)          HIWORD(loc)

 #define ASSERT_NULL_MONSRCLOC     0             // there is never line 0

-#ifdef VMM // {
-#ifdef MONITOR_APP // {
-
-#define ASSERT_MONSRCLOC() ASSERT_NULL_MONSRCLOC
-
-#else // } {
-
-#define ASSERT_MONSRCLOC() ({                                           \
-   const uintptr_t offset = ((__LINE__ << 16) +                         \
-                             (uintptr_t)__VMM__FILE__);                 \
-   const Assert_MonSrcLoc loc = offset;                                 \
-   loc;                                                                 \
-})
-
-extern const char __vmm_pathnames_start;
-#define ASSERT_MONSRCFILE(loc) \
-   (&__vmm_pathnames_start + ASSERT_MONSRCFILEOFFSET(loc))
-
-
+#if defined(VMM) && !defined(MONITOR_APP) // {
 /*
 * Assertion information is collected in a non-loadable section
 * named .assert_info.  Each record in this section contains
@@ -176,6 +147,18 @@ extern const char __vmm_pathnames_start;
 */

 #ifndef VMM_BOOTSTRAP
+/*
+ * The portion of the __attribute__ line after __FILE__ is there so that
+ * the .assert_pathname_* sections are not marked as ALLOC, since we only
+ * need them in the vmx and do not need them loaded.
+ */
+#define __VMM__FILE__SECTION \
+      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
+#define __VMM__FILE__ ({                                                \
+         static __VMM__FILE__SECTION const char file[] = "";            \
+         file;                                                          \
+      })
+
 #define ASSERT_RECORDINFO(assembly, assertType, bugNr)                   \
   __asm__ __volatile__(".pushsection .assert_info;"                     \
                        ".quad 0f;"                                      \
@@ -215,8 +198,6 @@ extern uint64 bsAssertRIP;
   ({COMPILER_MEM_BARRIER();                                             \
     ASSERT_RECORDINFO("", AssertType_##name##Bug, bug);                 \
     __builtin_trap();})
+#endif // VMM && !MONITOR_APP}

-#endif // MONITOR_APP }
-#endif // VMM }
-
-#endif
+#endif // _MON_ASSERT_H_
@@ -52,7 +52,7 @@ extern "C" {
 typedef struct PageListEntry {
   CompressedBPN cbpn;
   Bool voided;
-   uint8 _pad[2];
+   uint8 _pad[1];
 } PageListEntry;
 #pragma pack(pop) 

@@ -55,12 +55,16 @@ typedef enum PCIPassthruVectorIndex {
 } PCIPassthruVectorIndex;

 typedef enum PCIPassthru_IntrType {
-   PCIPASSTHRU_INTR_NONE   = 0x00,
-   PCIPASSTHRU_INTR_IOAPIC = 0x01,
-   PCIPASSTHRU_INTR_MSI    = 0x02,
-   PCIPASSTHRU_INTR_MSIX   = 0x04,
+   PCIPASSTHRU_INTR_NONE    = 0x00,
+   PCIPASSTHRU_INTR_IOAPIC  = 0x01,
+   PCIPASSTHRU_INTR_MSI     = 0x02,
+   PCIPASSTHRU_INTR_MSIX    = 0x04,
+   PCIPASSTHRU_INTR_MSI_INT = 0x08,
 } PCIPassthru_IntrType;

+#define PCIPASSTHRU_INTR_MSI_COMBINED                   \
+   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)
+
 typedef struct FPTIntrProxyInfo {
   uint32            adapterIndex;
   uint32            vectorIndex;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,12 @@
 *********************************************************/

 /*
- * perfctr_generic.h --
+ * perfctr.h --
 *
 */

-#ifndef _PERFCTR_GENERIC_H_
-#define _PERFCTR_GENERIC_H_
+#ifndef _PERFCTR_H_
+#define _PERFCTR_H_

 #define INCLUDE_ALLOW_USERLEVEL

@@ -33,7 +33,9 @@

 #include "includeCheck.h"
 #include "vm_basic_types.h"
+#include "perfctr_arch.h"

+#define PERF_EVENT_NAME_LEN                      64
 /*
 * nmiNo      -- vmm peer is not attempting to do nmi profiling this run
 * nmiYes     -- vmm peer is doing nmi profiling and nmis are currently enabled
@@ -69,4 +71,51 @@ typedef struct {
   uint8  _pad[3];
 } CrossProfShared;

-#endif // ifndef _PERFCTR_GENERIC_H_
+/*
+ * PerfCtr_Config --
+ *      Describes configuration for a single hardware performance counter
+ *
+ *      Since this is only used to record general performance counters, we
+ *      made the assumption in nmiProfiler.c that the type is GENERAL and
+ *      index is counter number of type GENERAL.
+ *
+ *      **** x86 ****
+ *      On AMD K8 and GH:
+ *      index:        Which perf ctr, 0 to 3.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On AMD with PerfCtrExtCore support:
+ *      index:        Which perf ctr, 0 to 5.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On Intel Core architecture:
+ *      <to be documented>
+ *
+ *      **** ARM ****
+ *      escrVal:     Value placed in PMEVTYPER<n>_EL0 to configure event counter
+ *      index:       Index of the event counter.
+ */
+typedef struct PerfCtr_Config {
+   uint64 escrVal;
+   uint32 index;
+   uint32 periodMean;
+
+   /*
+    * Random number (whose absolute value is capped at
+    * periodJitterMask) is used to randomize sampling interval.
+    */
+   uint32  periodJitterMask;
+   uint32  seed;    // seed is used to compute next random number
+   uint16  config;
+   Bool    valid;
+   PERFCTR_CONFIG_ARCH_FIELDS
+} PerfCtr_Config;
+
+
+#endif // ifndef _PERFCTR_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,17 @@
 *********************************************************/

 /*
- * x86perfctr.h --
+ * perfctr_arch.h --
 *
+ *      Performance counters (x64 specific).
 */

-#ifndef _X86PERFCTR_H_
-#define _X86PERFCTR_H_
+#ifndef _PERFCTR_ARCH_H
+#define _PERFCTR_ARCH_H
+
+#ifndef _PERFCTR_H_
+#error "This file can only be included by perfctr.h"
+#endif

 #define INCLUDE_ALLOW_USERLEVEL

@@ -30,25 +35,20 @@
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
-
 #include "includeCheck.h"
+
 #include "vm_asm.h"
 #include "x86cpuid_asm.h"
-#include "perfctr_generic.h"

-#define PERFCTR_PENTIUM4_NUM_COUNTERS            18
-#define PERFCTR_PENTIUM4_NUM_COUNTERS_WITH_L3    26
-#define PERFCTR_AMD_NUM_COUNTERS                 4 
+#define PERFCTR_AMD_NUM_COUNTERS                 4
 #define PERFCTR_AMD_EXT_NUM_COUNTERS             6
 #define PERFCTR_P6_NUM_COUNTERS                  2
 #define PERFCTR_NEHALEM_NUM_GEN_COUNTERS         4
 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS       3
 #define PERFCTR_SANDYBRIDGE_NUM_GEN_COUNTERS     8 /* When HT is disabled */
-#define PERFCTR_CORE_NUM_ARCH_EVENTS             7
-#define PERFCTR_CORE_NUM_FIXED_COUNTERS          3
-#define PERFCTR_PENTIUM4_VAL_MASK                0xffffffffffLL
+#define PERFCTR_CORE_NUM_ARCH_EVENTS             8
+#define PERFCTR_CORE_NUM_FIXED_COUNTERS          4
 #define PERFCTR_AMD_VAL_MASK                     0xffffffffffffLL
-#define PERF_EVENT_NAME_LEN                      64
 /*
 * Even though the performance counters in P6 are 40 bits,
 * we can only write to the lower 32 bits.  Bit 31 is
@@ -59,7 +59,7 @@

 /*
 * Performance counter width is determined at runtime in CORE.
- * But the writables bits are fixed. we can only write to the 
+ * But the writables bits are fixed. we can only write to the
 * lower 32 bits.  Bit 31 is used to sign-extend the upper 8 bits.
 */
 #define PERFCTR_CORE_WRITE_MASK                  0xffffffff
@@ -124,7 +124,7 @@
 #define PERFCTR_AMD_CPU_CLK_UNHALTED                           0x76

 /* AMD Load/Store unit events */
-#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20 
+#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20
 #define PERFCTR_AMD_LS_BUFFER2_FULL                            0x23

 /*
@@ -198,8 +198,8 @@

 #define PERFCTR_AMD_MEM_CTRL_PAGE_TABLE_OVERFLOWS                   0xe1
 #define PERFCTR_AMD_CPU_IO_REQUESTS_TO_MEMORY_IO                    0xe9
-#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec 
- 
+#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec
+
 /* AMD HyperTransport Interface Events */

 #define PERFCTR_AMD_HT_L0_TX_BW                                     0xf6
@@ -219,9 +219,9 @@
 */

 /* P6 Data Cache Unit (DCU) */
-#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043  
+#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043
 #define PERFCTR_P6_DCU_LINES_IN                  0x00000045
-#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046 
+#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046
 #define PERFCTR_P6_DCU_MISS_OUTSTANDING          0x00000048

 /* P6 Instruction Fetch Unit (IFU) */
@@ -379,7 +379,7 @@
 #define PERFCTR_P6_APIC_INTR                     0x00100000
 #define PERFCTR_P6_ENABLE                        0x00400000
 #define PERFCTR_P6_INVERT_COUNTER_MASK           0x00800000
-#define PERFCTR_P6_COUNTER_MASK                  0xff000000        
+#define PERFCTR_P6_COUNTER_MASK                  0xff000000
 #define PERFCTR_P6_COUNTER_MASK_SHIFT            24
 #define PERFCTR_P6_SHIFT_BY_UNITMASK(e)          (e << 8)

@@ -436,6 +436,7 @@
 #define PERFCTR_CORE_KERNEL_MODE                    PERFCTR_CPU_KERNEL_MODE
 #define PERFCTR_CORE_APIC_INTR                      PERFCTR_CPU_APIC_INTR
 #define PERFCTR_CORE_ENABLE                         PERFCTR_CPU_ENABLE
+/* AnyThread Counting deprecated since PMU v5. */
 #define PERFCTR_CORE_ANYTHREAD                      0x00200000
 #define PERFCTR_CORE_IN_TX                          (CONST64U(1) << 32)
 #define PERFCTR_CORE_IN_TXCP                        (CONST64U(1) << 33)
@@ -463,6 +464,7 @@
 #define PERFCTR_CORE_INST_RETIRED                0xc0
 /* bus cycles */
 #define PERFCTR_CORE_UNHALTED_REF_CYCLES         (0x3c | (0x01 << 8))
+#define PERFCTR_CORE_TOPDOWN_SLOTS               (0xa4 | (0x01 << 8))

 /*
 * See Tables 30-2, 30-4 of the
@@ -558,132 +560,20 @@
 #define PERFCTR_NEHALEM_UNCORE_ENABLE            0x400000
 #define PERFCTR_NEHALEM_UNCORE_L3_LINES_IN       (0x0a | (0x0f << 8))

-
-/*
- * ----------------------------------------------------------------------
- *
- * Pentium 4
- *
- * ----------------------------------------------------------------------
- */
-
-/*
- * Pentium 4 Counter Configuration Control Register flags and fields
- */
-#define PERFCTR_PENTIUM4_CCCR_ENABLE                   0x00001000
-#define PERFCTR_PENTIUM4_CCCR_SET_ESCR(c, e)           ((c) |= (((e) & 0x7) << 13))
-#define PERFCTR_PENTIUM4_CCCR_REQRSVD                  0x00030000 // always set these bits
-#define PERFCTR_PENTIUM4_CCCR_COMPARE                  0x00040000
-#define PERFCTR_PENTIUM4_CCCR_COMPLEMENT               0x00080000
-#define PERFCTR_PENTIUM4_CCCR_SET_THRESHOLD(c, e)      ((c) |= (((e) & 0xf) << 20))
-#define PERFCTR_PENTIUM4_CCCR_EDGE                     0x01000000
-#define PERFCTR_PENTIUM4_CCCR_FORCE_OVF                0x02000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T0               0x04000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T1               0x08000000
-#define PERFCTR_PENTIUM4_CCCR_CASCADE                  0x40000000
-#define PERFCTR_PENTIUM4_CCCR_OVF                      0x80000000
-
-#define PERFCTR_PENTIUM4_COUNTER_BASEADDR              0x00000300
-#define PERFCTR_PENTIUM4_CCCR_BASE_ADDR                0x00000360
-
-#define PERFCTR_PENTIUM4_CCCR_THRESHOLD(e)             ((e & 0xf) << 20)
-#define PERFCTR_PENTIUM4_OPT_EDGE_DETECT (PERFCTR_PENTIUM4_CCCR_COMPARE | PERFCTR_PENTIUM4_CCCR_EDGE)
-
-/*
- * Pentium 4 Event Selection Control Register flags
- */
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T0             0x00000004
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T0           0x00000008
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T1             0x00000001
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T1           0x00000002
-#define PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK(e)          (e << 9)
-#define PENTIUM4_EVTSEL(e)                             (e << 25)
-
-/*
- * Event definitions for Pentium4 Family Processors. 
- * There are many more of these possible.
- * See Appendix A.1 in Volume 3 of the IA32 manual.
- */
-
-#define PERFCTR_PENTIUM4_EVT_INSTR_RETIRED              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_INSTR_COMPLETED            PENTIUM4_EVTSEL(0x07)
-#define PERFCTR_PENTIUM4_EVT_BSQ_CACHE_REFERENCE        PENTIUM4_EVTSEL(0x0c)
-#define PERFCTR_PENTIUM4_EVT_ITLB_REFERENCE             PENTIUM4_EVTSEL(0x18)
-#define PERFCTR_PENTIUM4_EVT_PAGE_WALK_TYPE             PENTIUM4_EVTSEL(0x01)   
-#define PERFCTR_PENTIUM4_EVT_BPU_FETCH_REQUEST          PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_BRANCH_RETIRED             PENTIUM4_EVTSEL(0x06)  
-#define PERFCTR_PENTIUM4_EVT_MISPRED_BRANCH_RETIRED     PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_MACHINE_CLEAR              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_MEMORY_CANCEL              PENTIUM4_EVTSEL(0x02) 
-#define PERFCTR_PENTIUM4_GLOBAL_POWER_EVENTS            PENTIUM4_EVTSEL(0x13)
-
-/*
- * Pentium4 doesn't have an event for clock cycles, but Intel
- * outlines a method in the IA32 manual, vol 3, sec 14.9.9,
- * for measuring cycles which involves this event strategy.
- */
-
-#define PERFCTR_PENTIUM4_EVT_CLK_CYCLES \
-   PENTIUM4_EVTSEL(0x2) | \
-   PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK (0x01 | 0x02 | 0x04 | 0x08)
-
-
-/*
- * PerfCtr_Config --
- *      Describes configuration for a single hardware performance counter
- *
- *      Since this is only used to record general performance counters, we
- *      made the assumption in nmiProfiler.c  that the type is GENERAL and
- *      index is counter number of type GENERAL.
- *      
- *      
- *      On AMD K8 and GH:
- *      index:        Which perf ctr, 0 to 3.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- *
- *      On AMD with PerfCtrExtCore support:
- *      index:        Which perf ctr, 0 to 5.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- * 
- *      No field should be greater than 32-bit as it is shared with monitor.
- *
- *      On Intel Core architecture:
- *      <to be documented>
- */
-typedef struct PerfCtr_Config {
-   uint64 escrVal;
-   uint32 index;
-   uint32 addr;
-   uint32 escrAddr;
-   uint32 resetHi;
-   uint32 periodMean;
-
-   /*
-    * Random number (whose absolute value is capped at
-    * periodJitterMask) is used to randomize sampling interval.
-    */
-   uint32  periodJitterMask;
-   uint32  seed;    // seed is used to compute next random number
-   uint16  config;
-   Bool    valid;
-   Bool    pebsEnabled;
-} PerfCtr_Config;
-
+#define PERFCTR_CONFIG_ARCH_FIELDS \
+   uint32 addr;      \
+   uint32 escrAddr;  \
+   uint32 resetHi;   \
+   Bool pebsEnabled; \

 /*
 * Program/reprogram event reg(s) associated w/perfctrs & start or stop perfctrs
 */
 static INLINE void
-PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
-                   uint32 escrVal)       // IN: event register value
+PerfCtr_WriteEvtSel(uint32 addr,       // IN: Event register to write
+                    uint32 escrVal)    // IN: event register value
 {
-   __SET_MSR(ctr->escrAddr, escrVal);
+   X86MSR_SetMSR(addr, escrVal);
 }


@@ -691,11 +581,10 @@ PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
 * Set/reset performance counters to engender desired period before overflow
 */
 static INLINE void
-PerfCtrWriteCounter(PerfCtr_Config *ctr,   // IN: counter to write
-                    uint32 valueLo,        // IN: low 32 bits of value to write
-                    uint32 valueHi)        // IN: high 32 bits of value to write
+PerfCtr_WriteCounter(uint32 addr,   // IN: counter to write
+                     uint64 value)  // IN: value to write
 {
-   WRMSR(ctr->addr, valueLo, valueHi);
+   X86MSR_SetMSR(addr, value);
 }


@@ -814,8 +703,9 @@ PerfCtr_PEBSAvailable(void)
   CPUIDRegs regs;
   __GET_CPUID(0, &regs);
   if (CPUID_IsVendorIntel(&regs) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_EMON_AVAILABLE) != 0) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_PEBS_UNAVAILABLE) == 0)) {
+       (X86MSR_GetMSR(MSR_MISC_ENABLE) &
+        (MSR_MISC_ENABLE_EMON_AVAILABLE | MSR_MISC_ENABLE_PEBS_UNAVAILABLE))
+       == MSR_MISC_ENABLE_EMON_AVAILABLE) {
      CPUIDRegs hvendor;

      /*
@@ -827,7 +717,7 @@ PerfCtr_PEBSAvailable(void)
             !CPUID_IsRawVendor(&hvendor,
                                CPUID_HYPERV_HYPERVISOR_VENDOR_STRING);
   }
-   return FALSE; 
+   return FALSE;
 }

 /*
@@ -858,162 +748,4 @@ PerfCtr_PTAvailable(void)
   return FALSE;
 }

-/* The following are taken from the Intel Architecture Manual,
- * Book 3, Table 14-4. 
- */ 
-
-/* --------- BEGIN INTEL DEFINES ------------------------ */
-
-#define PENTIUM4_MSR_BPU_COUNTER0_IDX           0
-#define PENTIUM4_MSR_BPU_COUNTER0_ADDR          0x300
-#define PENTIUM4_MSR_BPU_CCCR0                  0x360
-#define PENTIUM4_MSR_BPU_COUNTER1_IDX           1
-#define PENTIUM4_MSR_BPU_COUNTER1_ADDR          0x301
-#define PENTIUM4_MSR_BPU_CCCR1                  0x361
-#define PENTIUM4_MSR_BPU_COUNTER2_IDX           2
-#define PENTIUM4_MSR_BPU_COUNTER2_ADDR          0x302
-#define PENTIUM4_MSR_BPU_CCCR2                  0x362
-#define PENTIUM4_MSR_BPU_COUNTER3_IDX           3
-#define PENTIUM4_MSR_BPU_COUNTER3_ADDR          0x303
-#define PENTIUM4_MSR_BPU_CCCR3                  0x363
-#define PENTIUM4_MSR_MS_COUNTER0_IDX            4
-#define PENTIUM4_MSR_MS_COUNTER0_ADDR           0x304
-#define PENTIUM4_MSR_MS_CCCR0                   0x364
-#define PENTIUM4_MSR_MS_COUNTER1_IDX            5
-#define PENTIUM4_MSR_MS_COUNTER1_ADDR           0x305
-#define PENTIUM4_MSR_MS_CCCR1                   0x365
-#define PENTIUM4_MSR_MS_COUNTER2_IDX            6
-#define PENTIUM4_MSR_MS_COUNTER2_ADDR           0x306
-#define PENTIUM4_MSR_MS_CCCR2                   0x366
-#define PENTIUM4_MSR_MS_COUNTER3_IDX            7
-#define PENTIUM4_MSR_MS_COUNTER3_ADDR           0x307
-#define PENTIUM4_MSR_MS_CCCR3                   0x367
-#define PENTIUM4_MSR_FLAME_COUNTER0_IDX         8
-#define PENTIUM4_MSR_FLAME_COUNTER0_ADDR        0x308
-#define PENTIUM4_MSR_FLAME_CCCR0                0x368
-#define PENTIUM4_MSR_FLAME_COUNTER1_IDX         9
-#define PENTIUM4_MSR_FLAME_COUNTER1_ADDR        0x309
-#define PENTIUM4_MSR_FLAME_CCCR1                0x369
-#define PENTIUM4_MSR_FLAME_COUNTER2_IDX         10
-#define PENTIUM4_MSR_FLAME_COUNTER2_ADDR        0x30A
-#define PENTIUM4_MSR_FLAME_CCCR2                0x36A
-#define PENTIUM4_MSR_FLAME_COUNTER3_IDX         11
-#define PENTIUM4_MSR_FLAME_COUNTER3_ADDR        0x30B
-#define PENTIUM4_MSR_FLAME_CCCR3                0x36B
-#define PENTIUM4_MSR_IQ_COUNTER0_IDX            12
-#define PENTIUM4_MSR_IQ_COUNTER0_ADDR           0x30C
-#define PENTIUM4_MSR_IQ_CCCR0                   0x36C
-#define PENTIUM4_MSR_IQ_COUNTER1_IDX            13
-#define PENTIUM4_MSR_IQ_COUNTER1_ADDR           0x30D
-#define PENTIUM4_MSR_IQ_CCCR1                   0x36D
-#define PENTIUM4_MSR_IQ_COUNTER2_IDX            14
-#define PENTIUM4_MSR_IQ_COUNTER2_ADDR           0x30E
-#define PENTIUM4_MSR_IQ_CCCR2                   0x36E
-#define PENTIUM4_MSR_IQ_COUNTER3_IDX            15
-#define PENTIUM4_MSR_IQ_COUNTER3_ADDR           0x30F
-#define PENTIUM4_MSR_IQ_CCCR3                   0x36F
-#define PENTIUM4_MSR_IQ_COUNTER4_IDX            16
-#define PENTIUM4_MSR_IQ_COUNTER4_ADDR           0x310
-#define PENTIUM4_MSR_IQ_CCCR4                   0x370
-#define PENTIUM4_MSR_IQ_COUNTER5_IDX            17
-#define PENTIUM4_MSR_IQ_COUNTER5_ADDR           0x311
-#define PENTIUM4_MSR_IQ_CCCR5                   0x371
-
-#define PENTIUM4_MSR_ALF_ESCR0_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR0_ADDR             0x3CA
-#define PENTIUM4_MSR_ALF_ESCR1_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR1_ADDR             0x3CB
-#define PENTIUM4_MSR_BPU_ESCR0_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR0_ADDR             0x3B2
-#define PENTIUM4_MSR_BPU_ESCR1_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR1_ADDR             0x3B3
-#define PENTIUM4_MSR_BSU_ESCR0_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR0_ADDR             0x3A0
-#define PENTIUM4_MSR_BSU_ESCR1_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR1_ADDR             0x3A1
-#define PENTIUM4_MSR_CRU_ESCR0_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR0_ADDR             0x3B8
-#define PENTIUM4_MSR_CRU_ESCR1_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR1_ADDR             0x3B9
-#define PENTIUM4_MSR_CRU_ESCR2_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR2_ADDR             0x3CC
-#define PENTIUM4_MSR_CRU_ESCR3_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR3_ADDR             0x3CD
-#define PENTIUM4_MSR_CRU_ESCR4_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR4_ADDR             0x3E0
-#define PENTIUM4_MSR_CRU_ESCR5_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR5_ADDR             0x3E1
-#define PENTIUM4_MSR_DAC_ESCR0_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR0_ADDR             0x3A8
-#define PENTIUM4_MSR_DAC_ESCR1_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR1_ADDR             0x3A9
-#define PENTIUM4_MSR_FIRM_ESCR0_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR0_ADDR            0x3A4
-#define PENTIUM4_MSR_FIRM_ESCR1_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR1_ADDR            0x3A5
-#define PENTIUM4_MSR_FLAME_ESCR0_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR0_ADDR           0x3A6
-#define PENTIUM4_MSR_FLAME_ESCR1_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR1_ADDR           0x3A7
-#define PENTIUM4_MSR_FSB_ESCR0_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR0_ADDR             0x3A2
-#define PENTIUM4_MSR_FSB_ESCR1_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR1_ADDR             0x3A3
-#define PENTIUM4_MSR_IQ_ESCR0_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR0_ADDR              0x3BA
-#define PENTIUM4_MSR_IQ_ESCR1_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR1_ADDR              0x3BB
-#define PENTIUM4_MSR_IS_ESCR0_IDX               1
-#define PENTIUM4_MSR_IS_ESCR0_ADDR              0x3B4
-#define PENTIUM4_MSR_IS_ESCR1_IDX               1
-#define PENTIUM4_MSR_IS_ESCR1_ADDR              0x3B5
-#define PENTIUM4_MSR_ITLB_ESCR0_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR0_ADDR            0x3B6
-#define PENTIUM4_MSR_ITLB_ESCR1_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR1_ADDR            0x3B7
-#define PENTIUM4_MSR_IX_ESCR0_IDX               5
-#define PENTIUM4_MSR_IX_ESCR0_ADDR              0x3C8
-#define PENTIUM4_MSR_IX_ESCR1_IDX               5
-#define PENTIUM4_MSR_IX_ESCR1_ADDR              0x3C9
-#define PENTIUM4_MSR_MOB_ESCR0_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR0_ADDR             0x3AA
-#define PENTIUM4_MSR_MOB_ESCR1_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR1_ADDR             0x3AB
-#define PENTIUM4_MSR_MS_ESCR0_IDX               0
-#define PENTIUM4_MSR_MS_ESCR0_ADDR              0x3C0
-#define PENTIUM4_MSR_MS_ESCR1_IDX               0
-#define PENTIUM4_MSR_MS_ESCR1_ADDR              0x3C1
-#define PENTIUM4_MSR_PMH_ESCR0_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR0_ADDR             0x3AC
-#define PENTIUM4_MSR_PMH_ESCR1_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR1_ADDR             0x3AD
-#define PENTIUM4_MSR_RAT_ESCR0_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR0_ADDR             0x3BC
-#define PENTIUM4_MSR_RAT_ESCR1_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR1_ADDR             0x3BD
-#define PENTIUM4_MSR_SAAT_ESCR0_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR0_ADDR            0x3AE
-#define PENTIUM4_MSR_SAAT_ESCR1_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR1_ADDR            0x3AF
-#define PENTIUM4_MSR_SSU_ESCR0_IDX              3
-#define PENTIUM4_MSR_SSU_ESCR0_ADDR             0x3BE
-#define PENTIUM4_MSR_TBPU_ESCR0_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR0_ADDR            0x3C2
-#define PENTIUM4_MSR_TBPU_ESCR1_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR1_ADDR            0x3C3
-#define PENTIUM4_MSR_TC_ESCR0_IDX               1
-#define PENTIUM4_MSR_TC_ESCR0_ADDR              0x3C4
-#define PENTIUM4_MSR_TC_ESCR1_IDX               1
-#define PENTIUM4_MSR_TC_ESCR1_ADDR              0x3C5
-#define PENTIUM4_MSR_U2L_ESCR0_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR0_ADDR             0x3B0
-#define PENTIUM4_MSR_U2L_ESCR1_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR1_ADDR             0x3B1
-
-#define PENTIUM4_MIN_ESCR_ADDR                  0x3A0
-#define PENTIUM4_MAX_ESCR_ADDR                  0x3E1
-#define PENTIUM4_NUM_ESCR_ADDRS                 (PENTIUM4_MAX_ESCR_ADDR - PENTIUM4_MIN_ESCR_ADDR + 1)
-
-/* -------- END INTEL DEFINES ------------------------- */
-
-#endif // ifndef _X86PERFCTR_H_ 
+#endif // _PERFCTR_ARCH_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -173,35 +173,6 @@ int64 PTSC_CyclesToUS(VmRelativeTS ts);
 */
 #include "user_layout.h"

-#ifdef VM_ARM_64
-static INLINE VmAbsoluteTS
-PTSC_Get(void)
-{
-   extern __thread User_ThreadData vmkUserTdata;
-
-   register RateConv_Params params;
-   register uint64 pseudoTSC;
-
-   /*
-    * On ARM64 the generic timer guarantees that the counters are synchronous
-    * and independent of CPU frequency.
-    */
-   params = vmkUserTdata.pseudoTSCConv;
-   pseudoTSC = RDTSC();
-
-   /*
-    *  assert that conversion rate is `identity'.
-    * we'll have to revisit that when/if this assumption changes
-    * on some platform
-    */
-   ASSERT((params.mult == 1 && params.shift == 0) ||
-          (params.mult == 0x80000000 && params.shift == 31));
-
-   pseudoTSC += params.add;
-
-   return pseudoTSC;
-}
-#else
 static INLINE VmAbsoluteTS
 PTSC_Get(void)
 {
@@ -215,7 +186,7 @@ PTSC_Get(void)
   ASSERT((int64)ptsc >= 0);
   return ptsc;
 }
-#endif
+
 #else

 /*
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,7 +97,7 @@ void RateConv_LogRatio(const char *prefix,
 static INLINE uint64
 RateConv_Unsigned(const RateConv_Params *conv, uint64 x)
 {
-   return Mul64x3264(x, conv->mult, conv->shift) + conv->add;
+   return Mul64x3264(x, conv->mult, conv->shift) + (uint64)conv->add;
 }


@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,28 +16,32 @@
 *
 *********************************************************/

-#ifndef _INITBLOCK_H
-#define _INITBLOCK_H
+/*
+ * sharedAreaType.h --
+ *
+ *      This file contains shared area type definitions.
+ */

+#ifndef _SHAREDAREATYPE_H_
+#define _SHAREDAREATYPE_H_

-#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+
 #include "includeCheck.h"

+typedef enum {
+   SHARED_AREA_PER_VM_VMX = 0,
+   SHARED_AREA_INTER_VCPU_VMX,
+   SHARED_AREA_PER_VCPU_VMX,
+   SHARED_AREA_PER_VM,
+   SHARED_AREA_INTER_VCPU,
+   SHARED_AREA_PER_VCPU,
+   NUM_SHARED_AREAS
+} SharedAreaType;

-#include "basic_initblock.h"
-
-
-#define MAX_LOGFILE_NAME_LENGTH 256
-#define MAX_MONITOR_REGIONS     8
-#define INIT_BLOCK_MAGIC     (0x1789+14)
-
-
-/*
- * Option flags 
- */
-/* None yet */
-
-
-#endif
+#endif // _SHAREDAREATYPE_H_
@@ -0,0 +1,62 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.h --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifndef SHARED_AREA_VMMON_H
+#define SHARED_AREA_VMMON_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#include "includeCheck.h"
+
+#include "iocontrols.h"
+
+struct VMDriver;
+
+/* A collection of pages backing a shared area region. */
+typedef struct SharedAreaVmmonRegion {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} SharedAreaVmmonRegion;
+
+/* The VMMon Driver shared area. */
+typedef struct SharedAreaVmmon {
+   SharedAreaVmmonRegion regions[NUM_SHARED_AREAS];
+} SharedAreaVmmon;
+
+/* Request for a backing MPN for a shared area region at a given VCPU/offset. */
+typedef struct SharedAreaVmmonRequest {
+   SharedAreaType type;
+   Vcpuid vcpu;
+   PageCnt offset;
+} SharedAreaVmmonRequest;
+
+SharedAreaVmmon *SharedAreaVmmon_Init(struct VMDriver *vm);
+void SharedAreaVmmon_Cleanup(SharedAreaVmmon *area);
+Bool SharedAreaVmmon_RegisterRegion(struct VMDriver *driver,
+                                    VMSharedAreaRegistrationBlock *block);
+Bool SharedAreaVmmon_ValidateRegionArgs(struct VMDriver *driver,
+                                        VMSharedAreaRegistrationBlock *block);
+MPN SharedAreaVmmon_GetRegionMPN(struct VMDriver *vm,
+                                 SharedAreaVmmonRequest *request);
+#endif /* SHARED_AREA_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,45 +17,32 @@
 *********************************************************/

 /*
- * basic_initblock.h --
+ * statVarsVmmon.h --
 *
- *    VM initialization block.
+ *     VMMon stat vars management.
 */

-#ifndef _BASIC_INITBLOCK_H_
-#define _BASIC_INITBLOCK_H_
+#ifndef STAT_VARS_VMMON_H
+#define STAT_VARS_VMMON_H

-
-#define INCLUDE_ALLOW_USERLEVEL
-
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_DISTRIBUTE
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

+#include "iocontrols.h"

-#include "vcpuid.h"
+struct VMDriver;

+/* The VMMon Driver stat vars area. */
+typedef struct StatVarsVmmon {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} StatVarsVmmon;

-#define MAX_INITBLOCK_CPUS     MAX_VCPUS
-
-
-typedef
-#include "vmware_pack_begin.h"
-struct InitBlock {
-   uint32 magicNumber;     /* Magic number (INIT_BLOCK_MAGIC) */
-   uint32 vmInitFailurePeriod;
-   VA64   crosspage[MAX_INITBLOCK_CPUS];
-   LPN64  monStartLPN;
-   LPN64  monEndLPN;
-   MPN    crossGDTMPN;
-   uint16 numPTPPages;
-}
-#include "vmware_pack_end.h"
-InitBlock;
-
-
-#endif // _BASIC_INITBLOCK_H_
+StatVarsVmmon *StatVarsVmmon_Init(struct VMDriver *vm);
+void StatVarsVmmon_Cleanup(StatVarsVmmon *statVars);
+Bool StatVarsVmmon_RegisterVCPU(struct VMDriver *driver,
+                                VMStatVarsRegistrationBlock *block);
+MPN StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpuid,
+                               PageCnt offset);
+#endif /* STAT_VARS_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 #ifndef USERCALLDEFS_H
 #define USERCALLDEFS_H

-#ifdef linux
+#ifdef __linux__
 #include <linux/errno.h>
 #define USERCALL_RESTART (-ERESTARTNOINTR)
 #else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,6 @@
 extern "C" {
 #endif

-
 typedef uint32 Vcpuid;                 // VCPU number

 #define VCPUID_INVALID  (~0U)
@@ -54,17 +53,11 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define MAX_CORES_PER_SOCKET   64
 #define MAX_VCPU_SOCKETS      128

-#ifdef VMM
-#include "vcpuset.h"
-
-/* In VMM, CurVcpuid() is available everywhere. */
-extern const Vcpuid      curVcpuid;
-extern const VCPUSet     curVcpuidSet;
-#define CurVcpuid()         (curVcpuid)
-#define CurVcpuidSet()      (&curVcpuidSet)
-#define IS_BOOT_VCPU()      IS_BOOT_VCPUID(CurVcpuid())
-
-#endif  /* VMM */
+/*
+ * There are several properties of the VM which change at the 128 VCPU
+ * boundary.  EFI firmware, x2APIC, and IOMMU are required among others.
+ */
+#define MAX_SMALL_VM_VCPUS 128

 #if defined __cplusplus
 } // extern "C"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -112,7 +112,7 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET(_setIndex)                                     \
   do {                                                                       \
-      int _setIndex;                                                          \
+      unsigned _setIndex;                                                     \
      for (_setIndex = 0; _setIndex < VCS_SUBSET_COUNT; _setIndex++) {

 #define ROF_EACH_SUBSET_IN_SET()                                              \
@@ -122,8 +122,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET_COUNTDOWN(_setIndex)                           \
   do {                                                                       \
-      int _setIndex;                                                          \
-      for (_setIndex = VCS_SUBSET_COUNT - 1; _setIndex >= 0; _setIndex--) {
+      unsigned _setIndex = VCS_SUBSET_COUNT;                                  \
+      while (_setIndex-- > 0) {

 #define ROF_EACH_SUBSET_IN_SET_COUNTDOWN()                                    \
      }                                                                       \
@@ -132,8 +132,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex, _numVcpus)                \
   do {                                                                       \
-      int _setIndex;                                                          \
-      int _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);                 \
+      unsigned _setIndex;                                                     \
+      unsigned _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);            \
      for (_setIndex = 0; _setIndex <= _maxSubsets; _setIndex++) {

 #define ROF_EACH_POPULATED_SUBSET_IN_SET()                                    \
@@ -307,7 +307,7 @@ VCPUSet_FindFirst(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return lssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return lssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET();
   return VCPUID_INVALID;
@@ -319,7 +319,7 @@ VCPUSet_FindLast(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET_COUNTDOWN(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return mssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return mssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET_COUNTDOWN();
   return VCPUID_INVALID;
@@ -356,8 +356,7 @@ VCPUSet_FindFirstInSubset(const VCPUSet *vcs, uint64 *subset,
   ASSERT(*subsetIdx < maxSubsets && maxSubsets <= VCS_SUBSET_COUNT);
   do {
      if (*subset != 0) {
-         int bit;
-         bit = lssb64_0(*subset);
+         unsigned bit = (unsigned)lssb64_0(*subset);
         *subset &= ~(CONST64U(1) << bit);
         return bit + (*subsetIdx << VCS_SUBSET_SHIFT);
      }
@@ -672,7 +671,7 @@ static INLINE Vcpuid
 VCPUSet_FindSingleton(const VCPUSet *vcs)
 {
   uint64 foundSub = 0;
-   uint32 foundIdx;
+   uint32 foundIdx = 0;
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 sub = vcs->subset[idx];
      if (sub != 0) {
@@ -683,8 +682,11 @@ VCPUSet_FindSingleton(const VCPUSet *vcs)
         foundIdx = idx;
      }
   } ROF_EACH_SUBSET_IN_SET();
-   return foundSub != 0 ? lssb64_0(foundSub) + (foundIdx << VCS_SUBSET_SHIFT) :
-                          VCPUID_INVALID;
+   if (foundSub != 0) {
+      return lssb64(foundSub) - 1 + (foundIdx << VCS_SUBSET_SHIFT);
+   } else {
+      return VCPUID_INVALID;
+   }
 }


@@ -863,27 +865,27 @@ VCPUSet_Intersection(VCPUSet *dest, const VCPUSet *src)

 #ifdef VCS_SNPRINTF
 static INLINE char *
-VCPUSet_LogFormat(char *buf, const int size, const VCPUSet *vcs)
+VCPUSet_LogFormat(char *buf, size_t size, const VCPUSet *vcs)
 {
-   int offset = 0;
-   Vcpuid highest = VCPUSet_FindLast(vcs);
-   int idx = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+   unsigned offset  = 0;
+   Vcpuid   highest = VCPUSet_FindLast(vcs);
+   unsigned idx     = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+
   ASSERT(size >= VCS_BUF_SIZE);
 #define VCS_LOGF(...)                                                   \
   {                                                                    \
      int ret = VCS_SNPRINTF(buf + offset, size - offset, __VA_ARGS__); \
-      ASSERT(0 <= ret && ret < size - offset);                          \
-      offset += ret;                                                    \
+      ASSERT(0 <= ret && size >= offset && ret < (int)(size - offset)); \
+      offset += (unsigned)ret;                                          \
   }
   /* Print the leading value with no zero-extension. */
-   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx--]);
+   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx]);

-   while (idx >= 0) {
+   while (idx-- > 0) {
      if ((idx + 1) % (VCS_SUBSET_WIDTH / 8) == 0) {
         VCS_LOGF(".");
      }
      VCS_LOGF("%02x", ((unsigned char *)vcs)[idx]);
-      idx--;
   }
   return buf;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,52 +41,47 @@

 #include "vm_basic_asm.h"
 #include "x86msr.h"
-
-#ifdef VM_X86_64
-#include "vm_asm_x86_64.h"
-#else
 #include "vm_asm_x86.h"
-#endif


 static INLINE void
 SET_FS64(uint64 fs64)
 {
-   __SET_MSR(MSR_FSBASE, fs64);
+   X86MSR_SetMSR(MSR_FSBASE, fs64);
 }


 static INLINE void
 SET_GS64(uint64 gs64)
 {
-   __SET_MSR(MSR_GSBASE, gs64);
+   X86MSR_SetMSR(MSR_GSBASE, gs64);
 }

 static INLINE void
 SET_KernelGS64(uint64 kgs64)
 {
-   __SET_MSR(MSR_KERNELGSBASE, kgs64);
+   X86MSR_SetMSR(MSR_KERNELGSBASE, kgs64);
 }


 static INLINE uint64
 GET_FS64(void)
 {
-   return __GET_MSR(MSR_FSBASE);
+   return X86MSR_GetMSR(MSR_FSBASE);
 }


 static INLINE uint64
 GET_GS64(void)
 {
-   return __GET_MSR(MSR_GSBASE);
+   return X86MSR_GetMSR(MSR_GSBASE);
 }


 static INLINE uint64
 GET_KernelGS64(void)
 {
-   return __GET_MSR(MSR_KERNELGSBASE);
+   return X86MSR_GetMSR(MSR_KERNELGSBASE);
 }

 #endif // VM_ARM_64
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * vm_asm_x86.h
 *
- *	IA32 asm macros
+ *      IA32 asm macros
 */

 #ifndef _VM_ASM_X86_H_
@@ -49,10 +49,6 @@

 #ifdef __GNUC__

-#if defined __APPLE__
-/* PR 352418: GCC produces error if the non-Apple version is used */
-#define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
-#else
 /* ASSERT_ON_COMPILE_SELECTOR_SIZE:
 *
 *  - Selector must be 16-bits.
@@ -66,26 +62,16 @@
 *                    ((__builtin_constant_p(expr) ? ((expr) >> 16) == 0) \
 *                                                 : sizeof(expr) <= 2)
 */
-#if (__GNUC__ >= 4) && (__GNUC_MINOR__ >= 1) && !defined(USE_UBSAN)
+#ifndef USE_UBSAN
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)                                \
   ASSERT_ON_COMPILE(sizeof(Selector) == 2 &&                                \
                     ((__builtin_constant_p(expr) && ((expr) >> 16) == 0) || \
                      sizeof(expr) <= 2))
 #else
-/* gcc 3.3.3 is not able to produce a constant expression (PR 356383) */
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
 #endif
-#endif


-/*
- * [GS]ET_[GI]DT() are defined as macros wrapping a function
- * so we can pass the argument implicitly by reference (requires
- * a macro) and get type checking too (requires a function).
- */
-
-#define SET_GDT(var) _Set_GDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
@@ -97,8 +83,6 @@ _Set_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define SET_IDT(var) _Set_IDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_IDT(_GETSET_DTR_TYPE *dtr)
@@ -110,8 +94,6 @@ _Set_IDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_GDT(var) _Get_GDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because there's a hidden input (the [IG]DTR) that can change
@@ -126,8 +108,6 @@ _Get_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_IDT(var) _Get_IDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because the [IG]DT can change without the compiler knowing it
@@ -218,22 +198,6 @@ _BUILD_GET_R(_GET_CR3, cr3)
 _BUILD_GET_R(_GET_CR4, cr4)
 _BUILD_GET_R(_GET_CR8, cr8)

-#if defined __APPLE__
-/* Mac OS gcc 4 uses DBx instead of DRx register names. */
-_BUILD_SET_R(_SET_DR0, db0)
-_BUILD_SET_R(_SET_DR1, db1)
-_BUILD_SET_R(_SET_DR2, db2)
-_BUILD_SET_R(_SET_DR3, db3)
-_BUILD_SET_R(_SET_DR6, db6)
-_BUILD_SET_R(_SET_DR7, db7)
-
-_BUILD_GET_R(_GET_DR0, db0)
-_BUILD_GET_R(_GET_DR1, db1)
-_BUILD_GET_R(_GET_DR2, db2)
-_BUILD_GET_R(_GET_DR3, db3)
-_BUILD_GET_R(_GET_DR6, db6)
-_BUILD_GET_R(_GET_DR7, db7)
-#else
 _BUILD_SET_R(_SET_DR0, dr0)
 _BUILD_SET_R(_SET_DR1, dr1)
 _BUILD_SET_R(_SET_DR2, dr2)
@@ -247,47 +211,8 @@ _BUILD_GET_R(_GET_DR2, dr2)
 _BUILD_GET_R(_GET_DR3, dr3)
 _BUILD_GET_R(_GET_DR6, dr6)
 _BUILD_GET_R(_GET_DR7, dr7)
-#endif

-#define SET_CR_DR(regType, regNum, expr)                     \
-   do {                                                      \
-      /* Ensure no implicit truncation of 'expr' */          \
-      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
-      _SET_##regType##regNum(expr);                          \
-   } while (0)
-
-#define GET_CR_DR(regType, regNum, var) \
-   do {                                 \
-      _GET_##regType##regNum(&(var));   \
-   } while (0)
-
-#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
 #define SET_CR2(expr) SET_CR_DR(CR, 2, expr)
-#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
-#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
-#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
-
-/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
-#undef GET_CR0
-#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
-#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
-#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
-#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
-#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
-
-#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
-#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
-#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
-#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
-#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
-#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
-
-#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
-#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
-#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
-#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
-#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
-#define GET_DR7(var)  GET_CR_DR(DR, 7, var)

 #define SET_SEGREG(reg, expr)                                          \
   do {                                                                \
@@ -426,440 +351,347 @@ CLTS(void)
 /* Beginning of the section whose correctness has NOT been checked */
 #define FNCLEX()         __asm__("fnclex" ::);

-/* TLB_INVALIDATE_xxx are not checked yet */
+/* TLB_INVALIDATE_PAGE is not checked yet */
 #define TLB_INVALIDATE_PAGE(_addr) do { \
     __asm__ __volatile__("invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
 } while (0)

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) do { \
-     __asm__ __volatile__("fs; invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
-} while (0)
-
 #define RESTORE_FLAGS _Set_flags
 #define ENABLE_INTERRUPTS() __asm__ __volatile__ ("sti": : :"memory")
 #define CLEAR_INTERRUPTS()  __asm__ __volatile__ ("cli": : :"memory")
 #define RAISE_INTERRUPT(_x)  __asm__ __volatile__("int %0" :: "g" (_x))
 #define RETURN_FROM_INT()   __asm__ __volatile__("iret" :: )

-#if ! defined(VMKERNEL)
-#define NO_INTERRUPTS_BEGIN()	do { \
-                                   uintptr_t _flags; \
-                                   SAVE_FLAGS(_flags); \
-                                   CLEAR_INTERRUPTS();
-
-#define NO_INTERRUPTS_END()	   RESTORE_FLAGS(_flags); \
-                                } while(0)
-#endif
-
 /* End of the section whose correctness has NOT been checked */

 #elif defined _MSC_VER  /* !__GNUC__ */

-#define SET_IDT(_idt)    _Set_IDT(&(_idt))
-#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define _BUILD_SET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writedr(reg, r);                              \
+   }
+
+#define _BUILD_GET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t *result)                            \
+   {                                                  \
+      *result = __readdr(reg);                        \
+   }
+
+#define _BUILD_SET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writecr##reg(r);                              \
+   }
+
+#define _BUILD_GET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t *result)                            \
+   {                                                  \
+      *result = __readcr##reg();                      \
+   }
+
+_BUILD_SET_DR(_SET_DR0, 0)
+_BUILD_SET_DR(_SET_DR1, 1)
+_BUILD_SET_DR(_SET_DR2, 2)
+_BUILD_SET_DR(_SET_DR3, 3)
+_BUILD_SET_DR(_SET_DR6, 6)
+_BUILD_SET_DR(_SET_DR7, 7)
+
+_BUILD_GET_DR(_GET_DR0, 0)
+_BUILD_GET_DR(_GET_DR1, 1)
+_BUILD_GET_DR(_GET_DR2, 2)
+_BUILD_GET_DR(_GET_DR3, 3)
+_BUILD_GET_DR(_GET_DR6, 6)
+_BUILD_GET_DR(_GET_DR7, 7)
+
+_BUILD_SET_CR(_SET_CR0, 0);
+_BUILD_SET_CR(_SET_CR3, 3);
+_BUILD_SET_CR(_SET_CR4, 4);
+_BUILD_SET_CR(_SET_CR8, 8);
+
+_BUILD_GET_CR(_GET_CR0, 0);
+_BUILD_GET_CR(_GET_CR2, 2);
+_BUILD_GET_CR(_GET_CR3, 3);
+_BUILD_GET_CR(_GET_CR4, 4);
+_BUILD_GET_CR(_GET_CR8, 8);
+
+static INLINE void
+_Set_GDT(_GETSET_DTR_TYPE *dtr)
+{
+#ifdef VM_X86_64
+   _lgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _lgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm lgdt [eax]
+#endif
+}
+
+static INLINE void
+_Get_GDT(_GETSET_DTR_TYPE *dtr)
+{
+#ifdef VM_X86_64
+   _sgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _sgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm sgdt [eax]
+#endif
+}
+
+static INLINE void
+_Set_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __lidt(dtr);
+}
+
+static INLINE void
+_Get_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __sidt(dtr);
+}
+
+
+#define ENABLE_INTERRUPTS() _enable()
+#define CLEAR_INTERRUPTS()  _disable()
+
+#define SAVE_FLAGS(x) do { \
+   (x) = __readeflags();   \
+} while (0)
+
+#define RESTORE_FLAGS(x) __writeeflags(x)
+
+
+#ifdef VM_X86_32
+
 #define SET_TR(_tr)      _Set_TR(_tr)
 #define SET_LDT(_tr)     _Set_LDT(_tr)

-#define GET_IDT(_idt)    _Get_IDT(&(_idt))
-#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
 #define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
 #define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)

-#define GET_CR0(_reg)    __asm mov eax, cr0 __asm mov _reg, eax
-#define SET_CR0(_reg)    __asm mov eax, _reg __asm mov cr0, eax
-#define GET_CR2(_reg)    __asm mov eax, cr2 __asm mov _reg, eax
 #define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
-#define GET_CR3(_reg)    __asm mov eax, cr3 __asm mov _reg, eax
-#define SET_CR3(_reg)    __asm mov eax, _reg __asm mov cr3, eax
-/*
- * MSC doesn't seem to like CR4 in __asm statements. We emit
- * the opcode for MOV EAX,CR4 = 0xf020e0 and MOV CR4,EAX = 0xf022e0
- */
-#define GET_CR4(_reg) { \
- __asm _emit 0x0f __asm _emit 0x20 __asm _emit 0xe0 \
- __asm mov _reg, eax \
-}
-#define SET_CR4(_reg)    { \
-  __asm mov eax, _reg \
-  __asm _emit 0x0f __asm _emit 0x22 __asm _emit 0xe0 \
-}
-
-
-#define GET_DR0(_reg) do { __asm mov eax,dr0 __asm mov _reg,eax } while (0)
-#define SET_DR0(_reg) do { __asm mov eax,_reg __asm mov dr0,eax } while (0)
-#define GET_DR1(_reg) do { __asm mov eax,dr1 __asm mov _reg,eax } while (0)
-#define SET_DR1(_reg) do { __asm mov eax,_reg __asm mov dr1,eax } while (0)
-#define GET_DR2(_reg) do { __asm mov eax,dr2 __asm mov _reg,eax } while (0)
-#define SET_DR2(_reg) do { __asm mov eax,_reg __asm mov dr2,eax } while (0)
-#define GET_DR3(_reg) do { __asm mov eax,dr3 __asm mov _reg,eax } while (0)
-#define SET_DR3(_reg) do { __asm mov eax,_reg __asm mov dr3,eax } while (0)
-#define GET_DR6(_reg) do { __asm mov eax,dr6 __asm mov _reg,eax } while (0)
-#define SET_DR6(_reg) do { __asm mov eax,_reg __asm mov dr6,eax } while (0)
-#define GET_DR7(_reg) do { __asm mov eax,dr7 __asm mov _reg,eax } while (0)
-#define SET_DR7(_reg) do { __asm mov eax,_reg __asm mov dr7,eax } while (0)
-
-
 #define CLTS()           __asm clts

 #define FNCLEX()         __asm fnclex

 #define TLB_INVALIDATE_PAGE(_addr) {  \
-	 void *_a = (_addr); \
-     __asm mov eax, _a __asm invlpg [eax] \
+   void *_a = (_addr); \
+   __asm mov eax, _a __asm invlpg [eax] \
 }

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) { \
-	uint32 __a = (uint32) (_addr); \
-	__asm mov eax, __a _asm invlpg fs:[eax] \
-}
-
-
-#define ENABLE_INTERRUPTS() { __asm sti }
-#define CLEAR_INTERRUPTS()  { __asm cli }
-
 #define RAISE_INTERRUPT(_x)  {__asm int _x }
 #define RETURN_FROM_INT()   {__asm iretd }


-#define SAVE_FLAGS(x) { \
-     __asm pushfd __asm pop eax __asm mov x, eax \
-}
-
-#define RESTORE_FLAGS(x) { \
-     __asm push x __asm popfd\
-}
-
-
-
-static INLINE void SET_DS(Selector val)
+static INLINE void
+SET_DS(Selector val)
 {
   __asm mov ax, val
   __asm mov ds, ax
 }

-static INLINE void SET_ES(Selector val)
+static INLINE void
+SET_ES(Selector val)
 {
   __asm mov ax, val
   __asm mov es, ax
 }

-static INLINE void SET_FS(Selector val)
+static INLINE void
+SET_FS(Selector val)
 {
   __asm mov ax, val
   __asm mov fs, ax
 }

-static INLINE void SET_GS(Selector val)
+static INLINE void
+SET_GS(Selector val)
 {
   __asm mov ax, val
   __asm mov gs, ax
 }

-static INLINE void SET_SS(Selector val)
+static INLINE void
+SET_SS(Selector val)
 {
   __asm mov ax, val
   __asm mov ss, ax
 }

-static INLINE Selector GET_FS(void)
+static INLINE Selector
+GET_FS(void)
 {
-	Selector _v;
-	__asm mov _v,fs
-	return _v;
+   Selector _v;
+   __asm mov _v,fs
+   return _v;
 }

-static INLINE Selector GET_GS(void)
+static INLINE Selector
+GET_GS(void)
 {
-	Selector _v;
-	__asm mov _v,gs
-	return _v;
+   Selector _v;
+   __asm mov _v,gs
+   return _v;
 }

-
-static INLINE Selector GET_DS(void)
+static INLINE Selector
+GET_DS(void)
 {
-	Selector _v;
-	__asm mov _v,ds
-	return _v;
+   Selector _v;
+   __asm mov _v,ds
+   return _v;
 }

-static INLINE Selector GET_ES(void)
+static INLINE Selector
+GET_ES(void)
 {
-	Selector _v;
-	__asm mov _v,es
-	return _v;
+   Selector _v;
+   __asm mov _v,es
+   return _v;
 }

-static INLINE Selector GET_SS(void)
+static INLINE Selector
+GET_SS(void)
 {
-	Selector _v;
-	__asm mov _v,ss
-	return _v;
+   Selector _v;
+   __asm mov _v,ss
+   return _v;
 }

-static INLINE Selector GET_CS(void)
+static INLINE Selector
+GET_CS(void)
 {
-	Selector _v;
-	__asm mov _v,cs
-	return _v;
+   Selector _v;
+   __asm mov _v,cs
+   return _v;
 }

-#pragma warning( disable : 4035)
-
-static INLINE uint32  GET_WORD_FROM_FS(uint32 *_addr) {
-	__asm mov eax, _addr
-    __asm mov eax, fs:[eax]
-}
-
-static INLINE uint16  GET_SHORT_FROM_FS(uint16 *_addr) {
-	__asm mov eax, _addr
-    __asm mov ax, fs:[eax]
-}
-
-static INLINE uint8  GET_BYTE_FROM_FS(uint8 *_addr) {
-	__asm mov eax, _addr
-     __asm mov al, fs:[eax]
-}
-
-#pragma warning (default: 4035)
-
-static INLINE void  SET_WORD_FS(uint32 *_addr, uint32 _val) {
-    __asm mov eax, _addr
-    __asm mov ebx, _val
-    __asm mov fs:[eax], ebx
-}
-
-static INLINE void  SET_SHORT_FS(uint32 *_addr, uint16 _val) {
-    __asm mov eax, _addr
-    __asm mov bx, _val
-    __asm mov fs:[eax], bx
-}
-
-static INLINE void  SET_BYTE_FS(uint32 *_addr, uint8 _val) {
-    __asm mov eax, _addr
-    __asm mov bl, _val
-    __asm mov fs:[eax], bl
-}
-
-static INLINE void _Set_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lgdt [eax]
-}
-
-static INLINE void _Set_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lidt [eax]
-}
-
-static INLINE void _Set_LDT(Selector val)
+static INLINE void
+_Set_LDT(Selector val)
 {
   __asm lldt val
 }

-static INLINE void _Set_TR(Selector val)
+static INLINE void
+_Set_TR(Selector val)
 {
   __asm ltr val
 }

-static INLINE void _Get_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sgdt [eax]
-}
-
-static INLINE void _Get_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sidt [eax]
-}
-
-static INLINE Selector _Get_LDT(void) {
+static INLINE Selector
+_Get_LDT(void)
+{
   Selector sel;
   __asm sldt sel
   return sel;
 }

-static INLINE Selector _Get_TR(void) {
+static INLINE Selector
+_Get_TR(void)
+{
   Selector sel;
   __asm str sel
   return sel;
 }

-
-static INLINE void
-MEMCOPY_TO_FS(VA to,
-              char * from,
-              unsigned long n)
-{
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x = *(uint32*) (from + i);
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, x
-	  __asm mov fs:[eax], ebx
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x = from[i];
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov bl, x
-	  __asm mov fs:[eax], bl
-      i++;
-   }
-}
-
-
-
-static INLINE void
-MEMCOPY_FROM_FS(char * to,
-                VA from,
-                unsigned long n)
-{
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x;
-	  uint32 _faddr = (uint32)(from+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, fs:[eax]
-	  __asm mov x,ebx
-      *(uint32*)(to+i)=x;
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x;
-	  uint32 _faddr = (uint32) (from+i);
-	  __asm mov eax, _faddr;
-      __asm mov bl, fs:[eax]
-	  __asm mov x, bl
-      *(uint8*)(to+i)=x;
-      i++;
-   }
-}
-
-#else
-#error No compiler defined for get/set
+#endif /* !VM_X86_32 */
 #endif /* !__GNUC__ && !_MSC_VER */


-#ifdef __GNUC__
-static INLINE unsigned CURRENT_CPL(void)
+#define SET_CR_DR(regType, regNum, expr)                     \
+   do {                                                      \
+      /* Ensure no implicit truncation of 'expr' */          \
+      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
+      _SET_##regType##regNum(expr);                          \
+   } while (0)
+
+#define GET_CR_DR(regType, regNum, var) \
+   do {                                 \
+      _GET_##regType##regNum(&(var));   \
+   } while (0)
+
+#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
+#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
+#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
+#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
+#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
+#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
+
+#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
+#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
+#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
+#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
+#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
+#define GET_DR7(var)  GET_CR_DR(DR, 7, var)
+
+/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
+#undef GET_CR0
+#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
+#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
+#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
+#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
+#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
+
+#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
+#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
+#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
+#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
+
+static INLINE Bool
+INTERRUPTS_ENABLED(void)
+{
+   uintptr_t flags;
+   SAVE_FLAGS(flags);
+   return ((flags & EFLAGS_IF) != 0);
+}
+
+/*
+ * [GS]ET_[GI]DT() are defined as macros wrapping a function
+ * so we can pass the argument implicitly by reference (requires
+ * a macro) and get type checking too (requires a function).
+ */
+
+#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
+
+#define SET_IDT(_idt)    _Set_IDT(&(_idt))
+#define GET_IDT(_idt)    _Get_IDT(&(_idt))
+
+#if !defined(VMKERNEL)
+#define NO_INTERRUPTS_BEGIN() do { \
+                                   uintptr_t _flags; \
+                                   SAVE_FLAGS(_flags); \
+                                   CLEAR_INTERRUPTS();
+
+#define NO_INTERRUPTS_END()        RESTORE_FLAGS(_flags); \
+                                 } while(0)
+#endif
+
+
+#if defined (__GNUC__) || (defined (_MSC_VER) && defined (VM_X86_32))
+static INLINE unsigned
+CURRENT_CPL(void)
 {
   return SELECTOR_RPL(GET_CS());
 }
-#elif defined _MSC_VER
-static INLINE unsigned CURRENT_CPL(void) {
-   volatile Selector _v;
-   __asm mov ax, cs _asm mov _v, ax
-   return SELECTOR_RPL(_v);
-}
-#else
-#error No compiler defined for CURRENT_CPL
 #endif


+static INLINE uint64
+RDPMC(int counter)
+{
 #ifdef __GNUC__
-/* Checked against the Intel manual and GCC --hpreg
-
-   volatile because the msr can change without the compiler knowing it
-   (when we use wrmsr). */
-static INLINE uint64 __GET_MSR(int cx)
-{
-   uint64 msr;
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
-      : "=a" (msr)
-      : "c" (cx)
-      : "%rdx"
-   );
-#else
-   __asm__ __volatile__(
-      "rdmsr"
-      : "=A" (msr)
-      : "c" (cx)
-   );
-#endif
-   return msr;
-}
-#elif defined _MSC_VER
-#pragma warning( disable : 4035)
-static INLINE uint64 __GET_MSR(int input)
-{
-   __asm push ecx
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x32
-   __asm pop ecx
-}
-
-static INLINE void __SET_MSR(int input, uint64 val)
-{
-   uint32 hival = (uint32)(val >> 32);
-   uint32 loval = (uint32)val;
-   __asm push edx
-   __asm push ecx
-   __asm push eax
-   __asm mov  eax, loval
-   __asm mov  edx, hival
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x30
-   __asm pop eax
-   __asm pop ecx
-   __asm pop edx
-}
-#pragma warning (default: 4035)
-#else
-#error
-#endif
-
-#ifdef __GNUC__
-static INLINE void __SET_MSR(int cx, uint64 val)
-{
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "a" ((uint32) val), "d" ((uint32)(val >> 32)), "c" (cx)
-    );
-#else
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "A" (val),
-        "c" (cx)
-    );
-#endif
-}
-#endif
-
-
-/*
- * RDMSR/WRMSR access the 64bit MSRs as two
- * 32 bit quantities, whereas GET_MSR/SET_MSR
- * above access the MSRs as one 64bit quantity.
- */
-#ifdef __GNUC__
-#undef RDMSR
-#undef WRMSR
-#define RDMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "rdmsr"                         \
-      : "=a" (low), "=d" (high)       \
-      : "c" (msrNum)                  \
-   );                                 \
-} while (0)
-
-#define WRMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "wrmsr"                         \
-      : /* no outputs */              \
-      : "c" (msrNum),                 \
-        "a" (low),                    \
-        "d" (high)                    \
-    );                                \
-} while (0)
-
-static INLINE uint64 RDPMC(int cx)
-{
 #ifdef VM_X86_64
   uint64 pmcLow;
   uint64 pmcHigh;
@@ -867,7 +699,7 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=a" (pmcLow), "=d" (pmcHigh)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmcHigh << 32 | pmcLow;
@@ -877,43 +709,26 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=A" (pmc)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmc;
 #endif
-}
 #elif defined _MSC_VER
-#ifndef VM_X86_64 // XXX Switch to intrinsics with the new 32 and 64-bit compilers.
-
-static INLINE uint64 RDPMC(int counter)
-{
-   __asm mov ecx, counter
-   __asm rdpmc
+   return __readpmc(counter);
+#endif
 }

-static INLINE void WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   __asm mov ecx, msrNum
-   __asm mov eax, lo
-   __asm mov edx, hi
-   __asm wrmsr
-}
-#endif
-#endif

-
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
-static INLINE uint64 __XGETBV(int cx)
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined (ULM)
+static INLINE uint64
+__XGETBV(int cx)
 {
+#ifdef __GNUC__
 #ifdef VM_X86_64
   uint64 lowval, hival;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=a" (lowval), "=d" (hival)
      : "c" (cx)
   );
@@ -921,39 +736,41 @@ static INLINE uint64 __XGETBV(int cx)
 #else
   uint64 val;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=A" (val)
      : "c" (cx)
   );
   return val;
 #endif
+#elif defined _MSC_VER
+   return _xgetbv((unsigned)cx);
+#endif
 }

-static INLINE void __XSETBV(int cx, uint64 val)
+static INLINE void
+__XSETBV(int cx, uint64 val)
 {
+#ifdef __GNUC__
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd1"
-#else
      "xsetbv"
-#endif
      : /* no outputs */
      : "a" ((uint32)val), "d" ((uint32)(val >> 32)), "c" (cx)
    );
+#elif defined _MSC_VER
+   _xsetbv((unsigned)cx, val);
+#endif
 }

-static INLINE uint64 GET_XCR0(void)
+static INLINE uint64
+GET_XCR0(void)
 {
   return __XGETBV(0);
 }

 #define SET_XCR0(val) __XSETBV(0, val)

-static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
+static INLINE void
+SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 {
 #ifndef VMM_BOOTSTRAP
   ASSERT(oldVal == GET_XCR0());
@@ -965,33 +782,45 @@ static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 #endif


-#define START_TRACING() { \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags |= EFLAGS_TF;    \
-   RESTORE_FLAGS(flags);  \
-}
-
-#define STOP_TRACING() {  \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags &= ~EFLAGS_TF;   \
-   RESTORE_FLAGS(flags);  \
-}
-
-
-static INLINE Bool
-INTERRUPTS_ENABLED(void)
+static INLINE uint32
+RDTSCP_AuxOnly(void)
 {
-   uintptr_t flags;
-   SAVE_FLAGS(flags);
-   return ((flags & EFLAGS_IF) != 0);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return tscAux;
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   __rdtscp(&tscAux);
+
+   return tscAux;
+#endif
 }

-static INLINE void
-SET_KERNEL_PER_CORE(uint64 val)
+
+static INLINE uint64
+RDTSCP(void)
 {
-   __SET_MSR(MSR_GSBASE, val);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      :"=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return QWORD(tscHigh, tscLow);
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   return __rdtscp(&tscAux);
+#endif
 }

 #endif
@@ -1,137 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * vm_asm_x86_64.h
- *
- *	x86-64 asm macros
- */
-
-#ifndef _VM_ASM_X86_64_H_
-#define _VM_ASM_X86_64_H_
-
-
-#define INCLUDE_ALLOW_VMX
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_DISTRIBUTE
-#include "includeCheck.h"
-
-#ifdef __GNUC__
-#include "vm_asm_x86.h"
-#endif
-
-/*
- * This file contains inline assembly routines used by x86_64 code.
- */
-
-#ifdef __GNUC__
-
-  /* nop; prevents #error for no compiler definition from firing */
-
-#elif defined _MSC_VER  /* !__GNUC__ */
-
-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-
-#ifdef _WIN64
-#ifdef __cplusplus
-extern "C" {
-#endif
-unsigned __int64  __readmsr(unsigned long);
-void              __writemsr(unsigned long, unsigned __int64);
-#pragma intrinsic(__readmsr, __writemsr)
-#ifdef __cplusplus
-}
-#endif
-
-
-static INLINE uint64
-RDPMC(int counter)
-{
-   return __readpmc(counter);
-}
-
-
-static INLINE void
-WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   uint64 value = QWORD(hi, lo);
-   __writemsr(msrNum, value);
-}
-
-
-static INLINE uint64
-__GET_MSR(int input)
-{
-   return __readmsr((unsigned long)input);
-}
-
-
-static INLINE void
-__SET_MSR(int cx, uint64 val)
-{
-  __writemsr((unsigned long)cx, (unsigned __int64)val);
-}
-
-#endif
-
-#else
-#error No compiler defined for get/set
-#endif /* !__GNUC__ && !_MSC_VER */
-
-
-#ifdef __GNUC__
-static INLINE void
-SWAPGS(void)
-{
-   __asm__ __volatile__("swapgs");
-}
-
-
-static INLINE uint64
-RDTSCP_AuxOnly(void)
-{
-   uint64 tscLow, tscHigh, tscAux;
-
-   __asm__ __volatile__(
-      "rdtscp"
-      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
-   );
-
-   return tscAux;
-}
-#endif
-
-#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -202,7 +202,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1317,7 +1317,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
+   __asm lock And [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_And
@@ -1382,7 +1382,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
+   __asm lock Or [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Or
@@ -1447,7 +1447,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Xor
@@ -2303,7 +2303,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2324,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3618,7 +3621,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -221,8 +221,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +263,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +376,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +392,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -768,12 +771,14 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
@@ -888,7 +893,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +907,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1103,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1161,101 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,12 +49,29 @@ uint64 _umul128(uint64 multiplier, uint64 multiplicand,
 int64 _mul128(int64 multiplier, int64 multiplicand,
              int64 *highProduct);
 uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
+#ifdef ULM
+void _fxsave64(void *save);
+void _fxsave(void *save);
+void _fxrstor64(const void *load);
+void _fxrstor(const void *load);
+void _xsave64(void *save, uint64 mask);
+void _xsave(void *save, uint64 mask);
+void _xsaveopt64(void *save, uint64 mask);
+void _xsavec(void *save, uint64 mask);
+void _xrstor64(const void *load, uint64 mask);
+void _xrstor(const void *load, uint64 mask);
+#endif /* ULM */
 #ifdef __cplusplus
 }
 #endif

 #pragma intrinsic(_umul128, _mul128, __shiftright128)

+#ifdef ULM
+#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
+                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
+#endif /* ULM */
+
 #endif // _MSC_VER

 #if defined(__GNUC__)
@@ -113,40 +130,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +204,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +307,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +351,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +433,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +518,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +542,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -244,9 +244,131 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+#if defined VM_X86_32
+   __asm sfence;
+#else
+   _mm_sfence();
+#endif
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +400,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +515,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +533,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +613,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,12 +44,44 @@
 #endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -58,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -159,18 +157,6 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

-/*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-

 /*
 * Token concatenation
@@ -209,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -255,6 +241,10 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif
@@ -286,6 +276,14 @@ Max(int a, int b)
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -556,10 +554,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -664,6 +662,20 @@ typedef int pid_t;
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
@@ -702,12 +714,6 @@ typedef int pid_t;
 #endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -718,7 +724,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,18 +63,9 @@
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/*
- * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -83,7 +74,6 @@
 #ifdef __x86_64__
 #define VM_X86_64
 #define vm_x86_64 1
-#define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
 #else
@@ -120,14 +110,91 @@

 #endif

-#if defined(__cplusplus) && __cplusplus >= 201103L || \
-    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
-    defined(__APPLE__) || defined(HAVE_STDINT_H)
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif
+
 /*
- * We're using <stdint.h> instead of <cstdint> below because some C++ code
- * deliberately compiles without C++ include paths.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif
+

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -138,40 +205,6 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-#else /* !HAVE_STDINT_H */
-
-/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
-#ifdef _MSC_VER
-
-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
-
-#elif __GNUC__
-#   if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#   else
-/*
- * Only strict c90 (without extensions) lacks a 'long long' type.
- * If this declaration fails ... use -std=c99 or -std=gnu90.
- */
-typedef unsigned long long uint64;
-typedef long long int64;
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-

 /*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
@@ -202,14 +235,6 @@ typedef char           Bool;
 #define IS_BOOL(x)     (((x) & ~1) == 0)


-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */
-
 /*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
@@ -218,9 +243,6 @@ typedef char           Bool;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -234,14 +256,11 @@ typedef char           Bool;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -249,46 +268,10 @@ typedef char           Bool;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -337,10 +320,11 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* macOS hosts use the same formatters for 32- and 64-bit. */
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
   #define FMT64         "ll"
-   #if KERNEL
+   #if defined(__APPLE__) && KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
@@ -452,7 +436,6 @@ typedef uintptr_t VPN;

 typedef uint64    PA;
 typedef uint64    PPN;
-typedef uint32    PPNTMP;

 typedef uint64    TPA;
 typedef uint64    TPPN;
@@ -470,9 +453,7 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint64    PgCnt64;
 typedef uint64    PageCnt;
-typedef uint64    PgNum64;
 typedef uint64    PageNum;
 typedef unsigned  MemHandle;
 typedef unsigned  IoHandle;
@@ -515,7 +496,7 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
@@ -598,15 +579,14 @@ typedef void * UserVA;


 /* Maximal observable PPN value. */
-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
 #define INVALID_PPN32     ((PPN32)0xffffffff)
-#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
-#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -624,7 +604,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((uint32)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -837,7 +818,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -934,82 +915,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,22 +40,25 @@
 /*
 * These definitions suit both the x64 and arm64 architectures. In particular
 * - At all levels, a page table is a 4KB page containing 512 8-byte entries.
- * - Each entry maps 39, 30, 21 and 12 bits at the respective page table
+ * - Each entry maps 48, 39, 30, 21 and 12 bits at the respective page table
 *   levels (with standard page sizes).
 */

-typedef uint64 PT_L4E;
-typedef uint64 PT_L3E;
-typedef uint64 PT_L2E;
-typedef uint64 PT_L1E;
+typedef uint64 PT_Entry;
+typedef PT_Entry PT_L5E;
+typedef PT_Entry PT_L4E;
+typedef PT_Entry PT_L3E;
+typedef PT_Entry PT_L2E;
+typedef PT_Entry PT_L1E;

 typedef enum {
   PT_LEVEL_1 = 1,
   PT_LEVEL_2,
   PT_LEVEL_3,
   PT_LEVEL_4,
+   PT_LEVEL_5,
   PT_LEVEL_STOP = PT_LEVEL_1,
-   PT_MAX_LEVELS = PT_LEVEL_4
+   PT_MAX_LEVELS = PT_LEVEL_5
 } PT_Level;

 #define PT_PTE_SIZE           8
@@ -76,6 +79,7 @@ typedef enum {

 /* Address space size (in 4KB pages) covered by a level-_l page table entry. */
 #define PT_PAGES_PER_LE(_l)   ((uint64)1 << PT_LE_PG_SHIFT(_l))
+#define PT_PAGES_PER_L5E      PT_PAGES_PER_LE(PT_LEVEL_5)
 #define PT_PAGES_PER_L4E      PT_PAGES_PER_LE(PT_LEVEL_4)
 #define PT_PAGES_PER_L3E      PT_PAGES_PER_LE(PT_LEVEL_3)
 #define PT_PAGES_PER_L2E      PT_PAGES_PER_LE(PT_LEVEL_2)
@@ -86,6 +90,7 @@ typedef enum {
 * address space size (in bytes) covered by a level-_l page table entry.
 */
 #define PT_LE_SHIFT(_l)       (PAGE_SHIFT + PT_LE_PG_SHIFT(_l))
+#define PT_L5E_SHIFT          PT_LE_SHIFT(PT_LEVEL_5)
 #define PT_L4E_SHIFT          PT_LE_SHIFT(PT_LEVEL_4)
 #define PT_L3E_SHIFT          PT_LE_SHIFT(PT_LEVEL_3)
 #define PT_L2E_SHIFT          PT_LE_SHIFT(PT_LEVEL_2)
@@ -93,6 +98,7 @@ typedef enum {

 /* Address space size (in bytes) covered by a level-_l page table entry. */
 #define PT_LE_SIZE(_l)        ((uint64)1 << PT_LE_SHIFT(_l))
+#define PT_L5E_SIZE           PT_LE_SIZE(PT_LEVEL_5)
 #define PT_L4E_SIZE           PT_LE_SIZE(PT_LEVEL_4)
 #define PT_L3E_SIZE           PT_LE_SIZE(PT_LEVEL_3)
 #define PT_L2E_SIZE           PT_LE_SIZE(PT_LEVEL_2)
@@ -103,6 +109,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LA_2_LOFF(_a, _l)  (((_a) >> PT_LE_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LA_2_L5OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LA_2_L4OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LA_2_L3OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LA_2_L2OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_2)
@@ -113,6 +120,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LPN_2_LOFF(_a, _l) (((_a) >> PT_LE_PG_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LPN_2_L5OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LPN_2_L4OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LPN_2_L3OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LPN_2_L2OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_2)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -30,9 +30,6 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-#define INTERRUPT_REDIRECTION_BITMAP_SIZE 32
-#define IO_PERMISSION_BITMAP_SIZE 0
-

 #define VMM_PANIC_NONE         0
 #define VMM_PANIC_MSG_COPYING  1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -83,7 +83,7 @@
 * Structure used to query platform about the page state.
 */
 typedef struct PlatformPageInfoList {
-   PgCnt64 numPages;
+   PageCnt numPages;
   BPN     bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
   MPN     mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
   uint8   flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
@@ -96,9 +96,10 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

-#define VMMEM_SERVICES_DEFS                               \
-   MDEF(VMMEM_SERVICES_TYPE_P2M,  P2MUpdate_FilterPages)  \
-   MDEF(VMMEM_SERVICES_TYPE_SWAP, BusMemSwap_FilterPages)
+#define VMMEM_SERVICES_DEFS                                       \
+   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
+   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
+   MDEF(VMMEM_SERVICES_TYPE_DIRTY_CLEAR, DirtyDrainer_ClearDirty)

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -89,6 +89,8 @@
 #define CR4_PCE        0x00000100
 #define CR4_OSFXSR     0x00000200 // CPU/OS supports SIMD insts
 #define CR4_OSXMMEXCPT 0x00000400 // #XF exception enable PIII only
+#define CR4_UMIP       0x00000800
+#define CR4_LA57       0x00001000
 #define CR4_VMXE       0x00002000
 #define CR4_SMXE       0x00004000
 #define CR4_FSGSBASE   0x00010000
@@ -97,7 +99,7 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
-#define CR4_RESERVED   CONST64U(0xffffffffff889800)
+#define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

 /*
@@ -201,8 +203,15 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

+/* Bitmap of the exception vectors that have associated error codes. */
+#define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
+                                (1u << EXC_NP) | (1u << EXC_SS) | \
+                                (1u << EXC_GP) | (1u << EXC_PF) | \
+                                (1u << EXC_AC))
+
 /*
 * eflag/rflag definitions.
 */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -119,38 +119,45 @@ void __cpuidex(int regs[4], int eax, int ecx);
 #endif

 static INLINE void
-__GET_CPUID(int eax,         // IN
+__GET_CPUID(uint32     eax,  // IN
            CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax)
      : "memory"
   );
 }

 static INLINE void
-__GET_CPUID2(int eax,         // IN
-             int ecx,         // IN
+__GET_CPUID2(uint32 eax,      // IN
+             uint32 ecx,      // IN
             CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax), "c" (ecx)
      : "memory"
   );
 }

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int eax) // IN
+__GET_EAX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx)
      : "a" (eax)
      : "memory", "%ecx", "%edx"
   );
@@ -159,7 +166,7 @@ __GET_EAX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int eax) // IN
+__GET_EBX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

@@ -174,14 +181,16 @@ __GET_EBX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int eax) // IN
+__GET_ECX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ecx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=c" (ecx)
      : "a" (eax)
      : "memory", "%edx"
   );
@@ -190,14 +199,16 @@ __GET_ECX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int eax) // IN
+__GET_EDX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 edx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=d" (edx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=d" (edx)
      : "a" (eax)
      : "memory", "%ecx"
   );
@@ -207,7 +218,7 @@ __GET_EDX_FROM_CPUID(int eax) // IN


 static INLINE uint32
-__GET_EAX_FROM_CPUID4(int ecx) // IN
+__GET_EAX_FROM_CPUID4(uint32 ecx) // IN
 {
   uint32 eax;
   uint32 ebx;
@@ -228,7 +239,7 @@ __GET_EAX_FROM_CPUID4(int ecx) // IN
 #elif defined(_MSC_VER) // } {

 static INLINE void
-__GET_CPUID(int input, CPUIDRegs *regs)
+__GET_CPUID(uint32 input, CPUIDRegs *regs)
 {
 #ifdef VM_X86_64
   __cpuid((int *)regs, input);
@@ -262,7 +273,7 @@ __GET_CPUID(int input, CPUIDRegs *regs)
 */

 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __cpuidex((int *)regs, inputEax, inputEcx);
 }
@@ -275,13 +286,13 @@ __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
 */

 extern void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs);
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs);
 #endif // _MSC_VER >= 1500

 #else // VM_X86_64

 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __asm push esi
   __asm push ebx
@@ -305,7 +316,7 @@ __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
 #endif

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int input)
+__GET_EAX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -332,7 +343,7 @@ __GET_EAX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int input)
+__GET_EBX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -359,7 +370,7 @@ __GET_EBX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int input)
+__GET_ECX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -386,7 +397,7 @@ __GET_ECX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int input)
+__GET_EDX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -420,12 +431,12 @@ __GET_EDX_FROM_CPUID(int input)
 */

 extern uint32
-__GET_EAX_FROM_CPUID4(int inputEcx);
+__GET_EAX_FROM_CPUID4(uint32 inputEcx);

 #else // VM_X86_64

 static INLINE uint32
-__GET_EAX_FROM_CPUID4(int inputEcx)
+__GET_EAX_FROM_CPUID4(uint32 inputEcx)
 {
   uint32 output;

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,43 +97,78 @@ static INLINE uint32 Desc_LongMode(const Descriptor *d) { return d->longmode; }
 static INLINE uint32 Desc_DB(const Descriptor *d)       { return d->DB; }
 static INLINE uint32 Desc_Gran(const Descriptor *d)     { return d->gran; }

-static INLINE uint32 Desc64_Type(const Descriptor64 *d)     { return d->type; }
-static INLINE uint32 Desc64_S(const Descriptor64 *d)        { return d->S; }
-static INLINE uint32 Desc64_DPL(const Descriptor64 *d)      { return d->DPL; }
-static INLINE uint32 Desc64_Present(const Descriptor64 *d)  { return d->present; }
-static INLINE uint32 Desc64_AVL(const Descriptor64 *d)      { return d->AVL; }
-static INLINE uint32 Desc64_Gran(const Descriptor64 *d)     { return d->gran; }
-static INLINE uint32 Desc64_ExtAttrs(const Descriptor64 *d) { return d->ext_attrs; }
+static INLINE uint32
+Desc64_Type(const Descriptor64 *d)
+{
+   return (uint32)d->type;
+}
+
+static INLINE uint32
+Desc64_S(const Descriptor64 *d)
+{
+   return (uint32)d->S;
+}
+
+static INLINE uint32
+Desc64_DPL(const Descriptor64 *d)
+{
+   return (uint32)d->DPL;
+}
+
+static INLINE uint32
+Desc64_Present(const Descriptor64 *d)
+{
+   return (uint32)d->present;
+}
+
+static INLINE uint32
+Desc64_AVL(const Descriptor64 *d)
+{
+   return (uint32)d->AVL;
+}
+
+static INLINE uint32
+Desc64_Gran(const Descriptor64 *d)
+{
+   return (uint32)d->gran;
+}
+
+static INLINE uint32
+Desc64_ExtAttrs(const Descriptor64 *d)
+{
+   return (uint32)d->ext_attrs;
+}

 static INLINE LA32
 Desc_GetBase(const Descriptor *d)
 {
-   return (d->base_hi << 24) | (d->base_mid << 16) | d->base_lo;
+   return ((unsigned)d->base_hi  << 24) |
+          ((unsigned)d->base_mid << 16) | d->base_lo;
 }

 static INLINE LA64
 Desc64_GetBase(const Descriptor64 *d)
 {
-   return ((uint64)d->base_hi << 32)  | 
-          ((uint64)d->base_mid << 24) | (uint32)d->base_lo;
+   return ((uint64)d->base_hi << 32)  |
+          ((uint64)d->base_mid << 24) | (uint64)d->base_lo;
 }

 static INLINE LA32
 Desc64_GetBaseHi(const Descriptor64 *d)
 {
-   return d->base_hi;
+   return (LA32)d->base_hi;
 }

 static INLINE VA32
 Desc_GetLimit(const Descriptor *d)
 {
-   return (d->limit_hi << 16) | d->limit_lo;
+   return ((unsigned)d->limit_hi << 16) | d->limit_lo;
 }

 static INLINE VA32
 Desc64_GetLimit(const Descriptor64 *d)
 {
-   return ((uint32)d->limit_hi << 16) | (uint32)d->limit_lo;
+   return (VA32)(((unsigned)d->limit_hi << 16) | d->limit_lo);
 }

 static INLINE Bool
@@ -310,12 +345,13 @@ static INLINE uint32 DESC_PRESENT(Descriptor d)    { return d.present; }
 *-----------------------------------------------------------------------------
 */

-static INLINE void 
-Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type, 
-                           uint32 DPL, uint32 present, uint32 DB, uint32 gran)
+static INLINE void
+Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit,
+                           uint32 type, uint32 DPL, uint32 present,
+                           uint32 DB, uint32 gran)
 {
-   uint32 baseHi = (uint32) (base >> 32);
-   uint32 baseLo = (uint32) base;
+   uint32 baseHi = (uint32)(base >> 32);
+   uint32 baseLo = (uint32)base;

   /*
    * The first half of a 16-byte descriptor is a valid 8-byte descriptor
@@ -336,15 +372,17 @@ Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type
 *   Given descriptor, return the code/stack size that it specifies.
 *----------------------------------------------------------------------
 */
-static INLINE int
+static INLINE unsigned
 Desc_DBSize(const Descriptor *desc)
 {
   /*
    * Code/stack size is determined by the D/B bit; bit 22 of the 2nd word.
-    * Shift the bit to position 1, mask it out, add 2. Result: 2 or 4.
+    * Shift the bit to position 1, mask it out, add 2.
+    *
+    * Result: 2 or 4.
    */
-   int cSz = ((((const uint32*)desc)[1] >> 21) & 2) + 2;
-   ASSERT(cSz == (Desc_DB(desc) ? 4 : 2));
+   unsigned cSz = ((((const uint32 *)desc)[1] >> 21) & 2) + 2;
+   ASSERT(cSz == (Desc_DB(desc) ? 4U : 2U));
   return cSz;
 }

@@ -526,7 +564,7 @@ typedef struct Gate {
 static INLINE VA
 CallGate_GetOffset(const Gate *cg)
 {
-   return (cg->offset_hi << 16) | cg->offset_lo;
+   return ((unsigned)cg->offset_hi << 16) | cg->offset_lo;
 }

 static INLINE void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-/* 
+/*
 * x86msr.h --
 *
 *      MSR number definitions.
@@ -59,11 +59,8 @@ struct MSRReply {
    * we use it to correlate the replies of multiple queries.
    */
   uint64 tag;              // OUT
-
   uint64 msrVal;           // OUT
-
   uint8  implemented;      // OUT
-
   uint8  _pad[7];
 }
 #include "vmware_pack_end.h"
@@ -120,25 +117,34 @@ MSRQuery;
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
 #define MSR_ARCH_CAPABILITIES_RSBA                (1ULL << 2)
+#define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
+#define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)
+#define MSR_ARCH_CAPABILITIES_MDS_NO              (1ULL << 5)

+#define MSR_FLUSH_CMD                        0x10b
+#define MSR_FLUSH_CMD_FLUSH_L1D                   (1ULL << 0)

 #define MSR_SPEC_CTRL_IBRS                        (1UL << 0)
 #define MSR_SPEC_CTRL_STIBP                       (1UL << 1)
+#define MSR_SPEC_CTRL_SSBD                        (1UL << 2)

 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

+
 #define MSR_MISC_FEATURES_ENABLES            0x140

 /* Intel Core Architecture and later: use only architected counters. */
-#define IA32_MSR_PERF_CAPABILITIES                0x345
-#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT        0
-#define MSR_PERF_CAPABILITIES_LBRFMT_MASK         0x3f
-#define MSR_PERF_CAPABILITIES_PEBSTRAP            (1u << 6)
-#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS    (1u << 7)
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT 8
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK  0xf
-#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM    (1u << 12)
-#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES   (1u << 13)
+#define IA32_MSR_PERF_CAPABILITIES                   0x345
+#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT           0
+#define MSR_PERF_CAPABILITIES_LBRFMT_MASK            0x3f
+#define MSR_PERF_CAPABILITIES_PEBSTRAP               (1u << 6)
+#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS       (1u << 7)
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT    8
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK     0xf
+#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM       (1u << 12)
+#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES      (1u << 13)
+#define MSR_PERF_CAPABILITIES_PEBS_ADAPTIVE_DATA     (1u << 14)
+#define MSR_PERF_CAPABILITIES_PERF_METRICS_AVAILABLE (1u << 15)

 #define IA32_MSR_PEBS_ENABLE                      0x3f1

@@ -265,6 +271,8 @@ typedef enum {

 #define MSR_XSS              0x00000da0  // Extended Supervisor State Mask

+#define MSR_MPX_LAX          0x00001000  // MPX Linear Address Extension
+
 /* RTIT MSRs */
 #define MSR_RTIT_CTL              0x00000570
 #define MSR_RTIT_STATUS           0x00000571
@@ -507,9 +515,28 @@ typedef enum {
 #define MSR_VM_CR_SVME_DISABLE     0x0000000000000010ULL
 #define MSR_VM_CR_RESERVED         0xffffffffffffffe0ULL

+/* SEV related MSRs. */
+#define MSR_VMPAGE_FLUSH           0xc001011e
+#define MSR_GHCB_PA                0xc0010130
+#define MSR_GHCB_PA_SEVINFO_HV              1
+#define MSR_GHCB_PA_SEVINFO_REQ             2
+#define MSR_GHCB_PA_AP_JUMP_TABLE           3
+#define MSR_SEV_STATUS             0xc0010131
+
+#define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
+#define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
+
 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)

+#define MSR_AMD_LS_CFG           0xc0011020  // load-store configuration
+#define MSR_AMD_LS_CFG_SSBD_BULLDOZER (1ULL<<54)  // BD family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_KYOTO     (1ULL<<33)  // Kyoto family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_ZEN       (1ULL<<10)  // Zen family non-arch SSBD
+
+#define MSR_AMD_VIRT_SPEC_CTRL   0xc001011f  // Virtual speculation control
+#define MSR_AMD_VIRT_SPEC_CTRL_SSBD   (1ULL<<2)   // Virtual SSBD
+
 /* Syscall/Sysret related MSRs (x86_64) */
 #define MSR_STAR             0xc0000081 // Also present on Athlons.
 #define MSR_LSTAR            0xc0000082
@@ -591,6 +618,8 @@ typedef enum {
 #define MSR_HYPERV_HYPERCALL_EN                  1ULL
 #define MSR_HYPERV_REFERENCE_TSC_EN              1ULL
 #define MSR_HYPERV_VP_ASSIST_EN                  1ULL
+#define MSR_HYPERV_SIEF_EN                       1ULL
+#define MSR_HYPERV_SIM_EN                        1ULL

 #define MSR_HYPERV_GUESTOSID_VENDOR_SHIFT        48
 #define MSR_HYPERV_GUESTOSID_VENDOR_MASK         0xfULL
@@ -600,6 +629,9 @@ typedef enum {
 #define MSR_HYPERV_GUESTOSID_OS_MASK             0xfULL
 #define MSR_HYPERV_GUESTOSID_OS_WINNT_DERIVATIVE 4ULL

+/* MSR for forcing RTM abort to recover PMC3 (see PR 2333817) */
+#define MSR_TSX_FORCE_ABORT                      0x0000010f
+#define MSR_TSX_FORCE_ABORT_RTM_BIT_INDEX        0

 /*
 * MTRR bit description
@@ -657,14 +689,19 @@ typedef unsigned char MTRRType;



+// Platform Quality of Service (PQM) MSRs
+#define MSR_INTEL_PQM_EVTSEL    0xc8d
+#define MSR_INTEL_PQM_CTR       0xc8e
+#define MSR_INTEL_PQM_ASSOC     0xc8f

-/*
- * Platform Quality of Service
- */
+ // Platform Quality Enforcement (PQE) MSRs
+#define MSR_INTEL_PQE_CLOS_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_MASK_MAX      0xd8f

-#define MSR_INTEL_QOS_EVTSEL    0xc8d
-#define MSR_INTEL_QOS_CTR       0xc8e
-#define MSR_INTEL_QOS_ASSOC     0xc8f
+#define MSR_INTEL_PQE_CLOS_L3_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_L3_MASK_MAX      0xd0f
+#define MSR_INTEL_PQE_CLOS_L2_MASK_BASE     0xd10
+#define MSR_INTEL_PQE_CLOS_L2_MASK_MAX      0xd4f

 static INLINE uint32
 X86MSR_SysCallEIP(uint64 star)
@@ -687,6 +724,126 @@ X86MSR_SysRetCS(uint64 star)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * X86MSR_{G,S}etMSR --
+ *
+ *      Get or set an MSR in hardware. The interface should remain
+ *      agnostic to the underlying compiler and architecture.
+ *
+ *----------------------------------------------------------------------
+ */
+
+#ifdef __GNUC__
+/*
+ * Checked against the Intel manual and GCC --hpreg
+ * volatile because the msr can change without the compiler knowing it
+ * (when we use wrmsr).
+ */
+#ifdef VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
+      : "=a" (msr)
+      : "c" (cx)
+      : "%rdx"
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "a" ((uint32) value),
+        "d" ((uint32)(value >> 32)),
+        "c" (cx)
+    );
+}
+#else // __GNUC__ && !VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr"
+      : "=A" (msr)
+      : "c"  (cx)
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "A" (value),
+        "c" (cx)
+    );
+}
+#endif
+#elif defined _MSC_VER // !__GNUC__ && _MSC_VER
+#ifdef _WIN64
+unsigned __int64  __readmsr(unsigned long);
+void              __writemsr(unsigned long, unsigned __int64);
+#pragma intrinsic(__readmsr, __writemsr)
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   return __readmsr((unsigned long)cx);
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
+}
+#else // !__GNUC__ && _MSC_VER && !_WIN64
+#pragma warning( disable : 4035)
+static INLINE uint64
+X86MSR_GetMSR(uint32 input)
+{
+   __asm push ecx
+   __asm mov  ecx, input
+   /* 0x0f 0x32 -> edx:eax = rdmsr[ecx] */
+   __asm _emit 0x0f __asm _emit 0x32
+   __asm pop ecx
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 input, uint64 value)
+{
+      uint32 hival = (uint32)((value) >> 32);
+      uint32 loval = (uint32)value;
+      __asm push edx
+      __asm push ecx
+      __asm push eax
+      __asm mov  eax, loval
+      __asm mov  edx, hival
+      __asm mov  ecx, input
+      /* 0x0f 0x30 -> wrmsr[ecx] = edx:eax */
+      __asm _emit 0x0f __asm _emit 0x30
+      __asm pop  eax
+      __asm pop  ecx
+      __asm pop  edx
+}
+#pragma warning (default: 4035)
+#endif
+#else // !__GNUC__ && !_MSC_VER
+#error No compiler defined for RDMSR/WRMSR.
+#endif
+
+
 #if defined __cplusplus
 }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,7 +52,6 @@
 #define LM_AVAIL_MASK         (CONST64(0x7) << LM_AVAIL_SHIFT)
 #define LM_FLAGS_MASK         CONST64(0x80000000000001ff)
 #define LM_CR3_FLAGS_MASK     CONST64(0x18)
-#define LM_L4_L3_RSVD_MASK    CONST64(0x180)
 #define LM_L3_1G_RSVD_MASK    CONST64(0x3fffe000)

 #define LM_MAKE_CR3(_mpfn, _flags) \
@@ -65,12 +64,62 @@
                    ((uint64)(_flags) & LM_FLAGS_MASK))

 #define LM_MAKE_PDE(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L5E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L4E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L3E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L2E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L1E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)


+/*
+ *----------------------------------------------------------------------
+ *
+ * LMPTEIsSafe --
+ *
+ *    A shadow PTE is considered "safe" if any of the following conditions are
+ *    met:
+ *
+ *      a) It is not terminal (i.e., present with no reserved bits set)
+ *      b) Terminal, but with MPN and PS fields set to zero
+ *      c) Terminal, but with MPN field specifying an uncachable page
+ *
+ *    In practice, for condition c), we require that bits 45:43 of the EPTE are
+ *    set to b'111.  The position of these bits is undocumented and not
+ *    architectural; they are truly magic.
+ *
+ *----------------------------------------------------------------------
+ */
+#ifdef VMX86_DEBUG
+
+#define LM_SAFE_BITS_MASK (MASK64(3) << 43)
+#define LM_SAFE_BITS(_v)  ((_v) & LM_SAFE_BITS_MASK)
+
+static INLINE Bool
+LMPTEIsTerminal(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   uint64 rsvd2m = (MASK(PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvd1g = (MASK(2 * PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvdl4 = PTE_PS;
+   uint64 rsvd = physMask & MASK64(52);
+
+   return (!PTE_PRESENT(pte) ||
+           (pte & LM_PTE_PFN_MASK) & rsvd) != 0 ||
+           ((pte & PTE_PS) != 0 &&
+            ((level == PT_LEVEL_2 && (pte & rsvd2m) != 0) ||
+             (level == PT_LEVEL_3 && (pte & rsvd1g) != 0))) ||
+           (level == PT_LEVEL_4 && (pte & rsvdl4) != 0);
+}
+
+static INLINE Bool
+LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   MPN mpn = (pte & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT;
+   Bool safe = !LMPTEIsTerminal(pte, level, physMask) ||
+               (mpn == 0 && !PTE_LARGEPAGE(pte)) ||
+               LM_SAFE_BITS(pte) == LM_SAFE_BITS_MASK;
+   return safe;
+}
+#endif /* VMX86_DEBUG */


 /*
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,21 +65,21 @@ typedef struct Descriptor {
 typedef 
 #include "vmware_pack_begin.h"
 struct Descriptor64 {
-   unsigned   limit_lo  : 16;   // Limit bits 15-0.
-   unsigned   base_lo   : 24;   // Base bits  23-0.
-   unsigned   type      : 4;
-   unsigned   S         : 1;
-   unsigned   DPL       : 2;
-   unsigned   present   : 1;
-   unsigned   limit_hi  : 4;    // Limit bits 19-16.
-   unsigned   AVL       : 1;
-   unsigned   reserved0 : 2;
-   unsigned   gran      : 1;
-   unsigned   base_mid  : 8;    // Base bits 31-24.
-   unsigned   base_hi   : 32;   // Base bits 63-32.
-   unsigned   reserved1 : 8;
-   unsigned   ext_attrs : 5;
-   unsigned   reserved2 : 19;
+   uint64   limit_lo  : 16;   // Limit bits 15-0.
+   uint64   base_lo   : 24;   // Base bits  23-0.
+   uint64   type      : 4;
+   uint64   S         : 1;
+   uint64   DPL       : 2;
+   uint64   present   : 1;
+   uint64   limit_hi  : 4;    // Limit bits 19-16.
+   uint64   AVL       : 1;
+   uint64   reserved0 : 2;
+   uint64   gran      : 1;
+   uint64   base_mid  : 8;    // Base bits 31-24.
+   uint64   base_hi   : 32;   // Base bits 63-32.
+   uint64   reserved1 : 8;
+   uint64   ext_attrs : 5;
+   uint64   reserved2 : 19;
 }
 #include "vmware_pack_end.h"
 Descriptor64;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -47,8 +47,10 @@
 #endif
 #include "x86cpuid_asm.h"

-#define SVM_VMCB_IO_BITMAP_SIZE    (3 * PAGE_SIZE)
-#define SVM_VMCB_MSR_BITMAP_SIZE   (2 * PAGE_SIZE)
+#define SVM_VMCB_IO_BITMAP_PAGES   (3)
+#define SVM_VMCB_IO_BITMAP_SIZE    PAGES_2_BYTES(SVM_VMCB_IO_BITMAP_PAGES)
+#define SVM_VMCB_MSR_BITMAP_PAGES  (2)
+#define SVM_VMCB_MSR_BITMAP_SIZE   PAGES_2_BYTES(SVM_VMCB_MSR_BITMAP_PAGES)

 /* Exit controls for the CR/DR access and hardware exceptions */
 #define SVM_CR_RD_CTL(num)         (0x1 << (num))
@@ -63,53 +65,56 @@
 #define SVM_XCP_CTL_ALL            0xffffffff

 /* Execution intercept controls */
-/* VMCB.exitCtl */
-#define SVM_VMCB_EXEC_CTL_INTR        0x0000000000000001ULL
-#define SVM_VMCB_EXEC_CTL_NMI         0x0000000000000002ULL
-#define SVM_VMCB_EXEC_CTL_SMI         0x0000000000000004ULL
-#define SVM_VMCB_EXEC_CTL_INIT        0x0000000000000008ULL
-#define SVM_VMCB_EXEC_CTL_VINTR       0x0000000000000010ULL
-#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR  0x0000000000000020ULL
-#define SVM_VMCB_EXEC_CTL_SIDT        0x0000000000000040ULL
-#define SVM_VMCB_EXEC_CTL_SGDT        0x0000000000000080ULL
-#define SVM_VMCB_EXEC_CTL_SLDT        0x0000000000000100ULL
-#define SVM_VMCB_EXEC_CTL_STR         0x0000000000000200ULL
-#define SVM_VMCB_EXEC_CTL_LIDT        0x0000000000000400ULL
-#define SVM_VMCB_EXEC_CTL_LGDT        0x0000000000000800ULL
-#define SVM_VMCB_EXEC_CTL_LLDT        0x0000000000001000ULL
-#define SVM_VMCB_EXEC_CTL_LTR         0x0000000000002000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSC       0x0000000000004000ULL
-#define SVM_VMCB_EXEC_CTL_RDPMC       0x0000000000008000ULL
-#define SVM_VMCB_EXEC_CTL_PUSHF       0x0000000000010000ULL
-#define SVM_VMCB_EXEC_CTL_POPF        0x0000000000020000ULL
-#define SVM_VMCB_EXEC_CTL_CPUID       0x0000000000040000ULL
-#define SVM_VMCB_EXEC_CTL_RSM         0x0000000000080000ULL
-#define SVM_VMCB_EXEC_CTL_IRET        0x0000000000100000ULL
-#define SVM_VMCB_EXEC_CTL_SWINT       0x0000000000200000ULL
-#define SVM_VMCB_EXEC_CTL_INVD        0x0000000000400000ULL
-#define SVM_VMCB_EXEC_CTL_PAUSE       0x0000000000800000ULL
-#define SVM_VMCB_EXEC_CTL_HLT         0x0000000001000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPG      0x0000000002000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPGA     0x0000000004000000ULL
-#define SVM_VMCB_EXEC_CTL_IOIO        0x0000000008000000ULL
-#define SVM_VMCB_EXEC_CTL_MSR         0x0000000010000000ULL
-#define SVM_VMCB_EXEC_CTL_TS          0x0000000020000000ULL
-#define SVM_VMCB_EXEC_CTL_FERR_FRZ    0x0000000040000000ULL
-#define SVM_VMCB_EXEC_CTL_SHUTDOWN    0x0000000080000000ULL
-#define SVM_VMCB_EXEC_CTL_VMRUN       0x0000000100000000ULL
-#define SVM_VMCB_EXEC_CTL_VMMCALL     0x0000000200000000ULL
-#define SVM_VMCB_EXEC_CTL_VMLOAD      0x0000000400000000ULL
-#define SVM_VMCB_EXEC_CTL_VMSAVE      0x0000000800000000ULL
-#define SVM_VMCB_EXEC_CTL_STGI        0x0000001000000000ULL
-#define SVM_VMCB_EXEC_CTL_CLGI        0x0000002000000000ULL
-#define SVM_VMCB_EXEC_CTL_SKINIT      0x0000004000000000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSCP      0x0000008000000000ULL
-#define SVM_VMCB_EXEC_CTL_ICEBP       0x0000010000000000ULL
-#define SVM_VMCB_EXEC_CTL_WBINVD      0x0000020000000000ULL
-#define SVM_VMCB_EXEC_CTL_MONITOR     0x0000040000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT       0x0000080000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT_COND  0x0000100000000000ULL
-#define SVM_VMCB_EXEC_CTL_XSETBV      0x0000200000000000ULL
+/* VMCB.execCtl */
+#define SVM_VMCB_EXEC_CTL_INTR           0x0000000000000001ULL
+#define SVM_VMCB_EXEC_CTL_NMI            0x0000000000000002ULL
+#define SVM_VMCB_EXEC_CTL_SMI            0x0000000000000004ULL
+#define SVM_VMCB_EXEC_CTL_INIT           0x0000000000000008ULL
+#define SVM_VMCB_EXEC_CTL_VINTR          0x0000000000000010ULL
+#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR     0x0000000000000020ULL
+#define SVM_VMCB_EXEC_CTL_SIDT           0x0000000000000040ULL
+#define SVM_VMCB_EXEC_CTL_SGDT           0x0000000000000080ULL
+#define SVM_VMCB_EXEC_CTL_SLDT           0x0000000000000100ULL
+#define SVM_VMCB_EXEC_CTL_STR            0x0000000000000200ULL
+#define SVM_VMCB_EXEC_CTL_LIDT           0x0000000000000400ULL
+#define SVM_VMCB_EXEC_CTL_LGDT           0x0000000000000800ULL
+#define SVM_VMCB_EXEC_CTL_LLDT           0x0000000000001000ULL
+#define SVM_VMCB_EXEC_CTL_LTR            0x0000000000002000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSC          0x0000000000004000ULL
+#define SVM_VMCB_EXEC_CTL_RDPMC          0x0000000000008000ULL
+#define SVM_VMCB_EXEC_CTL_PUSHF          0x0000000000010000ULL
+#define SVM_VMCB_EXEC_CTL_POPF           0x0000000000020000ULL
+#define SVM_VMCB_EXEC_CTL_CPUID          0x0000000000040000ULL
+#define SVM_VMCB_EXEC_CTL_RSM            0x0000000000080000ULL
+#define SVM_VMCB_EXEC_CTL_IRET           0x0000000000100000ULL
+#define SVM_VMCB_EXEC_CTL_SWINT          0x0000000000200000ULL
+#define SVM_VMCB_EXEC_CTL_INVD           0x0000000000400000ULL
+#define SVM_VMCB_EXEC_CTL_PAUSE          0x0000000000800000ULL
+#define SVM_VMCB_EXEC_CTL_HLT            0x0000000001000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPG         0x0000000002000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPGA        0x0000000004000000ULL
+#define SVM_VMCB_EXEC_CTL_IOIO           0x0000000008000000ULL
+#define SVM_VMCB_EXEC_CTL_MSR            0x0000000010000000ULL
+#define SVM_VMCB_EXEC_CTL_TS             0x0000000020000000ULL
+#define SVM_VMCB_EXEC_CTL_FERR_FRZ       0x0000000040000000ULL
+#define SVM_VMCB_EXEC_CTL_SHUTDOWN       0x0000000080000000ULL
+#define SVM_VMCB_EXEC_CTL_VMRUN          0x0000000100000000ULL
+#define SVM_VMCB_EXEC_CTL_VMMCALL        0x0000000200000000ULL
+#define SVM_VMCB_EXEC_CTL_VMLOAD         0x0000000400000000ULL
+#define SVM_VMCB_EXEC_CTL_VMSAVE         0x0000000800000000ULL
+#define SVM_VMCB_EXEC_CTL_STGI           0x0000001000000000ULL
+#define SVM_VMCB_EXEC_CTL_CLGI           0x0000002000000000ULL
+#define SVM_VMCB_EXEC_CTL_SKINIT         0x0000004000000000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSCP         0x0000008000000000ULL
+#define SVM_VMCB_EXEC_CTL_ICEBP          0x0000010000000000ULL
+#define SVM_VMCB_EXEC_CTL_WBINVD         0x0000020000000000ULL
+#define SVM_VMCB_EXEC_CTL_MONITOR        0x0000040000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT          0x0000080000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT_COND     0x0000100000000000ULL
+#define SVM_VMCB_EXEC_CTL_XSETBV         0x0000200000000000ULL
+#define SVM_VMCB_EXEC_CTL_EFER_WR_TRAP   0x0000800000000000ULL
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP(n) (0x0001000000000000ULL << (n))
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP_ALL 0xffff000000000000ULL

 /* VMCB.tlbCtl */
 #define SVM_VMCB_TLB_CTL_FLUSH              0x01
@@ -130,9 +135,12 @@

 /* VMCB.intrState */
 #define SVM_VMCB_INTRSTATE_INTR_SHADOW     (1 << 0)
+#define SVM_VMCB_INTRSTATE_GUEST_IF        (1 << 1)

 /* VMCB.nptCtl */
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)
+#define SVM_VMCB_NPTCTL_SEV_ENABLE         (1 << 1)
+#define SVM_VMCB_NPTCTL_SEV_ES_ENABLE      (1 << 2)

 /* VMCB.virtExt */
 #define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
@@ -234,9 +242,16 @@
 #define SVM_EXITCODE_MWAIT                139
 #define SVM_EXITCODE_MWAIT_COND           140
 #define SVM_EXITCODE_XSETBV               141
+#define SVM_EXITCODE_EFER_WRITE_TRAP      143
+#define SVM_EXITCODE_CR_WRITE_TRAP(n)    (144 + (n))
 #define SVM_EXITCODE_NPF                 1024
 #define SVM_EXITCODE_AVIC_INCOMPLETE_IPI 1025
 #define SVM_EXITCODE_AVIC_NOACCEL        1026
+#define SVM_EXITCODE_VMGEXIT             1027
+#define SVM_EXITCODE_MMIO_READ           0x80000001   // SW only
+#define SVM_EXITCODE_MMIO_WRITE          0x80000002   // SW only
+#define SVM_EXITCODE_NMI_COMPLETE        0x80000003   // SW only
+#define SVM_EXITCODE_UNSUPPORTED         0x8000FFFF   // SW only
 #define SVM_EXITCODE_INVALID             (-1ULL)

 /* ExitInfo1 for I/O exits */
@@ -260,6 +275,9 @@
 #define SVM_IOEXIT_MBZ           0x00000002
 #define SVM_IOEXIT_SBZ           0x0000e000

+/* ExitInfo1 for MSR exits */
+#define SVM_MSREXIT_WRMSR        0x00000001
+
 /* ExitInfo1 for CR exits */
 #define SVM_CREXIT_MOVCR         (1ull << 63)
 #define SVM_CREXIT_GPR_MASK      0xf
@@ -430,7 +448,7 @@ SVM_LockedFromFeatures(uint64 vmCR)
 static INLINE Bool
 SVM_EnabledCPU(void)
 {
-   return SVM_EnabledFromFeatures(__GET_MSR(MSR_VM_CR));
+   return SVM_EnabledFromFeatures(X86MSR_GetMSR(MSR_VM_CR));
 }
 #endif /* } !defined(USERLEVEL) */

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -62,13 +62,6 @@ typedef uint64 VM_EPTE;
 * Registers
 */

-typedef union SharedReg64 {
-   Reg8  reg8[2];
-   Reg16 reg16;
-   Reg32 reg32;
-   Reg64 reg64;
-} SharedReg64;
-
 typedef union SharedUReg64 {
   UReg8  ureg8[2];
   UReg16 ureg16;
@@ -178,11 +171,7 @@ typedef Task32 Task;

 #pragma pack(push, 1)
 typedef struct {
-#if defined(VMM) || defined(COREQUERY)
-   uint64 va;
-#else
-   uint32 va;
-#endif
+   uint64   va;
   Selector seg;
 } FarPtr;
 #pragma pack(pop)
@@ -468,4 +457,12 @@ typedef struct ExcFrame64ForCopy {
 } ExcFrame64ForCopy;
 #pragma pack(pop)

+/*
+ * Layout of the stack for performing a 64 bit lret instruction.
+ */
+typedef struct LretFrame64 {
+   uint64 rip;
+   uint64 cs;
+} LretFrame64;
+
 #endif // ifndef _X86TYPES_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,12 @@ VT_EXIT(RDSEED,               61)
 VT_EXIT(PML_LOGFULL,          62)
 VT_EXIT(XSAVES,               63)
 VT_EXIT(XRSTORS,              64)
-
+VT_EXIT(VMEXIT65,             65)
+VT_EXIT(VMEXIT66,             66)
+VT_EXIT(VMEXIT67,             67)
+VT_EXIT(VMEXIT68,             68)
+VT_EXIT(VMEXIT69,             69)
+VT_EXIT(ENCLV,                70)
+VT_EXIT(SGX_CONFLICT,         71)
 /* Bump this up if you add an exit reason. */
-#define VT_NUM_EXIT_REASONS   65
+#define VT_NUM_EXIT_REASONS   72
@@ -283,7 +283,7 @@ VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2030, _S64, _TC, 24)
 VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA,  _S)
 VMCS_UNUSED(                        0x2034, _S64, _TC, 26)
-VMCS_UNUSED(                        0x2036, _S64, _TC, 27)
+VMCS_FIELD(ENCLV_EXITING_BITMAP,    0x2036, _S64, _TC, 27, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2038, _S64, _TC, 28)
 VMCS_UNUSED(                        0x203A, _S64, _TC, 29)
 VMCS_UNUSED(                        0x203C, _S64, _TC, 30)
@@ -159,10 +159,10 @@
 #define VT_ENCODING_RSVD               0xffff9000

 /*
- * The highest index of any currently defined field is 25, for
- * TSC_MULTIPLIER.
+ * The highest index of any currently defined field is 27, for
+ * ENCLV_EXITING_BITMAP.
 */
-#define VT_ENCODING_MAX_INDEX                  25
+#define VT_ENCODING_MAX_INDEX                  27

 enum {
 #define VMCS_FIELD(_name, _val, ...) VT_VMCS_##_name = _val,
@@ -293,7 +293,9 @@ enum {
   VMX_CPU2(PT_SUPPRESS_NR_BIT, 19)                      \
   VMX_CPU2(XSAVES,             20)                      \
   VMX_CPU2(EPT_MBX,            22)                      \
-   VMX_CPU2(TSC_SCALING,        25)
+   VMX_CPU2(TSC_SCALING,        25)                      \
+   VMX_CPU2(ENCLV,              28)                      \
+   VMX_CPU2(EPC_VIRT_EXT,       29)

 #define VMX_PROCBASED_CTLS2_CAP                          \
        VMX_PROCBASED_CTLS2_CAP_NDA                      \
@@ -687,6 +689,12 @@ enum {
 #define VT_GUESTFAIL_QUAL_NMI          3
 #define VT_GUESTFAIL_QUAL_LINK         4

+/* SGX conflict VM-exit Qualification Codes */
+#define VT_SGX_TRACKING_RESOURCE_CONFLICT     0
+#define VT_SGX_TRACKING_REFERENCE_CONFLICT    1
+#define VT_SGX_EPC_PAGE_CONFLICT_EXCEPTION    2
+#define VT_SGX_EPC_PAGE_CONFLICT_ERROR        3
+
 /* VMX abort indicators. */

 #define VT_VMX_ABORT_GUEST_MSRS        1
@@ -710,7 +718,8 @@ enum {
 #define VT_REQUIRED_PINBASED_CTLS                      \
   (VT_PINBASED_CTLS_DEFAULT1                        | \
    VT_VMCS_PIN_VMEXEC_CTL_EXTINT_EXIT               | \
-    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT)
+    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT                  | \
+    VT_VMCS_PIN_VMEXEC_CTL_VNMI)

 #define VT_REQUIRED_PROCBASED_CTLS                     \
   (VT_PROCBASED_CTLS_DEFAULT1                       | \
@@ -726,6 +735,7 @@ enum {
    VT_VMCS_CPU_VMEXEC_CTL_LDCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_STCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_TPR_SHADOW                | \
+    VT_VMCS_CPU_VMEXEC_CTL_VNMI_WINDOW               | \
    VT_VMCS_CPU_VMEXEC_CTL_MONITOR)

 #define VT_REQUIRED_EXIT_CTLS                          \
@@ -1075,7 +1085,7 @@ VT_MBXSupportedFromFeatures(uint64 secondary)
 static INLINE Bool
 VT_EnabledCPU(void)
 {
-   return VT_EnabledFromFeatures(__GET_MSR(MSR_FEATCTL));
+   return VT_EnabledFromFeatures(X86MSR_GetMSR(MSR_FEATCTL));
 }


@@ -1096,16 +1106,14 @@ VT_EnabledCPU(void)
 static INLINE Bool
 VT_SupportedCPU(void)
 {
-   if (__GET_MSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) {
-      return VT_SupportedFromFeatures(__GET_MSR(MSR_VMX_TRUE_PINBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_PROCBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_ENTRY_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_EXIT_CTLS),
-                                      __GET_MSR(MSR_VMX_BASIC));
-   } else {
-      /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
-      return FALSE;
-   }
+
+   /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
+   return (X86MSR_GetMSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) != 0 &&
+          VT_SupportedFromFeatures(X86MSR_GetMSR(MSR_VMX_TRUE_PINBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_PROCBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_ENTRY_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_EXIT_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_BASIC));
 }

 #endif /* } !defined(USERLEVEL) */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -48,12 +48,10 @@
 #include "modulecall.h"
 #include "vm_asm.h"
 #include "vmx86.h"
-#include "initblock.h"
 #include "task.h"
 #include "memtrack.h"
 #include "task.h"
 #include "cpuid.h"
-#include "cpuid_info.h"
 #include "circList.h"
 #include "x86msr.h"

@@ -64,6 +62,7 @@
 #include "hostif.h"
 #include "hostif_priv.h"
 #include "vmhost.h"
+#include "sharedAreaVmmon.h"

 static void LinuxDriverQueue(Device *device);
 static void LinuxDriverDequeue(Device *device);
@@ -96,7 +95,9 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd,
                       unsigned long ioarg);

 static int LinuxDriver_Close(struct inode *inode, struct file *filp);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
+static vm_fault_t LinuxDriverFault(struct vm_fault *fault);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
 static int LinuxDriverFault(struct vm_fault *fault);
 #else
 static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault);
@@ -414,6 +415,7 @@ LinuxDriver_Open(struct inode *inode, // IN
   memset(device, 0, sizeof *device);

   sema_init(&device->lock4Gb, 1);
+   init_rwsem(&device->vmDriverRWSema);

   filp->private_data = device;
   LinuxDriverQueue(device);
@@ -443,17 +445,12 @@ LinuxDriver_Open(struct inode *inode, // IN
 */

 static int
-LinuxDriverAllocPages(unsigned int gfpFlag, // IN
-                      unsigned int order,   // IN
+LinuxDriverAllocPages(unsigned int order,   // IN
                      struct page **pg,     // OUT
-                      unsigned int size)    // IN
+                      PageCnt size)         // IN
 {
-   struct page* page;
-
-   page = alloc_pages(gfpFlag, order);
+   struct page* page = alloc_pages(GFP_HIGHUSER, order);
   if (page) {
-      unsigned int i;
-
      /*
       * Grab an extra reference on all pages except first one - first
       * one was already refcounted by alloc_pages.
@@ -466,9 +463,8 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
       * Note that this depends on Linux VM internals.  It works on all
       * kernels we care about.
       */
-
-      order = 1 << order;
-      for (i = 0; i < order; i++) {
+      PageCnt i, orderToNumPages = 1 << order;
+      for (i = 0; i < orderToNumPages; i++) {
         if (i) {
            /*
             * Debug kernels assert that page->_count is not zero when
@@ -486,17 +482,14 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
            put_page(page);
         } else {
            void *addr = kmap(page);
-
            memset(addr, 0, PAGE_SIZE);
            kunmap(page);
            *pg++ = page;
         }
         page++;
      }
-
      return 0;
   }
-
   return -ENOMEM;
 }

@@ -520,8 +513,7 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
 static void
 LinuxDriverDestructor4Gb(Device *device) // IN
 {
-   unsigned int pg;
-
+   PageCnt pg;
   if (!device->size4Gb) {
      return;
   }
@@ -554,10 +546,12 @@ LinuxDriver_Close(struct inode *inode, // IN
   ASSERT(device);

   LinuxDriverDequeue(device);
+   down_write(&device->vmDriverRWSema);
   if (device->vm != NULL) {
      Vmx86_ReleaseVM(device->vm);
      device->vm = NULL;
   }
+   up_write(&device->vmDriverRWSema);

   Vmx86_Close();

@@ -595,7 +589,12 @@ LinuxDriver_Close(struct inode *inode, // IN
 *-----------------------------------------------------------------------------
 */

-static int
+static
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
+vm_fault_t
+#else
+int
+#endif
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
 LinuxDriverFault(struct vm_fault *fault)     //IN/OUT
 #else
@@ -607,7 +606,7 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
   struct vm_area_struct *vma = fault->vma;
 #endif
   Device *device = (Device *) vma->vm_file->private_data;
-   unsigned long pg;
+   PageNum pg;
   struct page* page;

   pg = fault->pgoff;
@@ -644,12 +643,10 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
 static int LinuxDriverAllocContig(Device *device,
                                  struct vm_area_struct *vma,
                                  unsigned long off,
-                                  unsigned long size)
+                                  PageCnt size)
 {
   unsigned long vmaOrder = VMMON_MAP_ORDER(off);
-   unsigned long vmaAllocSize;
-   unsigned int gfpFlag;
-   unsigned long i;
+   PageCnt vmaAllocSize, i;

   if (VMMON_MAP_RSVD(off)) {
      /* Reserved bits set... */
@@ -659,24 +656,6 @@ static int LinuxDriverAllocContig(Device *device,
      /* We do not need non-zero offsets... */
      return -EINVAL;
   }
-   switch (VMMON_MAP_MT(off)) {
-      case VMMON_MAP_MT_LOW4GB:
-#ifdef GFP_DMA32
-         gfpFlag = GFP_USER | GFP_DMA32;
-#else
-         gfpFlag = GFP_USER | GFP_DMA;
-#endif
-         break;
-      case VMMON_MAP_MT_LOW16MB:
-         gfpFlag = GFP_USER | GFP_DMA;
-         break;
-      case VMMON_MAP_MT_ANY:
-         gfpFlag = GFP_HIGHUSER;
-         break;
-      default:
-         /* Invalid memory type */
-         return -EINVAL;
-   }
   if (size > VMMON_MAP_OFFSET_MASK + 1) {
      /* Size is too big to fit to our window. */
      return -ENOMEM;
@@ -694,23 +673,18 @@ static int LinuxDriverAllocContig(Device *device,
   }
   vmaAllocSize = 1 << vmaOrder;
   for (i = 0; i < size; i += vmaAllocSize) {
-      int err;
-
-      err = LinuxDriverAllocPages(gfpFlag, vmaOrder,
-                                  device->pages4Gb + i, size - i);
+      int err = LinuxDriverAllocPages(vmaOrder, device->pages4Gb + i, size - i);
      if (err) {
         while (i > 0) {
            put_page(device->pages4Gb[--i]);
         }
         up(&device->lock4Gb);
-
         return err;
      }
   }
   device->size4Gb = size;
   up(&device->lock4Gb);
   vma->vm_ops = &vmuser_mops;
-
   return 0;
 }

@@ -739,7 +713,7 @@ LinuxDriverMmap(struct file *filp,
                struct vm_area_struct *vma)
 {
   Device *device = (Device *) filp->private_data;
-   unsigned long size;
+   PageCnt size;
   int err;

   /* Only shared mappings */
@@ -753,17 +727,18 @@ LinuxDriverMmap(struct file *filp,
   if (size < 1) {
      return -EINVAL;
   }
+   down_read(&device->vmDriverRWSema);
   if (device->vm) {
      err = -EINVAL;
   } else {
      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
   }
+   up_read(&device->vmDriverRWSema);
   if (err) {
      return err;
   }
   /* Clear VM_IO, otherwise SuSE's kernels refuse to do get_user_pages */
   vma->vm_flags &= ~VM_IO;
-
   return 0;
 }

@@ -1034,26 +1009,30 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
                  unsigned long ioarg)  // IN:
 {
   Device *device = (Device *) filp->private_data;
-   int retval = 0;
+   long retval = 0;
   Vcpuid vcpuid;
   VMDriver *vm;
+   Bool needsWriteLock = iocmd == IOCTL_VMX86_CREATE_VM ||
+                         iocmd == IOCTL_VMX86_RELEASE_VM;

+   ASSERT_ON_COMPILE(sizeof(long) == sizeof(int64));
   if (device == NULL) {
      return -EINVAL;
   }

+   if (needsWriteLock) {
+      down_write(&device->vmDriverRWSema);
+   } else {
+      down_read(&device->vmDriverRWSema);
+   }
+
   vm = device->vm;
-
-   /*
-    * Validate the VM pointer for those IOCTLs that require it.
-    */
-
+   /* Validate the VM pointer for those IOCTLs that require it. */
   switch (iocmd) {
   case IOCTL_VMX86_VERSION:
   case IOCTL_VMX86_CREATE_VM:
   case IOCTL_VMX86_SET_UID:
   case IOCTL_VMX86_GET_NUM_VMS:
-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
   case IOCTL_VMX86_SET_HARD_LIMIT:
   case IOCTL_VMX86_GET_IPI_VECTORS:
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
@@ -1108,67 +1087,73 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:

   case IOCTL_VMX86_PROCESS_BOOTSTRAP: {
      VMProcessBootstrapBlock *args;
+      size_t argsSz = GetVMProcessBootstrapBlockSize(vm->numVCPUs);
      VA64 uAddr;
-      Bool res;

      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
      if (retval != 0) {
         break;
      }
-      args = HostIF_AllocKernelMem(sizeof *args, TRUE);
+      args = HostIF_AllocKernelMem(argsSz, TRUE);
      if (args == NULL) {
         retval = -ENOMEM;
         break;
      }
-      retval = HostIF_CopyFromUser(args, uAddr, sizeof *args);
+      retval = HostIF_CopyFromUser(args, uAddr, argsSz);
      if (retval != 0) {
         HostIF_FreeKernelMem(args);
         break;
      }
-      res = Vmx86_ProcessBootstrap(vm,
-                                   args->bsBlobAddr,
-                                   args->numBytes,
-                                   args->headerOffset,
-                                   args->numVCPUs,
-                                   args->ptRootVAs,
-                                   args->shRegions);
-      if (!res) {
+      if (args->numVCPUs != vm->numVCPUs) {
+         retval = -EINVAL;
+      } else if (!Vmx86_ProcessBootstrap(vm,
+                                         args->bsBlobAddr,
+                                         args->numBytes,
+                                         args->headerOffset,
+                                         args->numVCPUs,
+                                         args->perVcpuPages,
+                                         args->shRegions)) {
         retval = -ENOMEM;
      }
      HostIF_FreeKernelMem(args);
      break;
   }

+   case IOCTL_VMX86_REGISTER_SHARED: {
+      VMSharedAreaRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!SharedAreaVmmon_ValidateRegionArgs(vm, &args)) {
+         retval = -EINVAL;
+         break;
+      }
+      if (!SharedAreaVmmon_RegisterRegion(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
+   case IOCTL_VMX86_REGISTER_STATVARS: {
+      VMStatVarsRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!StatVarsVmmon_RegisterVCPU(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
   case IOCTL_VMX86_RELEASE_VM:
      device->vm = NULL;
      Vmx86_ReleaseVM(vm);
      break;

-   case IOCTL_VMX86_INIT_VM: {
-      InitBlock *initParams;
-      VA64 uAddr;
-
-      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
-      if (retval != 0) {
-         break;
-      }
-      initParams = HostIF_AllocKernelMem(sizeof *initParams, TRUE);
-      if (initParams == NULL) {
-         retval = -ENOMEM;
-         break;
-      }
-      retval = HostIF_CopyFromUser(initParams, uAddr, sizeof *initParams);
-      if (retval == 0) {
-         if (Vmx86_InitVM(vm, initParams) == 0) {
-            retval = HostIF_CopyToUser(uAddr, initParams, sizeof *initParams);
-         } else {
-            retval = -EINVAL;
-         }
-      }
-      HostIF_FreeKernelMem(initParams);
-      break;
-   }
-
   case IOCTL_VMX86_RUN_VM:
      vcpuid = ioarg;

@@ -1236,17 +1221,28 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_VMM_PAGE_ROOT: {
+      VcpuPageRoot args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->ptRootMpns == NULL) {
+         retval = -EINVAL;
+      } else {
+         args.pageRoot = vm->ptRootMpns[args.vcpuid];
+         retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      }
+      break;
+   }
+
   case IOCTL_VMX86_GET_NUM_VMS:
      retval = Vmx86_GetNumVMs();
      break;

-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
-      retval = Vmx86_GetTotalMemUsage();
-      break;
-
   case IOCTL_VMX86_SET_HARD_LIMIT: {
-      int32 limit;
-
+      PageCnt limit;
      retval = HostIF_CopyFromUser(&limit, ioarg, sizeof limit);
      if (retval != 0) {
         break;
@@ -1516,6 +1512,13 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_NUM_ANON_PAGES: {
+      PageCnt numAnonPages;
+      numAnonPages = Vmx86_GetNumAnonPages(vm);
+      retval = HostIF_CopyToUser(ioarg, &numAnonPages, sizeof numAnonPages);
+      break;
+   }
+
   case IOCTL_VMX86_READ_PAGE: {
         VMMReadWritePage req;

@@ -1530,6 +1533,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
+#if VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1538,6 +1542,9 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
         }
         retval = HostIF_WritePhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE,
                                       PAGE_SIZE);
+#else
+         retval = -EINVAL;
+#endif
         break;
      }

@@ -1596,7 +1603,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      uint64 swapSize;
      retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize);
      if (retval != 0) {
-         Warning("Could not copy swap size from user, status %d\n", retval);
+         Warning("Could not copy swap size from user, status %ld\n", retval);
         break;
      }
      linuxState.swapSize = swapSize;
@@ -1629,6 +1636,11 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

 exit:
+   if (needsWriteLock) {
+      up_write(&device->vmDriverRWSema);
+   } else {
+      up_read(&device->vmDriverRWSema);
+   }
   return retval;
 }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,27 +27,14 @@
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
+#include <linux/rwsem.h>
 #include <linux/wait.h>

 #include "vmx86.h"
 #include "driver_vmcore.h"


-/*
- * Used to track a piece of memory that's been mapped into the kernel from
- * userlevel.
- */
-typedef struct VMMappedUserMem {
-   void *addr;
-   size_t numPages;
-   struct page *pages[0];
-} VMMappedUserMem;
-
-
-/*
- * Per-instance driver state
- */
-
+/* Per-instance driver state */
 struct VMDriver;

 /* 16 pages (64KB) looks as a good limit for one allocation */
@@ -56,7 +43,15 @@ struct VMDriver;
 typedef struct Device {
   struct Device   *next;
   struct VMDriver *vm;
-
+   /*
+    * This RW semaphore protects accesses to the VMDriver to
+    * avoid racing between various ioctls, and the creation
+    * and removal of the VM in question. The lock is read-acquired
+    * by ioctls that reference the VMDriver, and write-acquired by
+    * ioctls or device callbacks that allocate or destory the
+    * VMDriver.
+    */
+   struct rw_semaphore vmDriverRWSema;
   /*
    * The semaphore protect accesses to size4Gb and pages4Gb
    * in mmap(). mmap() may happen only once, and all other
@@ -64,7 +59,7 @@ typedef struct Device {
    * only after successful mmap.
    */
   struct semaphore lock4Gb;
-   unsigned int size4Gb;
+   PageCnt size4Gb;
   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
 } Device;

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,7 +63,7 @@
 #include "memtrack.h"
 #include "phystrack.h"
 #include "cpuid.h"
-#include "cpuid_info.h"
+#include "x86cpuid_asm.h"
 #include "hostif.h"
 #include "hostif_priv.h"
 #include "vmhost.h"
@@ -787,21 +787,21 @@ HostIF_FreeMachinePage(MPN mpn)  // IN:
 *----------------------------------------------------------------------
 */

-int
+int64
 HostIF_AllocLockedPages(VMDriver *vm,         // IN: VM instance pointer
                        VA64 addr,            // OUT: buffer address
-                        unsigned numPages,    // IN: number of pages to allocate
+                        PageCnt numPages,     // IN: number of pages to allocate
                        Bool kernelMPNBuffer) // IN: kernel vs user space
 {
   VMHost *vmh = vm->vmhost;
-   unsigned int cnt;
-   int err = 0;
+   PageCnt cnt;
+   int64 err = 0;

   if (!vmh || !vmh->AWEPages) {
      return -EINVAL;
   }
   for (cnt = 0; cnt < numPages; cnt++) {
-      struct page* pg;
+      struct page *pg;
      MPN mpn;

      pg = alloc_page(GFP_HIGHUSER);
@@ -848,66 +848,38 @@ HostIF_AllocLockedPages(VMDriver *vm,         // IN: VM instance pointer

 int
 HostIF_FreeLockedPages(VMDriver *vm,         // IN: VM instance pointer
-                       VA64 addr,            // IN: array of MPNs
-                       unsigned numPages,    // IN: number of pages to free
-                       Bool kernelMPNBuffer) // IN: kernel vs user address
+                       MPN *mpns,            // IN: array of MPNs
+                       PageCnt numPages)     // IN: number of pages to free
 {
-   const int MPN_BATCH = 64;
-   MPN const *pmpn = VA64ToPtr(addr);
   VMHost *vmh = vm->vmhost;
-   unsigned int cnt;
+   PageCnt cnt;
   struct page *pg;
-   MPN *mpns;

-   mpns = HostIF_AllocKernelMem(sizeof *mpns * MPN_BATCH, TRUE);
-
-   if (mpns == NULL) {
-      return -ENOMEM;
-   }
   if (!vmh || !vmh->AWEPages) {
-      HostIF_FreeKernelMem(mpns);
      return -EINVAL;
   }

-   if (!kernelMPNBuffer) {
-      if (numPages > MPN_BATCH) {
-         HostIF_FreeKernelMem(mpns);
-         return -EINVAL;
-      }
-
-      if (HostIF_CopyFromUser(mpns, addr, numPages * sizeof *pmpn)) {
-         printk(KERN_DEBUG "Cannot read from process address space at %p\n",
-                pmpn);
-         HostIF_FreeKernelMem(mpns);
-         return -EINVAL;
-      }
-
-      pmpn = mpns;
-   }
-
   for (cnt = 0; cnt < numPages; cnt++) {
-      if (!PhysTrack_Test(vmh->AWEPages, pmpn[cnt])) {
+      if (!PhysTrack_Test(vmh->AWEPages, mpns[cnt])) {
         printk(KERN_DEBUG "Attempted to free unallocated MPN %016" FMT64 "X\n",
-                pmpn[cnt]);
-         HostIF_FreeKernelMem(mpns);
+                mpns[cnt]);
         return -EINVAL;
      }

-      pg = pfn_to_page(pmpn[cnt]);
+      pg = pfn_to_page(mpns[cnt]);
      if (page_count(pg) != 1) {
         // should this case be considered a failure?
         printk(KERN_DEBUG "Page %016" FMT64 "X is still used by someone "
-                "(use count %u, VM %p)\n", pmpn[cnt],
+                "(use count %u, VM %p)\n", mpns[cnt],
                 page_count(pg), vm);
      }
   }

   for (cnt = 0; cnt < numPages; cnt++) {
-      pg = pfn_to_page(pmpn[cnt]);
-      PhysTrack_Remove(vmh->AWEPages, pmpn[cnt]);
+      pg = pfn_to_page(mpns[cnt]);
+      PhysTrack_Remove(vmh->AWEPages, mpns[cnt]);
      __free_page(pg);
   }
-   HostIF_FreeKernelMem(mpns);
   return 0;
 }

@@ -1042,7 +1014,7 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
 *      Initialize the host-dependent part of the driver.
 *
 * Results:
- *      zero on success, non-zero on error.
+ *      TRUE on success, FALSE on error.
 *
 * Side effects:
 *      None
@@ -1050,12 +1022,12 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
 *----------------------------------------------------------------------
 */

-int
+Bool
 HostIF_Init(VMDriver *vm, uint32 numVCPUs)
 {
   vm->memtracker = MemTrack_Init(vm);
   if (vm->memtracker == NULL) {
-      return -1;
+      return FALSE;
   }

   vm->vmhost = HostIFAllocVMHost(numVCPUs);
@@ -1068,7 +1040,7 @@ HostIF_Init(VMDriver *vm, uint32 numVCPUs)
   }
   MutexInit(&vm->vmhost->vmMutex, "vm");

-   return 0;
+   return TRUE;
 error:
   if (vm->vmhost != NULL) {
      HostIFHostMemCleanup(vm);
@@ -1079,7 +1051,7 @@ error:
      MemTrack_Cleanup(vm->memtracker, UnlockEntry, vm);
      vm->memtracker = NULL;
   }
-   return -1;
+   return FALSE;
 }


@@ -1248,11 +1220,7 @@ HostIF_LockPage(VMDriver *vm,                // IN: VMDriver
   vpn = PTR_2_VPN(uvAddr);
   if (!allowMultipleMPNsPerVA) {
      entryPtr = MemTrack_LookupVPN(vm->memtracker, vpn);
-
-      /*
-       * Already tracked and locked
-       */
-
+      /* Already tracked and locked */
      if (entryPtr != NULL && entryPtr->mpn != 0) {
         return PAGE_LOCK_ALREADY_LOCKED;
      }
@@ -1265,10 +1233,7 @@ HostIF_LockPage(VMDriver *vm,                // IN: VMDriver
   *mpn = (MPN)page_to_pfn(page);

   if (allowMultipleMPNsPerVA) {
-      /*
-       *  Add the MPN to the PhysTracker that tracks locked pages.
-       */
-
+      /* Add the MPN to the PhysTracker that tracks locked pages */
      struct PhysTracker* const pt = vm->vmhost->lockedPages;

      if (PhysTrack_Test(pt, *mpn)) {
@@ -1485,7 +1450,7 @@ HostIF_FreeAllResources(VMDriver *vm) // IN

 void *
 HostIF_AllocKernelMem(size_t size,  // IN:
-                      int wired)    // IN:
+                      Bool wired)   // IN:
 {
   void * ptr = kmalloc(size, GFP_KERNEL);

@@ -1629,19 +1594,21 @@ HostIF_UnmapPage(VPN vpn) // IN:
 *----------------------------------------------------------------------
 */

-unsigned int
+PageCnt
 HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
-                               unsigned int currentlyLockedPages) // IN
+                               PageCnt currentlyLockedPages)      // IN
 {
   /*
    * This variable is available and exported to modules,
    * since at least 2.6.0.
    */

+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
   extern unsigned long totalram_pages;
-
-   unsigned int totalPhysicalPages = totalram_pages;
-
+   PageCnt totalPhysicalPages = totalram_pages;
+#else
+   PageCnt totalPhysicalPages = totalram_pages();
+#endif
   /*
    * Use the memory information linux exports as of late for a more
    * precise estimate of locked memory.  All kernel page-related structures
@@ -1650,14 +1617,13 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
    * also as good as locked, since we don't use them.  Lastly, without
    * available swap, anonymous pages become locked in memory as well.
    */
-
-   unsigned int forHost;
-   unsigned int reservedPages = MEMDEFAULTS_MIN_HOST_PAGES;
-   unsigned int hugePages = (vm == NULL) ? 0 :
-      BYTES_2_PAGES(vm->memInfo.hugePageBytes);
-   unsigned int lockedPages = hugePages + reservedPages;
-   unsigned int anonPages;
-   unsigned int swapPages = BYTES_2_PAGES(linuxState.swapSize);
+   PageCnt forHost;
+   PageCnt reservedPages = MEMDEFAULTS_MIN_HOST_PAGES;
+   PageCnt hugePages = (vm == NULL) ? 0
+                                    : BYTES_2_PAGES(vm->memInfo.hugePageBytes);
+   PageCnt lockedPages = hugePages + reservedPages;
+   PageCnt anonPages;
+   PageCnt swapPages = BYTES_2_PAGES(linuxState.swapSize);

   /* global_page_state is global_zone_page_state in 4.14. */
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
@@ -1738,6 +1704,49 @@ HostIF_WaitForFreePages(unsigned int timeoutMs)  // IN:
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIFGetTime --
+ *
+ *      Reads the current time in UPTIME_FREQ units.
+ *
+ * Results:
+ *      The uptime, in units of UPTIME_FREQ.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static uint64
+HostIFGetTime(void)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+   struct timeval tv;
+
+   do_gettimeofday(&tv);
+   return tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ;
+#else
+   struct timespec64 now;
+
+   /*
+    * Use raw time used by Posix timers.  This time is not affected by
+    * NTP adjustments, so it may drift from real time and monotonic time,
+    * but it will stay in sync with other timers.
+    */
+   ktime_get_raw_ts64(&now);
+   /*
+    * UPTIME_FREQ resolution is lower than tv_nsec,
+    * so we have to do division...
+    */
+   ASSERT_ON_COMPILE(1000000000 % UPTIME_FREQ == 0);
+   return now.tv_nsec / (1000000000 / UPTIME_FREQ) + now.tv_sec * UPTIME_FREQ;
+#endif
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1769,7 +1778,6 @@ HostIF_WaitForFreePages(unsigned int timeoutMs)  // IN:
 static uint64
 HostIFReadUptimeWork(unsigned long *j)  // OUT: current jiffies
 {
-   struct timeval tv;
   uint64 monotime, uptime, upBase, monoBase;
   int64 diff;
   uint32 version;
@@ -1784,13 +1792,12 @@ HostIFReadUptimeWork(unsigned long *j)  // OUT: current jiffies
      monoBase = uptimeState.monotimeBase;
   } while (!VersionedAtomic_EndTryRead(&uptimeState.version, version));

-   do_gettimeofday(&tv);
+   uptime = HostIFGetTime();
   upBase = Atomic_Read64(&uptimeState.uptimeBase);

   monotime = (uint64)(jifs - jifBase) * (UPTIME_FREQ / HZ);
   monotime += monoBase;

-   uptime = tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ;
   uptime += upBase;

   /*
@@ -1895,13 +1902,11 @@ HostIFUptimeResyncMono(struct timer_list *timer)  // IN: ignored
 void
 HostIF_InitUptime(void)
 {
-   struct timeval tv;
+   uint64 tm;

   uptimeState.jiffiesBase = jiffies;
-   do_gettimeofday(&tv);
-   Atomic_Write64(&uptimeState.uptimeBase,
-                  -(tv.tv_usec * (UPTIME_FREQ / 1000000) +
-                    tv.tv_sec * UPTIME_FREQ));
+   tm = HostIFGetTime();
+   Atomic_Write64(&uptimeState.uptimeBase, -tm);

 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
   init_timer(&uptimeState.timer);
@@ -2113,10 +2118,10 @@ HostIF_MapCrossPage(VMDriver *vm, // IN
 */

 void *
-HostIF_AllocKernelPages(unsigned numPages, // IN: Number of pages
+HostIF_AllocKernelPages(PageCnt numPages,  // IN: Number of pages
                        MPN     *mpns)     // OUT: Array of MPNs
 {
-   unsigned i;
+   PageCnt i;
   void *ptr = vmalloc(numPages * PAGE_SIZE);

   if (ptr == NULL) {
@@ -2147,7 +2152,7 @@ HostIF_AllocKernelPages(unsigned numPages, // IN: Number of pages
 */

 void
-HostIF_FreeKernelPages(unsigned numPages, // IN: Number of pages
+HostIF_FreeKernelPages(PageCnt numPages,  // IN: Number of pages
                       void    *ptr)      // IN: Kernel VA of first page
 {
   vfree(ptr);
@@ -2285,7 +2290,7 @@ isVAReadable(VA r)  // IN:
   int ret;

   old_fs = get_fs();
-   set_fs(get_ds());
+   set_fs(KERNEL_DS);
   r = APICR_TO_ADDR(r, APICR_VERSION);
   ret = HostIF_CopyFromUser(&dummy, r, sizeof dummy);
   set_fs(old_fs);
@@ -2318,7 +2323,7 @@ SetVMAPICAddr(VMDriver *vm, // IN/OUT: driver state
   volatile void *hostapic;

   ASSERT_ON_COMPILE(APICR_SIZE <= PAGE_SIZE);
-   hostapic = (volatile void *) ioremap_nocache(ma, PAGE_SIZE);
+   hostapic = (volatile void *) ioremap(ma, PAGE_SIZE);
   if (hostapic) {
      if ((APIC_VERSIONREG(hostapic) & 0xF0) == 0x10) {
         vm->hostAPIC.base = (volatile uint32 (*)[4]) hostapic;
@@ -2421,7 +2426,7 @@ HostIF_APICInit(VMDriver *vm,   // IN:
      apicIPILogged = TRUE;
   }

-   if ((__GET_MSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
      if (setVMPtr) {
         vm->hostAPIC.base = NULL;
         vm->vmhost->hostAPICIsMapped = FALSE;
@@ -2562,7 +2567,7 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   }

   old_fs = get_fs();
-   set_fs(get_ds());
+   set_fs(KERNEL_DS);

   {
      struct poll_wqueues table;
@@ -2647,8 +2652,12 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
                            const VCPUSet *vcs) // IN:
 {
   FOR_EACH_VCPU_IN_SET_WITH_MAX(vcs, vcpuid, vm->numVCPUs) {
-      struct task_struct *t = vm->vmhost->vcpuSemaTask[vcpuid];
-      vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
+      /*
+       * To avoid lost wake-up race and to avoid redundant wakeups, use
+       * "xchg" to read and clear atomically; wake_up_process() is non-lossy.
+       */
+      struct task_struct *t =
+         (struct task_struct *)xchg(&vm->vmhost->vcpuSemaTask[vcpuid], NULL);
      if (t && (t->state & TASK_INTERRUPTIBLE)) {
         wake_up_process(t);
      }
@@ -2691,7 +2700,7 @@ HostIF_SemaphoreSignal(uint64 *args)  // IN:
   }

   old_fs = get_fs();
-   set_fs(get_ds());
+   set_fs(KERNEL_DS);

   /*
    * Always write sizeof(uint64) bytes. This works fine for eventfd and
@@ -3142,6 +3151,26 @@ HostIF_GetNextAnonPage(VMDriver *vm, MPN inMPN)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * HostIF_GetNumAnonPages --
+ *
+ *      Returns the number of anon MPNs used by this VM.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+PageCnt
+HostIF_GetNumAnonPages(VMDriver *vm)
+{
+   if (!vm->vmhost || !vm->vmhost->AWEPages) {
+      return 0;
+   }
+   return PhysTrack_GetNumTrackedPages(vm->vmhost->AWEPages);
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -3359,7 +3388,6 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz.
      }
   } else {
      if (linuxState.fastClockThread) {
-         force_sig(SIGKILL, linuxState.fastClockThread);
         kthread_stop(linuxState.fastClockThread);

         linuxState.fastClockThread = NULL;
@@ -3369,140 +3397,6 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz.
   return 0;
 }

-
-/*
- *-----------------------------------------------------------------------------
- *
- * HostIF_MapUserMem --
- *
- *      Obtain kernel pointer to user memory. The pages backing the user memory
- *      address are locked into memory (this allows the pointer to be used in
- *      contexts where paging is undesirable or impossible).
- *
- * Results:
- *      On success, returns the kernel virtual address, along with a handle to
- *      be used for unmapping.
- *      On failure, returns NULL.
- *
- * Side effects:
- *      Yes.
- *
- *-----------------------------------------------------------------------------
- */
-
-void *
-HostIF_MapUserMem(VA addr,                  // IN: User memory virtual address
-                  size_t size,              // IN: Size of memory desired
-                  VMMappedUserMem **handle) // OUT: Handle to mapped memory
-{
-   void *p = (void *) (uintptr_t) addr;
-   VMMappedUserMem *newHandle;
-   VA offset = addr & (PAGE_SIZE - 1);
-   size_t numPagesNeeded = ((offset + size) / PAGE_SIZE) + 1;
-   size_t handleSize =
-      sizeof *newHandle + numPagesNeeded * sizeof newHandle->pages[0];
-   void *mappedAddr;
-
-   ASSERT(handle);
-
-   if (!access_ok(VERIFY_WRITE, p, size)) {
-      printk(KERN_ERR "%s: Couldn't verify write to uva 0x%p with size %"
-             FMTSZ"u\n", __func__, p, size);
-
-      return NULL;
-   }
-
-   newHandle = kmalloc(handleSize, GFP_KERNEL);
-   if (newHandle == NULL) {
-      printk(KERN_ERR "%s: Couldn't allocate %"FMTSZ"u bytes of memory\n",
-             __func__, handleSize);
-
-      return NULL;
-   }
-
-   if (HostIFGetUserPages(p, newHandle->pages, numPagesNeeded)) {
-      kfree(newHandle);
-      printk(KERN_ERR "%s: Couldn't get %"FMTSZ"u %s for uva 0x%p\n", __func__,
-             numPagesNeeded, numPagesNeeded > 1 ? "pages" : "page", p);
-
-      return NULL;
-   }
-
-   if (numPagesNeeded > 1) {
-      /*
-       * Unlike kmap(), vmap() can fail. If it does, we need to release the
-       * pages that we acquired in HostIFGetUserPages().
-       */
-
-      mappedAddr = vmap(newHandle->pages, numPagesNeeded, VM_MAP, PAGE_KERNEL);
-      if (mappedAddr == NULL) {
-         unsigned int i;
-         for (i = 0; i < numPagesNeeded; i++) {
-            put_page(newHandle->pages[i]);
-         }
-         kfree(newHandle);
-         printk(KERN_ERR "%s: Couldn't vmap %"FMTSZ"u %s for uva 0x%p\n",
-                __func__, numPagesNeeded,
-                numPagesNeeded > 1 ? "pages" : "page", p);
-
-         return NULL;
-      }
-   } else {
-      mappedAddr = kmap(newHandle->pages[0]);
-   }
-
-   printk(KERN_DEBUG "%s: p = 0x%p, offset = 0x%p, numPagesNeeded = %"FMTSZ"u,"
-          " handleSize = %"FMTSZ"u, mappedAddr = 0x%p\n",
-          __func__, p, (void *)offset, numPagesNeeded, handleSize, mappedAddr);
-
-   newHandle->numPages = numPagesNeeded;
-   newHandle->addr = mappedAddr;
-   *handle = newHandle;
-
-   return mappedAddr + offset;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * HostIF_UnmapUserMem --
- *
- *      Unmap user memory from HostIF_MapUserMem().
- *
- * Results:
- *      None.
- *
- * Side effects:
- *      Yes.
- *
- *-----------------------------------------------------------------------------
- */
-
-void
-HostIF_UnmapUserMem(VMMappedUserMem *handle) // IN: Handle to mapped memory
-{
-   unsigned int i;
-
-   if (handle == NULL) {
-      return;
-   }
-
-   printk(KERN_DEBUG "%s: numPages = %"FMTSZ"u, addr = 0x%p\n",
-          __func__, handle->numPages, handle->addr);
-
-   if (handle->numPages > 1) {
-      vunmap(handle->addr);
-   } else {
-      kunmap(handle->pages[0]);
-   }
-
-   for (i = 0; i < handle->numPages; i++) {
-      put_page(handle->pages[i]);
-   }
-   kfree(handle);
-}
-
 /*
 *-----------------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,12 +25,6 @@
 #ifndef _HOSTIF_PRIV_H_
 #define _HOSTIF_PRIV_H_

-/* Functions for mapping and unmapping userspace memory. */
-struct VMMappedUserMem;
-EXTERN void  *HostIF_MapUserMem(VA addr, size_t size,
-                                struct VMMappedUserMem **handle);
-EXTERN void   HostIF_UnmapUserMem(struct VMMappedUserMem *handle);
-
 /* Uptime-related functions. */
 EXTERN void   HostIF_InitUptime(void);
 EXTERN void   HostIF_CleanupUptime(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,7 +24,7 @@
 *
 */

-#if defined(linux)
+#if defined(__linux__)
 /* Must come before any kernel header file */
 #   include "driver-config.h"
 #   include <linux/kernel.h>
@@ -34,7 +34,6 @@
 #include "modulecall.h"
 #include "vmx86.h"
 #include "task.h"
-#include "initblock.h"
 #include "vm_basic_asm.h"
 #include "iocontrols.h"
 #include "hostif.h"
@@ -43,6 +42,7 @@
 #include "usercalldefs.h"
 #include "cpuid.h"
 #include "vmmblob.h"
+#include "sharedAreaVmmon.h"

 /*
 *----------------------------------------------------------------------
@@ -72,13 +72,15 @@ int
 Vmx86_RunVM(VMDriver *vm,   // IN:
            Vcpuid vcpuid)  // IN:
 {
-   uint32           retval    = MODULECALL_USERRETURN;
+   uint64           retval    = MODULECALL_USERRETURN;
   VMCrossPageData *crosspage;
   int              bailValue = 0;

-   ASSERT(vcpuid < vm->numVCPUs && CPUID_HostSupportsHV());
+   ASSERT(vcpuid < vm->numVCPUs);
+   if (vm->crosspage[vcpuid] == NULL) {
+      return USERCALL_VMX86ALLOCERR;
+   }
   crosspage = &vm->crosspage[vcpuid]->crosspageData;
-   ASSERT(crosspage);

   /*
    * Check if we were interrupted by signal.
@@ -123,7 +125,8 @@ skipTaskSwitch:;
          * from the ioctl (back to the userlevel side of a VCPU thread).
          */
         bailValue = crosspage->userCallType;
-         crosspage->retval = retval;
+         ASSERT(retval == (uint64)((uint32)retval));
+         crosspage->retval = (uint32)retval;
         goto bailOut;
      }

@@ -136,18 +139,19 @@ skipTaskSwitch:;

      case MODULECALL_GET_RECYCLED_PAGES: {
         MPN mpns[MODULECALL_NUM_ARGS];
-         int nPages = MIN((int)crosspage->args[0], MODULECALL_NUM_ARGS);
-
-         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages, TRUE,
-                                         FALSE);
+         PageCnt nPages = MIN(crosspage->args[0], MODULECALL_NUM_ARGS);
+         ASSERT((int64)crosspage->args[0] >= 0);
+         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages,
+                                         TRUE, FALSE);
         if (retval <= nPages) {
-            int i;
+            PageCnt i;
            for (i = 0; i < retval; i++) {
               crosspage->args[i] = mpns[i];
            }
         } else {
-            // retval is holding an error code
-            Warning("Failed to alloc %u pages: %d\n", nPages, (int)retval);
+            /* retval is holding an error code */
+            Warning("Failed to alloc %"FMT64"u pages: %"FMT64"d\n", nPages,
+                    (int64)retval);
            retval = 0;
         }
         break;
@@ -197,7 +201,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_RELEASE_ANON_PAGES: {
-         unsigned count;
+         PageCnt count;
         MPN mpns[MODULECALL_NUM_ARGS];
         for (count = 0; count < MODULECALL_NUM_ARGS; count++) {
            mpns[count] = (MPN)crosspage->args[count];
@@ -206,15 +210,14 @@ skipTaskSwitch:;
            }
         }
         ASSERT(count > 0);
-         retval = Vmx86_FreeLockedPages(vm, PtrToVA64(mpns), count, TRUE);
+         retval = Vmx86_FreeLockedPages(vm, mpns, count);
         break;
      }

      case MODULECALL_LOOKUP_MPN: {
-         int i;
-         VPN64  vpn    = (VPN64)crosspage->args[0];
-         uint32 nPages = (uint32)crosspage->args[1];
-         VA64   uAddr  = (VA64)VPN_2_VA(vpn);
+         VPN64   vpn        = (VPN64)crosspage->args[0];
+         PageCnt i, nPages  = crosspage->args[1];
+         VA64    uAddr      = (VA64)VPN_2_VA(vpn);
         ASSERT(nPages <= MODULECALL_NUM_ARGS);
         HostIF_VMLock(vm, 38);
         for (i = 0; i < nPages; i++) {
@@ -292,17 +295,36 @@ skipTaskSwitch:;
         vm->blobInfo = NULL;
      } break;

+      case MODULECALL_GET_SHARED_AREA: {
+         SharedAreaVmmonRequest request;
+         request.type       = (SharedAreaType)crosspage->args[0];
+         request.vcpu       = (Vcpuid)crosspage->args[1];
+         request.offset     = (PageCnt)crosspage->args[2];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[3] = SharedAreaVmmon_GetRegionMPN(vm, &request);
+      } break;
+
+      case MODULECALL_GET_STAT_VARS: {
+         Vcpuid  vcpu   = (Vcpuid)crosspage->args[0];
+         PageCnt offset = (PageCnt)crosspage->args[1];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[2] = StatVarsVmmon_GetRegionMPN(vm, vcpu, offset);
+      } break;
+
+      case MODULECALL_GET_NUM_PTP_PAGES: {
+         /* Store PageCnt in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[1] = vm->numPTPPages;
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
-
-      crosspage->retval = retval;
-
-#if defined(linux)
+      ASSERT(retval == (uint64)((uint32)retval));
+      crosspage->retval = (uint32)retval;
+#if defined(__linux__)
      cond_resched(); // Other kernels are preemptable
 #endif
   }
-
 bailOut:
   return bailValue;
 }
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -129,6 +136,9 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

+# Add Spectre options when available
+CC_OPTS += $(call vm_check_gcc,-mindirect-branch=thunk -mindirect-branch-register,)
+
 include $(SRCROOT)/Makefile.kernel

 else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,8 @@

 #define ETHER_MAX_QUEUED_PACKET 1600

+/* Most Ethernet equipment can support jumbo frames up to 9216 bytes. */
+#define ETHER_MAX_JUMBO_FRAME_LEN 9216

 /*
 * State's that a NIC can be in currently we only use this
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -28,6 +28,7 @@

 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
 #include <linux/mm.h>
 #include "compat_skbuff.h"
 #include <linux/sockios.h>
@@ -44,6 +45,22 @@
 #include "vmnetInt.h"


+/*
+ * Default min MTU value as defined by kernel versions >= 4.10.0.
+ * Use the same value for earlier versions of the kernel which do not
+ * enforce a minimum MTU size.
+ */
+#define VMNET_MIN_MTU 68
+#define VMNET_MAX_MTU (ETHER_MAX_JUMBO_FRAME_LEN - \
+                      (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
+
+#ifdef RHEL_RELEASE_CODE
+#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 5) && \
+     RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8, 0))
+#define ndo_change_mtu ndo_change_mtu_rh74
+#endif
+#endif /* RHEL_RELEASE_CODE */
+
 typedef struct VNetNetIF {
   VNetPort                port;
   struct net_device      *dev;
@@ -64,6 +81,11 @@ static int  VNetNetifSetMAC(struct net_device *dev, void *addr);
 static void VNetNetifSetMulticast(struct net_device *dev);
 static int  VNetNetIfProcRead(char *page, char **start, off_t off,
                              int count, int *eof, void *data);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+static int VNetNetifChangeMtu(struct net_device *dev, int new_mtu);
+#endif
+

 /*
 *----------------------------------------------------------------------
@@ -84,7 +106,7 @@ static int  VNetNetIfProcRead(char *page, char **start, off_t off,
 static void
 VNetNetIfSetup(struct net_device *dev)  // IN:
 {
-#if !COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) || defined(HAVE_NET_DEVICE_OPS)
+#ifdef HAVE_NET_DEVICE_OPS
   static const struct net_device_ops vnetNetifOps = {
      .ndo_init = VNetNetifProbe,
      .ndo_open = VNetNetifOpen,
@@ -97,12 +119,23 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
 #else
      .ndo_set_rx_mode = VNetNetifSetMulticast,
 #endif
-
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
+      .ndo_change_mtu = VNetNetifChangeMtu,
+#endif
   };
+#endif /* HAVE_NET_DEVICE_OPS */
+
+   /* Turns on IFF_BROADCAST, IFF_MULTICAST. */
+   ether_setup(dev);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+   /* Set the maximum allowed MTU value. */
+   dev->max_mtu = VMNET_MAX_MTU;
 #endif

-   ether_setup(dev); // turns on IFF_BROADCAST, IFF_MULTICAST
-#if COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) && !defined(HAVE_NET_DEVICE_OPS) 
+#ifdef HAVE_NET_DEVICE_OPS
+   dev->netdev_ops = &vnetNetifOps;
+#else
   dev->init = VNetNetifProbe;
   dev->open = VNetNetifOpen;
   dev->hard_start_xmit = VNetNetifStartXmit;
@@ -110,9 +143,10 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
   dev->get_stats = VNetNetifGetStats;
   dev->set_mac_address = VNetNetifSetMAC;
   dev->set_multicast_list = VNetNetifSetMulticast;
-#else
-   dev->netdev_ops = &vnetNetifOps;
-   #endif /* HAVE_NET_DEVICE_OPS */
+#ifdef HAVE_CHANGE_MTU
+   dev->change_mtu = VNetNetifChangeMtu;
+#endif /* HAVE_CHANGE_MTU */
+#endif /* HAVE_NET_DEVICE_OPS */

 }

@@ -530,6 +564,38 @@ VNetNetifSetMulticast(struct net_device *dev) // IN: unused
 }


+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+/*
+ *----------------------------------------------------------------------
+ *
+ * VNetNetifChangeMtu --
+ *
+ *      Changes the current MTU value of a given vmnet interface.
+ *
+ * Results:
+ *      Returns zero on success, or invalid argument error if an incorrect
+ *      MTU size is being set.
+ *
+ * Side effects:
+ *      None.
+ *----------------------------------------------------------------------
+ */
+
+int
+VNetNetifChangeMtu(struct net_device *dev, // IN/OUT: assigning the mtu member.
+                   int new_mtu)            // IN:     new mtu value.
+{
+   if (new_mtu < VMNET_MIN_MTU || new_mtu > VMNET_MAX_MTU) {
+      return -EINVAL;
+   }
+
+   dev->mtu = new_mtu;
+   return 0;
+}
+#endif
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013, 2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -137,6 +137,7 @@ VNetProcShow(struct seq_file *p, // IN:
 }


+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0)
 /*
 *----------------------------------------------------------------------
 *
@@ -168,6 +169,7 @@ static struct file_operations fops = {
   .release = single_release,
 };
 #endif
+#endif


 /*
@@ -203,7 +205,12 @@ VNetProcMakeEntryInt(VNetProcEntry   *parent,   // IN:
      } else {
         ent->data   = data;
         ent->fn     = fn;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+         ent->pde    = proc_create_single_data(name, mode, parent->pde,
+                                               VNetProcShow, ent);
+#else
         ent->pde    = proc_create_data(name, mode, parent->pde, &fops, ent);
+#endif
      }
      if (ent->pde != NULL) {
         *ret = ent;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,8 +41,6 @@

 #else /* _WIN32 */

-#undef __KERNEL__ //To prevent including any kernel specific stuff 
-
 #ifdef VMX86_DEVEL
 #define DBG 1
 #else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -78,11 +78,11 @@ static int  VNetUserIfSetUplinkState(VNetPort *port, uint8 linkUp);
 extern unsigned int  vnet_max_qlen;

 #if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
-#   define compat_kmap(page) kmap(page)
-#   define compat_kunmap(page) kunmap(page)
-#else
-#   define compat_kmap(page) kmap((page).p)
-#   define compat_kunmap(page) kunmap((page).p)
+#   define skb_frag_page(frag) (frag)->page
+#   define skb_frag_size(frag) (frag)->size
+#endif
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0)
+#   define skb_frag_off(frag) (frag)->page_offset
 #endif

 /*
@@ -137,16 +137,21 @@ UserifLockPage(VA addr) // IN
 */

 static INLINE int
-VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapPtr(VA64 uAddr,      // IN: pointer to user memory
                 size_t size,     // IN: size of data
                 struct page **p, // OUT: locked page
                 void **ptr)      // OUT: kernel mapped pointer
 {
-   if (!access_ok(VERIFY_WRITE, (void *)uAddr, size) ||
-       (((uAddr + size - 1) & ~(PAGE_SIZE - 1)) !=
-        (uAddr & ~(PAGE_SIZE - 1)))) {
+   uint8 v;
+
+   /* Check area does not straddle two pages. */
+   if ((uAddr & (PAGE_SIZE - 1)) + size > PAGE_SIZE) {
      return -EINVAL;
   }
+   /* Check if it is user's area.  UserifLockPage() checks writability. */
+   if (copy_from_user(&v, (void *)(unsigned long)uAddr, sizeof v) != 0) {
+      return -EFAULT;
+   }

   *p = UserifLockPage(uAddr);
   if (*p == NULL) {
@@ -158,7 +163,7 @@ VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
 }

 static INLINE int
-VNetUserIfMapUint32Ptr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapUint32Ptr(VA64 uAddr,      // IN: pointer to user memory
                       struct page **p, // OUT: locked page
                       uint32 **ptr)    // OUT: kernel mapped pointer
 {
@@ -201,7 +206,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      return -EBUSY;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->pollPtr, &pollPage,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->pollPtr, &pollPage,
                                        &pollPtr)) < 0) {
      return retval;
   }
@@ -213,7 +218,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      goto error_free;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->recvClusterPtr,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->recvClusterPtr,
                                        &recvClusterPage,
                                        &recvClusterCount)) < 0) {
      goto error_free;
@@ -362,12 +367,12 @@ VNetUserIfReceive(VNetJack       *this, // IN
   VNetUserIF *userIf = (VNetUserIF*)this->private;
   uint8 *dest = SKB_2_DESTMAC(skb);
   unsigned long flags;
-   
+
   if (!UP_AND_RUNNING(userIf->port.flags)) {
      userIf->stats.droppedDown++;
      goto drop_packet;
   }
-   
+
   if (!VNetPacketMatch(dest,
                        userIf->port.paddr,
                        (const uint8 *)userIf->port.exactFilter,
@@ -377,13 +382,17 @@ VNetUserIfReceive(VNetJack       *this, // IN
      userIf->stats.droppedMismatch++;
      goto drop_packet;
   }
-   
+
   if (skb_queue_len(&userIf->packetQueue) >= vnet_max_qlen) {
      userIf->stats.droppedOverflow++;
      goto drop_packet;
   }
-   
-   if (skb->len > ETHER_MAX_QUEUED_PACKET) {
+
+   /*
+    * The check would be more accurate if based on the current MTU value
+    * of the corresponding vmnet interface. PR 2267716 is filed to track this.
+    */
+   if (skb->len > ETHER_MAX_JUMBO_FRAME_LEN) {
      userIf->stats.droppedLargePacket++;
      goto drop_packet;
   }
@@ -559,20 +568,20 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
   for (frag = skb_shinfo(skb)->frags;
 	frag != skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
 	frag++) {
-      if (frag->size > 0) {
+      if (skb_frag_size(frag) > 0) {
 	 unsigned int tmpCsum;
 	 const void *vaddr;

-	 vaddr = compat_kmap(frag->page);
-	 tmpCsum = csum_and_copy_to_user(vaddr + frag->page_offset,
-					 curr, frag->size, 0, &err);
-	 compat_kunmap(frag->page);
+	 vaddr = kmap(skb_frag_page(frag));
+	 tmpCsum = csum_and_copy_to_user(vaddr + skb_frag_off(frag),
+					 curr, skb_frag_size(frag), 0, &err);
+	 kunmap(skb_frag_page(frag));

 	 if (err) {
 	    return err;
 	 }
 	 csum = csum_block_add(csum, tmpCsum, curr - buf);
-	 curr += frag->size;
+	 curr += skb_frag_size(frag);
      }
   }

@@ -747,7 +756,7 @@ VNetUserIfRead(VNetPort    *port, // IN
 *----------------------------------------------------------------------
 */

-static int 
+static int
 VNetUserIfWrite(VNetPort    *port, // IN
                struct file *filp, // IN
                const char  *buf,  // IN
@@ -757,11 +766,14 @@ VNetUserIfWrite(VNetPort    *port, // IN
   struct sk_buff *skb;

   /*
-    * Check size
+    * Check size.
+    *
+    * Regarding the upper count limit the check would be more accurate
+    * if based on the current MTU value of the corresponding vmnet interface.
+    * PR 2267716 is filed to track this.
    */
-   
-   if (count < sizeof (struct ethhdr) || 
-       count > ETHER_MAX_QUEUED_PACKET) {
+   if (count < sizeof (struct ethhdr) ||
+       count > ETHER_MAX_JUMBO_FRAME_LEN) {
      return -EINVAL;
   }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -202,7 +202,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1317,7 +1317,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
+   __asm lock And [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_And
@@ -1382,7 +1382,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
+   __asm lock Or [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Or
@@ -1447,7 +1447,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Xor
@@ -2303,7 +2303,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2324,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3618,7 +3621,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -221,8 +221,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +263,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +376,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +392,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -768,12 +771,14 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
@@ -888,7 +893,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +907,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1103,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1161,101 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,12 +49,29 @@ uint64 _umul128(uint64 multiplier, uint64 multiplicand,
 int64 _mul128(int64 multiplier, int64 multiplicand,
              int64 *highProduct);
 uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
+#ifdef ULM
+void _fxsave64(void *save);
+void _fxsave(void *save);
+void _fxrstor64(const void *load);
+void _fxrstor(const void *load);
+void _xsave64(void *save, uint64 mask);
+void _xsave(void *save, uint64 mask);
+void _xsaveopt64(void *save, uint64 mask);
+void _xsavec(void *save, uint64 mask);
+void _xrstor64(const void *load, uint64 mask);
+void _xrstor(const void *load, uint64 mask);
+#endif /* ULM */
 #ifdef __cplusplus
 }
 #endif

 #pragma intrinsic(_umul128, _mul128, __shiftright128)

+#ifdef ULM
+#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
+                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
+#endif /* ULM */
+
 #endif // _MSC_VER

 #if defined(__GNUC__)
@@ -113,40 +130,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +204,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +307,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +351,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +433,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +518,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +542,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -244,9 +244,131 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+#if defined VM_X86_32
+   __asm sfence;
+#else
+   _mm_sfence();
+#endif
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +400,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +515,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +533,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +613,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,12 +44,44 @@
 #endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -58,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -159,18 +157,6 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

-/*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-

 /*
 * Token concatenation
@@ -209,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -255,6 +241,10 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif
@@ -286,6 +276,14 @@ Max(int a, int b)
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -556,10 +554,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -664,6 +662,20 @@ typedef int pid_t;
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
@@ -702,12 +714,6 @@ typedef int pid_t;
 #endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -718,7 +724,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,18 +63,9 @@
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/*
- * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -83,7 +74,6 @@
 #ifdef __x86_64__
 #define VM_X86_64
 #define vm_x86_64 1
-#define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
 #else
@@ -120,14 +110,91 @@

 #endif

-#if defined(__cplusplus) && __cplusplus >= 201103L || \
-    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
-    defined(__APPLE__) || defined(HAVE_STDINT_H)
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif
+
 /*
- * We're using <stdint.h> instead of <cstdint> below because some C++ code
- * deliberately compiles without C++ include paths.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif
+

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -138,40 +205,6 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-#else /* !HAVE_STDINT_H */
-
-/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
-#ifdef _MSC_VER
-
-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
-
-#elif __GNUC__
-#   if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#   else
-/*
- * Only strict c90 (without extensions) lacks a 'long long' type.
- * If this declaration fails ... use -std=c99 or -std=gnu90.
- */
-typedef unsigned long long uint64;
-typedef long long int64;
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-

 /*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
@@ -202,14 +235,6 @@ typedef char           Bool;
 #define IS_BOOL(x)     (((x) & ~1) == 0)


-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */
-
 /*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
@@ -218,9 +243,6 @@ typedef char           Bool;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -234,14 +256,11 @@ typedef char           Bool;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -249,46 +268,10 @@ typedef char           Bool;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -337,10 +320,11 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* macOS hosts use the same formatters for 32- and 64-bit. */
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
   #define FMT64         "ll"
-   #if KERNEL
+   #if defined(__APPLE__) && KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
@@ -452,7 +436,6 @@ typedef uintptr_t VPN;

 typedef uint64    PA;
 typedef uint64    PPN;
-typedef uint32    PPNTMP;

 typedef uint64    TPA;
 typedef uint64    TPPN;
@@ -470,9 +453,7 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint64    PgCnt64;
 typedef uint64    PageCnt;
-typedef uint64    PgNum64;
 typedef uint64    PageNum;
 typedef unsigned  MemHandle;
 typedef unsigned  IoHandle;
@@ -515,7 +496,7 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
@@ -598,15 +579,14 @@ typedef void * UserVA;


 /* Maximal observable PPN value. */
-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
 #define INVALID_PPN32     ((PPN32)0xffffffff)
-#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
-#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -624,7 +604,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((uint32)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -837,7 +818,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -934,82 +915,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2005-2012,2014-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2005-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,7 @@
 *    VMware HD Audio controller
 */
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
+#define PCI_DEVICE_ID_VMWARE_SVGA3              0x0406
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
 #define PCI_DEVICE_ID_VMWARE_VGA                0x0711
@@ -138,23 +139,24 @@
 *    Intel 82545EM (e1000, server adapter, single port)
 *    Intel 82546EB (e1000, server adapter, dual port)
 *    Intel HECI (as embedded in ich9m)
- *    Intel XHCI (Panther Point / Intel 7 Series)
+ *    Intel XHCI (Panther Point / Intel 7 Series, 5Gbps)
+ *    Intel XHCI (Cannon Lake / Intel 300 Series, 10Gbps)
 */
-#define PCI_VENDOR_ID_INTEL             0x8086
-#define PCI_DEVICE_ID_INTEL_82439TX     0x7100
-#define PCI_DEVICE_ID_INTEL_82371AB_0   0x7110
-#define PCI_DEVICE_ID_INTEL_82371AB_2   0x7112
-#define PCI_DEVICE_ID_INTEL_82371AB_3   0x7113
-#define PCI_DEVICE_ID_INTEL_82371AB     0x7111
-#define PCI_DEVICE_ID_INTEL_82443BX     0x7190
-#define PCI_DEVICE_ID_INTEL_82443BX_1   0x7191
-#define PCI_DEVICE_ID_INTEL_82443BX_2   0x7192 /* Used when no AGP support */
-#define PCI_DEVICE_ID_INTEL_82545EM     0x100f
-#define PCI_DEVICE_ID_INTEL_82546EB     0x1010
-#define PCI_DEVICE_ID_INTEL_82574       0x10d3
-#define PCI_DEVICE_ID_INTEL_82574_APPLE 0x10f6
-#define PCI_DEVICE_ID_INTEL_HECI        0x2a74
+#define PCI_VENDOR_ID_INTEL                   0x8086
+#define PCI_DEVICE_ID_INTEL_82439TX           0x7100
+#define PCI_DEVICE_ID_INTEL_82371AB_0         0x7110
+#define PCI_DEVICE_ID_INTEL_82371AB_2         0x7112
+#define PCI_DEVICE_ID_INTEL_82371AB_3         0x7113
+#define PCI_DEVICE_ID_INTEL_82371AB           0x7111
+#define PCI_DEVICE_ID_INTEL_82443BX           0x7190
+#define PCI_DEVICE_ID_INTEL_82443BX_1         0x7191
+#define PCI_DEVICE_ID_INTEL_82443BX_2         0x7192 /* Used when no AGP support */
+#define PCI_DEVICE_ID_INTEL_82545EM           0x100f
+#define PCI_DEVICE_ID_INTEL_82546EB           0x1010
+#define PCI_DEVICE_ID_INTEL_82574             0x10d3
+#define PCI_DEVICE_ID_INTEL_82574_APPLE       0x10f6
 #define PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI 0x1e31
+#define PCI_DEVICE_ID_INTEL_CANNONLAKE_XHCI   0xa36d

 /*
 *  From drivers/usb/host/xhci-pci.c:
@@ -347,6 +349,9 @@
 /************* QAT implementation limits ********************/
 #define MAX_QAT_PCI_DEVICES 4

+/************* PrecisionClock implementation limits ********************/
+#define MAX_PRECISIONCLOCK_DEVICES 1
+
 /************* Strings for Host USB Driver *******************************/

 #ifdef _WIN32
Author	SHA1	Message	Date
Michal Kubecek	cbd3374fd0	import Player 15.5.2 module sources	2020-03-14 09:59:24 +01:00
Michal Kubecek	87feab5f03	import Player 15.5.1 module sources	2019-11-14 07:56:57 +01:00
Michal Kubecek	6daa6921e7	import Player 15.5.0 module sources	2019-09-21 00:11:26 +02:00
Michal Kubecek	193db5638c	Merge branch 'master' into player	2019-09-21 00:10:31 +02:00
Michal Kubecek	85ede6aae9	misc: fix wrong directory name in INSTALL (fixes #36 ) Fix wrong directory name in one of the examples.	2019-09-09 08:38:27 +02:00
Michal Kubecek	1f8ca44e74	import Player 15.1.0 module sources	2019-05-15 07:06:02 +02:00
Michal Kubecek	3686288fcb	import Player 15.0.1 module sources	2018-11-12 07:48:28 +01:00
Michal Kubecek	3d8f309653	import Player 15.0.0 module sources	2018-10-02 19:09:36 +02:00
Michal Kubecek	56b2ba7686	import Player 14.1.3 module sources	2018-08-20 07:28:17 +02:00
Michal Kubecek	7c61516e73	Merge branch 'master' into player	2018-05-27 16:13:48 +02:00
Michal Kubecek	7ee6cd4520	import Player 14.1.2 module sources	2018-05-23 18:14:05 +02:00
Michal Kubecek	5791987f56	Merge branch 'master' into player	2018-04-02 15:37:18 +02:00
Michal Kubecek	0b2713797e	Merge branch 'master' into player	2018-02-04 21:50:37 +01:00
Michal Kubecek	3e445144a1	import Player 14.1.1 module sources	2018-01-12 13:51:34 +01:00
Michal Kubecek	82f569a212	import Player 14.1.0 module sources	2017-12-25 09:47:38 +01:00
Michal Kubecek	455d4d9368	import Player 14.0.0 module sources	2017-09-30 20:31:15 +02:00
Michal Kubecek	027dd56418	import Player 12.5.7 module sources	2017-06-30 19:11:22 +02:00
Michal Kubecek	0e634ffbf1	import Player 12.5.6 module sources	2017-05-31 10:06:51 +02:00
Michal Kubecek	c62c78e908	import Player 12.5.5 module sources	2017-05-31 09:20:00 +02:00
Michal Kubecek	4f648692e4	import Player 12.5.2 module sources	2017-05-31 09:19:26 +02:00