vmnet: work around skb_frag_t build failure for SLE15-SP2 and openSUSE-15.2

The kernels used in (upcoming) SLE15 SP2 and openSUSE Leap 15.2 are based on upstream version 5.3 but have a backport of the skb_frag_t conversion to bio_vec which reached mainline in v5.4-rc1. As we cannot easily check availability compat_skb_frag_off() (which was also added in v5.4-rc1) and have to resort to version check, add an exception for SLE15-SP2 and openSUSE-15.2 kernels.
vmnet: handle switch of skb_frag_t to bio_vec
2019-11-25 08:19:03 +01:00 · 2019-11-14 08:06:57 +01:00 · 2019-11-14 08:06:57 +01:00 · 2019-11-14 08:06:57 +01:00 · 2019-11-14 08:06:57 +01:00 · 2019-11-14 08:06:57 +01:00
91 changed files with 5383 additions and 4337 deletions
@@ -150,7 +150,7 @@ test build).
 When using a downloaded tarball, simply run

  tar -cf vmmon.tar vmmon-only
-  tar -cf vmnet.tar vmmon-only
+  tar -cf vmnet.tar vmnet-only

 to create the tarballs. In this case, there is no protection against
 unwanted local changes.
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -31,7 +31,7 @@ $(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, \
 		$(SRCROOT)/bootstrap/*.c)))

 clean:
-	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \
+	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions .cache.mk \
 		Module.symvers Modules.symvers Module.markers modules.order \
 		$(foreach dir,linux/ common/ vmcore/ bootstrap/ \
 		./,$(addprefix $(dir),.*.cmd .*.o.flags *.o)))
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,14 +24,7 @@
 *      See monLoader.h for a full description.
 */

-#ifdef VMX86_VMX
-
-#include <string.h>
-#include <limits.h>
-#include "vmx.h"
-#include "vm_assert.h"
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #include "vmkernel.h"
 #include "libc.h"
@@ -57,15 +50,15 @@

 #include "vm_assert.h"

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

 #include "vm_basic_types.h"
 #include "monLoader.h"
 #include "vcpuid.h"
-#include "x86types.h"
-#include "x86paging_64.h"
+#include "vm_pagetable.h"
+#include "address_defs.h"
 #include "monLoaderLog.h"

 #define CANONICAL_MASK MASK64(36)
@@ -75,12 +68,6 @@
 #define L2EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 18))
 #define L1EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 9))

-/*
- * Import address space from VMX rather than building it.  Once monitor loading
- * is removed from VMX entirely, code to import will be deleted.
- */
-#define IMPORT_AS_FROM_VMX (TRUE)
-
 /* The maximum (canonical-address) VPN */
 #define VPN_MAX MASK64(52)

@@ -111,15 +98,11 @@ typedef struct MonLoaderContext {
      VPN          ASLastVPN;   /* last VPN in the address space (inclusive) */
      uint64       ASPTEFlags;  /* PTE flags for L4->L1 connection. */
      Vcpuid       currentVCPU;
-      Bool         ASImported;  /* Was this AS imported or created anew? */
      Bool         hasAddrSpace;
   } vcpu;
 } MonLoaderContext;


-static const Bool importFromVMX = IMPORT_AS_FROM_VMX;
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -192,7 +175,7 @@ MonLoaderIsMapped(MonLoaderContext *ctx,    // IN/OUT
   if (ret != ML_OK) {
      return ret;
   }
-   *mapped = PTE_PRESENT(pte);
+   *mapped = ML_PERM_PRESENT(pte);
   return ML_OK;
 }

@@ -280,15 +263,40 @@ MonLoaderMapMPN(MonLoaderContext *ctx,    // IN/OUT
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderBuildsPtLevel --
+ *
+ *      Determines whether MonLoader allocates and maps the page table(s) for
+ *      the monitor at the given level.
+ *
+ * Result:
+ *      TRUE if MonLoader creates the page table at the given level, FALSE
+ *      otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+static Bool
+MonLoaderCreatesPtLevel(PT_Level level)
+{
+   ASSERT(level >= PT_LEVEL_STOP && level <= PT_MAX_LEVELS);
+   return FALSE;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
 * MonLoaderCreateAddressSpace --
 *
- *      Creates or (if importAS is set) verifies an address space.  The VPN
- *      range specified by firstVPN and size is used to determine page counts
- *      at each page table level to map every page.  Sufficient pages are then
- *      allocated or verified.
+ *      Creates or verifies an address space.  The VPN range specified by
+ *      firstVPN and size is used to determine page counts at each page table
+ *      level to map every page.  Sufficient pages are then allocated or
+ *      verified.
 *
 * Result:
 *      ML_OK if successful, some other value on error.
@@ -302,8 +310,7 @@ static MonLoaderError
 MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
                            VPN               firstVPN, // IN
                            uint64            flags,    // IN
-                            uint64            monPages, // IN
-                            Bool              importAS) // IN
+                            uint64            monPages) // IN
 {
   Vcpuid vcpu = ctx->vcpu.currentVCPU;
   VPN lastVPN = firstVPN + monPages - 1;
@@ -329,11 +336,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
   ctx->vcpu.ASFirstVPN = firstVPN;
   ctx->vcpu.ASLastVPN = lastVPN;
   ctx->vcpu.ASPTEFlags = flags;
-   ctx->vcpu.ASImported = importAS;

   ptMPNs = &ctx->vcpu.ptMPNs;

-   if (importAS) {
+   if (MonLoaderCreatesPtLevel(PT_LEVEL_4)) {
+      NOT_IMPLEMENTED();
+   } else {
+      /* Verify the VMX's allocation. */
      mpn = MonLoaderCallout_GetPageRoot(ctx->envCtx, vcpu);
      if (mpn == INVALID_MPN) {
         return ML_ERROR_CALLOUT_PAGEROOT_GET;
@@ -341,13 +350,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
      LOG(5, "%s: vcpu %u page root=0x%"FMT64"x\n", __FUNCTION__, vcpu, mpn);
      ptMPNs->L4MPNs[0] = mpn;
      MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-   } else {
-      NOT_IMPLEMENTED();
   }
   ptMPNs->L4MPNCount = 1;

   for (i = 0; i < L3MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_3)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L4E;
         unsigned L4Off = PT_LPN_2_L4OFF(monVPN);
         PT_L1E pte;
@@ -357,20 +366,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L4E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L3MPNs[i] = mpn;
   }
   ptMPNs->L3MPNCount = L3MPNsNeeded;
      
   for (i = 0; i < L2MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_2)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L3E;
         unsigned L3Off = PT_LPN_2_L3OFF(monVPN);
         unsigned L3Page = L3EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -381,20 +390,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L3E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L2MPNs[i] = mpn;
   }
   ptMPNs->L2MPNCount = L2MPNsNeeded;

   for (i = 0; i < L1MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_1)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L2E;
         unsigned L2Off = PT_LPN_2_L2OFF(monVPN);
         unsigned L2Page = L2EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -405,17 +414,15 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         if (!PTE_PRESENT(pte)) {
+         if (!ML_PERM_PRESENT(pte)) {
            ptMPNs->L1MPNs[i] = INVALID_MPN;
            continue;
         }
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L1MPNs[i] = mpn;
   }
@@ -446,12 +453,12 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
 */
 static MonLoaderError
 MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
-                       unsigned          level,    // IN
+                       PT_Level          level,    // IN
                       uint64            flags,    // IN
                       VPN               monVPN,   // IN
                       uint64            monPages) // IN
 {
-   Bool verify = ctx->vcpu.ASImported;
+   Bool verify = !MonLoaderCreatesPtLevel(level);
   uint64    i;
   unsigned  count;
   MPN      *ptMPNs;
@@ -460,25 +467,25 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
      if (!ctx->vcpu.hasAddrSpace) {
         return ML_ERROR_NO_ADDRSPACE;
      }
-      if (level < 1 || level > 4) {
+      if (level < PT_LEVEL_STOP || level > PT_MAX_LEVELS) {
         return ML_ERROR_ARGS;
      }
   }

   switch (level) {
-      case 4:
+      case PT_LEVEL_4:
         count = ctx->vcpu.ptMPNs.L4MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L4MPNs;
         break;
-      case 3:
+      case PT_LEVEL_3:
         count = ctx->vcpu.ptMPNs.L3MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L3MPNs;
         break;
-      case 2:
+      case PT_LEVEL_2:
         count = ctx->vcpu.ptMPNs.L2MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L2MPNs;
         break;
-      case 1:
+      case PT_LEVEL_1:
         count = ctx->vcpu.ptMPNs.L1MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L1MPNs;
         break;
@@ -494,11 +501,11 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
         VPN vpn = monVPN + i;
         if (verify) {
            PT_L1E l1e;
-            if (ptMPNs[i] == INVALID_MPN && level == 1) {
+            if (ptMPNs[i] == INVALID_MPN && level == PT_LEVEL_1) {
               continue;
            }
            if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK ||
-                LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
+                ML_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
               return ML_ERROR_PAGE_TABLE_VERIFY;
            }
         } else {
@@ -629,7 +636,7 @@ MonLoaderCopyFromBlob(MonLoaderContext *ctx,        // IN/OUT
 *      ML_OK if successful, some other value on error.
 *
 * Side effects:
- *      User pages may be mapped in.
+ *      User or host pages may be mapped in.
 *
 *----------------------------------------------------------------------
 */
@@ -660,6 +667,14 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
                                                  vcpu);
      }
      if (mpn == INVALID_MPN) {
+         if (subIndex == MONLOADER_HT_MAP_IDX) {
+            /*
+             * This item is tied to a vmkernel feature.
+             * When the feature is disabled, there is nothing to share.
+             */
+            ASSERT(vmx86_server);
+            return ML_OK;
+         }
         /* Partial sharing is allowed.  Return success if any occurred. */
         return i != 0 ? ML_OK : ML_ERROR_SHARE;
      }
@@ -672,45 +687,6 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
 }


-#ifndef VMMON
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderSetEntrypoint --
- *
- *      Calls out to set the code and stack entrypoint.
- *
- * Result:
- *      ML_OK if successful, some other value on error.
- *
- * Side effects:
- *      The monitor entrypoint for the current VCPU may be set.
- *
- *----------------------------------------------------------------------
- */
-static MonLoaderError
-MonLoaderSetEntrypoint(MonLoaderContext *ctx,           // IN
-                       uint16            codeSelector,  // IN
-                       VA64              code,          // IN
-                       uint16            stackSelector, // IN
-                       VA64              stack)         // IN
-{
-   if (VA_2_VPN(code) < ctx->vcpu.ASFirstVPN  ||
-       VA_2_VPN(code) > ctx->vcpu.ASLastVPN   ||
-       VA_2_VPN(stack) < ctx->vcpu.ASFirstVPN ||
-       VA_2_VPN(stack) > ctx->vcpu.ASLastVPN) {
-      return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
-   }
-   if (MonLoaderCallout_SetEntrypoint(ctx->envCtx, codeSelector, code,
-                                      stackSelector, stack)) {
-      return ML_OK;
-   } else {
-      return ML_ERROR_CALLOUT_ENTRYPOINT;
-   }
-}
-#endif
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -795,7 +771,7 @@ MonLoaderShareFromBlob(MonLoaderContext *ctx,        // IN/OUT

   if ((blobOffset & (PAGE_SIZE - 1)) != 0 ||
       (blobSize & (PAGE_SIZE - 1)) != 0 ||
-       (flags & PTE_RW) != 0) {
+       ML_PERM_WRITEABLE(flags)) {
      return ML_ERROR_SHARE;
   }
   if (blobSize > monBytes || blobSize == 0) {
@@ -866,7 +842,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
      return ML_ERROR_TABLE_MISSING;
   }

-   if (!MonLoaderCallout_Init(args, &ctx.envCtx)) {
+   if (!MonLoaderCallout_Init(args, &ctx.envCtx, numVCPUs)) {
      return ML_ERROR_CALLOUT_INIT;
   }
   ret = ML_OK;
@@ -887,7 +863,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         uint64               monBytes   = PAGES_2_BYTES(monPages);
         uint64               subIndex   = entry->subIndex;
         uint64               bspOnly    = entry->bspOnly;
-         uint64               isPriv     = vmx86_server || entry->procVmmon;
         *line = i;

         /*
@@ -896,17 +871,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         if (bspOnly && !IS_BOOT_VCPUID(*vcpu)) {
            continue;
         }
-         /*
-          * Entries marked as privileged (isPriv == TRUE) are processed
-          * by the privileged layer (VMKernel/vmmon). Others are
-          * processed by the VMX. One exception is the entry of type
-          * ML_CONTENT_ADDRSPACE: it is always processed by the VMX and,
-          * if marked as privileged, also by the privileged layer.
-          */
-         if (isPriv != MonLoaderCallout_IsPrivileged(ctx.envCtx) &&
-             (content != ML_CONTENT_ADDRSPACE || isPriv == 0)) {
-            continue;
-         }
         if (monPages == 0 || monVPN + monPages - 1 > VPN_MAX) {
            ret = ML_ERROR_SIZE;
            continue;
@@ -919,8 +883,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
                  continue;
               }
               /* Create or verify the address space and page table MPNs. */
-               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages,
-                                                 importFromVMX);
+               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages);
               break;
            case ML_CONTENT_PAGETABLE_L4:
            case ML_CONTENT_PAGETABLE_L3:
@@ -988,15 +951,10 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         }
      }
   }
-#ifndef VMMON
+#ifdef VMKERNEL
   if (ret == ML_OK) {
      if (ctx.vcpu.ASLastVPN == 0) {
         ret = ML_ERROR_NO_ADDRSPACE;
-      } else {
-         ret = MonLoaderSetEntrypoint(&ctx, header->codeSelector,
-                                      header->codeEntrypoint,
-                                      header->stackSelector,
-                                      header->stackEntrypoint);
      }
   }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,7 +65,7 @@ typedef struct MonLoaderEnvContext {
 */

 Bool
-MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx)
+MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx, unsigned numVCPUs)
 {
   MonLoaderEnvContext *c;
   MonLoaderArgs *mlArgs = (MonLoaderArgs *)args;
@@ -215,7 +215,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
   vpn = Vmx86_MapPage(mpn);
   if (vpn == 0) {
      Log("Failed to map MPN 0x%"FMT64"x\n", mpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   HostIF_VMLock(vm, 41);
@@ -225,7 +225,7 @@ MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
      Log("Failed to track mapping from VPN 0x%"FMTVPN"x to MPN 0x%"FMT64"x\n",
          vpn, mpn);
      Vmx86_UnmapPage(vpn);
-      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
      return INVALID_MPN;
   }
   return mpn;
@@ -386,10 +386,8 @@ MonLoaderFindSharedRegion(MonLoaderEnvContext *ctx,
   int i;

   for (i = 0; i < ML_SHARED_REGIONS_MAX; i++) {
-      if (ctx->shRegions[i].baseVpn == INVALID_VPN) {
-         /* Not found. */
-         return NULL;
-      } else if (ctx->shRegions[i].index == index) {
+      if (ctx->shRegions[i].index == index &&
+          ctx->shRegions[i].baseVpn != INVALID_VPN) {
         return &ctx->shRegions[i];
      }
   }
@@ -425,10 +423,6 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,

   ASSERT(IS_BOOT_VCPUID(vcpuid));

-   if (index >= ML_SHARED_REGIONS_MAX) {
-      Log("Invalid shared region %"FMT64"x", index);
-      return INVALID_MPN;
-   }
   s = MonLoaderFindSharedRegion(ctx, index);
   if (s == NULL) {
      return INVALID_MPN;
@@ -465,10 +459,10 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 */

 MPN
-MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx,
-                                   uint64 subIndex, // IN
-                                   unsigned page,   // IN
-                                   Vcpuid vcpu)     // IN
+MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx, // IN
+                                   uint64 subIndex,          // IN
+                                   unsigned page,            // IN
+                                   Vcpuid vcpu)              // IN
 {
   if (subIndex == MONLOADER_HEADER_IDX) {
      return VmmBlob_GetHeaderMpn(ctx->vm);
@@ -484,30 +478,7 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   NOT_IMPLEMENTED();
-}
-
-
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderCallout_IsPrivileged --
- *
- *      Returns whether or not this is a privileged environment.
- *
- * Returns:
- *      TRUE if privileged, FALSE if not.
- *
- * Side effects:
- *      None.
- *
- *----------------------------------------------------------------------
- */
-
-Bool
-MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN
-{
-   return TRUE;
+   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
 }


@@ -534,13 +505,3 @@ MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx,    // IN
   ASSERT((offset & (PAGE_SIZE - 1)) == 0);
   return VmmBlob_GetMpn(ctx->vm, BYTES_2_PAGES(offset));
 }
-
-Bool
-MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *ctx,           // IN
-                               uint16               codeSelector,  // IN
-                               VA64                 code,          // IN
-                               uint16               stackSelector, // IN
-                               VA64                 stack)         // IN
-{
-   NOT_IMPLEMENTED();
-}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,7 +31,8 @@
 #include "vmmblob.h"
 #include "monLoader.h"

-#define VMMBLOB_SIZE_MAX (12 * 1024 * 1024)
+#define VMMBLOB_SIZE_MAX (22 * 1024 * 1024) /* Ensure enough space for
+                                             * obj build with * GCOV_VMM=1. */

 /*
 *----------------------------------------------------------------------
@@ -65,7 +65,7 @@ APIC_GetMA(void)
    * Check if X2 APIC mode is enabled.
    */

-   if ((__GET_MSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
      return (MA)-1;
   }

@@ -75,7 +75,7 @@ APIC_GetMA(void)
    */

   // Mask out goo in the low 12 bits, which is unrelated to the address.
-   result = __GET_MSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);
+   result = X86MSR_GetMSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);

   /*
    * On Intel, the high bits are reserved so we mask.
@@ -122,7 +122,7 @@ APIC_Read(const APICDescriptor *desc, // IN
          int regNum)                 // IN
 {
   if (desc->isX2) {
-      return (uint32 )__GET_MSR(MSR_X2APIC_BASE + regNum);
+      return (uint32)X86MSR_GetMSR(MSR_X2APIC_BASE + regNum);
   } else {
      return desc->base[regNum][0];
   }
@@ -182,7 +182,7 @@ APIC_Write(const APICDescriptor *desc, // IN
           uint32 val)                 // IN
 {
   if (desc->isX2) {
-      __SET_MSR(MSR_X2APIC_BASE + regNum, val);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + regNum, val);
   } else {
      desc->base[regNum][0] = val;
   }
@@ -210,7 +210,7 @@ uint64
 APIC_ReadICR(const APICDescriptor *desc) // IN
 {
   if (desc->isX2) {
-      return __GET_MSR(MSR_X2APIC_BASE + APICR_ICRLO);
+      return X86MSR_GetMSR(MSR_X2APIC_BASE + APICR_ICRLO);
   } else {
      uint32 icrHi = desc->base[APICR_ICRHI][0];
      uint32 icrLo = desc->base[APICR_ICRLO][0];
@@ -244,7 +244,7 @@ APIC_WriteICR(const APICDescriptor *desc, // IN
 {
   if (desc->isX2) {
      uint64 icr = (uint64) id << 32 | icrLo;
-      __SET_MSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
   } else {
      ASSERT(!(id & ~(APIC_ICRHI_DEST_MASK >> APIC_ICRHI_DEST_OFFSET)));
      desc->base[APICR_ICRHI][0] = id << APIC_ICRHI_DEST_OFFSET;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -40,6 +40,7 @@ uint32      cpuidVersion;
 Bool        hostSupportsVT;
 Bool        hostSupportsSVM;
 Bool        hostHasSpecCtrl;
+Bool        hostSupportsXSave;


 /*
@@ -84,6 +85,7 @@ CPUID_Init(void)
   __GET_CPUID(1, &regs);
   cpuidVersion = regs.eax;
   cpuidFeatures = regs.edx;
+   hostSupportsXSave = CPUID_ISSET(1, ECX, XSAVE, regs.ecx);

   __GET_CPUID(0, &regs);
   ptr = (uint32 *)name;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,6 +37,7 @@ extern CpuidVendor cpuidVendor;
 extern Bool        hostSupportsVT;
 extern Bool        hostSupportsSVM;
 extern Bool        hostHasSpecCtrl;
+extern Bool        hostSupportsXSave;

 void CPUID_Init(void);
 Bool CPUID_AddressSizeSupported(void);
@@ -84,6 +85,12 @@ CPUID_HostSupportsSpecCtrl(void)
   return hostHasSpecCtrl;
 }

+static INLINE Bool
+CPUID_HostSupportsXSave(void)
+{
+   return hostSupportsXSave;
+}
+
 static INLINE Bool
 CPUID_SSE2Supported(void)
 {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,7 +27,7 @@
 #   define __HOST_KERNEL_H__


-#   ifdef linux
+#   ifdef __linux__
 /*
 * In some cases, this files needs to include Linux kernel header file
 * asm/page.h.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,7 +59,7 @@

 #define CROSSCALL_SLEEP_US 1000

-EXTERN int   HostIF_Init(VMDriver *vm, uint32 numVCPUs);
+EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);

@@ -76,12 +76,12 @@ EXTERN Bool  HostIF_IsLockedByMPN(VMDriver *vm, MPN mpn);
 EXTERN void  HostIF_FreeAllResources(VMDriver *vm);
 EXTERN uint64 HostIF_ReadUptime(void);
 EXTERN uint64 HostIF_UptimeFrequency(void);
-EXTERN unsigned int HostIF_EstimateLockedPageLimit(const VMDriver *vm,
-                                                   unsigned int lockedPages);
+EXTERN PageCnt HostIF_EstimateLockedPageLimit(const VMDriver *vm,
+                                              PageCnt lockedPages);
 EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
-EXTERN void *HostIF_AllocKernelPages(unsigned numPages, MPN *mpns);
-EXTERN void  HostIF_FreeKernelPages(unsigned numPages, void *ptr);
+EXTERN void *HostIF_AllocKernelPages(PageCnt numPages, MPN *mpns);
+EXTERN void  HostIF_FreeKernelPages(PageCnt numPages, void *ptr);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
@@ -111,11 +111,11 @@ EXTERN void HostIF_WaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_CancelWaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_WakeUpYielders(VMDriver *vm, Vcpuid currVcpu);

-EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                   unsigned int numPages, Bool kernelMPNBuffer);
-EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned int numPages, Bool kernelMPNBuffer);
+EXTERN int64 HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                     PageCnt numPages, Bool kernelMPNBuffer);
+EXTERN int HostIF_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn);
+EXTERN PageCnt HostIF_GetNumAnonPages(VMDriver *vm);
 EXTERN MPN HostIF_AllocLowPage(VMDriver *vm);

 EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr,
@@ -29,9 +29,10 @@
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
+#include "vm_basic_types.h"


-void *HostIF_AllocKernelMem(size_t size, int nonPageable);
+void *HostIF_AllocKernelMem(size_t size, Bool nonPageable);
 void  HostIF_FreeKernelMem(void *ptr);

 #endif // ifdef _HOSTIFMEM_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -96,7 +96,7 @@
 * Modify this value to increase the maximum number of tracked pages
 * per MemTrack instance.
 */
-#define MEMTRACK_MAX_TRACKED        MBYTES_2_PAGES(4096)
+#define MEMTRACK_MAX_TRACKED        GBYTES_2_PAGES(119)

 /*
 * Linux uses a 3-level directory, because we want to keep allocations
@@ -148,7 +148,7 @@ typedef uint64 MemTrackHTKey;

 typedef struct MemTrack {
   VMDriver         *vm;            /* The VM instance. */
-   unsigned          numPages;      /* Number of pages tracked. */
+   PageCnt           numPages;      /* Number of pages tracked. */
   MemTrackDir1      dir1;          /* First level directory. */
   MemTrackHT        vpnHashTable;  /* VPN to entry hashtable. */
   MemTrackHT       *mpnHashTable;  /* MPN to entry hashtable. */
@@ -197,7 +197,7 @@ MemTrackAllocPage(void)

 #define MEMTRACK_ALLOCDFN(_name, _itype, _otype)   \
   static INLINE _otype *                          \
-   _name(_itype *arg, unsigned pos)                \
+   _name(_itype *arg, uint64 pos)                  \
   {                                               \
      if (arg->dir[pos] == NULL) {                 \
         arg->dir[pos] = MemTrackAllocPage();      \
@@ -273,8 +273,8 @@ MemTrackHTInsert(MemTrackHT *ht,          // IN
 static void
 MemTrackCleanup(MemTrack *mt)    // IN
 {
-   unsigned idx;
-   unsigned p1;
+   PageCnt idx;
+   uint64 p1;
   MemTrackDir1 *dir1;
   Bool freeBackMap = mt != NULL && mt->mpnHashTable != NULL;

@@ -284,7 +284,7 @@ MemTrackCleanup(MemTrack *mt)    // IN
   dir1 = &mt->dir1;

   for (p1 = 0; p1 < MEMTRACK_DIR1_ENTRIES; p1++) {
-      unsigned p2;
+      uint64 p2;
      MemTrackDir2 *dir2 = MEMTRACK_GETDIR2(dir1, p1);

      if (dir2 == NULL) {
@@ -339,7 +339,7 @@ MemTrack *
 MemTrack_Init(VMDriver *vm) // IN:
 {
   MemTrack *mt;
-   unsigned idx;
+   PageCnt idx;

 #if defined(MEMTRACK_3LEVEL)
   ASSERT_ON_COMPILE(sizeof *mt <= PAGE_SIZE);
@@ -359,7 +359,8 @@ MemTrack_Init(VMDriver *vm) // IN:
      MemTrackHTPage *htPage = MemTrackAllocPage();

      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate VPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate VPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->vpnHashTable.pages[idx] = htPage;
@@ -374,7 +375,8 @@ MemTrack_Init(VMDriver *vm) // IN:
   for (idx = 0; idx < MEMTRACK_HT_PAGES; idx++) {
      MemTrackHTPage *htPage = MemTrackAllocPage();
      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate MPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate MPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->mpnHashTable->pages[idx] = htPage;
@@ -409,8 +411,8 @@ MemTrack_Add(MemTrack *mt,    // IN
             VPN64 vpn,       // IN
             MPN mpn)         // IN
 {
-   unsigned idx = mt->numPages;
-   unsigned p1, p2, p3;
+   PageCnt idx = mt->numPages;
+   uint64 p1, p2, p3;
   MemTrackEntry *ent;
   MemTrackDir1 *dir1 = &mt->dir1;
   MemTrackDir2 *dir2;
@@ -535,16 +537,16 @@ MemTrack_LookupMPN(MemTrack *mt, // IN
 *----------------------------------------------------------------------
 */

-unsigned
+PageCnt
 MemTrack_Cleanup(MemTrack *mt,            // IN
                 MemTrackCleanupCb *cb,   // IN
                 void *cData)             // IN
 {
-   unsigned idx;
-   unsigned count = 0;
+   PageCnt idx;
+   PageCnt count = 0;

   for (idx = 0; idx < mt->numPages; idx++) {
-      unsigned p1, p2, p3;
+      uint64 p1, p2, p3;
      MemTrackEntry *ent;
      MemTrackDir1 *dir1 = &mt->dir1;
      MEMTRACK_IDX2DIR(idx, p1, p2, p3);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -43,8 +43,8 @@ typedef struct MemTrackEntry {
 typedef void (MemTrackCleanupCb)(void *cData, MemTrackEntry *entry);

 extern struct MemTrack *MemTrack_Init(VMDriver *vm);
-extern unsigned MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
-                                 void *cbData);
+extern PageCnt MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
+                                void *cbData);
 extern MemTrackEntry *MemTrack_Add(struct MemTrack *mt, VPN64 vpn, MPN mpn);
 extern MemTrackEntry *MemTrack_LookupVPN(struct MemTrack *mt, VPN64 vpn);
 extern MemTrackEntry *MemTrack_LookupMPN(struct MemTrack *mt, MPN mpn);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,16 +24,15 @@
 *    Depending on configuration phystracker provides either 2-level or
 *    3-level structure to track whether page (specified by its MPN) is
 *    locked or no.  Linux uses 3-level structures with top limit of
- *    1TB (32bit) or 16TB (64bit).  Windows use 2-level structures
- *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.  On Mac
- *    limit is 2-level 128GB.
+ *    1TB (32bit) or 16TB (64bit).  Windows and Mac use 2-level structures
+ *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.
 *
 *    2-level phystracker is built on top of 3-level one by collapsing
 *    middle level.
 */


-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -50,9 +49,7 @@
 #define BYTES_PER_ENTRY      (PAGE_SIZE)
 #define PHYSTRACK_L3_ENTRIES (8 * BYTES_PER_ENTRY) /* 128MB */

-#if defined(WINNT_DDK)
-#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
-#elif defined(__linux__)
+#if defined(__linux__)
 #define PHYSTRACK_L2_ENTRIES (BYTES_PER_ENTRY / sizeof(void *)) /* 64GB or 128GB */
 /*
 * Currently MPN is 32 bits.  15 bits are in L3, 9 bits are in L2,
@@ -60,7 +57,7 @@
 */
 #define PHYSTRACK_L1_ENTRIES (256) /* 16TB. */
 #else
-#define PHYSTRACK_L1_ENTRIES ((128 + 4) * 8) /* 128 GB */
+#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
 #endif

 #ifndef PHYSTRACK_L2_ENTRIES
@@ -493,3 +490,32 @@ PhysTrack_GetNext(const PhysTracker *tracker, // IN
   return INVALID_MPN;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PhysTrack_GetNumTrackedPages --
+ *
+ *      Returns the total number of tracked pages
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+PageCnt
+PhysTrack_GetNumTrackedPages(const PhysTracker *tracker)
+{
+   PageCnt numTrackedMPNs = 0;
+   MPN nextMPN;
+   ASSERT(tracker);
+   ASSERT(HostIF_VMLockIsHeld(tracker->vm));
+   nextMPN = PhysTrack_GetNext(tracker, INVALID_MPN);
+   while (nextMPN != INVALID_MPN) {
+      numTrackedMPNs++;
+      nextMPN = PhysTrack_GetNext(tracker, nextMPN);
+   }
+   return numTrackedMPNs;
+}
+
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined(WINNT_DDK)
+#if !defined (__linux__)
 /* XXX: can be more efficient based on server vs. desktop and version of Windows */
 #define PHYSTRACK_MAX_SUPPORTED_GB (2048 + 4) /* 2 TB 64-bit W2k8 + 4 GB PCI */
 #endif
@@ -44,6 +44,7 @@ EXTERN void PhysTrack_Add(struct PhysTracker *, MPN);
 EXTERN void PhysTrack_Remove(struct PhysTracker *, MPN);
 EXTERN Bool PhysTrack_Test(const struct PhysTracker *, MPN);
 EXTERN MPN PhysTrack_GetNext(const struct PhysTracker *, MPN);
+EXTERN PageCnt PhysTrack_GetNumTrackedPages(const struct PhysTracker *);

 #endif

@@ -0,0 +1,299 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.c --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "sharedAreaVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmonIsMultiVCPU --
+ *
+ *      Helper function for determining if a shared area type represents
+ *      a region that has VCPU-specific memory.
+ *
+ * Results:
+ *      TRUE if type has VCPU-specific memory, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+SharedAreaVmmonIsMultiVCPU(SharedAreaType type)
+{
+   switch (type) {
+   case SHARED_AREA_PER_VM:
+   case SHARED_AREA_PER_VM_VMX:
+      return FALSE;
+   default:
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Init --
+ *
+ *      Initializes the shared area component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon shared area.
+ *
+ *----------------------------------------------------------------------
+ */
+
+SharedAreaVmmon *
+SharedAreaVmmon_Init(VMDriver *vm)
+{
+   SharedAreaVmmon *sa;
+
+   /*
+    * Allocate the shared area pointer here. The pages in each region are
+    * lazily allocated when a region is registered.
+    */
+   sa = HostIF_AllocKernelMem(sizeof *sa, FALSE);
+   if (sa == NULL) {
+      Warning("SharedArea failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(sa, 0, sizeof *sa);
+   return sa;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Cleanup --
+ *
+ *      Cleans up the shared area component of the VM Driver by freeing all
+ *      previously allocated VMMon shared area memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+SharedAreaVmmon_Cleanup(SharedAreaVmmon *area)
+{
+   SharedAreaType t;
+
+   if (area == NULL) {
+      return;
+   }
+
+   for (t = 0; t < NUM_SHARED_AREAS; t++) {
+      SharedAreaVmmonRegion *region = &area->regions[t];
+
+      if (region->pages != NULL) {
+         HostIF_FreeKernelMem(region->pages);
+      }
+   }
+   HostIF_FreeKernelMem(area);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_ValidateRegionArgs --
+ *
+ *      Validate the given VMMon shared area region registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register a shared
+ *      area region for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_ValidateRegionArgs(VMDriver *vm,
+                                   VMSharedAreaRegistrationBlock *block)
+{
+   SharedAreaVmmonRegion *region;
+   Vcpuid vcpu         = block->vcpu;
+   SharedAreaType type = block->region.index;
+   PageCnt numPages    = block->region.numPages;
+
+   if (vm == NULL               ||
+       vm->sharedArea == NULL   ||
+       vcpu >= vm->numVCPUs     ||
+       numPages == 0            ||
+       type >= NUM_SHARED_AREAS ||
+       (!SharedAreaVmmonIsMultiVCPU(type) && vcpu > 0)) {
+      return FALSE;
+   }
+
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu != 0) {
+
+      /*
+       * A region that was previously registered for a given VCPU should have
+       * the same number of pages as originally specified. Also, a region can
+       * only be reserved once for a given VCPU throughout the runtime of a VM.
+       */
+      return region->pagesPerVcpu == numPages &&
+             region->pages[vcpu * region->pagesPerVcpu] == INVALID_MPN;
+   } else {
+      /* First registration for this region. */
+      ASSERT(region->pages == NULL);
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_RegisterRegion --
+ *
+ *      Register a VMMon shared area region for a given VCPU with the VM Driver.
+ *      A successful return value indicates that the backing pages of the
+ *      shared area region are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the region was successfully registered for the given VCPU,
+ *      FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_RegisterRegion(VMDriver *vm,
+                               VMSharedAreaRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   SharedAreaType type = block->region.index;
+   SharedAreaVmmonRegion *region;
+   MPN *pages;
+   PageCnt page;
+
+   ASSERT(SharedAreaVmmon_ValidateRegionArgs(vm, block));
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu == 0) {
+      PageCnt pagesInRegion = block->region.numPages;
+
+      if (SharedAreaVmmonIsMultiVCPU(type)) {
+         pagesInRegion *= vm->numVCPUs;
+      }
+      region->pages = HostIF_AllocKernelMem(sizeof *pages * pagesInRegion,
+                                            FALSE);
+      if (region->pages == NULL) {
+         Warning("Failed to allocate pages array for region %u\n", type);
+         return FALSE;
+      }
+      for (page = 0; page < pagesInRegion; page++) {
+         region->pages[page] = INVALID_MPN;
+      }
+      region->pagesPerVcpu = block->region.numPages;
+   }
+
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+
+   /*
+    * Lock all shared area backing pages throughout the runtime of the monitor.
+    * The pages remain locked until they are freed by HostIF during VM Driver
+    * tear-down.
+    */
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      VPN vmxVPN = block->region.baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, pages + page);
+
+      if (status != PAGE_LOCK_SUCCESS) {
+         PageCnt resetPage;
+
+         /* Reset the region pages for this VCPU to their original state. */
+         for (resetPage = 0; resetPage < page; resetPage++) {
+            status = Vmx86_UnlockPage(vm, uAddr);
+            ASSERT(status == PAGE_UNLOCK_SUCCESS);
+            pages[resetPage] = INVALID_MPN;
+         }
+         return FALSE;
+      }
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_GetRegionMPN --
+ *
+ *      For the shared area region corresponding to the given type and VCPU,
+ *      get the MPN backing the region at the given offset.
+ *
+ * Results:
+ *      The MPN backing the region, or INVALID_MPN if no MPN is found.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+SharedAreaVmmon_GetRegionMPN(VMDriver *vm, SharedAreaVmmonRequest *request)
+{
+   SharedAreaType type = request->type;
+   Vcpuid vcpu = request->vcpu;
+   SharedAreaVmmonRegion *region;
+   PageCnt pgOffset = request->offset;
+   MPN *pages;
+
+   ASSERT(vcpu < vm->numVCPUs);
+   ASSERT(type < NUM_SHARED_AREAS);
+   ASSERT(IMPLIES(!SharedAreaVmmonIsMultiVCPU(type), vcpu == 0));
+   region = &vm->sharedArea->regions[type];
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   if (region->pages == NULL || region->pagesPerVcpu == 0) {
+      Warning("(%s) Requested unregistered region %u, VCPU %u\n", __FUNCTION__,
+              type, vcpu);
+      return INVALID_MPN;
+   }
+   if (pgOffset >= region->pagesPerVcpu) {
+      Warning("(%s) Offset %"FMT64"u (type %u, per-VCPU size %"FMT64"u)\n",
+              __FUNCTION__, pgOffset, type, region->pagesPerVcpu);
+      return INVALID_MPN;
+   }
+
+   return pages[pgOffset];
+}
@@ -0,0 +1,282 @@
+/*********************************************************
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * statVarsVmmon.c --
+ *
+ *     VMMon stat variables management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "statVarsVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Init --
+ *
+ *      Initializes the stat vars component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon stat vars struct.
+ *
+ *----------------------------------------------------------------------
+ */
+
+StatVarsVmmon *
+StatVarsVmmon_Init(VMDriver *vm)
+{
+   StatVarsVmmon *statVars;
+
+   statVars = HostIF_AllocKernelMem(sizeof *statVars, FALSE);
+   if (statVars == NULL) {
+      Warning("StatVars failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(statVars, 0, sizeof *statVars);
+   return statVars;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Cleanup --
+ *
+ *      Cleans up the stat vars component of the VM Driver by freeing all
+ *      previously allocated VMMon stat vars memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+StatVarsVmmon_Cleanup(StatVarsVmmon *statVars)
+{
+   if (statVars == NULL) {
+      return;
+   }
+
+   if (statVars->pages != NULL) {
+      HostIF_FreeKernelMem(statVars->pages);
+   }
+   HostIF_FreeKernelMem(statVars);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmonValidateRegistration --
+ *
+ *      Validate the given VMMon stat vars registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register stat vars
+ *      backing pages for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+StatVarsVmmonValidateRegistration(VMDriver *vm,
+                                   VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu      = block->vcpu;
+   PageCnt numPages = block->numPages;
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+
+   if (vm == NULL           ||
+       vm->statVars == NULL ||
+       numPages == 0        ||
+       vcpu >= vm->numVCPUs) {
+      return FALSE;
+   }
+
+   pagesPerVcpu = vm->statVars->pagesPerVcpu;
+   pages = vm->statVars->pages;
+   if (pagesPerVcpu == 0) {
+      ASSERT(pages == NULL);
+      return TRUE;
+   } else {
+      return pagesPerVcpu == numPages &&
+             pages[vcpu * pagesPerVcpu] == INVALID_MPN;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_RegisterVCPU --
+ *
+ *      Register stat vars backing pages for the given VCPU with VMMon.
+ *      A successful return value indicates that stat vars backing pages
+ *      for the given VCPU are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the backing pages were successfully registered for the
+ *      given VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+StatVarsVmmon_RegisterVCPU(VMDriver *vm,
+                           VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   PageCnt pagesPerVcpu = block->numPages;
+   StatVarsVmmon *statVars = vm->statVars;
+   MPN *pages;
+   PageCnt page;
+
+   HostIF_VMLock(vm, 45);
+   if (!StatVarsVmmonValidateRegistration(vm, block)) {
+      HostIF_VMUnlock(vm, 45);
+      return FALSE;
+   }
+   if (statVars->pagesPerVcpu == 0) {
+      PageCnt totalPages = pagesPerVcpu * vm->numVCPUs;
+      statVars->pages = HostIF_AllocKernelMem(totalPages * sizeof *pages,
+                                              FALSE);
+      if (statVars->pages == NULL) {
+         HostIF_VMUnlock(vm, 45);
+         return FALSE;
+      }
+      for (page = 0; page < totalPages; page++) {
+         statVars->pages[page] = INVALID_MPN;
+      }
+      statVars->pagesPerVcpu = pagesPerVcpu;
+   }
+
+   /*
+    * Validate that all pages in this VCPU's region are still invalid. Before
+    * unlocking, temporarily set the first page to 0 to prevent a malicious user
+    * from triggering an ASSERT by firing parallel ioctls for the same VCPU.
+    */
+   pages = statVars->pages + vcpu * pagesPerVcpu;
+   for (page = 0; page < pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+   ASSERT_ON_COMPILE(INVALID_MPN != 0);
+   pages[0] = 0;
+   HostIF_VMUnlock(vm, 45);
+
+   /*
+    * Verify that all stat vars backing pages are locked.  The pages are
+    * assumed to remain locked either until they are unlocked by the VMX
+    * following a NUMA migration, or by HostIF during VM Driver tear-down.
+    */
+   for (page = 0; page < pagesPerVcpu; page++) {
+      VPN vmxVPN = block->baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      PageCnt resetPage;
+      MPN *currPage = pages + page;
+      Bool verifiedLocked = FALSE;
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, currPage);
+
+      /*
+       * Backing pages are expected to be locked by the VMX before being
+       * registered.
+       */
+      if (status == PAGE_LOCK_ALREADY_LOCKED) {
+         verifiedLocked = TRUE;
+         status = Vmx86_LookupUserMPN(vm, uAddr, currPage);
+         /*
+          * PR 2260615: Some platforms return PAGE_LOCK_SUCCESS for a successful
+          * user MPN lookup, others return PAGE_LOOKUP_SUCCESS, but they both
+          * happen to equal the same value.
+          */
+         ASSERT_ON_COMPILE(PAGE_LOOKUP_SUCCESS == PAGE_LOCK_SUCCESS);
+         if (status == PAGE_LOOKUP_SUCCESS) {
+            /*
+             * The backing page was verified as previously locked, and was
+             * successfully recorded.
+             */
+            continue;
+         }
+      }
+      /*
+       * The backing page was either not previously locked, or was not recorded.
+       * In either case, reset all previously recorded pages to INVALID_MPN.
+       */
+      if (!verifiedLocked && status == PAGE_LOCK_SUCCESS) {
+         status = Vmx86_UnlockPage(vm, uAddr);
+         ASSERT(status == PAGE_UNLOCK_SUCCESS);
+      }
+      for (resetPage = 0; resetPage <= page; resetPage++) {
+         pages[resetPage] = INVALID_MPN;
+      }
+      return FALSE;
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_GetRegionMPN --
+ *
+ *      Obtain the backing MPN for stat vars for the given VCPU at
+ *      the given offset.
+ *
+ * Results:
+ *      The backing stat vars MPN for the given VCPU at the given offset
+ *      if it has been registered, or INVALID_MPN otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpu, PageCnt offset)
+{
+   MPN *pages;
+   MPN backingPage;
+   StatVarsVmmon *statVars = vm->statVars;
+
+   ASSERT(vcpu < vm->numVCPUs);
+
+   HostIF_VMLock(vm, 46);
+   if (offset >= statVars->pagesPerVcpu) {
+      HostIF_VMUnlock(vm, 46);
+      Warning("(%s) Requested MPN at invalid offset %"FMT64"u for VCPU %u\n",
+              __FUNCTION__, offset, vcpu);
+      return INVALID_MPN;
+   }
+
+   pages = statVars->pages + vcpu * statVars->pagesPerVcpu;
+   backingPage = pages[offset];
+   HostIF_VMUnlock(vm, 46);
+   return backingPage;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -33,7 +33,7 @@
 *
 */

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"
 #   include <linux/string.h> /* memset() in the kernel */
@@ -61,10 +61,11 @@
 #include "x86vt.h"
 #include "x86vtinstr.h"
 #include "apic.h"
-#include "x86perfctr.h"
+#include "perfctr.h"
 #include "x86paging_common.h"
 #include "x86paging_64.h"
 #include "memtrack.h"
+#include "monLoader.h"

 #ifdef LINUX_GDT_IS_RO
 #   include <asm/desc.h>
@@ -846,7 +847,7 @@ TaskApplyPTPatches(VMDriver *vm, VMCrossPageData *cpData)
      VMMPageTablePatch *patch = &cpData->vmmPTP[i];
      unsigned l4idx = PT_LPN_2_L4OFF(patch->lpn);
      unsigned l3idx = PT_LPN_2_L3OFF(patch->lpn);
-      PT_L4E pte;
+      PT_L4E pte = 0;
      MA ma = cpData->monCR3 + l4idx * sizeof pte; /* PTE machine address. */

      if (patch->level == PTP_EMPTY) {
@@ -943,8 +944,8 @@ static Bool
 TaskVerifyPTPatches(VMDriver *vm,
                    VMCrossPageData *cpData)
 {
-   LPN64 xgCPN = LA_2_LPN(cpData->vmmCrossGDTLA);
-   LPN64 xpCPN = LA_2_LPN(cpData->vmmCrossPageLA);
+   LPN64 xgCPN = LA_2_LPN(cpData->crossGDTLA);
+   LPN64 xpCPN = LA_2_LPN(cpData->crossPageLA);
   MPN   xpMPN = MA_2_MPN(cpData->crosspageMA);
   MA    cr3   = cpData->monCR3;
   return TaskVerifyPTMap(vm, cr3, xpCPN, xpMPN) &&
@@ -1092,9 +1093,8 @@ TaskSetCrossGDTVMM(BSVMM_GDTInit *gdt)
 * TaskSetCrossGDTHost --
 *
 *      Initializes the host portion of the crossGDT by copying it directly
- *      from the host kernel's GDT. We assume that all the host segments
- *      we will ever need come from the first page of the host's GDT and
- *      precede the lowest segment needed by the monitor.
+ *      from the host kernel's GDT. We assume that all the host segments we
+ *      will ever need come from the first page of the host's GDT.
 *
 * Results:
 *      TRUE on success, FALSE otherwise.
@@ -1106,30 +1106,17 @@ TaskSetCrossGDTVMM(BSVMM_GDTInit *gdt)
 */

 static void
-TaskSetCrossGDTHost(const BSVMM_GDTInit *gdt)
+TaskSetCrossGDTHost(void)
 {
   unsigned len;
   DTR64 hostGDT;
-   unsigned i;
-   /* Copy descriptors up to at most the size of the CrossGDT. */
-   unsigned minVMMIndex = sizeof(CrossGDT) / sizeof(Descriptor);
-
   /*
    * All copied host segment descriptors will come from the first page of
-    * the host kernel GDT and precede the first monitor segment descriptor.
+    * the host kernel GDT.
    */
-   ASSERT_ON_COMPILE(sizeof(CrossGDT) % sizeof(Descriptor) == 0);
   ASSERT(HostIF_GlobalLockIsHeld());
-   for (i = 0; i < ARRAYSIZE(gdt->entries); i++) {
-      const BSVMM_GDTInitEntry *entry = &gdt->entries[i];
-      if (entry->present == 1) {
-         minVMMIndex = MIN(entry->index, minVMMIndex);
-      }
-   }
   TaskSaveGDT(&hostGDT);
-   /* PR 2142795: Only copy up to at most the monitor's first descriptor. */
-   len = MIN((unsigned)hostGDT.limit + 1, minVMMIndex * sizeof(Descriptor));
-   ASSERT(len <= sizeof(CrossGDT));
+   len = MIN((unsigned)hostGDT.limit + 1, sizeof(CrossGDT));
   memcpy(crossGDT->gdtes, (void*)HOST_KERNEL_LA_2_VA((LA)hostGDT.offset), len);
 }

@@ -1171,7 +1158,7 @@ Task_CreateCrossGDT(BSVMM_GDTInit *gdt)
         return FALSE;
      }
      memset(crossGDT, 0, sizeof *crossGDT);
-      TaskSetCrossGDTHost(gdt);
+      TaskSetCrossGDTHost();
   }

   populatedCrossGDT = TaskSetCrossGDTVMM(gdt);
@@ -1589,8 +1576,8 @@ TaskCreatePTPatches(VMDriver    *vm,
                    uint16      *numPages)
 {
   VMCrossPageData * const cpData = &crosspage->crosspageData;
-   LPN64                   xgCPN  = LA_2_LPN(cpData->vmmCrossGDTLA);
-   LPN64                   xpCPN  = LA_2_LPN(cpData->vmmCrossPageLA);
+   LPN64                   xgCPN  = LA_2_LPN(cpData->crossGDTLA);
+   LPN64                   xpCPN  = LA_2_LPN(cpData->crossPageLA);
   MPN                     xpMPN  = MA_2_MPN(cpData->crosspageMA);
   /*
    * Set up patches that the BackToHost code will use to map the
@@ -1616,8 +1603,7 @@ TaskCreatePTPatches(VMDriver    *vm,
 *    Initialize the crosspage used to switch to the monitor task.
 *
 * Results:
- *    0 on success
- *    != 0 on failure
+ *    TRUE on success, FALSE on failure.
 *
 * Side effects:
 *    None
@@ -1625,29 +1611,28 @@ TaskCreatePTPatches(VMDriver    *vm,
 *-----------------------------------------------------------------------------
 */

-int
-Task_InitCrosspage(VMDriver *vm,          // IN
-                   InitBlock *initParams) // IN/OUT: Initial params from the
-                                          //         VM
+Bool
+Task_InitCrosspage(VMDriver *vm,               // IN
+                   LPN monStartLPN,            // IN
+                   LPN monEndLPN,              // IN
+                   PerVcpuPages *perVcpuPages) // IN
 {
   Vcpuid vcpuid;
   uint16 numPTPPages = 0;

   if (crossGDT == NULL) {
-      return 1;
+      return FALSE;
   }

-   initParams->crossGDTMPN = crossGDTMPN;
-
   ASSERT(0 < vm->numVCPUs && vm->numVCPUs <= MAX_VCPUS);
   for (vcpuid = 0; vcpuid < vm->numVCPUs; vcpuid++) {
-      VA64             crossPageUserAddr = initParams->crosspage[vcpuid];
+      VA64             crossPageUserAddr = perVcpuPages[vcpuid].crosspage;
      VMCrossPage     *p = HostIF_MapCrossPage(vm, crossPageUserAddr);
      VMCrossPageData *cpData;
      MPN              crossPageMPN;

      if (p == NULL) {
-         return 1;
+         return FALSE;
      }
      cpData = &p->crosspageData;

@@ -1656,7 +1641,7 @@ Task_InitCrosspage(VMDriver *vm,          // IN
          PAGE_LOOKUP_SUCCESS ||
          crossPageMPN == 0) {
         HostIF_VMUnlock(vm, 38);
-         return 1;
+         return FALSE;
      }
      HostIF_VMUnlock(vm, 38);

@@ -1673,39 +1658,39 @@ Task_InitCrosspage(VMDriver *vm,          // IN
         Warning("%s: crosspage version mismatch: vmmon claims %#x, must "
                 "match vmx version of %#x.\n", __FUNCTION__,
                 (int)CROSSPAGE_VERSION, cpData->version);
-         return 1;
+         return FALSE;
      }
      if (!pseudoTSC.initialized) {
         Warning("%s*: PseudoTSC has not been initialized\n", __FUNCTION__);
-         return 1;
+         return FALSE;
      }
      cpData->crosspageMA = MPN_2_MA(crossPageMPN);
-      cpData->hostCrossPageLA = (LA64)(uintptr_t)p;
-      cpData->vmmCrossPageLA = cpData->hostCrossPageLA;
-      cpData->vmmCrossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT);
+      cpData->crossPageLA = (LA64)(uintptr_t)p;
+      cpData->crossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.offset = HOST_KERNEL_VA_2_LA((VA)crossGDT);
      cpData->crossGDTHKLADesc.limit  = sizeof(CrossGDT) - 1;
+      if (CPUID_HostSupportsXSave()) {
+         cpData->wsCR4 |= CR4_OSXSAVE;
+      }
+      cpData->monCR3 = MPN_2_MA(vm->ptRootMpns[vcpuid]);

      HostIF_VMLock(vm, 39);
-      if (!TaskCreatePTPatches(vm, p,
-                               initParams->monStartLPN,
-                               initParams->monEndLPN,
-                               &numPTPPages)) {
+      if (!TaskCreatePTPatches(vm, p, monStartLPN, monEndLPN, &numPTPPages)) {
         HostIF_VMUnlock(vm, 39);
         Warning("%s: Could not create page table patches for VCPU %d\n",
                 __FUNCTION__, vcpuid);
-         return 1;
+         return FALSE;
      }
      HostIF_VMUnlock(vm, 39);
      if (!TaskApplyPTPatches(vm, cpData)) {
         Warning("%s: Could not apply page table patches for VCPU %d\n",
                 __FUNCTION__, vcpuid);
-         return 1;
+         return FALSE;
      }
      if (!TaskVerifyPTPatches(vm, cpData)) {
         Warning("%s: Page table patches for VCPU %d failed verification\n",
                 __FUNCTION__, vcpuid);
-         return 1;
+         return FALSE;
      }

      /*
@@ -1733,12 +1718,12 @@ Task_InitCrosspage(VMDriver *vm,          // IN
      TaskInitHostSwitchIDT(p);
   }
   /*
-    * Report back to the VMX the number of pages allocated for this VM's
-    * page table patches.
+    * Store the number of pages allocated for this VM's page table patches so
+    * the bootstrap can account for the memory overhead later in a module call.
    */
-   initParams->numPTPPages = numPTPPages;
+   vm->numPTPPages = numPTPPages;

-   return 0;
+   return TRUE;
 }


@@ -2322,12 +2307,23 @@ TaskSwitchToMonitor(VMCrossPage *crosspage)
   {
      uint64 raxGetsWiped, rcxGetsWiped;

+#ifdef CALL_NOSPEC
+      __asm__ __volatile__(CALL_NOSPEC
+                           : "=a" (raxGetsWiped),
+                             "=c" (rcxGetsWiped)
+                           : "0" (codePtr),
+                             "1" (crosspage),
+                             THUNK_TARGET(codePtr)
+                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
+#else
      __asm__ __volatile__("call *%%rax"
                           : "=a" (raxGetsWiped),
                             "=c" (rcxGetsWiped)
                           : "0" (codePtr),
                             "1" (crosspage)
                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
+#endif
+
   }
 #elif defined(_MSC_VER)
   /*
@@ -2520,7 +2516,7 @@ Task_Switch(VMDriver *vm,  // IN
    * world switch.  We must be careful not to overwrite the
    * crosspages arguments when doing this though, see bug 820257.
    */
-   if (hvRootMPN == INVALID_MPN) {
+   if (hvRootMPN == INVALID_MPN && CPUID_HostSupportsHV()) {
      crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE;
      crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_VMX_PAGE;
      crosspage->crosspageData.pcpuNum = pCPU;
@@ -2545,9 +2541,9 @@ Task_Switch(VMDriver *vm,  // IN
          * hosts IDT - PR 848701.
          */
         if (pebsAvailable) {
-            pebsMSR = __GET_MSR(IA32_MSR_PEBS_ENABLE);
+            pebsMSR = X86MSR_GetMSR(IA32_MSR_PEBS_ENABLE);
            if (pebsMSR != 0) {
-               __SET_MSR(IA32_MSR_PEBS_ENABLE, 0);
+               X86MSR_SetMSR(IA32_MSR_PEBS_ENABLE, 0);
            }
         }

@@ -2556,9 +2552,9 @@ Task_Switch(VMDriver *vm,  // IN
          * enabled.
          */
         if (ptAvailable) {
-            ptMSR = __GET_MSR(MSR_RTIT_CTL);
+            ptMSR = X86MSR_GetMSR(MSR_RTIT_CTL);
            if ((ptMSR & MSR_RTIT_CTL_TRACE_EN) != 0) {
-               __SET_MSR(MSR_RTIT_CTL, ptMSR & ~MSR_RTIT_CTL_TRACE_EN);
+               X86MSR_SetMSR(MSR_RTIT_CTL, ptMSR & ~MSR_RTIT_CTL_TRACE_EN);
            }
         }

@@ -2578,12 +2574,12 @@ Task_Switch(VMDriver *vm,  // IN
             * so that we can set CR4.VMXE to activate VMX.
             */
            uint64 bits = MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE;
-            uint64 featCtl = __GET_MSR(MSR_FEATCTL);
+            uint64 featCtl = X86MSR_GetMSR(MSR_FEATCTL);
            if ((featCtl & bits) != bits) {
               if ((featCtl & MSR_FEATCTL_LOCK) != 0) {
                  Panic("Intel VT-x is disabled and locked on CPU %d\n", pCPU);
               }
-               __SET_MSR(MSR_FEATCTL, featCtl | bits);
+               X86MSR_SetMSR(MSR_FEATCTL, featCtl | bits);
            }
         }

@@ -2651,12 +2647,12 @@ Task_Switch(VMDriver *vm,  // IN
         }

         if (CPUID_HostSupportsSVM()) {
-            efer = __GET_MSR(MSR_EFER);
+            efer = X86MSR_GetMSR(MSR_EFER);
            if ((efer & MSR_EFER_SVME) == 0) {
-               __SET_MSR(MSR_EFER, efer | MSR_EFER_SVME);
+               X86MSR_SetMSR(MSR_EFER, efer | MSR_EFER_SVME);
            }
-            foreignHSAVE = __GET_MSR(MSR_VM_HSAVE_PA);
-            __SET_MSR(MSR_VM_HSAVE_PA, MPN_2_MA(hvRootMPN));
+            foreignHSAVE = X86MSR_GetMSR(MSR_VM_HSAVE_PA);
+            X86MSR_SetMSR(MSR_VM_HSAVE_PA, MPN_2_MA(hvRootMPN));
         }

         /*
@@ -2721,9 +2717,9 @@ Task_Switch(VMDriver *vm,  // IN
         if (CPUID_HostSupportsSpecCtrl()) {
 #ifdef CYCLE_SPEC_CTRL
            currentSpecCtrlValue = (currentSpecCtrlValue + 1) % 4;
-            __SET_MSR(MSR_SPEC_CTRL, currentSpecCtrlValue);
+            X86MSR_SetMSR(MSR_SPEC_CTRL, currentSpecCtrlValue);
 #endif
-            crosspage->crosspageData.specCtrl = __GET_MSR(MSR_SPEC_CTRL);
+            crosspage->crosspageData.specCtrl = X86MSR_GetMSR(MSR_SPEC_CTRL);
         }

         DEBUG_ONLY(crosspage->crosspageData.monTinyStack[0] = 0xDEADBEEF;)
@@ -2737,11 +2733,11 @@ Task_Switch(VMDriver *vm,  // IN

 #ifdef CYCLE_SPEC_CTRL
         if (CPUID_HostSupportsSpecCtrl()) {
-            readSpecCtrlValue = __GET_MSR(MSR_SPEC_CTRL);
+            readSpecCtrlValue = X86MSR_GetMSR(MSR_SPEC_CTRL);
            specCtrlEqual = readSpecCtrlValue ==
                            crosspage->crosspageData.specCtrl;
            /* Do not leak cycling SPEC_CTRL value back to host. */
-            __SET_MSR(MSR_SPEC_CTRL, 0);
+            X86MSR_SetMSR(MSR_SPEC_CTRL, 0);
         }
 #endif

@@ -2764,9 +2760,9 @@ Task_Switch(VMDriver *vm,  // IN
         }

         if (CPUID_HostSupportsSVM()) {
-            __SET_MSR(MSR_VM_HSAVE_PA, foreignHSAVE);
+            X86MSR_SetMSR(MSR_VM_HSAVE_PA, foreignHSAVE);
            if ((efer & MSR_EFER_SVME) == 0) {
-               __SET_MSR(MSR_EFER, efer);
+               X86MSR_SetMSR(MSR_EFER, efer);
            }
         }

@@ -2840,11 +2836,11 @@ Task_Switch(VMDriver *vm,  // IN
         TaskLoadIDT(&hostIDT);

         if (pebsMSR != 0) {
-            __SET_MSR(IA32_MSR_PEBS_ENABLE, pebsMSR);
+            X86MSR_SetMSR(IA32_MSR_PEBS_ENABLE, pebsMSR);
         }

         if ((ptMSR & MSR_RTIT_CTL_TRACE_EN) != 0) {
-            __SET_MSR(MSR_RTIT_CTL, ptMSR);
+            X86MSR_SetMSR(MSR_RTIT_CTL, ptMSR);
         }

         TaskUpdateLatestPTSC(vm, &crosspage->crosspageData);
@@ -2909,6 +2905,18 @@ Task_Switch(VMDriver *vm,  // IN
   }

   if (crosspage->crosspageData.moduleCallType == MODULECALL_INTR) {
+
+      /*
+       * Newer versions of Window expect EFLAGS_AC to be set when handling an
+       * interrupt - PR 2248661.
+       */
+      if ((flags & EFLAGS_AC) != 0) {
+         uintptr_t curFlags;
+         SAVE_FLAGS(curFlags);
+         curFlags |= EFLAGS_AC;
+         RESTORE_FLAGS(curFlags);
+      }
+
      /*
       * Note we must do the RAISE_INTERRUPT before ever enabling
       * interrupts or bad things have happened (might want to know exactly
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,10 +24,11 @@
 #include "includeCheck.h"
 #include "bootstrap_vmm.h"

-struct InitBlock;
+struct MonLoaderHeader;

 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
-extern int Task_InitCrosspage(VMDriver *vm, struct InitBlock *params);
+extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
+                               PerVcpuPages *perVcpuPages);
 extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,6 @@
 #include "vm_basic_math.h"
 #include "vmx86.h"
 #include "task.h"
-#include "initblock.h"
 #include "vm_asm.h"
 #include "iocontrols.h"
 #include "hostif.h"
@@ -56,14 +55,16 @@
 #include "x86vt.h"
 #include "x86svm.h"
 #include "x86cpuid_asm.h"
-#if defined(linux)
+#if defined(__linux__)
 #include <asm/timex.h>
 #endif
-#include "x86perfctr.h"
+#include "perfctr.h"
 #include "x86vtinstr.h"
 #include "bootstrap_vmm.h"
 #include "monLoader.h"
 #include "vmmblob.h"
+#include "sharedAreaVmmon.h"
+#include "statVarsVmmon.h"

 PseudoTSC pseudoTSC;

@@ -77,14 +78,14 @@ static VMDriver *vmDriverList = NULL;
 static LockedPageLimit lockedPageLimit = {
   0,                        // host: does not need to be initialized.
   0,                        // configured: must be set by some VM as it is powered on.
-   (uint32)MAX_LOCKED_PAGES, // dynamic
+   MAX_LOCKED_PAGES,         // dynamic
 };

 /* Percentage of guest "paged" memory that must fit within the hard limit. */
-static unsigned minVmMemPct;
+static Percent minVmMemPct;

 /* Number of pages actually locked by all virtual machines */
-static unsigned numLockedPages;
+static PageCnt numLockedPages;

 /* Total virtual machines on this host */
 static unsigned vmCount;
@@ -141,13 +142,12 @@ typedef struct NXData {
 *----------------------------------------------------------------------
 */

-static INLINE unsigned
+static INLINE PageCnt
 Vmx86AdjustLimitForOverheads(const VMDriver* vm,
-                             const uint32 limit)
+                             const PageCnt limit)
 {
-   uint32 extraCost = (vm != NULL) ? vmCount * vm->memInfo.perVMOverhead : 0;
+   PageCnt extraCost = (vm != NULL) ? vmCount * vm->memInfo.perVMOverhead : 0;
   ASSERT(HostIF_GlobalLockIsHeld());
-
   return (extraCost < limit) ?  (limit - extraCost) : 0;
 }

@@ -176,12 +176,11 @@ Vmx86AdjustLimitForOverheads(const VMDriver* vm,
 *----------------------------------------------------------------------
 */

-static INLINE unsigned
+static INLINE PageCnt
 Vmx86LockedPageLimit(const VMDriver* vm)  // IN:
 {
-   uint32 overallLimit;
+   PageCnt overallLimit;
   ASSERT(HostIF_GlobalLockIsHeld());
-
   lockedPageLimit.host = HostIF_EstimateLockedPageLimit(vm, numLockedPages);
   overallLimit = MIN(MIN(lockedPageLimit.configured, lockedPageLimit.dynamic),
                      lockedPageLimit.host);
@@ -212,7 +211,7 @@ Vmx86LockedPageLimit(const VMDriver* vm)  // IN:

 static INLINE Bool
 Vmx86HasFreePages(VMDriver *vm,
-                  unsigned int numPages,
+                  PageCnt numPages,
                  Bool checkVM)
 {
   /*
@@ -226,10 +225,7 @@ Vmx86HasFreePages(VMDriver *vm,
          (!checkVM || HostIF_VMLockIsHeld(vm)));

   if (checkVM) {
-      /*
-       * Check the per-vm limit.
-       */
-
+      /* Check the per-vm limit. */
      ASSERT(HostIF_VMLockIsHeld(vm));
      if (vm->memInfo.admitted) {
         if (vm->memInfo.maxAllocation <= vm->memInfo.locked) {
@@ -239,12 +235,8 @@ Vmx86HasFreePages(VMDriver *vm,
         }
      }
   } else {
-      /*
-       * Check the global limit.
-       */
-
-      unsigned limit = Vmx86LockedPageLimit(vm);
-
+      /* Check the global limit. */
+      PageCnt limit = Vmx86LockedPageLimit(vm);
      if (limit <= numLockedPages) {
         return FALSE;
      } else if (limit - numLockedPages < numPages) {
@@ -555,7 +547,7 @@ Vmx86_Free(void *ptr)
 void *
 Vmx86_Calloc(size_t numElements, // IN
             size_t elementSize, // IN
-             int nonPageable)    // IN
+             Bool nonPageable)   // IN
 {
   size_t numBytes = numElements * elementSize;
   void *retval;
@@ -732,6 +724,14 @@ Vmx86FreeAllVMResources(VMDriver *vm)
         VmmBlob_Cleanup(vm->blobInfo);
         vm->blobInfo = NULL;
      }
+      if (vm->sharedArea != NULL) {
+         SharedAreaVmmon_Cleanup(vm->sharedArea);
+         vm->sharedArea = NULL;
+      }
+      if (vm->statVars != NULL) {
+         StatVarsVmmon_Cleanup(vm->statVars);
+         vm->statVars = NULL;
+      }

      HostIF_FreeAllResources(vm);

@@ -768,7 +768,7 @@ Vmx86FreeAllVMResources(VMDriver *vm)

 static Bool
 Vmx86ReserveFreePages(VMDriver *vm,
-                      unsigned int numPages,
+                      PageCnt numPages,
                      Bool ignoreLimits)
 {
   Bool retval = FALSE;
@@ -779,15 +779,14 @@ Vmx86ReserveFreePages(VMDriver *vm,
   for (retries = 3; !retval && (retries > 0); retries--) {
      HostIF_GlobalLock(17);
      HostIF_VMLock(vm, 0);
-
-      // Check VM's limit and don't wait.
+      /* Check VM's limit and don't wait. */
      retval = Vmx86HasFreePages(vm, numPages, TRUE);
      if (!retval) {
         HostIF_VMUnlock(vm, 0);
         HostIF_GlobalUnlock(17);
         break;
      } else {
-         // Wait to satisfy the global limit.
+         /* Wait to satisfy the global limit. */
         retval = Vmx86HasFreePages(vm, numPages, FALSE);
         if (retval) {
            numLockedPages += numPages;
@@ -841,7 +840,7 @@ Vmx86ReserveFreePages(VMDriver *vm,

 static void
 Vmx86UnreserveFreePages(VMDriver *vm,
-                        unsigned int numPages)
+                        PageCnt numPages)
 {
   ASSERT(vm);

@@ -879,7 +878,7 @@ static void
 Vmx86GetNX(void *clientData) // IN/OUT: A NXData *
 {
   NXData *nxData = (NXData *)clientData;
-   uint64 efer = __GET_MSR(MSR_EFER);
+   uint64 efer = X86MSR_GetMSR(MSR_EFER);

   Atomic_Inc32(&nxData->responded);
   if ((efer & MSR_EFER_NXE) == MSR_EFER_NXE) {
@@ -946,11 +945,6 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)
      return NULL;
   }

-   /* Disallow VM creation if HV is not available, as the VM cannot be run. */
-   if (!CPUID_HostSupportsHV()) {
-      return NULL;
-   }
-
   /* Disallow VM creation if NX is disabled on the host as VMM requires NX. */
   if (!hostUsesNX) {
      Log("NX/XD must be enabled.  Cannot create VM.\n");
@@ -970,25 +964,30 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)
      Atomic_Write32(&vm->currentHostCpu[v], INVALID_PCPU);
      vm->ptRootMpns[v] = INVALID_MPN;
   }
-   if (HostIF_Init(vm, numVCPUs)) {
-      goto cleanup;
-   }
-   bsBuf = HostIF_AllocKernelMem(bsBlobSize, FALSE);
-   if (bsBuf == NULL) {
-      goto cleanup;
-   }
-   if (HostIF_CopyFromUser(bsBuf, bsBlob, bsBlobSize) != 0) {
-      goto cleanup;
-   }
-   bsParams = BSVMM_Validate(bsBuf, bsBlobSize);
-   if (bsParams == NULL) {
-      Warning("Could not validate the VMM bootstrap blob");
+   if (!HostIF_Init(vm, numVCPUs)) {
      goto cleanup;
   }

-   if (!Task_CreateCrossGDT(&bsParams->gdtInit)) {
-      goto cleanup;
+   /* The ULM does not use the cross GDT. */
+   if (bsBlobSize != 0) {
+      bsBuf = HostIF_AllocKernelMem(bsBlobSize, FALSE);
+      if (bsBuf == NULL) {
+         goto cleanup;
+      }
+      if (HostIF_CopyFromUser(bsBuf, bsBlob, bsBlobSize) != 0) {
+         goto cleanup;
+      }
+      bsParams = BSVMM_Validate(bsBuf, bsBlobSize);
+      if (bsParams == NULL) {
+         Warning("Could not validate the VMM bootstrap blob");
+         goto cleanup;
+      }
+
+      if (!Task_CreateCrossGDT(&bsParams->gdtInit)) {
+         goto cleanup;
+      }
   }
+
   vm->ptpTracker = MemTrack_Init(vm);
   if (vm->ptpTracker == NULL) {
      goto cleanup;
@@ -997,6 +996,14 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)
   if (vm->vmmTracker == NULL) {
      goto cleanup;
   }
+   vm->sharedArea = SharedAreaVmmon_Init(vm);
+   if (vm->sharedArea == NULL) {
+      goto cleanup;
+   }
+   vm->statVars = StatVarsVmmon_Init(vm);
+   if (vm->statVars == NULL) {
+      goto cleanup;
+   }

   HostIF_GlobalLock(0);

@@ -1015,7 +1022,9 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)

   HostIF_GlobalUnlock(0);

-   HostIF_FreeKernelMem(bsBuf);
+   if (bsBuf != NULL) {
+      HostIF_FreeKernelMem(bsBuf);
+   }
   return vm;

 cleanup:
@@ -1052,7 +1061,8 @@ cleanup:
 */

 static Bool
-Vmx86SetPageTableRoots(VMDriver *vm, UserVA64 *ptRootVAs, uint16 numVCPUs)
+Vmx86SetPageTableRoots(VMDriver *vm, PerVcpuPages *perVcpuPages,
+                       uint16 numVCPUs)
 {
   uint16 vcpu;

@@ -1060,14 +1070,16 @@ Vmx86SetPageTableRoots(VMDriver *vm, UserVA64 *ptRootVAs, uint16 numVCPUs)
      return FALSE;
   }
   for (vcpu = 0; vcpu < numVCPUs; vcpu++) {
-      if ((ptRootVAs[vcpu] & (PAGE_SIZE - 1)) != 0) {
+      VA64 ptRoot = perVcpuPages[vcpu].ptRoot;
+
+      if ((ptRoot & (PAGE_SIZE - 1)) != 0) {
         Warning("Error: page table VA %"FMT64"x is not page-aligned\n",
-                 ptRootVAs[vcpu]);
+                 ptRoot);
         return FALSE;
      }
      ASSERT(vm->ptRootMpns[vcpu] == INVALID_MPN);
      HostIF_VMLock(vm, 38);
-      if (HostIF_LookupUserMPN(vm, ptRootVAs[vcpu], &vm->ptRootMpns[vcpu]) !=
+      if (HostIF_LookupUserMPN(vm, ptRoot, &vm->ptRootMpns[vcpu]) !=
          PAGE_LOOKUP_SUCCESS) {
         HostIF_VMUnlock(vm, 38);
         Warning("Failure looking up page table root MPN for VCPU %d\n", vcpu);
@@ -1128,7 +1140,7 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
                       uint32 numBytes,
                       uint32 headerOffset,
                       uint16 numVCPUs,
-                       UserVA64 *ptRootVAs,
+                       PerVcpuPages *perVcpuPages,
                       VMSharedRegion *shRegions)
 {
   VmmBlobInfo *bi = NULL;
@@ -1136,18 +1148,31 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
   Vcpuid errVcpu;
   MonLoaderError ret;
   MonLoaderArgs args;
+   MonLoaderHeader *header;

   if (!VmmBlob_Load(bsBlobAddr, numBytes, headerOffset, &bi)) {
      Warning("Error loading VMM bootstrap blob\n");
      goto error;
   }
   vm->blobInfo = bi;
-   if (!Vmx86SetPageTableRoots(vm, ptRootVAs, numVCPUs)) {
+   header = bi->header;
+   if (!Vmx86SetPageTableRoots(vm, perVcpuPages, numVCPUs)) {
      goto error;
   }
+
+   /*
+    * Initialize the driver's part of the cross-over page used to
+    * talk to the monitor.
+    */
+   if (!Task_InitCrosspage(vm, header->monStartLPN, header->monEndLPN,
+                           perVcpuPages)) {
+      Warning("Error initializing crosspage\n");
+      goto error;
+   }
+
   args.vm = vm;
   args.shRegions = shRegions;
-   ret = MonLoader_Process(bi->header, numVCPUs, &args, &errLine, &errVcpu);
+   ret = MonLoader_Process(header, numVCPUs, &args, &errLine, &errVcpu);
   if (ret != ML_OK) {
      Warning("Error processing bootstrap: error %d at line %u, vcpu %u\n",
               ret, errLine, errVcpu);
@@ -1268,78 +1293,6 @@ Vmx86_Close(void)
 }


-/*
- *-----------------------------------------------------------------------------
- *
- * Vmx86_InitVM --
- *
- *    Initializaiton of the VM.  Expects all initial arguments
- *    to be part of the InitBlock structure.
- *
- * Results:
- *    0 on success
- *    != 0 on failure
- *
- * Side effects:
- *    Many
- *
- *-----------------------------------------------------------------------------
- */
-
-int
-Vmx86_InitVM(VMDriver *vm,          // IN
-             InitBlock *initParams) // IN/OUT: Initial params from the VM
-{
-   int retval;
-
-   if (initParams->magicNumber != INIT_BLOCK_MAGIC) {
-      Warning("Bad magic number for init block 0x%x\n",
-              initParams->magicNumber);
-
-      return 1;
-   }
-   if (vm->numVCPUs == 0 || vm->numVCPUs > MAX_VCPUS) {
-      Warning("Invalid number of VCPUs %d\n", vm->numVCPUs);
-
-      return 1;
-   }
-
-   /*
-    * Initialize the driver's part of the cross-over page used to
-    * talk to the monitor
-    */
-
-   retval = Task_InitCrosspage(vm, initParams);
-   if (retval) {
-      Warning("Task crosspage init died with retval=%d\n", retval);
-      /*
-       *  Note that any clean-up of resources will be handled during
-       *  power-off when Vmx86_ReleaseVM() is called as part of
-       *  MonitorLoop_PowerOff().
-       */
-
-      return 1;
-   }
-
-   /*
-    *  Check if we want to arbitrarily fail every N VM initializations.
-    *  Useful in testing PR 72482.
-    */
-
-   if (initParams->vmInitFailurePeriod != 0) {
-      static uint32 counter = 0;
-
-      if ((++counter) % initParams->vmInitFailurePeriod == 0) {
-         Warning("VM initialization failed on %d iteration\n", counter);
-
-         return 1;
-      }
-   }
-
-   return 0;
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -1690,42 +1643,13 @@ Vmx86_GetNumVMs(void)
 }


-int32
-Vmx86_GetTotalMemUsage(void)
+static INLINE PageCnt
+Vmx86MinAllocationFunc(PageCnt nonpaged,
+                       PageCnt anonymous,
+                       PageCnt mainmem,
+                       Percent memPct)
 {
-   VMDriver *vm;
-   int totalmem = 0;
-
-   HostIF_GlobalLock(15);
-   vm = vmDriverList;
-
-   for (vm = vmDriverList; vm != NULL; vm = vm->nextDriver) {
-      /*
-       * The VM lock is not strictly necessary as the vm will
-       * stay on the list until we release the global lock and
-       * because of order in which "admitted" and "mainMemSize"
-       * are set when each VM is admitted.
-       */
-
-      if (vm->memInfo.admitted) {
-          totalmem += PAGES_2_MBYTES(ROUNDUP(vm->memInfo.mainMemSize,
-                                             MBYTES_2_PAGES(1)));
-      }
-   }
-
-   HostIF_GlobalUnlock(15);
-
-   return totalmem;
-}
-
-
-static INLINE unsigned
-Vmx86MinAllocationFunc(unsigned nonpaged,
-                       unsigned anonymous,
-                       unsigned mainmem,
-                       unsigned memPct)
-{
-   return RatioOf(memPct, mainmem, 100) + nonpaged + anonymous;
+   return (memPct * mainmem) / 100 + nonpaged + anonymous;
 }


@@ -1749,12 +1673,11 @@ Vmx86MinAllocationFunc(unsigned nonpaged,
 *----------------------------------------------------------------------
 */

-static INLINE unsigned
+static INLINE PageCnt
 Vmx86MinAllocation(VMDriver *vm,
-                   unsigned memPct)
+                   Percent memPct)
 {
   ASSERT(HostIF_VMLockIsHeld(vm));
-
   return Vmx86MinAllocationFunc(vm->memInfo.nonpaged, vm->memInfo.anonymous,
                                 vm->memInfo.mainMemSize, memPct);
 }
@@ -1780,12 +1703,11 @@ Vmx86MinAllocation(VMDriver *vm,
 *----------------------------------------------------------------------
 */

-static unsigned
-Vmx86CalculateGlobalMinAllocation(unsigned memPct)
+static PageCnt
+Vmx86CalculateGlobalMinAllocation(Percent memPct)
 {
   VMDriver *vm;
-   unsigned minAllocation = 0;
-
+   PageCnt minAllocation = 0;
   ASSERT(HostIF_GlobalLockIsHeld());
   /* Pages of other vms required to fit inside the hard limit. */
   for (vm = vmDriverList; vm; vm = vm->nextDriver) {
@@ -1819,10 +1741,9 @@ Vmx86CalculateGlobalMinAllocation(unsigned memPct)
 */

 static INLINE_SINGLE_CALLER void
-Vmx86UpdateMinAllocations(unsigned memPct)  // IN:
+Vmx86UpdateMinAllocations(Percent memPct)  // IN:
 {
   VMDriver *vm;
-
   ASSERT(HostIF_GlobalLockIsHeld());
   /* Pages of other vms required to fit inside the hard limit. */
   for (vm = vmDriverList; vm; vm = vm->nextDriver) {
@@ -1855,7 +1776,7 @@ Vmx86UpdateMinAllocations(unsigned memPct)  // IN:
 */

 Bool
-Vmx86_SetConfiguredLockedPagesLimit(unsigned limit)  // IN:
+Vmx86_SetConfiguredLockedPagesLimit(PageCnt limit)  // IN:
 {
   Bool retval = FALSE;

@@ -1889,7 +1810,7 @@ Vmx86_SetConfiguredLockedPagesLimit(unsigned limit)  // IN:
 */

 void
-Vmx86_SetDynamicLockedPagesLimit(unsigned limit)  // IN:
+Vmx86_SetDynamicLockedPagesLimit(PageCnt limit)  // IN:
 {
   HostIF_GlobalLock(11);
   lockedPageLimit.dynamic = limit;
@@ -2031,33 +1952,29 @@ Vmx86_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver
 *-----------------------------------------------------------------------------
 */

-int
+int64
 Vmx86_AllocLockedPages(VMDriver *vm,         // IN: VMDriver
                       VA64 addr,            // OUT: VA of an array for
                                             //      allocated MPNs.
-                       unsigned numPages,    // IN: number of pages to allocate
+                       PageCnt numPages,     // IN: number of pages to allocate
                       Bool kernelMPNBuffer, // IN: is the MPN buffer in kernel
                                             //     or user address space?
                       Bool ignoreLimits)    // IN: should limits be ignored?
 {
-   int allocatedPages;
-
+   int64 allocatedPages;
   if (!Vmx86ReserveFreePages(vm, numPages, ignoreLimits)) {
      // XXX What kind of system-specific error code is that? --hpreg
      return PAGE_LOCK_LIMIT_EXCEEDED;
   }
-
   HostIF_VMLock(vm, 7);
   allocatedPages = HostIF_AllocLockedPages(vm, addr, numPages,
                                            kernelMPNBuffer);
   HostIF_VMUnlock(vm, 7);
-
   if (allocatedPages < 0) {
      Vmx86UnreserveFreePages(vm, numPages);
   } else if (allocatedPages < numPages) {
      Vmx86UnreserveFreePages(vm, numPages - allocatedPages);
   }
-
   return allocatedPages;
 }

@@ -2082,14 +1999,13 @@ Vmx86_AllocLockedPages(VMDriver *vm,         // IN: VMDriver

 int
 Vmx86_FreeLockedPages(VMDriver *vm,         // IN: VM instance pointer
-                      VA64 addr,            // IN: user or kernel array of MPNs to free
-                      unsigned numPages,    // IN: number of pages to free
-                      Bool kernelMPNBuffer) // IN: is the MPN buffer in kernel or user address space?
+                      MPN *mpns,            // IN: MPNs to free
+                      PageCnt numPages)     // IN: number of pages to free
 {
   int ret;

   HostIF_VMLock(vm, 8);
-   ret = HostIF_FreeLockedPages(vm, addr, numPages, kernelMPNBuffer);
+   ret = HostIF_FreeLockedPages(vm, mpns, numPages);
   HostIF_VMUnlock(vm, 8);

   if (ret == 0) {
@@ -2170,6 +2086,33 @@ Vmx86_GetNextAnonPage(VMDriver *vm,       // IN: VM instance pointer
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_GetNumAnonPages --
+ *
+ *      Queries the driver for the total number of anonymous pages.
+ *
+ * Results:
+ *      Total number of anonymous pages
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+PageCnt
+Vmx86_GetNumAnonPages(VMDriver *vm)       // IN: VM instance pointer
+{
+   PageCnt ret;
+   HostIF_VMLock(vm, 45);
+   ret = HostIF_GetNumAnonPages(vm);
+   HostIF_VMUnlock(vm, 45);
+   return ret;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -2275,10 +2218,10 @@ Vmx86_GetMemInfo(VMDriver *curVM,

 static void
 Vmx86SetMemoryUsage(VMDriver *curVM,       // IN/OUT
-                    unsigned paged,        // IN
-                    unsigned nonpaged,     // IN
-                    unsigned anonymous,    // IN
-                    unsigned aminVmMemPct) // IN
+                    PageCnt  paged,        // IN
+                    PageCnt  nonpaged,     // IN
+                    PageCnt  anonymous,    // IN
+                    Percent  aminVmMemPct) // IN
 {
   ASSERT(HostIF_VMLockIsHeld(curVM));
   curVM->memInfo.paged         = paged;
@@ -2317,8 +2260,7 @@ Vmx86_Admit(VMDriver *curVM,     // IN
            VMMemInfoArgs *args) // IN/OUT
 {
   Bool allowAdmissionCheck = FALSE;
-   unsigned int globalMinAllocation;
-
+   PageCnt globalMinAllocation;
   HostIF_GlobalLock(9);

   /*
@@ -2360,8 +2302,8 @@ Vmx86_Admit(VMDriver *curVM,     // IN

 #if defined _WIN32
   if (curVM->memInfo.admitted) {
-      unsigned int allocatedPages, nonpaged;
-      signed int pages;
+      PageCnt allocatedPages, nonpaged;
+      int64 pages;
      MPN *mpns;

      /*
@@ -2397,10 +2339,7 @@ Vmx86_Admit(VMDriver *curVM,     // IN
       * XXX Do not free the pages but hand them directly to the admitted VM.
       */

-      for (pages = 0; pages < allocatedPages; pages += ALLOCATE_CHUNK_SIZE) {
-         Vmx86_FreeLockedPages(curVM, PtrToVA64(mpns + pages),
-                               MIN(ALLOCATE_CHUNK_SIZE, allocatedPages - pages), TRUE);
-      }
+      Vmx86_FreeLockedPages(curVM, mpns, allocatedPages);
      HostIF_FreeKernelMem(mpns);
 #undef ALLOCATE_CHUNK_SIZE

@@ -2429,11 +2368,11 @@ undoAdmission:
 Bool
 Vmx86_Readmit(VMDriver *curVM, OvhdMem_Deltas *delta)
 {
-   unsigned globalMinAllocation, newMinAllocation;
+   PageCnt globalMinAllocation, newMinAllocation;
   Bool retval = FALSE;
-   int paged;
-   int nonpaged;
-   int anonymous;
+   int64 paged;
+   int64 nonpaged;
+   int64 anonymous;

   HostIF_GlobalLock(31);
   globalMinAllocation = Vmx86CalculateGlobalMinAllocation(minVmMemPct);
@@ -2542,19 +2481,20 @@ static void
 Vmx86EnableHVOnCPU(void)
 {
   if (CPUID_HostSupportsSVM()) {
-      uint64 vmCR = __GET_MSR(MSR_VM_CR);
+      uint64 vmCR = X86MSR_GetMSR(MSR_VM_CR);
      if (!SVM_LockedFromFeatures(vmCR)) {
         CPUIDRegs regs;
         __GET_CPUID(0x8000000A, &regs);
         if (CPUID_GET(0x8000000A, EDX, SVM_LOCK, regs.edx) != 0) {
-            __SET_MSR(MSR_VM_CR, (vmCR & ~MSR_VM_CR_SVME_DISABLE) |
-                                  MSR_VM_CR_SVM_LOCK);
+            X86MSR_SetMSR(MSR_VM_CR, (vmCR & ~MSR_VM_CR_SVME_DISABLE) |
+                                      MSR_VM_CR_SVM_LOCK);
         }
      }
   } else if (CPUID_HostSupportsVT()) {
-      uint64 featCtl = __GET_MSR(MSR_FEATCTL);
+      uint64 featCtl = X86MSR_GetMSR(MSR_FEATCTL);
      if (!VT_LockedFromFeatures(featCtl)) {
-         __SET_MSR(MSR_FEATCTL, featCtl | MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE);
+         X86MSR_SetMSR(MSR_FEATCTL,
+                       featCtl | MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE);
      }
   }
 }
@@ -2925,9 +2865,9 @@ Vmx86CheckVMXStatus(const char *operation, // IN: Operation string
 #ifdef __GNUC__
 #if __GNUC__== 4 && __GNUC_MINOR__ > 3
      if (status == VMX_FailValid) {
-         uint64 errorCode;
+         size_t errorCode;
         VMREAD_2_STATUS(VT_VMCS_VMINSTR_ERR, &errorCode);
-         Log("VM-instruction error: Error %"FMT64"d\n", errorCode);
+         Log("VM-instruction error: Error %"FMTSZ"d\n", errorCode);
      }
 #endif
 #endif
@@ -3169,12 +3109,12 @@ Vmx86PerfCtrInUse(Bool isGen, unsigned pmcNum, unsigned ctrlMSR,
                  unsigned cntMSR, Bool hasPGC)
 {
   volatile unsigned delay;
-   uint64 origPGC = hasPGC ? __GET_MSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR) : 0;
+   uint64 origPGC = hasPGC ? X86MSR_GetMSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR) : 0;
   uint64 pmcCtrl;
   uint64 pmcCount, count;
   uint64 ctrlEna, pgcEna;

-   pmcCtrl = __GET_MSR(ctrlMSR);
+   pmcCtrl = X86MSR_GetMSR(ctrlMSR);
   if (isGen) {
      ASSERT(pmcNum < 32);
      if ((pmcCtrl & PERFCTR_CPU_ENABLE) != 0) {
@@ -3191,21 +3131,21 @@ Vmx86PerfCtrInUse(Bool isGen, unsigned pmcNum, unsigned ctrlMSR,
      ctrlEna = pmcCtrl | PERFCTR_CORE_FIXED_KERNEL_MASKn(pmcNum);
      pgcEna = CONST64U(1) << (pmcNum + 32);
   }
-   pmcCount = __GET_MSR(cntMSR);
+   pmcCount = X86MSR_GetMSR(cntMSR);
   /* Enable the counter. */
-   __SET_MSR(ctrlMSR, ctrlEna);
+   X86MSR_SetMSR(ctrlMSR, ctrlEna);
   if (hasPGC) {
-      __SET_MSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, pgcEna | origPGC);
+      X86MSR_SetMSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, pgcEna | origPGC);
   }
   /* Retire some instructions and wait a few cycles. */
   for (delay = 0; delay < 100; delay++) ;
   /* Disable the counter. */
   if (hasPGC) {
-      __SET_MSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, origPGC);
+      X86MSR_SetMSR(PERFCTR_CORE_GLOBAL_CTRL_ADDR, origPGC);
   }
-   count = __GET_MSR(cntMSR);
-   __SET_MSR(ctrlMSR, pmcCtrl);
-   __SET_MSR(cntMSR, pmcCount);
+   count = X86MSR_GetMSR(cntMSR);
+   X86MSR_SetMSR(ctrlMSR, pmcCtrl);
+   X86MSR_SetMSR(cntMSR, pmcCount);
   return count == pmcCount;
 }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,13 +32,14 @@
 #include "x86msr.h"
 #include "modulecall.h"
 #include "vcpuid.h"
-#include "initblock.h"
 #include "iocontrols.h"
 #include "numa_defs.h"
 #include "rateconv.h"
 #include "vmmem_shared.h"
 #include "apic.h"
 #include "bootstrap_vmm.h"
+#include "sharedAreaVmmon.h"
+#include "statVarsVmmon.h"

 typedef struct TSCDelta {
   Atomic_uint64 min;
@@ -55,23 +56,19 @@ struct VmmBlobInfo;
 typedef struct VMDriver {
   /* Unique (in the driver), strictly positive, VM ID used by userland. */
   int                     userID;
-
   Vcpuid                  numVCPUs;         /* Number of vcpus in VM. */
-
   struct VMDriver        *nextDriver;       /* Next on list of all VMDrivers */
-
   struct VMHost          *vmhost;           /* Host-specific fields. */
-
   MPN                    *ptRootMpns;       /* numVCPUs-sized array. */
   struct VmmBlobInfo     *blobInfo;         /* VMM bootstrap blob info. */
-
+   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
+   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
   APICDescriptor          hostAPIC;
-
   struct MemTrack        *memtracker;       /* Memory tracker pointer */
   Bool                    checkFuncFailed;
   struct PerfCounter     *perfCounter;
@@ -81,6 +78,7 @@ typedef struct VMDriver {
   Atomic_uint64           ptscLatest;
   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
+   PageCnt                 numPTPPages;      /* Num PTP pages allocated. */
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -110,7 +108,7 @@ typedef struct PseudoTSC {

 extern PseudoTSC pseudoTSC;

-#define MAX_LOCKED_PAGES (-1)
+#define MAX_LOCKED_PAGES MAX_PPN

 extern void Vmx86_CacheNXState(void);
 extern VMDriver *Vmx86_CreateVM(VA64 bsBlob,
@@ -121,11 +119,10 @@ extern Bool Vmx86_ProcessBootstrap(VMDriver *vm,
                                   uint32 numBytes,
                                   uint32 headerOffset,
                                   uint16 numVCPUs,
-                                   UserVA64 *ptRootVAs,
+                                   PerVcpuPages *perVcpuPages,
                                   VMSharedRegion *shRegions);
 extern int Vmx86_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 extern int Vmx86_ReleaseVM(VMDriver *vm);
-extern int Vmx86_InitVM(VMDriver *vm, InitBlock *initParams);
 extern int Vmx86_LateInitVM(VMDriver *vm);
 extern int Vmx86_RunVM(VMDriver *vm, Vcpuid vcpuid);
 extern void Vmx86_YieldToSet(VMDriver *vm, Vcpuid currVcpu, const VCPUSet *req,
@@ -142,22 +139,21 @@ extern int Vmx86_UnlockPage(VMDriver *vm, VA64 uAddr);
 extern int Vmx86_UnlockPageByMPN(VMDriver *vm, MPN mpn, VA64 uAddr);
 extern MPN Vmx86_GetRecycledPage(VMDriver *vm);
 extern int Vmx86_ReleaseAnonPage(VMDriver *vm, MPN mpn);
-extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned numPages, Bool kernelMPNBuffer,
-                                  Bool ignoreLimits);
-extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                 unsigned numPages, Bool kernelMPNBuffer);
+extern int64 Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                    PageCnt numPages, Bool kernelMPNBuffer,
+                                    Bool ignoreLimits);
+extern int Vmx86_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn);
+extern MPN Vmx86_GetNumAnonPages(VMDriver *vm);
 extern MPN Vmx86_AllocLowPage(VMDriver *vm, Bool ignoreLimits);
 extern void *Vmx86_Calloc(size_t numElements,
                          size_t elementSize,
-                          int nonPageable);
+                          Bool nonPageable);
 extern void  Vmx86_Free(void *ptr);

 extern int32 Vmx86_GetNumVMs(void);
-extern int32 Vmx86_GetTotalMemUsage(void);
-extern Bool Vmx86_SetConfiguredLockedPagesLimit(unsigned limit);
-extern void Vmx86_SetDynamicLockedPagesLimit(unsigned limit);
+extern Bool Vmx86_SetConfiguredLockedPagesLimit(PageCnt limit);
+extern void Vmx86_SetDynamicLockedPagesLimit(PageCnt limit);
 extern Bool Vmx86_GetMemInfo(VMDriver *curVM,
                             Bool curVMOnly,
                             VMMemInfoArgs *args,
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -345,6 +345,34 @@ CircList_Push(ListItem *p,               // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_InsertAfter --
+ *
+ *      Adds a new member to the list after the provided item.  Assumes p
+ *      is not a member of a list already.
+ *
+ * Result:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE void
+CircList_InsertAfter(ListItem *p,      // IN:
+                     ListItem *after)  // IN:
+{
+   p->prev = after;
+   p->next = after->next;
+   p->next->prev = p;
+   after->next = p;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -0,0 +1,30 @@
+#ifndef __COMPAT_POLL_H__
+#define __COMPAT_POLL_H__
+
+#include <linux/poll.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0))
+
+#ifndef __poll_t
+typedef unsigned int __poll_t;
+#endif
+
+static inline __poll_t compat_vfs_poll(struct file *file,
+				       struct poll_table_struct *pt)
+{
+	if (unlikely(!file->f_op->poll))
+		return DEFAULT_POLLMASK;
+	return file->f_op->poll(file, pt);
+}
+
+#else
+
+static inline __poll_t compat_vfs_poll(struct file *file,
+				       struct poll_table_struct *pt)
+{
+	return vfs_poll(file, pt);
+}
+
+#endif
+
+#endif /* __COMPAT_POLL_H__ */
@@ -1,77 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _CPUID_INFO_H
-#define _CPUID_INFO_H
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-
-#include "includeCheck.h"
-
-#include "vm_basic_asm.h"
-#include "x86cpuid_asm.h"
-
-#if defined __cplusplus
-extern "C" {
-#endif
-
-
-typedef struct CPUID0 {
-   int numEntries;
-   char name[16];      // 4 extra bytes to null terminate
-} CPUID0;
-
-typedef struct CPUID1 {
-   uint32 version;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID1;
-
-typedef struct CPUID80 {
-   uint32 numEntries;
-   uint32 ebx;
-   uint32 ecx;
-   uint32 edx;
-} CPUID80;
-
-typedef struct CPUID81 {
-   uint32 eax;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID81;
-
-typedef struct CPUIDSummary {
-   CPUID0  id0;
-   CPUID1  id1;
-   CPUIDRegs ida;
-   CPUID80 id80;
-   CPUID81 id81;
-   CPUIDRegs id88, id8a;
-} CPUIDSummary;
-
-
-#if defined __cplusplus
-} // extern "C"
-#endif
-
-#endif // _CPUID_INFO_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,7 +37,6 @@
 #error iocontrols.h is for hosted vmmon, do not use on visor
 #endif

-#include "basic_initblock.h"
 #include "x86segdescrs.h"
 #include "rateconv.h"
 #include "overheadmem_types.h"
@@ -45,6 +44,8 @@
 #include "numa_defs.h"
 #include "bootstrap_vmm.h"
 #include "contextinfo.h"
+#include "vcpuid.h"
+#include "sharedAreaType.h"

 #if defined __cplusplus
 extern "C" {
@@ -144,7 +145,7 @@ PtrToVA64(void const *ptr) // IN
 * See bora/doc/vmcore details.
 */

-#define VMMON_VERSION           (360 << 16 | 0)
+#define VMMON_VERSION           (385 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -153,11 +154,8 @@ PtrToVA64(void const *ptr) // IN
 * ENOMEM returned after MAX_VMS virtual machines created
 */

-#ifdef VMX86_SERVER
-#define MAX_VMS 128
-#else
 #define MAX_VMS 64
-#endif
+
 /*
 * MsgWaitForMultipleObjects doesn't scale well enough on Win32.
 * Allocate with MAX_VMS so static buffers are large, but do
@@ -209,11 +207,13 @@ enum IOCTLCmd {
   IOCTLCMD(VERSION) = IOCTLCMD(FIRST),
   IOCTLCMD(CREATE_VM),
   IOCTLCMD(PROCESS_BOOTSTRAP),
+   IOCTLCMD(REGISTER_SHARED),
+   IOCTLCMD(REGISTER_STATVARS),
   IOCTLCMD(RELEASE_VM),
   IOCTLCMD(GET_NUM_VMS),
-   IOCTLCMD(INIT_VM),
   IOCTLCMD(RUN_VM),
   IOCTLCMD(LOOK_UP_MPN),
+   IOCTLCMD(GET_VMM_PAGE_ROOT),
   IOCTLCMD(LOCK_PAGE),
   IOCTLCMD(UNLOCK_PAGE),
   IOCTLCMD(APIC_INIT),
@@ -222,7 +222,6 @@ enum IOCTLCmd {
   IOCTLCMD(ADMIT),
   IOCTLCMD(UPDATE_MEM_INFO),
   IOCTLCMD(READMIT),
-   IOCTLCMD(GET_TOTAL_MEM_USAGE),
   IOCTLCMD(GET_KHZ_ESTIMATE),
   IOCTLCMD(SET_HOST_CLOCK_RATE),
   IOCTLCMD(READ_PAGE),
@@ -232,6 +231,7 @@ enum IOCTLCmd {
    /* AWE calls */
   IOCTLCMD(ALLOC_LOCKED_PAGES),
   IOCTLCMD(GET_NEXT_ANON_PAGE),
+   IOCTLCMD(GET_NUM_ANON_PAGES),

   IOCTLCMD(GET_ALL_MSRS),

@@ -276,7 +276,7 @@ enum IOCTLCmd {
 #endif

 #if defined __APPLE__
-   IOCTLCMD(ALLOC_LOW_PAGES),
+   IOCTLCMD(GET_NUM_RESPONDING_CPUS),
   IOCTLCMD(INIT_DRIVER),
   IOCTLCMD(BLUEPILL),
 #endif
@@ -317,13 +317,15 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_CREATE_VM           VMIOCTL_BUFFERED(CREATE_VM)
 #define IOCTL_VMX86_RELEASE_VM          VMIOCTL_BUFFERED(RELEASE_VM)
 #define IOCTL_VMX86_PROCESS_BOOTSTRAP   VMIOCTL_BUFFERED(PROCESS_BOOTSTRAP)
+#define IOCTL_VMX86_REGISTER_SHARED     VMIOCTL_BUFFERED(REGISTER_SHARED)
+#define IOCTL_VMX86_REGISTER_STATVARS   VMIOCTL_BUFFERED(REGISTER_STATVARS)
 #define IOCTL_VMX86_GET_NUM_VMS         VMIOCTL_BUFFERED(GET_NUM_VMS)
-#define IOCTL_VMX86_INIT_VM             VMIOCTL_BUFFERED(INIT_VM)
 #define IOCTL_VMX86_RUN_VM              VMIOCTL_NEITHER(RUN_VM)
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
+#define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
 #define IOCTL_VMX86_UNLOCK_PAGE         VMIOCTL_BUFFERED(UNLOCK_PAGE)
 #define IOCTL_VMX86_APIC_INIT           VMIOCTL_BUFFERED(APIC_INIT)
@@ -347,6 +349,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UNLOCK_PAGE_BY_MPN  VMIOCTL_BUFFERED(UNLOCK_PAGE_BY_MPN)
 #define IOCTL_VMX86_ALLOC_LOCKED_PAGES  VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES)
 #define IOCTL_VMX86_GET_NEXT_ANON_PAGE  VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE)
+#define IOCTL_VMX86_GET_NUM_ANON_PAGES  VMIOCTL_BUFFERED(GET_NUM_ANON_PAGES)

 #define IOCTL_VMX86_READ_DISASM_PROC_BINARY \
                                      VMIOCTL_BUFFERED(READ_DISASM_PROC_BINARY)
@@ -369,6 +372,8 @@ enum IOCTLCmd {
 #endif


+#define INIT_BLOCK_MAGIC     (0x1789 + 14)
+
 /*
 * Flags sent into APICBASE ioctl
 */
@@ -395,6 +400,14 @@ union {
 #include "vmware_pack_end.h"
 VMLockPage;

+typedef
+#include "vmware_pack_begin.h"
+union {
+   Vcpuid vcpuid; // IN: VCPU
+   MPN pageRoot;  // OUT: MPN of the VCPU's page root
+}
+#include "vmware_pack_end.h"
+VcpuPageRoot;

 typedef struct VMAPICInfo {
   uint32 flags;
@@ -413,9 +426,9 @@ typedef struct VMAPICInfo {
 * all these values at any given time.
 */
 typedef struct LockedPageLimit {
-   uint32 host;        // driver calculated maximum for this host
-   uint32 configured;  // user defined maximum pages to lock
-   uint32 dynamic;     // authd hardLimitMonitor pages to lock
+   PageCnt host;        // driver calculated maximum for this host
+   PageCnt configured;  // user defined maximum pages to lock
+   PageCnt dynamic;     // authd hardLimitMonitor pages to lock
 } LockedPageLimit;

 /*
@@ -436,15 +449,15 @@ typedef struct LockedPageLimit {
 */

 typedef struct VMMemMgmtInfo {
-   uint32          minAllocation;   // minimum pages for vm
-   uint32          maxAllocation;   // maximum pages the vm could lock
+   PageCnt         minAllocation;   // minimum pages for vm
+   PageCnt         maxAllocation;   // maximum pages the vm could lock
+   PageCnt         nonpaged;        // overhead memory (guest, mmap)
+   PageCnt         paged;           // vmx memory (malloc, statics)
+   PageCnt         anonymous;       // vmm memory
+   PageCnt         mainMemSize;     // guest main memory size
+   PageCnt         locked;          // number of pages locked by this vm
+   PageCnt         perVMOverhead;   // memory for vmx/vmmon overheads
   uint32          shares;          // proportional sharing weight
-   uint32          nonpaged;        // overhead memory (guest, mmap)
-   uint32          paged;           // vmx memory (malloc, statics)
-   uint32          anonymous;       // vmm memory
-   uint32          mainMemSize;     // guest main memory size
-   uint32          locked;          // number of pages locked by this vm
-   uint32          perVMOverhead;   // memory for vmx/vmmon overheads
   Percent         touchedPct;      // % of guest memory being touched
   Percent         dirtiedPct;      // % of guest memory being dirtied
   Bool            admitted;        // admission control
@@ -467,14 +480,14 @@ typedef struct VMMemMgmtInfoPatch {

 typedef struct VMMemInfoArgs {
   uint64          currentTime;        // Host time in secs of the call.
-   uint32          minVmMemPct;        // % of vm that must fit in memory
-   uint32          globalMinAllocation;// pages that must fit in maxLockedPages
-   uint32          numLockedPages;     // total locked pages by all vms
+   PageCnt         globalMinAllocation;// pages that must fit in maxLockedPages
+   PageCnt         numLockedPages;     // total locked pages by all vms
   LockedPageLimit lockedPageLimit;    // set of locked page limits
-   uint32          maxLockedPages;     // effective limit on locked pages
+   PageCnt         maxLockedPages;     // effective limit on locked pages
   uint32          callerIndex;        // this vm's index memInfo array
   uint32          numVMs;             // number of running VMs
-   uint8           _pad[4];
+   Percent         minVmMemPct;        // % of vm that must fit in memory
+   uint8           _pad[7];
   VMMemMgmtInfo   memInfo[1];
 } VMMemInfoArgs;

@@ -487,9 +500,9 @@ typedef struct VMMPNNext {
 } VMMPNNext;

 typedef struct VMMPNList {
-   uint32    mpnCount;   // IN (and OUT on Mac OS)
+   PageCnt   mpnCount;   // IN (and OUT on Mac OS)
   Bool      ignoreLimits;
-   uint8     _pad[3];
+   uint8     _pad[7];
   VA64      mpnList;    // IN: User VA of an array of 64-bit MPNs.
 } VMMPNList;

@@ -525,8 +538,6 @@ typedef struct PTSCCheckParams {
   uint8  _pad[7];
 } PTSCCheckParams;

-#ifndef VMX86_SERVER
-
 typedef struct IPIVectors {
   /*
    * Vectors we have allocated or stolen for the monitor interrupts.
@@ -535,8 +546,6 @@ typedef struct IPIVectors {
   uint8 hvIPIVector;
 } IPIVectors;

-#endif
-
 /*
 * Arguments and return value for VM creation.
 */
@@ -551,11 +560,33 @@ typedef struct VMCreateBlock {
 * Information about a shared region.
 */
 typedef struct VMSharedRegion {
-   uint32 index;
-   VPN    baseVpn;
-   uint32 numPages;
+   SharedAreaType index;
+   VPN            baseVpn;
+   uint32         numPages;
 } VMSharedRegion;

+/*
+ * Arguments for VMM shared area registration.
+ */
+typedef struct VMSharedAreaRegistrationBlock {
+   Vcpuid vcpu;            // IN: VCPU being registered.
+   VMSharedRegion region;  // IN: Shared region being registered.
+} VMSharedAreaRegistrationBlock;
+
+/*
+ * Information about a VM's statvars.
+ */
+typedef struct VMStatVarsRegistrationBlock {
+   VPN     baseVpn;
+   PageCnt numPages;
+   Vcpuid  vcpu;
+} VMStatVarsRegistrationBlock;
+
+typedef struct {
+   VA64   crosspage; // IN: User VA of VCPU crosspage.
+   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
+} PerVcpuPages;
+
 /*
 * Arguments for VMM bootstrap processing.
 */
@@ -564,10 +595,17 @@ typedef struct VMProcessBootstrapBlock {
   uint32         numBytes;      // IN: Size of VMM bootstrap blob.
   uint32         headerOffset;  // IN: Offset of header in blob.
   uint16         numVCPUs;      // IN: Number of VCPUs.
-   VA64           ptRootVAs[MAX_VCPUS];  // IN: User VA of PT roots.
   VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions.
+   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

+static INLINE size_t
+GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
+{
+   VMProcessBootstrapBlock *args = NULL;
+   return sizeof *args + numVCPUs * sizeof args->perVcpuPages[0];
+}
+
 /*
 * Arguments for VMM context retrieval.
 */
@@ -583,29 +621,20 @@ typedef union {
 * macros to marshall real arguments to mmap's made-up 'offset' argument.
 */

-#define VMMON_MAP_MT_LOW4GB     0
-#define VMMON_MAP_MT_LOW16MB    1
-#define VMMON_MAP_MT_ANY        2
-
 #define VMMON_MAP_OFFSET_SHIFT  0
 #define VMMON_MAP_OFFSET_MASK   0x00000FFF
 #define VMMON_MAP_ORDER_SHIFT   12
 #define VMMON_MAP_ORDER_MASK    0xF
-#define VMMON_MAP_MT_SHIFT      16
-#define VMMON_MAP_MT_MASK       0x7
-#define VMMON_MAP_RSVD_SHIFT    19
+#define VMMON_MAP_RSVD_SHIFT    16

 #define VMMON_MAP_RSVD(base)    \
                ((base) >> VMMON_MAP_RSVD_SHIFT)
-#define VMMON_MAP_MT(base)      \
-                (((base) >> VMMON_MAP_MT_SHIFT) & VMMON_MAP_MT_MASK)
 #define VMMON_MAP_ORDER(base)   \
                (((base) >> VMMON_MAP_ORDER_SHIFT) & VMMON_MAP_ORDER_MASK)
 #define VMMON_MAP_OFFSET(base)  \
                (((base) >> VMMON_MAP_OFFSET_SHIFT) & VMMON_MAP_OFFSET_MASK)

-#define VMMON_MAP_BASE(mt, order)   (((mt) << VMMON_MAP_MT_SHIFT) | \
-                                     ((order) << VMMON_MAP_ORDER_SHIFT))
+#define VMMON_MAP_BASE(order)   ((order) << VMMON_MAP_ORDER_SHIFT)

 #elif defined _WIN32
 /*
@@ -616,7 +645,6 @@ typedef struct VMAllocContiguousMem {
   VA64   mpnList;  // IN: User VA of an array of 64-bit MPNs.
   uint32 mpnCount; // IN
   uint32 order;    // IN
-   MPN    maxMPN;   // IN
 } VMAllocContiguousMem;
 #elif defined __APPLE__
 #   include "iocontrolsMacos.h"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,8 +42,8 @@
 #include "mon_assert.h"

 #define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
-#define CP_STUB_SIZE 16     /* A relative jmp instruction (5 bytes)
-                               padded to the next 16 byte boundary. */
+#define CP_STUB_SIZE     16     /* A relative jmp instruction (5 bytes)
+                                   padded to the next 16 byte boundary. */

 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
@@ -66,7 +66,10 @@
   MC(GET_MON_IPI_VECTOR)                                                     \
   MC(GET_HV_IPI_VECTOR)                                                      \
   MC(GET_HOST_TIMER_VECTORS)                                                 \
-   MC(BOOTSTRAP_CLEANUP)
+   MC(BOOTSTRAP_CLEANUP)                                                      \
+   MC(GET_SHARED_AREA)                                                        \
+   MC(GET_STAT_VARS)                                                          \
+   MC(GET_NUM_PTP_PAGES)

 /*
 *----------------------------------------------------------------------
@@ -292,7 +295,8 @@ struct VMCrossPageData {
   uint64   crosspageMA;

   uint64   hostDR[8];
-   LA64     hostCrossPageLA;   // where host has crosspage mapped
+   LA64     crossPageLA;       // where host/PTP map the cross page
+   LA64     crossGDTLA;        // where host/PTP map the cross GDT
   uint16   hostInitial64CS;
   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
@@ -310,8 +314,6 @@ struct VMCrossPageData {
   Task64   monTask;          /* vmm's task */

   VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
-   LA64              vmmCrossPageLA;
-   LA64              vmmCrossGDTLA;   // where crossGDT is mapped by PT patch

   /*
    * The monitor may requests up to two actions when returning to the
@@ -391,6 +393,7 @@ struct VMCrossPageData {
   uint8         _pad7[4];
   uint64        wsUD2;                       // IP of ud2 instr or 0 if unset.
   uint64        specCtrl; /* host MSR_SPEC_CTRL value before world switch. */
+   uint8         _pad8[8];
 }
 #include "vmware_pack_end.h"
 VMCrossPageData;
@@ -447,17 +450,14 @@ struct VMCrossPage {
 #include "vmware_pack_end.h"
 VMCrossPage;

-#define CROSSPAGE_VERSION_BASE 0xc09 /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc0c /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE  ((VMCrossPage * const) VPN_2_VA(CROSS_PAGE_START))
+#define CROSS_PAGE             ((VMCrossPage * const)VPN_2_VA(CROSS_PAGE_START))
 #define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
 #endif

-#define NULLPAGE_LINEAR_START  (MONITOR_LINEAR_START + \
-                                PAGE_SIZE * CPL0_GUARD_PAGE_START)
-
 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
 #define MX_WAITNORMAL          1  // Must equal one; see linux module code.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,6 +25,8 @@
 #ifndef _MON_ADDR_LAYOUT_H
 #define _MON_ADDR_LAYOUT_H

+#include "vm_pagetable.h"
+
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMKERNEL
@@ -37,6 +39,8 @@ typedef struct VMM64_AddrLayout {
    *All address are VPNs and all lengths are numPages
    */
   uint64 monBase;            // MONITOR_BASE_VPN
+   uint64 monL5Start;         // MON_PAGE_TABLE_L5_START
+   uint32 monL5Len;           // MON_PAGE_TABLE_L5_LEN
   uint64 monL4Start;         // MON_PAGE_TABLE_L4_START
   uint32 monL4Len;           // MON_PAGE_TABLE_L4_LEN
   uint64 monL3Start;         // MON_PAGE_TABLE_L3_START
@@ -51,7 +55,21 @@ typedef struct VMM64_AddrLayout {
 } VMM64_AddrLayout;
 #pragma pack(pop)

-#define VMM_MONAS_FIRST_L4OFF   1
-#define VMM_MONAS_LAST_L4OFF  127
+#define VMM_MONAS_4LP_FIRST_L4OFF   1
+#define VMM_MONAS_4LP_LAST_L4OFF  130

+#define VMM_MONAS_5LP_FIRST_L5OFF   1
+#define VMM_MONAS_5LP_LAST_L5OFF   34
+
+static INLINE PT_Level
+MonAS_GetPagingLevel(void)
+{
+   return PT_LEVEL_4;
+}
+
+static INLINE Bool
+MonAS_Uses5LevelPaging(void)
+{
+   return MonAS_GetPagingLevel() == PT_LEVEL_5;
+}
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,8 +31,8 @@
 * contains code and data statically built, empty but allocated space, shared
 * and run-time initialized content.  The monitor loader header regularizes
 * encoding of address space information, allowing a common representation and
- * common code to be re-used for different contexts (vmmon for hosted, vmkernel
- * for ESX, and the VMX while deprivileging of bootstrap is ongoing).
+ * common code to be re-used for different contexts (vmmon for hosted
+ * and vmkernel for ESX).
 *
 * The Header
 * ==========
@@ -87,7 +87,6 @@
 * MonLoaderCallout_Init(): Initialize MonLoader callouts.
 * MonLoaderCallout_GetSharedHostPage(subIdx, page): Get shared host page MPN.
 * MonLoaderCallout_GetSharedUserPage(subIdx, page): Get shared user page MPN.
- * MonLoaderCallout_SetEntrypoint(): Sets the entry code/stack for execution.
 *
 * Building vs Importing
 * =====================
@@ -119,21 +118,54 @@
 #define _MON_LOADER

 #include "vm_basic_types.h"
-#include "x86paging_common.h"
-#include "x86types.h" /* PT_L1E */
+#include "vm_pagetable.h"
 #include "vcpuid.h"   /* Vcpuid */

+#if defined VM_X86_64
+#include "x86paging_64.h"
+#elif defined VM_ARM_64
+#include "arm64_vmsa.h"
+#endif
+
 #define ML_NAME_MAX 16

 /* ML perms are simple and abbreviated. */

+#if defined VM_X86_64
+
 #define ML_PERM_RWX (PTE_P | PTE_RW)
 #define ML_PERM_RW  (PTE_P | PTE_RW | PTE_NX)
 #define ML_PERM_RO  (PTE_P |          PTE_NX)
 #define ML_PERM_RX   PTE_P

+#define ML_PERM_TBL   ML_PERM_RWX
 #define ML_PERM_MASK (PTE_P| PTE_RW | PTE_NX | PTE_US)

+#define ML_PERM_PRESENT(_flags)     (((_flags) & PTE_P) != 0)
+#define ML_PERM_WRITEABLE(_flags)   (((_flags) & PTE_RW) != 0)
+
+#define ML_PTE_2_PFN(_pte)          LM_PTE_2_PFN(_pte)
+
+#elif defined VM_ARM_64
+
+#define _ML_PERM_COMMON (ARM_PTE_BLOCK_AP(ARM_AP_PL0)   | ARM_PTE_BLOCK_AF | \
+                         ARM_PTE_BLOCK_SH(ARM_SH_OUTER) | ARM_PTE_BLOCK_L3_TYPE)
+#define ARM_PTE_BLOCK_AP_RO ARM_PTE_BLOCK_AP(ARM_AP_RO)
+
+#define ML_PERM_RW   (_ML_PERM_COMMON |                       ARM_PTE_BLOCK_XN)
+#define ML_PERM_RO   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+#define ML_PERM_RX   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO                   )
+
+#define ML_PERM_TBL   ML_PERM_RW
+#define ML_PERM_MASK (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+
+#define ML_PERM_PRESENT(_flags)   (((_flags) & ARM_PTE_VALID) != 0)
+#define ML_PERM_WRITEABLE(_flags) (((_flags) & ARM_PTE_BLOCK_AP_RO) == 0)
+
+#define ML_PTE_2_PFN(_pte)    (((_pte) & ARM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
+
+#endif
+
 #define ML_PERMS_MATCH(x,p) (((x) & ML_PERM_MASK) == ((p) & ML_PERM_MASK))

 #define LOADER_HEADER_MAGIC 0x8675309E98675309
@@ -150,10 +182,10 @@ typedef enum {
   ML_CONTENT_SHARE,         /* Share data from external source. */
 } MonLoaderContentType;

-#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? 4 : \
-                               x == ML_CONTENT_PAGETABLE_L3 ? 3 : \
-                               x == ML_CONTENT_PAGETABLE_L2 ? 2 : \
-                               x == ML_CONTENT_PAGETABLE_L1 ? 1 : \
+#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? PT_LEVEL_4 : \
+                               x == ML_CONTENT_PAGETABLE_L3 ? PT_LEVEL_3 : \
+                               x == ML_CONTENT_PAGETABLE_L2 ? PT_LEVEL_2 : \
+                               x == ML_CONTENT_PAGETABLE_L1 ? PT_LEVEL_1 : \
                               0)

 /* Sources of content, for pages not zeroed, unmapped or otherwise special. */
@@ -183,11 +215,14 @@ typedef struct {
   } blobSrc;
   uint64               bspOnly;           /* Process only on BSP. */
   uint64               subIndex;          /* Region ID for ML_CONTENT_COPY. */
-   uint64               procVmmon;         /* Processed by the vmmon on hosted. */
 } MonLoaderEntry;


-/* Packed for easy consumption by bootstrap-offsets.pl. */
+/*
+ * Packed for easy consumption by bootstrap-offsets.pl. If the contents of the
+ * MonLoaderHeader struct are changed then $HEADER_SIZE must be updated
+ * accordingly in bootstrap-offsets.pl.
+ */
 #pragma pack(push, 1)
 typedef struct MonLoaderHeader {
   uint64         magic;
@@ -199,6 +234,8 @@ typedef struct MonLoaderHeader {
   /* ss:rip */
   uint16         stackSelector;
   VA64           stackEntrypoint;
+   LPN64          monStartLPN;
+   LPN64          monEndLPN;
   MonLoaderEntry entries[];
 } MonLoaderHeader;
 #pragma pack(pop)
@@ -217,16 +254,13 @@ MPN  MonLoaderCallout_GetPageRoot(struct MonLoaderEnvContext *, Vcpuid);
 Bool MonLoaderCallout_GetPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                             Vcpuid, PT_L1E *);
 Bool MonLoaderCallout_ImportPage(struct MonLoaderEnvContext *, MPN, Vcpuid);
-Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **);
+Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **, unsigned);
 Bool MonLoaderCallout_MapMPNInPTE(struct MonLoaderEnvContext *, MPN, unsigned,
                                  uint64, MPN, Vcpuid);
 MPN  MonLoaderCallout_GetSharedUserPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
 MPN  MonLoaderCallout_GetSharedHostPage(struct MonLoaderEnvContext *, uint64,
                                        unsigned, Vcpuid);
-Bool MonLoaderCallout_IsPrivileged(struct MonLoaderEnvContext *);
-Bool MonLoaderCallout_SetEntrypoint(struct MonLoaderEnvContext *, uint16, VA64,
-                                    uint16, VA64);
 MPN  MonLoaderCallout_GetBlobMpn(struct MonLoaderEnvContext *, uint64);

 typedef enum MonLoaderError {
@@ -237,7 +271,6 @@ typedef enum MonLoaderError {
   ML_ERROR_ARGS,
   ML_ERROR_CALLOUT_INIT,
   ML_ERROR_CALLOUT_COPY,
-   ML_ERROR_CALLOUT_ENTRYPOINT,
   ML_ERROR_CALLOUT_GETPTE,
   ML_ERROR_CALLOUT_MAPINPTE,
   ML_ERROR_CALLOUT_PAGEROOT_GET,
@@ -259,8 +292,12 @@ typedef enum MonLoaderError {
 } MonLoaderError;


-/* A subindex above shared area subindices for sharing of MonLoaderHeader. */
+/*
+ * Values above shared area subindices for sharing of MonLoaderHeader
+ * and htSchedStateMap.
+ */
 #define MONLOADER_HEADER_IDX 6
+#define MONLOADER_HT_MAP_IDX 7


 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
@@ -25,23 +25,7 @@
 #ifndef _MONLOADER_LOG_H_
 #define _MONLOADER_LOG_H_

-#ifdef VMX86_VMX
-
-#define LOGLEVEL_MODULE main
-#include "loglevel_vmcore.h"
-/*
- * Redefine LOG() to match the vmkernel calling convention.
- */
-#undef LOG_BYNAME
-#undef LOG
-
-#define LOG_BYNAME(_mod, _min, _fmt, ...) \
-        (DOLOG_BYNAME(_mod, _min) ? Log(_fmt "\n", ## __VA_ARGS__) : (void) 0)
-
-#define LOG(_min, _fmt, ...) \
-        LOG_BYNAME(LOGLEVEL_MODULE, _min, _fmt, ## __VA_ARGS__)
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #define LOGLEVEL_MODULE MonLoader
 #include "log.h"
@@ -59,7 +43,7 @@ do {                                                 \
   }                                                 \
 } while (0)

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2007-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2007-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -60,20 +60,21 @@
 *     lengths, and this causes the 'monitor-modular-size' script to
 *     report differences in '.rodata'.
 *
- *   o To avoid differences in '.rodata', each __FILE__ is put into
- *     its own section.  The monitor's linker (not ld) will use the
- *     name of the section to recover the name of the source file.
+ *   o Each __FILE__ is put into its own .assert_pathname_##__FILE
+ *     section.  The monitor's runtime linker (not ld) then uses the name
+ *     of this section to recover the name of the corresponding source file.
 *
 *   o At run time, prior to loading, when our linker is creating an
 *     executable image of the monitor and extensions, all the file
 *     names are extracted from these sections, the '${VMTREE}' prefix
 *     is removed, and the resulting table of shortened file names is
- *     added to '.rodata'.
+ *     added to a non-loadable section called '.assert_pathnames' in the
+ *     VMX linker (not the bootstrap linker which does not need them).
 *
 *     Further, during linkage, each relocation to the original
 *     section containing the path name is modified so that the low
- *     16-bits contain an offset from '__vmm_pathnames_start' rather
- *     than the base of the original containing section.
+ *     16-bits contain an offset from the start of the .assert_pathnames
+ *     section rather than the base of the original containing section.
 *
 *     Only three types of relocations to the assertion strings are
 *     supported (32-bit PC-relative and 32-bit/64-bit absolute) because that
@@ -111,42 +112,12 @@ typedef struct Assert_Info {
   Assert_MonSrcLoc loc;
 } Assert_Info;

-/*
- * The portion of the __attribute__ line after __FILE__ is there so that
- * the .assert_pathname_* sections are not marked as ALLOC, since we only
- * need them in the vmx and do not need them loaded.
- */
-#define __VMM__FILE__SECTION \
-      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
-#define __VMM__FILE__ ({                                                \
-         static __VMM__FILE__SECTION const char file[] = "";            \
-         file;                                                          \
-      })
-
 #define ASSERT_MONSRCFILEOFFSET(loc)    LOWORD(loc)
 #define ASSERT_MONSRCLINE(loc)          HIWORD(loc)

 #define ASSERT_NULL_MONSRCLOC     0             // there is never line 0

-#ifdef VMM // {
-#ifdef MONITOR_APP // {
-
-#define ASSERT_MONSRCLOC() ASSERT_NULL_MONSRCLOC
-
-#else // } {
-
-#define ASSERT_MONSRCLOC() ({                                           \
-   const uintptr_t offset = ((__LINE__ << 16) +                         \
-                             (uintptr_t)__VMM__FILE__);                 \
-   const Assert_MonSrcLoc loc = offset;                                 \
-   loc;                                                                 \
-})
-
-extern const char __vmm_pathnames_start;
-#define ASSERT_MONSRCFILE(loc) \
-   (&__vmm_pathnames_start + ASSERT_MONSRCFILEOFFSET(loc))
-
-
+#if defined(VMM) && !defined(MONITOR_APP) // {
 /*
 * Assertion information is collected in a non-loadable section
 * named .assert_info.  Each record in this section contains
@@ -176,6 +147,18 @@ extern const char __vmm_pathnames_start;
 */

 #ifndef VMM_BOOTSTRAP
+/*
+ * The portion of the __attribute__ line after __FILE__ is there so that
+ * the .assert_pathname_* sections are not marked as ALLOC, since we only
+ * need them in the vmx and do not need them loaded.
+ */
+#define __VMM__FILE__SECTION \
+      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
+#define __VMM__FILE__ ({                                                \
+         static __VMM__FILE__SECTION const char file[] = "";            \
+         file;                                                          \
+      })
+
 #define ASSERT_RECORDINFO(assembly, assertType, bugNr)                   \
   __asm__ __volatile__(".pushsection .assert_info;"                     \
                        ".quad 0f;"                                      \
@@ -215,8 +198,6 @@ extern uint64 bsAssertRIP;
   ({COMPILER_MEM_BARRIER();                                             \
     ASSERT_RECORDINFO("", AssertType_##name##Bug, bug);                 \
     __builtin_trap();})
+#endif // VMM && !MONITOR_APP}

-#endif // MONITOR_APP }
-#endif // VMM }
-
-#endif
+#endif // _MON_ASSERT_H_
@@ -52,7 +52,7 @@ extern "C" {
 typedef struct PageListEntry {
   CompressedBPN cbpn;
   Bool voided;
-   uint8 _pad[2];
+   uint8 _pad[1];
 } PageListEntry;
 #pragma pack(pop) 

@@ -55,12 +55,16 @@ typedef enum PCIPassthruVectorIndex {
 } PCIPassthruVectorIndex;

 typedef enum PCIPassthru_IntrType {
-   PCIPASSTHRU_INTR_NONE   = 0x00,
-   PCIPASSTHRU_INTR_IOAPIC = 0x01,
-   PCIPASSTHRU_INTR_MSI    = 0x02,
-   PCIPASSTHRU_INTR_MSIX   = 0x04,
+   PCIPASSTHRU_INTR_NONE    = 0x00,
+   PCIPASSTHRU_INTR_IOAPIC  = 0x01,
+   PCIPASSTHRU_INTR_MSI     = 0x02,
+   PCIPASSTHRU_INTR_MSIX    = 0x04,
+   PCIPASSTHRU_INTR_MSI_INT = 0x08,
 } PCIPassthru_IntrType;

+#define PCIPASSTHRU_INTR_MSI_COMBINED                   \
+   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)
+
 typedef struct FPTIntrProxyInfo {
   uint32            adapterIndex;
   uint32            vectorIndex;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,12 @@
 *********************************************************/

 /*
- * perfctr_generic.h --
+ * perfctr.h --
 *
 */

-#ifndef _PERFCTR_GENERIC_H_
-#define _PERFCTR_GENERIC_H_
+#ifndef _PERFCTR_H_
+#define _PERFCTR_H_

 #define INCLUDE_ALLOW_USERLEVEL

@@ -33,7 +33,9 @@

 #include "includeCheck.h"
 #include "vm_basic_types.h"
+#include "perfctr_arch.h"

+#define PERF_EVENT_NAME_LEN                      64
 /*
 * nmiNo      -- vmm peer is not attempting to do nmi profiling this run
 * nmiYes     -- vmm peer is doing nmi profiling and nmis are currently enabled
@@ -69,4 +71,51 @@ typedef struct {
   uint8  _pad[3];
 } CrossProfShared;

-#endif // ifndef _PERFCTR_GENERIC_H_
+/*
+ * PerfCtr_Config --
+ *      Describes configuration for a single hardware performance counter
+ *
+ *      Since this is only used to record general performance counters, we
+ *      made the assumption in nmiProfiler.c that the type is GENERAL and
+ *      index is counter number of type GENERAL.
+ *
+ *      **** x86 ****
+ *      On AMD K8 and GH:
+ *      index:        Which perf ctr, 0 to 3.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On AMD with PerfCtrExtCore support:
+ *      index:        Which perf ctr, 0 to 5.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On Intel Core architecture:
+ *      <to be documented>
+ *
+ *      **** ARM ****
+ *      escrVal:     Value placed in PMEVTYPER<n>_EL0 to configure event counter
+ *      index:       Index of the event counter.
+ */
+typedef struct PerfCtr_Config {
+   uint64 escrVal;
+   uint32 index;
+   uint32 periodMean;
+
+   /*
+    * Random number (whose absolute value is capped at
+    * periodJitterMask) is used to randomize sampling interval.
+    */
+   uint32  periodJitterMask;
+   uint32  seed;    // seed is used to compute next random number
+   uint16  config;
+   Bool    valid;
+   PERFCTR_CONFIG_ARCH_FIELDS
+} PerfCtr_Config;
+
+
+#endif // ifndef _PERFCTR_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,17 @@
 *********************************************************/

 /*
- * x86perfctr.h --
+ * perfctr_arch.h --
 *
+ *      Performance counters (x64 specific).
 */

-#ifndef _X86PERFCTR_H_
-#define _X86PERFCTR_H_
+#ifndef _PERFCTR_ARCH_H
+#define _PERFCTR_ARCH_H
+
+#ifndef _PERFCTR_H_
+#error "This file can only be included by perfctr.h"
+#endif

 #define INCLUDE_ALLOW_USERLEVEL

@@ -30,25 +35,20 @@
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
-
 #include "includeCheck.h"
+
 #include "vm_asm.h"
 #include "x86cpuid_asm.h"
-#include "perfctr_generic.h"

-#define PERFCTR_PENTIUM4_NUM_COUNTERS            18
-#define PERFCTR_PENTIUM4_NUM_COUNTERS_WITH_L3    26
-#define PERFCTR_AMD_NUM_COUNTERS                 4 
+#define PERFCTR_AMD_NUM_COUNTERS                 4
 #define PERFCTR_AMD_EXT_NUM_COUNTERS             6
 #define PERFCTR_P6_NUM_COUNTERS                  2
 #define PERFCTR_NEHALEM_NUM_GEN_COUNTERS         4
 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS       3
 #define PERFCTR_SANDYBRIDGE_NUM_GEN_COUNTERS     8 /* When HT is disabled */
-#define PERFCTR_CORE_NUM_ARCH_EVENTS             7
-#define PERFCTR_CORE_NUM_FIXED_COUNTERS          3
-#define PERFCTR_PENTIUM4_VAL_MASK                0xffffffffffLL
+#define PERFCTR_CORE_NUM_ARCH_EVENTS             8
+#define PERFCTR_CORE_NUM_FIXED_COUNTERS          4
 #define PERFCTR_AMD_VAL_MASK                     0xffffffffffffLL
-#define PERF_EVENT_NAME_LEN                      64
 /*
 * Even though the performance counters in P6 are 40 bits,
 * we can only write to the lower 32 bits.  Bit 31 is
@@ -59,7 +59,7 @@

 /*
 * Performance counter width is determined at runtime in CORE.
- * But the writables bits are fixed. we can only write to the 
+ * But the writables bits are fixed. we can only write to the
 * lower 32 bits.  Bit 31 is used to sign-extend the upper 8 bits.
 */
 #define PERFCTR_CORE_WRITE_MASK                  0xffffffff
@@ -124,7 +124,7 @@
 #define PERFCTR_AMD_CPU_CLK_UNHALTED                           0x76

 /* AMD Load/Store unit events */
-#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20 
+#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20
 #define PERFCTR_AMD_LS_BUFFER2_FULL                            0x23

 /*
@@ -198,8 +198,8 @@

 #define PERFCTR_AMD_MEM_CTRL_PAGE_TABLE_OVERFLOWS                   0xe1
 #define PERFCTR_AMD_CPU_IO_REQUESTS_TO_MEMORY_IO                    0xe9
-#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec 
- 
+#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec
+
 /* AMD HyperTransport Interface Events */

 #define PERFCTR_AMD_HT_L0_TX_BW                                     0xf6
@@ -219,9 +219,9 @@
 */

 /* P6 Data Cache Unit (DCU) */
-#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043  
+#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043
 #define PERFCTR_P6_DCU_LINES_IN                  0x00000045
-#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046 
+#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046
 #define PERFCTR_P6_DCU_MISS_OUTSTANDING          0x00000048

 /* P6 Instruction Fetch Unit (IFU) */
@@ -379,7 +379,7 @@
 #define PERFCTR_P6_APIC_INTR                     0x00100000
 #define PERFCTR_P6_ENABLE                        0x00400000
 #define PERFCTR_P6_INVERT_COUNTER_MASK           0x00800000
-#define PERFCTR_P6_COUNTER_MASK                  0xff000000        
+#define PERFCTR_P6_COUNTER_MASK                  0xff000000
 #define PERFCTR_P6_COUNTER_MASK_SHIFT            24
 #define PERFCTR_P6_SHIFT_BY_UNITMASK(e)          (e << 8)

@@ -436,6 +436,7 @@
 #define PERFCTR_CORE_KERNEL_MODE                    PERFCTR_CPU_KERNEL_MODE
 #define PERFCTR_CORE_APIC_INTR                      PERFCTR_CPU_APIC_INTR
 #define PERFCTR_CORE_ENABLE                         PERFCTR_CPU_ENABLE
+/* AnyThread Counting deprecated since PMU v5. */
 #define PERFCTR_CORE_ANYTHREAD                      0x00200000
 #define PERFCTR_CORE_IN_TX                          (CONST64U(1) << 32)
 #define PERFCTR_CORE_IN_TXCP                        (CONST64U(1) << 33)
@@ -463,6 +464,7 @@
 #define PERFCTR_CORE_INST_RETIRED                0xc0
 /* bus cycles */
 #define PERFCTR_CORE_UNHALTED_REF_CYCLES         (0x3c | (0x01 << 8))
+#define PERFCTR_CORE_TOPDOWN_SLOTS               (0xa4 | (0x01 << 8))

 /*
 * See Tables 30-2, 30-4 of the
@@ -558,132 +560,20 @@
 #define PERFCTR_NEHALEM_UNCORE_ENABLE            0x400000
 #define PERFCTR_NEHALEM_UNCORE_L3_LINES_IN       (0x0a | (0x0f << 8))

-
-/*
- * ----------------------------------------------------------------------
- *
- * Pentium 4
- *
- * ----------------------------------------------------------------------
- */
-
-/*
- * Pentium 4 Counter Configuration Control Register flags and fields
- */
-#define PERFCTR_PENTIUM4_CCCR_ENABLE                   0x00001000
-#define PERFCTR_PENTIUM4_CCCR_SET_ESCR(c, e)           ((c) |= (((e) & 0x7) << 13))
-#define PERFCTR_PENTIUM4_CCCR_REQRSVD                  0x00030000 // always set these bits
-#define PERFCTR_PENTIUM4_CCCR_COMPARE                  0x00040000
-#define PERFCTR_PENTIUM4_CCCR_COMPLEMENT               0x00080000
-#define PERFCTR_PENTIUM4_CCCR_SET_THRESHOLD(c, e)      ((c) |= (((e) & 0xf) << 20))
-#define PERFCTR_PENTIUM4_CCCR_EDGE                     0x01000000
-#define PERFCTR_PENTIUM4_CCCR_FORCE_OVF                0x02000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T0               0x04000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T1               0x08000000
-#define PERFCTR_PENTIUM4_CCCR_CASCADE                  0x40000000
-#define PERFCTR_PENTIUM4_CCCR_OVF                      0x80000000
-
-#define PERFCTR_PENTIUM4_COUNTER_BASEADDR              0x00000300
-#define PERFCTR_PENTIUM4_CCCR_BASE_ADDR                0x00000360
-
-#define PERFCTR_PENTIUM4_CCCR_THRESHOLD(e)             ((e & 0xf) << 20)
-#define PERFCTR_PENTIUM4_OPT_EDGE_DETECT (PERFCTR_PENTIUM4_CCCR_COMPARE | PERFCTR_PENTIUM4_CCCR_EDGE)
-
-/*
- * Pentium 4 Event Selection Control Register flags
- */
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T0             0x00000004
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T0           0x00000008
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T1             0x00000001
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T1           0x00000002
-#define PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK(e)          (e << 9)
-#define PENTIUM4_EVTSEL(e)                             (e << 25)
-
-/*
- * Event definitions for Pentium4 Family Processors. 
- * There are many more of these possible.
- * See Appendix A.1 in Volume 3 of the IA32 manual.
- */
-
-#define PERFCTR_PENTIUM4_EVT_INSTR_RETIRED              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_INSTR_COMPLETED            PENTIUM4_EVTSEL(0x07)
-#define PERFCTR_PENTIUM4_EVT_BSQ_CACHE_REFERENCE        PENTIUM4_EVTSEL(0x0c)
-#define PERFCTR_PENTIUM4_EVT_ITLB_REFERENCE             PENTIUM4_EVTSEL(0x18)
-#define PERFCTR_PENTIUM4_EVT_PAGE_WALK_TYPE             PENTIUM4_EVTSEL(0x01)   
-#define PERFCTR_PENTIUM4_EVT_BPU_FETCH_REQUEST          PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_BRANCH_RETIRED             PENTIUM4_EVTSEL(0x06)  
-#define PERFCTR_PENTIUM4_EVT_MISPRED_BRANCH_RETIRED     PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_MACHINE_CLEAR              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_MEMORY_CANCEL              PENTIUM4_EVTSEL(0x02) 
-#define PERFCTR_PENTIUM4_GLOBAL_POWER_EVENTS            PENTIUM4_EVTSEL(0x13)
-
-/*
- * Pentium4 doesn't have an event for clock cycles, but Intel
- * outlines a method in the IA32 manual, vol 3, sec 14.9.9,
- * for measuring cycles which involves this event strategy.
- */
-
-#define PERFCTR_PENTIUM4_EVT_CLK_CYCLES \
-   PENTIUM4_EVTSEL(0x2) | \
-   PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK (0x01 | 0x02 | 0x04 | 0x08)
-
-
-/*
- * PerfCtr_Config --
- *      Describes configuration for a single hardware performance counter
- *
- *      Since this is only used to record general performance counters, we
- *      made the assumption in nmiProfiler.c  that the type is GENERAL and
- *      index is counter number of type GENERAL.
- *      
- *      
- *      On AMD K8 and GH:
- *      index:        Which perf ctr, 0 to 3.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- *
- *      On AMD with PerfCtrExtCore support:
- *      index:        Which perf ctr, 0 to 5.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- * 
- *      No field should be greater than 32-bit as it is shared with monitor.
- *
- *      On Intel Core architecture:
- *      <to be documented>
- */
-typedef struct PerfCtr_Config {
-   uint64 escrVal;
-   uint32 index;
-   uint32 addr;
-   uint32 escrAddr;
-   uint32 resetHi;
-   uint32 periodMean;
-
-   /*
-    * Random number (whose absolute value is capped at
-    * periodJitterMask) is used to randomize sampling interval.
-    */
-   uint32  periodJitterMask;
-   uint32  seed;    // seed is used to compute next random number
-   uint16  config;
-   Bool    valid;
-   Bool    pebsEnabled;
-} PerfCtr_Config;
-
+#define PERFCTR_CONFIG_ARCH_FIELDS \
+   uint32 addr;      \
+   uint32 escrAddr;  \
+   uint32 resetHi;   \
+   Bool pebsEnabled; \

 /*
 * Program/reprogram event reg(s) associated w/perfctrs & start or stop perfctrs
 */
 static INLINE void
-PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
-                   uint32 escrVal)       // IN: event register value
+PerfCtr_WriteEvtSel(uint32 addr,       // IN: Event register to write
+                    uint32 escrVal)    // IN: event register value
 {
-   __SET_MSR(ctr->escrAddr, escrVal);
+   X86MSR_SetMSR(addr, escrVal);
 }


@@ -691,11 +581,10 @@ PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
 * Set/reset performance counters to engender desired period before overflow
 */
 static INLINE void
-PerfCtrWriteCounter(PerfCtr_Config *ctr,   // IN: counter to write
-                    uint32 valueLo,        // IN: low 32 bits of value to write
-                    uint32 valueHi)        // IN: high 32 bits of value to write
+PerfCtr_WriteCounter(uint32 addr,   // IN: counter to write
+                     uint64 value)  // IN: value to write
 {
-   WRMSR(ctr->addr, valueLo, valueHi);
+   X86MSR_SetMSR(addr, value);
 }


@@ -814,8 +703,9 @@ PerfCtr_PEBSAvailable(void)
   CPUIDRegs regs;
   __GET_CPUID(0, &regs);
   if (CPUID_IsVendorIntel(&regs) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_EMON_AVAILABLE) != 0) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_PEBS_UNAVAILABLE) == 0)) {
+       (X86MSR_GetMSR(MSR_MISC_ENABLE) &
+        (MSR_MISC_ENABLE_EMON_AVAILABLE | MSR_MISC_ENABLE_PEBS_UNAVAILABLE))
+       == MSR_MISC_ENABLE_EMON_AVAILABLE) {
      CPUIDRegs hvendor;

      /*
@@ -827,7 +717,7 @@ PerfCtr_PEBSAvailable(void)
             !CPUID_IsRawVendor(&hvendor,
                                CPUID_HYPERV_HYPERVISOR_VENDOR_STRING);
   }
-   return FALSE; 
+   return FALSE;
 }

 /*
@@ -858,162 +748,4 @@ PerfCtr_PTAvailable(void)
   return FALSE;
 }

-/* The following are taken from the Intel Architecture Manual,
- * Book 3, Table 14-4. 
- */ 
-
-/* --------- BEGIN INTEL DEFINES ------------------------ */
-
-#define PENTIUM4_MSR_BPU_COUNTER0_IDX           0
-#define PENTIUM4_MSR_BPU_COUNTER0_ADDR          0x300
-#define PENTIUM4_MSR_BPU_CCCR0                  0x360
-#define PENTIUM4_MSR_BPU_COUNTER1_IDX           1
-#define PENTIUM4_MSR_BPU_COUNTER1_ADDR          0x301
-#define PENTIUM4_MSR_BPU_CCCR1                  0x361
-#define PENTIUM4_MSR_BPU_COUNTER2_IDX           2
-#define PENTIUM4_MSR_BPU_COUNTER2_ADDR          0x302
-#define PENTIUM4_MSR_BPU_CCCR2                  0x362
-#define PENTIUM4_MSR_BPU_COUNTER3_IDX           3
-#define PENTIUM4_MSR_BPU_COUNTER3_ADDR          0x303
-#define PENTIUM4_MSR_BPU_CCCR3                  0x363
-#define PENTIUM4_MSR_MS_COUNTER0_IDX            4
-#define PENTIUM4_MSR_MS_COUNTER0_ADDR           0x304
-#define PENTIUM4_MSR_MS_CCCR0                   0x364
-#define PENTIUM4_MSR_MS_COUNTER1_IDX            5
-#define PENTIUM4_MSR_MS_COUNTER1_ADDR           0x305
-#define PENTIUM4_MSR_MS_CCCR1                   0x365
-#define PENTIUM4_MSR_MS_COUNTER2_IDX            6
-#define PENTIUM4_MSR_MS_COUNTER2_ADDR           0x306
-#define PENTIUM4_MSR_MS_CCCR2                   0x366
-#define PENTIUM4_MSR_MS_COUNTER3_IDX            7
-#define PENTIUM4_MSR_MS_COUNTER3_ADDR           0x307
-#define PENTIUM4_MSR_MS_CCCR3                   0x367
-#define PENTIUM4_MSR_FLAME_COUNTER0_IDX         8
-#define PENTIUM4_MSR_FLAME_COUNTER0_ADDR        0x308
-#define PENTIUM4_MSR_FLAME_CCCR0                0x368
-#define PENTIUM4_MSR_FLAME_COUNTER1_IDX         9
-#define PENTIUM4_MSR_FLAME_COUNTER1_ADDR        0x309
-#define PENTIUM4_MSR_FLAME_CCCR1                0x369
-#define PENTIUM4_MSR_FLAME_COUNTER2_IDX         10
-#define PENTIUM4_MSR_FLAME_COUNTER2_ADDR        0x30A
-#define PENTIUM4_MSR_FLAME_CCCR2                0x36A
-#define PENTIUM4_MSR_FLAME_COUNTER3_IDX         11
-#define PENTIUM4_MSR_FLAME_COUNTER3_ADDR        0x30B
-#define PENTIUM4_MSR_FLAME_CCCR3                0x36B
-#define PENTIUM4_MSR_IQ_COUNTER0_IDX            12
-#define PENTIUM4_MSR_IQ_COUNTER0_ADDR           0x30C
-#define PENTIUM4_MSR_IQ_CCCR0                   0x36C
-#define PENTIUM4_MSR_IQ_COUNTER1_IDX            13
-#define PENTIUM4_MSR_IQ_COUNTER1_ADDR           0x30D
-#define PENTIUM4_MSR_IQ_CCCR1                   0x36D
-#define PENTIUM4_MSR_IQ_COUNTER2_IDX            14
-#define PENTIUM4_MSR_IQ_COUNTER2_ADDR           0x30E
-#define PENTIUM4_MSR_IQ_CCCR2                   0x36E
-#define PENTIUM4_MSR_IQ_COUNTER3_IDX            15
-#define PENTIUM4_MSR_IQ_COUNTER3_ADDR           0x30F
-#define PENTIUM4_MSR_IQ_CCCR3                   0x36F
-#define PENTIUM4_MSR_IQ_COUNTER4_IDX            16
-#define PENTIUM4_MSR_IQ_COUNTER4_ADDR           0x310
-#define PENTIUM4_MSR_IQ_CCCR4                   0x370
-#define PENTIUM4_MSR_IQ_COUNTER5_IDX            17
-#define PENTIUM4_MSR_IQ_COUNTER5_ADDR           0x311
-#define PENTIUM4_MSR_IQ_CCCR5                   0x371
-
-#define PENTIUM4_MSR_ALF_ESCR0_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR0_ADDR             0x3CA
-#define PENTIUM4_MSR_ALF_ESCR1_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR1_ADDR             0x3CB
-#define PENTIUM4_MSR_BPU_ESCR0_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR0_ADDR             0x3B2
-#define PENTIUM4_MSR_BPU_ESCR1_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR1_ADDR             0x3B3
-#define PENTIUM4_MSR_BSU_ESCR0_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR0_ADDR             0x3A0
-#define PENTIUM4_MSR_BSU_ESCR1_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR1_ADDR             0x3A1
-#define PENTIUM4_MSR_CRU_ESCR0_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR0_ADDR             0x3B8
-#define PENTIUM4_MSR_CRU_ESCR1_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR1_ADDR             0x3B9
-#define PENTIUM4_MSR_CRU_ESCR2_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR2_ADDR             0x3CC
-#define PENTIUM4_MSR_CRU_ESCR3_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR3_ADDR             0x3CD
-#define PENTIUM4_MSR_CRU_ESCR4_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR4_ADDR             0x3E0
-#define PENTIUM4_MSR_CRU_ESCR5_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR5_ADDR             0x3E1
-#define PENTIUM4_MSR_DAC_ESCR0_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR0_ADDR             0x3A8
-#define PENTIUM4_MSR_DAC_ESCR1_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR1_ADDR             0x3A9
-#define PENTIUM4_MSR_FIRM_ESCR0_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR0_ADDR            0x3A4
-#define PENTIUM4_MSR_FIRM_ESCR1_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR1_ADDR            0x3A5
-#define PENTIUM4_MSR_FLAME_ESCR0_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR0_ADDR           0x3A6
-#define PENTIUM4_MSR_FLAME_ESCR1_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR1_ADDR           0x3A7
-#define PENTIUM4_MSR_FSB_ESCR0_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR0_ADDR             0x3A2
-#define PENTIUM4_MSR_FSB_ESCR1_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR1_ADDR             0x3A3
-#define PENTIUM4_MSR_IQ_ESCR0_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR0_ADDR              0x3BA
-#define PENTIUM4_MSR_IQ_ESCR1_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR1_ADDR              0x3BB
-#define PENTIUM4_MSR_IS_ESCR0_IDX               1
-#define PENTIUM4_MSR_IS_ESCR0_ADDR              0x3B4
-#define PENTIUM4_MSR_IS_ESCR1_IDX               1
-#define PENTIUM4_MSR_IS_ESCR1_ADDR              0x3B5
-#define PENTIUM4_MSR_ITLB_ESCR0_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR0_ADDR            0x3B6
-#define PENTIUM4_MSR_ITLB_ESCR1_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR1_ADDR            0x3B7
-#define PENTIUM4_MSR_IX_ESCR0_IDX               5
-#define PENTIUM4_MSR_IX_ESCR0_ADDR              0x3C8
-#define PENTIUM4_MSR_IX_ESCR1_IDX               5
-#define PENTIUM4_MSR_IX_ESCR1_ADDR              0x3C9
-#define PENTIUM4_MSR_MOB_ESCR0_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR0_ADDR             0x3AA
-#define PENTIUM4_MSR_MOB_ESCR1_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR1_ADDR             0x3AB
-#define PENTIUM4_MSR_MS_ESCR0_IDX               0
-#define PENTIUM4_MSR_MS_ESCR0_ADDR              0x3C0
-#define PENTIUM4_MSR_MS_ESCR1_IDX               0
-#define PENTIUM4_MSR_MS_ESCR1_ADDR              0x3C1
-#define PENTIUM4_MSR_PMH_ESCR0_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR0_ADDR             0x3AC
-#define PENTIUM4_MSR_PMH_ESCR1_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR1_ADDR             0x3AD
-#define PENTIUM4_MSR_RAT_ESCR0_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR0_ADDR             0x3BC
-#define PENTIUM4_MSR_RAT_ESCR1_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR1_ADDR             0x3BD
-#define PENTIUM4_MSR_SAAT_ESCR0_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR0_ADDR            0x3AE
-#define PENTIUM4_MSR_SAAT_ESCR1_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR1_ADDR            0x3AF
-#define PENTIUM4_MSR_SSU_ESCR0_IDX              3
-#define PENTIUM4_MSR_SSU_ESCR0_ADDR             0x3BE
-#define PENTIUM4_MSR_TBPU_ESCR0_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR0_ADDR            0x3C2
-#define PENTIUM4_MSR_TBPU_ESCR1_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR1_ADDR            0x3C3
-#define PENTIUM4_MSR_TC_ESCR0_IDX               1
-#define PENTIUM4_MSR_TC_ESCR0_ADDR              0x3C4
-#define PENTIUM4_MSR_TC_ESCR1_IDX               1
-#define PENTIUM4_MSR_TC_ESCR1_ADDR              0x3C5
-#define PENTIUM4_MSR_U2L_ESCR0_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR0_ADDR             0x3B0
-#define PENTIUM4_MSR_U2L_ESCR1_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR1_ADDR             0x3B1
-
-#define PENTIUM4_MIN_ESCR_ADDR                  0x3A0
-#define PENTIUM4_MAX_ESCR_ADDR                  0x3E1
-#define PENTIUM4_NUM_ESCR_ADDRS                 (PENTIUM4_MAX_ESCR_ADDR - PENTIUM4_MIN_ESCR_ADDR + 1)
-
-/* -------- END INTEL DEFINES ------------------------- */
-
-#endif // ifndef _X86PERFCTR_H_ 
+#endif // _PERFCTR_ARCH_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -173,35 +173,6 @@ int64 PTSC_CyclesToUS(VmRelativeTS ts);
 */
 #include "user_layout.h"

-#ifdef VM_ARM_64
-static INLINE VmAbsoluteTS
-PTSC_Get(void)
-{
-   extern __thread User_ThreadData vmkUserTdata;
-
-   register RateConv_Params params;
-   register uint64 pseudoTSC;
-
-   /*
-    * On ARM64 the generic timer guarantees that the counters are synchronous
-    * and independent of CPU frequency.
-    */
-   params = vmkUserTdata.pseudoTSCConv;
-   pseudoTSC = RDTSC();
-
-   /*
-    *  assert that conversion rate is `identity'.
-    * we'll have to revisit that when/if this assumption changes
-    * on some platform
-    */
-   ASSERT((params.mult == 1 && params.shift == 0) ||
-          (params.mult == 0x80000000 && params.shift == 31));
-
-   pseudoTSC += params.add;
-
-   return pseudoTSC;
-}
-#else
 static INLINE VmAbsoluteTS
 PTSC_Get(void)
 {
@@ -215,7 +186,7 @@ PTSC_Get(void)
   ASSERT((int64)ptsc >= 0);
   return ptsc;
 }
-#endif
+
 #else

 /*
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,7 +97,7 @@ void RateConv_LogRatio(const char *prefix,
 static INLINE uint64
 RateConv_Unsigned(const RateConv_Params *conv, uint64 x)
 {
-   return Mul64x3264(x, conv->mult, conv->shift) + conv->add;
+   return Mul64x3264(x, conv->mult, conv->shift) + (uint64)conv->add;
 }


@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,28 +16,32 @@
 *
 *********************************************************/

-#ifndef _INITBLOCK_H
-#define _INITBLOCK_H
+/*
+ * sharedAreaType.h --
+ *
+ *      This file contains shared area type definitions.
+ */

+#ifndef _SHAREDAREATYPE_H_
+#define _SHAREDAREATYPE_H_

-#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+
 #include "includeCheck.h"

+typedef enum {
+   SHARED_AREA_PER_VM_VMX = 0,
+   SHARED_AREA_INTER_VCPU_VMX,
+   SHARED_AREA_PER_VCPU_VMX,
+   SHARED_AREA_PER_VM,
+   SHARED_AREA_INTER_VCPU,
+   SHARED_AREA_PER_VCPU,
+   NUM_SHARED_AREAS
+} SharedAreaType;

-#include "basic_initblock.h"
-
-
-#define MAX_LOGFILE_NAME_LENGTH 256
-#define MAX_MONITOR_REGIONS     8
-#define INIT_BLOCK_MAGIC     (0x1789+14)
-
-
-/*
- * Option flags 
- */
-/* None yet */
-
-
-#endif
+#endif // _SHAREDAREATYPE_H_
@@ -0,0 +1,62 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.h --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifndef SHARED_AREA_VMMON_H
+#define SHARED_AREA_VMMON_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#include "includeCheck.h"
+
+#include "iocontrols.h"
+
+struct VMDriver;
+
+/* A collection of pages backing a shared area region. */
+typedef struct SharedAreaVmmonRegion {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} SharedAreaVmmonRegion;
+
+/* The VMMon Driver shared area. */
+typedef struct SharedAreaVmmon {
+   SharedAreaVmmonRegion regions[NUM_SHARED_AREAS];
+} SharedAreaVmmon;
+
+/* Request for a backing MPN for a shared area region at a given VCPU/offset. */
+typedef struct SharedAreaVmmonRequest {
+   SharedAreaType type;
+   Vcpuid vcpu;
+   PageCnt offset;
+} SharedAreaVmmonRequest;
+
+SharedAreaVmmon *SharedAreaVmmon_Init(struct VMDriver *vm);
+void SharedAreaVmmon_Cleanup(SharedAreaVmmon *area);
+Bool SharedAreaVmmon_RegisterRegion(struct VMDriver *driver,
+                                    VMSharedAreaRegistrationBlock *block);
+Bool SharedAreaVmmon_ValidateRegionArgs(struct VMDriver *driver,
+                                        VMSharedAreaRegistrationBlock *block);
+MPN SharedAreaVmmon_GetRegionMPN(struct VMDriver *vm,
+                                 SharedAreaVmmonRequest *request);
+#endif /* SHARED_AREA_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,45 +17,32 @@
 *********************************************************/

 /*
- * basic_initblock.h --
+ * statVarsVmmon.h --
 *
- *    VM initialization block.
+ *     VMMon stat vars management.
 */

-#ifndef _BASIC_INITBLOCK_H_
-#define _BASIC_INITBLOCK_H_
+#ifndef STAT_VARS_VMMON_H
+#define STAT_VARS_VMMON_H

-
-#define INCLUDE_ALLOW_USERLEVEL
-
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_DISTRIBUTE
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

+#include "iocontrols.h"

-#include "vcpuid.h"
+struct VMDriver;

+/* The VMMon Driver stat vars area. */
+typedef struct StatVarsVmmon {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} StatVarsVmmon;

-#define MAX_INITBLOCK_CPUS     MAX_VCPUS
-
-
-typedef
-#include "vmware_pack_begin.h"
-struct InitBlock {
-   uint32 magicNumber;     /* Magic number (INIT_BLOCK_MAGIC) */
-   uint32 vmInitFailurePeriod;
-   VA64   crosspage[MAX_INITBLOCK_CPUS];
-   LPN64  monStartLPN;
-   LPN64  monEndLPN;
-   MPN    crossGDTMPN;
-   uint16 numPTPPages;
-}
-#include "vmware_pack_end.h"
-InitBlock;
-
-
-#endif // _BASIC_INITBLOCK_H_
+StatVarsVmmon *StatVarsVmmon_Init(struct VMDriver *vm);
+void StatVarsVmmon_Cleanup(StatVarsVmmon *statVars);
+Bool StatVarsVmmon_RegisterVCPU(struct VMDriver *driver,
+                                VMStatVarsRegistrationBlock *block);
+MPN StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpuid,
+                               PageCnt offset);
+#endif /* STAT_VARS_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 #ifndef USERCALLDEFS_H
 #define USERCALLDEFS_H

-#ifdef linux
+#ifdef __linux__
 #include <linux/errno.h>
 #define USERCALL_RESTART (-ERESTARTNOINTR)
 #else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,6 @@
 extern "C" {
 #endif

-
 typedef uint32 Vcpuid;                 // VCPU number

 #define VCPUID_INVALID  (~0U)
@@ -54,17 +53,11 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define MAX_CORES_PER_SOCKET   64
 #define MAX_VCPU_SOCKETS      128

-#ifdef VMM
-#include "vcpuset.h"
-
-/* In VMM, CurVcpuid() is available everywhere. */
-extern const Vcpuid      curVcpuid;
-extern const VCPUSet     curVcpuidSet;
-#define CurVcpuid()         (curVcpuid)
-#define CurVcpuidSet()      (&curVcpuidSet)
-#define IS_BOOT_VCPU()      IS_BOOT_VCPUID(CurVcpuid())
-
-#endif  /* VMM */
+/*
+ * There are several properties of the VM which change at the 128 VCPU
+ * boundary.  EFI firmware, x2APIC, and IOMMU are required among others.
+ */
+#define MAX_SMALL_VM_VCPUS 128

 #if defined __cplusplus
 } // extern "C"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -112,7 +112,7 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET(_setIndex)                                     \
   do {                                                                       \
-      int _setIndex;                                                          \
+      unsigned _setIndex;                                                     \
      for (_setIndex = 0; _setIndex < VCS_SUBSET_COUNT; _setIndex++) {

 #define ROF_EACH_SUBSET_IN_SET()                                              \
@@ -122,8 +122,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET_COUNTDOWN(_setIndex)                           \
   do {                                                                       \
-      int _setIndex;                                                          \
-      for (_setIndex = VCS_SUBSET_COUNT - 1; _setIndex >= 0; _setIndex--) {
+      unsigned _setIndex = VCS_SUBSET_COUNT;                                  \
+      while (_setIndex-- > 0) {

 #define ROF_EACH_SUBSET_IN_SET_COUNTDOWN()                                    \
      }                                                                       \
@@ -132,8 +132,8 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex, _numVcpus)                \
   do {                                                                       \
-      int _setIndex;                                                          \
-      int _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);                 \
+      unsigned _setIndex;                                                     \
+      unsigned _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);            \
      for (_setIndex = 0; _setIndex <= _maxSubsets; _setIndex++) {

 #define ROF_EACH_POPULATED_SUBSET_IN_SET()                                    \
@@ -307,7 +307,7 @@ VCPUSet_FindFirst(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return lssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return lssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET();
   return VCPUID_INVALID;
@@ -319,7 +319,7 @@ VCPUSet_FindLast(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET_COUNTDOWN(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return mssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return mssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET_COUNTDOWN();
   return VCPUID_INVALID;
@@ -356,8 +356,7 @@ VCPUSet_FindFirstInSubset(const VCPUSet *vcs, uint64 *subset,
   ASSERT(*subsetIdx < maxSubsets && maxSubsets <= VCS_SUBSET_COUNT);
   do {
      if (*subset != 0) {
-         int bit;
-         bit = lssb64_0(*subset);
+         unsigned bit = (unsigned)lssb64_0(*subset);
         *subset &= ~(CONST64U(1) << bit);
         return bit + (*subsetIdx << VCS_SUBSET_SHIFT);
      }
@@ -672,7 +671,7 @@ static INLINE Vcpuid
 VCPUSet_FindSingleton(const VCPUSet *vcs)
 {
   uint64 foundSub = 0;
-   uint32 foundIdx;
+   uint32 foundIdx = 0;
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 sub = vcs->subset[idx];
      if (sub != 0) {
@@ -683,8 +682,11 @@ VCPUSet_FindSingleton(const VCPUSet *vcs)
         foundIdx = idx;
      }
   } ROF_EACH_SUBSET_IN_SET();
-   return foundSub != 0 ? lssb64_0(foundSub) + (foundIdx << VCS_SUBSET_SHIFT) :
-                          VCPUID_INVALID;
+   if (foundSub != 0) {
+      return lssb64(foundSub) - 1 + (foundIdx << VCS_SUBSET_SHIFT);
+   } else {
+      return VCPUID_INVALID;
+   }
 }


@@ -863,27 +865,27 @@ VCPUSet_Intersection(VCPUSet *dest, const VCPUSet *src)

 #ifdef VCS_SNPRINTF
 static INLINE char *
-VCPUSet_LogFormat(char *buf, const int size, const VCPUSet *vcs)
+VCPUSet_LogFormat(char *buf, size_t size, const VCPUSet *vcs)
 {
-   int offset = 0;
-   Vcpuid highest = VCPUSet_FindLast(vcs);
-   int idx = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+   unsigned offset  = 0;
+   Vcpuid   highest = VCPUSet_FindLast(vcs);
+   unsigned idx     = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+
   ASSERT(size >= VCS_BUF_SIZE);
 #define VCS_LOGF(...)                                                   \
   {                                                                    \
      int ret = VCS_SNPRINTF(buf + offset, size - offset, __VA_ARGS__); \
-      ASSERT(0 <= ret && ret < size - offset);                          \
-      offset += ret;                                                    \
+      ASSERT(0 <= ret && size >= offset && ret < (int)(size - offset)); \
+      offset += (unsigned)ret;                                          \
   }
   /* Print the leading value with no zero-extension. */
-   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx--]);
+   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx]);

-   while (idx >= 0) {
+   while (idx-- > 0) {
      if ((idx + 1) % (VCS_SUBSET_WIDTH / 8) == 0) {
         VCS_LOGF(".");
      }
      VCS_LOGF("%02x", ((unsigned char *)vcs)[idx]);
-      idx--;
   }
   return buf;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,52 +41,47 @@

 #include "vm_basic_asm.h"
 #include "x86msr.h"
-
-#ifdef VM_X86_64
-#include "vm_asm_x86_64.h"
-#else
 #include "vm_asm_x86.h"
-#endif


 static INLINE void
 SET_FS64(uint64 fs64)
 {
-   __SET_MSR(MSR_FSBASE, fs64);
+   X86MSR_SetMSR(MSR_FSBASE, fs64);
 }


 static INLINE void
 SET_GS64(uint64 gs64)
 {
-   __SET_MSR(MSR_GSBASE, gs64);
+   X86MSR_SetMSR(MSR_GSBASE, gs64);
 }

 static INLINE void
 SET_KernelGS64(uint64 kgs64)
 {
-   __SET_MSR(MSR_KERNELGSBASE, kgs64);
+   X86MSR_SetMSR(MSR_KERNELGSBASE, kgs64);
 }


 static INLINE uint64
 GET_FS64(void)
 {
-   return __GET_MSR(MSR_FSBASE);
+   return X86MSR_GetMSR(MSR_FSBASE);
 }


 static INLINE uint64
 GET_GS64(void)
 {
-   return __GET_MSR(MSR_GSBASE);
+   return X86MSR_GetMSR(MSR_GSBASE);
 }


 static INLINE uint64
 GET_KernelGS64(void)
 {
-   return __GET_MSR(MSR_KERNELGSBASE);
+   return X86MSR_GetMSR(MSR_KERNELGSBASE);
 }

 #endif // VM_ARM_64
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * vm_asm_x86.h
 *
- *	IA32 asm macros
+ *      IA32 asm macros
 */

 #ifndef _VM_ASM_X86_H_
@@ -49,10 +49,6 @@

 #ifdef __GNUC__

-#if defined __APPLE__
-/* PR 352418: GCC produces error if the non-Apple version is used */
-#define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
-#else
 /* ASSERT_ON_COMPILE_SELECTOR_SIZE:
 *
 *  - Selector must be 16-bits.
@@ -66,26 +62,16 @@
 *                    ((__builtin_constant_p(expr) ? ((expr) >> 16) == 0) \
 *                                                 : sizeof(expr) <= 2)
 */
-#if (__GNUC__ >= 4) && (__GNUC_MINOR__ >= 1) && !defined(USE_UBSAN)
+#ifndef USE_UBSAN
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)                                \
   ASSERT_ON_COMPILE(sizeof(Selector) == 2 &&                                \
                     ((__builtin_constant_p(expr) && ((expr) >> 16) == 0) || \
                      sizeof(expr) <= 2))
 #else
-/* gcc 3.3.3 is not able to produce a constant expression (PR 356383) */
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
 #endif
-#endif


-/*
- * [GS]ET_[GI]DT() are defined as macros wrapping a function
- * so we can pass the argument implicitly by reference (requires
- * a macro) and get type checking too (requires a function).
- */
-
-#define SET_GDT(var) _Set_GDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
@@ -97,8 +83,6 @@ _Set_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define SET_IDT(var) _Set_IDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_IDT(_GETSET_DTR_TYPE *dtr)
@@ -110,8 +94,6 @@ _Set_IDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_GDT(var) _Get_GDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because there's a hidden input (the [IG]DTR) that can change
@@ -126,8 +108,6 @@ _Get_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_IDT(var) _Get_IDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because the [IG]DT can change without the compiler knowing it
@@ -218,22 +198,6 @@ _BUILD_GET_R(_GET_CR3, cr3)
 _BUILD_GET_R(_GET_CR4, cr4)
 _BUILD_GET_R(_GET_CR8, cr8)

-#if defined __APPLE__
-/* Mac OS gcc 4 uses DBx instead of DRx register names. */
-_BUILD_SET_R(_SET_DR0, db0)
-_BUILD_SET_R(_SET_DR1, db1)
-_BUILD_SET_R(_SET_DR2, db2)
-_BUILD_SET_R(_SET_DR3, db3)
-_BUILD_SET_R(_SET_DR6, db6)
-_BUILD_SET_R(_SET_DR7, db7)
-
-_BUILD_GET_R(_GET_DR0, db0)
-_BUILD_GET_R(_GET_DR1, db1)
-_BUILD_GET_R(_GET_DR2, db2)
-_BUILD_GET_R(_GET_DR3, db3)
-_BUILD_GET_R(_GET_DR6, db6)
-_BUILD_GET_R(_GET_DR7, db7)
-#else
 _BUILD_SET_R(_SET_DR0, dr0)
 _BUILD_SET_R(_SET_DR1, dr1)
 _BUILD_SET_R(_SET_DR2, dr2)
@@ -247,47 +211,8 @@ _BUILD_GET_R(_GET_DR2, dr2)
 _BUILD_GET_R(_GET_DR3, dr3)
 _BUILD_GET_R(_GET_DR6, dr6)
 _BUILD_GET_R(_GET_DR7, dr7)
-#endif

-#define SET_CR_DR(regType, regNum, expr)                     \
-   do {                                                      \
-      /* Ensure no implicit truncation of 'expr' */          \
-      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
-      _SET_##regType##regNum(expr);                          \
-   } while (0)
-
-#define GET_CR_DR(regType, regNum, var) \
-   do {                                 \
-      _GET_##regType##regNum(&(var));   \
-   } while (0)
-
-#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
 #define SET_CR2(expr) SET_CR_DR(CR, 2, expr)
-#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
-#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
-#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
-
-/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
-#undef GET_CR0
-#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
-#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
-#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
-#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
-#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
-
-#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
-#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
-#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
-#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
-#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
-#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
-
-#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
-#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
-#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
-#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
-#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
-#define GET_DR7(var)  GET_CR_DR(DR, 7, var)

 #define SET_SEGREG(reg, expr)                                          \
   do {                                                                \
@@ -426,440 +351,347 @@ CLTS(void)
 /* Beginning of the section whose correctness has NOT been checked */
 #define FNCLEX()         __asm__("fnclex" ::);

-/* TLB_INVALIDATE_xxx are not checked yet */
+/* TLB_INVALIDATE_PAGE is not checked yet */
 #define TLB_INVALIDATE_PAGE(_addr) do { \
     __asm__ __volatile__("invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
 } while (0)

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) do { \
-     __asm__ __volatile__("fs; invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
-} while (0)
-
 #define RESTORE_FLAGS _Set_flags
 #define ENABLE_INTERRUPTS() __asm__ __volatile__ ("sti": : :"memory")
 #define CLEAR_INTERRUPTS()  __asm__ __volatile__ ("cli": : :"memory")
 #define RAISE_INTERRUPT(_x)  __asm__ __volatile__("int %0" :: "g" (_x))
 #define RETURN_FROM_INT()   __asm__ __volatile__("iret" :: )

-#if ! defined(VMKERNEL)
-#define NO_INTERRUPTS_BEGIN()	do { \
-                                   uintptr_t _flags; \
-                                   SAVE_FLAGS(_flags); \
-                                   CLEAR_INTERRUPTS();
-
-#define NO_INTERRUPTS_END()	   RESTORE_FLAGS(_flags); \
-                                } while(0)
-#endif
-
 /* End of the section whose correctness has NOT been checked */

 #elif defined _MSC_VER  /* !__GNUC__ */

-#define SET_IDT(_idt)    _Set_IDT(&(_idt))
-#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define _BUILD_SET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writedr(reg, r);                              \
+   }
+
+#define _BUILD_GET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t *result)                            \
+   {                                                  \
+      *result = __readdr(reg);                        \
+   }
+
+#define _BUILD_SET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writecr##reg(r);                              \
+   }
+
+#define _BUILD_GET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t *result)                            \
+   {                                                  \
+      *result = __readcr##reg();                      \
+   }
+
+_BUILD_SET_DR(_SET_DR0, 0)
+_BUILD_SET_DR(_SET_DR1, 1)
+_BUILD_SET_DR(_SET_DR2, 2)
+_BUILD_SET_DR(_SET_DR3, 3)
+_BUILD_SET_DR(_SET_DR6, 6)
+_BUILD_SET_DR(_SET_DR7, 7)
+
+_BUILD_GET_DR(_GET_DR0, 0)
+_BUILD_GET_DR(_GET_DR1, 1)
+_BUILD_GET_DR(_GET_DR2, 2)
+_BUILD_GET_DR(_GET_DR3, 3)
+_BUILD_GET_DR(_GET_DR6, 6)
+_BUILD_GET_DR(_GET_DR7, 7)
+
+_BUILD_SET_CR(_SET_CR0, 0);
+_BUILD_SET_CR(_SET_CR3, 3);
+_BUILD_SET_CR(_SET_CR4, 4);
+_BUILD_SET_CR(_SET_CR8, 8);
+
+_BUILD_GET_CR(_GET_CR0, 0);
+_BUILD_GET_CR(_GET_CR2, 2);
+_BUILD_GET_CR(_GET_CR3, 3);
+_BUILD_GET_CR(_GET_CR4, 4);
+_BUILD_GET_CR(_GET_CR8, 8);
+
+static INLINE void
+_Set_GDT(_GETSET_DTR_TYPE *dtr)
+{
+#ifdef VM_X86_64
+   _lgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _lgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm lgdt [eax]
+#endif
+}
+
+static INLINE void
+_Get_GDT(_GETSET_DTR_TYPE *dtr)
+{
+#ifdef VM_X86_64
+   _sgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _sgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm sgdt [eax]
+#endif
+}
+
+static INLINE void
+_Set_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __lidt(dtr);
+}
+
+static INLINE void
+_Get_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __sidt(dtr);
+}
+
+
+#define ENABLE_INTERRUPTS() _enable()
+#define CLEAR_INTERRUPTS()  _disable()
+
+#define SAVE_FLAGS(x) do { \
+   (x) = __readeflags();   \
+} while (0)
+
+#define RESTORE_FLAGS(x) __writeeflags(x)
+
+
+#ifdef VM_X86_32
+
 #define SET_TR(_tr)      _Set_TR(_tr)
 #define SET_LDT(_tr)     _Set_LDT(_tr)

-#define GET_IDT(_idt)    _Get_IDT(&(_idt))
-#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
 #define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
 #define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)

-#define GET_CR0(_reg)    __asm mov eax, cr0 __asm mov _reg, eax
-#define SET_CR0(_reg)    __asm mov eax, _reg __asm mov cr0, eax
-#define GET_CR2(_reg)    __asm mov eax, cr2 __asm mov _reg, eax
 #define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
-#define GET_CR3(_reg)    __asm mov eax, cr3 __asm mov _reg, eax
-#define SET_CR3(_reg)    __asm mov eax, _reg __asm mov cr3, eax
-/*
- * MSC doesn't seem to like CR4 in __asm statements. We emit
- * the opcode for MOV EAX,CR4 = 0xf020e0 and MOV CR4,EAX = 0xf022e0
- */
-#define GET_CR4(_reg) { \
- __asm _emit 0x0f __asm _emit 0x20 __asm _emit 0xe0 \
- __asm mov _reg, eax \
-}
-#define SET_CR4(_reg)    { \
-  __asm mov eax, _reg \
-  __asm _emit 0x0f __asm _emit 0x22 __asm _emit 0xe0 \
-}
-
-
-#define GET_DR0(_reg) do { __asm mov eax,dr0 __asm mov _reg,eax } while (0)
-#define SET_DR0(_reg) do { __asm mov eax,_reg __asm mov dr0,eax } while (0)
-#define GET_DR1(_reg) do { __asm mov eax,dr1 __asm mov _reg,eax } while (0)
-#define SET_DR1(_reg) do { __asm mov eax,_reg __asm mov dr1,eax } while (0)
-#define GET_DR2(_reg) do { __asm mov eax,dr2 __asm mov _reg,eax } while (0)
-#define SET_DR2(_reg) do { __asm mov eax,_reg __asm mov dr2,eax } while (0)
-#define GET_DR3(_reg) do { __asm mov eax,dr3 __asm mov _reg,eax } while (0)
-#define SET_DR3(_reg) do { __asm mov eax,_reg __asm mov dr3,eax } while (0)
-#define GET_DR6(_reg) do { __asm mov eax,dr6 __asm mov _reg,eax } while (0)
-#define SET_DR6(_reg) do { __asm mov eax,_reg __asm mov dr6,eax } while (0)
-#define GET_DR7(_reg) do { __asm mov eax,dr7 __asm mov _reg,eax } while (0)
-#define SET_DR7(_reg) do { __asm mov eax,_reg __asm mov dr7,eax } while (0)
-
-
 #define CLTS()           __asm clts

 #define FNCLEX()         __asm fnclex

 #define TLB_INVALIDATE_PAGE(_addr) {  \
-	 void *_a = (_addr); \
-     __asm mov eax, _a __asm invlpg [eax] \
+   void *_a = (_addr); \
+   __asm mov eax, _a __asm invlpg [eax] \
 }

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) { \
-	uint32 __a = (uint32) (_addr); \
-	__asm mov eax, __a _asm invlpg fs:[eax] \
-}
-
-
-#define ENABLE_INTERRUPTS() { __asm sti }
-#define CLEAR_INTERRUPTS()  { __asm cli }
-
 #define RAISE_INTERRUPT(_x)  {__asm int _x }
 #define RETURN_FROM_INT()   {__asm iretd }


-#define SAVE_FLAGS(x) { \
-     __asm pushfd __asm pop eax __asm mov x, eax \
-}
-
-#define RESTORE_FLAGS(x) { \
-     __asm push x __asm popfd\
-}
-
-
-
-static INLINE void SET_DS(Selector val)
+static INLINE void
+SET_DS(Selector val)
 {
   __asm mov ax, val
   __asm mov ds, ax
 }

-static INLINE void SET_ES(Selector val)
+static INLINE void
+SET_ES(Selector val)
 {
   __asm mov ax, val
   __asm mov es, ax
 }

-static INLINE void SET_FS(Selector val)
+static INLINE void
+SET_FS(Selector val)
 {
   __asm mov ax, val
   __asm mov fs, ax
 }

-static INLINE void SET_GS(Selector val)
+static INLINE void
+SET_GS(Selector val)
 {
   __asm mov ax, val
   __asm mov gs, ax
 }

-static INLINE void SET_SS(Selector val)
+static INLINE void
+SET_SS(Selector val)
 {
   __asm mov ax, val
   __asm mov ss, ax
 }

-static INLINE Selector GET_FS(void)
+static INLINE Selector
+GET_FS(void)
 {
-	Selector _v;
-	__asm mov _v,fs
-	return _v;
+   Selector _v;
+   __asm mov _v,fs
+   return _v;
 }

-static INLINE Selector GET_GS(void)
+static INLINE Selector
+GET_GS(void)
 {
-	Selector _v;
-	__asm mov _v,gs
-	return _v;
+   Selector _v;
+   __asm mov _v,gs
+   return _v;
 }

-
-static INLINE Selector GET_DS(void)
+static INLINE Selector
+GET_DS(void)
 {
-	Selector _v;
-	__asm mov _v,ds
-	return _v;
+   Selector _v;
+   __asm mov _v,ds
+   return _v;
 }

-static INLINE Selector GET_ES(void)
+static INLINE Selector
+GET_ES(void)
 {
-	Selector _v;
-	__asm mov _v,es
-	return _v;
+   Selector _v;
+   __asm mov _v,es
+   return _v;
 }

-static INLINE Selector GET_SS(void)
+static INLINE Selector
+GET_SS(void)
 {
-	Selector _v;
-	__asm mov _v,ss
-	return _v;
+   Selector _v;
+   __asm mov _v,ss
+   return _v;
 }

-static INLINE Selector GET_CS(void)
+static INLINE Selector
+GET_CS(void)
 {
-	Selector _v;
-	__asm mov _v,cs
-	return _v;
+   Selector _v;
+   __asm mov _v,cs
+   return _v;
 }

-#pragma warning( disable : 4035)
-
-static INLINE uint32  GET_WORD_FROM_FS(uint32 *_addr) {
-	__asm mov eax, _addr
-    __asm mov eax, fs:[eax]
-}
-
-static INLINE uint16  GET_SHORT_FROM_FS(uint16 *_addr) {
-	__asm mov eax, _addr
-    __asm mov ax, fs:[eax]
-}
-
-static INLINE uint8  GET_BYTE_FROM_FS(uint8 *_addr) {
-	__asm mov eax, _addr
-     __asm mov al, fs:[eax]
-}
-
-#pragma warning (default: 4035)
-
-static INLINE void  SET_WORD_FS(uint32 *_addr, uint32 _val) {
-    __asm mov eax, _addr
-    __asm mov ebx, _val
-    __asm mov fs:[eax], ebx
-}
-
-static INLINE void  SET_SHORT_FS(uint32 *_addr, uint16 _val) {
-    __asm mov eax, _addr
-    __asm mov bx, _val
-    __asm mov fs:[eax], bx
-}
-
-static INLINE void  SET_BYTE_FS(uint32 *_addr, uint8 _val) {
-    __asm mov eax, _addr
-    __asm mov bl, _val
-    __asm mov fs:[eax], bl
-}
-
-static INLINE void _Set_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lgdt [eax]
-}
-
-static INLINE void _Set_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lidt [eax]
-}
-
-static INLINE void _Set_LDT(Selector val)
+static INLINE void
+_Set_LDT(Selector val)
 {
   __asm lldt val
 }

-static INLINE void _Set_TR(Selector val)
+static INLINE void
+_Set_TR(Selector val)
 {
   __asm ltr val
 }

-static INLINE void _Get_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sgdt [eax]
-}
-
-static INLINE void _Get_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sidt [eax]
-}
-
-static INLINE Selector _Get_LDT(void) {
+static INLINE Selector
+_Get_LDT(void)
+{
   Selector sel;
   __asm sldt sel
   return sel;
 }

-static INLINE Selector _Get_TR(void) {
+static INLINE Selector
+_Get_TR(void)
+{
   Selector sel;
   __asm str sel
   return sel;
 }

-
-static INLINE void
-MEMCOPY_TO_FS(VA to,
-              char * from,
-              unsigned long n)
-{
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x = *(uint32*) (from + i);
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, x
-	  __asm mov fs:[eax], ebx
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x = from[i];
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov bl, x
-	  __asm mov fs:[eax], bl
-      i++;
-   }
-}
-
-
-
-static INLINE void
-MEMCOPY_FROM_FS(char * to,
-                VA from,
-                unsigned long n)
-{
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x;
-	  uint32 _faddr = (uint32)(from+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, fs:[eax]
-	  __asm mov x,ebx
-      *(uint32*)(to+i)=x;
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x;
-	  uint32 _faddr = (uint32) (from+i);
-	  __asm mov eax, _faddr;
-      __asm mov bl, fs:[eax]
-	  __asm mov x, bl
-      *(uint8*)(to+i)=x;
-      i++;
-   }
-}
-
-#else
-#error No compiler defined for get/set
+#endif /* !VM_X86_32 */
 #endif /* !__GNUC__ && !_MSC_VER */


-#ifdef __GNUC__
-static INLINE unsigned CURRENT_CPL(void)
+#define SET_CR_DR(regType, regNum, expr)                     \
+   do {                                                      \
+      /* Ensure no implicit truncation of 'expr' */          \
+      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
+      _SET_##regType##regNum(expr);                          \
+   } while (0)
+
+#define GET_CR_DR(regType, regNum, var) \
+   do {                                 \
+      _GET_##regType##regNum(&(var));   \
+   } while (0)
+
+#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
+#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
+#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
+#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
+#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
+#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
+
+#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
+#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
+#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
+#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
+#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
+#define GET_DR7(var)  GET_CR_DR(DR, 7, var)
+
+/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
+#undef GET_CR0
+#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
+#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
+#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
+#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
+#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
+
+#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
+#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
+#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
+#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
+
+static INLINE Bool
+INTERRUPTS_ENABLED(void)
+{
+   uintptr_t flags;
+   SAVE_FLAGS(flags);
+   return ((flags & EFLAGS_IF) != 0);
+}
+
+/*
+ * [GS]ET_[GI]DT() are defined as macros wrapping a function
+ * so we can pass the argument implicitly by reference (requires
+ * a macro) and get type checking too (requires a function).
+ */
+
+#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
+
+#define SET_IDT(_idt)    _Set_IDT(&(_idt))
+#define GET_IDT(_idt)    _Get_IDT(&(_idt))
+
+#if !defined(VMKERNEL)
+#define NO_INTERRUPTS_BEGIN() do { \
+                                   uintptr_t _flags; \
+                                   SAVE_FLAGS(_flags); \
+                                   CLEAR_INTERRUPTS();
+
+#define NO_INTERRUPTS_END()        RESTORE_FLAGS(_flags); \
+                                 } while(0)
+#endif
+
+
+#if defined (__GNUC__) || (defined (_MSC_VER) && defined (VM_X86_32))
+static INLINE unsigned
+CURRENT_CPL(void)
 {
   return SELECTOR_RPL(GET_CS());
 }
-#elif defined _MSC_VER
-static INLINE unsigned CURRENT_CPL(void) {
-   volatile Selector _v;
-   __asm mov ax, cs _asm mov _v, ax
-   return SELECTOR_RPL(_v);
-}
-#else
-#error No compiler defined for CURRENT_CPL
 #endif


+static INLINE uint64
+RDPMC(int counter)
+{
 #ifdef __GNUC__
-/* Checked against the Intel manual and GCC --hpreg
-
-   volatile because the msr can change without the compiler knowing it
-   (when we use wrmsr). */
-static INLINE uint64 __GET_MSR(int cx)
-{
-   uint64 msr;
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
-      : "=a" (msr)
-      : "c" (cx)
-      : "%rdx"
-   );
-#else
-   __asm__ __volatile__(
-      "rdmsr"
-      : "=A" (msr)
-      : "c" (cx)
-   );
-#endif
-   return msr;
-}
-#elif defined _MSC_VER
-#pragma warning( disable : 4035)
-static INLINE uint64 __GET_MSR(int input)
-{
-   __asm push ecx
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x32
-   __asm pop ecx
-}
-
-static INLINE void __SET_MSR(int input, uint64 val)
-{
-   uint32 hival = (uint32)(val >> 32);
-   uint32 loval = (uint32)val;
-   __asm push edx
-   __asm push ecx
-   __asm push eax
-   __asm mov  eax, loval
-   __asm mov  edx, hival
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x30
-   __asm pop eax
-   __asm pop ecx
-   __asm pop edx
-}
-#pragma warning (default: 4035)
-#else
-#error
-#endif
-
-#ifdef __GNUC__
-static INLINE void __SET_MSR(int cx, uint64 val)
-{
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "a" ((uint32) val), "d" ((uint32)(val >> 32)), "c" (cx)
-    );
-#else
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "A" (val),
-        "c" (cx)
-    );
-#endif
-}
-#endif
-
-
-/*
- * RDMSR/WRMSR access the 64bit MSRs as two
- * 32 bit quantities, whereas GET_MSR/SET_MSR
- * above access the MSRs as one 64bit quantity.
- */
-#ifdef __GNUC__
-#undef RDMSR
-#undef WRMSR
-#define RDMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "rdmsr"                         \
-      : "=a" (low), "=d" (high)       \
-      : "c" (msrNum)                  \
-   );                                 \
-} while (0)
-
-#define WRMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "wrmsr"                         \
-      : /* no outputs */              \
-      : "c" (msrNum),                 \
-        "a" (low),                    \
-        "d" (high)                    \
-    );                                \
-} while (0)
-
-static INLINE uint64 RDPMC(int cx)
-{
 #ifdef VM_X86_64
   uint64 pmcLow;
   uint64 pmcHigh;
@@ -867,7 +699,7 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=a" (pmcLow), "=d" (pmcHigh)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmcHigh << 32 | pmcLow;
@@ -877,43 +709,26 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=A" (pmc)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmc;
 #endif
-}
 #elif defined _MSC_VER
-#ifndef VM_X86_64 // XXX Switch to intrinsics with the new 32 and 64-bit compilers.
-
-static INLINE uint64 RDPMC(int counter)
-{
-   __asm mov ecx, counter
-   __asm rdpmc
+   return __readpmc(counter);
+#endif
 }

-static INLINE void WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   __asm mov ecx, msrNum
-   __asm mov eax, lo
-   __asm mov edx, hi
-   __asm wrmsr
-}
-#endif
-#endif

-
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
-static INLINE uint64 __XGETBV(int cx)
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined (ULM)
+static INLINE uint64
+__XGETBV(int cx)
 {
+#ifdef __GNUC__
 #ifdef VM_X86_64
   uint64 lowval, hival;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=a" (lowval), "=d" (hival)
      : "c" (cx)
   );
@@ -921,39 +736,41 @@ static INLINE uint64 __XGETBV(int cx)
 #else
   uint64 val;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=A" (val)
      : "c" (cx)
   );
   return val;
 #endif
+#elif defined _MSC_VER
+   return _xgetbv((unsigned)cx);
+#endif
 }

-static INLINE void __XSETBV(int cx, uint64 val)
+static INLINE void
+__XSETBV(int cx, uint64 val)
 {
+#ifdef __GNUC__
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd1"
-#else
      "xsetbv"
-#endif
      : /* no outputs */
      : "a" ((uint32)val), "d" ((uint32)(val >> 32)), "c" (cx)
    );
+#elif defined _MSC_VER
+   _xsetbv((unsigned)cx, val);
+#endif
 }

-static INLINE uint64 GET_XCR0(void)
+static INLINE uint64
+GET_XCR0(void)
 {
   return __XGETBV(0);
 }

 #define SET_XCR0(val) __XSETBV(0, val)

-static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
+static INLINE void
+SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 {
 #ifndef VMM_BOOTSTRAP
   ASSERT(oldVal == GET_XCR0());
@@ -965,33 +782,45 @@ static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 #endif


-#define START_TRACING() { \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags |= EFLAGS_TF;    \
-   RESTORE_FLAGS(flags);  \
-}
-
-#define STOP_TRACING() {  \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags &= ~EFLAGS_TF;   \
-   RESTORE_FLAGS(flags);  \
-}
-
-
-static INLINE Bool
-INTERRUPTS_ENABLED(void)
+static INLINE uint32
+RDTSCP_AuxOnly(void)
 {
-   uintptr_t flags;
-   SAVE_FLAGS(flags);
-   return ((flags & EFLAGS_IF) != 0);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return tscAux;
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   __rdtscp(&tscAux);
+
+   return tscAux;
+#endif
 }

-static INLINE void
-SET_KERNEL_PER_CORE(uint64 val)
+
+static INLINE uint64
+RDTSCP(void)
 {
-   __SET_MSR(MSR_GSBASE, val);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      :"=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return QWORD(tscHigh, tscLow);
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   return __rdtscp(&tscAux);
+#endif
 }

 #endif
@@ -1,137 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * vm_asm_x86_64.h
- *
- *	x86-64 asm macros
- */
-
-#ifndef _VM_ASM_X86_64_H_
-#define _VM_ASM_X86_64_H_
-
-
-#define INCLUDE_ALLOW_VMX
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_DISTRIBUTE
-#include "includeCheck.h"
-
-#ifdef __GNUC__
-#include "vm_asm_x86.h"
-#endif
-
-/*
- * This file contains inline assembly routines used by x86_64 code.
- */
-
-#ifdef __GNUC__
-
-  /* nop; prevents #error for no compiler definition from firing */
-
-#elif defined _MSC_VER  /* !__GNUC__ */
-
-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-
-#ifdef _WIN64
-#ifdef __cplusplus
-extern "C" {
-#endif
-unsigned __int64  __readmsr(unsigned long);
-void              __writemsr(unsigned long, unsigned __int64);
-#pragma intrinsic(__readmsr, __writemsr)
-#ifdef __cplusplus
-}
-#endif
-
-
-static INLINE uint64
-RDPMC(int counter)
-{
-   return __readpmc(counter);
-}
-
-
-static INLINE void
-WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   uint64 value = QWORD(hi, lo);
-   __writemsr(msrNum, value);
-}
-
-
-static INLINE uint64
-__GET_MSR(int input)
-{
-   return __readmsr((unsigned long)input);
-}
-
-
-static INLINE void
-__SET_MSR(int cx, uint64 val)
-{
-  __writemsr((unsigned long)cx, (unsigned __int64)val);
-}
-
-#endif
-
-#else
-#error No compiler defined for get/set
-#endif /* !__GNUC__ && !_MSC_VER */
-
-
-#ifdef __GNUC__
-static INLINE void
-SWAPGS(void)
-{
-   __asm__ __volatile__("swapgs");
-}
-
-
-static INLINE uint64
-RDTSCP_AuxOnly(void)
-{
-   uint64 tscLow, tscHigh, tscAux;
-
-   __asm__ __volatile__(
-      "rdtscp"
-      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
-   );
-
-   return tscAux;
-}
-#endif
-
-#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -67,6 +67,7 @@ extern "C" {
 #if defined (VMKPANIC) 
 #include "vmk_assert.h"
 #else /* !VMKPANIC */
+#include <linux/kernel.h>
 #define _ASSERT_PANIC(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
 #define _ASSERT_PANIC_BUG(bug, name) \
@@ -107,7 +108,7 @@ NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
 } while(0)

 #else
-NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+#define Panic panic
 #endif

 void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
@@ -202,7 +203,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1317,7 +1317,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
+   __asm lock And [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_And
@@ -1382,7 +1382,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
+   __asm lock Or [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Or
@@ -1447,7 +1447,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Xor
@@ -2303,7 +2303,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2324,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3618,7 +3621,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -221,8 +221,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +263,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +376,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +392,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -768,12 +771,14 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
@@ -888,7 +893,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +907,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1103,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1161,101 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,12 +49,29 @@ uint64 _umul128(uint64 multiplier, uint64 multiplicand,
 int64 _mul128(int64 multiplier, int64 multiplicand,
              int64 *highProduct);
 uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
+#ifdef ULM
+void _fxsave64(void *save);
+void _fxsave(void *save);
+void _fxrstor64(const void *load);
+void _fxrstor(const void *load);
+void _xsave64(void *save, uint64 mask);
+void _xsave(void *save, uint64 mask);
+void _xsaveopt64(void *save, uint64 mask);
+void _xsavec(void *save, uint64 mask);
+void _xrstor64(const void *load, uint64 mask);
+void _xrstor(const void *load, uint64 mask);
+#endif /* ULM */
 #ifdef __cplusplus
 }
 #endif

 #pragma intrinsic(_umul128, _mul128, __shiftright128)

+#ifdef ULM
+#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
+                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
+#endif /* ULM */
+
 #endif // _MSC_VER

 #if defined(__GNUC__)
@@ -113,40 +130,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +204,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +307,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +351,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +433,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +518,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +542,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -244,9 +244,131 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+#if defined VM_X86_32
+   __asm sfence;
+#else
+   _mm_sfence();
+#endif
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +400,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +515,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +533,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +613,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,12 +44,44 @@
 #endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -58,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -159,18 +157,6 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

-/*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-

 /*
 * Token concatenation
@@ -209,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -255,6 +241,10 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif
@@ -286,6 +276,14 @@ Max(int a, int b)
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -556,10 +554,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -664,6 +662,20 @@ typedef int pid_t;
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
@@ -702,12 +714,6 @@ typedef int pid_t;
 #endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -718,7 +724,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,18 +63,9 @@
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/*
- * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -83,7 +74,6 @@
 #ifdef __x86_64__
 #define VM_X86_64
 #define vm_x86_64 1
-#define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
 #else
@@ -120,14 +110,91 @@

 #endif

-#if defined(__cplusplus) && __cplusplus >= 201103L || \
-    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
-    defined(__APPLE__) || defined(HAVE_STDINT_H)
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif
+
 /*
- * We're using <stdint.h> instead of <cstdint> below because some C++ code
- * deliberately compiles without C++ include paths.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif
+

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -138,40 +205,6 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-#else /* !HAVE_STDINT_H */
-
-/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
-#ifdef _MSC_VER
-
-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
-
-#elif __GNUC__
-#   if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#   else
-/*
- * Only strict c90 (without extensions) lacks a 'long long' type.
- * If this declaration fails ... use -std=c99 or -std=gnu90.
- */
-typedef unsigned long long uint64;
-typedef long long int64;
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-

 /*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
@@ -202,14 +235,6 @@ typedef char           Bool;
 #define IS_BOOL(x)     (((x) & ~1) == 0)


-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */
-
 /*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
@@ -218,9 +243,6 @@ typedef char           Bool;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -234,14 +256,11 @@ typedef char           Bool;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -249,46 +268,10 @@ typedef char           Bool;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -337,10 +320,11 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* macOS hosts use the same formatters for 32- and 64-bit. */
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
   #define FMT64         "ll"
-   #if KERNEL
+   #if defined(__APPLE__) && KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
@@ -452,7 +436,6 @@ typedef uintptr_t VPN;

 typedef uint64    PA;
 typedef uint64    PPN;
-typedef uint32    PPNTMP;

 typedef uint64    TPA;
 typedef uint64    TPPN;
@@ -470,9 +453,7 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint64    PgCnt64;
 typedef uint64    PageCnt;
-typedef uint64    PgNum64;
 typedef uint64    PageNum;
 typedef unsigned  MemHandle;
 typedef unsigned  IoHandle;
@@ -515,7 +496,7 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
@@ -598,15 +579,14 @@ typedef void * UserVA;


 /* Maximal observable PPN value. */
-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
 #define INVALID_PPN32     ((PPN32)0xffffffff)
-#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
-#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -624,7 +604,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((uint32)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -837,7 +818,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -934,82 +915,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,22 +40,25 @@
 /*
 * These definitions suit both the x64 and arm64 architectures. In particular
 * - At all levels, a page table is a 4KB page containing 512 8-byte entries.
- * - Each entry maps 39, 30, 21 and 12 bits at the respective page table
+ * - Each entry maps 48, 39, 30, 21 and 12 bits at the respective page table
 *   levels (with standard page sizes).
 */

-typedef uint64 PT_L4E;
-typedef uint64 PT_L3E;
-typedef uint64 PT_L2E;
-typedef uint64 PT_L1E;
+typedef uint64 PT_Entry;
+typedef PT_Entry PT_L5E;
+typedef PT_Entry PT_L4E;
+typedef PT_Entry PT_L3E;
+typedef PT_Entry PT_L2E;
+typedef PT_Entry PT_L1E;

 typedef enum {
   PT_LEVEL_1 = 1,
   PT_LEVEL_2,
   PT_LEVEL_3,
   PT_LEVEL_4,
+   PT_LEVEL_5,
   PT_LEVEL_STOP = PT_LEVEL_1,
-   PT_MAX_LEVELS = PT_LEVEL_4
+   PT_MAX_LEVELS = PT_LEVEL_5
 } PT_Level;

 #define PT_PTE_SIZE           8
@@ -76,6 +79,7 @@ typedef enum {

 /* Address space size (in 4KB pages) covered by a level-_l page table entry. */
 #define PT_PAGES_PER_LE(_l)   ((uint64)1 << PT_LE_PG_SHIFT(_l))
+#define PT_PAGES_PER_L5E      PT_PAGES_PER_LE(PT_LEVEL_5)
 #define PT_PAGES_PER_L4E      PT_PAGES_PER_LE(PT_LEVEL_4)
 #define PT_PAGES_PER_L3E      PT_PAGES_PER_LE(PT_LEVEL_3)
 #define PT_PAGES_PER_L2E      PT_PAGES_PER_LE(PT_LEVEL_2)
@@ -86,6 +90,7 @@ typedef enum {
 * address space size (in bytes) covered by a level-_l page table entry.
 */
 #define PT_LE_SHIFT(_l)       (PAGE_SHIFT + PT_LE_PG_SHIFT(_l))
+#define PT_L5E_SHIFT          PT_LE_SHIFT(PT_LEVEL_5)
 #define PT_L4E_SHIFT          PT_LE_SHIFT(PT_LEVEL_4)
 #define PT_L3E_SHIFT          PT_LE_SHIFT(PT_LEVEL_3)
 #define PT_L2E_SHIFT          PT_LE_SHIFT(PT_LEVEL_2)
@@ -93,6 +98,7 @@ typedef enum {

 /* Address space size (in bytes) covered by a level-_l page table entry. */
 #define PT_LE_SIZE(_l)        ((uint64)1 << PT_LE_SHIFT(_l))
+#define PT_L5E_SIZE           PT_LE_SIZE(PT_LEVEL_5)
 #define PT_L4E_SIZE           PT_LE_SIZE(PT_LEVEL_4)
 #define PT_L3E_SIZE           PT_LE_SIZE(PT_LEVEL_3)
 #define PT_L2E_SIZE           PT_LE_SIZE(PT_LEVEL_2)
@@ -103,6 +109,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LA_2_LOFF(_a, _l)  (((_a) >> PT_LE_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LA_2_L5OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LA_2_L4OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LA_2_L3OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LA_2_L2OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_2)
@@ -113,6 +120,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LPN_2_LOFF(_a, _l) (((_a) >> PT_LE_PG_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LPN_2_L5OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LPN_2_L4OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LPN_2_L3OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LPN_2_L2OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_2)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -30,9 +30,6 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-#define INTERRUPT_REDIRECTION_BITMAP_SIZE 32
-#define IO_PERMISSION_BITMAP_SIZE 0
-

 #define VMM_PANIC_NONE         0
 #define VMM_PANIC_MSG_COPYING  1
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -83,7 +83,7 @@
 * Structure used to query platform about the page state.
 */
 typedef struct PlatformPageInfoList {
-   PgCnt64 numPages;
+   PageCnt numPages;
   BPN     bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
   MPN     mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
   uint8   flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
@@ -96,9 +96,10 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

-#define VMMEM_SERVICES_DEFS                               \
-   MDEF(VMMEM_SERVICES_TYPE_P2M,  P2MUpdate_FilterPages)  \
-   MDEF(VMMEM_SERVICES_TYPE_SWAP, BusMemSwap_FilterPages)
+#define VMMEM_SERVICES_DEFS                                       \
+   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
+   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
+   MDEF(VMMEM_SERVICES_TYPE_DIRTY_CLEAR, DirtyDrainer_ClearDirty)

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,8 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#include <asm/processor-flags.h>
+
 #define X86_MAX_INSTR_LEN  15   /* Max byte length of an x86 instruction. */

 #define NUM_IDT_VECTORS 256
@@ -75,7 +77,9 @@
 #define CR3_PDB_MASK   0xfffff000
 #define CR3_IGNORE     0xFFF
 #define PAE_CR3_IGNORE 0x1F
+#ifndef CR3_PCID_MASK
 #define CR3_PCID_MASK  0xFFF
+#endif
 #define CR3_NO_FLUSH   (1ULL << 63)

 #define CR4_VME        0x00000001
@@ -89,6 +93,8 @@
 #define CR4_PCE        0x00000100
 #define CR4_OSFXSR     0x00000200 // CPU/OS supports SIMD insts
 #define CR4_OSXMMEXCPT 0x00000400 // #XF exception enable PIII only
+#define CR4_UMIP       0x00000800
+#define CR4_LA57       0x00001000
 #define CR4_VMXE       0x00002000
 #define CR4_SMXE       0x00004000
 #define CR4_FSGSBASE   0x00010000
@@ -97,7 +103,7 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
-#define CR4_RESERVED   CONST64U(0xffffffffff889800)
+#define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

 /*
@@ -201,8 +207,15 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

+/* Bitmap of the exception vectors that have associated error codes. */
+#define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
+                                (1u << EXC_NP) | (1u << EXC_SS) | \
+                                (1u << EXC_GP) | (1u << EXC_PF) | \
+                                (1u << EXC_AC))
+
 /*
 * eflag/rflag definitions.
 */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -119,38 +119,45 @@ void __cpuidex(int regs[4], int eax, int ecx);
 #endif

 static INLINE void
-__GET_CPUID(int eax,         // IN
+__GET_CPUID(uint32     eax,  // IN
            CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax)
      : "memory"
   );
 }

 static INLINE void
-__GET_CPUID2(int eax,         // IN
-             int ecx,         // IN
+__GET_CPUID2(uint32 eax,      // IN
+             uint32 ecx,      // IN
             CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax), "c" (ecx)
      : "memory"
   );
 }

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int eax) // IN
+__GET_EAX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx)
      : "a" (eax)
      : "memory", "%ecx", "%edx"
   );
@@ -159,7 +166,7 @@ __GET_EAX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int eax) // IN
+__GET_EBX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

@@ -174,14 +181,16 @@ __GET_EBX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int eax) // IN
+__GET_ECX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ecx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=c" (ecx)
      : "a" (eax)
      : "memory", "%edx"
   );
@@ -190,14 +199,16 @@ __GET_ECX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int eax) // IN
+__GET_EDX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 edx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=d" (edx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=d" (edx)
      : "a" (eax)
      : "memory", "%ecx"
   );
@@ -207,7 +218,7 @@ __GET_EDX_FROM_CPUID(int eax) // IN


 static INLINE uint32
-__GET_EAX_FROM_CPUID4(int ecx) // IN
+__GET_EAX_FROM_CPUID4(uint32 ecx) // IN
 {
   uint32 eax;
   uint32 ebx;
@@ -228,7 +239,7 @@ __GET_EAX_FROM_CPUID4(int ecx) // IN
 #elif defined(_MSC_VER) // } {

 static INLINE void
-__GET_CPUID(int input, CPUIDRegs *regs)
+__GET_CPUID(uint32 input, CPUIDRegs *regs)
 {
 #ifdef VM_X86_64
   __cpuid((int *)regs, input);
@@ -262,7 +273,7 @@ __GET_CPUID(int input, CPUIDRegs *regs)
 */

 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __cpuidex((int *)regs, inputEax, inputEcx);
 }
@@ -275,13 +286,13 @@ __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
 */

 extern void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs);
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs);
 #endif // _MSC_VER >= 1500

 #else // VM_X86_64

 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __asm push esi
   __asm push ebx
@@ -305,7 +316,7 @@ __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
 #endif

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int input)
+__GET_EAX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -332,7 +343,7 @@ __GET_EAX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int input)
+__GET_EBX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -359,7 +370,7 @@ __GET_EBX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int input)
+__GET_ECX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -386,7 +397,7 @@ __GET_ECX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int input)
+__GET_EDX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -420,12 +431,12 @@ __GET_EDX_FROM_CPUID(int input)
 */

 extern uint32
-__GET_EAX_FROM_CPUID4(int inputEcx);
+__GET_EAX_FROM_CPUID4(uint32 inputEcx);

 #else // VM_X86_64

 static INLINE uint32
-__GET_EAX_FROM_CPUID4(int inputEcx)
+__GET_EAX_FROM_CPUID4(uint32 inputEcx)
 {
   uint32 output;

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,43 +97,78 @@ static INLINE uint32 Desc_LongMode(const Descriptor *d) { return d->longmode; }
 static INLINE uint32 Desc_DB(const Descriptor *d)       { return d->DB; }
 static INLINE uint32 Desc_Gran(const Descriptor *d)     { return d->gran; }

-static INLINE uint32 Desc64_Type(const Descriptor64 *d)     { return d->type; }
-static INLINE uint32 Desc64_S(const Descriptor64 *d)        { return d->S; }
-static INLINE uint32 Desc64_DPL(const Descriptor64 *d)      { return d->DPL; }
-static INLINE uint32 Desc64_Present(const Descriptor64 *d)  { return d->present; }
-static INLINE uint32 Desc64_AVL(const Descriptor64 *d)      { return d->AVL; }
-static INLINE uint32 Desc64_Gran(const Descriptor64 *d)     { return d->gran; }
-static INLINE uint32 Desc64_ExtAttrs(const Descriptor64 *d) { return d->ext_attrs; }
+static INLINE uint32
+Desc64_Type(const Descriptor64 *d)
+{
+   return (uint32)d->type;
+}
+
+static INLINE uint32
+Desc64_S(const Descriptor64 *d)
+{
+   return (uint32)d->S;
+}
+
+static INLINE uint32
+Desc64_DPL(const Descriptor64 *d)
+{
+   return (uint32)d->DPL;
+}
+
+static INLINE uint32
+Desc64_Present(const Descriptor64 *d)
+{
+   return (uint32)d->present;
+}
+
+static INLINE uint32
+Desc64_AVL(const Descriptor64 *d)
+{
+   return (uint32)d->AVL;
+}
+
+static INLINE uint32
+Desc64_Gran(const Descriptor64 *d)
+{
+   return (uint32)d->gran;
+}
+
+static INLINE uint32
+Desc64_ExtAttrs(const Descriptor64 *d)
+{
+   return (uint32)d->ext_attrs;
+}

 static INLINE LA32
 Desc_GetBase(const Descriptor *d)
 {
-   return (d->base_hi << 24) | (d->base_mid << 16) | d->base_lo;
+   return ((unsigned)d->base_hi  << 24) |
+          ((unsigned)d->base_mid << 16) | d->base_lo;
 }

 static INLINE LA64
 Desc64_GetBase(const Descriptor64 *d)
 {
-   return ((uint64)d->base_hi << 32)  | 
-          ((uint64)d->base_mid << 24) | (uint32)d->base_lo;
+   return ((uint64)d->base_hi << 32)  |
+          ((uint64)d->base_mid << 24) | (uint64)d->base_lo;
 }

 static INLINE LA32
 Desc64_GetBaseHi(const Descriptor64 *d)
 {
-   return d->base_hi;
+   return (LA32)d->base_hi;
 }

 static INLINE VA32
 Desc_GetLimit(const Descriptor *d)
 {
-   return (d->limit_hi << 16) | d->limit_lo;
+   return ((unsigned)d->limit_hi << 16) | d->limit_lo;
 }

 static INLINE VA32
 Desc64_GetLimit(const Descriptor64 *d)
 {
-   return ((uint32)d->limit_hi << 16) | (uint32)d->limit_lo;
+   return (VA32)(((unsigned)d->limit_hi << 16) | d->limit_lo);
 }

 static INLINE Bool
@@ -310,12 +345,13 @@ static INLINE uint32 DESC_PRESENT(Descriptor d)    { return d.present; }
 *-----------------------------------------------------------------------------
 */

-static INLINE void 
-Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type, 
-                           uint32 DPL, uint32 present, uint32 DB, uint32 gran)
+static INLINE void
+Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit,
+                           uint32 type, uint32 DPL, uint32 present,
+                           uint32 DB, uint32 gran)
 {
-   uint32 baseHi = (uint32) (base >> 32);
-   uint32 baseLo = (uint32) base;
+   uint32 baseHi = (uint32)(base >> 32);
+   uint32 baseLo = (uint32)base;

   /*
    * The first half of a 16-byte descriptor is a valid 8-byte descriptor
@@ -336,15 +372,17 @@ Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type
 *   Given descriptor, return the code/stack size that it specifies.
 *----------------------------------------------------------------------
 */
-static INLINE int
+static INLINE unsigned
 Desc_DBSize(const Descriptor *desc)
 {
   /*
    * Code/stack size is determined by the D/B bit; bit 22 of the 2nd word.
-    * Shift the bit to position 1, mask it out, add 2. Result: 2 or 4.
+    * Shift the bit to position 1, mask it out, add 2.
+    *
+    * Result: 2 or 4.
    */
-   int cSz = ((((const uint32*)desc)[1] >> 21) & 2) + 2;
-   ASSERT(cSz == (Desc_DB(desc) ? 4 : 2));
+   unsigned cSz = ((((const uint32 *)desc)[1] >> 21) & 2) + 2;
+   ASSERT(cSz == (Desc_DB(desc) ? 4U : 2U));
   return cSz;
 }

@@ -526,7 +564,7 @@ typedef struct Gate {
 static INLINE VA
 CallGate_GetOffset(const Gate *cg)
 {
-   return (cg->offset_hi << 16) | cg->offset_lo;
+   return ((unsigned)cg->offset_hi << 16) | cg->offset_lo;
 }

 static INLINE void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-/* 
+/*
 * x86msr.h --
 *
 *      MSR number definitions.
@@ -24,6 +24,7 @@

 #ifndef _X86MSR_H_
 #define _X86MSR_H_
+#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX

@@ -59,11 +60,8 @@ struct MSRReply {
    * we use it to correlate the replies of multiple queries.
    */
   uint64 tag;              // OUT
-
   uint64 msrVal;           // OUT
-
   uint8  implemented;      // OUT
-
   uint8  _pad[7];
 }
 #include "vmware_pack_end.h"
@@ -120,25 +118,36 @@ MSRQuery;
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
 #define MSR_ARCH_CAPABILITIES_RSBA                (1ULL << 2)
+#define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
+#define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)
+#define MSR_ARCH_CAPABILITIES_MDS_NO              (1ULL << 5)

+#define MSR_FLUSH_CMD                        0x10b
+#define MSR_FLUSH_CMD_FLUSH_L1D                   (1ULL << 0)

 #define MSR_SPEC_CTRL_IBRS                        (1UL << 0)
 #define MSR_SPEC_CTRL_STIBP                       (1UL << 1)
+#define MSR_SPEC_CTRL_SSBD                        (1UL << 2)

 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

+
+#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
+#endif

 /* Intel Core Architecture and later: use only architected counters. */
-#define IA32_MSR_PERF_CAPABILITIES                0x345
-#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT        0
-#define MSR_PERF_CAPABILITIES_LBRFMT_MASK         0x3f
-#define MSR_PERF_CAPABILITIES_PEBSTRAP            (1u << 6)
-#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS    (1u << 7)
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT 8
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK  0xf
-#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM    (1u << 12)
-#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES   (1u << 13)
+#define IA32_MSR_PERF_CAPABILITIES                   0x345
+#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT           0
+#define MSR_PERF_CAPABILITIES_LBRFMT_MASK            0x3f
+#define MSR_PERF_CAPABILITIES_PEBSTRAP               (1u << 6)
+#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS       (1u << 7)
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT    8
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK     0xf
+#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM       (1u << 12)
+#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES      (1u << 13)
+#define MSR_PERF_CAPABILITIES_PEBS_ADAPTIVE_DATA     (1u << 14)
+#define MSR_PERF_CAPABILITIES_PERF_METRICS_AVAILABLE (1u << 15)

 #define IA32_MSR_PEBS_ENABLE                      0x3f1

@@ -265,6 +274,8 @@ typedef enum {

 #define MSR_XSS              0x00000da0  // Extended Supervisor State Mask

+#define MSR_MPX_LAX          0x00001000  // MPX Linear Address Extension
+
 /* RTIT MSRs */
 #define MSR_RTIT_CTL              0x00000570
 #define MSR_RTIT_STATUS           0x00000571
@@ -463,7 +474,9 @@ typedef enum {
 #define MSR_K7_HWCR_SSEDIS         0x00008000ULL // Disable SSE bit
 #define MSR_K7_HWCR_MONMWAITUSEREN 0x00000400ULL // Enable MONITOR/MWAIT CPL>0
 #define MSR_K7_HWCR_TLBFFDIS       0x00000040ULL // Disable TLB Flush Filter
+#ifndef MSR_K7_HWCR_SMMLOCK
 #define MSR_K7_HWCR_SMMLOCK        0x00000001ULL // Lock SMM environment
+#endif

 #ifndef MSR_K8_SYSCFG
 #define MSR_K8_SYSCFG        0xc0010010
@@ -507,9 +520,28 @@ typedef enum {
 #define MSR_VM_CR_SVME_DISABLE     0x0000000000000010ULL
 #define MSR_VM_CR_RESERVED         0xffffffffffffffe0ULL

+/* SEV related MSRs. */
+#define MSR_VMPAGE_FLUSH           0xc001011e
+#define MSR_GHCB_PA                0xc0010130
+#define MSR_GHCB_PA_SEVINFO_HV              1
+#define MSR_GHCB_PA_SEVINFO_REQ             2
+#define MSR_GHCB_PA_AP_JUMP_TABLE           3
+#define MSR_SEV_STATUS             0xc0010131
+
+#define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
+#define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
+
 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)

+#define MSR_AMD_LS_CFG           0xc0011020  // load-store configuration
+#define MSR_AMD_LS_CFG_SSBD_BULLDOZER (1ULL<<54)  // BD family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_KYOTO     (1ULL<<33)  // Kyoto family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_ZEN       (1ULL<<10)  // Zen family non-arch SSBD
+
+#define MSR_AMD_VIRT_SPEC_CTRL   0xc001011f  // Virtual speculation control
+#define MSR_AMD_VIRT_SPEC_CTRL_SSBD   (1ULL<<2)   // Virtual SSBD
+
 /* Syscall/Sysret related MSRs (x86_64) */
 #define MSR_STAR             0xc0000081 // Also present on Athlons.
 #define MSR_LSTAR            0xc0000082
@@ -591,6 +623,8 @@ typedef enum {
 #define MSR_HYPERV_HYPERCALL_EN                  1ULL
 #define MSR_HYPERV_REFERENCE_TSC_EN              1ULL
 #define MSR_HYPERV_VP_ASSIST_EN                  1ULL
+#define MSR_HYPERV_SIEF_EN                       1ULL
+#define MSR_HYPERV_SIM_EN                        1ULL

 #define MSR_HYPERV_GUESTOSID_VENDOR_SHIFT        48
 #define MSR_HYPERV_GUESTOSID_VENDOR_MASK         0xfULL
@@ -600,6 +634,11 @@ typedef enum {
 #define MSR_HYPERV_GUESTOSID_OS_MASK             0xfULL
 #define MSR_HYPERV_GUESTOSID_OS_WINNT_DERIVATIVE 4ULL

+/* MSR for forcing RTM abort to recover PMC3 (see PR 2333817) */
+#ifndef MSR_TSX_FORCE_ABORT
+#define MSR_TSX_FORCE_ABORT                      0x0000010f
+#endif
+#define MSR_TSX_FORCE_ABORT_RTM_BIT_INDEX        0

 /*
 * MTRR bit description
@@ -652,19 +691,28 @@ typedef unsigned char MTRRType;
 /*
 * MISC_FEATURES_ENABLES bits
 */
+#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
+#endif




+// Platform Quality of Service (PQM) MSRs
+#define MSR_INTEL_PQM_EVTSEL    0xc8d
+#define MSR_INTEL_PQM_CTR       0xc8e
+#define MSR_INTEL_PQM_ASSOC     0xc8f

-/*
- * Platform Quality of Service
- */
+ // Platform Quality Enforcement (PQE) MSRs
+#define MSR_INTEL_PQE_CLOS_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_MASK_MAX      0xd8f

-#define MSR_INTEL_QOS_EVTSEL    0xc8d
-#define MSR_INTEL_QOS_CTR       0xc8e
-#define MSR_INTEL_QOS_ASSOC     0xc8f
+#define MSR_INTEL_PQE_CLOS_L3_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_L3_MASK_MAX      0xd0f
+#define MSR_INTEL_PQE_CLOS_L2_MASK_BASE     0xd10
+#define MSR_INTEL_PQE_CLOS_L2_MASK_MAX      0xd4f

 static INLINE uint32
 X86MSR_SysCallEIP(uint64 star)
@@ -687,6 +735,126 @@ X86MSR_SysRetCS(uint64 star)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * X86MSR_{G,S}etMSR --
+ *
+ *      Get or set an MSR in hardware. The interface should remain
+ *      agnostic to the underlying compiler and architecture.
+ *
+ *----------------------------------------------------------------------
+ */
+
+#ifdef __GNUC__
+/*
+ * Checked against the Intel manual and GCC --hpreg
+ * volatile because the msr can change without the compiler knowing it
+ * (when we use wrmsr).
+ */
+#ifdef VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
+      : "=a" (msr)
+      : "c" (cx)
+      : "%rdx"
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "a" ((uint32) value),
+        "d" ((uint32)(value >> 32)),
+        "c" (cx)
+    );
+}
+#else // __GNUC__ && !VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr"
+      : "=A" (msr)
+      : "c"  (cx)
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "A" (value),
+        "c" (cx)
+    );
+}
+#endif
+#elif defined _MSC_VER // !__GNUC__ && _MSC_VER
+#ifdef _WIN64
+unsigned __int64  __readmsr(unsigned long);
+void              __writemsr(unsigned long, unsigned __int64);
+#pragma intrinsic(__readmsr, __writemsr)
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   return __readmsr((unsigned long)cx);
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
+}
+#else // !__GNUC__ && _MSC_VER && !_WIN64
+#pragma warning( disable : 4035)
+static INLINE uint64
+X86MSR_GetMSR(uint32 input)
+{
+   __asm push ecx
+   __asm mov  ecx, input
+   /* 0x0f 0x32 -> edx:eax = rdmsr[ecx] */
+   __asm _emit 0x0f __asm _emit 0x32
+   __asm pop ecx
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 input, uint64 value)
+{
+      uint32 hival = (uint32)((value) >> 32);
+      uint32 loval = (uint32)value;
+      __asm push edx
+      __asm push ecx
+      __asm push eax
+      __asm mov  eax, loval
+      __asm mov  edx, hival
+      __asm mov  ecx, input
+      /* 0x0f 0x30 -> wrmsr[ecx] = edx:eax */
+      __asm _emit 0x0f __asm _emit 0x30
+      __asm pop  eax
+      __asm pop  ecx
+      __asm pop  edx
+}
+#pragma warning (default: 4035)
+#endif
+#else // !__GNUC__ && !_MSC_VER
+#error No compiler defined for RDMSR/WRMSR.
+#endif
+
+
 #if defined __cplusplus
 }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,7 +52,6 @@
 #define LM_AVAIL_MASK         (CONST64(0x7) << LM_AVAIL_SHIFT)
 #define LM_FLAGS_MASK         CONST64(0x80000000000001ff)
 #define LM_CR3_FLAGS_MASK     CONST64(0x18)
-#define LM_L4_L3_RSVD_MASK    CONST64(0x180)
 #define LM_L3_1G_RSVD_MASK    CONST64(0x3fffe000)

 #define LM_MAKE_CR3(_mpfn, _flags) \
@@ -65,12 +64,62 @@
                    ((uint64)(_flags) & LM_FLAGS_MASK))

 #define LM_MAKE_PDE(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L5E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L4E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L3E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L2E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L1E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)


+/*
+ *----------------------------------------------------------------------
+ *
+ * LMPTEIsSafe --
+ *
+ *    A shadow PTE is considered "safe" if any of the following conditions are
+ *    met:
+ *
+ *      a) It is not terminal (i.e., present with no reserved bits set)
+ *      b) Terminal, but with MPN and PS fields set to zero
+ *      c) Terminal, but with MPN field specifying an uncachable page
+ *
+ *    In practice, for condition c), we require that bits 45:43 of the EPTE are
+ *    set to b'111.  The position of these bits is undocumented and not
+ *    architectural; they are truly magic.
+ *
+ *----------------------------------------------------------------------
+ */
+#ifdef VMX86_DEBUG
+
+#define LM_SAFE_BITS_MASK (MASK64(3) << 43)
+#define LM_SAFE_BITS(_v)  ((_v) & LM_SAFE_BITS_MASK)
+
+static INLINE Bool
+LMPTEIsTerminal(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   uint64 rsvd2m = (MASK(PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvd1g = (MASK(2 * PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvdl4 = PTE_PS;
+   uint64 rsvd = physMask & MASK64(52);
+
+   return (!PTE_PRESENT(pte) ||
+           (pte & LM_PTE_PFN_MASK) & rsvd) != 0 ||
+           ((pte & PTE_PS) != 0 &&
+            ((level == PT_LEVEL_2 && (pte & rsvd2m) != 0) ||
+             (level == PT_LEVEL_3 && (pte & rsvd1g) != 0))) ||
+           (level == PT_LEVEL_4 && (pte & rsvdl4) != 0);
+}
+
+static INLINE Bool
+LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   MPN mpn = (pte & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT;
+   Bool safe = !LMPTEIsTerminal(pte, level, physMask) ||
+               (mpn == 0 && !PTE_LARGEPAGE(pte)) ||
+               LM_SAFE_BITS(pte) == LM_SAFE_BITS_MASK;
+   return safe;
+}
+#endif /* VMX86_DEBUG */


 /*
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,21 +65,21 @@ typedef struct Descriptor {
 typedef 
 #include "vmware_pack_begin.h"
 struct Descriptor64 {
-   unsigned   limit_lo  : 16;   // Limit bits 15-0.
-   unsigned   base_lo   : 24;   // Base bits  23-0.
-   unsigned   type      : 4;
-   unsigned   S         : 1;
-   unsigned   DPL       : 2;
-   unsigned   present   : 1;
-   unsigned   limit_hi  : 4;    // Limit bits 19-16.
-   unsigned   AVL       : 1;
-   unsigned   reserved0 : 2;
-   unsigned   gran      : 1;
-   unsigned   base_mid  : 8;    // Base bits 31-24.
-   unsigned   base_hi   : 32;   // Base bits 63-32.
-   unsigned   reserved1 : 8;
-   unsigned   ext_attrs : 5;
-   unsigned   reserved2 : 19;
+   uint64   limit_lo  : 16;   // Limit bits 15-0.
+   uint64   base_lo   : 24;   // Base bits  23-0.
+   uint64   type      : 4;
+   uint64   S         : 1;
+   uint64   DPL       : 2;
+   uint64   present   : 1;
+   uint64   limit_hi  : 4;    // Limit bits 19-16.
+   uint64   AVL       : 1;
+   uint64   reserved0 : 2;
+   uint64   gran      : 1;
+   uint64   base_mid  : 8;    // Base bits 31-24.
+   uint64   base_hi   : 32;   // Base bits 63-32.
+   uint64   reserved1 : 8;
+   uint64   ext_attrs : 5;
+   uint64   reserved2 : 19;
 }
 #include "vmware_pack_end.h"
 Descriptor64;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -47,8 +47,10 @@
 #endif
 #include "x86cpuid_asm.h"

-#define SVM_VMCB_IO_BITMAP_SIZE    (3 * PAGE_SIZE)
-#define SVM_VMCB_MSR_BITMAP_SIZE   (2 * PAGE_SIZE)
+#define SVM_VMCB_IO_BITMAP_PAGES   (3)
+#define SVM_VMCB_IO_BITMAP_SIZE    PAGES_2_BYTES(SVM_VMCB_IO_BITMAP_PAGES)
+#define SVM_VMCB_MSR_BITMAP_PAGES  (2)
+#define SVM_VMCB_MSR_BITMAP_SIZE   PAGES_2_BYTES(SVM_VMCB_MSR_BITMAP_PAGES)

 /* Exit controls for the CR/DR access and hardware exceptions */
 #define SVM_CR_RD_CTL(num)         (0x1 << (num))
@@ -63,53 +65,56 @@
 #define SVM_XCP_CTL_ALL            0xffffffff

 /* Execution intercept controls */
-/* VMCB.exitCtl */
-#define SVM_VMCB_EXEC_CTL_INTR        0x0000000000000001ULL
-#define SVM_VMCB_EXEC_CTL_NMI         0x0000000000000002ULL
-#define SVM_VMCB_EXEC_CTL_SMI         0x0000000000000004ULL
-#define SVM_VMCB_EXEC_CTL_INIT        0x0000000000000008ULL
-#define SVM_VMCB_EXEC_CTL_VINTR       0x0000000000000010ULL
-#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR  0x0000000000000020ULL
-#define SVM_VMCB_EXEC_CTL_SIDT        0x0000000000000040ULL
-#define SVM_VMCB_EXEC_CTL_SGDT        0x0000000000000080ULL
-#define SVM_VMCB_EXEC_CTL_SLDT        0x0000000000000100ULL
-#define SVM_VMCB_EXEC_CTL_STR         0x0000000000000200ULL
-#define SVM_VMCB_EXEC_CTL_LIDT        0x0000000000000400ULL
-#define SVM_VMCB_EXEC_CTL_LGDT        0x0000000000000800ULL
-#define SVM_VMCB_EXEC_CTL_LLDT        0x0000000000001000ULL
-#define SVM_VMCB_EXEC_CTL_LTR         0x0000000000002000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSC       0x0000000000004000ULL
-#define SVM_VMCB_EXEC_CTL_RDPMC       0x0000000000008000ULL
-#define SVM_VMCB_EXEC_CTL_PUSHF       0x0000000000010000ULL
-#define SVM_VMCB_EXEC_CTL_POPF        0x0000000000020000ULL
-#define SVM_VMCB_EXEC_CTL_CPUID       0x0000000000040000ULL
-#define SVM_VMCB_EXEC_CTL_RSM         0x0000000000080000ULL
-#define SVM_VMCB_EXEC_CTL_IRET        0x0000000000100000ULL
-#define SVM_VMCB_EXEC_CTL_SWINT       0x0000000000200000ULL
-#define SVM_VMCB_EXEC_CTL_INVD        0x0000000000400000ULL
-#define SVM_VMCB_EXEC_CTL_PAUSE       0x0000000000800000ULL
-#define SVM_VMCB_EXEC_CTL_HLT         0x0000000001000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPG      0x0000000002000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPGA     0x0000000004000000ULL
-#define SVM_VMCB_EXEC_CTL_IOIO        0x0000000008000000ULL
-#define SVM_VMCB_EXEC_CTL_MSR         0x0000000010000000ULL
-#define SVM_VMCB_EXEC_CTL_TS          0x0000000020000000ULL
-#define SVM_VMCB_EXEC_CTL_FERR_FRZ    0x0000000040000000ULL
-#define SVM_VMCB_EXEC_CTL_SHUTDOWN    0x0000000080000000ULL
-#define SVM_VMCB_EXEC_CTL_VMRUN       0x0000000100000000ULL
-#define SVM_VMCB_EXEC_CTL_VMMCALL     0x0000000200000000ULL
-#define SVM_VMCB_EXEC_CTL_VMLOAD      0x0000000400000000ULL
-#define SVM_VMCB_EXEC_CTL_VMSAVE      0x0000000800000000ULL
-#define SVM_VMCB_EXEC_CTL_STGI        0x0000001000000000ULL
-#define SVM_VMCB_EXEC_CTL_CLGI        0x0000002000000000ULL
-#define SVM_VMCB_EXEC_CTL_SKINIT      0x0000004000000000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSCP      0x0000008000000000ULL
-#define SVM_VMCB_EXEC_CTL_ICEBP       0x0000010000000000ULL
-#define SVM_VMCB_EXEC_CTL_WBINVD      0x0000020000000000ULL
-#define SVM_VMCB_EXEC_CTL_MONITOR     0x0000040000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT       0x0000080000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT_COND  0x0000100000000000ULL
-#define SVM_VMCB_EXEC_CTL_XSETBV      0x0000200000000000ULL
+/* VMCB.execCtl */
+#define SVM_VMCB_EXEC_CTL_INTR           0x0000000000000001ULL
+#define SVM_VMCB_EXEC_CTL_NMI            0x0000000000000002ULL
+#define SVM_VMCB_EXEC_CTL_SMI            0x0000000000000004ULL
+#define SVM_VMCB_EXEC_CTL_INIT           0x0000000000000008ULL
+#define SVM_VMCB_EXEC_CTL_VINTR          0x0000000000000010ULL
+#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR     0x0000000000000020ULL
+#define SVM_VMCB_EXEC_CTL_SIDT           0x0000000000000040ULL
+#define SVM_VMCB_EXEC_CTL_SGDT           0x0000000000000080ULL
+#define SVM_VMCB_EXEC_CTL_SLDT           0x0000000000000100ULL
+#define SVM_VMCB_EXEC_CTL_STR            0x0000000000000200ULL
+#define SVM_VMCB_EXEC_CTL_LIDT           0x0000000000000400ULL
+#define SVM_VMCB_EXEC_CTL_LGDT           0x0000000000000800ULL
+#define SVM_VMCB_EXEC_CTL_LLDT           0x0000000000001000ULL
+#define SVM_VMCB_EXEC_CTL_LTR            0x0000000000002000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSC          0x0000000000004000ULL
+#define SVM_VMCB_EXEC_CTL_RDPMC          0x0000000000008000ULL
+#define SVM_VMCB_EXEC_CTL_PUSHF          0x0000000000010000ULL
+#define SVM_VMCB_EXEC_CTL_POPF           0x0000000000020000ULL
+#define SVM_VMCB_EXEC_CTL_CPUID          0x0000000000040000ULL
+#define SVM_VMCB_EXEC_CTL_RSM            0x0000000000080000ULL
+#define SVM_VMCB_EXEC_CTL_IRET           0x0000000000100000ULL
+#define SVM_VMCB_EXEC_CTL_SWINT          0x0000000000200000ULL
+#define SVM_VMCB_EXEC_CTL_INVD           0x0000000000400000ULL
+#define SVM_VMCB_EXEC_CTL_PAUSE          0x0000000000800000ULL
+#define SVM_VMCB_EXEC_CTL_HLT            0x0000000001000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPG         0x0000000002000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPGA        0x0000000004000000ULL
+#define SVM_VMCB_EXEC_CTL_IOIO           0x0000000008000000ULL
+#define SVM_VMCB_EXEC_CTL_MSR            0x0000000010000000ULL
+#define SVM_VMCB_EXEC_CTL_TS             0x0000000020000000ULL
+#define SVM_VMCB_EXEC_CTL_FERR_FRZ       0x0000000040000000ULL
+#define SVM_VMCB_EXEC_CTL_SHUTDOWN       0x0000000080000000ULL
+#define SVM_VMCB_EXEC_CTL_VMRUN          0x0000000100000000ULL
+#define SVM_VMCB_EXEC_CTL_VMMCALL        0x0000000200000000ULL
+#define SVM_VMCB_EXEC_CTL_VMLOAD         0x0000000400000000ULL
+#define SVM_VMCB_EXEC_CTL_VMSAVE         0x0000000800000000ULL
+#define SVM_VMCB_EXEC_CTL_STGI           0x0000001000000000ULL
+#define SVM_VMCB_EXEC_CTL_CLGI           0x0000002000000000ULL
+#define SVM_VMCB_EXEC_CTL_SKINIT         0x0000004000000000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSCP         0x0000008000000000ULL
+#define SVM_VMCB_EXEC_CTL_ICEBP          0x0000010000000000ULL
+#define SVM_VMCB_EXEC_CTL_WBINVD         0x0000020000000000ULL
+#define SVM_VMCB_EXEC_CTL_MONITOR        0x0000040000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT          0x0000080000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT_COND     0x0000100000000000ULL
+#define SVM_VMCB_EXEC_CTL_XSETBV         0x0000200000000000ULL
+#define SVM_VMCB_EXEC_CTL_EFER_WR_TRAP   0x0000800000000000ULL
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP(n) (0x0001000000000000ULL << (n))
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP_ALL 0xffff000000000000ULL

 /* VMCB.tlbCtl */
 #define SVM_VMCB_TLB_CTL_FLUSH              0x01
@@ -130,9 +135,12 @@

 /* VMCB.intrState */
 #define SVM_VMCB_INTRSTATE_INTR_SHADOW     (1 << 0)
+#define SVM_VMCB_INTRSTATE_GUEST_IF        (1 << 1)

 /* VMCB.nptCtl */
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)
+#define SVM_VMCB_NPTCTL_SEV_ENABLE         (1 << 1)
+#define SVM_VMCB_NPTCTL_SEV_ES_ENABLE      (1 << 2)

 /* VMCB.virtExt */
 #define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
@@ -234,9 +242,16 @@
 #define SVM_EXITCODE_MWAIT                139
 #define SVM_EXITCODE_MWAIT_COND           140
 #define SVM_EXITCODE_XSETBV               141
+#define SVM_EXITCODE_EFER_WRITE_TRAP      143
+#define SVM_EXITCODE_CR_WRITE_TRAP(n)    (144 + (n))
 #define SVM_EXITCODE_NPF                 1024
 #define SVM_EXITCODE_AVIC_INCOMPLETE_IPI 1025
 #define SVM_EXITCODE_AVIC_NOACCEL        1026
+#define SVM_EXITCODE_VMGEXIT             1027
+#define SVM_EXITCODE_MMIO_READ           0x80000001   // SW only
+#define SVM_EXITCODE_MMIO_WRITE          0x80000002   // SW only
+#define SVM_EXITCODE_NMI_COMPLETE        0x80000003   // SW only
+#define SVM_EXITCODE_UNSUPPORTED         0x8000FFFF   // SW only
 #define SVM_EXITCODE_INVALID             (-1ULL)

 /* ExitInfo1 for I/O exits */
@@ -260,6 +275,9 @@
 #define SVM_IOEXIT_MBZ           0x00000002
 #define SVM_IOEXIT_SBZ           0x0000e000

+/* ExitInfo1 for MSR exits */
+#define SVM_MSREXIT_WRMSR        0x00000001
+
 /* ExitInfo1 for CR exits */
 #define SVM_CREXIT_MOVCR         (1ull << 63)
 #define SVM_CREXIT_GPR_MASK      0xf
@@ -430,7 +448,7 @@ SVM_LockedFromFeatures(uint64 vmCR)
 static INLINE Bool
 SVM_EnabledCPU(void)
 {
-   return SVM_EnabledFromFeatures(__GET_MSR(MSR_VM_CR));
+   return SVM_EnabledFromFeatures(X86MSR_GetMSR(MSR_VM_CR));
 }
 #endif /* } !defined(USERLEVEL) */

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -62,13 +62,6 @@ typedef uint64 VM_EPTE;
 * Registers
 */

-typedef union SharedReg64 {
-   Reg8  reg8[2];
-   Reg16 reg16;
-   Reg32 reg32;
-   Reg64 reg64;
-} SharedReg64;
-
 typedef union SharedUReg64 {
   UReg8  ureg8[2];
   UReg16 ureg16;
@@ -178,11 +171,7 @@ typedef Task32 Task;

 #pragma pack(push, 1)
 typedef struct {
-#if defined(VMM) || defined(COREQUERY)
-   uint64 va;
-#else
-   uint32 va;
-#endif
+   uint64   va;
   Selector seg;
 } FarPtr;
 #pragma pack(pop)
@@ -468,4 +457,12 @@ typedef struct ExcFrame64ForCopy {
 } ExcFrame64ForCopy;
 #pragma pack(pop)

+/*
+ * Layout of the stack for performing a 64 bit lret instruction.
+ */
+typedef struct LretFrame64 {
+   uint64 rip;
+   uint64 cs;
+} LretFrame64;
+
 #endif // ifndef _X86TYPES_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,12 @@ VT_EXIT(RDSEED,               61)
 VT_EXIT(PML_LOGFULL,          62)
 VT_EXIT(XSAVES,               63)
 VT_EXIT(XRSTORS,              64)
-
+VT_EXIT(VMEXIT65,             65)
+VT_EXIT(VMEXIT66,             66)
+VT_EXIT(VMEXIT67,             67)
+VT_EXIT(VMEXIT68,             68)
+VT_EXIT(VMEXIT69,             69)
+VT_EXIT(ENCLV,                70)
+VT_EXIT(SGX_CONFLICT,         71)
 /* Bump this up if you add an exit reason. */
-#define VT_NUM_EXIT_REASONS   65
+#define VT_NUM_EXIT_REASONS   72
@@ -283,7 +283,7 @@ VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2030, _S64, _TC, 24)
 VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA,  _S)
 VMCS_UNUSED(                        0x2034, _S64, _TC, 26)
-VMCS_UNUSED(                        0x2036, _S64, _TC, 27)
+VMCS_FIELD(ENCLV_EXITING_BITMAP,    0x2036, _S64, _TC, 27, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2038, _S64, _TC, 28)
 VMCS_UNUSED(                        0x203A, _S64, _TC, 29)
 VMCS_UNUSED(                        0x203C, _S64, _TC, 30)
@@ -159,10 +159,10 @@
 #define VT_ENCODING_RSVD               0xffff9000

 /*
- * The highest index of any currently defined field is 25, for
- * TSC_MULTIPLIER.
+ * The highest index of any currently defined field is 27, for
+ * ENCLV_EXITING_BITMAP.
 */
-#define VT_ENCODING_MAX_INDEX                  25
+#define VT_ENCODING_MAX_INDEX                  27

 enum {
 #define VMCS_FIELD(_name, _val, ...) VT_VMCS_##_name = _val,
@@ -293,7 +293,9 @@ enum {
   VMX_CPU2(PT_SUPPRESS_NR_BIT, 19)                      \
   VMX_CPU2(XSAVES,             20)                      \
   VMX_CPU2(EPT_MBX,            22)                      \
-   VMX_CPU2(TSC_SCALING,        25)
+   VMX_CPU2(TSC_SCALING,        25)                      \
+   VMX_CPU2(ENCLV,              28)                      \
+   VMX_CPU2(EPC_VIRT_EXT,       29)

 #define VMX_PROCBASED_CTLS2_CAP                          \
        VMX_PROCBASED_CTLS2_CAP_NDA                      \
@@ -687,6 +689,12 @@ enum {
 #define VT_GUESTFAIL_QUAL_NMI          3
 #define VT_GUESTFAIL_QUAL_LINK         4

+/* SGX conflict VM-exit Qualification Codes */
+#define VT_SGX_TRACKING_RESOURCE_CONFLICT     0
+#define VT_SGX_TRACKING_REFERENCE_CONFLICT    1
+#define VT_SGX_EPC_PAGE_CONFLICT_EXCEPTION    2
+#define VT_SGX_EPC_PAGE_CONFLICT_ERROR        3
+
 /* VMX abort indicators. */

 #define VT_VMX_ABORT_GUEST_MSRS        1
@@ -710,7 +718,8 @@ enum {
 #define VT_REQUIRED_PINBASED_CTLS                      \
   (VT_PINBASED_CTLS_DEFAULT1                        | \
    VT_VMCS_PIN_VMEXEC_CTL_EXTINT_EXIT               | \
-    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT)
+    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT                  | \
+    VT_VMCS_PIN_VMEXEC_CTL_VNMI)

 #define VT_REQUIRED_PROCBASED_CTLS                     \
   (VT_PROCBASED_CTLS_DEFAULT1                       | \
@@ -726,6 +735,7 @@ enum {
    VT_VMCS_CPU_VMEXEC_CTL_LDCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_STCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_TPR_SHADOW                | \
+    VT_VMCS_CPU_VMEXEC_CTL_VNMI_WINDOW               | \
    VT_VMCS_CPU_VMEXEC_CTL_MONITOR)

 #define VT_REQUIRED_EXIT_CTLS                          \
@@ -1075,7 +1085,7 @@ VT_MBXSupportedFromFeatures(uint64 secondary)
 static INLINE Bool
 VT_EnabledCPU(void)
 {
-   return VT_EnabledFromFeatures(__GET_MSR(MSR_FEATCTL));
+   return VT_EnabledFromFeatures(X86MSR_GetMSR(MSR_FEATCTL));
 }


@@ -1096,16 +1106,14 @@ VT_EnabledCPU(void)
 static INLINE Bool
 VT_SupportedCPU(void)
 {
-   if (__GET_MSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) {
-      return VT_SupportedFromFeatures(__GET_MSR(MSR_VMX_TRUE_PINBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_PROCBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_ENTRY_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_EXIT_CTLS),
-                                      __GET_MSR(MSR_VMX_BASIC));
-   } else {
-      /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
-      return FALSE;
-   }
+
+   /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
+   return (X86MSR_GetMSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) != 0 &&
+          VT_SupportedFromFeatures(X86MSR_GetMSR(MSR_VMX_TRUE_PINBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_PROCBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_ENTRY_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_EXIT_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_BASIC));
 }

 #endif /* } !defined(USERLEVEL) */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -48,12 +48,10 @@
 #include "modulecall.h"
 #include "vm_asm.h"
 #include "vmx86.h"
-#include "initblock.h"
 #include "task.h"
 #include "memtrack.h"
 #include "task.h"
 #include "cpuid.h"
-#include "cpuid_info.h"
 #include "circList.h"
 #include "x86msr.h"

@@ -64,6 +62,7 @@
 #include "hostif.h"
 #include "hostif_priv.h"
 #include "vmhost.h"
+#include "sharedAreaVmmon.h"

 static void LinuxDriverQueue(Device *device);
 static void LinuxDriverDequeue(Device *device);
@@ -96,7 +95,9 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd,
                       unsigned long ioarg);

 static int LinuxDriver_Close(struct inode *inode, struct file *filp);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
+static vm_fault_t LinuxDriverFault(struct vm_fault *fault);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
 static int LinuxDriverFault(struct vm_fault *fault);
 #else
 static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault);
@@ -414,6 +415,7 @@ LinuxDriver_Open(struct inode *inode, // IN
   memset(device, 0, sizeof *device);

   sema_init(&device->lock4Gb, 1);
+   init_rwsem(&device->vmDriverRWSema);

   filp->private_data = device;
   LinuxDriverQueue(device);
@@ -443,17 +445,12 @@ LinuxDriver_Open(struct inode *inode, // IN
 */

 static int
-LinuxDriverAllocPages(unsigned int gfpFlag, // IN
-                      unsigned int order,   // IN
+LinuxDriverAllocPages(unsigned int order,   // IN
                      struct page **pg,     // OUT
-                      unsigned int size)    // IN
+                      PageCnt size)         // IN
 {
-   struct page* page;
-
-   page = alloc_pages(gfpFlag, order);
+   struct page* page = alloc_pages(GFP_HIGHUSER, order);
   if (page) {
-      unsigned int i;
-
      /*
       * Grab an extra reference on all pages except first one - first
       * one was already refcounted by alloc_pages.
@@ -466,9 +463,8 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
       * Note that this depends on Linux VM internals.  It works on all
       * kernels we care about.
       */
-
-      order = 1 << order;
-      for (i = 0; i < order; i++) {
+      PageCnt i, orderToNumPages = 1 << order;
+      for (i = 0; i < orderToNumPages; i++) {
         if (i) {
            /*
             * Debug kernels assert that page->_count is not zero when
@@ -486,17 +482,14 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
            put_page(page);
         } else {
            void *addr = kmap(page);
-
            memset(addr, 0, PAGE_SIZE);
            kunmap(page);
            *pg++ = page;
         }
         page++;
      }
-
      return 0;
   }
-
   return -ENOMEM;
 }

@@ -520,8 +513,7 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
 static void
 LinuxDriverDestructor4Gb(Device *device) // IN
 {
-   unsigned int pg;
-
+   PageCnt pg;
   if (!device->size4Gb) {
      return;
   }
@@ -554,10 +546,12 @@ LinuxDriver_Close(struct inode *inode, // IN
   ASSERT(device);

   LinuxDriverDequeue(device);
+   down_write(&device->vmDriverRWSema);
   if (device->vm != NULL) {
      Vmx86_ReleaseVM(device->vm);
      device->vm = NULL;
   }
+   up_write(&device->vmDriverRWSema);

   Vmx86_Close();

@@ -595,7 +589,12 @@ LinuxDriver_Close(struct inode *inode, // IN
 *-----------------------------------------------------------------------------
 */

-static int
+static
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
+vm_fault_t
+#else
+int
+#endif
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
 LinuxDriverFault(struct vm_fault *fault)     //IN/OUT
 #else
@@ -607,7 +606,7 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
   struct vm_area_struct *vma = fault->vma;
 #endif
   Device *device = (Device *) vma->vm_file->private_data;
-   unsigned long pg;
+   PageNum pg;
   struct page* page;

   pg = fault->pgoff;
@@ -644,12 +643,10 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
 static int LinuxDriverAllocContig(Device *device,
                                  struct vm_area_struct *vma,
                                  unsigned long off,
-                                  unsigned long size)
+                                  PageCnt size)
 {
   unsigned long vmaOrder = VMMON_MAP_ORDER(off);
-   unsigned long vmaAllocSize;
-   unsigned int gfpFlag;
-   unsigned long i;
+   PageCnt vmaAllocSize, i;

   if (VMMON_MAP_RSVD(off)) {
      /* Reserved bits set... */
@@ -659,24 +656,6 @@ static int LinuxDriverAllocContig(Device *device,
      /* We do not need non-zero offsets... */
      return -EINVAL;
   }
-   switch (VMMON_MAP_MT(off)) {
-      case VMMON_MAP_MT_LOW4GB:
-#ifdef GFP_DMA32
-         gfpFlag = GFP_USER | GFP_DMA32;
-#else
-         gfpFlag = GFP_USER | GFP_DMA;
-#endif
-         break;
-      case VMMON_MAP_MT_LOW16MB:
-         gfpFlag = GFP_USER | GFP_DMA;
-         break;
-      case VMMON_MAP_MT_ANY:
-         gfpFlag = GFP_HIGHUSER;
-         break;
-      default:
-         /* Invalid memory type */
-         return -EINVAL;
-   }
   if (size > VMMON_MAP_OFFSET_MASK + 1) {
      /* Size is too big to fit to our window. */
      return -ENOMEM;
@@ -694,23 +673,18 @@ static int LinuxDriverAllocContig(Device *device,
   }
   vmaAllocSize = 1 << vmaOrder;
   for (i = 0; i < size; i += vmaAllocSize) {
-      int err;
-
-      err = LinuxDriverAllocPages(gfpFlag, vmaOrder,
-                                  device->pages4Gb + i, size - i);
+      int err = LinuxDriverAllocPages(vmaOrder, device->pages4Gb + i, size - i);
      if (err) {
         while (i > 0) {
            put_page(device->pages4Gb[--i]);
         }
         up(&device->lock4Gb);
-
         return err;
      }
   }
   device->size4Gb = size;
   up(&device->lock4Gb);
   vma->vm_ops = &vmuser_mops;
-
   return 0;
 }

@@ -739,7 +713,7 @@ LinuxDriverMmap(struct file *filp,
                struct vm_area_struct *vma)
 {
   Device *device = (Device *) filp->private_data;
-   unsigned long size;
+   PageCnt size;
   int err;

   /* Only shared mappings */
@@ -753,17 +727,18 @@ LinuxDriverMmap(struct file *filp,
   if (size < 1) {
      return -EINVAL;
   }
+   down_read(&device->vmDriverRWSema);
   if (device->vm) {
      err = -EINVAL;
   } else {
      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
   }
+   up_read(&device->vmDriverRWSema);
   if (err) {
      return err;
   }
   /* Clear VM_IO, otherwise SuSE's kernels refuse to do get_user_pages */
   vma->vm_flags &= ~VM_IO;
-
   return 0;
 }

@@ -982,7 +957,7 @@ LinuxDriverReadTSC(void *data,   // OUT: TSC values
 *-----------------------------------------------------------------------------
 */

-__attribute__((always_inline)) static Bool
+__always_inline static Bool
 LinuxDriverSyncReadTSCs(uint64 *delta) // OUT: TSC max - TSC min
 {
   TSCDelta tscDelta;
@@ -1034,26 +1009,30 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
                  unsigned long ioarg)  // IN:
 {
   Device *device = (Device *) filp->private_data;
-   int retval = 0;
+   long retval = 0;
   Vcpuid vcpuid;
   VMDriver *vm;
+   Bool needsWriteLock = iocmd == IOCTL_VMX86_CREATE_VM ||
+                         iocmd == IOCTL_VMX86_RELEASE_VM;

+   ASSERT_ON_COMPILE(sizeof(long) == sizeof(int64));
   if (device == NULL) {
      return -EINVAL;
   }

+   if (needsWriteLock) {
+      down_write(&device->vmDriverRWSema);
+   } else {
+      down_read(&device->vmDriverRWSema);
+   }
+
   vm = device->vm;
-
-   /*
-    * Validate the VM pointer for those IOCTLs that require it.
-    */
-
+   /* Validate the VM pointer for those IOCTLs that require it. */
   switch (iocmd) {
   case IOCTL_VMX86_VERSION:
   case IOCTL_VMX86_CREATE_VM:
   case IOCTL_VMX86_SET_UID:
   case IOCTL_VMX86_GET_NUM_VMS:
-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
   case IOCTL_VMX86_SET_HARD_LIMIT:
   case IOCTL_VMX86_GET_IPI_VECTORS:
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
@@ -1108,67 +1087,73 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:

   case IOCTL_VMX86_PROCESS_BOOTSTRAP: {
      VMProcessBootstrapBlock *args;
+      size_t argsSz = GetVMProcessBootstrapBlockSize(vm->numVCPUs);
      VA64 uAddr;
-      Bool res;

      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
      if (retval != 0) {
         break;
      }
-      args = HostIF_AllocKernelMem(sizeof *args, TRUE);
+      args = HostIF_AllocKernelMem(argsSz, TRUE);
      if (args == NULL) {
         retval = -ENOMEM;
         break;
      }
-      retval = HostIF_CopyFromUser(args, uAddr, sizeof *args);
+      retval = HostIF_CopyFromUser(args, uAddr, argsSz);
      if (retval != 0) {
         HostIF_FreeKernelMem(args);
         break;
      }
-      res = Vmx86_ProcessBootstrap(vm,
-                                   args->bsBlobAddr,
-                                   args->numBytes,
-                                   args->headerOffset,
-                                   args->numVCPUs,
-                                   args->ptRootVAs,
-                                   args->shRegions);
-      if (!res) {
+      if (args->numVCPUs != vm->numVCPUs) {
+         retval = -EINVAL;
+      } else if (!Vmx86_ProcessBootstrap(vm,
+                                         args->bsBlobAddr,
+                                         args->numBytes,
+                                         args->headerOffset,
+                                         args->numVCPUs,
+                                         args->perVcpuPages,
+                                         args->shRegions)) {
         retval = -ENOMEM;
      }
      HostIF_FreeKernelMem(args);
      break;
   }

+   case IOCTL_VMX86_REGISTER_SHARED: {
+      VMSharedAreaRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!SharedAreaVmmon_ValidateRegionArgs(vm, &args)) {
+         retval = -EINVAL;
+         break;
+      }
+      if (!SharedAreaVmmon_RegisterRegion(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
+   case IOCTL_VMX86_REGISTER_STATVARS: {
+      VMStatVarsRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!StatVarsVmmon_RegisterVCPU(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
   case IOCTL_VMX86_RELEASE_VM:
      device->vm = NULL;
      Vmx86_ReleaseVM(vm);
      break;

-   case IOCTL_VMX86_INIT_VM: {
-      InitBlock *initParams;
-      VA64 uAddr;
-
-      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
-      if (retval != 0) {
-         break;
-      }
-      initParams = HostIF_AllocKernelMem(sizeof *initParams, TRUE);
-      if (initParams == NULL) {
-         retval = -ENOMEM;
-         break;
-      }
-      retval = HostIF_CopyFromUser(initParams, uAddr, sizeof *initParams);
-      if (retval == 0) {
-         if (Vmx86_InitVM(vm, initParams) == 0) {
-            retval = HostIF_CopyToUser(uAddr, initParams, sizeof *initParams);
-         } else {
-            retval = -EINVAL;
-         }
-      }
-      HostIF_FreeKernelMem(initParams);
-      break;
-   }
-
   case IOCTL_VMX86_RUN_VM:
      vcpuid = ioarg;

@@ -1236,17 +1221,28 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_VMM_PAGE_ROOT: {
+      VcpuPageRoot args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->ptRootMpns == NULL) {
+         retval = -EINVAL;
+      } else {
+         args.pageRoot = vm->ptRootMpns[args.vcpuid];
+         retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      }
+      break;
+   }
+
   case IOCTL_VMX86_GET_NUM_VMS:
      retval = Vmx86_GetNumVMs();
      break;

-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
-      retval = Vmx86_GetTotalMemUsage();
-      break;
-
   case IOCTL_VMX86_SET_HARD_LIMIT: {
-      int32 limit;
-
+      PageCnt limit;
      retval = HostIF_CopyFromUser(&limit, ioarg, sizeof limit);
      if (retval != 0) {
         break;
@@ -1516,6 +1512,13 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_NUM_ANON_PAGES: {
+      PageCnt numAnonPages;
+      numAnonPages = Vmx86_GetNumAnonPages(vm);
+      retval = HostIF_CopyToUser(ioarg, &numAnonPages, sizeof numAnonPages);
+      break;
+   }
+
   case IOCTL_VMX86_READ_PAGE: {
         VMMReadWritePage req;

@@ -1530,6 +1533,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
+#ifdef VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1538,6 +1542,9 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
         }
         retval = HostIF_WritePhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE,
                                       PAGE_SIZE);
+#else
+         retval = -EINVAL;
+#endif
         break;
      }

@@ -1596,7 +1603,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      uint64 swapSize;
      retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize);
      if (retval != 0) {
-         Warning("Could not copy swap size from user, status %d\n", retval);
+         Warning("Could not copy swap size from user, status %ld\n", retval);
         break;
      }
      linuxState.swapSize = swapSize;
@@ -1629,6 +1636,11 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

 exit:
+   if (needsWriteLock) {
+      up_write(&device->vmDriverRWSema);
+   } else {
+      up_read(&device->vmDriverRWSema);
+   }
   return retval;
 }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,27 +27,14 @@
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
+#include <linux/rwsem.h>
 #include <linux/wait.h>

 #include "vmx86.h"
 #include "driver_vmcore.h"


-/*
- * Used to track a piece of memory that's been mapped into the kernel from
- * userlevel.
- */
-typedef struct VMMappedUserMem {
-   void *addr;
-   size_t numPages;
-   struct page *pages[0];
-} VMMappedUserMem;
-
-
-/*
- * Per-instance driver state
- */
-
+/* Per-instance driver state */
 struct VMDriver;

 /* 16 pages (64KB) looks as a good limit for one allocation */
@@ -56,7 +43,15 @@ struct VMDriver;
 typedef struct Device {
   struct Device   *next;
   struct VMDriver *vm;
-
+   /*
+    * This RW semaphore protects accesses to the VMDriver to
+    * avoid racing between various ioctls, and the creation
+    * and removal of the VM in question. The lock is read-acquired
+    * by ioctls that reference the VMDriver, and write-acquired by
+    * ioctls or device callbacks that allocate or destory the
+    * VMDriver.
+    */
+   struct rw_semaphore vmDriverRWSema;
   /*
    * The semaphore protect accesses to size4Gb and pages4Gb
    * in mmap(). mmap() may happen only once, and all other
@@ -64,7 +59,7 @@ typedef struct Device {
    * only after successful mmap.
    */
   struct semaphore lock4Gb;
-   unsigned int size4Gb;
+   PageCnt size4Gb;
   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
 } Device;

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,7 +63,7 @@
 #include "memtrack.h"
 #include "phystrack.h"
 #include "cpuid.h"
-#include "cpuid_info.h"
+#include "x86cpuid_asm.h"
 #include "hostif.h"
 #include "hostif_priv.h"
 #include "vmhost.h"
@@ -74,6 +74,7 @@

 #include "pgtbl.h"
 #include "versioned_atomic.h"
+#include "compat_poll.h"

 #if !defined(CONFIG_HIGH_RES_TIMERS)
 #error CONFIG_HIGH_RES_TIMERS required for acceptable performance
@@ -787,21 +788,21 @@ HostIF_FreeMachinePage(MPN mpn)  // IN:
 *----------------------------------------------------------------------
 */

-int
+int64
 HostIF_AllocLockedPages(VMDriver *vm,         // IN: VM instance pointer
                        VA64 addr,            // OUT: buffer address
-                        unsigned numPages,    // IN: number of pages to allocate
+                        PageCnt numPages,     // IN: number of pages to allocate
                        Bool kernelMPNBuffer) // IN: kernel vs user space
 {
   VMHost *vmh = vm->vmhost;
-   unsigned int cnt;
-   int err = 0;
+   PageCnt cnt;
+   int64 err = 0;

   if (!vmh || !vmh->AWEPages) {
      return -EINVAL;
   }
   for (cnt = 0; cnt < numPages; cnt++) {
-      struct page* pg;
+      struct page *pg;
      MPN mpn;

      pg = alloc_page(GFP_HIGHUSER);
@@ -848,66 +849,38 @@ HostIF_AllocLockedPages(VMDriver *vm,         // IN: VM instance pointer

 int
 HostIF_FreeLockedPages(VMDriver *vm,         // IN: VM instance pointer
-                       VA64 addr,            // IN: array of MPNs
-                       unsigned numPages,    // IN: number of pages to free
-                       Bool kernelMPNBuffer) // IN: kernel vs user address
+                       MPN *mpns,            // IN: array of MPNs
+                       PageCnt numPages)     // IN: number of pages to free
 {
-   const int MPN_BATCH = 64;
-   MPN const *pmpn = VA64ToPtr(addr);
   VMHost *vmh = vm->vmhost;
-   unsigned int cnt;
+   PageCnt cnt;
   struct page *pg;
-   MPN *mpns;

-   mpns = HostIF_AllocKernelMem(sizeof *mpns * MPN_BATCH, TRUE);
-
-   if (mpns == NULL) {
-      return -ENOMEM;
-   }
   if (!vmh || !vmh->AWEPages) {
-      HostIF_FreeKernelMem(mpns);
      return -EINVAL;
   }

-   if (!kernelMPNBuffer) {
-      if (numPages > MPN_BATCH) {
-         HostIF_FreeKernelMem(mpns);
-         return -EINVAL;
-      }
-
-      if (HostIF_CopyFromUser(mpns, addr, numPages * sizeof *pmpn)) {
-         printk(KERN_DEBUG "Cannot read from process address space at %p\n",
-                pmpn);
-         HostIF_FreeKernelMem(mpns);
-         return -EINVAL;
-      }
-
-      pmpn = mpns;
-   }
-
   for (cnt = 0; cnt < numPages; cnt++) {
-      if (!PhysTrack_Test(vmh->AWEPages, pmpn[cnt])) {
+      if (!PhysTrack_Test(vmh->AWEPages, mpns[cnt])) {
         printk(KERN_DEBUG "Attempted to free unallocated MPN %016" FMT64 "X\n",
-                pmpn[cnt]);
-         HostIF_FreeKernelMem(mpns);
+                mpns[cnt]);
         return -EINVAL;
      }

-      pg = pfn_to_page(pmpn[cnt]);
+      pg = pfn_to_page(mpns[cnt]);
      if (page_count(pg) != 1) {
         // should this case be considered a failure?
         printk(KERN_DEBUG "Page %016" FMT64 "X is still used by someone "
-                "(use count %u, VM %p)\n", pmpn[cnt],
+                "(use count %u, VM %p)\n", mpns[cnt],
                 page_count(pg), vm);
      }
   }

   for (cnt = 0; cnt < numPages; cnt++) {
-      pg = pfn_to_page(pmpn[cnt]);
-      PhysTrack_Remove(vmh->AWEPages, pmpn[cnt]);
+      pg = pfn_to_page(mpns[cnt]);
+      PhysTrack_Remove(vmh->AWEPages, mpns[cnt]);
      __free_page(pg);
   }
-   HostIF_FreeKernelMem(mpns);
   return 0;
 }

@@ -1042,7 +1015,7 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
 *      Initialize the host-dependent part of the driver.
 *
 * Results:
- *      zero on success, non-zero on error.
+ *      TRUE on success, FALSE on error.
 *
 * Side effects:
 *      None
@@ -1050,12 +1023,12 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
 *----------------------------------------------------------------------
 */

-int
+Bool
 HostIF_Init(VMDriver *vm, uint32 numVCPUs)
 {
   vm->memtracker = MemTrack_Init(vm);
   if (vm->memtracker == NULL) {
-      return -1;
+      return FALSE;
   }

   vm->vmhost = HostIFAllocVMHost(numVCPUs);
@@ -1068,7 +1041,7 @@ HostIF_Init(VMDriver *vm, uint32 numVCPUs)
   }
   MutexInit(&vm->vmhost->vmMutex, "vm");

-   return 0;
+   return TRUE;
 error:
   if (vm->vmhost != NULL) {
      HostIFHostMemCleanup(vm);
@@ -1079,7 +1052,7 @@ error:
      MemTrack_Cleanup(vm->memtracker, UnlockEntry, vm);
      vm->memtracker = NULL;
   }
-   return -1;
+   return FALSE;
 }


@@ -1248,11 +1221,7 @@ HostIF_LockPage(VMDriver *vm,                // IN: VMDriver
   vpn = PTR_2_VPN(uvAddr);
   if (!allowMultipleMPNsPerVA) {
      entryPtr = MemTrack_LookupVPN(vm->memtracker, vpn);
-
-      /*
-       * Already tracked and locked
-       */
-
+      /* Already tracked and locked */
      if (entryPtr != NULL && entryPtr->mpn != 0) {
         return PAGE_LOCK_ALREADY_LOCKED;
      }
@@ -1265,10 +1234,7 @@ HostIF_LockPage(VMDriver *vm,                // IN: VMDriver
   *mpn = (MPN)page_to_pfn(page);

   if (allowMultipleMPNsPerVA) {
-      /*
-       *  Add the MPN to the PhysTracker that tracks locked pages.
-       */
-
+      /* Add the MPN to the PhysTracker that tracks locked pages */
      struct PhysTracker* const pt = vm->vmhost->lockedPages;

      if (PhysTrack_Test(pt, *mpn)) {
@@ -1485,7 +1451,7 @@ HostIF_FreeAllResources(VMDriver *vm) // IN

 void *
 HostIF_AllocKernelMem(size_t size,  // IN:
-                      int wired)    // IN:
+                      Bool wired)   // IN:
 {
   void * ptr = kmalloc(size, GFP_KERNEL);

@@ -1629,19 +1595,21 @@ HostIF_UnmapPage(VPN vpn) // IN:
 *----------------------------------------------------------------------
 */

-unsigned int
+PageCnt
 HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
-                               unsigned int currentlyLockedPages) // IN
+                               PageCnt currentlyLockedPages)      // IN
 {
   /*
    * This variable is available and exported to modules,
    * since at least 2.6.0.
    */

+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
   extern unsigned long totalram_pages;
-
-   unsigned int totalPhysicalPages = totalram_pages;
-
+   PageCnt totalPhysicalPages = totalram_pages;
+#else
+   PageCnt totalPhysicalPages = totalram_pages();
+#endif
   /*
    * Use the memory information linux exports as of late for a more
    * precise estimate of locked memory.  All kernel page-related structures
@@ -1650,14 +1618,13 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
    * also as good as locked, since we don't use them.  Lastly, without
    * available swap, anonymous pages become locked in memory as well.
    */
-
-   unsigned int forHost;
-   unsigned int reservedPages = MEMDEFAULTS_MIN_HOST_PAGES;
-   unsigned int hugePages = (vm == NULL) ? 0 :
-      BYTES_2_PAGES(vm->memInfo.hugePageBytes);
-   unsigned int lockedPages = hugePages + reservedPages;
-   unsigned int anonPages;
-   unsigned int swapPages = BYTES_2_PAGES(linuxState.swapSize);
+   PageCnt forHost;
+   PageCnt reservedPages = MEMDEFAULTS_MIN_HOST_PAGES;
+   PageCnt hugePages = (vm == NULL) ? 0
+                                    : BYTES_2_PAGES(vm->memInfo.hugePageBytes);
+   PageCnt lockedPages = hugePages + reservedPages;
+   PageCnt anonPages;
+   PageCnt swapPages = BYTES_2_PAGES(linuxState.swapSize);

   /* global_page_state is global_zone_page_state in 4.14. */
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
@@ -1738,6 +1705,49 @@ HostIF_WaitForFreePages(unsigned int timeoutMs)  // IN:
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIFGetTime --
+ *
+ *      Reads the current time in UPTIME_FREQ units.
+ *
+ * Results:
+ *      The uptime, in units of UPTIME_FREQ.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static uint64
+HostIFGetTime(void)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+   struct timeval tv;
+
+   do_gettimeofday(&tv);
+   return tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ;
+#else
+   struct timespec64 now;
+
+   /*
+    * Use raw time used by Posix timers.  This time is not affected by
+    * NTP adjustments, so it may drift from real time and monotonic time,
+    * but it will stay in sync with other timers.
+    */
+   ktime_get_raw_ts64(&now);
+   /*
+    * UPTIME_FREQ resolution is lower than tv_nsec,
+    * so we have to do division...
+    */
+   ASSERT_ON_COMPILE(1000000000 % UPTIME_FREQ == 0);
+   return now.tv_nsec / (1000000000 / UPTIME_FREQ) + now.tv_sec * UPTIME_FREQ;
+#endif
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1769,7 +1779,6 @@ HostIF_WaitForFreePages(unsigned int timeoutMs)  // IN:
 static uint64
 HostIFReadUptimeWork(unsigned long *j)  // OUT: current jiffies
 {
-   struct timeval tv;
   uint64 monotime, uptime, upBase, monoBase;
   int64 diff;
   uint32 version;
@@ -1784,13 +1793,12 @@ HostIFReadUptimeWork(unsigned long *j)  // OUT: current jiffies
      monoBase = uptimeState.monotimeBase;
   } while (!VersionedAtomic_EndTryRead(&uptimeState.version, version));

-   do_gettimeofday(&tv);
+   uptime = HostIFGetTime();
   upBase = Atomic_Read64(&uptimeState.uptimeBase);

   monotime = (uint64)(jifs - jifBase) * (UPTIME_FREQ / HZ);
   monotime += monoBase;

-   uptime = tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ;
   uptime += upBase;

   /*
@@ -1895,13 +1903,11 @@ HostIFUptimeResyncMono(struct timer_list *timer)  // IN: ignored
 void
 HostIF_InitUptime(void)
 {
-   struct timeval tv;
+   uint64 tm;

   uptimeState.jiffiesBase = jiffies;
-   do_gettimeofday(&tv);
-   Atomic_Write64(&uptimeState.uptimeBase,
-                  -(tv.tv_usec * (UPTIME_FREQ / 1000000) +
-                    tv.tv_sec * UPTIME_FREQ));
+   tm = HostIFGetTime();
+   Atomic_Write64(&uptimeState.uptimeBase, -tm);

 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
   init_timer(&uptimeState.timer);
@@ -2113,10 +2119,10 @@ HostIF_MapCrossPage(VMDriver *vm, // IN
 */

 void *
-HostIF_AllocKernelPages(unsigned numPages, // IN: Number of pages
+HostIF_AllocKernelPages(PageCnt numPages,  // IN: Number of pages
                        MPN     *mpns)     // OUT: Array of MPNs
 {
-   unsigned i;
+   PageCnt i;
   void *ptr = vmalloc(numPages * PAGE_SIZE);

   if (ptr == NULL) {
@@ -2147,7 +2153,7 @@ HostIF_AllocKernelPages(unsigned numPages, // IN: Number of pages
 */

 void
-HostIF_FreeKernelPages(unsigned numPages, // IN: Number of pages
+HostIF_FreeKernelPages(PageCnt numPages,  // IN: Number of pages
                       void    *ptr)      // IN: Kernel VA of first page
 {
   vfree(ptr);
@@ -2285,7 +2291,7 @@ isVAReadable(VA r)  // IN:
   int ret;

   old_fs = get_fs();
-   set_fs(get_ds());
+   set_fs(KERNEL_DS);
   r = APICR_TO_ADDR(r, APICR_VERSION);
   ret = HostIF_CopyFromUser(&dummy, r, sizeof dummy);
   set_fs(old_fs);
@@ -2421,7 +2427,7 @@ HostIF_APICInit(VMDriver *vm,   // IN:
      apicIPILogged = TRUE;
   }

-   if ((__GET_MSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
      if (setVMPtr) {
         vm->hostAPIC.base = NULL;
         vm->vmhost->hostAPICIsMapped = FALSE;
@@ -2562,7 +2568,7 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   }

   old_fs = get_fs();
-   set_fs(get_ds());
+   set_fs(KERNEL_DS);

   {
      struct poll_wqueues table;
@@ -2570,7 +2576,7 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:

      poll_initwait(&table);
      current->state = TASK_INTERRUPTIBLE;
-      mask = file->f_op->poll(file, &table.pt);
+      mask = compat_vfs_poll(file, &table.pt);
      if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
         vm->vmhost->vcpuSemaTask[vcpuid] = current;
         schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
@@ -2647,8 +2653,12 @@ HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
                            const VCPUSet *vcs) // IN:
 {
   FOR_EACH_VCPU_IN_SET_WITH_MAX(vcs, vcpuid, vm->numVCPUs) {
-      struct task_struct *t = vm->vmhost->vcpuSemaTask[vcpuid];
-      vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
+      /*
+       * To avoid lost wake-up race and to avoid redundant wakeups, use
+       * "xchg" to read and clear atomically; wake_up_process() is non-lossy.
+       */
+      struct task_struct *t =
+         (struct task_struct *)xchg(&vm->vmhost->vcpuSemaTask[vcpuid], NULL);
      if (t && (t->state & TASK_INTERRUPTIBLE)) {
         wake_up_process(t);
      }
@@ -2691,7 +2701,7 @@ HostIF_SemaphoreSignal(uint64 *args)  // IN:
   }

   old_fs = get_fs();
-   set_fs(get_ds());
+   set_fs(KERNEL_DS);

   /*
    * Always write sizeof(uint64) bytes. This works fine for eventfd and
@@ -3142,6 +3152,26 @@ HostIF_GetNextAnonPage(VMDriver *vm, MPN inMPN)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * HostIF_GetNumAnonPages --
+ *
+ *      Returns the number of anon MPNs used by this VM.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+PageCnt
+HostIF_GetNumAnonPages(VMDriver *vm)
+{
+   if (!vm->vmhost || !vm->vmhost->AWEPages) {
+      return 0;
+   }
+   return PhysTrack_GetNumTrackedPages(vm->vmhost->AWEPages);
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -3359,7 +3389,6 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz.
      }
   } else {
      if (linuxState.fastClockThread) {
-         force_sig(SIGKILL, linuxState.fastClockThread);
         kthread_stop(linuxState.fastClockThread);

         linuxState.fastClockThread = NULL;
@@ -3369,140 +3398,6 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz.
   return 0;
 }

-
-/*
- *-----------------------------------------------------------------------------
- *
- * HostIF_MapUserMem --
- *
- *      Obtain kernel pointer to user memory. The pages backing the user memory
- *      address are locked into memory (this allows the pointer to be used in
- *      contexts where paging is undesirable or impossible).
- *
- * Results:
- *      On success, returns the kernel virtual address, along with a handle to
- *      be used for unmapping.
- *      On failure, returns NULL.
- *
- * Side effects:
- *      Yes.
- *
- *-----------------------------------------------------------------------------
- */
-
-void *
-HostIF_MapUserMem(VA addr,                  // IN: User memory virtual address
-                  size_t size,              // IN: Size of memory desired
-                  VMMappedUserMem **handle) // OUT: Handle to mapped memory
-{
-   void *p = (void *) (uintptr_t) addr;
-   VMMappedUserMem *newHandle;
-   VA offset = addr & (PAGE_SIZE - 1);
-   size_t numPagesNeeded = ((offset + size) / PAGE_SIZE) + 1;
-   size_t handleSize =
-      sizeof *newHandle + numPagesNeeded * sizeof newHandle->pages[0];
-   void *mappedAddr;
-
-   ASSERT(handle);
-
-   if (!access_ok(VERIFY_WRITE, p, size)) {
-      printk(KERN_ERR "%s: Couldn't verify write to uva 0x%p with size %"
-             FMTSZ"u\n", __func__, p, size);
-
-      return NULL;
-   }
-
-   newHandle = kmalloc(handleSize, GFP_KERNEL);
-   if (newHandle == NULL) {
-      printk(KERN_ERR "%s: Couldn't allocate %"FMTSZ"u bytes of memory\n",
-             __func__, handleSize);
-
-      return NULL;
-   }
-
-   if (HostIFGetUserPages(p, newHandle->pages, numPagesNeeded)) {
-      kfree(newHandle);
-      printk(KERN_ERR "%s: Couldn't get %"FMTSZ"u %s for uva 0x%p\n", __func__,
-             numPagesNeeded, numPagesNeeded > 1 ? "pages" : "page", p);
-
-      return NULL;
-   }
-
-   if (numPagesNeeded > 1) {
-      /*
-       * Unlike kmap(), vmap() can fail. If it does, we need to release the
-       * pages that we acquired in HostIFGetUserPages().
-       */
-
-      mappedAddr = vmap(newHandle->pages, numPagesNeeded, VM_MAP, PAGE_KERNEL);
-      if (mappedAddr == NULL) {
-         unsigned int i;
-         for (i = 0; i < numPagesNeeded; i++) {
-            put_page(newHandle->pages[i]);
-         }
-         kfree(newHandle);
-         printk(KERN_ERR "%s: Couldn't vmap %"FMTSZ"u %s for uva 0x%p\n",
-                __func__, numPagesNeeded,
-                numPagesNeeded > 1 ? "pages" : "page", p);
-
-         return NULL;
-      }
-   } else {
-      mappedAddr = kmap(newHandle->pages[0]);
-   }
-
-   printk(KERN_DEBUG "%s: p = 0x%p, offset = 0x%p, numPagesNeeded = %"FMTSZ"u,"
-          " handleSize = %"FMTSZ"u, mappedAddr = 0x%p\n",
-          __func__, p, (void *)offset, numPagesNeeded, handleSize, mappedAddr);
-
-   newHandle->numPages = numPagesNeeded;
-   newHandle->addr = mappedAddr;
-   *handle = newHandle;
-
-   return mappedAddr + offset;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * HostIF_UnmapUserMem --
- *
- *      Unmap user memory from HostIF_MapUserMem().
- *
- * Results:
- *      None.
- *
- * Side effects:
- *      Yes.
- *
- *-----------------------------------------------------------------------------
- */
-
-void
-HostIF_UnmapUserMem(VMMappedUserMem *handle) // IN: Handle to mapped memory
-{
-   unsigned int i;
-
-   if (handle == NULL) {
-      return;
-   }
-
-   printk(KERN_DEBUG "%s: numPages = %"FMTSZ"u, addr = 0x%p\n",
-          __func__, handle->numPages, handle->addr);
-
-   if (handle->numPages > 1) {
-      vunmap(handle->addr);
-   } else {
-      kunmap(handle->pages[0]);
-   }
-
-   for (i = 0; i < handle->numPages; i++) {
-      put_page(handle->pages[i]);
-   }
-   kfree(handle);
-}
-
 /*
 *-----------------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,12 +25,6 @@
 #ifndef _HOSTIF_PRIV_H_
 #define _HOSTIF_PRIV_H_

-/* Functions for mapping and unmapping userspace memory. */
-struct VMMappedUserMem;
-EXTERN void  *HostIF_MapUserMem(VA addr, size_t size,
-                                struct VMMappedUserMem **handle);
-EXTERN void   HostIF_UnmapUserMem(struct VMMappedUserMem *handle);
-
 /* Uptime-related functions. */
 EXTERN void   HostIF_InitUptime(void);
 EXTERN void   HostIF_CleanupUptime(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,7 +24,7 @@
 *
 */

-#if defined(linux)
+#if defined(__linux__)
 /* Must come before any kernel header file */
 #   include "driver-config.h"
 #   include <linux/kernel.h>
@@ -34,7 +34,6 @@
 #include "modulecall.h"
 #include "vmx86.h"
 #include "task.h"
-#include "initblock.h"
 #include "vm_basic_asm.h"
 #include "iocontrols.h"
 #include "hostif.h"
@@ -43,6 +42,7 @@
 #include "usercalldefs.h"
 #include "cpuid.h"
 #include "vmmblob.h"
+#include "sharedAreaVmmon.h"

 /*
 *----------------------------------------------------------------------
@@ -72,13 +72,15 @@ int
 Vmx86_RunVM(VMDriver *vm,   // IN:
            Vcpuid vcpuid)  // IN:
 {
-   uint32           retval    = MODULECALL_USERRETURN;
+   uint64           retval    = MODULECALL_USERRETURN;
   VMCrossPageData *crosspage;
   int              bailValue = 0;

-   ASSERT(vcpuid < vm->numVCPUs && CPUID_HostSupportsHV());
+   ASSERT(vcpuid < vm->numVCPUs);
+   if (vm->crosspage[vcpuid] == NULL) {
+      return USERCALL_VMX86ALLOCERR;
+   }
   crosspage = &vm->crosspage[vcpuid]->crosspageData;
-   ASSERT(crosspage);

   /*
    * Check if we were interrupted by signal.
@@ -123,7 +125,8 @@ skipTaskSwitch:;
          * from the ioctl (back to the userlevel side of a VCPU thread).
          */
         bailValue = crosspage->userCallType;
-         crosspage->retval = retval;
+         ASSERT(retval == (uint64)((uint32)retval));
+         crosspage->retval = (uint32)retval;
         goto bailOut;
      }

@@ -136,18 +139,19 @@ skipTaskSwitch:;

      case MODULECALL_GET_RECYCLED_PAGES: {
         MPN mpns[MODULECALL_NUM_ARGS];
-         int nPages = MIN((int)crosspage->args[0], MODULECALL_NUM_ARGS);
-
-         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages, TRUE,
-                                         FALSE);
+         PageCnt nPages = MIN(crosspage->args[0], MODULECALL_NUM_ARGS);
+         ASSERT((int64)crosspage->args[0] >= 0);
+         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages,
+                                         TRUE, FALSE);
         if (retval <= nPages) {
-            int i;
+            PageCnt i;
            for (i = 0; i < retval; i++) {
               crosspage->args[i] = mpns[i];
            }
         } else {
-            // retval is holding an error code
-            Warning("Failed to alloc %u pages: %d\n", nPages, (int)retval);
+            /* retval is holding an error code */
+            Warning("Failed to alloc %"FMT64"u pages: %"FMT64"d\n", nPages,
+                    (int64)retval);
            retval = 0;
         }
         break;
@@ -197,7 +201,7 @@ skipTaskSwitch:;
      }

      case MODULECALL_RELEASE_ANON_PAGES: {
-         unsigned count;
+         PageCnt count;
         MPN mpns[MODULECALL_NUM_ARGS];
         for (count = 0; count < MODULECALL_NUM_ARGS; count++) {
            mpns[count] = (MPN)crosspage->args[count];
@@ -206,15 +210,14 @@ skipTaskSwitch:;
            }
         }
         ASSERT(count > 0);
-         retval = Vmx86_FreeLockedPages(vm, PtrToVA64(mpns), count, TRUE);
+         retval = Vmx86_FreeLockedPages(vm, mpns, count);
         break;
      }

      case MODULECALL_LOOKUP_MPN: {
-         int i;
-         VPN64  vpn    = (VPN64)crosspage->args[0];
-         uint32 nPages = (uint32)crosspage->args[1];
-         VA64   uAddr  = (VA64)VPN_2_VA(vpn);
+         VPN64   vpn        = (VPN64)crosspage->args[0];
+         PageCnt i, nPages  = crosspage->args[1];
+         VA64    uAddr      = (VA64)VPN_2_VA(vpn);
         ASSERT(nPages <= MODULECALL_NUM_ARGS);
         HostIF_VMLock(vm, 38);
         for (i = 0; i < nPages; i++) {
@@ -292,17 +295,36 @@ skipTaskSwitch:;
         vm->blobInfo = NULL;
      } break;

+      case MODULECALL_GET_SHARED_AREA: {
+         SharedAreaVmmonRequest request;
+         request.type       = (SharedAreaType)crosspage->args[0];
+         request.vcpu       = (Vcpuid)crosspage->args[1];
+         request.offset     = (PageCnt)crosspage->args[2];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[3] = SharedAreaVmmon_GetRegionMPN(vm, &request);
+      } break;
+
+      case MODULECALL_GET_STAT_VARS: {
+         Vcpuid  vcpu   = (Vcpuid)crosspage->args[0];
+         PageCnt offset = (PageCnt)crosspage->args[1];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[2] = StatVarsVmmon_GetRegionMPN(vm, vcpu, offset);
+      } break;
+
+      case MODULECALL_GET_NUM_PTP_PAGES: {
+         /* Store PageCnt in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[1] = vm->numPTPPages;
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
-
-      crosspage->retval = retval;
-
-#if defined(linux)
+      ASSERT(retval == (uint64)((uint32)retval));
+      crosspage->retval = (uint32)retval;
+#if defined(__linux__)
      cond_resched(); // Other kernels are preemptable
 #endif
   }
-
 bailOut:
   return bailValue;
 }
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -32,6 +32,6 @@ $(DRIVER)-y := driver.o hub.o userif.o netif.o bridge.o procfs.o smac_compat.o \
 ####

 clean:
-	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko \
+	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko .cache.mk \
 	       .tmp_versions Module.symvers Modules.symvers Module.markers \
 	       modules.order $($(DRIVER)-y) .*.cmd .*.o.flags
@@ -67,6 +67,18 @@

 #if LOGLEVEL >= 4
 static struct timeval vnetTime;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)
+static void do_gettimeofday(struct timeval *tv)
+{
+	struct timespec64 now;
+
+	ktime_get_real_ts64(&now);
+	tv->tv_sec = now.tv_sec;
+	tv->tv_usec = now.tv_nsec / 1000;
+}
+#endif
+
 #endif

 typedef struct VNetBridge VNetBridge;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,8 @@

 #define ETHER_MAX_QUEUED_PACKET 1600

+/* Most Ethernet equipment can support jumbo frames up to 9216 bytes. */
+#define ETHER_MAX_JUMBO_FRAME_LEN 9216

 /*
 * State's that a NIC can be in currently we only use this
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -28,6 +28,7 @@

 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
 #include <linux/mm.h>
 #include "compat_skbuff.h"
 #include <linux/sockios.h>
@@ -44,6 +45,22 @@
 #include "vmnetInt.h"


+/*
+ * Default min MTU value as defined by kernel versions >= 4.10.0.
+ * Use the same value for earlier versions of the kernel which do not
+ * enforce a minimum MTU size.
+ */
+#define VMNET_MIN_MTU 68
+#define VMNET_MAX_MTU (ETHER_MAX_JUMBO_FRAME_LEN - \
+                      (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
+
+#ifdef RHEL_RELEASE_CODE
+#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 5) && \
+     RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8, 0))
+#define ndo_change_mtu ndo_change_mtu_rh74
+#endif
+#endif /* RHEL_RELEASE_CODE */
+
 typedef struct VNetNetIF {
   VNetPort                port;
   struct net_device      *dev;
@@ -64,6 +81,11 @@ static int  VNetNetifSetMAC(struct net_device *dev, void *addr);
 static void VNetNetifSetMulticast(struct net_device *dev);
 static int  VNetNetIfProcRead(char *page, char **start, off_t off,
                              int count, int *eof, void *data);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+static int VNetNetifChangeMtu(struct net_device *dev, int new_mtu);
+#endif
+

 /*
 *----------------------------------------------------------------------
@@ -84,7 +106,7 @@ static int  VNetNetIfProcRead(char *page, char **start, off_t off,
 static void
 VNetNetIfSetup(struct net_device *dev)  // IN:
 {
-#if !COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) || defined(HAVE_NET_DEVICE_OPS)
+#ifdef HAVE_NET_DEVICE_OPS
   static const struct net_device_ops vnetNetifOps = {
      .ndo_init = VNetNetifProbe,
      .ndo_open = VNetNetifOpen,
@@ -97,12 +119,23 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
 #else
      .ndo_set_rx_mode = VNetNetifSetMulticast,
 #endif
-
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
+      .ndo_change_mtu = VNetNetifChangeMtu,
+#endif
   };
+#endif /* HAVE_NET_DEVICE_OPS */
+
+   /* Turns on IFF_BROADCAST, IFF_MULTICAST. */
+   ether_setup(dev);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+   /* Set the maximum allowed MTU value. */
+   dev->max_mtu = VMNET_MAX_MTU;
 #endif

-   ether_setup(dev); // turns on IFF_BROADCAST, IFF_MULTICAST
-#if COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) && !defined(HAVE_NET_DEVICE_OPS) 
+#ifdef HAVE_NET_DEVICE_OPS
+   dev->netdev_ops = &vnetNetifOps;
+#else
   dev->init = VNetNetifProbe;
   dev->open = VNetNetifOpen;
   dev->hard_start_xmit = VNetNetifStartXmit;
@@ -110,9 +143,10 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
   dev->get_stats = VNetNetifGetStats;
   dev->set_mac_address = VNetNetifSetMAC;
   dev->set_multicast_list = VNetNetifSetMulticast;
-#else
-   dev->netdev_ops = &vnetNetifOps;
-   #endif /* HAVE_NET_DEVICE_OPS */
+#ifdef HAVE_CHANGE_MTU
+   dev->change_mtu = VNetNetifChangeMtu;
+#endif /* HAVE_CHANGE_MTU */
+#endif /* HAVE_NET_DEVICE_OPS */

 }

@@ -530,6 +564,38 @@ VNetNetifSetMulticast(struct net_device *dev) // IN: unused
 }


+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+/*
+ *----------------------------------------------------------------------
+ *
+ * VNetNetifChangeMtu --
+ *
+ *      Changes the current MTU value of a given vmnet interface.
+ *
+ * Results:
+ *      Returns zero on success, or invalid argument error if an incorrect
+ *      MTU size is being set.
+ *
+ * Side effects:
+ *      None.
+ *----------------------------------------------------------------------
+ */
+
+int
+VNetNetifChangeMtu(struct net_device *dev, // IN/OUT: assigning the mtu member.
+                   int new_mtu)            // IN:     new mtu value.
+{
+   if (new_mtu < VMNET_MIN_MTU || new_mtu > VMNET_MAX_MTU) {
+      return -EINVAL;
+   }
+
+   dev->mtu = new_mtu;
+   return 0;
+}
+#endif
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,8 +41,6 @@

 #else /* _WIN32 */

-#undef __KERNEL__ //To prevent including any kernel specific stuff 
-
 #ifdef VMX86_DEVEL
 #define DBG 1
 #else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -78,13 +78,33 @@ static int  VNetUserIfSetUplinkState(VNetPort *port, uint8 linkUp);
 extern unsigned int  vnet_max_qlen;

 #if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
-#   define compat_kmap(page) kmap(page)
-#   define compat_kunmap(page) kunmap(page)
+#   define compat_kmap_frag(frag) kmap((frag)->page)
+#   define compat_kunmap_frag(page) kunmap((frag)->page)
 #else
-#   define compat_kmap(page) kmap((page).p)
-#   define compat_kunmap(page) kunmap((page).p)
+#   define compat_kmap_frag(frag) kmap(skb_frag_page(frag))
+#   define compat_kunmap_frag(frag) kunmap(skb_frag_page(frag))
 #endif

+static unsigned int compat_skb_frag_size(const skb_frag_t *frag)
+{
+#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
+	return frag->size;
+#else
+	return skb_frag_size(frag);
+#endif
+}
+
+static unsigned int compat_skb_frag_off(const skb_frag_t *frag)
+{
+#if COMPAT_LINUX_VERSION_CHECK_LT(5, 4, 0) && \
+	!(defined(CONFIG_SUSE_VERSION) && CONFIG_SUSE_VERSION == 15 && \
+	  defined(CONFIG_SUSE_PATCHLEVEL) && CONFIG_SUSE_PATCHLEVEL >= 2)
+	return frag->page_offset;
+#else
+	return skb_frag_off(frag);
+#endif
+}
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -137,16 +157,21 @@ UserifLockPage(VA addr) // IN
 */

 static INLINE int
-VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapPtr(VA64 uAddr,      // IN: pointer to user memory
                 size_t size,     // IN: size of data
                 struct page **p, // OUT: locked page
                 void **ptr)      // OUT: kernel mapped pointer
 {
-   if (!access_ok(VERIFY_WRITE, (void *)uAddr, size) ||
-       (((uAddr + size - 1) & ~(PAGE_SIZE - 1)) !=
-        (uAddr & ~(PAGE_SIZE - 1)))) {
+   uint8 v;
+
+   /* Check area does not straddle two pages. */
+   if ((uAddr & (PAGE_SIZE - 1)) + size > PAGE_SIZE) {
      return -EINVAL;
   }
+   /* Check if it is user's area.  UserifLockPage() checks writability. */
+   if (copy_from_user(&v, (void *)(unsigned long)uAddr, sizeof v) != 0) {
+      return -EFAULT;
+   }

   *p = UserifLockPage(uAddr);
   if (*p == NULL) {
@@ -158,7 +183,7 @@ VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
 }

 static INLINE int
-VNetUserIfMapUint32Ptr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapUint32Ptr(VA64 uAddr,      // IN: pointer to user memory
                       struct page **p, // OUT: locked page
                       uint32 **ptr)    // OUT: kernel mapped pointer
 {
@@ -201,7 +226,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      return -EBUSY;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->pollPtr, &pollPage,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->pollPtr, &pollPage,
                                        &pollPtr)) < 0) {
      return retval;
   }
@@ -213,7 +238,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      goto error_free;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->recvClusterPtr,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->recvClusterPtr,
                                        &recvClusterPage,
                                        &recvClusterCount)) < 0) {
      goto error_free;
@@ -362,12 +387,12 @@ VNetUserIfReceive(VNetJack       *this, // IN
   VNetUserIF *userIf = (VNetUserIF*)this->private;
   uint8 *dest = SKB_2_DESTMAC(skb);
   unsigned long flags;
-   
+
   if (!UP_AND_RUNNING(userIf->port.flags)) {
      userIf->stats.droppedDown++;
      goto drop_packet;
   }
-   
+
   if (!VNetPacketMatch(dest,
                        userIf->port.paddr,
                        (const uint8 *)userIf->port.exactFilter,
@@ -377,13 +402,17 @@ VNetUserIfReceive(VNetJack       *this, // IN
      userIf->stats.droppedMismatch++;
      goto drop_packet;
   }
-   
+
   if (skb_queue_len(&userIf->packetQueue) >= vnet_max_qlen) {
      userIf->stats.droppedOverflow++;
      goto drop_packet;
   }
-   
-   if (skb->len > ETHER_MAX_QUEUED_PACKET) {
+
+   /*
+    * The check would be more accurate if based on the current MTU value
+    * of the corresponding vmnet interface. PR 2267716 is filed to track this.
+    */
+   if (skb->len > ETHER_MAX_JUMBO_FRAME_LEN) {
      userIf->stats.droppedLargePacket++;
      goto drop_packet;
   }
@@ -559,20 +588,21 @@ VNetCsumCopyDatagram(const struct sk_buff *skb,	// IN: skb to copy
   for (frag = skb_shinfo(skb)->frags;
 	frag != skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
 	frag++) {
-      if (frag->size > 0) {
+      if (compat_skb_frag_size(frag) > 0) {
 	 unsigned int tmpCsum;
 	 const void *vaddr;

-	 vaddr = compat_kmap(frag->page);
-	 tmpCsum = csum_and_copy_to_user(vaddr + frag->page_offset,
-					 curr, frag->size, 0, &err);
-	 compat_kunmap(frag->page);
+	 vaddr = compat_kmap_frag(frag);
+	 tmpCsum = csum_and_copy_to_user(vaddr + compat_skb_frag_off(frag),
+					 curr, compat_skb_frag_size(frag), 0,
+					 &err);
+	 compat_kunmap_frag(frag);

 	 if (err) {
 	    return err;
 	 }
 	 csum = csum_block_add(csum, tmpCsum, curr - buf);
-	 curr += frag->size;
+	 curr += compat_skb_frag_size(frag);
      }
   }

@@ -747,7 +777,7 @@ VNetUserIfRead(VNetPort    *port, // IN
 *----------------------------------------------------------------------
 */

-static int 
+static int
 VNetUserIfWrite(VNetPort    *port, // IN
                struct file *filp, // IN
                const char  *buf,  // IN
@@ -757,11 +787,14 @@ VNetUserIfWrite(VNetPort    *port, // IN
   struct sk_buff *skb;

   /*
-    * Check size
+    * Check size.
+    *
+    * Regarding the upper count limit the check would be more accurate
+    * if based on the current MTU value of the corresponding vmnet interface.
+    * PR 2267716 is filed to track this.
    */
-   
-   if (count < sizeof (struct ethhdr) || 
-       count > ETHER_MAX_QUEUED_PACKET) {
+   if (count < sizeof (struct ethhdr) ||
+       count > ETHER_MAX_JUMBO_FRAME_LEN) {
      return -EINVAL;
   }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -202,7 +202,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1317,7 +1317,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
+   __asm lock And [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_And
@@ -1382,7 +1382,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
+   __asm lock Or [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Or
@@ -1447,7 +1447,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Xor
@@ -2303,7 +2303,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2321,7 +2324,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -(int64)val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -3618,7 +3621,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -221,8 +221,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +263,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +376,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +392,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -768,12 +771,14 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
@@ -888,7 +893,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +907,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1170,7 +1103,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1161,101 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,12 +49,29 @@ uint64 _umul128(uint64 multiplier, uint64 multiplicand,
 int64 _mul128(int64 multiplier, int64 multiplicand,
              int64 *highProduct);
 uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
+#ifdef ULM
+void _fxsave64(void *save);
+void _fxsave(void *save);
+void _fxrstor64(const void *load);
+void _fxrstor(const void *load);
+void _xsave64(void *save, uint64 mask);
+void _xsave(void *save, uint64 mask);
+void _xsaveopt64(void *save, uint64 mask);
+void _xsavec(void *save, uint64 mask);
+void _xrstor64(const void *load, uint64 mask);
+void _xrstor(const void *load, uint64 mask);
+#endif /* ULM */
 #ifdef __cplusplus
 }
 #endif

 #pragma intrinsic(_umul128, _mul128, __shiftright128)

+#ifdef ULM
+#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
+                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
+#endif /* ULM */
+
 #endif // _MSC_VER

 #if defined(__GNUC__)
@@ -113,40 +130,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +204,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +307,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +351,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +433,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +518,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +542,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -244,9 +244,131 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+#if defined VM_X86_32
+   __asm sfence;
+#else
+   _mm_sfence();
+#endif
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +400,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +515,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,47 +533,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed.
- * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc", "memory");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -427,18 +613,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -44,12 +44,44 @@
 #endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -58,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -159,18 +157,6 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

-/*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-

 /*
 * Token concatenation
@@ -209,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -255,6 +241,10 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif
@@ -286,6 +276,14 @@ Max(int a, int b)
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -556,10 +554,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -664,6 +662,20 @@ typedef int pid_t;
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
@@ -702,12 +714,6 @@ typedef int pid_t;
 #endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -718,7 +724,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -63,18 +63,9 @@
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/*
- * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -83,7 +74,6 @@
 #ifdef __x86_64__
 #define VM_X86_64
 #define vm_x86_64 1
-#define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
 #else
@@ -120,14 +110,91 @@

 #endif

-#if defined(__cplusplus) && __cplusplus >= 201103L || \
-    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
-    defined(__APPLE__) || defined(HAVE_STDINT_H)
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif
+
 /*
- * We're using <stdint.h> instead of <cstdint> below because some C++ code
- * deliberately compiles without C++ include paths.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif
+

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -138,40 +205,6 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-#else /* !HAVE_STDINT_H */
-
-/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
-#ifdef _MSC_VER
-
-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
-
-#elif __GNUC__
-#   if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#   else
-/*
- * Only strict c90 (without extensions) lacks a 'long long' type.
- * If this declaration fails ... use -std=c99 or -std=gnu90.
- */
-typedef unsigned long long uint64;
-typedef long long int64;
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-

 /*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
@@ -202,14 +235,6 @@ typedef char           Bool;
 #define IS_BOOL(x)     (((x) & ~1) == 0)


-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */
-
 /*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
@@ -218,9 +243,6 @@ typedef char           Bool;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -234,14 +256,11 @@ typedef char           Bool;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -249,46 +268,10 @@ typedef char           Bool;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -337,10 +320,11 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* macOS hosts use the same formatters for 32- and 64-bit. */
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
   #define FMT64         "ll"
-   #if KERNEL
+   #if defined(__APPLE__) && KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
@@ -452,7 +436,6 @@ typedef uintptr_t VPN;

 typedef uint64    PA;
 typedef uint64    PPN;
-typedef uint32    PPNTMP;

 typedef uint64    TPA;
 typedef uint64    TPPN;
@@ -470,9 +453,7 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint64    PgCnt64;
 typedef uint64    PageCnt;
-typedef uint64    PgNum64;
 typedef uint64    PageNum;
 typedef unsigned  MemHandle;
 typedef unsigned  IoHandle;
@@ -515,7 +496,7 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
@@ -598,15 +579,14 @@ typedef void * UserVA;


 /* Maximal observable PPN value. */
-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
 #define INVALID_PPN32     ((PPN32)0xffffffff)
-#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
-#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -624,7 +604,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((uint32)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -837,7 +818,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -934,82 +915,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2005-2012,2014-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2005-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,8 @@
 #endif
 #endif

+#include <linux/pci_ids.h>
+
 /* LSILogic 53C1030 Parallel SCSI controller
 * LSILogic SAS1068 SAS controller
 */
@@ -53,7 +55,11 @@
 *    VMware HD Audio codec
 *    VMware HD Audio controller
 */
+#ifndef PCI_VENDOR_ID_VMWARE
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
+#endif
+
+#define PCI_DEVICE_ID_VMWARE_SVGA3              0x0406
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
 #define PCI_DEVICE_ID_VMWARE_VGA                0x0711
@@ -70,7 +76,11 @@
 #define PCI_DEVICE_ID_VMWARE_1394               0x0780
 #define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
 #define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
+
+#ifndef PCI_DEVICE_ID_VMWARE_VMXNET3
 #define PCI_DEVICE_ID_VMWARE_VMXNET3            0x07B0
+#endif
+
 #define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
 #define PCI_DEVICE_ID_VMWARE_82574              0x07D0
 #define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
@@ -138,23 +148,24 @@
 *    Intel 82545EM (e1000, server adapter, single port)
 *    Intel 82546EB (e1000, server adapter, dual port)
 *    Intel HECI (as embedded in ich9m)
- *    Intel XHCI (Panther Point / Intel 7 Series)
+ *    Intel XHCI (Panther Point / Intel 7 Series, 5Gbps)
+ *    Intel XHCI (Cannon Lake / Intel 300 Series, 10Gbps)
 */
-#define PCI_VENDOR_ID_INTEL             0x8086
-#define PCI_DEVICE_ID_INTEL_82439TX     0x7100
-#define PCI_DEVICE_ID_INTEL_82371AB_0   0x7110
-#define PCI_DEVICE_ID_INTEL_82371AB_2   0x7112
-#define PCI_DEVICE_ID_INTEL_82371AB_3   0x7113
-#define PCI_DEVICE_ID_INTEL_82371AB     0x7111
-#define PCI_DEVICE_ID_INTEL_82443BX     0x7190
-#define PCI_DEVICE_ID_INTEL_82443BX_1   0x7191
-#define PCI_DEVICE_ID_INTEL_82443BX_2   0x7192 /* Used when no AGP support */
-#define PCI_DEVICE_ID_INTEL_82545EM     0x100f
-#define PCI_DEVICE_ID_INTEL_82546EB     0x1010
-#define PCI_DEVICE_ID_INTEL_82574       0x10d3
-#define PCI_DEVICE_ID_INTEL_82574_APPLE 0x10f6
-#define PCI_DEVICE_ID_INTEL_HECI        0x2a74
+#define PCI_VENDOR_ID_INTEL                   0x8086
+#define PCI_DEVICE_ID_INTEL_82439TX           0x7100
+#define PCI_DEVICE_ID_INTEL_82371AB_0         0x7110
+#define PCI_DEVICE_ID_INTEL_82371AB_2         0x7112
+#define PCI_DEVICE_ID_INTEL_82371AB_3         0x7113
+#define PCI_DEVICE_ID_INTEL_82371AB           0x7111
+#define PCI_DEVICE_ID_INTEL_82443BX           0x7190
+#define PCI_DEVICE_ID_INTEL_82443BX_1         0x7191
+#define PCI_DEVICE_ID_INTEL_82443BX_2         0x7192 /* Used when no AGP support */
+#define PCI_DEVICE_ID_INTEL_82545EM           0x100f
+#define PCI_DEVICE_ID_INTEL_82546EB           0x1010
+#define PCI_DEVICE_ID_INTEL_82574             0x10d3
+#define PCI_DEVICE_ID_INTEL_82574_APPLE       0x10f6
 #define PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI 0x1e31
+#define PCI_DEVICE_ID_INTEL_CANNONLAKE_XHCI   0xa36d

 /*
 *  From drivers/usb/host/xhci-pci.c:
@@ -347,6 +358,9 @@
 /************* QAT implementation limits ********************/
 #define MAX_QAT_PCI_DEVICES 4

+/************* PrecisionClock implementation limits ********************/
+#define MAX_PRECISIONCLOCK_DEVICES 1
+
 /************* Strings for Host USB Driver *******************************/

 #ifdef _WIN32
Author	SHA1	Message	Date
Michal Kubecek	41bfaa9de7	vmnet: work around skb_frag_t build failure for SLE15-SP2 and openSUSE-15.2 The kernels used in (upcoming) SLE15 SP2 and openSUSE Leap 15.2 are based on upstream version 5.3 but have a backport of the skb_frag_t conversion to bio_vec which reached mainline in v5.4-rc1. As we cannot easily check availability compat_skb_frag_off() (which was also added in v5.4-rc1) and have to resort to version check, add an exception for SLE15-SP2 and openSUSE-15.2 kernels.	2019-11-25 08:19:03 +01:00
Michal Kubecek	50d6105ced	vmnet: handle switch of skb_frag_t to bio_vec The switch from custom skb_frag_t implementation to bio_vec in v5.4-rc1 is mostly transparent for modules which use accessor for skb_frag_t members. Unfortunately many users access the members directly and function VNetCsumCopyDatagram() in vmnet is one of those. Use accessors everywhere so that vmnet code is compatible with kernel 5.4 and newer. Use "compat_" prefix to avoid clashes with backports adding the accessors to older codebase.	2019-11-14 08:06:57 +01:00
Michal Kubecek	525d443329	vmmon: use standard definition of MSR_TSX_FORCE_ABORT if available The MSR_TSX_FORCE_ABORT macro is defined in mainline since commit 52f64909409c ("x86: Add TSX Force Abort CPUID/MSR") in v5.1-rc1.	2019-11-14 08:06:57 +01:00
Michal Kubecek	ca784dc94e	vmnet: bring back the do_gettimeofday() helper The do_gettimeofday() helper was removed by commit e4b92b108c6c ("timekeeping: remove obsolete time accessors") in v5.0-rc1. Bring it back for users in vmnet-only/bridge.c Note that Workstation/Player 15.1.0 did in fact resolve the absence of do_gettimeofday() in kernel 5.0 and newer but it did so only for calls in vmmon-only/linux/hostif.c. Calls in vmnet-only/bridge.c were left in place as those are only compiled when building with LOGLEVEL >= 4.	2019-11-14 08:06:57 +01:00
Michal Kubecek	821a4a5d76	vmmon: check presence of file_operations::poll() Commit 11c5ad0ec441 ("eventpoll: switch to ->poll_mask") in v4.18-rc1, made eventpoll switch from ->poll() to ->poll_mask(), resulting in a null pointer dereference as vmmon calls the callback directly without checking its availability. Rather than calling the callback directly (which would result in null pointer dereference), use vfs_poll() wrapper. As this wrapper is only available since 4.18-rc1 cycle as well, provide a copy to use when building against older kernels. Note: even if the commit revealing the problem was reverted in 4.18-rc3 (and is unlikely to go back in the same form) so that vmmon no longer crashes with 4.18 kernel, calling the callback without a check is a bad practice so let's keep the patch in place.	2019-11-14 08:06:57 +01:00
Michal Kubecek	829e2f4f9a	vmmon: fix indirect call with retpoline build Build against kernel with retpoline support issues warning objtool: Task_Switch()+0x425: indirect call found in RETPOLINE build This is because an indirect call in TaskSwitchToMonitor() is encoded using inline assembler so that it bypasses retpoline generation. For this purpose, macro CALL_NOSPEC exists since v4.15-rc8 (and has been backported into some distribution kernels with the rest of retpoline support). Use the macro if available and fallback to the original code if not.	2019-11-14 08:06:57 +01:00
Michal Kubecek	f647af56af	vmmon: fix always_inline attribute usage Function declared with __attribute__((always_inline)) should also be declared as inline, otherwise gcc issues a warning "always_inline function might not be inlinable". It's just cosmetic but getting rid of known harmless warnings makes it easier to spot actual problems. Use the __always_inline macro for LinuxDriverSyncReadTSCs() as this is how always inline functions should be declared in kernel code.	2019-11-14 08:06:57 +01:00
Michal Kubecek	b4de1e2436	vmmon: use standard definition of MSR_K7_HWCR_SMMLOCK if available The MSR_K7_HWCR_SMMLOCK macro is defined in mainline since commit 18c71ce9c882 ("x86/CPU/AMD: Add the Secure Encrypted Virtualization CPU feature") in v4.16-rc1.	2019-11-14 08:06:57 +01:00
Theo Bertacchini	8893d11a72	modules: remove .cache.mk on make clean Remove .cache.mk inside vmmon-only and vmnet-only when executing "make clean". This file can cause issues when upgrading gcc as make will still look for includes inside older gcc includes directory. File .cache.mk with cache of generated variables was created by build since kernel v4.15-rc1, commit 3298b690b21c ("kbuild: Add a cache for generated variables") until the feature was removed in v4.18-rc1, commit e08d6de4e532 ("kbuild: remove kbuild cache").	2019-11-14 08:06:57 +01:00
Michal Kubecek	fca70ebea0	vmmon: quick workaround for objtool warnings As discussed in https://bugzilla.suse.com/show_bug.cgi?id=1059674 the reason for multiple objtool warnings is the fact that vmmon module defines its own Panic() function which never returns. While it is marked as such which is used by the compiler for optimization, there is no way to find this from object file. While this seems innocuous, it might result in problems with unwinder later. The quickest way around is to replace vmmon's own Panic() with standard kernel panic() until a cleaner solution is found.	2019-11-14 08:06:57 +01:00
Michal Kubecek	ef8704d97d	vmmon: use standard definition of CR3_PCID_MASK if available The CR3_PCID_MASK is defined in mainline asm/processor-flags.h since commit 6c690ee1039b ("x86/mm: Split read_cr3() into read_cr3_pa() and __read_cr3()") in v4.13-rc1.	2019-11-14 08:06:57 +01:00
Michal Kubecek	38c3889146	vmmon: use standard definition of MSR_MISC_FEATURES_ENABLES if available The MSR_MISC_FEATURES_ENABLES macro is defined in mainline since commit ab6d9468631a ("x86/msr: Rename MISC_FEATURE_ENABLES to MISC_FEATURES_ENABLES") in v4.12-rc1. The MSR_MISC_FEATURES_ENABLES_CPUID_FAULT macro is defined in mainline since commit e9ea1e7f53b8 ("x86/arch_prctl: Add ARCH_[GET\|SET]_CPUID") in v4.12-rc1.	2019-11-14 08:06:54 +01:00
Michal Kubecek	e056593a8b	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE_VMXNET3 if available The PCI_DEVICE_ID_VMWARE_VMXNET3 macro is defined in mainline pci_ids.h since commit b1226c7db1d9 ("vmxnet3: Move PCI Id to pci_ids.h") in v4.10-rc1.	2019-11-14 08:02:08 +01:00
Michal Kubecek	1390fbdb75	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE if available The PCI_VENDOR_ID_VMWARE macro is defined in mainline pci_ids.h since commit 94e57fea6202 ("PCI: Move PCI_VENDOR_ID_VMWARE to pci_ids.h") in v3.18-rc1.	2019-11-14 08:02:08 +01:00
Michal Kubecek	4b2c2a0bca	modules: do not override CC command line options Workstation/Player 15.5.0 tries to guess the right retpoline (Spectre) related compiler command line options, resulting in objtool warnings like warning: objtool: .text.__x86_indirect_thunk_rbp+0x0: unsupported intra-function call warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate e.g. in vmmon-only/common/memtrack.o (but also many others). This seems wrong; out-of-tree modules should be built with the same compiler options that were used to build the kernel they are built against. Distribution kernel devel packages usually make sure this is the way so let them do their job.	2019-11-14 08:02:08 +01:00
Michal Kubecek	5fa25885c8	vmmon: fix wrong VMX86_DEVEL test If VMX86_DEVEL macro is not defined, preprocessor warns when it is tested with "#if" in vmmon-only/linux/driver.c. As VMX86_DEVEL is tested with "#ifdef" everywhere else, change this one test to match the rest.	2019-11-14 08:02:08 +01:00
Michal Kubecek	bf5aeaca48	import Workstation 15.5.1 module sources	2019-11-14 08:00:40 +01:00
Michal Kubecek	51949a3872	import Workstation 15.5.0 module sources	2019-09-21 00:18:33 +02:00
Michal Kubecek	4d36d6a1d2	Merge branch 'master' into workstation	2019-09-21 00:10:22 +02:00
Michal Kubecek	85ede6aae9	misc: fix wrong directory name in INSTALL (fixes #36 ) Fix wrong directory name in one of the examples.	2019-09-09 08:38:27 +02:00
Michal Kubecek	a93f44ee44	import Workstation 15.1.0 module sources	2019-05-15 07:10:51 +02:00