vmmon: quick workaround for objtool warnings

As discussed in https://bugzilla.suse.com/show_bug.cgi?id=1059674 the reason for multiple objtool warnings is the fact that vmmon module defines its own Panic() function which never returns. While it is marked as such which is used by the compiler for optimization, there is no way to find this from object file. While this seems innocuous, it might result in problems with unwinder later. The quickest way around is to replace vmmon's own Panic() with standard kernel panic() until a cleaner solution is found.
vmmon: use standard definition of CR3_PCID_MASK if available
2019-11-14 08:06:57 +01:00 · 2019-11-14 08:06:57 +01:00 · 2019-11-14 08:06:54 +01:00 · 2019-11-14 08:02:08 +01:00 · 2019-11-14 08:02:08 +01:00 · 2019-11-14 08:02:08 +01:00
106 changed files with 8462 additions and 6809 deletions
@@ -150,7 +150,7 @@ test build).
 When using a downloaded tarball, simply run

  tar -cf vmmon.tar vmmon-only
-  tar -cf vmnet.tar vmmon-only
+  tar -cf vmnet.tar vmnet-only

 to create the tarballs. In this case, there is no protection against
 unwanted local changes.
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998,2015 VMware, Inc. All rights reserved.
+# Copyright (C) 1998,2015,2017 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -72,7 +72,7 @@ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')

 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
-C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o hashFunc.o
+C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o
 C_TARGETS_VMCORE := moduleloop.o
 C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,14 +24,7 @@
 *      See monLoader.h for a full description.
 */

-#ifdef VMX86_VMX
-
-#include <string.h>
-#include <limits.h>
-#include "vmx.h"
-#include "vm_assert.h"
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #include "vmkernel.h"
 #include "libc.h"
@@ -57,15 +50,15 @@

 #include "vm_assert.h"

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

 #include "vm_basic_types.h"
 #include "monLoader.h"
 #include "vcpuid.h"
-#include "x86types.h"
-#include "x86paging_64.h"
+#include "vm_pagetable.h"
+#include "address_defs.h"
 #include "monLoaderLog.h"

 #define CANONICAL_MASK MASK64(36)
@@ -75,12 +68,6 @@
 #define L2EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 18))
 #define L1EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 9))

-/*
- * Import address space from VMX rather than building it.  Once monitor loading
- * is removed from VMX entirely, code to import will be deleted.
- */
-#define IMPORT_AS_FROM_VMX (TRUE)
-
 /* The maximum (canonical-address) VPN */
 #define VPN_MAX MASK64(52)

@@ -104,22 +91,18 @@ typedef struct MonPTMPNs {
 } MonPTMPNs;
 
 typedef struct MonLoaderContext {
-   MonLoaderEnvContext *envCtx; /* Environment-specific context */
+   struct MonLoaderEnvContext *envCtx; /* Environment-specific context */
   struct {
      MonPTMPNs    ptMPNs;      /* Mappings into the AS for the current VCPU. */
      VPN          ASFirstVPN;  /* first VPN in the address space */
      VPN          ASLastVPN;   /* last VPN in the address space (inclusive) */
      uint64       ASPTEFlags;  /* PTE flags for L4->L1 connection. */
      Vcpuid       currentVCPU;
-      Bool         ASImported;  /* Was this AS imported or created anew? */
      Bool         hasAddrSpace;
   } vcpu;
 } MonLoaderContext;


-static const Bool importFromVMX = IMPORT_AS_FROM_VMX;
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -192,7 +175,7 @@ MonLoaderIsMapped(MonLoaderContext *ctx,    // IN/OUT
   if (ret != ML_OK) {
      return ret;
   }
-   *mapped = PTE_PRESENT(pte);
+   *mapped = ML_PERM_PRESENT(pte);
   return ML_OK;
 }

@@ -280,15 +263,40 @@ MonLoaderMapMPN(MonLoaderContext *ctx,    // IN/OUT
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderBuildsPtLevel --
+ *
+ *      Determines whether MonLoader allocates and maps the page table(s) for
+ *      the monitor at the given level.
+ *
+ * Result:
+ *      TRUE if MonLoader creates the page table at the given level, FALSE
+ *      otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+static Bool
+MonLoaderCreatesPtLevel(PT_Level level)
+{
+   ASSERT(level >= PT_LEVEL_STOP && level <= PT_MAX_LEVELS);
+   return FALSE;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
 * MonLoaderCreateAddressSpace --
 *
- *      Creates or (if importAS is set) verifies an address space.  The VPN
- *      range specified by firstVPN and size is used to determine page counts
- *      at each page table level to map every page.  Sufficient pages are then
- *      allocated or verified.
+ *      Creates or verifies an address space.  The VPN range specified by
+ *      firstVPN and size is used to determine page counts at each page table
+ *      level to map every page.  Sufficient pages are then allocated or
+ *      verified.
 *
 * Result:
 *      ML_OK if successful, some other value on error.
@@ -302,8 +310,7 @@ static MonLoaderError
 MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
                            VPN               firstVPN, // IN
                            uint64            flags,    // IN
-                            uint64            monPages, // IN
-                            Bool              importAS) // IN
+                            uint64            monPages) // IN
 {
   Vcpuid vcpu = ctx->vcpu.currentVCPU;
   VPN lastVPN = firstVPN + monPages - 1;
@@ -329,11 +336,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
   ctx->vcpu.ASFirstVPN = firstVPN;
   ctx->vcpu.ASLastVPN = lastVPN;
   ctx->vcpu.ASPTEFlags = flags;
-   ctx->vcpu.ASImported = importAS;

   ptMPNs = &ctx->vcpu.ptMPNs;

-   if (importAS) {
+   if (MonLoaderCreatesPtLevel(PT_LEVEL_4)) {
+      NOT_IMPLEMENTED();
+   } else {
+      /* Verify the VMX's allocation. */
      mpn = MonLoaderCallout_GetPageRoot(ctx->envCtx, vcpu);
      if (mpn == INVALID_MPN) {
         return ML_ERROR_CALLOUT_PAGEROOT_GET;
@@ -341,13 +350,13 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
      LOG(5, "%s: vcpu %u page root=0x%"FMT64"x\n", __FUNCTION__, vcpu, mpn);
      ptMPNs->L4MPNs[0] = mpn;
      MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-   } else {
-      NOT_IMPLEMENTED();
   }
   ptMPNs->L4MPNCount = 1;

   for (i = 0; i < L3MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_3)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L4E;
         unsigned L4Off = PT_LPN_2_L4OFF(monVPN);
         PT_L1E pte;
@@ -357,20 +366,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L4E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L3MPNs[i] = mpn;
   }
   ptMPNs->L3MPNCount = L3MPNsNeeded;
      
   for (i = 0; i < L2MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_2)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L3E;
         unsigned L3Off = PT_LPN_2_L3OFF(monVPN);
         unsigned L3Page = L3EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -381,20 +390,20 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L3E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L2MPNs[i] = mpn;
   }
   ptMPNs->L2MPNCount = L2MPNsNeeded;

   for (i = 0; i < L1MPNsNeeded; i++) {
-      if (importAS) {
+      if (MonLoaderCreatesPtLevel(PT_LEVEL_1)) {
+         NOT_IMPLEMENTED();
+      } else {
         VPN monVPN = firstVPN + i * PT_PAGES_PER_L2E;
         unsigned L2Off = PT_LPN_2_L2OFF(monVPN);
         unsigned L2Page = L2EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
@@ -405,13 +414,15 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
-         mpn = LM_PTE_2_PFN(pte);
+         if (!ML_PERM_PRESENT(pte)) {
+            ptMPNs->L1MPNs[i] = INVALID_MPN;
+            continue;
+         }
+         mpn = ML_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
         }
         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
-      } else {
-         NOT_IMPLEMENTED();
      }
      ptMPNs->L1MPNs[i] = mpn;
   }
@@ -442,12 +453,12 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
 */
 static MonLoaderError
 MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
-                       unsigned          level,    // IN
+                       PT_Level          level,    // IN
                       uint64            flags,    // IN
                       VPN               monVPN,   // IN
                       uint64            monPages) // IN
 {
-   Bool verify = ctx->vcpu.ASImported;
+   Bool verify = !MonLoaderCreatesPtLevel(level);
   uint64    i;
   unsigned  count;
   MPN      *ptMPNs;
@@ -456,25 +467,25 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
      if (!ctx->vcpu.hasAddrSpace) {
         return ML_ERROR_NO_ADDRSPACE;
      }
-      if (level < 1 || level > 4) {
+      if (level < PT_LEVEL_STOP || level > PT_MAX_LEVELS) {
         return ML_ERROR_ARGS;
      }
   }

   switch (level) {
-      case 4:
+      case PT_LEVEL_4:
         count = ctx->vcpu.ptMPNs.L4MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L4MPNs;
         break;
-      case 3:
+      case PT_LEVEL_3:
         count = ctx->vcpu.ptMPNs.L3MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L3MPNs;
         break;
-      case 2:
+      case PT_LEVEL_2:
         count = ctx->vcpu.ptMPNs.L2MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L2MPNs;
         break;
-      case 1:
+      case PT_LEVEL_1:
         count = ctx->vcpu.ptMPNs.L1MPNCount;
         ptMPNs = ctx->vcpu.ptMPNs.L1MPNs;
         break;
@@ -490,8 +501,11 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
         VPN vpn = monVPN + i;
         if (verify) {
            PT_L1E l1e;
+            if (ptMPNs[i] == INVALID_MPN && level == PT_LEVEL_1) {
+               continue;
+            }
            if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK ||
-                LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
+                ML_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
               return ML_ERROR_PAGE_TABLE_VERIFY;
            }
         } else {
@@ -622,7 +636,7 @@ MonLoaderCopyFromBlob(MonLoaderContext *ctx,        // IN/OUT
 *      ML_OK if successful, some other value on error.
 *
 * Side effects:
- *      User pages may be mapped in.
+ *      User or host pages may be mapped in.
 *
 *----------------------------------------------------------------------
 */
@@ -653,6 +667,14 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
                                                  vcpu);
      }
      if (mpn == INVALID_MPN) {
+         if (subIndex == MONLOADER_HT_MAP_IDX) {
+            /*
+             * This item is tied to a vmkernel feature.
+             * When the feature is disabled, there is nothing to share.
+             */
+            ASSERT(vmx86_server);
+            return ML_OK;
+         }
         /* Partial sharing is allowed.  Return success if any occurred. */
         return i != 0 ? ML_OK : ML_ERROR_SHARE;
      }
@@ -665,45 +687,6 @@ MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
 }


-#ifndef VMMON
-/*
- *----------------------------------------------------------------------
- *
- * MonLoaderSetEntrypoint --
- *
- *      Calls out to set the code and stack entrypoint.
- *
- * Result:
- *      ML_OK if successful, some other value on error.
- *
- * Side effects:
- *      The monitor entrypoint for the current VCPU may be set.
- *
- *----------------------------------------------------------------------
- */
-static MonLoaderError
-MonLoaderSetEntrypoint(MonLoaderContext *ctx,           // IN
-                       uint16            codeSelector,  // IN
-                       VA64              code,          // IN
-                       uint16            stackSelector, // IN
-                       VA64              stack)         // IN
-{
-   if (VA_2_VPN(code) < ctx->vcpu.ASFirstVPN  ||
-       VA_2_VPN(code) > ctx->vcpu.ASLastVPN   ||
-       VA_2_VPN(stack) < ctx->vcpu.ASFirstVPN ||
-       VA_2_VPN(stack) > ctx->vcpu.ASLastVPN) {
-      return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
-   }
-   if (MonLoaderCallout_SetEntrypoint(ctx->envCtx, codeSelector, code,
-                                      stackSelector, stack)) {
-      return ML_OK;
-   } else {
-      return ML_ERROR_CALLOUT_ENTRYPOINT;
-   }
-}
-#endif
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -786,8 +769,9 @@ MonLoaderShareFromBlob(MonLoaderContext *ctx,        // IN/OUT
 {
   uint64 endOff = blobOffset + blobSize;

-   if ((blobOffset & PAGE_MASK) != 0 || (blobSize & PAGE_MASK) != 0 ||
-       (flags & PTE_RW) != 0) {
+   if ((blobOffset & (PAGE_SIZE - 1)) != 0 ||
+       (blobSize & (PAGE_SIZE - 1)) != 0 ||
+       ML_PERM_WRITEABLE(flags)) {
      return ML_ERROR_SHARE;
   }
   if (blobSize > monBytes || blobSize == 0) {
@@ -858,7 +842,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
      return ML_ERROR_TABLE_MISSING;
   }

-   if (!MonLoaderCallout_Init(args, &ctx.envCtx)) {
+   if (!MonLoaderCallout_Init(args, &ctx.envCtx, numVCPUs)) {
      return ML_ERROR_CALLOUT_INIT;
   }
   ret = ML_OK;
@@ -879,7 +863,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         uint64               monBytes   = PAGES_2_BYTES(monPages);
         uint64               subIndex   = entry->subIndex;
         uint64               bspOnly    = entry->bspOnly;
-         uint64               isPriv     = vmx86_server || entry->procVmmon;
         *line = i;

         /*
@@ -888,17 +871,6 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         if (bspOnly && !IS_BOOT_VCPUID(*vcpu)) {
            continue;
         }
-         /*
-          * Entries marked as privileged (isPriv == TRUE) are processed
-          * by the privileged layer (VMKernel/vmmon). Others are
-          * processed by the VMX. One exception is the entry of type
-          * ML_CONTENT_ADDRSPACE: it is always processed by the VMX and,
-          * if marked as privileged, also by the privileged layer.
-          */
-         if (isPriv != MonLoaderCallout_IsPrivileged(ctx.envCtx) &&
-             (content != ML_CONTENT_ADDRSPACE || isPriv == 0)) {
-            continue;
-         }
         if (monPages == 0 || monVPN + monPages - 1 > VPN_MAX) {
            ret = ML_ERROR_SIZE;
            continue;
@@ -911,8 +883,7 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
                  continue;
               }
               /* Create or verify the address space and page table MPNs. */
-               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages,
-                                                 importFromVMX);
+               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages);
               break;
            case ML_CONTENT_PAGETABLE_L4:
            case ML_CONTENT_PAGETABLE_L3:
@@ -980,15 +951,10 @@ MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
         }
      }
   }
-#ifndef VMMON
+#ifdef VMKERNEL
   if (ret == ML_OK) {
      if (ctx.vcpu.ASLastVPN == 0) {
         ret = ML_ERROR_NO_ADDRSPACE;
-      } else {
-         ret = MonLoaderSetEntrypoint(&ctx, header->codeSelector,
-                                      header->codeEntrypoint,
-                                      header->stackSelector,
-                                      header->stackEntrypoint);
      }
   }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,7 +37,10 @@
 #include "x86paging_common.h"
 #include "x86paging_64.h"
 #include "vmx86.h"
+#include "monLoader.h"
 #include "monLoaderLog.h"
+#include "vmmblob.h"
+#include "memtrack.h"

 typedef struct MonLoaderEnvContext {
   VMDriver *vm;
@@ -62,7 +65,7 @@ typedef struct MonLoaderEnvContext {
 */

 Bool
-MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx)
+MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx, unsigned numVCPUs)
 {
   MonLoaderEnvContext *c;
   MonLoaderArgs *mlArgs = (MonLoaderArgs *)args;
@@ -101,6 +104,7 @@ void
 MonLoaderCallout_CleanUp(MonLoaderEnvContext *ctx)
 {
   ASSERT(ctx != NULL);
+   Vmx86_CleanupVMMPages(ctx->vm);
   HostIF_FreeKernelMem(ctx);
 }

@@ -179,11 +183,52 @@ MonLoaderCallout_ImportPage(MonLoaderEnvContext *ctx, // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_AllocMPN --
+ *
+ *      Allocates and maps a new VMM page for the specified VCPU.
+ *
+ * Returns:
+ *      An MPN on success, INVALID_MPN on failure.
+ *
+ * Side effects:
+ *      The allocated page is accounted for and its mapping tracked.
+ *
+ *----------------------------------------------------------------------
+ */
+
 MPN
 MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
                          Vcpuid               vcpu) // IN
 {
-   NOT_IMPLEMENTED();
+   VMDriver *vm = ctx->vm;
+   MemTrackEntry *entry;
+   MPN mpn;
+   VPN vpn;
+
+   if (Vmx86_AllocLockedPages(vm, (VA64)&mpn, 1, TRUE, FALSE) != 1) {
+      Log("Failed to allocate page\n");
+      return INVALID_MPN;
+   }
+   vpn = Vmx86_MapPage(mpn);
+   if (vpn == 0) {
+      Log("Failed to map MPN 0x%"FMT64"x\n", mpn);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
+      return INVALID_MPN;
+   }
+   HostIF_VMLock(vm, 41);
+   entry = MemTrack_Add(vm->vmmTracker, vpn, mpn);
+   HostIF_VMUnlock(vm, 41);
+   if (entry == NULL) {
+      Log("Failed to track mapping from VPN 0x%"FMTVPN"x to MPN 0x%"FMT64"x\n",
+          vpn, mpn);
+      Vmx86_UnmapPage(vpn);
+      Vmx86_FreeLockedPages(vm, &mpn, 1);
+      return INVALID_MPN;
+   }
+   return mpn;
 }


@@ -227,16 +272,62 @@ MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *ctx, // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_FillPage --
+ *
+ *      Fills a page with a pattern, given the MPN of the page.
+ *
+ * Returns:
+ *      TRUE on success, FALSE if a mapping of the page was not found or
+ *      invalid.
+ *
+ * Side effects:
+ *      The page is filled.
+ *
+ *----------------------------------------------------------------------
+ */
+
 Bool
 MonLoaderCallout_FillPage(MonLoaderEnvContext *ctx,     // IN
                          uint8                pattern, // IN
                          MPN                  mpn,     // IN
                          Vcpuid               vcpu)    // IN
 {
-   NOT_IMPLEMENTED();
+   VMDriver *vm = ctx->vm;
+   MemTrackEntry *entry;
+
+   HostIF_VMLock(vm, 42);
+   entry = MemTrack_LookupMPN(vm->vmmTracker, mpn);
+   HostIF_VMUnlock(vm, 42);
+   if (entry == NULL || entry->mpn != mpn || entry->vpn == 0) {
+      Log("Failed to look up MPN 0x%"FMT64"x\n", mpn);
+      return FALSE;
+   }
+   memset((void *)VPN_2_VA(entry->vpn), pattern, PAGE_SIZE);
+   return TRUE;
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_CopyFromBlob --
+ *
+ *     Copies contents of up to PAGE_SIZE length from the blob at a
+ *     given offset into the page specified by the given MPN. Zero-fills
+ *     the remaining space.
+ *
+ * Returns:
+ *      TRUE on success, FALSE on failure.
+ *
+ * Side effects:
+ *      The page is filled with blob contents.
+ *
+ *----------------------------------------------------------------------
+ */
+
 Bool
 MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *ctx,        // IN
                              uint64               blobOffset, // IN
@@ -244,7 +335,30 @@ MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *ctx,        // IN
                              MPN                  mpn,        // IN
                              Vcpuid               vcpu)       // IN
 {
-   NOT_IMPLEMENTED();
+   MemTrackEntry *entry;
+   VMDriver *vm = ctx->vm;
+   uint64 blobSize = VmmBlob_GetSize(vm);
+   uint8 *blob = VmmBlob_GetPtr(vm);
+   uint8 *buf;
+
+   if (copySize > PAGE_SIZE || copySize == 0 || blobSize < copySize ||
+       blobOffset > blobSize - copySize) {
+      Log("Invalid VMM blob copy parameters: blobOffset 0x%"FMT64"x, "
+          "copySize 0x%"FMTSZ"x, blobSize 0x%"FMT64"x\n", blobOffset, copySize,
+          blobSize);
+      return FALSE;
+   }
+   HostIF_VMLock(vm, 44);
+   entry = MemTrack_LookupMPN(vm->vmmTracker, mpn);
+   HostIF_VMUnlock(vm, 44);
+   if (entry == NULL || entry->vpn == 0) {
+      Log("Failed to look up MPN 0x%"FMT64"x\n", mpn);
+      return FALSE;
+   }
+   buf = (uint8 *)VPN_2_VA(entry->vpn);
+   memcpy(buf, blob + blobOffset, copySize);
+   memset(buf + copySize, 0, PAGE_SIZE - copySize);
+   return TRUE;
 }


@@ -272,10 +386,8 @@ MonLoaderFindSharedRegion(MonLoaderEnvContext *ctx,
   int i;

   for (i = 0; i < ML_SHARED_REGIONS_MAX; i++) {
-      if (ctx->shRegions[i].baseVpn == INVALID_VPN) {
-         /* Not found. */
-         return NULL;
-      } else if (ctx->shRegions[i].index == index) {
+      if (ctx->shRegions[i].index == index &&
+          ctx->shRegions[i].baseVpn != INVALID_VPN) {
         return &ctx->shRegions[i];
      }
   }
@@ -311,10 +423,6 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,

   ASSERT(IS_BOOT_VCPUID(vcpuid));

-   if (index >= ML_SHARED_REGIONS_MAX) {
-      Log("Invalid shared region %"FMT64"x", index);
-      return INVALID_MPN;
-   }
   s = MonLoaderFindSharedRegion(ctx, index);
   if (s == NULL) {
      return INVALID_MPN;
@@ -322,15 +430,14 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
   if (pgOffset >= s->numPages) {
      return INVALID_MPN;
   }
-
   addr = VPN_2_VA(s->baseVpn) + pgOffset * PAGE_SIZE;
-   HostIF_VMLock(ctx->vm, 38);
+   HostIF_VMLock(ctx->vm, 43);
   if (HostIF_LookupUserMPN(ctx->vm, addr, &mpn) != PAGE_LOOKUP_SUCCESS) {
-      HostIF_VMUnlock(ctx->vm, 38);
+      HostIF_VMUnlock(ctx->vm, 43);
      Log("Failed to lookup MPN for shared region VA %"FMTVA"x\n", addr);
      return INVALID_MPN;
   }
-   HostIF_VMUnlock(ctx->vm, 38);
+   HostIF_VMUnlock(ctx->vm, 43);
   return mpn;
 }

@@ -352,12 +459,16 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
 */

 MPN
-MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx,
-                                   uint64 subIndex, // IN
-                                   unsigned page,   // IN
-                                   Vcpuid vcpu)     // IN
+MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx, // IN
+                                   uint64 subIndex,          // IN
+                                   unsigned page,            // IN
+                                   Vcpuid vcpu)              // IN
 {
-   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   if (subIndex == MONLOADER_HEADER_IDX) {
+      return VmmBlob_GetHeaderMpn(ctx->vm);
+   } else {
+      return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   }
 }


@@ -367,19 +478,19 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
                                   unsigned             page,     // IN
                                   Vcpuid               vcpu)     // IN
 {
-   NOT_IMPLEMENTED();
+   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
 }


 /*
 *----------------------------------------------------------------------
 *
- * MonLoaderCallout_IsPrivileged --
+ * MonLoaderCallout_GetBlobMpn --
 *
- *      Returns whether or not this is a privileged environment.
+ *      Returns the MPN backing the given VMM blob offset.
 *
 * Returns:
- *      TRUE if privileged, FALSE if not.
+ *      An MPN, or INVALID_MPN
 *
 * Side effects:
 *      None.
@@ -387,25 +498,10 @@ MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
 *----------------------------------------------------------------------
 */

-Bool
-MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN
-{
-   return TRUE;
-}
-
 MPN
 MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx,    // IN
                            uint64               offset) // IN
 {
-   return INVALID_MPN;
-}
-
-Bool
-MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *ctx,           // IN
-                               uint16               codeSelector,  // IN
-                               VA64                 code,          // IN
-                               uint16               stackSelector, // IN
-                               VA64                 stack)         // IN
-{
-   NOT_IMPLEMENTED();
+   ASSERT((offset & (PAGE_SIZE - 1)) == 0);
+   return VmmBlob_GetMpn(ctx->vm, BYTES_2_PAGES(offset));
 }
@@ -0,0 +1,231 @@
+/*********************************************************
+ * Copyright (C) 2017-2019 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vmmblob.c --
+ *
+ *     VMM blob management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "vmmblob.h"
+#include "monLoader.h"
+
+#define VMMBLOB_SIZE_MAX (22 * 1024 * 1024) /* Ensure enough space for
+                                             * obj build with * GCOV_VMM=1. */
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetPtr --
+ *
+ *      Returns a pointer to the buffer containing the VMM blob.
+ *
+ * Results:
+ *      A pointer.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint8 *
+VmmBlob_GetPtr(VMDriver *vm)
+{
+   return vm->blobInfo->blobPtr;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetSize --
+ *
+ *      Returns the size of the VMM blob in bytes.
+ *
+ * Results:
+ *      Number of bytes.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint64
+VmmBlob_GetSize(VMDriver *vm)
+{
+   return vm->blobInfo->numBytes;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetMpn --
+ *
+ *      Returns the MPN backing a VMM blob page.
+ *
+ * Results:
+ *      An MPN, or INVALID_MPN.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+VmmBlob_GetMpn(VMDriver *vm, uint64 pgOffset)
+{
+   VmmBlobInfo *bi = vm->blobInfo;
+   uint32 blobNumPages = CEILING(bi->numBytes, PAGE_SIZE);
+   MPN mpn = INVALID_MPN;
+
+   if (pgOffset < blobNumPages) {
+      mpn = bi->mpns[pgOffset];
+   }
+   return mpn;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetHeaderMpn --
+ *
+ *      Returns the MPN backing the MonLoader header.
+ *
+ * Results:
+ *      An MPN, or INVALID_MPN.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+VmmBlob_GetHeaderMpn(VMDriver *vm)
+{
+   VmmBlobInfo *bi = vm->blobInfo;
+   uint64 headerOffset = (uint8 *)bi->header - (uint8 *)bi->blobPtr;
+   ASSERT((headerOffset & (PAGE_SIZE - 1)) == 0);
+   return VmmBlob_GetMpn(vm, BYTES_2_PAGES(headerOffset));
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_Cleanup --
+ *
+ *      Cleans up VMM blob state by freeing its memory and associated
+ *      metadata.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+VmmBlob_Cleanup(VmmBlobInfo *bi)
+{
+   const uint32 numPages = CEILING(bi->numBytes, PAGE_SIZE);
+
+   HostIF_FreeKernelPages(numPages, bi->blobPtr);
+   HostIF_FreeKernelMem(bi->mpns);
+   HostIF_FreeKernelMem(bi);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_Load --
+ *
+ *      Copies and instantiates a VMM bootstrap blob from userspace.
+ *
+ * Results:
+ *      TRUE if successful, FALSE if not.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+VmmBlob_Load(UserVA64 blobAddr,
+             uint32 numBytes,
+             uint32 headerOffset,
+             VmmBlobInfo **blobInfo)
+{
+   uint8 *blob = NULL;
+   MPN *mpns = NULL;
+   MonLoaderHeader *header;
+   VmmBlobInfo *bi = NULL;
+   uint32 numPages = CEILING(numBytes, PAGE_SIZE);
+   size_t fixedHdrSize = MonLoader_GetFixedHeaderSize();
+
+   if (numBytes > VMMBLOB_SIZE_MAX || headerOffset > numBytes ||
+       fixedHdrSize > numBytes - headerOffset) {
+      Warning("Invalid arguments for processing bootstrap. "
+              "Header offset: %u, Fixed header size: %"FMTSZ"u bytes, "
+              "Blob size: %u bytes\n", headerOffset, fixedHdrSize, numBytes);
+      goto error;
+   }
+   mpns = HostIF_AllocKernelMem(numPages * sizeof(*mpns), FALSE);
+   if (mpns == NULL) {
+      Warning("Memory allocation for VMM bootstrap blob failed\n");
+      goto error;
+   }
+   bi = HostIF_AllocKernelMem(sizeof *bi, FALSE);
+   if (bi == NULL) {
+      Warning("Memory allocation for VMM bootstrap blob failed\n");
+      goto error;
+   }
+   blob = HostIF_AllocKernelPages(numPages, mpns);
+   if (blob == NULL) {
+      Warning("Memory allocation for VMM bootstrap blob failed\n");
+      goto error;
+   }
+   if (HostIF_CopyFromUser(blob, blobAddr, numBytes) != 0) {
+      Warning("Error copying VMM bootstrap blob from userspace\n");
+      goto error;
+   }
+   header = (MonLoaderHeader *)(blob + headerOffset);
+   if (header->count > (numBytes - headerOffset - fixedHdrSize) /
+                        sizeof (MonLoaderEntry)) {
+      Warning("Invalid arguments for processing bootstrap. "
+              "Header offset: %u, Number of header entries: %u, "
+              "Blob size: %u bytes\n", headerOffset, header->count, numBytes);
+      goto error;
+   }
+   bi->mpns = mpns;
+   bi->blobPtr = blob;
+   bi->numBytes = numBytes;
+   bi->header = header;
+   *blobInfo = bi;
+
+   return TRUE;
+
+error:
+   if (blob != NULL) {
+      HostIF_FreeKernelPages(numPages, blob);
+   }
+   if (bi != NULL) {
+      HostIF_FreeKernelMem(bi);
+   }
+   if (mpns != NULL) {
+      HostIF_FreeKernelMem(mpns);
+   }
+   return FALSE;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2011 VMware, Inc. All rights reserved.
+ * Copyright (C) 2011, 2016, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,7 +17,8 @@
 *********************************************************/

 #include "vmware.h"
-#include "hostif.h"
+#include "x86apic.h"
+#include "x86msr.h"
 #include "x86cpuid_asm.h"
 #include "vm_asm.h"
 #include "cpuid.h"
@@ -56,7 +57,7 @@ APIC_GetMA(void)

   if (cpuVendor != CPUID_VENDOR_INTEL &&
       cpuVendor != CPUID_VENDOR_AMD &&
-       cpuVendor != CPUID_VENDOR_VIA) {
+       cpuVendor != CPUID_VENDOR_HYGON) {
      return (MA)-1;
   }

@@ -64,7 +65,7 @@ APIC_GetMA(void)
    * Check if X2 APIC mode is enabled.
    */

-   if ((__GET_MSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
      return (MA)-1;
   }

@@ -74,18 +75,16 @@ APIC_GetMA(void)
    */

   // Mask out goo in the low 12 bits, which is unrelated to the address.
-   result = __GET_MSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);
+   result = X86MSR_GetMSR(MSR_APIC_BASE) & ~MASK64(PAGE_SHIFT);

   /*
    * On Intel, the high bits are reserved so we mask.
-    * On AMD, high bits are explicitly MBZ, so no need.
-    * Via doesn't specify, so we'll assume reserved.
+    * On AMD and Hygon, high bits are explicitly MBZ, so no need.
    */
-   if (cpuVendor == CPUID_VENDOR_INTEL || cpuVendor == CPUID_VENDOR_VIA) {
+   if (cpuVendor == CPUID_VENDOR_INTEL) {
      /*
       * Intel suggests using CPUID 0x80000008.eax[7-0] (physical
       * address size), with 36 (24 bit MPNs) as a fallback.
-       * Via has that cpuid leaf as well.
       */
      unsigned numPhysicalBits = 36;

@@ -123,7 +122,7 @@ APIC_Read(const APICDescriptor *desc, // IN
          int regNum)                 // IN
 {
   if (desc->isX2) {
-      return (uint32 )__GET_MSR(MSR_X2APIC_BASE + regNum);
+      return (uint32)X86MSR_GetMSR(MSR_X2APIC_BASE + regNum);
   } else {
      return desc->base[regNum][0];
   }
@@ -183,7 +182,7 @@ APIC_Write(const APICDescriptor *desc, // IN
           uint32 val)                 // IN
 {
   if (desc->isX2) {
-      __SET_MSR(MSR_X2APIC_BASE + regNum, val);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + regNum, val);
   } else {
      desc->base[regNum][0] = val;
   }
@@ -211,7 +210,7 @@ uint64
 APIC_ReadICR(const APICDescriptor *desc) // IN
 {
   if (desc->isX2) {
-      return __GET_MSR(MSR_X2APIC_BASE + APICR_ICRLO);
+      return X86MSR_GetMSR(MSR_X2APIC_BASE + APICR_ICRLO);
   } else {
      uint32 icrHi = desc->base[APICR_ICRHI][0];
      uint32 icrLo = desc->base[APICR_ICRLO][0];
@@ -245,7 +244,7 @@ APIC_WriteICR(const APICDescriptor *desc, // IN
 {
   if (desc->isX2) {
      uint64 icr = (uint64) id << 32 | icrLo;
-      __SET_MSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
+      X86MSR_SetMSR(MSR_X2APIC_BASE + APICR_ICRLO, icr);
   } else {
      ASSERT(!(id & ~(APIC_ICRHI_DEST_MASK >> APIC_ICRHI_DEST_OFFSET)));
      desc->base[APICR_ICRHI][0] = id << APIC_ICRHI_DEST_OFFSET;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -34,10 +34,13 @@
 #include "x86svm.h"
 #include "x86vt.h"

-uint32 cpuidFeatures;
-static CpuidVendor vendor = CPUID_NUM_VENDORS;
-static uint32 version;
-static Bool hasSpecCtrl;
+uint32      cpuidFeatures;
+CpuidVendor cpuidVendor = CPUID_NUM_VENDORS;
+uint32      cpuidVersion;
+Bool        hostSupportsVT;
+Bool        hostSupportsSVM;
+Bool        hostHasSpecCtrl;
+Bool        hostSupportsXSave;


 /*
@@ -80,8 +83,9 @@ CPUID_Init(void)
   char name[16];

   __GET_CPUID(1, &regs);
-   version = regs.eax;
+   cpuidVersion = regs.eax;
   cpuidFeatures = regs.edx;
+   hostSupportsXSave = CPUID_ISSET(1, ECX, XSAVE, regs.ecx);

   __GET_CPUID(0, &regs);
   ptr = (uint32 *)name;
@@ -91,33 +95,25 @@ CPUID_Init(void)
   ptr[3] = 0;

   if (strcmp(name, CPUID_INTEL_VENDOR_STRING_FIXED) == 0) {
-      vendor = CPUID_VENDOR_INTEL;
+      cpuidVendor = CPUID_VENDOR_INTEL;
   } else if (strcmp(name, CPUID_AMD_VENDOR_STRING_FIXED) == 0) {
-      vendor = CPUID_VENDOR_AMD;
+      cpuidVendor = CPUID_VENDOR_AMD;
   } else if (strcmp(name, CPUID_CYRIX_VENDOR_STRING_FIXED) == 0) {
-      vendor = CPUID_VENDOR_CYRIX;
+      cpuidVendor = CPUID_VENDOR_CYRIX;
+   } else if (strcmp(name, CPUID_HYGON_VENDOR_STRING_FIXED) == 0) {
+      cpuidVendor = CPUID_VENDOR_HYGON;
   } else {
      Warning("VMMON CPUID: Unrecognized CPU\n");
-      vendor = CPUID_VENDOR_UNKNOWN;
+      cpuidVendor = CPUID_VENDOR_UNKNOWN;
   }

   __GET_CPUID2(7, 0, &regs);
-   hasSpecCtrl = vendor == CPUID_VENDOR_INTEL && ((regs.edx >> 26) & 0x3) != 0;
-}
+   hostHasSpecCtrl = cpuidVendor == CPUID_VENDOR_INTEL &&
+                     (CPUID_ISSET(7, EDX, IBRSIBPB, regs.edx) ||
+                      CPUID_ISSET(7, EDX, STIBP, regs.edx));

-
-CpuidVendor
-CPUID_GetVendor(void)
-{
-   ASSERT(vendor != CPUID_NUM_VENDORS);
-   return vendor;
-}
-
-
-uint32
-CPUID_GetVersion(void)
-{
-   return version;
+   hostSupportsVT = VT_CapableCPU();
+   hostSupportsSVM = SVM_CapableCPU();
 }


@@ -155,51 +151,3 @@ CPUID_AddressSizeSupported(void)

   return result;
 }
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * CPUID_HostSupportsHV --
- *
- *     Determine whether processor supports hardware virtualization.  Two
- *     possibilities are valid: VMX on Intel or SVM on AMD.
- *
- * Results:
- *     True iff the processor supports the required features.
- *
- * Side effects:
- *     None.
- *
- *-----------------------------------------------------------------------------
- */
-
-Bool
-CPUID_HostSupportsHV(void)
-{
-   return (vendor == CPUID_VENDOR_AMD   && SVM_CapableCPU()) ||
-          (vendor == CPUID_VENDOR_INTEL && VT_CapableCPU());
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * CPUID_HostSupportsSpecCtrl --
- *
- *     Determine whether the processor supports MSR_SPEC_CTRL.
- *
- * Results:
- *     True iff the processor supports the required features.
- *
- * Side effects:
- *     None.
- *
- *-----------------------------------------------------------------------------
- */
-
-Bool
-CPUID_HostSupportsSpecCtrl(void)
-{
-   return hasSpecCtrl;
-}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998, 2016-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,14 +31,16 @@

 #include "x86cpuid.h"

-extern uint32 cpuidFeatures;
+extern uint32      cpuidFeatures;
+extern uint32      cpuidVersion;
+extern CpuidVendor cpuidVendor;
+extern Bool        hostSupportsVT;
+extern Bool        hostSupportsSVM;
+extern Bool        hostHasSpecCtrl;
+extern Bool        hostSupportsXSave;

 void CPUID_Init(void);
-CpuidVendor CPUID_GetVendor(void);
-uint32 CPUID_GetVersion(void);
 Bool CPUID_AddressSizeSupported(void);
-Bool CPUID_HostSupportsHV(void);
-Bool CPUID_HostSupportsSpecCtrl(void);

 static INLINE uint32
 CPUID_GetFeatures(void)
@@ -46,6 +48,49 @@ CPUID_GetFeatures(void)
   return cpuidFeatures;
 }

+static INLINE CpuidVendor
+CPUID_GetVendor(void)
+{
+   ASSERT(cpuidVendor != CPUID_NUM_VENDORS);
+   return cpuidVendor;
+}
+
+static INLINE uint32
+CPUID_GetVersion(void)
+{
+   return cpuidVersion;
+}
+
+static INLINE Bool
+CPUID_HostSupportsVT(void)
+{
+   return hostSupportsVT;
+}
+
+static INLINE Bool
+CPUID_HostSupportsSVM(void)
+{
+   return hostSupportsSVM;
+}
+
+static INLINE Bool
+CPUID_HostSupportsHV(void)
+{
+   return hostSupportsVT || hostSupportsSVM;
+}
+
+static INLINE Bool
+CPUID_HostSupportsSpecCtrl(void)
+{
+   return hostHasSpecCtrl;
+}
+
+static INLINE Bool
+CPUID_HostSupportsXSave(void)
+{
+   return hostSupportsXSave;
+}
+
 static INLINE Bool
 CPUID_SSE2Supported(void)
 {
@@ -1,41 +0,0 @@
-/*********************************************************
- * Copyright (C) 2001 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-
-/*
- * hashFunc.c --
- *
- *      The core implementation lives in lib/shared/hashFunc.h because it
- *      is shared by the vmkernel and vmmon.
- */
-
-#include "hashFunc.h"
-
-/*
- * Wrappers
- */
-
-// arbitrary constant
-#define HASH_INIT_VALUE (42)
-
-// 64-bit hash for one 4K page
-uint64
-HashFunc_HashPage(const void *addr)
-{
-   return hash2((uint64 *)addr, PAGE_SIZE / sizeof (uint64), HASH_INIT_VALUE);
-}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,7 +27,7 @@
 #   define __HOST_KERNEL_H__


-#   ifdef linux
+#   ifdef __linux__
 /*
 * In some cases, this files needs to include Linux kernel header file
 * asm/page.h.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,14 +59,7 @@

 #define CROSSCALL_SLEEP_US 1000

-/* Return value for HostIF_IPI. */
-typedef enum {
-   IPI_NONE,      // No IPI was sent.
-   IPI_UNICAST,   // Unicast IPIs were sent.
-   IPI_BROADCAST  // A broadcast IPI was sent to all PCPUs.
-} HostIFIPIMode;
-
-EXTERN int   HostIF_Init(VMDriver *vm);
+EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);

@@ -83,12 +76,12 @@ EXTERN Bool  HostIF_IsLockedByMPN(VMDriver *vm, MPN mpn);
 EXTERN void  HostIF_FreeAllResources(VMDriver *vm);
 EXTERN uint64 HostIF_ReadUptime(void);
 EXTERN uint64 HostIF_UptimeFrequency(void);
-EXTERN unsigned int HostIF_EstimateLockedPageLimit(const VMDriver *vm,
-                                                   unsigned int lockedPages);
+EXTERN PageCnt HostIF_EstimateLockedPageLimit(const VMDriver *vm,
+                                              PageCnt lockedPages);
 EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
-EXTERN void *HostIF_AllocKernelPages(unsigned numPages, MPN *mpns);
-EXTERN void  HostIF_FreeKernelPages(unsigned numPages, void *ptr);
+EXTERN void *HostIF_AllocKernelPages(PageCnt numPages, MPN *mpns);
+EXTERN void  HostIF_FreeKernelPages(PageCnt numPages, void *ptr);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
@@ -96,6 +89,9 @@ EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 #endif

 EXTERN Bool  HostIF_APICInit(VMDriver *vm, Bool setVMPtr, Bool probe);
+EXTERN uint8 HostIF_GetMonitorIPIVector(void);
+EXTERN uint8 HostIF_GetHVIPIVector(void);
+EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);

 EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
                                  Vcpuid vcpuid,
@@ -104,7 +100,8 @@ EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
 EXTERN int   HostIF_SemaphoreSignal(uint64 *args);

 EXTERN void  HostIF_SemaphoreForceWakeup(VMDriver *vm, const VCPUSet *vcs);
-EXTERN HostIFIPIMode HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
+EXTERN void  HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
+EXTERN void  HostIF_OneIPI(VMDriver *vm, Vcpuid v);

 EXTERN uint32 HostIF_GetCurrentPCPU(void);
 EXTERN void HostIF_CallOnEachCPU(void (*func)(void *), void *data);
@@ -114,11 +111,12 @@ EXTERN void HostIF_WaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_CancelWaitForThreads(VMDriver *vm, Vcpuid currVcpu);
 EXTERN void HostIF_WakeUpYielders(VMDriver *vm, Vcpuid currVcpu);

-EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                   unsigned int numPages, Bool kernelMPNBuffer);
-EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned int numPages, Bool kernelMPNBuffer);
+EXTERN int64 HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                     PageCnt numPages, Bool kernelMPNBuffer);
+EXTERN int HostIF_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn);
+EXTERN PageCnt HostIF_GetNumAnonPages(VMDriver *vm);
+EXTERN MPN HostIF_AllocLowPage(VMDriver *vm);

 EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr,
                               Bool kernelBuffer, size_t len);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2008, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,9 +29,10 @@
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
+#include "vm_basic_types.h"


-void *HostIF_AllocKernelMem(size_t size, int nonPageable);
+void *HostIF_AllocKernelMem(size_t size, Bool nonPageable);
 void  HostIF_FreeKernelMem(void *ptr);

 #endif // ifdef _HOSTIFMEM_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -96,7 +96,7 @@
 * Modify this value to increase the maximum number of tracked pages
 * per MemTrack instance.
 */
-#define MEMTRACK_MAX_TRACKED        MBYTES_2_PAGES(4096)
+#define MEMTRACK_MAX_TRACKED        GBYTES_2_PAGES(119)

 /*
 * Linux uses a 3-level directory, because we want to keep allocations
@@ -148,7 +148,7 @@ typedef uint64 MemTrackHTKey;

 typedef struct MemTrack {
   VMDriver         *vm;            /* The VM instance. */
-   unsigned          numPages;      /* Number of pages tracked. */
+   PageCnt           numPages;      /* Number of pages tracked. */
   MemTrackDir1      dir1;          /* First level directory. */
   MemTrackHT        vpnHashTable;  /* VPN to entry hashtable. */
   MemTrackHT       *mpnHashTable;  /* MPN to entry hashtable. */
@@ -197,7 +197,7 @@ MemTrackAllocPage(void)

 #define MEMTRACK_ALLOCDFN(_name, _itype, _otype)   \
   static INLINE _otype *                          \
-   _name(_itype *arg, unsigned pos)                \
+   _name(_itype *arg, uint64 pos)                  \
   {                                               \
      if (arg->dir[pos] == NULL) {                 \
         arg->dir[pos] = MemTrackAllocPage();      \
@@ -273,8 +273,8 @@ MemTrackHTInsert(MemTrackHT *ht,          // IN
 static void
 MemTrackCleanup(MemTrack *mt)    // IN
 {
-   unsigned idx;
-   unsigned p1;
+   PageCnt idx;
+   uint64 p1;
   MemTrackDir1 *dir1;
   Bool freeBackMap = mt != NULL && mt->mpnHashTable != NULL;

@@ -284,7 +284,7 @@ MemTrackCleanup(MemTrack *mt)    // IN
   dir1 = &mt->dir1;

   for (p1 = 0; p1 < MEMTRACK_DIR1_ENTRIES; p1++) {
-      unsigned p2;
+      uint64 p2;
      MemTrackDir2 *dir2 = MEMTRACK_GETDIR2(dir1, p1);

      if (dir2 == NULL) {
@@ -339,7 +339,7 @@ MemTrack *
 MemTrack_Init(VMDriver *vm) // IN:
 {
   MemTrack *mt;
-   unsigned idx;
+   PageCnt idx;

 #if defined(MEMTRACK_3LEVEL)
   ASSERT_ON_COMPILE(sizeof *mt <= PAGE_SIZE);
@@ -359,7 +359,8 @@ MemTrack_Init(VMDriver *vm) // IN:
      MemTrackHTPage *htPage = MemTrackAllocPage();

      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate VPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate VPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->vpnHashTable.pages[idx] = htPage;
@@ -374,7 +375,8 @@ MemTrack_Init(VMDriver *vm) // IN:
   for (idx = 0; idx < MEMTRACK_HT_PAGES; idx++) {
      MemTrackHTPage *htPage = MemTrackAllocPage();
      if (htPage == NULL) {
-         Warning("MemTrack failed to allocate MPN hash table (%d).\n", idx);
+         Warning("MemTrack failed to allocate MPN hash table (%"FMT64"d).\n",
+                 idx);
         goto error;
      }
      mt->mpnHashTable->pages[idx] = htPage;
@@ -409,8 +411,8 @@ MemTrack_Add(MemTrack *mt,    // IN
             VPN64 vpn,       // IN
             MPN mpn)         // IN
 {
-   unsigned idx = mt->numPages;
-   unsigned p1, p2, p3;
+   PageCnt idx = mt->numPages;
+   uint64 p1, p2, p3;
   MemTrackEntry *ent;
   MemTrackDir1 *dir1 = &mt->dir1;
   MemTrackDir2 *dir2;
@@ -535,16 +537,16 @@ MemTrack_LookupMPN(MemTrack *mt, // IN
 *----------------------------------------------------------------------
 */

-unsigned
+PageCnt
 MemTrack_Cleanup(MemTrack *mt,            // IN
                 MemTrackCleanupCb *cb,   // IN
                 void *cData)             // IN
 {
-   unsigned idx;
-   unsigned count = 0;
+   PageCnt idx;
+   PageCnt count = 0;

   for (idx = 0; idx < mt->numPages; idx++) {
-      unsigned p1, p2, p3;
+      uint64 p1, p2, p3;
      MemTrackEntry *ent;
      MemTrackDir1 *dir1 = &mt->dir1;
      MEMTRACK_IDX2DIR(idx, p1, p2, p3);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -43,8 +43,8 @@ typedef struct MemTrackEntry {
 typedef void (MemTrackCleanupCb)(void *cData, MemTrackEntry *entry);

 extern struct MemTrack *MemTrack_Init(VMDriver *vm);
-extern unsigned MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
-                                 void *cbData);
+extern PageCnt MemTrack_Cleanup(struct MemTrack *mt, MemTrackCleanupCb *cb,
+                                void *cbData);
 extern MemTrackEntry *MemTrack_Add(struct MemTrack *mt, VPN64 vpn, MPN mpn);
 extern MemTrackEntry *MemTrack_LookupVPN(struct MemTrack *mt, VPN64 vpn);
 extern MemTrackEntry *MemTrack_LookupMPN(struct MemTrack *mt, MPN mpn);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,16 +24,15 @@
 *    Depending on configuration phystracker provides either 2-level or
 *    3-level structure to track whether page (specified by its MPN) is
 *    locked or no.  Linux uses 3-level structures with top limit of
- *    1TB (32bit) or 16TB (64bit).  Windows use 2-level structures
- *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.  On Mac
- *    limit is 2-level 128GB.
+ *    1TB (32bit) or 16TB (64bit).  Windows and Mac use 2-level structures
+ *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.
 *
 *    2-level phystracker is built on top of 3-level one by collapsing
 *    middle level.
 */


-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -50,9 +49,7 @@
 #define BYTES_PER_ENTRY      (PAGE_SIZE)
 #define PHYSTRACK_L3_ENTRIES (8 * BYTES_PER_ENTRY) /* 128MB */

-#if defined(WINNT_DDK)
-#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
-#elif defined(__linux__)
+#if defined(__linux__)
 #define PHYSTRACK_L2_ENTRIES (BYTES_PER_ENTRY / sizeof(void *)) /* 64GB or 128GB */
 /*
 * Currently MPN is 32 bits.  15 bits are in L3, 9 bits are in L2,
@@ -60,7 +57,7 @@
 */
 #define PHYSTRACK_L1_ENTRIES (256) /* 16TB. */
 #else
-#define PHYSTRACK_L1_ENTRIES ((128 + 4) * 8) /* 128 GB */
+#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
 #endif

 #ifndef PHYSTRACK_L2_ENTRIES
@@ -493,3 +490,32 @@ PhysTrack_GetNext(const PhysTracker *tracker, // IN
   return INVALID_MPN;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * PhysTrack_GetNumTrackedPages --
+ *
+ *      Returns the total number of tracked pages
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+PageCnt
+PhysTrack_GetNumTrackedPages(const PhysTracker *tracker)
+{
+   PageCnt numTrackedMPNs = 0;
+   MPN nextMPN;
+   ASSERT(tracker);
+   ASSERT(HostIF_VMLockIsHeld(tracker->vm));
+   nextMPN = PhysTrack_GetNext(tracker, INVALID_MPN);
+   while (nextMPN != INVALID_MPN) {
+      numTrackedMPNs++;
+      nextMPN = PhysTrack_GetNext(tracker, nextMPN);
+   }
+   return numTrackedMPNs;
+}
+
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined(WINNT_DDK)
+#if !defined (__linux__)
 /* XXX: can be more efficient based on server vs. desktop and version of Windows */
 #define PHYSTRACK_MAX_SUPPORTED_GB (2048 + 4) /* 2 TB 64-bit W2k8 + 4 GB PCI */
 #endif
@@ -44,6 +44,7 @@ EXTERN void PhysTrack_Add(struct PhysTracker *, MPN);
 EXTERN void PhysTrack_Remove(struct PhysTracker *, MPN);
 EXTERN Bool PhysTrack_Test(const struct PhysTracker *, MPN);
 EXTERN MPN PhysTrack_GetNext(const struct PhysTracker *, MPN);
+EXTERN PageCnt PhysTrack_GetNumTrackedPages(const struct PhysTracker *);

 #endif

@@ -0,0 +1,299 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.c --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "sharedAreaVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmonIsMultiVCPU --
+ *
+ *      Helper function for determining if a shared area type represents
+ *      a region that has VCPU-specific memory.
+ *
+ * Results:
+ *      TRUE if type has VCPU-specific memory, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+SharedAreaVmmonIsMultiVCPU(SharedAreaType type)
+{
+   switch (type) {
+   case SHARED_AREA_PER_VM:
+   case SHARED_AREA_PER_VM_VMX:
+      return FALSE;
+   default:
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Init --
+ *
+ *      Initializes the shared area component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon shared area.
+ *
+ *----------------------------------------------------------------------
+ */
+
+SharedAreaVmmon *
+SharedAreaVmmon_Init(VMDriver *vm)
+{
+   SharedAreaVmmon *sa;
+
+   /*
+    * Allocate the shared area pointer here. The pages in each region are
+    * lazily allocated when a region is registered.
+    */
+   sa = HostIF_AllocKernelMem(sizeof *sa, FALSE);
+   if (sa == NULL) {
+      Warning("SharedArea failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(sa, 0, sizeof *sa);
+   return sa;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_Cleanup --
+ *
+ *      Cleans up the shared area component of the VM Driver by freeing all
+ *      previously allocated VMMon shared area memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+SharedAreaVmmon_Cleanup(SharedAreaVmmon *area)
+{
+   SharedAreaType t;
+
+   if (area == NULL) {
+      return;
+   }
+
+   for (t = 0; t < NUM_SHARED_AREAS; t++) {
+      SharedAreaVmmonRegion *region = &area->regions[t];
+
+      if (region->pages != NULL) {
+         HostIF_FreeKernelMem(region->pages);
+      }
+   }
+   HostIF_FreeKernelMem(area);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_ValidateRegionArgs --
+ *
+ *      Validate the given VMMon shared area region registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register a shared
+ *      area region for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_ValidateRegionArgs(VMDriver *vm,
+                                   VMSharedAreaRegistrationBlock *block)
+{
+   SharedAreaVmmonRegion *region;
+   Vcpuid vcpu         = block->vcpu;
+   SharedAreaType type = block->region.index;
+   PageCnt numPages    = block->region.numPages;
+
+   if (vm == NULL               ||
+       vm->sharedArea == NULL   ||
+       vcpu >= vm->numVCPUs     ||
+       numPages == 0            ||
+       type >= NUM_SHARED_AREAS ||
+       (!SharedAreaVmmonIsMultiVCPU(type) && vcpu > 0)) {
+      return FALSE;
+   }
+
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu != 0) {
+
+      /*
+       * A region that was previously registered for a given VCPU should have
+       * the same number of pages as originally specified. Also, a region can
+       * only be reserved once for a given VCPU throughout the runtime of a VM.
+       */
+      return region->pagesPerVcpu == numPages &&
+             region->pages[vcpu * region->pagesPerVcpu] == INVALID_MPN;
+   } else {
+      /* First registration for this region. */
+      ASSERT(region->pages == NULL);
+      return TRUE;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_RegisterRegion --
+ *
+ *      Register a VMMon shared area region for a given VCPU with the VM Driver.
+ *      A successful return value indicates that the backing pages of the
+ *      shared area region are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the region was successfully registered for the given VCPU,
+ *      FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+SharedAreaVmmon_RegisterRegion(VMDriver *vm,
+                               VMSharedAreaRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   SharedAreaType type = block->region.index;
+   SharedAreaVmmonRegion *region;
+   MPN *pages;
+   PageCnt page;
+
+   ASSERT(SharedAreaVmmon_ValidateRegionArgs(vm, block));
+   region = &vm->sharedArea->regions[type];
+   if (region->pagesPerVcpu == 0) {
+      PageCnt pagesInRegion = block->region.numPages;
+
+      if (SharedAreaVmmonIsMultiVCPU(type)) {
+         pagesInRegion *= vm->numVCPUs;
+      }
+      region->pages = HostIF_AllocKernelMem(sizeof *pages * pagesInRegion,
+                                            FALSE);
+      if (region->pages == NULL) {
+         Warning("Failed to allocate pages array for region %u\n", type);
+         return FALSE;
+      }
+      for (page = 0; page < pagesInRegion; page++) {
+         region->pages[page] = INVALID_MPN;
+      }
+      region->pagesPerVcpu = block->region.numPages;
+   }
+
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+
+   /*
+    * Lock all shared area backing pages throughout the runtime of the monitor.
+    * The pages remain locked until they are freed by HostIF during VM Driver
+    * tear-down.
+    */
+   for (page = 0; page < region->pagesPerVcpu; page++) {
+      VPN vmxVPN = block->region.baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, pages + page);
+
+      if (status != PAGE_LOCK_SUCCESS) {
+         PageCnt resetPage;
+
+         /* Reset the region pages for this VCPU to their original state. */
+         for (resetPage = 0; resetPage < page; resetPage++) {
+            status = Vmx86_UnlockPage(vm, uAddr);
+            ASSERT(status == PAGE_UNLOCK_SUCCESS);
+            pages[resetPage] = INVALID_MPN;
+         }
+         return FALSE;
+      }
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SharedAreaVmmon_GetRegionMPN --
+ *
+ *      For the shared area region corresponding to the given type and VCPU,
+ *      get the MPN backing the region at the given offset.
+ *
+ * Results:
+ *      The MPN backing the region, or INVALID_MPN if no MPN is found.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+SharedAreaVmmon_GetRegionMPN(VMDriver *vm, SharedAreaVmmonRequest *request)
+{
+   SharedAreaType type = request->type;
+   Vcpuid vcpu = request->vcpu;
+   SharedAreaVmmonRegion *region;
+   PageCnt pgOffset = request->offset;
+   MPN *pages;
+
+   ASSERT(vcpu < vm->numVCPUs);
+   ASSERT(type < NUM_SHARED_AREAS);
+   ASSERT(IMPLIES(!SharedAreaVmmonIsMultiVCPU(type), vcpu == 0));
+   region = &vm->sharedArea->regions[type];
+   pages = region->pages + vcpu * region->pagesPerVcpu;
+   if (region->pages == NULL || region->pagesPerVcpu == 0) {
+      Warning("(%s) Requested unregistered region %u, VCPU %u\n", __FUNCTION__,
+              type, vcpu);
+      return INVALID_MPN;
+   }
+   if (pgOffset >= region->pagesPerVcpu) {
+      Warning("(%s) Offset %"FMT64"u (type %u, per-VCPU size %"FMT64"u)\n",
+              __FUNCTION__, pgOffset, type, region->pagesPerVcpu);
+      return INVALID_MPN;
+   }
+
+   return pages[pgOffset];
+}
@@ -0,0 +1,282 @@
+/*********************************************************
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * statVarsVmmon.c --
+ *
+ *     VMMon stat variables management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h> /* memset() in the kernel. */
+#else
+#   include <string.h>
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "statVarsVmmon.h"
+#include "vmx86.h"
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Init --
+ *
+ *      Initializes the stat vars component of the VM Driver.
+ *
+ * Results:
+ *      A pointer to the VMMon stat vars struct.
+ *
+ *----------------------------------------------------------------------
+ */
+
+StatVarsVmmon *
+StatVarsVmmon_Init(VMDriver *vm)
+{
+   StatVarsVmmon *statVars;
+
+   statVars = HostIF_AllocKernelMem(sizeof *statVars, FALSE);
+   if (statVars == NULL) {
+      Warning("StatVars failed to allocate handle.\n");
+      return NULL;
+   }
+   memset(statVars, 0, sizeof *statVars);
+   return statVars;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_Cleanup --
+ *
+ *      Cleans up the stat vars component of the VM Driver by freeing all
+ *      previously allocated VMMon stat vars memory.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+StatVarsVmmon_Cleanup(StatVarsVmmon *statVars)
+{
+   if (statVars == NULL) {
+      return;
+   }
+
+   if (statVars->pages != NULL) {
+      HostIF_FreeKernelMem(statVars->pages);
+   }
+   HostIF_FreeKernelMem(statVars);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmonValidateRegistration --
+ *
+ *      Validate the given VMMon stat vars registration block.
+ *
+ * Results:
+ *      TRUE if the block is valid and can be used to register stat vars
+ *      backing pages for a VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static Bool
+StatVarsVmmonValidateRegistration(VMDriver *vm,
+                                   VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu      = block->vcpu;
+   PageCnt numPages = block->numPages;
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+
+   if (vm == NULL           ||
+       vm->statVars == NULL ||
+       numPages == 0        ||
+       vcpu >= vm->numVCPUs) {
+      return FALSE;
+   }
+
+   pagesPerVcpu = vm->statVars->pagesPerVcpu;
+   pages = vm->statVars->pages;
+   if (pagesPerVcpu == 0) {
+      ASSERT(pages == NULL);
+      return TRUE;
+   } else {
+      return pagesPerVcpu == numPages &&
+             pages[vcpu * pagesPerVcpu] == INVALID_MPN;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_RegisterVCPU --
+ *
+ *      Register stat vars backing pages for the given VCPU with VMMon.
+ *      A successful return value indicates that stat vars backing pages
+ *      for the given VCPU are locked and tracked by the VM Driver.
+ *
+ * Results:
+ *      TRUE if the backing pages were successfully registered for the
+ *      given VCPU, FALSE otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+StatVarsVmmon_RegisterVCPU(VMDriver *vm,
+                           VMStatVarsRegistrationBlock *block)
+{
+   Vcpuid vcpu = block->vcpu;
+   PageCnt pagesPerVcpu = block->numPages;
+   StatVarsVmmon *statVars = vm->statVars;
+   MPN *pages;
+   PageCnt page;
+
+   HostIF_VMLock(vm, 45);
+   if (!StatVarsVmmonValidateRegistration(vm, block)) {
+      HostIF_VMUnlock(vm, 45);
+      return FALSE;
+   }
+   if (statVars->pagesPerVcpu == 0) {
+      PageCnt totalPages = pagesPerVcpu * vm->numVCPUs;
+      statVars->pages = HostIF_AllocKernelMem(totalPages * sizeof *pages,
+                                              FALSE);
+      if (statVars->pages == NULL) {
+         HostIF_VMUnlock(vm, 45);
+         return FALSE;
+      }
+      for (page = 0; page < totalPages; page++) {
+         statVars->pages[page] = INVALID_MPN;
+      }
+      statVars->pagesPerVcpu = pagesPerVcpu;
+   }
+
+   /*
+    * Validate that all pages in this VCPU's region are still invalid. Before
+    * unlocking, temporarily set the first page to 0 to prevent a malicious user
+    * from triggering an ASSERT by firing parallel ioctls for the same VCPU.
+    */
+   pages = statVars->pages + vcpu * pagesPerVcpu;
+   for (page = 0; page < pagesPerVcpu; page++) {
+      ASSERT(pages[page] == INVALID_MPN);
+   }
+   ASSERT_ON_COMPILE(INVALID_MPN != 0);
+   pages[0] = 0;
+   HostIF_VMUnlock(vm, 45);
+
+   /*
+    * Verify that all stat vars backing pages are locked.  The pages are
+    * assumed to remain locked either until they are unlocked by the VMX
+    * following a NUMA migration, or by HostIF during VM Driver tear-down.
+    */
+   for (page = 0; page < pagesPerVcpu; page++) {
+      VPN vmxVPN = block->baseVpn + page;
+      VA64 uAddr = VPN_2_VA(vmxVPN);
+      PageCnt resetPage;
+      MPN *currPage = pages + page;
+      Bool verifiedLocked = FALSE;
+      int status = Vmx86_LockPage(vm, uAddr, FALSE, currPage);
+
+      /*
+       * Backing pages are expected to be locked by the VMX before being
+       * registered.
+       */
+      if (status == PAGE_LOCK_ALREADY_LOCKED) {
+         verifiedLocked = TRUE;
+         status = Vmx86_LookupUserMPN(vm, uAddr, currPage);
+         /*
+          * PR 2260615: Some platforms return PAGE_LOCK_SUCCESS for a successful
+          * user MPN lookup, others return PAGE_LOOKUP_SUCCESS, but they both
+          * happen to equal the same value.
+          */
+         ASSERT_ON_COMPILE(PAGE_LOOKUP_SUCCESS == PAGE_LOCK_SUCCESS);
+         if (status == PAGE_LOOKUP_SUCCESS) {
+            /*
+             * The backing page was verified as previously locked, and was
+             * successfully recorded.
+             */
+            continue;
+         }
+      }
+      /*
+       * The backing page was either not previously locked, or was not recorded.
+       * In either case, reset all previously recorded pages to INVALID_MPN.
+       */
+      if (!verifiedLocked && status == PAGE_LOCK_SUCCESS) {
+         status = Vmx86_UnlockPage(vm, uAddr);
+         ASSERT(status == PAGE_UNLOCK_SUCCESS);
+      }
+      for (resetPage = 0; resetPage <= page; resetPage++) {
+         pages[resetPage] = INVALID_MPN;
+      }
+      return FALSE;
+   }
+
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * StatVarsVmmon_GetRegionMPN --
+ *
+ *      Obtain the backing MPN for stat vars for the given VCPU at
+ *      the given offset.
+ *
+ * Results:
+ *      The backing stat vars MPN for the given VCPU at the given offset
+ *      if it has been registered, or INVALID_MPN otherwise.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpu, PageCnt offset)
+{
+   MPN *pages;
+   MPN backingPage;
+   StatVarsVmmon *statVars = vm->statVars;
+
+   ASSERT(vcpu < vm->numVCPUs);
+
+   HostIF_VMLock(vm, 46);
+   if (offset >= statVars->pagesPerVcpu) {
+      HostIF_VMUnlock(vm, 46);
+      Warning("(%s) Requested MPN at invalid offset %"FMT64"u for VCPU %u\n",
+              __FUNCTION__, offset, vcpu);
+      return INVALID_MPN;
+   }
+
+   pages = statVars->pages + vcpu * statVars->pagesPerVcpu;
+   backingPage = pages[offset];
+   HostIF_VMUnlock(vm, 46);
+   return backingPage;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,10 +24,11 @@
 #include "includeCheck.h"
 #include "bootstrap_vmm.h"

-struct InitBlock;
+struct MonLoaderHeader;

 extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
-extern int Task_InitCrosspage(VMDriver *vm, struct InitBlock *params);
+extern Bool Task_InitCrosspage(VMDriver *vm, LPN monStartLPN, LPN monEndLPN,
+                               PerVcpuPages *perVcpuPages);
 extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -32,24 +32,22 @@
 #include "x86msr.h"
 #include "modulecall.h"
 #include "vcpuid.h"
-#include "initblock.h"
 #include "iocontrols.h"
 #include "numa_defs.h"
 #include "rateconv.h"
 #include "vmmem_shared.h"
 #include "apic.h"
 #include "bootstrap_vmm.h"
-
-typedef struct PseudoTSCOffsetInfo {
-   uint32 inVMMCnt;                  /* Number of vcpus executing in the VMM. */
-   uint32 vcpuid;                    /* Index into VMDriver.ptscOffsets. */
-} PseudoTSCOffsetInfo;
+#include "sharedAreaVmmon.h"
+#include "statVarsVmmon.h"

 typedef struct TSCDelta {
   Atomic_uint64 min;
   Atomic_uint64 max;
 } TSCDelta;

+struct VmmBlobInfo;
+
 /*
 * VMDriver - the main data structure for the driver side of a
 *            virtual machine.
@@ -57,32 +55,30 @@ typedef struct TSCDelta {

 typedef struct VMDriver {
   /* Unique (in the driver), strictly positive, VM ID used by userland. */
-   int                          userID;
-
-   struct VMDriver             *nextDriver;   /* Next on list of all VMDrivers */
-
-   Vcpuid                       numVCPUs;     /* Number of vcpus in VM. */
-   struct VMHost               *vmhost;       /* Host-specific fields. */
-
-   BSVMM_HostParams             bsParams;
-   MPN                          ptRootMpns[MAX_VCPUS];
-
+   int                     userID;
+   Vcpuid                  numVCPUs;         /* Number of vcpus in VM. */
+   struct VMDriver        *nextDriver;       /* Next on list of all VMDrivers */
+   struct VMHost          *vmhost;           /* Host-specific fields. */
+   MPN                    *ptRootMpns;       /* numVCPUs-sized array. */
+   struct VmmBlobInfo     *blobInfo;         /* VMM bootstrap blob info. */
+   SharedAreaVmmon        *sharedArea;       /* VMMon shared area info. */
+   StatVarsVmmon          *statVars;         /* VMMon stat vars info. */
   /* Pointers to the crossover pages shared with the monitor. */
-   struct VMCrossPage          *crosspage[MAX_INITBLOCK_CPUS];
-   struct MemTrack             *ptpTracker;
-   volatile uint32              currentHostCpu[MAX_INITBLOCK_CPUS];
-   VCPUSet                      crosscallWaitSet[MAX_INITBLOCK_CPUS];
-   APICDescriptor               hostAPIC;
-
-   struct MemTrack             *memtracker;   /* Memory tracker pointer */
-   Bool                         checkFuncFailed;
-   struct PerfCounter          *perfCounter;
-   VMMemMgmtInfo                memInfo;
-   unsigned                     fastClockRate;/* Protected by FastClockLock. */
-
-   volatile PseudoTSCOffsetInfo ptscOffsetInfo; /* Volatile per PR 699101#29. */
-   Atomic_uint64                ptscLatest;
-   int64                        ptscOffsets[MAX_INITBLOCK_CPUS];
+   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
+   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
+   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
+   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
+   APICDescriptor          hostAPIC;
+   struct MemTrack        *memtracker;       /* Memory tracker pointer */
+   Bool                    checkFuncFailed;
+   struct PerfCounter     *perfCounter;
+   VMMemMgmtInfo           memInfo;
+   unsigned                fastClockRate;    /* Protected by FastClockLock. */
+   Atomic_uint64           ptscOffsetInfo;   /* Volatile per PR 699101#29. */
+   Atomic_uint64           ptscLatest;
+   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
+   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
+   PageCnt                 numPTPPages;      /* Num PTP pages allocated. */
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -112,19 +108,21 @@ typedef struct PseudoTSC {

 extern PseudoTSC pseudoTSC;

-#define MAX_LOCKED_PAGES (-1)
+#define MAX_LOCKED_PAGES MAX_PPN

-extern VMDriver *Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize);
+extern void Vmx86_CacheNXState(void);
+extern VMDriver *Vmx86_CreateVM(VA64 bsBlob,
+                                uint32 bsBlobSize,
+                                uint32 numVCPUs);
 extern Bool Vmx86_ProcessBootstrap(VMDriver *vm,
                                   VA64 bsBlobAddr,
                                   uint32 numBytes,
                                   uint32 headerOffset,
                                   uint16 numVCPUs,
-                                   UserVA64 *ptRootVAs,
+                                   PerVcpuPages *perVcpuPages,
                                   VMSharedRegion *shRegions);
 extern int Vmx86_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 extern int Vmx86_ReleaseVM(VMDriver *vm);
-extern int Vmx86_InitVM(VMDriver *vm, InitBlock *initParams);
 extern int Vmx86_LateInitVM(VMDriver *vm);
 extern int Vmx86_RunVM(VMDriver *vm, Vcpuid vcpuid);
 extern void Vmx86_YieldToSet(VMDriver *vm, Vcpuid currVcpu, const VCPUSet *req,
@@ -141,17 +139,21 @@ extern int Vmx86_UnlockPage(VMDriver *vm, VA64 uAddr);
 extern int Vmx86_UnlockPageByMPN(VMDriver *vm, MPN mpn, VA64 uAddr);
 extern MPN Vmx86_GetRecycledPage(VMDriver *vm);
 extern int Vmx86_ReleaseAnonPage(VMDriver *vm, MPN mpn);
-extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
-                                  unsigned numPages, Bool kernelMPNBuffer,
-                                  Bool ignoreLimits);
-extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr,
-                                 unsigned numPages, Bool kernelMPNBuffer);
+extern int64 Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
+                                    PageCnt numPages, Bool kernelMPNBuffer,
+                                    Bool ignoreLimits);
+extern int Vmx86_FreeLockedPages(VMDriver *vm, MPN *mpns, PageCnt numPages);
 extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn);
+extern MPN Vmx86_GetNumAnonPages(VMDriver *vm);
+extern MPN Vmx86_AllocLowPage(VMDriver *vm, Bool ignoreLimits);
+extern void *Vmx86_Calloc(size_t numElements,
+                          size_t elementSize,
+                          Bool nonPageable);
+extern void  Vmx86_Free(void *ptr);

 extern int32 Vmx86_GetNumVMs(void);
-extern int32 Vmx86_GetTotalMemUsage(void);
-extern Bool Vmx86_SetConfiguredLockedPagesLimit(unsigned limit);
-extern void Vmx86_SetDynamicLockedPagesLimit(unsigned limit);
+extern Bool Vmx86_SetConfiguredLockedPagesLimit(PageCnt limit);
+extern void Vmx86_SetDynamicLockedPagesLimit(PageCnt limit);
 extern Bool Vmx86_GetMemInfo(VMDriver *curVM,
                             Bool curVMOnly,
                             VMMemInfoArgs *args,
@@ -210,4 +212,11 @@ extern uint64 Vmx86_GetPseudoTSC(void);

 extern uint64 Vmx86_GetUnavailablePerfCtrs(void);

+extern Bool Vmx86_GetMonitorContext(VMDriver *vm, Vcpuid vcpuid,
+                                    Context64 *context);
+extern void Vmx86_CleanupVMMPages(VMDriver *vm);
+
+extern VPN Vmx86_MapPage(MPN mpn);
+extern void Vmx86_UnmapPage(VPN vpn);
+
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -267,8 +267,8 @@ CircList_DeleteItem(ListItem *p,         // IN
 {
   ListItem *next;

-   ASSERT(p);
-   ASSERT(headp);
+   ASSERT(p != NULL);
+   ASSERT(headp != NULL);

   next = p->next;
   if (p == next) {
@@ -345,6 +345,34 @@ CircList_Push(ListItem *p,               // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_InsertAfter --
+ *
+ *      Adds a new member to the list after the provided item.  Assumes p
+ *      is not a member of a list already.
+ *
+ * Result:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE void
+CircList_InsertAfter(ListItem *p,      // IN:
+                     ListItem *after)  // IN:
+{
+   p->prev = after;
+   p->next = after->next;
+   p->next->prev = p;
+   after->next = p;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -26,9 +26,11 @@
 #ifndef _CONTEXTINFO_H
 #define _CONTEXTINFO_H

+#define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMX
 #include "includeCheck.h"

 #include "x86desc.h"
@@ -42,7 +42,7 @@
 typedef uint32 PCPU;
 #define INVALID_PCPU		((PCPU) -1)

-#define MAX_PCPUS       576
+#define MAX_PCPUS       1024
 #define MAX_PCPUS_BITS  10  // MAX_PCPUS <= (1 << MAX_PCPUS_BITS)
 #define MAX_PCPUS_MASK  ((1 << MAX_PCPUS_BITS) - 1)

@@ -1,104 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _CPUID_INFO_H
-#define _CPUID_INFO_H
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-
-#include "includeCheck.h"
-
-#include "vm_basic_asm.h"
-#include "x86cpuid_asm.h"
-
-#if defined __cplusplus
-extern "C" {
-#endif
-
-
-typedef struct CPUID0 {
-   int numEntries;
-   char name[16];      // 4 extra bytes to null terminate
-} CPUID0;
-
-typedef struct CPUID1 {
-   uint32 version;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID1;
-
-typedef struct CPUID80 {
-   uint32 numEntries;
-   uint32 ebx;
-   uint32 ecx;
-   uint32 edx;
-} CPUID80;
-
-typedef struct CPUID81 {
-   uint32 eax;
-   uint32 ebx;
-   uint32 ecxFeatures;
-   uint32 edxFeatures;
-} CPUID81;
-
-typedef struct CPUIDSummary {
-   CPUID0  id0;
-   CPUID1  id1;
-   CPUIDRegs ida;
-   CPUID80 id80;
-   CPUID81 id81;
-   CPUIDRegs id88, id8a;
-} CPUIDSummary;
-
-
-/*
- *----------------------------------------------------------------------
- *
- * CPUIDSummary_RegsFromCpuid0 --
- *
- *      Fills in the given CPUIDRegs struct with the values from the CPUID0 struct.
- *
- * Results:
- *      Returns the CPUIDRegs pointer passed in.
- *
- * Side effects:
- *	None.
- *
- *----------------------------------------------------------------------
- */
-static INLINE CPUIDRegs*
-CPUIDSummary_RegsFromCpuid0(CPUID0* id0In,
-                            CPUIDRegs* id0Out)
-{
-   id0Out->eax = id0In->numEntries;
-   id0Out->ebx = *(uint32 *) (id0In->name + 0);
-   id0Out->edx = *(uint32 *) (id0In->name + 4);
-   id0Out->ecx = *(uint32 *) (id0In->name + 8);
-   return id0Out;
-}
-
-
-#if defined __cplusplus
-} // extern "C"
-#endif
-
-#endif // _CPUID_INFO_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,6 +24,19 @@
 *      There is only one in the whole host system, shared by all VMs.
 *      It is allocated when the first VCPU is started and freed when the
 *      driver is unloaded.
+ *
+ *      The crossGDT size is exactly one page.
+ *
+ *      The hosted world switch code is based on the assumption that by
+ *      placing VMM descriptors at the end of the page, they will not
+ *      overlap with host kernel descriptors in use when "crossing over".
+ *
+ *      All necessary host segments must be from first page of GDT.
+ *      In Nov 2006, host GDT limits easily met this constraint:
+ *
+ *          Linux 64 bit:  80 (yes 80, not 7F)
+ *          MacOS 64 bit:  8F
+ *        Windows 64 bit:  6F
 */

 #ifndef _CROSSGDT_H_
@@ -38,31 +51,9 @@
 #include "vm_basic_defs.h"  // PAGE_SIZE
 #include "x86types.h"       // Descriptor

+
 typedef struct CrossGDT {
-   Descriptor gdtes[0x5000 / sizeof (Descriptor)];  // 0x5000 > GDT_LIMIT
+   Descriptor gdtes[PAGE_SIZE / sizeof(Descriptor)];
 } CrossGDT;

-#define CROSSGDT_NUMPAGES CEILING(sizeof (CrossGDT), PAGE_SIZE)
-
-/*
- * Out of the 5 pages, only the first and last are really used.
- *
- * All we need to map are the first and last pages.  This mask tells
- * the setup code which pages it can put stuff in and it tells the
- * mapping and invalidation code which pages are mapped and unmapped.
- */
-#define CROSSGDT_PAGEMASK 0x11
-#define CROSSGDT_GETINDEXMASK(i) (1 << ((i) * sizeof (Descriptor) / PAGE_SIZE))
-#define CROSSGDT_TESTINDEXMASK(i) (CROSSGDT_GETINDEXMASK(i) & CROSSGDT_PAGEMASK)
-
-/*
- * All necessary host segments must be below CROSSGDT_HOSTLIMIT. In Nov 2006,
- * host GDT limits for the various guest OSes were:
- *
- *     Linux 64 bit:  80 (yes 80, not 7F)
- *     MacOS 64 bit:  8F
- *   Windows 64 bit:  6F
- */
-#define CROSSGDT_HOSTLIMIT (PAGE_SIZE / sizeof (Descriptor)) /* 1st page */
-
 #endif
@@ -1,345 +0,0 @@
-/*********************************************************
- * Copyright (C) 2001,2016 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _HASH_FUNC_H
-#define _HASH_FUNC_H
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-
-#include "includeCheck.h"
-
-/*
- * hashFunc.h --
- *
- *	Hash functions defined as simple wrappers around code placed
- *	in the public domain by Bob Jenkins.  His original code can
- *	be used freely for any purpose, including commercial (see the
- *	original comments in the enclosed code).
- */
-
-#include "vm_basic_types.h"
-#include "vm_basic_defs.h"
-#include "vm_assert.h"
-
-/*
- * operations
- */
-
-extern uint64 HashFunc_HashBytes(const uint8 *key,  uint32 nBytes);
-extern uint64 HashFunc_HashMoreBytes(const uint8 *key, uint32 nBytes, 
-                                     uint64 initialValue);
-extern uint64 HashFunc_HashBytesSlow(const uint8 *key, uint32 nBytes);
-extern uint64 HashFunc_HashQuads(const uint64 *key, uint32 nQuads);
-extern uint64 HashFunc_HashPage(const void *addr);
-
-/*
--------------------------------------------------------------------
-Hashes 32-bit key and returns a value that can fit into numBits bits.
--------------------------------------------------------------------
-*/
-
-static INLINE uint32
-HashFunc_TruncHash32(uint32 key, unsigned numBits)
-{
-   const uint32 hashConst = 0x9e3779b9; /* the golden ratio */
-   ASSERT(numBits > 0 && numBits <= 32);
-   return (key * hashConst) >> (32 - numBits);
-}
-
-/*
--------------------------------------------------------------------
-Hashes 64-bit key and returns a value that can fit into numBits bits.
--------------------------------------------------------------------
-*/
-
-static INLINE uint32
-HashFunc_TruncHash64(uint64 key, unsigned numBits)
-{
-   return HashFunc_TruncHash32((uint32)(key ^ (key >> 32)), numBits);
-}
-
-
-/*
- * Original Bob Jenkins public-domain code with minor modifications
- * (static functions, 32-bit length args, disabled self-test code).
- */
-
-
-typedef uint64 ub8;   /* unsigned 8-byte quantities */
-typedef uint32 ub4;   /* unsigned 4-byte quantities */
-typedef uint8  ub1;
-
-#define hashsize(n) ((ub8)1<<(n))
-#define hashmask(n) (hashsize(n)-1)
-
-/*
--------------------------------------------------------------------
-mix -- mix 3 64-bit values reversibly.
-mix() takes 48 machine instructions, but only 24 cycles on a superscalar
-  machine (like Intel's new MMX architecture).  It requires 4 64-bit
-  registers for 4::2 parallelism.
-All 1-bit deltas, all 2-bit deltas, all deltas composed of top bits of
-  (a,b,c), and all deltas of bottom bits were tested.  All deltas were
-  tested both on random keys and on keys that were nearly all zero.
-  These deltas all cause every bit of c to change between 1/3 and 2/3
-  of the time (well, only 113/400 to 287/400 of the time for some
-  2-bit delta).  These deltas all cause at least 80 bits to change
-  among (a,b,c) when the mix is run either forward or backward (yes it
-  is reversible).
-This implies that a hash using mix64 has no funnels.  There may be
-  characteristics with 3-bit deltas or bigger, I didn't test for
-  those.
--------------------------------------------------------------------
-*/
-#define mix64(a,b,c) \
-{ \
-  a -= b; a -= c; a ^= (c>>43); \
-  b -= c; b -= a; b ^= (a<<9); \
-  c -= a; c -= b; c ^= (b>>8); \
-  a -= b; a -= c; a ^= (c>>38); \
-  b -= c; b -= a; b ^= (a<<23); \
-  c -= a; c -= b; c ^= (b>>5); \
-  a -= b; a -= c; a ^= (c>>35); \
-  b -= c; b -= a; b ^= (a<<49); \
-  c -= a; c -= b; c ^= (b>>11); \
-  a -= b; a -= c; a ^= (c>>12); \
-  b -= c; b -= a; b ^= (a<<18); \
-  c -= a; c -= b; c ^= (b>>22); \
-}
-
-/*
--------------------------------------------------------------------
-hash() -- hash a variable-length key into a 64-bit value
-  k     : the key (the unaligned variable-length array of bytes)
-  len   : the length of the key, counting by bytes
-  level : can be any 8-byte value
-Returns a 64-bit value.  Every bit of the key affects every bit of
-the return value.  No funnels.  Every 1-bit and 2-bit delta achieves
-avalanche.  About 41+5len instructions.
-
-The best hash table sizes are powers of 2.  There is no need to do
-mod a prime (mod is sooo slow!).  If you need less than 64 bits,
-use a bitmask.  For example, if you need only 10 bits, do
-  h = (h & hashmask(10));
-In which case, the hash table should have hashsize(10) elements.
-
-If you are hashing n strings (ub1 **)k, do it like this:
-  for (i=0, h=0; i<n; ++i) h = hash( k[i], len[i], h);
-
-By Bob Jenkins, Jan 4 1997.  bob_jenkins@burtleburtle.net.  You may
-use this code any way you wish, private, educational, or commercial,
-as long as this whole comment accompanies it.
-
-See http://burtleburtle.net/bob/hash/evahash.html
-Use for hash table lookup, or anything where one collision in 2^^64
-is acceptable.  Do NOT use for cryptographic purposes.
--------------------------------------------------------------------
-*/
-
-static INLINE ub8 hash(register const ub1 *k, /* the key */
-                       register ub4 length,   /* the length of the key */
-                       register ub8 level)    /* the previous hash, or an arbitrary value */
-{
-  register ub8 a,b,c;
-  ub4 len;
-
-  /* Set up the internal state */
-  len = length;
-  a = b = level;                         /* the previous hash value */
-  c = CONST64U(0x9e3779b97f4a7c13); /* the golden ratio; an arbitrary value */
-
-  /*---------------------------------------- handle most of the key */
-  while (len >= 24)
-  {
-    a += (k[0]        +((ub8)k[ 1]<< 8)+((ub8)k[ 2]<<16)+((ub8)k[ 3]<<24)
-     +((ub8)k[4 ]<<32)+((ub8)k[ 5]<<40)+((ub8)k[ 6]<<48)+((ub8)k[ 7]<<56));
-    b += (k[8]        +((ub8)k[ 9]<< 8)+((ub8)k[10]<<16)+((ub8)k[11]<<24)
-     +((ub8)k[12]<<32)+((ub8)k[13]<<40)+((ub8)k[14]<<48)+((ub8)k[15]<<56));
-    c += (k[16]       +((ub8)k[17]<< 8)+((ub8)k[18]<<16)+((ub8)k[19]<<24)
-     +((ub8)k[20]<<32)+((ub8)k[21]<<40)+((ub8)k[22]<<48)+((ub8)k[23]<<56));
-    mix64(a,b,c);
-    k += 24; len -= 24;
-  }
-
-  /*------------------------------------- handle the last 23 bytes */
-  c += length;
-  switch(len)              /* all the case statements fall through */
-  {
-  case 23: c+=((ub8)k[22]<<56);
-  case 22: c+=((ub8)k[21]<<48);
-  case 21: c+=((ub8)k[20]<<40);
-  case 20: c+=((ub8)k[19]<<32);
-  case 19: c+=((ub8)k[18]<<24);
-  case 18: c+=((ub8)k[17]<<16);
-  case 17: c+=((ub8)k[16]<<8);
-    /* the first byte of c is reserved for the length */
-  case 16: b+=((ub8)k[15]<<56);
-  case 15: b+=((ub8)k[14]<<48);
-  case 14: b+=((ub8)k[13]<<40);
-  case 13: b+=((ub8)k[12]<<32);
-  case 12: b+=((ub8)k[11]<<24);
-  case 11: b+=((ub8)k[10]<<16);
-  case 10: b+=((ub8)k[ 9]<<8);
-  case  9: b+=((ub8)k[ 8]);
-  case  8: a+=((ub8)k[ 7]<<56);
-  case  7: a+=((ub8)k[ 6]<<48);
-  case  6: a+=((ub8)k[ 5]<<40);
-  case  5: a+=((ub8)k[ 4]<<32);
-  case  4: a+=((ub8)k[ 3]<<24);
-  case  3: a+=((ub8)k[ 2]<<16);
-  case  2: a+=((ub8)k[ 1]<<8);
-  case  1: a+=((ub8)k[ 0]);
-    /* case 0: nothing left to add */
-  }
-  mix64(a,b,c);
-  /*-------------------------------------------- report the result */
-  return c;
-}
-
-/*
--------------------------------------------------------------------
- This works on all machines, is identical to hash() on little-endian 
- machines, and it is much faster than hash(), but it requires
- -- that the key be an array of ub8's, and
- -- that all your machines have the same endianness, and
- -- that the length be the number of ub8's in the key
--------------------------------------------------------------------
-*/
-static INLINE ub8 hash2(register const ub8 *k, /* the key */
-                        register ub4 length,   /* the length of the key */
-                        register ub8 level)    /* the previous hash, or an arbitrary value */
-{
-  register ub8 a,b,c;
-  ub4 len;
-
-  /* Set up the internal state */
-  len = length;
-  a = b = level;                         /* the previous hash value */
-  c = CONST64U(0x9e3779b97f4a7c13); /* the golden ratio; an arbitrary value */
-
-  /*---------------------------------------- handle most of the key */
-  while (len >= 3)
-  {
-    a += k[0];
-    b += k[1];
-    c += k[2];
-    mix64(a,b,c);
-    k += 3; len -= 3;
-  }
-
-  /*-------------------------------------- handle the last 2 ub8's */
-  c += length;
-  switch(len)              /* all the case statements fall through */
-  {
-    /* c is reserved for the length */
-  case  2: b+=k[1];
-  case  1: a+=k[0];
-    /* case 0: nothing left to add */
-  }
-  mix64(a,b,c);
-  /*-------------------------------------------- report the result */
-  return c;
-}
-
-/*
--------------------------------------------------------------------
- This is identical to hash() on little-endian machines, and it is much
- faster than hash(), but a little slower than hash2(), and it requires
- -- that all your machines be little-endian, for example all Intel x86
-    chips or all VAXen.  It gives wrong results on big-endian machines.
--------------------------------------------------------------------
-*/
-
-static INLINE ub8 hash3(register const ub1 *k, /* the key */
-                        register ub4 length,   /* the length of the key */
-                        register ub8 level)    /* the previous hash, or an arbitrary value */
-{
-  register ub8 a,b,c;
-  ub4 len;
-
-  /* Set up the internal state */
-  len = length;
-  a = b = level;                         /* the previous hash value */
-  c = CONST64U(0x9e3779b97f4a7c13); /* the golden ratio; an arbitrary value */
-
-  /*---------------------------------------- handle most of the key */
-  if (((size_t)k)&7)
-  {
-    while (len >= 24)
-    {
-      a += (k[0]        +((ub8)k[ 1]<< 8)+((ub8)k[ 2]<<16)+((ub8)k[ 3]<<24)
-       +((ub8)k[4 ]<<32)+((ub8)k[ 5]<<40)+((ub8)k[ 6]<<48)+((ub8)k[ 7]<<56));
-      b += (k[8]        +((ub8)k[ 9]<< 8)+((ub8)k[10]<<16)+((ub8)k[11]<<24)
-       +((ub8)k[12]<<32)+((ub8)k[13]<<40)+((ub8)k[14]<<48)+((ub8)k[15]<<56));
-      c += (k[16]       +((ub8)k[17]<< 8)+((ub8)k[18]<<16)+((ub8)k[19]<<24)
-       +((ub8)k[20]<<32)+((ub8)k[21]<<40)+((ub8)k[22]<<48)+((ub8)k[23]<<56));
-      mix64(a,b,c);
-      k += 24; len -= 24;
-    }
-  }
-  else
-  {
-    while (len >= 24)    /* aligned */
-    {
-      a += *(ub8 *)(k+0);
-      b += *(ub8 *)(k+8);
-      c += *(ub8 *)(k+16);
-      mix64(a,b,c);
-      k += 24; len -= 24;
-    }
-  }
-
-  /*------------------------------------- handle the last 23 bytes */
-  c += length;
-  switch(len)              /* all the case statements fall through */
-  {
-  case 23: c+=((ub8)k[22]<<56);
-  case 22: c+=((ub8)k[21]<<48);
-  case 21: c+=((ub8)k[20]<<40);
-  case 20: c+=((ub8)k[19]<<32);
-  case 19: c+=((ub8)k[18]<<24);
-  case 18: c+=((ub8)k[17]<<16);
-  case 17: c+=((ub8)k[16]<<8);
-    /* the first byte of c is reserved for the length */
-  case 16: b+=((ub8)k[15]<<56);
-  case 15: b+=((ub8)k[14]<<48);
-  case 14: b+=((ub8)k[13]<<40);
-  case 13: b+=((ub8)k[12]<<32);
-  case 12: b+=((ub8)k[11]<<24);
-  case 11: b+=((ub8)k[10]<<16);
-  case 10: b+=((ub8)k[ 9]<<8);
-  case  9: b+=((ub8)k[ 8]);
-  case  8: a+=((ub8)k[ 7]<<56);
-  case  7: a+=((ub8)k[ 6]<<48);
-  case  6: a+=((ub8)k[ 5]<<40);
-  case  5: a+=((ub8)k[ 4]<<32);
-  case  4: a+=((ub8)k[ 3]<<24);
-  case  3: a+=((ub8)k[ 2]<<16);
-  case  2: a+=((ub8)k[ 1]<<8);
-  case  1: a+=((ub8)k[ 0]);
-    /* case 0: nothing left to add */
-  }
-  mix64(a,b,c);
-  /*-------------------------------------------- report the result */
-  return c;
-}
-
-#endif //_HASH_FUNC_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,13 +37,15 @@
 #error iocontrols.h is for hosted vmmon, do not use on visor
 #endif

-#include "basic_initblock.h"
 #include "x86segdescrs.h"
 #include "rateconv.h"
 #include "overheadmem_types.h"
 #include "pageLock_defs.h"
 #include "numa_defs.h"
 #include "bootstrap_vmm.h"
+#include "contextinfo.h"
+#include "vcpuid.h"
+#include "sharedAreaType.h"

 #if defined __cplusplus
 extern "C" {
@@ -138,13 +140,12 @@ PtrToVA64(void const *ptr) // IN
 * vmcore, and rest-of-bora. The vmmon driver is largely outside
 * vmcore and vmcore imports functionality from vmmon. Addition,
 * deletion or modification of an iocontrol used only by rest-of-bora
- * does not break vmcore compatibility. 
+ * does not break vmcore compatibility.
 *
 * See bora/doc/vmcore details.
- *
 */

-#define VMMON_VERSION           (331 << 16 | 0)
+#define VMMON_VERSION           (385 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -153,13 +154,10 @@ PtrToVA64(void const *ptr) // IN
 * ENOMEM returned after MAX_VMS virtual machines created
 */

-#ifdef VMX86_SERVER
-#define MAX_VMS 128
-#else
 #define MAX_VMS 64
-#endif
-/* 
- * MsgWaitForMultipleObjects doesn't scale well enough on Win32. 
+
+/*
+ * MsgWaitForMultipleObjects doesn't scale well enough on Win32.
 * Allocate with MAX_VMS so static buffers are large, but do
 * admissions control with this value on Win32 until we check
 * scalability (probably in authd).
@@ -209,11 +207,13 @@ enum IOCTLCmd {
   IOCTLCMD(VERSION) = IOCTLCMD(FIRST),
   IOCTLCMD(CREATE_VM),
   IOCTLCMD(PROCESS_BOOTSTRAP),
+   IOCTLCMD(REGISTER_SHARED),
+   IOCTLCMD(REGISTER_STATVARS),
   IOCTLCMD(RELEASE_VM),
   IOCTLCMD(GET_NUM_VMS),
-   IOCTLCMD(INIT_VM),
   IOCTLCMD(RUN_VM),
   IOCTLCMD(LOOK_UP_MPN),
+   IOCTLCMD(GET_VMM_PAGE_ROOT),
   IOCTLCMD(LOCK_PAGE),
   IOCTLCMD(UNLOCK_PAGE),
   IOCTLCMD(APIC_INIT),
@@ -222,7 +222,6 @@ enum IOCTLCmd {
   IOCTLCMD(ADMIT),
   IOCTLCMD(UPDATE_MEM_INFO),
   IOCTLCMD(READMIT),
-   IOCTLCMD(GET_TOTAL_MEM_USAGE),
   IOCTLCMD(GET_KHZ_ESTIMATE),
   IOCTLCMD(SET_HOST_CLOCK_RATE),
   IOCTLCMD(READ_PAGE),
@@ -231,13 +230,11 @@ enum IOCTLCmd {
   IOCTLCMD(UNLOCK_PAGE_BY_MPN),
    /* AWE calls */
   IOCTLCMD(ALLOC_LOCKED_PAGES),
-   IOCTLCMD(FREE_LOCKED_PAGES),
   IOCTLCMD(GET_NEXT_ANON_PAGE),
+   IOCTLCMD(GET_NUM_ANON_PAGES),

   IOCTLCMD(GET_ALL_MSRS),

-   IOCTLCMD(COUNT_PRESENT_PAGES),
-  
   IOCTLCMD(GET_REFERENCE_CLOCK_HZ),
   IOCTLCMD(INIT_PSEUDO_TSC),
   IOCTLCMD(CHECK_PSEUDO_TSC),
@@ -247,6 +244,7 @@ enum IOCTLCmd {

   IOCTLCMD(GET_IPI_VECTORS),
   IOCTLCMD(SEND_IPI),
+   IOCTLCMD(SEND_ONE_IPI),

   /*
    * Keep host-specific calls at the end so they can be undefined
@@ -269,8 +267,8 @@ enum IOCTLCmd {
   IOCTLCMD(FREE_CONTIG_PAGES),
   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows
   IOCTLCMD(CHANGE_HARD_LIMIT),         // used by vmauthd on Windows
-   IOCTLCMD(GET_KERNEL_PROC_ADDRESS),
-   IOCTLCMD(READ_VA64),
+   IOCTLCMD(READ_DISASM_PROC_BINARY),
+   IOCTLCMD(CHECK_CANDIDATE_VA64),
   IOCTLCMD(SET_MEMORY_PARAMS),
   IOCTLCMD(REMEMBER_KHZ_ESTIMATE),
   IOCTLCMD(REMAP_SCATTER_LIST),
@@ -278,7 +276,7 @@ enum IOCTLCmd {
 #endif

 #if defined __APPLE__
-   IOCTLCMD(ALLOC_LOW_PAGES),
+   IOCTLCMD(GET_NUM_RESPONDING_CPUS),
   IOCTLCMD(INIT_DRIVER),
   IOCTLCMD(BLUEPILL),
 #endif
@@ -288,6 +286,7 @@ enum IOCTLCmd {
 #endif

   IOCTLCMD(GET_UNAVAIL_PERF_CTRS),
+   IOCTLCMD(GET_MONITOR_CONTEXT),
   // Must be last.
   IOCTLCMD(LAST)
 };
@@ -318,12 +317,15 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_CREATE_VM           VMIOCTL_BUFFERED(CREATE_VM)
 #define IOCTL_VMX86_RELEASE_VM          VMIOCTL_BUFFERED(RELEASE_VM)
 #define IOCTL_VMX86_PROCESS_BOOTSTRAP   VMIOCTL_BUFFERED(PROCESS_BOOTSTRAP)
+#define IOCTL_VMX86_REGISTER_SHARED     VMIOCTL_BUFFERED(REGISTER_SHARED)
+#define IOCTL_VMX86_REGISTER_STATVARS   VMIOCTL_BUFFERED(REGISTER_STATVARS)
 #define IOCTL_VMX86_GET_NUM_VMS         VMIOCTL_BUFFERED(GET_NUM_VMS)
-#define IOCTL_VMX86_INIT_VM             VMIOCTL_BUFFERED(INIT_VM)
 #define IOCTL_VMX86_RUN_VM              VMIOCTL_NEITHER(RUN_VM)
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
+#define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
+#define IOCTL_VMX86_GET_VMM_PAGE_ROOT   VMIOCTL_BUFFERED(GET_VMM_PAGE_ROOT)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
 #define IOCTL_VMX86_UNLOCK_PAGE         VMIOCTL_BUFFERED(UNLOCK_PAGE)
 #define IOCTL_VMX86_APIC_INIT           VMIOCTL_BUFFERED(APIC_INIT)
@@ -346,11 +348,13 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_LOCK_PAGE_NEW       VMIOCTL_BUFFERED(LOCK_PAGE_NEW)
 #define IOCTL_VMX86_UNLOCK_PAGE_BY_MPN  VMIOCTL_BUFFERED(UNLOCK_PAGE_BY_MPN)
 #define IOCTL_VMX86_ALLOC_LOCKED_PAGES  VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES)
-#define IOCTL_VMX86_FREE_LOCKED_PAGES   VMIOCTL_BUFFERED(FREE_LOCKED_PAGES)
 #define IOCTL_VMX86_GET_NEXT_ANON_PAGE  VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE)
+#define IOCTL_VMX86_GET_NUM_ANON_PAGES  VMIOCTL_BUFFERED(GET_NUM_ANON_PAGES)
+
+#define IOCTL_VMX86_READ_DISASM_PROC_BINARY \
+                                      VMIOCTL_BUFFERED(READ_DISASM_PROC_BINARY)
+#define IOCTL_VMX86_CHECK_CANDIDATE_VA64 VMIOCTL_BUFFERED(CHECK_CANDIDATE_VA64)

-#define IOCTL_VMX86_GET_KERNEL_PROC_ADDRESS  VMIOCTL_BUFFERED(GET_KERNEL_PROC_ADDRESS)
-#define IOCTL_VMX86_READ_VA64           VMIOCTL_BUFFERED(READ_VA64)
 #define IOCTL_VMX86_SET_MEMORY_PARAMS   VMIOCTL_BUFFERED(SET_MEMORY_PARAMS)

 #define IOCTL_VMX86_REMEMBER_KHZ_ESTIMATE VMIOCTL_BUFFERED(REMEMBER_KHZ_ESTIMATE)
@@ -362,11 +366,14 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_CHECK_PSEUDO_TSC         VMIOCTL_BUFFERED(CHECK_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_PSEUDO_TSC           VMIOCTL_NEITHER(GET_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS    VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS)
+#define IOCTL_VMX86_GET_MONITOR_CONTEXT      VMIOCTL_BUFFERED(GET_MONITOR_CONTEXT)
 #define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
 #define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
 #endif


+#define INIT_BLOCK_MAGIC     (0x1789 + 14)
+
 /*
 * Flags sent into APICBASE ioctl
 */
@@ -393,27 +400,35 @@ union {
 #include "vmware_pack_end.h"
 VMLockPage;

+typedef
+#include "vmware_pack_begin.h"
+union {
+   Vcpuid vcpuid; // IN: VCPU
+   MPN pageRoot;  // OUT: MPN of the VCPU's page root
+}
+#include "vmware_pack_end.h"
+VcpuPageRoot;

 typedef struct VMAPICInfo {
   uint32 flags;
-} VMAPICInfo; 
+} VMAPICInfo;

-#define VMX86_DRIVER_VCPUID_OFFSET	1000
+#define VMX86_DRIVER_VCPUID_OFFSET 1000


 /*
 * We keep track of 3 different limits on the number of pages we can lock.
 * The host limit is determined at driver load time (in windows only) to
 * make sure we do not starve the host by locking too many pages.
- * The static limit is user defined in the UI and the dynamic limit is 
+ * The static limit is user defined in the UI and the dynamic limit is
 * set by authd's hardLimitMonitor code (windows only), which queries
- * host load and adjusts the limit accordingly.  We lock the minimum of 
+ * host load and adjusts the limit accordingly.  We lock the minimum of
 * all these values at any given time.
 */
 typedef struct LockedPageLimit {
-   uint32 host;        // driver calculated maximum for this host
-   uint32 configured;  // user defined maximum pages to lock
-   uint32 dynamic;     // authd hardLimitMonitor pages to lock
+   PageCnt host;        // driver calculated maximum for this host
+   PageCnt configured;  // user defined maximum pages to lock
+   PageCnt dynamic;     // authd hardLimitMonitor pages to lock
 } LockedPageLimit;

 /*
@@ -434,15 +449,15 @@ typedef struct LockedPageLimit {
 */

 typedef struct VMMemMgmtInfo {
-   uint32          minAllocation;   // minimum pages for vm
-   uint32          maxAllocation;   // maximum pages the vm could lock
+   PageCnt         minAllocation;   // minimum pages for vm
+   PageCnt         maxAllocation;   // maximum pages the vm could lock
+   PageCnt         nonpaged;        // overhead memory (guest, mmap)
+   PageCnt         paged;           // vmx memory (malloc, statics)
+   PageCnt         anonymous;       // vmm memory
+   PageCnt         mainMemSize;     // guest main memory size
+   PageCnt         locked;          // number of pages locked by this vm
+   PageCnt         perVMOverhead;   // memory for vmx/vmmon overheads
   uint32          shares;          // proportional sharing weight
-   uint32          nonpaged;        // overhead memory (guest, mmap)
-   uint32          paged;           // vmx memory (malloc, statics)
-   uint32          anonymous;       // vmm memory
-   uint32          mainMemSize;     // guest main memory size
-   uint32          locked;          // number of pages locked by this vm
-   uint32          perVMOverhead;   // memory for vmx/vmmon overheads
   Percent         touchedPct;      // % of guest memory being touched
   Percent         dirtiedPct;      // % of guest memory being dirtied
   Bool            admitted;        // admission control
@@ -464,15 +479,15 @@ typedef struct VMMemMgmtInfoPatch {
 */

 typedef struct VMMemInfoArgs {
-   uint64          currentTime;        // Host time in secs of the call. 
-   uint32          minVmMemPct;        // % of vm that must fit in memory
-   uint32          globalMinAllocation;// pages that must fit in maxLockedPages
-   uint32          numLockedPages;     // total locked pages by all vms
+   uint64          currentTime;        // Host time in secs of the call.
+   PageCnt         globalMinAllocation;// pages that must fit in maxLockedPages
+   PageCnt         numLockedPages;     // total locked pages by all vms
   LockedPageLimit lockedPageLimit;    // set of locked page limits
-   uint32          maxLockedPages;     // effective limit on locked pages
+   PageCnt         maxLockedPages;     // effective limit on locked pages
   uint32          callerIndex;        // this vm's index memInfo array
   uint32          numVMs;             // number of running VMs
-   uint8           _pad[4];
+   Percent         minVmMemPct;        // % of vm that must fit in memory
+   uint8           _pad[7];
   VMMemMgmtInfo   memInfo[1];
 } VMMemInfoArgs;

@@ -485,19 +500,12 @@ typedef struct VMMPNNext {
 } VMMPNNext;

 typedef struct VMMPNList {
-   uint32    mpnCount;   // IN (and OUT on Mac OS)
+   PageCnt   mpnCount;   // IN (and OUT on Mac OS)
   Bool      ignoreLimits;
-   uint8     _pad[3];
+   uint8     _pad[7];
   VA64      mpnList;    // IN: User VA of an array of 64-bit MPNs.
 } VMMPNList;

-typedef struct VARange {
-   VA64     addr;
-   VA64     bv;
-   unsigned len;
-   uint32   pad;
-} VARange;
-
 typedef struct VMMUnlockPageByMPN {
   MPN       mpn;
   VA64      uAddr;         /* IN: User VA of the page (optional). */
@@ -508,11 +516,6 @@ typedef struct VMMReadWritePage {
   VA64         uAddr; // IN: User VA of a PAGE_SIZE-large buffer.
 } VMMReadWritePage;

-struct passthrough_iorange {
-   unsigned short ioBase;   /* Base of range to pass through. */
-   unsigned short numPorts; /* Length of range. */
-};
-
 /*
 * Data structure for the INIT_PSEUDO_TSC and CHECK_PSEUDO_TSC.
 */
@@ -535,29 +538,21 @@ typedef struct PTSCCheckParams {
   uint8  _pad[7];
 } PTSCCheckParams;

-#ifndef VMX86_SERVER
-
 typedef struct IPIVectors {
   /*
-    * Vector(s) the host uses for its own IPIs; we use this as a performance
-    * hint.
-    */
-   uint8 hostIPIVectors[2];
-   /* 
    * Vectors we have allocated or stolen for the monitor interrupts.
    */
-   uint8 monitorIPIVector; 
+   uint8 monitorIPIVector;
   uint8 hvIPIVector;
 } IPIVectors;

-#endif
-
 /*
 * Arguments and return value for VM creation.
 */
 typedef struct VMCreateBlock {
   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
+   uint32             numVCPUs;      // IN: Number of VCPUs.
   uint16             vmid;          // OUT: VM ID for the created VM.
 } VMCreateBlock;

@@ -565,11 +560,33 @@ typedef struct VMCreateBlock {
 * Information about a shared region.
 */
 typedef struct VMSharedRegion {
-   uint32 index;
-   VPN    baseVpn;
-   uint32 numPages;
+   SharedAreaType index;
+   VPN            baseVpn;
+   uint32         numPages;
 } VMSharedRegion;

+/*
+ * Arguments for VMM shared area registration.
+ */
+typedef struct VMSharedAreaRegistrationBlock {
+   Vcpuid vcpu;            // IN: VCPU being registered.
+   VMSharedRegion region;  // IN: Shared region being registered.
+} VMSharedAreaRegistrationBlock;
+
+/*
+ * Information about a VM's statvars.
+ */
+typedef struct VMStatVarsRegistrationBlock {
+   VPN     baseVpn;
+   PageCnt numPages;
+   Vcpuid  vcpu;
+} VMStatVarsRegistrationBlock;
+
+typedef struct {
+   VA64   crosspage; // IN: User VA of VCPU crosspage.
+   VA64   ptRoot;    // IN: User VA of VCPU L4 page table root.
+} PerVcpuPages;
+
 /*
 * Arguments for VMM bootstrap processing.
 */
@@ -578,10 +595,25 @@ typedef struct VMProcessBootstrapBlock {
   uint32         numBytes;      // IN: Size of VMM bootstrap blob.
   uint32         headerOffset;  // IN: Offset of header in blob.
   uint16         numVCPUs;      // IN: Number of VCPUs.
-   VA64           ptRootVAs[MAX_VCPUS];  // IN: User VA of PT roots.
   VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions.
+   PerVcpuPages   perVcpuPages[0];
 } VMProcessBootstrapBlock;

+static INLINE size_t
+GetVMProcessBootstrapBlockSize(unsigned numVCPUs)
+{
+   VMProcessBootstrapBlock *args = NULL;
+   return sizeof *args + numVCPUs * sizeof args->perVcpuPages[0];
+}
+
+/*
+ * Arguments for VMM context retrieval.
+ */
+typedef union {
+   Vcpuid vcpuid;     // IN
+   Context64 context; // OUT
+} VMMonContext;
+
 #if defined __linux__

 /*
@@ -589,29 +621,20 @@ typedef struct VMProcessBootstrapBlock {
 * macros to marshall real arguments to mmap's made-up 'offset' argument.
 */

-#define VMMON_MAP_MT_LOW4GB     0
-#define VMMON_MAP_MT_LOW16MB    1
-#define VMMON_MAP_MT_ANY        2
-
 #define VMMON_MAP_OFFSET_SHIFT  0
 #define VMMON_MAP_OFFSET_MASK   0x00000FFF
 #define VMMON_MAP_ORDER_SHIFT   12
 #define VMMON_MAP_ORDER_MASK    0xF
-#define VMMON_MAP_MT_SHIFT      16
-#define VMMON_MAP_MT_MASK       0x7
-#define VMMON_MAP_RSVD_SHIFT    19
+#define VMMON_MAP_RSVD_SHIFT    16

 #define VMMON_MAP_RSVD(base)    \
                ((base) >> VMMON_MAP_RSVD_SHIFT)
-#define VMMON_MAP_MT(base)      \
-                (((base) >> VMMON_MAP_MT_SHIFT) & VMMON_MAP_MT_MASK)
 #define VMMON_MAP_ORDER(base)   \
                (((base) >> VMMON_MAP_ORDER_SHIFT) & VMMON_MAP_ORDER_MASK)
 #define VMMON_MAP_OFFSET(base)  \
                (((base) >> VMMON_MAP_OFFSET_SHIFT) & VMMON_MAP_OFFSET_MASK)

-#define VMMON_MAP_BASE(mt, order)   (((mt) << VMMON_MAP_MT_SHIFT) | \
-                                     ((order) << VMMON_MAP_ORDER_SHIFT))
+#define VMMON_MAP_BASE(order)   ((order) << VMMON_MAP_ORDER_SHIFT)

 #elif defined _WIN32
 /*
@@ -622,7 +645,6 @@ typedef struct VMAllocContiguousMem {
   VA64   mpnList;  // IN: User VA of an array of 64-bit MPNs.
   uint32 mpnCount; // IN
   uint32 order;    // IN
-   MPN    maxMPN;   // IN
 } VMAllocContiguousMem;
 #elif defined __APPLE__
 #   include "iocontrolsMacos.h"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,14 +41,17 @@
 #include "modulecallstructs.h"
 #include "mon_assert.h"

-#define NUM_EXCEPTIONS 20       /* EXC_DE ... EXC_XF. */
- 
+#define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
+#define CP_STUB_SIZE     16     /* A relative jmp instruction (5 bytes)
+                                   padded to the next 16 byte boundary. */
+
 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
   MC(SEMAWAIT)                                                               \
   MC(SEMASIGNAL)                                                             \
   MC(SEMAFORCEWAKEUP)                                                        \
-   MC(IPI)          /* Hit thread with IPI. */                                \
+   MC(IPI)          /* Hit multiple threads with IPI. */                      \
+   MC(ONE_IPI)      /* Hit a single thread with IPI. */                       \
   MC(USERRETURN)   /* Return codes for user calls. */                        \
   MC(GET_RECYCLED_PAGES)                                                     \
   MC(RELEASE_ANON_PAGES)                                                     \
@@ -58,7 +61,15 @@
   MC(ALLOC_TMP_GDT)                                                          \
   MC(PIN_MPN)                                                                \
   MC(VMCLEAR_VMCS_ALL_CPUS)                                                  \
-   MC(GET_PAGE_ROOT)
+   MC(GET_PAGE_ROOT)                                                          \
+   MC(ALLOC_ANON_LOW_PAGE)                                                    \
+   MC(GET_MON_IPI_VECTOR)                                                     \
+   MC(GET_HV_IPI_VECTOR)                                                      \
+   MC(GET_HOST_TIMER_VECTORS)                                                 \
+   MC(BOOTSTRAP_CLEANUP)                                                      \
+   MC(GET_SHARED_AREA)                                                        \
+   MC(GET_STAT_VARS)                                                          \
+   MC(GET_NUM_PTP_PAGES)

 /*
 *----------------------------------------------------------------------
@@ -67,9 +78,6 @@
 *
 *      Enumeration of support calls done by the module.
 *
- *      If anything changes in the enum, please update kstatModuleCallPtrs
- *      for stats purposes.
- *
 *----------------------------------------------------------------------
 */

@@ -126,15 +134,6 @@ typedef enum UCCostStamp {
 } UCCostStamp;
 #endif // VMX86_SERVER

-typedef
-#include "vmware_pack_begin.h"
-struct CodeOffsets {
-   uint16 hostToVmm;
-   uint16 vmmToHost;
-}
-#include "vmware_pack_end.h"
-CodeOffsets;
-
 #define SHADOW_DR(cpData, n)    (cpData)->shadowDR[n].ureg64


@@ -143,12 +142,12 @@ CodeOffsets;
 * MAX_SWITCH_PT_PATCHES
 *
 *   This is the maximum number of patches that must be placed into
- *   the monitor page tables so that two pages of the host GDT and the
- *   crosspage can be accessed during worldswitch.
+ *   the monitor page tables so that the host GDT and the crosspage
+ *   can be accessed during worldswitch.
 *
 *----------------------------------------------------------------------
 */
-#define MAX_SWITCH_PT_PATCHES 3
+#define MAX_SWITCH_PT_PATCHES 2

 /*----------------------------------------------------------------------
 *
@@ -214,6 +213,7 @@ struct VMMPageTablePatch {
   uint32   level;              /* [0, 4]  (maximal size: 3 bits) */
   uint32   pteIdx;             /* Index of the PTE in the page table. */
   uint64   pteGlobalIdx;       /* Global index of the PTE in 'level'. */
+   LPN      lpn;                /* Logical page number mapped by patch. */
   VM_PDPTE pte;                /* PTE.                                */
 }
 #include "vmware_pack_end.h"
@@ -221,6 +221,23 @@ VMMPageTablePatch;

 #define MODULECALL_NUM_ARGS  4

+/*
+ * The cross page contains tiny stacks upon which interrupt and exception
+ * handlers in the switch path may temporarily run.  Each stack must be
+ * large enough for the sum of:
+ *
+ * - 1 #DB exception frame (5 * uint64)
+ * - 1 #NMI exception frame (5 * uint64)
+ * - 1 #MCE exception frame (5 * uint64)
+ * - the largest stack use instantaneously possible by #MCE handling code
+ * - the largest stack use instantaneously possible by #NMI handling code
+ * - the largest stack use instantaneously possible by #DB handling code
+ * - one high-water uint32 used to detect stack overflows when debugging
+ *
+ * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+ */
+#define TINY_STACK_SIZE      184
+
 /*
 *----------------------------------------------------------------------
 *
@@ -233,72 +250,70 @@ VMMPageTablePatch;
 typedef
 #include "vmware_pack_begin.h"
 struct VMCrossPageData {
+   uint32   version;           // CROSSPAGE_VERSION
+   uint32   vmmonVersion;      // VMMON_VERSION
+
   /*
-    * A tiny stack upon which interrupt and exception handlers in the switch
-    * path temporarily run.  Keep the end 16-byte aligned.  This stack must
-    * be large enough for the sum of:
-    *
-    * - 1 #DB exception frame (5 * uint64)
-    * - 1 #NMI exception frame (5 * uint64)
-    * - 1 #MCE exception frame (5 * uint64)
-    * - the largest stack use instantaneously possible by #MCE handling code
-    * - the largest stack use instantaneously possible by #NMI handling code
-    * - the largest stack use instantaneously possible by #DB handling code
-    * - one high-water uint32 used to detect stack overflows when debugging
-    * - remaining pad bytes to align to 16 bytes
-    *
-    * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+    * The following stacks and contexts are ordered for performance and code
+    * simplicity.  Both HostToVmm and VmmToHost strictly require this ordering.
+    * For any change to this data, both functions must be updated.
+    * Use VMX86_UCCOST to measure performance when changing the layout.
    */
-   uint32   tinyStack[46];

-   uint64   hostCR3;
-   uint32   crosspageMA;
-
-   uint8    hostDRSaved;        // Host DR spilled to hostDR[x].
-   uint8    hostDRInHW;         // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
-                                //   contains host-sized DB,NMI,MCE entries
-   uint16   hostSS;
-   uint64   hostRSP;
-   uint64   hostDR[8];
+   /* A tiny stack and the host context. */
+   uint64   hostTinyStack[TINY_STACK_SIZE / sizeof(uint64)];
+   uint64   hostCR3; /* Edge of context saved/restored in assembly. */
   uint64   hostRBX;
+   uint64   hostRBP;
   uint64   hostRSI;
   uint64   hostRDI;
-   uint64   hostRBP;
   uint64   hostR12;
   uint64   hostR13;
   uint64   hostR14;
   uint64   hostR15;
-   LA64     hostCrossPageLA;   // where host has crosspage mapped
+   uint64   hostRSP;
+   uint16   hostSS;  /* Edge of context saved/restored in assembly. */
+   uint16   hostDS;  /* Not saved/restored in assembly switch */
+   uint16   hostES;  /* Not saved/restored in assembly switch */
+   uint16   hostPad;
+
+   /* A tiny stack and the monitor context. */
+   uint64   monTinyStack[TINY_STACK_SIZE / sizeof(uint64)];
+   uint64   monCR3; /* Edge of context saved/restored in assembly. */
+   uint64   monRBX;
+   uint64   monRBP;
+   uint64   monR12;
+   uint64   monR13;
+   uint64   monR14;
+   uint64   monR15;
+   uint64   monRSP;
+   uint16   monSS;  /* Edge of context saved/restored in assembly. */
+   uint16   monDS;  /* Not saved/restored in assembly switch */
+   uint16   monES;  /* Not saved/restored in assembly switch */
+   uint16   monPad;
+
+   uint64   crosspageMA;
+
+   uint64   hostDR[8];
+   LA64     crossPageLA;       // where host/PTP map the cross page
+   LA64     crossGDTLA;        // where host/PTP map the cross GDT
   uint16   hostInitial64CS;
-   uint16   _pad0[3];
+   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
+   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
+   uint32   _pad0;

   uint64   wsCR0;
   uint64   wsCR4;

   DTR64    crossGDTHKLADesc;   // always uses host kernel linear address
   uint16   _pad1[3];
-   DTR64    mon64GDTR;
-   uint16   mon64ES;
-   uint16   mon64SS;
-   uint16   mon64DS;
-   uint64   mon64CR3;
-   uint64   mon64RBX;
-   uint64   mon64RSP;
-   uint64   mon64RBP;
-   uint64   mon64RSI;
-   uint64   mon64RDI;
-   uint64   mon64R12;
-   uint64   mon64R13;
-   uint64   mon64R14;
-   uint64   mon64R15;
-   uint64   mon64RIP;
-   Task64   monTask64;          /* vmm64's task */
+   DTR64    monGDTR;
+   uint16   _pad2[3];
+   /* A hardcoded value for monitor %rip which facilitates backtraces. */
+   uint64   monRIP;
+   Task64   monTask;          /* vmm's task */

   VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
-   LA64              vmm64CrossPageLA;
-   LA64              vmm64CrossGDTLA;   // where crossGDT mapped by PT patch
-                                        //  64-bit host: host kernel linear
-                                        // address

   /*
    * The monitor may requests up to two actions when returning to the
@@ -318,12 +333,12 @@ struct VMCrossPageData {
   int            userCallType;
   uint32         pcpuNum;   /* Used as extra module call arg within vmmon. */

-   VCPUSet        yieldVCPUs;
-
 #if !defined(VMX86_SERVER)
   uint64 ucTimeStamps[UCCOST_MAX];
 #endif

+   SwitchedMSRState switchedMSRState;
+
   /*
    * The values in the shadow debug registers must match those in the
    * hardware debug register immediately after a task switch in
@@ -333,8 +348,11 @@ struct VMCrossPageData {
   SharedUReg64     shadowDR[8];
   uint8            shadowDRInHW; // bit n set iff %DRn == shadowDR[n]

-   SwitchedMSRState switchedMSRState;
-   uint8            _pad2[7];
+   /* TRUE if no bits are set in yieldVCPUs, FALSE otherwise. */
+   Bool             yieldVCPUsIsEmpty;
+   uint8            _pad3[6];
+
+   VCPUSet          yieldVCPUs;

   /*
    * Adjustment for machines where the hardware TSC does not run
@@ -357,24 +375,25 @@ struct VMCrossPageData {
    * restart RunVM call, nobody else should look at it.
    */
   Bool     moduleCallInterrupted;
-   uint8    _pad3[6];
+   uint8    _pad4[6];

   DTR64    switchHostIDTR;   // baseLA = switchHostIDT's host kernel LA
-   uint16   _pad4[3];
-   DTR64    switchMonIDTR;    // baseLA = switchMonIDT's monitor LA
   uint16   _pad5[3];
+   DTR64    switchMonIDTR;    // baseLA = switchMonIDT's monitor LA
+   uint16   _pad6[3];

   /*
    * Descriptors and interrupt tables for switchNMI handlers.  Each
    * IDT has only enough space for the hardware exceptions; they are
    * sized to accommodate 64-bit descriptors.
    */
-   uint8 switchHostIDT[sizeof(Gate64) * NUM_EXCEPTIONS]; // hostCS:hostVA
-   uint8 switchMonIDT[sizeof(Gate64) * NUM_EXCEPTIONS];  // monCS:monVA
-
+   Gate64 switchHostIDT[NUM_EXCEPTIONS];      // hostCS:hostVA
+   Gate64 switchMonIDT[NUM_EXCEPTIONS];       // monCS:monVA
   volatile Bool wsException[NUM_EXCEPTIONS]; // Tracks faults in worldswitch.
+   uint8         _pad7[4];
   uint64        wsUD2;                       // IP of ud2 instr or 0 if unset.
   uint64        specCtrl; /* host MSR_SPEC_CTRL value before world switch. */
+   uint8         _pad8[8];
 }
 #include "vmware_pack_end.h"
 VMCrossPageData;
@@ -389,13 +408,19 @@ VMCrossPageData;
 *
 *----------------------------------------------------------------------
 */
+
+#define CODE_BLOCK_SIZE (PAGE_SIZE - sizeof(VMCrossPageData) - \
+                         (2 + NUM_EXCEPTIONS) * CP_STUB_SIZE)
+
 typedef
 #include "vmware_pack_begin.h"
 struct VMCrossPageCode {
-   uint16        size;           // Size of the code module.
-   uint32        vmmonVersion;   // VMMON_VERSION
-   CodeOffsets   offsets;        // Offsets to handlers.
-   uint8         codeBlock[512]; // Code for worldswitch and fault handling.
+   uint8         toVmmFunc[CP_STUB_SIZE];    // Fixed-position stubs to jump
+   uint8         toHostFunc[CP_STUB_SIZE];   // to world switch functions
+   uint8         gateStubs[NUM_EXCEPTIONS *
+                           CP_STUB_SIZE];    // Stubs for crossIDT.
+   uint8         codeBlock[CODE_BLOCK_SIZE]; // Code for worldswitch and
+                                             // fault handling.
 }
 #include "vmware_pack_end.h"
 VMCrossPageCode;
@@ -408,10 +433,10 @@ VMCrossPageCode;
 *
 *      Data structure shared between the monitor and the module
 *      that is used for crossing between the two.
- *      Accessible as vm->cross (kernel module) and CROSS_PAGE
+ *      Accessible as vm->crosspage (kernel module) and CROSS_PAGE
 *      (monitor)
 *
- *      Exactly one page long
+ *      Must be exactly one page long.
 *
 *----------------------------------------------------------------------
 */
@@ -419,29 +444,20 @@ VMCrossPageCode;
 typedef
 #include "vmware_pack_begin.h"
 struct VMCrossPage {
-   uint32          version;         /* 4 bytes. Must be at offset zero. */
-   uint32          crosspage_size;  /* 4 bytes. Must be at offset 4.    */
   VMCrossPageData crosspageData;
   VMCrossPageCode crosspageCode;
-   uint8           _pad[PAGE_SIZE - (sizeof(uint32) /* version */        +
-                                     sizeof(uint32) /* crosspage_size */ +
-                                     sizeof(VMCrossPageData)             +
-                                     sizeof(VMCrossPageCode))];
 }
 #include "vmware_pack_end.h"
 VMCrossPage;

-#define CROSSPAGE_VERSION_BASE 0xbfc /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc0c /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
-#define CROSS_PAGE  ((VMCrossPage * const) VPN_2_VA(CROSS_PAGE_START))
+#define CROSS_PAGE             ((VMCrossPage * const)VPN_2_VA(CROSS_PAGE_START))
 #define VMM_SWITCH_SHARED_DATA ((VMCrossPageData *)&CROSS_PAGE->crosspageData)
 #endif

-#define NULLPAGE_LINEAR_START  (MONITOR_LINEAR_START + \
-                                PAGE_SIZE * CPL0_GUARD_PAGE_START)
-
 #define MX_WAITINTERRUPTED     3
 #define MX_WAITTIMEDOUT        2
 #define MX_WAITNORMAL          1  // Must equal one; see linux module code.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005,2007,2009,2013,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,12 +19,14 @@
 /*
 * monAddrLayout.h --
 *
- *    Address layout of the monitor
+ *    Address layout of the monitor.
 */

 #ifndef _MON_ADDR_LAYOUT_H
 #define _MON_ADDR_LAYOUT_H

+#include "vm_pagetable.h"
+
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMKERNEL
@@ -33,25 +35,41 @@

 #pragma pack(push, 1)
 typedef struct VMM64_AddrLayout {
-   /* 
+   /*
    *All address are VPNs and all lengths are numPages
    */
   uint64 monBase;            // MONITOR_BASE_VPN
-   uint64 mmuRootStart;       // MMU_ROOT_START
-   uint32 mmuRootLen;         // MMU_ROOT_LEN
-   uint64 mmuL3Start;         // MMU_L3_START
-   uint32 mmuL3Len;           // MMU_L3_LEN
-   uint64 mmuL2Start;         // MMU_L2_START
-   uint32 mmuL2Len;           // MMU_L2_LEN
-   uint64 monPageTableStart;  // MON_PAGE_TABLE_START
-   uint32 monPageTableLen;    // MON_PAGE_TABLE_LEN
+   uint64 monL5Start;         // MON_PAGE_TABLE_L5_START
+   uint32 monL5Len;           // MON_PAGE_TABLE_L5_LEN
+   uint64 monL4Start;         // MON_PAGE_TABLE_L4_START
+   uint32 monL4Len;           // MON_PAGE_TABLE_L4_LEN
+   uint64 monL3Start;         // MON_PAGE_TABLE_L3_START
+   uint32 monL3Len;           // MON_PAGE_TABLE_L3_LEN
+   uint64 monL2Start;         // MON_PAGE_TABLE_L2_START
+   uint32 monL2Len;           // MON_PAGE_TABLE_L2_LEN
+   uint64 monL1Start;         // MON_PAGE_TABLE_L1_START
+   uint32 monL1Len;           // MON_PAGE_TABLE_L1_LEN
 #ifndef VMX86_SERVER
   uint64 monCpStart;         // CROSS_PAGE_START
 #endif
 } VMM64_AddrLayout;
 #pragma pack(pop)

-#define VMM_SCRATCHAS_FIRST_L4OFF   1
-#define VMM_SCRATCHAS_LAST_L4OFF  127
+#define VMM_MONAS_4LP_FIRST_L4OFF   1
+#define VMM_MONAS_4LP_LAST_L4OFF  130

+#define VMM_MONAS_5LP_FIRST_L5OFF   1
+#define VMM_MONAS_5LP_LAST_L5OFF   34
+
+static INLINE PT_Level
+MonAS_GetPagingLevel(void)
+{
+   return PT_LEVEL_4;
+}
+
+static INLINE Bool
+MonAS_Uses5LevelPaging(void)
+{
+   return MonAS_GetPagingLevel() == PT_LEVEL_5;
+}
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,8 +31,8 @@
 * contains code and data statically built, empty but allocated space, shared
 * and run-time initialized content.  The monitor loader header regularizes
 * encoding of address space information, allowing a common representation and
- * common code to be re-used for different contexts (vmmon for hosted, vmkernel
- * for ESX, and the VMX while deprivileging of bootstrap is ongoing).
+ * common code to be re-used for different contexts (vmmon for hosted
+ * and vmkernel for ESX).
 *
 * The Header
 * ==========
@@ -59,9 +59,10 @@
 * ML_CONTENT_COPY: Allocates new MPNs, maps them and copies from the specified
 * source.
 *
- * ML_CONTENT_SHARE: Memory provided by user or host kernel and mapped into the
- * monitor address space.  The subIndex field specifies which region, as
- * multiple regions may be shared for each source.
+ * ML_CONTENT_SHARE: Memory provided by user, host kernel, or VMM blob,
+ * and mapped into the monitor address space.  The subIndex field
+ * specifies which region, as multiple regions may be shared for each
+ * source.
 *
 * Processing
 * ==========
@@ -86,7 +87,6 @@
 * MonLoaderCallout_Init(): Initialize MonLoader callouts.
 * MonLoaderCallout_GetSharedHostPage(subIdx, page): Get shared host page MPN.
 * MonLoaderCallout_GetSharedUserPage(subIdx, page): Get shared user page MPN.
- * MonLoaderCallout_SetEntrypoint(): Sets the entry code/stack for execution.
 *
 * Building vs Importing
 * =====================
@@ -118,21 +118,54 @@
 #define _MON_LOADER

 #include "vm_basic_types.h"
-#include "x86paging_common.h"
-#include "x86types.h" /* PT_L1E */
+#include "vm_pagetable.h"
 #include "vcpuid.h"   /* Vcpuid */

+#if defined VM_X86_64
+#include "x86paging_64.h"
+#elif defined VM_ARM_64
+#include "arm64_vmsa.h"
+#endif
+
 #define ML_NAME_MAX 16

 /* ML perms are simple and abbreviated. */

+#if defined VM_X86_64
+
 #define ML_PERM_RWX (PTE_P | PTE_RW)
 #define ML_PERM_RW  (PTE_P | PTE_RW | PTE_NX)
 #define ML_PERM_RO  (PTE_P |          PTE_NX)
 #define ML_PERM_RX   PTE_P

+#define ML_PERM_TBL   ML_PERM_RWX
 #define ML_PERM_MASK (PTE_P| PTE_RW | PTE_NX | PTE_US)

+#define ML_PERM_PRESENT(_flags)     (((_flags) & PTE_P) != 0)
+#define ML_PERM_WRITEABLE(_flags)   (((_flags) & PTE_RW) != 0)
+
+#define ML_PTE_2_PFN(_pte)          LM_PTE_2_PFN(_pte)
+
+#elif defined VM_ARM_64
+
+#define _ML_PERM_COMMON (ARM_PTE_BLOCK_AP(ARM_AP_PL0)   | ARM_PTE_BLOCK_AF | \
+                         ARM_PTE_BLOCK_SH(ARM_SH_OUTER) | ARM_PTE_BLOCK_L3_TYPE)
+#define ARM_PTE_BLOCK_AP_RO ARM_PTE_BLOCK_AP(ARM_AP_RO)
+
+#define ML_PERM_RW   (_ML_PERM_COMMON |                       ARM_PTE_BLOCK_XN)
+#define ML_PERM_RO   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+#define ML_PERM_RX   (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO                   )
+
+#define ML_PERM_TBL   ML_PERM_RW
+#define ML_PERM_MASK (_ML_PERM_COMMON | ARM_PTE_BLOCK_AP_RO | ARM_PTE_BLOCK_XN)
+
+#define ML_PERM_PRESENT(_flags)   (((_flags) & ARM_PTE_VALID) != 0)
+#define ML_PERM_WRITEABLE(_flags) (((_flags) & ARM_PTE_BLOCK_AP_RO) == 0)
+
+#define ML_PTE_2_PFN(_pte)    (((_pte) & ARM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
+
+#endif
+
 #define ML_PERMS_MATCH(x,p) (((x) & ML_PERM_MASK) == ((p) & ML_PERM_MASK))

 #define LOADER_HEADER_MAGIC 0x8675309E98675309
@@ -149,10 +182,10 @@ typedef enum {
   ML_CONTENT_SHARE,         /* Share data from external source. */
 } MonLoaderContentType;

-#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? 4 : \
-                               x == ML_CONTENT_PAGETABLE_L3 ? 3 : \
-                               x == ML_CONTENT_PAGETABLE_L2 ? 2 : \
-                               x == ML_CONTENT_PAGETABLE_L1 ? 1 : \
+#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? PT_LEVEL_4 : \
+                               x == ML_CONTENT_PAGETABLE_L3 ? PT_LEVEL_3 : \
+                               x == ML_CONTENT_PAGETABLE_L2 ? PT_LEVEL_2 : \
+                               x == ML_CONTENT_PAGETABLE_L1 ? PT_LEVEL_1 : \
                               0)

 /* Sources of content, for pages not zeroed, unmapped or otherwise special. */
@@ -182,13 +215,16 @@ typedef struct {
   } blobSrc;
   uint64               bspOnly;           /* Process only on BSP. */
   uint64               subIndex;          /* Region ID for ML_CONTENT_COPY. */
-   uint64               procVmmon;         /* Processed by the vmmon on hosted. */
 } MonLoaderEntry;


-/* Packed for easy consumption by bootstrap-offsets.pl. */
+/*
+ * Packed for easy consumption by bootstrap-offsets.pl. If the contents of the
+ * MonLoaderHeader struct are changed then $HEADER_SIZE must be updated
+ * accordingly in bootstrap-offsets.pl.
+ */
 #pragma pack(push, 1)
-typedef struct {
+typedef struct MonLoaderHeader {
   uint64         magic;
   uint32         entrySize;
   uint32         count;
@@ -198,35 +234,34 @@ typedef struct {
   /* ss:rip */
   uint16         stackSelector;
   VA64           stackEntrypoint;
+   LPN64          monStartLPN;
+   LPN64          monEndLPN;
   MonLoaderEntry entries[];
 } MonLoaderHeader;
 #pragma pack(pop)

 /* Environment context structure, defined by the environment. */
 struct MonLoaderEnvContext;
-typedef struct MonLoaderEnvContext MonLoaderEnvContext;

 /* Callout prototypes */
-MPN  MonLoaderCallout_AllocMPN(MonLoaderEnvContext *, Vcpuid);
-void MonLoaderCallout_CleanUp(MonLoaderEnvContext *);
-Bool MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *, uint64, size_t, MPN,
-                                   Vcpuid);
-Bool MonLoaderCallout_FillPage(MonLoaderEnvContext *, uint8, MPN, Vcpuid);
-MPN  MonLoaderCallout_GetPageRoot(MonLoaderEnvContext *, Vcpuid);
-Bool MonLoaderCallout_GetPTE(MonLoaderEnvContext *, MPN, unsigned, Vcpuid,
-                             PT_L1E *);
-Bool MonLoaderCallout_ImportPage(MonLoaderEnvContext *, MPN, Vcpuid);
-Bool MonLoaderCallout_Init(void *, MonLoaderEnvContext **);
-Bool MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *, MPN, unsigned, uint64,
-                                  MPN, Vcpuid);
-MPN  MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *, uint64, unsigned,
-                                        Vcpuid);
-MPN  MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *, uint64, unsigned,
-                                        Vcpuid);
-Bool MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *);
-Bool MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *, uint16, VA64, uint16,
-                                    VA64);
-MPN  MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *, uint64);
+MPN  MonLoaderCallout_AllocMPN(struct MonLoaderEnvContext *, Vcpuid);
+void MonLoaderCallout_CleanUp(struct MonLoaderEnvContext *);
+Bool MonLoaderCallout_CopyFromBlob(struct MonLoaderEnvContext *, uint64,
+                                   size_t, MPN, Vcpuid);
+Bool MonLoaderCallout_FillPage(struct MonLoaderEnvContext *, uint8, MPN,
+                               Vcpuid);
+MPN  MonLoaderCallout_GetPageRoot(struct MonLoaderEnvContext *, Vcpuid);
+Bool MonLoaderCallout_GetPTE(struct MonLoaderEnvContext *, MPN, unsigned,
+                             Vcpuid, PT_L1E *);
+Bool MonLoaderCallout_ImportPage(struct MonLoaderEnvContext *, MPN, Vcpuid);
+Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **, unsigned);
+Bool MonLoaderCallout_MapMPNInPTE(struct MonLoaderEnvContext *, MPN, unsigned,
+                                  uint64, MPN, Vcpuid);
+MPN  MonLoaderCallout_GetSharedUserPage(struct MonLoaderEnvContext *, uint64,
+                                        unsigned, Vcpuid);
+MPN  MonLoaderCallout_GetSharedHostPage(struct MonLoaderEnvContext *, uint64,
+                                        unsigned, Vcpuid);
+MPN  MonLoaderCallout_GetBlobMpn(struct MonLoaderEnvContext *, uint64);

 typedef enum MonLoaderError {
   ML_OK = 0,
@@ -236,7 +271,6 @@ typedef enum MonLoaderError {
   ML_ERROR_ARGS,
   ML_ERROR_CALLOUT_INIT,
   ML_ERROR_CALLOUT_COPY,
-   ML_ERROR_CALLOUT_ENTRYPOINT,
   ML_ERROR_CALLOUT_GETPTE,
   ML_ERROR_CALLOUT_MAPINPTE,
   ML_ERROR_CALLOUT_PAGEROOT_GET,
@@ -258,8 +292,12 @@ typedef enum MonLoaderError {
 } MonLoaderError;


-/* A subindex above shared area subindices for sharing of MonLoaderHeader. */
-#define MONLOADER_HEADER_IDX (NUM_SHARED_AREAS + 1)
+/*
+ * Values above shared area subindices for sharing of MonLoaderHeader
+ * and htSchedStateMap.
+ */
+#define MONLOADER_HEADER_IDX 6
+#define MONLOADER_HT_MAP_IDX 7


 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
@@ -25,23 +25,7 @@
 #ifndef _MONLOADER_LOG_H_
 #define _MONLOADER_LOG_H_

-#ifdef VMX86_VMX
-
-#define LOGLEVEL_MODULE main
-#include "loglevel_vmcore.h"
-/*
- * Redefine LOG() to match the vmkernel calling convention.
- */
-#undef LOG_BYNAME
-#undef LOG
-
-#define LOG_BYNAME(_mod, _min, _fmt, ...) \
-        (DOLOG_BYNAME(_mod, _min) ? Log(_fmt "\n", ## __VA_ARGS__) : (void) 0)
-
-#define LOG(_min, _fmt, ...) \
-        LOG_BYNAME(LOGLEVEL_MODULE, _min, _fmt, ## __VA_ARGS__)
-
-#elif defined VMKERNEL
+#if defined VMKERNEL

 #define LOGLEVEL_MODULE MonLoader
 #include "log.h"
@@ -59,7 +43,7 @@ do {                                                 \
   }                                                 \
 } while (0)

-#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#else /* !defined VMKERNEL && !defined VMMON */
 #error MonLoader cannot be built as part of this environment
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2007-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2007-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -60,20 +60,21 @@
 *     lengths, and this causes the 'monitor-modular-size' script to
 *     report differences in '.rodata'.
 *
- *   o To avoid differences in '.rodata', each __FILE__ is put into
- *     its own section.  The monitor's linker (not ld) will use the
- *     name of the section to recover the name of the source file.
+ *   o Each __FILE__ is put into its own .assert_pathname_##__FILE
+ *     section.  The monitor's runtime linker (not ld) then uses the name
+ *     of this section to recover the name of the corresponding source file.
 *
 *   o At run time, prior to loading, when our linker is creating an
 *     executable image of the monitor and extensions, all the file
 *     names are extracted from these sections, the '${VMTREE}' prefix
 *     is removed, and the resulting table of shortened file names is
- *     added to '.rodata'.
+ *     added to a non-loadable section called '.assert_pathnames' in the
+ *     VMX linker (not the bootstrap linker which does not need them).
 *
 *     Further, during linkage, each relocation to the original
 *     section containing the path name is modified so that the low
- *     16-bits contain an offset from '__vmm_pathnames_start' rather
- *     than the base of the original containing section.
+ *     16-bits contain an offset from the start of the .assert_pathnames
+ *     section rather than the base of the original containing section.
 *
 *     Only three types of relocations to the assertion strings are
 *     supported (32-bit PC-relative and 32-bit/64-bit absolute) because that
@@ -111,42 +112,12 @@ typedef struct Assert_Info {
   Assert_MonSrcLoc loc;
 } Assert_Info;

-/*
- * The portion of the __attribute__ line after __FILE__ is there so that
- * the .assert_pathname_* sections are not marked as ALLOC, since we only
- * need them in the vmx and do not need them loaded.
- */
-#define __VMM__FILE__SECTION \
-      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
-#define __VMM__FILE__ ({                                                \
-         static __VMM__FILE__SECTION const char file[] = "";            \
-         file;                                                          \
-      })
-
 #define ASSERT_MONSRCFILEOFFSET(loc)    LOWORD(loc)
 #define ASSERT_MONSRCLINE(loc)          HIWORD(loc)

 #define ASSERT_NULL_MONSRCLOC     0             // there is never line 0

-#ifdef VMM // {
-#ifdef MONITOR_APP // {
-
-#define ASSERT_MONSRCLOC() ASSERT_NULL_MONSRCLOC
-
-#else // } {
-
-#define ASSERT_MONSRCLOC() ({                                           \
-   const uintptr_t offset = ((__LINE__ << 16) +                         \
-                             (uintptr_t)__VMM__FILE__);                 \
-   const Assert_MonSrcLoc loc = offset;                                 \
-   loc;                                                                 \
-})
-
-extern const char __vmm_pathnames_start;
-#define ASSERT_MONSRCFILE(loc) \
-   (&__vmm_pathnames_start + ASSERT_MONSRCFILEOFFSET(loc))
-
-
+#if defined(VMM) && !defined(MONITOR_APP) // {
 /*
 * Assertion information is collected in a non-loadable section
 * named .assert_info.  Each record in this section contains
@@ -176,6 +147,18 @@ extern const char __vmm_pathnames_start;
 */

 #ifndef VMM_BOOTSTRAP
+/*
+ * The portion of the __attribute__ line after __FILE__ is there so that
+ * the .assert_pathname_* sections are not marked as ALLOC, since we only
+ * need them in the vmx and do not need them loaded.
+ */
+#define __VMM__FILE__SECTION \
+      __attribute__((section (".assert_pathname_" __FILE__ ",\"\"#")))
+#define __VMM__FILE__ ({                                                \
+         static __VMM__FILE__SECTION const char file[] = "";            \
+         file;                                                          \
+      })
+
 #define ASSERT_RECORDINFO(assembly, assertType, bugNr)                   \
   __asm__ __volatile__(".pushsection .assert_info;"                     \
                        ".quad 0f;"                                      \
@@ -215,8 +198,6 @@ extern uint64 bsAssertRIP;
   ({COMPILER_MEM_BARRIER();                                             \
     ASSERT_RECORDINFO("", AssertType_##name##Bug, bug);                 \
     __builtin_trap();})
+#endif // VMM && !MONITOR_APP}

-#endif // MONITOR_APP }
-#endif // VMM }
-
-#endif
+#endif // _MON_ASSERT_H_
@@ -26,137 +26,16 @@
 #define INCLUDE_ALLOW_DISTRIBUTE
 #include "includeCheck.h"

-#include "vm_assert.h"
-#include "vm_atomic.h"
 #include "vm_basic_types.h"

 #if defined __cplusplus
 extern "C" {
 #endif

-
-/*
- * Please bump the version number if your change will break the
- * compatability to the drivers.
- */
-#define ACTION_EXPORTED_VERSION   2
-
-#define ACTION_WORD_SIZE   (sizeof(uint64) * 8)
-#define ACTION_NUM_WORDS   (2)
-#define ACTION_NUM_IDS     (ACTION_NUM_WORDS * ACTION_WORD_SIZE)
-
 #define MONACTION_INVALID  MAX_UINT32

 typedef uint32 MonitorIdemAction;

-/*
- * Representation of a set of actions.
- */
-typedef struct MonitorActionSet {
-   volatile uint64 word[ACTION_NUM_WORDS];
-} MonitorActionSet;
-
-#ifndef __cplusplus
-typedef enum MonitorActionSetName MonitorActionSetName;
-#endif
-
-/*
- * Summary of action and interrupt states.
- */
-typedef struct MonitorActionIntr {
-   MonitorActionSet  pendingSet;
-   volatile Bool     action;
-   Bool              intr;
-   Bool              nmi;
-   Bool              db;
-   uint32            _pad;
-} MonitorActionIntr;
-
-/*
- *------------------------------------------------------------------------
- * MonitorActionSet_AtomicInclude --
- *
- *    This function atomically adds an action to an action set.
- *
- * Results:
- *    TRUE if the action being added did not exist in the action set.
- *    FALSE otherwise.
- *
- * Side effects:
- *    The given action set will be updated.
- *------------------------------------------------------------------------
- */
-static INLINE Bool
-MonitorActionSet_AtomicInclude(MonitorActionSet *set, const uint32 actionID)
-{
-   Atomic_uint64 *atomicSet =
-      Atomic_VolatileToAtomic64(&set->word[actionID / ACTION_WORD_SIZE]);
-   uint64 mask = (uint64)1 << (actionID % ACTION_WORD_SIZE);
-   uint64 oldWord;
-   uint64 newWord;
-
-   ASSERT_ON_COMPILE((ACTION_WORD_SIZE & (ACTION_WORD_SIZE - 1)) == 0);
-#ifdef VMX86_DEBUG
-   /* If ASSERT is not desirable, do explicit check. Please see PR 567811. */
-#ifdef MODULE
-   if (UNLIKELY(actionID / ACTION_WORD_SIZE >= ACTION_NUM_WORDS)) {
-      return FALSE;
-   }
-#else
-   ASSERT(actionID / ACTION_WORD_SIZE < ACTION_NUM_WORDS);
-#endif // MODULE
-#endif // VMX86_DEBUG
-   do {
-      oldWord = Atomic_Read64(atomicSet);
-      newWord = oldWord | mask;
-   } while (!Atomic_CMPXCHG64(atomicSet, oldWord, newWord));
-   return (oldWord & mask) == 0;
-}
-
-
-/*
- *----------------------------------------------------------------------------
- * MonitorAction_SetBits --
- *
- *      The core logic for posting an action. Update the set of pending
- *      actions of the target VCPU in the shared area to mark the action
- *      as present. Make sure the bit is set in the pendingSet first to
- *      avoid a race with the drain loop.
- *
- *      It's the responsibility of the callers to ensure that the change
- *      to actionIntr->action is globally visible before any IPI is sent
- *      (the change to pendingSet is pushed out by the cmpxchg in
- *      MonitorActionSet_AtomicInclude).
- *
- * Results:
- *      TRUE if the action being posted was not pending before.
- *      FALSE otherwise (other threads could have posted the same action).
- *
- * Side effects:
- *      None.
- *----------------------------------------------------------------------------
- */
-static INLINE Bool
-MonitorAction_SetBits(MonitorActionIntr *actionIntr, MonitorIdemAction actionID)
-{
-   /* Careful if optimizing this: see PR70016. */
-   Bool newAction =
-      MonitorActionSet_AtomicInclude(&actionIntr->pendingSet, actionID);
-   actionIntr->action = TRUE;
-   return newAction;
-}
-
-/*
- * C1 states entered by monitor while waiting for an action
- */
-typedef enum {
-   VMM_C1_STATE_INVALID = 0,
-   VMM_C1_STATE_HLT,
-   VMM_C1_STATE_MWAIT,
-   VMM_C1_STATE_PAUSE
-} vmmC1StateType;
-
-
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -48,7 +48,11 @@ typedef uint8  NUMA_MemRangeID;
 */
 #define NUMA_MAX_NODES              64
 #define NUMA_MAX_CPUS_PER_NODE      (vmx86_server ? MAX_PCPUS : 32)
-#define NUMA_MAX_MEM_RANGES         (vm_arm_64 ? 128 : 64)
+#ifdef VMKBOOT
+#define NUMA_MAX_MEM_RANGES         64
+#else
+#define NUMA_MAX_MEM_RANGES         256
+#endif
 #define INVALID_NUMANODE            ((NUMA_Node)-1)
 #define NUMA_NODE_MASK_ANY          ((NUMA_NodeMask)-1)
 #define NUMA_NODE_MASK_NONE         ((NUMA_NodeMask)0)
@@ -70,4 +74,10 @@ typedef struct NUMA_MemRangesList {
   NUMA_MemRange memRange[NUMA_MAX_MEM_RANGES];
 } NUMA_MemRangesList;

+
+typedef struct NUMA_MemRangesListRef {
+   uint64        numMemRanges;
+   const NUMA_MemRange *memRange;
+} NUMA_MemRangesListRef;
+
 #endif // _NUMA_DEFS_H
@@ -52,7 +52,7 @@ extern "C" {
 typedef struct PageListEntry {
   CompressedBPN cbpn;
   Bool voided;
-   uint8 _pad[2];
+   uint8 _pad[1];
 } PageListEntry;
 #pragma pack(pop) 

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -55,26 +55,20 @@ typedef enum PCIPassthruVectorIndex {
 } PCIPassthruVectorIndex;

 typedef enum PCIPassthru_IntrType {
-   PCIPASSTHRU_INTR_NONE   = 0x00,
-   PCIPASSTHRU_INTR_IOAPIC = 0x01,
-   PCIPASSTHRU_INTR_MSI    = 0x02,
-   PCIPASSTHRU_INTR_MSIX   = 0x04,
+   PCIPASSTHRU_INTR_NONE    = 0x00,
+   PCIPASSTHRU_INTR_IOAPIC  = 0x01,
+   PCIPASSTHRU_INTR_MSI     = 0x02,
+   PCIPASSTHRU_INTR_MSIX    = 0x04,
+   PCIPASSTHRU_INTR_MSI_INT = 0x08,
 } PCIPassthru_IntrType;

+#define PCIPASSTHRU_INTR_MSI_COMBINED                   \
+   (PCIPASSTHRU_INTR_MSI | PCIPASSTHRU_INTR_MSI_INT)
+
 typedef struct FPTIntrProxyInfo {
   uint32            adapterIndex;
   uint32            vectorIndex;
   MonitorIdemAction actionID;
-   uint32 _pad0;
-
-   /*
-    * These addresses are only needed for hosted platforms, where our kernel
-    * modules lack a SharedArea API.
-    */
-#ifndef VMX86_SERVER
-   VA64              notifyUVA;
-   VA64              vectorIndexUVA;
-#endif
 } FPTIntrProxyInfo;

 typedef struct UPTIntrProxyInfo {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012,2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,12 @@
 *********************************************************/

 /*
- * perfctr_generic.h --
+ * perfctr.h --
 *
 */

-#ifndef _PERFCTR_GENERIC_H_
-#define _PERFCTR_GENERIC_H_
+#ifndef _PERFCTR_H_
+#define _PERFCTR_H_

 #define INCLUDE_ALLOW_USERLEVEL

@@ -33,7 +33,9 @@

 #include "includeCheck.h"
 #include "vm_basic_types.h"
+#include "perfctr_arch.h"

+#define PERF_EVENT_NAME_LEN                      64
 /*
 * nmiNo      -- vmm peer is not attempting to do nmi profiling this run
 * nmiYes     -- vmm peer is doing nmi profiling and nmis are currently enabled
@@ -69,4 +71,51 @@ typedef struct {
   uint8  _pad[3];
 } CrossProfShared;

-#endif // ifndef _PERFCTR_GENERIC_H_
+/*
+ * PerfCtr_Config --
+ *      Describes configuration for a single hardware performance counter
+ *
+ *      Since this is only used to record general performance counters, we
+ *      made the assumption in nmiProfiler.c that the type is GENERAL and
+ *      index is counter number of type GENERAL.
+ *
+ *      **** x86 ****
+ *      On AMD K8 and GH:
+ *      index:        Which perf ctr, 0 to 3.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On AMD with PerfCtrExtCore support:
+ *      index:        Which perf ctr, 0 to 5.  RDPMC argument
+ *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
+ *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
+ *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
+ *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
+ *
+ *      On Intel Core architecture:
+ *      <to be documented>
+ *
+ *      **** ARM ****
+ *      escrVal:     Value placed in PMEVTYPER<n>_EL0 to configure event counter
+ *      index:       Index of the event counter.
+ */
+typedef struct PerfCtr_Config {
+   uint64 escrVal;
+   uint32 index;
+   uint32 periodMean;
+
+   /*
+    * Random number (whose absolute value is capped at
+    * periodJitterMask) is used to randomize sampling interval.
+    */
+   uint32  periodJitterMask;
+   uint32  seed;    // seed is used to compute next random number
+   uint16  config;
+   Bool    valid;
+   PERFCTR_CONFIG_ARCH_FIELDS
+} PerfCtr_Config;
+
+
+#endif // ifndef _PERFCTR_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014, 2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,12 +17,17 @@
 *********************************************************/

 /*
- * x86perfctr.h --
+ * perfctr_arch.h --
 *
+ *      Performance counters (x64 specific).
 */

-#ifndef _X86PERFCTR_H_
-#define _X86PERFCTR_H_
+#ifndef _PERFCTR_ARCH_H
+#define _PERFCTR_ARCH_H
+
+#ifndef _PERFCTR_H_
+#error "This file can only be included by perfctr.h"
+#endif

 #define INCLUDE_ALLOW_USERLEVEL

@@ -30,24 +35,20 @@
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
-
 #include "includeCheck.h"
+
 #include "vm_asm.h"
 #include "x86cpuid_asm.h"
-#include "perfctr_generic.h"

-#define PERFCTR_PENTIUM4_NUM_COUNTERS            18
-#define PERFCTR_PENTIUM4_NUM_COUNTERS_WITH_L3    26
-#define PERFCTR_AMD_NUM_COUNTERS                 4 
+#define PERFCTR_AMD_NUM_COUNTERS                 4
 #define PERFCTR_AMD_EXT_NUM_COUNTERS             6
 #define PERFCTR_P6_NUM_COUNTERS                  2
 #define PERFCTR_NEHALEM_NUM_GEN_COUNTERS         4
 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS       3
 #define PERFCTR_SANDYBRIDGE_NUM_GEN_COUNTERS     8 /* When HT is disabled */
-#define PERFCTR_CORE_NUM_ARCH_EVENTS             7
-#define PERFCTR_PENTIUM4_VAL_MASK                0xffffffffffLL
+#define PERFCTR_CORE_NUM_ARCH_EVENTS             8
+#define PERFCTR_CORE_NUM_FIXED_COUNTERS          4
 #define PERFCTR_AMD_VAL_MASK                     0xffffffffffffLL
-#define PERF_EVENT_NAME_LEN                      64
 /*
 * Even though the performance counters in P6 are 40 bits,
 * we can only write to the lower 32 bits.  Bit 31 is
@@ -58,7 +59,7 @@

 /*
 * Performance counter width is determined at runtime in CORE.
- * But the writables bits are fixed. we can only write to the 
+ * But the writables bits are fixed. we can only write to the
 * lower 32 bits.  Bit 31 is used to sign-extend the upper 8 bits.
 */
 #define PERFCTR_CORE_WRITE_MASK                  0xffffffff
@@ -109,16 +110,21 @@
 /* AMD Performance Counter MSR Definitions */
 #define PERFCTR_AMD_PERFEVTSEL0_ADDR             0xC0010000
 #define PERFCTR_AMD_PERFCTR0_ADDR                0xC0010004
+/* AMD with PerfCtrExtCore (PERFCORE) support MSR Definitions */
 #define PERFCTR_AMD_EXT_BASE_ADDR                0xC0010200
 #define PERFCTR_AMD_EXT_EVENTSEL                 0
 #define PERFCTR_AMD_EXT_CTR                      1
 #define PERFCTR_AMD_EXT_MSR_STRIDE               2
+#define PERFCTR_AMD_AMD_EXT_CNTR_BASE_ADDR \
+        (PERFCTR_AMD_EXT_BASE_ADDR + PERFCTR_AMD_EXT_CTR)
+#define PERFCTR_AMD_AMD_EXT_EVSL_BASE_ADDR \
+        (PERFCTR_AMD_EXT_BASE_ADDR + PERFCTR_AMD_EXT_EVENTSEL)

 /* AMD Clocks */
 #define PERFCTR_AMD_CPU_CLK_UNHALTED                           0x76

 /* AMD Load/Store unit events */
-#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20 
+#define PERFCTR_AMD_SEGMENT_REGISTER_LOADS                     0x20
 #define PERFCTR_AMD_LS_BUFFER2_FULL                            0x23

 /*
@@ -192,8 +198,8 @@

 #define PERFCTR_AMD_MEM_CTRL_PAGE_TABLE_OVERFLOWS                   0xe1
 #define PERFCTR_AMD_CPU_IO_REQUESTS_TO_MEMORY_IO                    0xe9
-#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec 
- 
+#define PERFCTR_AMD_PROBE_RESPONSE_AND_UPSTREAM_REQ                 0xec
+
 /* AMD HyperTransport Interface Events */

 #define PERFCTR_AMD_HT_L0_TX_BW                                     0xf6
@@ -213,9 +219,9 @@
 */

 /* P6 Data Cache Unit (DCU) */
-#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043  
+#define PERFCTR_P6_DATA_MEM_REFS                 0x00000043
 #define PERFCTR_P6_DCU_LINES_IN                  0x00000045
-#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046 
+#define PERFCTR_P6_DCU_M_LINES_IN                0x00000046
 #define PERFCTR_P6_DCU_MISS_OUTSTANDING          0x00000048

 /* P6 Instruction Fetch Unit (IFU) */
@@ -373,7 +379,7 @@
 #define PERFCTR_P6_APIC_INTR                     0x00100000
 #define PERFCTR_P6_ENABLE                        0x00400000
 #define PERFCTR_P6_INVERT_COUNTER_MASK           0x00800000
-#define PERFCTR_P6_COUNTER_MASK                  0xff000000        
+#define PERFCTR_P6_COUNTER_MASK                  0xff000000
 #define PERFCTR_P6_COUNTER_MASK_SHIFT            24
 #define PERFCTR_P6_SHIFT_BY_UNITMASK(e)          (e << 8)

@@ -430,6 +436,7 @@
 #define PERFCTR_CORE_KERNEL_MODE                    PERFCTR_CPU_KERNEL_MODE
 #define PERFCTR_CORE_APIC_INTR                      PERFCTR_CPU_APIC_INTR
 #define PERFCTR_CORE_ENABLE                         PERFCTR_CPU_ENABLE
+/* AnyThread Counting deprecated since PMU v5. */
 #define PERFCTR_CORE_ANYTHREAD                      0x00200000
 #define PERFCTR_CORE_IN_TX                          (CONST64U(1) << 32)
 #define PERFCTR_CORE_IN_TXCP                        (CONST64U(1) << 33)
@@ -457,6 +464,7 @@
 #define PERFCTR_CORE_INST_RETIRED                0xc0
 /* bus cycles */
 #define PERFCTR_CORE_UNHALTED_REF_CYCLES         (0x3c | (0x01 << 8))
+#define PERFCTR_CORE_TOPDOWN_SLOTS               (0xa4 | (0x01 << 8))

 /*
 * See Tables 30-2, 30-4 of the
@@ -552,132 +560,20 @@
 #define PERFCTR_NEHALEM_UNCORE_ENABLE            0x400000
 #define PERFCTR_NEHALEM_UNCORE_L3_LINES_IN       (0x0a | (0x0f << 8))

-
-/*
- * ----------------------------------------------------------------------
- *
- * Pentium 4
- *
- * ----------------------------------------------------------------------
- */
-
-/*
- * Pentium 4 Counter Configuration Control Register flags and fields
- */
-#define PERFCTR_PENTIUM4_CCCR_ENABLE                   0x00001000
-#define PERFCTR_PENTIUM4_CCCR_SET_ESCR(c, e)           ((c) |= (((e) & 0x7) << 13))
-#define PERFCTR_PENTIUM4_CCCR_REQRSVD                  0x00030000 // always set these bits
-#define PERFCTR_PENTIUM4_CCCR_COMPARE                  0x00040000
-#define PERFCTR_PENTIUM4_CCCR_COMPLEMENT               0x00080000
-#define PERFCTR_PENTIUM4_CCCR_SET_THRESHOLD(c, e)      ((c) |= (((e) & 0xf) << 20))
-#define PERFCTR_PENTIUM4_CCCR_EDGE                     0x01000000
-#define PERFCTR_PENTIUM4_CCCR_FORCE_OVF                0x02000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T0               0x04000000
-#define PERFCTR_PENTIUM4_CCCR_OVF_PMI_T1               0x08000000
-#define PERFCTR_PENTIUM4_CCCR_CASCADE                  0x40000000
-#define PERFCTR_PENTIUM4_CCCR_OVF                      0x80000000
-
-#define PERFCTR_PENTIUM4_COUNTER_BASEADDR              0x00000300
-#define PERFCTR_PENTIUM4_CCCR_BASE_ADDR                0x00000360
-
-#define PERFCTR_PENTIUM4_CCCR_THRESHOLD(e)             ((e & 0xf) << 20)
-#define PERFCTR_PENTIUM4_OPT_EDGE_DETECT (PERFCTR_PENTIUM4_CCCR_COMPARE | PERFCTR_PENTIUM4_CCCR_EDGE)
-
-/*
- * Pentium 4 Event Selection Control Register flags
- */
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T0             0x00000004
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T0           0x00000008
-#define PERFCTR_PENTIUM4_ESCR_USER_MODE_T1             0x00000001
-#define PERFCTR_PENTIUM4_ESCR_KERNEL_MODE_T1           0x00000002
-#define PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK(e)          (e << 9)
-#define PENTIUM4_EVTSEL(e)                             (e << 25)
-
-/*
- * Event definitions for Pentium4 Family Processors. 
- * There are many more of these possible.
- * See Appendix A.1 in Volume 3 of the IA32 manual.
- */
-
-#define PERFCTR_PENTIUM4_EVT_INSTR_RETIRED              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_INSTR_COMPLETED            PENTIUM4_EVTSEL(0x07)
-#define PERFCTR_PENTIUM4_EVT_BSQ_CACHE_REFERENCE        PENTIUM4_EVTSEL(0x0c)
-#define PERFCTR_PENTIUM4_EVT_ITLB_REFERENCE             PENTIUM4_EVTSEL(0x18)
-#define PERFCTR_PENTIUM4_EVT_PAGE_WALK_TYPE             PENTIUM4_EVTSEL(0x01)   
-#define PERFCTR_PENTIUM4_EVT_BPU_FETCH_REQUEST          PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_BRANCH_RETIRED             PENTIUM4_EVTSEL(0x06)  
-#define PERFCTR_PENTIUM4_EVT_MISPRED_BRANCH_RETIRED     PENTIUM4_EVTSEL(0x03)
-#define PERFCTR_PENTIUM4_EVT_MACHINE_CLEAR              PENTIUM4_EVTSEL(0x02)
-#define PERFCTR_PENTIUM4_EVT_MEMORY_CANCEL              PENTIUM4_EVTSEL(0x02) 
-#define PERFCTR_PENTIUM4_GLOBAL_POWER_EVENTS            PENTIUM4_EVTSEL(0x13)
-
-/*
- * Pentium4 doesn't have an event for clock cycles, but Intel
- * outlines a method in the IA32 manual, vol 3, sec 14.9.9,
- * for measuring cycles which involves this event strategy.
- */
-
-#define PERFCTR_PENTIUM4_EVT_CLK_CYCLES \
-   PENTIUM4_EVTSEL(0x2) | \
-   PERFCTR_PENTIUM4_SHIFT_BY_UNITMASK (0x01 | 0x02 | 0x04 | 0x08)
-
-
-/*
- * PerfCtr_Config --
- *      Describes configuration for a single hardware performance counter
- *
- *      Since this is only used to record general performance counters, we
- *      made the assumption in nmiProfiler.c  that the type is GENERAL and
- *      index is counter number of type GENERAL.
- *      
- *      
- *      On AMD K8 and GH:
- *      index:        Which perf ctr, 0 to 3.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- *
- *      On AMD with PerfCtrExtCore support:
- *      index:        Which perf ctr, 0 to 5.  RDPMC argument
- *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
- *      escrAddr:     MSR # of the Perf Event Selector (0xc0010200 + 2 * index).
- *                                  aliased PMCs 0 - 3 (0xc0010000 + index).
- *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
- * 
- *      No field should be greater than 32-bit as it is shared with monitor.
- *
- *      On Intel Core architecture:
- *      <to be documented>
- */
-typedef struct PerfCtr_Config {
-   uint64 escrVal;
-   uint32 index;
-   uint32 addr;
-   uint32 escrAddr;
-   uint32 resetHi;
-   uint32 periodMean;
-
-   /*
-    * Random number (whose absolute value is capped at
-    * periodJitterMask) is used to randomize sampling interval.
-    */
-   uint32  periodJitterMask;
-   uint32  seed;    // seed is used to compute next random number
-   uint16  config;
-   Bool    valid;
-   Bool    pebsEnabled;
-} PerfCtr_Config;
-
+#define PERFCTR_CONFIG_ARCH_FIELDS \
+   uint32 addr;      \
+   uint32 escrAddr;  \
+   uint32 resetHi;   \
+   Bool pebsEnabled; \

 /*
 * Program/reprogram event reg(s) associated w/perfctrs & start or stop perfctrs
 */
 static INLINE void
-PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
-                   uint32 escrVal)       // IN: event register value
+PerfCtr_WriteEvtSel(uint32 addr,       // IN: Event register to write
+                    uint32 escrVal)    // IN: event register value
 {
-   __SET_MSR(ctr->escrAddr, escrVal);
+   X86MSR_SetMSR(addr, escrVal);
 }


@@ -685,11 +581,10 @@ PerfCtrWriteEvtSel(PerfCtr_Config *ctr,  // IN: counter to write
 * Set/reset performance counters to engender desired period before overflow
 */
 static INLINE void
-PerfCtrWriteCounter(PerfCtr_Config *ctr,   // IN: counter to write
-                    uint32 valueLo,        // IN: low 32 bits of value to write
-                    uint32 valueHi)        // IN: high 32 bits of value to write
+PerfCtr_WriteCounter(uint32 addr,   // IN: counter to write
+                     uint64 value)  // IN: value to write
 {
-   WRMSR(ctr->addr, valueLo, valueHi);
+   X86MSR_SetMSR(addr, value);
 }


@@ -808,8 +703,9 @@ PerfCtr_PEBSAvailable(void)
   CPUIDRegs regs;
   __GET_CPUID(0, &regs);
   if (CPUID_IsVendorIntel(&regs) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_EMON_AVAILABLE) != 0) &&
-       ((__GET_MSR(MSR_MISC_ENABLE) & MSR_MISC_ENABLE_PEBS_UNAVAILABLE) == 0)) {
+       (X86MSR_GetMSR(MSR_MISC_ENABLE) &
+        (MSR_MISC_ENABLE_EMON_AVAILABLE | MSR_MISC_ENABLE_PEBS_UNAVAILABLE))
+       == MSR_MISC_ENABLE_EMON_AVAILABLE) {
      CPUIDRegs hvendor;

      /*
@@ -821,7 +717,7 @@ PerfCtr_PEBSAvailable(void)
             !CPUID_IsRawVendor(&hvendor,
                                CPUID_HYPERV_HYPERVISOR_VENDOR_STRING);
   }
-   return FALSE; 
+   return FALSE;
 }

 /*
@@ -847,167 +743,9 @@ PerfCtr_PTAvailable(void)
   __GET_CPUID(0, &regs);
   if (CPUID_IsVendorIntel(&regs)) {
      __GET_CPUID2(7, 0, &regs);
-      return (regs.ecx & CPUID_INTERNAL_MASK_PT) != 0;
+      return (regs.ebx & CPUID_INTERNAL_MASK_PT) != 0;
   }
   return FALSE;
 }

-/* The following are taken from the Intel Architecture Manual,
- * Book 3, Table 14-4. 
- */ 
-
-/* --------- BEGIN INTEL DEFINES ------------------------ */
-
-#define PENTIUM4_MSR_BPU_COUNTER0_IDX           0
-#define PENTIUM4_MSR_BPU_COUNTER0_ADDR          0x300
-#define PENTIUM4_MSR_BPU_CCCR0                  0x360
-#define PENTIUM4_MSR_BPU_COUNTER1_IDX           1
-#define PENTIUM4_MSR_BPU_COUNTER1_ADDR          0x301
-#define PENTIUM4_MSR_BPU_CCCR1                  0x361
-#define PENTIUM4_MSR_BPU_COUNTER2_IDX           2
-#define PENTIUM4_MSR_BPU_COUNTER2_ADDR          0x302
-#define PENTIUM4_MSR_BPU_CCCR2                  0x362
-#define PENTIUM4_MSR_BPU_COUNTER3_IDX           3
-#define PENTIUM4_MSR_BPU_COUNTER3_ADDR          0x303
-#define PENTIUM4_MSR_BPU_CCCR3                  0x363
-#define PENTIUM4_MSR_MS_COUNTER0_IDX            4
-#define PENTIUM4_MSR_MS_COUNTER0_ADDR           0x304
-#define PENTIUM4_MSR_MS_CCCR0                   0x364
-#define PENTIUM4_MSR_MS_COUNTER1_IDX            5
-#define PENTIUM4_MSR_MS_COUNTER1_ADDR           0x305
-#define PENTIUM4_MSR_MS_CCCR1                   0x365
-#define PENTIUM4_MSR_MS_COUNTER2_IDX            6
-#define PENTIUM4_MSR_MS_COUNTER2_ADDR           0x306
-#define PENTIUM4_MSR_MS_CCCR2                   0x366
-#define PENTIUM4_MSR_MS_COUNTER3_IDX            7
-#define PENTIUM4_MSR_MS_COUNTER3_ADDR           0x307
-#define PENTIUM4_MSR_MS_CCCR3                   0x367
-#define PENTIUM4_MSR_FLAME_COUNTER0_IDX         8
-#define PENTIUM4_MSR_FLAME_COUNTER0_ADDR        0x308
-#define PENTIUM4_MSR_FLAME_CCCR0                0x368
-#define PENTIUM4_MSR_FLAME_COUNTER1_IDX         9
-#define PENTIUM4_MSR_FLAME_COUNTER1_ADDR        0x309
-#define PENTIUM4_MSR_FLAME_CCCR1                0x369
-#define PENTIUM4_MSR_FLAME_COUNTER2_IDX         10
-#define PENTIUM4_MSR_FLAME_COUNTER2_ADDR        0x30A
-#define PENTIUM4_MSR_FLAME_CCCR2                0x36A
-#define PENTIUM4_MSR_FLAME_COUNTER3_IDX         11
-#define PENTIUM4_MSR_FLAME_COUNTER3_ADDR        0x30B
-#define PENTIUM4_MSR_FLAME_CCCR3                0x36B
-#define PENTIUM4_MSR_IQ_COUNTER0_IDX            12
-#define PENTIUM4_MSR_IQ_COUNTER0_ADDR           0x30C
-#define PENTIUM4_MSR_IQ_CCCR0                   0x36C
-#define PENTIUM4_MSR_IQ_COUNTER1_IDX            13
-#define PENTIUM4_MSR_IQ_COUNTER1_ADDR           0x30D
-#define PENTIUM4_MSR_IQ_CCCR1                   0x36D
-#define PENTIUM4_MSR_IQ_COUNTER2_IDX            14
-#define PENTIUM4_MSR_IQ_COUNTER2_ADDR           0x30E
-#define PENTIUM4_MSR_IQ_CCCR2                   0x36E
-#define PENTIUM4_MSR_IQ_COUNTER3_IDX            15
-#define PENTIUM4_MSR_IQ_COUNTER3_ADDR           0x30F
-#define PENTIUM4_MSR_IQ_CCCR3                   0x36F
-#define PENTIUM4_MSR_IQ_COUNTER4_IDX            16
-#define PENTIUM4_MSR_IQ_COUNTER4_ADDR           0x310
-#define PENTIUM4_MSR_IQ_CCCR4                   0x370
-#define PENTIUM4_MSR_IQ_COUNTER5_IDX            17
-#define PENTIUM4_MSR_IQ_COUNTER5_ADDR           0x311
-#define PENTIUM4_MSR_IQ_CCCR5                   0x371
-
-#define PENTIUM4_MSR_ALF_ESCR0_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR0_ADDR             0x3CA
-#define PENTIUM4_MSR_ALF_ESCR1_IDX              1
-#define PENTIUM4_MSR_ALF_ESCR1_ADDR             0x3CB
-#define PENTIUM4_MSR_BPU_ESCR0_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR0_ADDR             0x3B2
-#define PENTIUM4_MSR_BPU_ESCR1_IDX              0
-#define PENTIUM4_MSR_BPU_ESCR1_ADDR             0x3B3
-#define PENTIUM4_MSR_BSU_ESCR0_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR0_ADDR             0x3A0
-#define PENTIUM4_MSR_BSU_ESCR1_IDX              7
-#define PENTIUM4_MSR_BSU_ESCR1_ADDR             0x3A1
-#define PENTIUM4_MSR_CRU_ESCR0_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR0_ADDR             0x3B8
-#define PENTIUM4_MSR_CRU_ESCR1_IDX              4
-#define PENTIUM4_MSR_CRU_ESCR1_ADDR             0x3B9
-#define PENTIUM4_MSR_CRU_ESCR2_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR2_ADDR             0x3CC
-#define PENTIUM4_MSR_CRU_ESCR3_IDX              5
-#define PENTIUM4_MSR_CRU_ESCR3_ADDR             0x3CD
-#define PENTIUM4_MSR_CRU_ESCR4_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR4_ADDR             0x3E0
-#define PENTIUM4_MSR_CRU_ESCR5_IDX              6
-#define PENTIUM4_MSR_CRU_ESCR5_ADDR             0x3E1
-#define PENTIUM4_MSR_DAC_ESCR0_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR0_ADDR             0x3A8
-#define PENTIUM4_MSR_DAC_ESCR1_IDX              5
-#define PENTIUM4_MSR_DAC_ESCR1_ADDR             0x3A9
-#define PENTIUM4_MSR_FIRM_ESCR0_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR0_ADDR            0x3A4
-#define PENTIUM4_MSR_FIRM_ESCR1_IDX             1
-#define PENTIUM4_MSR_FIRM_ESCR1_ADDR            0x3A5
-#define PENTIUM4_MSR_FLAME_ESCR0_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR0_ADDR           0x3A6
-#define PENTIUM4_MSR_FLAME_ESCR1_IDX            0
-#define PENTIUM4_MSR_FLAME_ESCR1_ADDR           0x3A7
-#define PENTIUM4_MSR_FSB_ESCR0_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR0_ADDR             0x3A2
-#define PENTIUM4_MSR_FSB_ESCR1_IDX              6
-#define PENTIUM4_MSR_FSB_ESCR1_ADDR             0x3A3
-#define PENTIUM4_MSR_IQ_ESCR0_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR0_ADDR              0x3BA
-#define PENTIUM4_MSR_IQ_ESCR1_IDX               0
-#define PENTIUM4_MSR_IQ_ESCR1_ADDR              0x3BB
-#define PENTIUM4_MSR_IS_ESCR0_IDX               1
-#define PENTIUM4_MSR_IS_ESCR0_ADDR              0x3B4
-#define PENTIUM4_MSR_IS_ESCR1_IDX               1
-#define PENTIUM4_MSR_IS_ESCR1_ADDR              0x3B5
-#define PENTIUM4_MSR_ITLB_ESCR0_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR0_ADDR            0x3B6
-#define PENTIUM4_MSR_ITLB_ESCR1_IDX             3
-#define PENTIUM4_MSR_ITLB_ESCR1_ADDR            0x3B7
-#define PENTIUM4_MSR_IX_ESCR0_IDX               5
-#define PENTIUM4_MSR_IX_ESCR0_ADDR              0x3C8
-#define PENTIUM4_MSR_IX_ESCR1_IDX               5
-#define PENTIUM4_MSR_IX_ESCR1_ADDR              0x3C9
-#define PENTIUM4_MSR_MOB_ESCR0_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR0_ADDR             0x3AA
-#define PENTIUM4_MSR_MOB_ESCR1_IDX              2
-#define PENTIUM4_MSR_MOB_ESCR1_ADDR             0x3AB
-#define PENTIUM4_MSR_MS_ESCR0_IDX               0
-#define PENTIUM4_MSR_MS_ESCR0_ADDR              0x3C0
-#define PENTIUM4_MSR_MS_ESCR1_IDX               0
-#define PENTIUM4_MSR_MS_ESCR1_ADDR              0x3C1
-#define PENTIUM4_MSR_PMH_ESCR0_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR0_ADDR             0x3AC
-#define PENTIUM4_MSR_PMH_ESCR1_IDX              4
-#define PENTIUM4_MSR_PMH_ESCR1_ADDR             0x3AD
-#define PENTIUM4_MSR_RAT_ESCR0_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR0_ADDR             0x3BC
-#define PENTIUM4_MSR_RAT_ESCR1_IDX              2
-#define PENTIUM4_MSR_RAT_ESCR1_ADDR             0x3BD
-#define PENTIUM4_MSR_SAAT_ESCR0_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR0_ADDR            0x3AE
-#define PENTIUM4_MSR_SAAT_ESCR1_IDX             2
-#define PENTIUM4_MSR_SAAT_ESCR1_ADDR            0x3AF
-#define PENTIUM4_MSR_SSU_ESCR0_IDX              3
-#define PENTIUM4_MSR_SSU_ESCR0_ADDR             0x3BE
-#define PENTIUM4_MSR_TBPU_ESCR0_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR0_ADDR            0x3C2
-#define PENTIUM4_MSR_TBPU_ESCR1_IDX             2
-#define PENTIUM4_MSR_TBPU_ESCR1_ADDR            0x3C3
-#define PENTIUM4_MSR_TC_ESCR0_IDX               1
-#define PENTIUM4_MSR_TC_ESCR0_ADDR              0x3C4
-#define PENTIUM4_MSR_TC_ESCR1_IDX               1
-#define PENTIUM4_MSR_TC_ESCR1_ADDR              0x3C5
-#define PENTIUM4_MSR_U2L_ESCR0_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR0_ADDR             0x3B0
-#define PENTIUM4_MSR_U2L_ESCR1_IDX              3
-#define PENTIUM4_MSR_U2L_ESCR1_ADDR             0x3B1
-
-#define PENTIUM4_MIN_ESCR_ADDR                  0x3A0
-#define PENTIUM4_MAX_ESCR_ADDR                  0x3E1
-#define PENTIUM4_NUM_ESCR_ADDRS                 (PENTIUM4_MAX_ESCR_ADDR - PENTIUM4_MIN_ESCR_ADDR + 1)
-
-/* -------- END INTEL DEFINES ------------------------- */
-
-#endif // ifndef _X86PERFCTR_H_ 
+#endif // _PERFCTR_ARCH_H
@@ -41,8 +41,8 @@
 */

 #define PSHARE_PAGELIST_MAX             (PAGELIST_MAX)
-#define PSHARE_P2M_BUFFER_MPNS_MAX      (16)
-#define PSHARE_P2M_BUFFER_MPNS_DEFAULT  (4)
+#define PSHARE_P2M_BUFFER_MPNS_MAX      (32)
+#define PSHARE_P2M_BUFFER_MPNS_DEFAULT  (5)
 #define PSHARE_P2M_BUFFER_SLOTS_PER_MPN (PAGE_SIZE / sizeof(PShare_P2MUpdate))
 #define PSHARE_P2M_BUFFER_SLOTS_MIN     (2)

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,6 +97,7 @@ extern PTSCInfo ptscInfo;
 Bool PTSC_Init(uint64 tscHz);
 VmAbsoluteTS PTSC_InitialCount(const char *module,
                               const char *option,
+                               Bool stress,
                               VmIntervalTS freq,
                               VmAbsoluteTS defaultCnt);
 Bool PTSC_HasPerfectlySynchronizedTSCs(void);
@@ -172,35 +173,6 @@ int64 PTSC_CyclesToUS(VmRelativeTS ts);
 */
 #include "user_layout.h"

-#ifdef VM_ARM_64
-static INLINE VmAbsoluteTS
-PTSC_Get(void)
-{
-   extern __thread User_ThreadData vmkUserTdata;
-
-   register RateConv_Params params;
-   register uint64 pseudoTSC;
-
-   /*
-    * On ARM64 the generic timer guarantees that the counters are synchronous
-    * and independent of CPU frequency.
-    */
-   params = vmkUserTdata.pseudoTSCConv;
-   pseudoTSC = RDTSC();
-
-   /*
-    *  assert that conversion rate is `identity'.
-    * we'll have to revisit that when/if this assumption changes
-    * on some platform
-    */
-   ASSERT((params.mult == 1 && params.shift == 0) ||
-          (params.mult == 0x80000000 && params.shift == 31));
-
-   pseudoTSC += params.add;
-
-   return pseudoTSC;
-}
-#else
 static INLINE VmAbsoluteTS
 PTSC_Get(void)
 {
@@ -214,7 +186,7 @@ PTSC_Get(void)
   ASSERT((int64)ptsc >= 0);
   return ptsc;
 }
-#endif
+
 #else

 /*
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,7 +97,7 @@ void RateConv_LogRatio(const char *prefix,
 static INLINE uint64
 RateConv_Unsigned(const RateConv_Params *conv, uint64 x)
 {
-   return Mul64x3264(x, conv->mult, conv->shift) + conv->add;
+   return Mul64x3264(x, conv->mult, conv->shift) + (uint64)conv->add;
 }


@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,28 +16,32 @@
 *
 *********************************************************/

-#ifndef _INITBLOCK_H
-#define _INITBLOCK_H
+/*
+ * sharedAreaType.h --
+ *
+ *      This file contains shared area type definitions.
+ */

+#ifndef _SHAREDAREATYPE_H_
+#define _SHAREDAREATYPE_H_

-#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+
 #include "includeCheck.h"

+typedef enum {
+   SHARED_AREA_PER_VM_VMX = 0,
+   SHARED_AREA_INTER_VCPU_VMX,
+   SHARED_AREA_PER_VCPU_VMX,
+   SHARED_AREA_PER_VM,
+   SHARED_AREA_INTER_VCPU,
+   SHARED_AREA_PER_VCPU,
+   NUM_SHARED_AREAS
+} SharedAreaType;

-#include "basic_initblock.h"
-
-
-#define MAX_LOGFILE_NAME_LENGTH 256
-#define MAX_MONITOR_REGIONS     8
-#define INIT_BLOCK_MAGIC     (0x1789+14)
-
-
-/*
- * Option flags 
- */
-/* None yet */
-
-
-#endif
+#endif // _SHAREDAREATYPE_H_
@@ -0,0 +1,62 @@
+/*********************************************************
+ * Copyright (C) 2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * sharedAreaVmmon.h --
+ *
+ *     VMMon shared area management.
+ */
+
+#ifndef SHARED_AREA_VMMON_H
+#define SHARED_AREA_VMMON_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#include "includeCheck.h"
+
+#include "iocontrols.h"
+
+struct VMDriver;
+
+/* A collection of pages backing a shared area region. */
+typedef struct SharedAreaVmmonRegion {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} SharedAreaVmmonRegion;
+
+/* The VMMon Driver shared area. */
+typedef struct SharedAreaVmmon {
+   SharedAreaVmmonRegion regions[NUM_SHARED_AREAS];
+} SharedAreaVmmon;
+
+/* Request for a backing MPN for a shared area region at a given VCPU/offset. */
+typedef struct SharedAreaVmmonRequest {
+   SharedAreaType type;
+   Vcpuid vcpu;
+   PageCnt offset;
+} SharedAreaVmmonRequest;
+
+SharedAreaVmmon *SharedAreaVmmon_Init(struct VMDriver *vm);
+void SharedAreaVmmon_Cleanup(SharedAreaVmmon *area);
+Bool SharedAreaVmmon_RegisterRegion(struct VMDriver *driver,
+                                    VMSharedAreaRegistrationBlock *block);
+Bool SharedAreaVmmon_ValidateRegionArgs(struct VMDriver *driver,
+                                        VMSharedAreaRegistrationBlock *block);
+MPN SharedAreaVmmon_GetRegionMPN(struct VMDriver *vm,
+                                 SharedAreaVmmonRequest *request);
+#endif /* SHARED_AREA_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006, 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,47 +17,32 @@
 *********************************************************/

 /*
- * basic_initblock.h --
+ * statVarsVmmon.h --
 *
- *    VM initialization block.
+ *     VMMon stat vars management.
 */

-#ifndef _BASIC_INITBLOCK_H_
-#define _BASIC_INITBLOCK_H_
+#ifndef STAT_VARS_VMMON_H
+#define STAT_VARS_VMMON_H

-
-#define INCLUDE_ALLOW_USERLEVEL
-
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_DISTRIBUTE
 #define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

+#include "iocontrols.h"

-#include "vcpuid.h"
+struct VMDriver;

+/* The VMMon Driver stat vars area. */
+typedef struct StatVarsVmmon {
+   PageCnt pagesPerVcpu;
+   MPN *pages;
+} StatVarsVmmon;

-#define MAX_INITBLOCK_CPUS     MAX_VCPUS
-
-
-typedef
-#include "vmware_pack_begin.h"
-struct InitBlock {
-   uint32 magicNumber;     /* Magic number (INIT_BLOCK_MAGIC) */
-   Vcpuid numVCPUs;
-   VA64   crosspage[MAX_INITBLOCK_CPUS];
-   LPN64  monStartLPN;
-   LPN64  monEndLPN;
-   uint32 vmInitFailurePeriod;
-   LA64   crossGDTHKLA;
-   MPN    crossGDTMPNs[5];  // CROSSGDT_NUMPAGES
-   uint16 numPTPPages;
-}
-#include "vmware_pack_end.h"
-InitBlock;
-
-
-#endif // _BASIC_INITBLOCK_H_
+StatVarsVmmon *StatVarsVmmon_Init(struct VMDriver *vm);
+void StatVarsVmmon_Cleanup(StatVarsVmmon *statVars);
+Bool StatVarsVmmon_RegisterVCPU(struct VMDriver *driver,
+                                VMStatVarsRegistrationBlock *block);
+MPN StatVarsVmmon_GetRegionMPN(struct VMDriver *vm, Vcpuid vcpuid,
+                               PageCnt offset);
+#endif /* STAT_VARS_VMMON_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 #ifndef USERCALLDEFS_H
 #define USERCALLDEFS_H

-#ifdef linux
+#ifdef __linux__
 #include <linux/errno.h>
 #define USERCALL_RESTART (-ERESTARTNOINTR)
 #else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,6 @@
 extern "C" {
 #endif

-
 typedef uint32 Vcpuid;                 // VCPU number

 #define VCPUID_INVALID  (~0U)
@@ -50,21 +49,15 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define BOOT_VCPU_ID     0
 #define IS_BOOT_VCPUID(vcpuid)  ((vcpuid) == BOOT_VCPU_ID)

-#define MAX_VCPUS      192
-
+#define MAX_VCPUS            2048
 #define MAX_CORES_PER_SOCKET   64
+#define MAX_VCPU_SOCKETS      128

-#ifdef VMM
-#include "vcpuset.h"
-
-/* In VMM, CurVcpuid() is available everywhere. */
-extern const Vcpuid      curVcpuid;
-extern const VCPUSet     curVcpuidSet;
-#define CurVcpuid()         (curVcpuid)
-#define CurVcpuidSet()      (&curVcpuidSet)
-#define IS_BOOT_VCPU()      IS_BOOT_VCPUID(CurVcpuid())
-
-#endif  /* VMM */
+/*
+ * There are several properties of the VM which change at the 128 VCPU
+ * boundary.  EFI firmware, x2APIC, and IOMMU are required among others.
+ */
+#define MAX_SMALL_VM_VCPUS 128

 #if defined __cplusplus
 } // extern "C"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * vcpuset.h --
 *
- *	ADT for a set of VCPUs.  Implemented as an array of bitmasks.
+ *      ADT for a set of VCPUs.  Implemented as an array of bitmasks.
 *
 */

@@ -80,22 +80,39 @@

 extern VCPUSet  vcpuSetFull;

-#define FOR_EACH_VCPU_IN_SET(_vcpuSet, _v)                                    \
-   do {                                                                       \
-      Vcpuid  _v;                                                             \
-      VCPUSet __vcs;                                                          \
-      VCPUSet_Copy(&__vcs, _vcpuSet);                                         \
-      while ((_v = VCPUSet_FindFirst(&__vcs)) != VCPUID_INVALID) {            \
-         VCPUSet_Remove(&__vcs, _v);
+#define FOR_EACH_VCPU_IN_SET_WITH_MAX(_vcpuSet, _v, _numVcpus)                 \
+   do {                                                                        \
+      Vcpuid _v;                                                               \
+      const VCPUSet *__vcs = (_vcpuSet);                                       \
+      unsigned _subsetIdx = 0;                                                 \
+      uint64 _subset = VCPUSet_Subset(__vcs, _subsetIdx);                      \
+      unsigned _maxSubsets = VCS_VCPUID_SUBSET_IDX((_numVcpus) - 1) + 1;       \
+      ASSERT(_maxSubsets <= VCS_SUBSET_COUNT);                                 \
+      while ((_v = VCPUSet_FindFirstInSubset(__vcs, &_subset, &_subsetIdx,     \
+                                             _maxSubsets)) != VCPUID_INVALID) {\

-#define ROF_EACH_VCPU_IN_SET()                                                \
-      }                                                                       \
+#define ROF_EACH_VCPU_IN_SET_WITH_MAX()                                 \
+      }                                                                 \
   } while (0)


+#define FOR_EACH_VCPU_IN_SET(_vcpuSet, _v)                              \
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(_vcpuSet, _v, MAX_VCPUS)
+
+#define ROF_EACH_VCPU_IN_SET()                                          \
+      }                                                                 \
+   } while (0)
+
+#define FOR_EACH_VCPU_IN_POPULATED_VCPUS(_vcpuSet, _v)                  \
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(_vcpuSet, _v, NumVCPUs())
+
+#define ROF_EACH_VCPU_IN_POPULATED_VCPUS()                              \
+      }                                                                 \
+   } while (0)
+
 #define FOR_EACH_SUBSET_IN_SET(_setIndex)                                     \
   do {                                                                       \
-      int _setIndex;                                                          \
+      unsigned _setIndex;                                                     \
      for (_setIndex = 0; _setIndex < VCS_SUBSET_COUNT; _setIndex++) {

 #define ROF_EACH_SUBSET_IN_SET()                                              \
@@ -105,18 +122,18 @@ extern VCPUSet  vcpuSetFull;

 #define FOR_EACH_SUBSET_IN_SET_COUNTDOWN(_setIndex)                           \
   do {                                                                       \
-      int _setIndex;                                                          \
-      for (_setIndex = VCS_SUBSET_COUNT - 1; _setIndex >= 0; _setIndex--) {
+      unsigned _setIndex = VCS_SUBSET_COUNT;                                  \
+      while (_setIndex-- > 0) {

 #define ROF_EACH_SUBSET_IN_SET_COUNTDOWN()                                    \
      }                                                                       \
   } while (0)


-#define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex)                           \
+#define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex, _numVcpus)                \
   do {                                                                       \
-      int _setIndex;                                                          \
-      int _maxSubsets = VCS_VCPUID_SUBSET_IDX(NumVCPUs() - 1);                \
+      unsigned _setIndex;                                                     \
+      unsigned _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);            \
      for (_setIndex = 0; _setIndex <= _maxSubsets; _setIndex++) {

 #define ROF_EACH_POPULATED_SUBSET_IN_SET()                                    \
@@ -128,7 +145,7 @@ extern VCPUSet  vcpuSetFull;
 *----------------------------------------------------------------------
 *
 * VCPUSet_Empty --
- *      
+ *
 *      Clear all bits in a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -147,7 +164,7 @@ VCPUSet_Empty(VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsEmpty --
- *      
+ *
 *      Return TRUE iff a VCPUSet has no bits set.
 *
 *----------------------------------------------------------------------
@@ -192,7 +209,7 @@ VCPUSet_Full(void)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Copy --
- *      
+ *
 *      Copy one VCPUSet's contents to another.
 *
 *----------------------------------------------------------------------
@@ -211,7 +228,7 @@ VCPUSet_Copy(VCPUSet *dest, const VCPUSet *src)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Equals --
- *      
+ *
 *      Compare two VCPUSets, return TRUE iff their contents match.
 *
 *----------------------------------------------------------------------
@@ -233,7 +250,7 @@ VCPUSet_Equals(const VCPUSet *vcs1, const VCPUSet *vcs2)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsMember --
- *      
+ *
 *      Return TRUE iff the given Vcpuid is present in a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -252,7 +269,7 @@ VCPUSet_IsMember(const VCPUSet *vcs, Vcpuid v)
 *----------------------------------------------------------------------
 *
 * VCPUSet_AtomicIsMember --
- *      
+ *
 *      Return TRUE iff the given Vcpuid is present in a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -273,7 +290,7 @@ VCPUSet_AtomicIsMember(VCPUSet *vcs, Vcpuid v)
 *
 * VCPUSet_FindFirst --
 * VCPUSet_FindLast --
- *      
+ *
 *      Find the first (lowest-numbered) or last (highest-numbered)
 *      Vcpuid in a VCPUSet.
 *
@@ -290,7 +307,7 @@ VCPUSet_FindFirst(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return lssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return lssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET();
   return VCPUID_INVALID;
@@ -302,19 +319,62 @@ VCPUSet_FindLast(const VCPUSet *vcs)
   FOR_EACH_SUBSET_IN_SET_COUNTDOWN(idx) {
      uint64 subset = vcs->subset[idx];
      if (subset != 0) {
-         return mssb64_0(subset) + (idx << VCS_SUBSET_SHIFT);
+         return mssb64(subset) - 1 + (idx << VCS_SUBSET_SHIFT);
      }
   } ROF_EACH_SUBSET_IN_SET_COUNTDOWN();
   return VCPUID_INVALID;
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * VCPUSet_FindFirstInSubset --
+ *
+ *      Find the first (lowest-numbered) Vcpuid in the given subset of a
+ *      VCPUSet object.  It is required that *subset is initialized to
+ *      a subset of *vcs and *subsetIdx must hold the index of the subset
+ *      stored in *subset.
+ *
+ * Results:
+ *      Vcpuid if at least one is present in a set.
+ *      VCPUID_INVALID if the set is empty.
+ *
+ * Side effects:
+ *      This function is intended to be used for iterating over all bits
+ *      of a VCPUSet, it will modify *subset to clear the bit associated
+ *      with the returned Vcpuid (if any).  The *subsetIdx argument may
+ *      also be updated.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Vcpuid
+VCPUSet_FindFirstInSubset(const VCPUSet *vcs, uint64 *subset,
+                          unsigned *subsetIdx, unsigned maxSubsets)
+{
+   ASSERT(*subsetIdx < maxSubsets && maxSubsets <= VCS_SUBSET_COUNT);
+   do {
+      if (*subset != 0) {
+         unsigned bit = (unsigned)lssb64_0(*subset);
+         *subset &= ~(CONST64U(1) << bit);
+         return bit + (*subsetIdx << VCS_SUBSET_SHIFT);
+      }
+      ++*subsetIdx;
+      if (*subsetIdx < maxSubsets) {
+         *subset = vcs->subset[*subsetIdx];
+      }
+   } while (*subsetIdx < maxSubsets);
+   return VCPUID_INVALID;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
 * VCPUSet_Remove --
 * VCPUSet_AtomicRemove --
- *      
+ *
 *      Remove or atomically remove a single Vcpuid from a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -371,7 +431,7 @@ VCPUSet_RemoveSet(VCPUSet *dest, const VCPUSet *src)
 *
 * VCPUSet_Include --
 * VCPUSet_AtomicInclude --
- *      
+ *
 *      Add or atomically add a single Vcpuid to a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -394,12 +454,33 @@ VCPUSet_AtomicInclude(VCPUSet *vcs, Vcpuid v)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * VCPUSet_AtomicTestInclude  --
+ *
+ *      Atomically add a single Vcpuid to a VCPUSet, and
+ *      return TRUE iff the given Vcpuid was present in the VCPUSet.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Bool
+VCPUSet_AtomicTestInclude(VCPUSet *vcs, Vcpuid v)
+{
+   volatile uint64 *subset = &vcs->subset[VCS_VCPUID_SUBSET_IDX(v)];
+   ASSERT(v < MAX_VCPUS);
+   return Atomic_TestSetBit64(Atomic_VolatileToAtomic64(subset),
+                              v & VCS_SUBSET_MASK);
+}
+
+
 #if defined(VMM) && !defined(MONITOR_APP)
 /*
 *----------------------------------------------------------------------
 *
 * VCPUSet_PackCareful --
- *      
+ *
 *      Pack a VCPUSet into the bytes at "ptr".
 *
 *----------------------------------------------------------------------
@@ -416,7 +497,7 @@ VCPUSet_PackCareful(unsigned numVCPUs, const VCPUSet *vcs, void *ptr)
 *----------------------------------------------------------------------
 *
 * VCPUSet_UnpackCareful --
- *      
+ *
 *      Unpack a VCPUSet from the bytes at "ptr".
 *
 *----------------------------------------------------------------------
@@ -483,7 +564,7 @@ VCPUSet_Populate(VCPUSet *vcs, unsigned numVCPUs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Subset --
- *      
+ *
 *      Return the specified subset of a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -502,7 +583,7 @@ VCPUSet_Subset(const VCPUSet *vcs,
 *----------------------------------------------------------------------
 *
 * VCPUSet_SubsetPtr --
- *      
+ *
 *      Return a pointer to the specified subset of a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -520,7 +601,7 @@ VCPUSet_SubsetPtr(VCPUSet *vcs, unsigned subset)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsSupersetOrEqual --
- *      
+ *
 *      Return TRUE iff vcs1 contains a superset of the VCPUs in vcs2
 *      or vcs1 and vcs2 contain exactly the same VCPUs.
 *
@@ -543,7 +624,7 @@ VCPUSet_IsSupersetOrEqual(const VCPUSet *vcs1, const VCPUSet *vcs2)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsSubsetOrEqual --
- *      
+ *
 *      Return TRUE iff vcs1 contains a subset of the VCPUs in vcs2
 *      or vcs1 and vcs2 contain exactly the same VCPUs.
 *
@@ -561,7 +642,7 @@ VCPUSet_IsSubsetOrEqual(const VCPUSet *vcs1, const VCPUSet *vcs2)
 *----------------------------------------------------------------------
 *
 * VCPUSet_MakeSingleton --
- *      
+ *
 *      Add a single Vcpuid to a VCPUSet and remove all others.
 *
 *----------------------------------------------------------------------
@@ -578,27 +659,34 @@ VCPUSet_MakeSingleton(VCPUSet *vcs, Vcpuid v)
 /*
 *----------------------------------------------------------------------
 *
- * VCPUSet_IsSingleton --
- *      
- *      Return TRUE iff vcs contains exactly one VCPU.
+ * VCPUSet_FindSingleton --
+ *
+ *      Return the VCPU in the set iff vcs contains exactly one VCPU.
+ *      Return VCPUID_INVALID otherwise.
 *
 *----------------------------------------------------------------------
 */

-static INLINE Bool
-VCPUSet_IsSingleton(const VCPUSet *vcs)
+static INLINE Vcpuid
+VCPUSet_FindSingleton(const VCPUSet *vcs)
 {
-   Bool foundOnce = FALSE;
+   uint64 foundSub = 0;
+   uint32 foundIdx = 0;
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 sub = vcs->subset[idx];
      if (sub != 0) {
-         if (foundOnce || (sub & (sub - 1)) != 0) {
-            return FALSE;
+         if (foundSub != 0 || (sub & (sub - 1)) != 0) {
+            return VCPUID_INVALID;
         }
-         foundOnce = TRUE;
+         foundSub = sub;
+         foundIdx = idx;
      }
   } ROF_EACH_SUBSET_IN_SET();
-   return foundOnce;
+   if (foundSub != 0) {
+      return lssb64(foundSub) - 1 + (foundIdx << VCS_SUBSET_SHIFT);
+   } else {
+      return VCPUID_INVALID;
+   }
 }


@@ -622,7 +710,7 @@ VCPUSet_IsFull(const VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_AtomicReadWriteSubset --
- *      
+ *
 *      For the nth aligned 64-VCPU subset of a VCPU set, atomically
 *      read then write.  Return the contents read.  Set 0 is VCPUs
 *      0-63 and set 1 is VCPUs 64-127.
@@ -667,7 +755,7 @@ VCPUSet_Size(const VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_UnionSubset --
- *      
+ *
 *      Given an 64-bit value and a subset number, add the VCPUs
 *      represented to the set.
 *
@@ -686,7 +774,7 @@ VCPUSet_UnionSubset(VCPUSet *vcs, uint64 vcpus, unsigned n)
 *----------------------------------------------------------------------
 *
 * VCPUSet_SubtractSubset --
- *      
+ *
 *      Given an 64-bit value and a subset number, remove the VCPUs
 *      represented in the subset from the set.
 *
@@ -705,7 +793,7 @@ VCPUSet_SubtractSubset(VCPUSet *vcs, uint64 vcpus, unsigned n)
 *----------------------------------------------------------------------
 *
 * VCPUSet_AtomicUnionSubset --
- *      
+ *
 *      Given a 64-bit value and a subset number, atomically add
 *      the VCPUs represented to the set.
 *
@@ -725,7 +813,7 @@ VCPUSet_AtomicUnionSubset(VCPUSet *vcs, uint64 vcpus, unsigned n)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Invert --
- *      
+ *
 *      Makes all non-present valid VCPUs in a set present and all
 *      VCPUs present non-present.
 *
@@ -746,7 +834,7 @@ VCPUSet_Invert(VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Intersection
- *      
+ *
 *      Given two VCPUSets, populate the destination set with only the
 *      VCPUs common to both.
 *
@@ -766,7 +854,7 @@ VCPUSet_Intersection(VCPUSet *dest, const VCPUSet *src)
 *----------------------------------------------------------------------
 *
 * VCPUSet_LogFormat --
- *      
+ *
 *      Given a buffer of at least VCS_BUF_SIZE to fill, write into it a
 *      string suitable for use in Log() or LOG().
 *      Returns the buffer which was passed as an argument, after
@@ -777,27 +865,27 @@ VCPUSet_Intersection(VCPUSet *dest, const VCPUSet *src)

 #ifdef VCS_SNPRINTF
 static INLINE char *
-VCPUSet_LogFormat(char *buf, const int size, const VCPUSet *vcs)
+VCPUSet_LogFormat(char *buf, size_t size, const VCPUSet *vcs)
 {
-   int offset = 0;
-   Vcpuid highest = VCPUSet_FindLast(vcs);
-   int idx = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+   unsigned offset  = 0;
+   Vcpuid   highest = VCPUSet_FindLast(vcs);
+   unsigned idx     = (highest == VCPUID_INVALID) ? 0 : highest / 8;
+
   ASSERT(size >= VCS_BUF_SIZE);
 #define VCS_LOGF(...)                                                   \
   {                                                                    \
      int ret = VCS_SNPRINTF(buf + offset, size - offset, __VA_ARGS__); \
-      ASSERT(0 <= ret && ret < size - offset);                          \
-      offset += ret;                                                    \
+      ASSERT(0 <= ret && size >= offset && ret < (int)(size - offset)); \
+      offset += (unsigned)ret;                                          \
   }
   /* Print the leading value with no zero-extension. */
-   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx--]);
+   VCS_LOGF("%#x", ((unsigned char *)vcs)[idx]);

-   while (idx >= 0) {
+   while (idx-- > 0) {
      if ((idx + 1) % (VCS_SUBSET_WIDTH / 8) == 0) {
         VCS_LOGF(".");
      }
      VCS_LOGF("%02x", ((unsigned char *)vcs)[idx]);
-      idx--;
   }
   return buf;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2013, 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2013, 2016-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,7 +41,7 @@
 #define VCS_SUBSET_WIDTH                                                   64
 #define VCS_SUBSET_SHIFT                                                    6
 #define VCS_SUBSET_MASK               ((CONST64U(1) << VCS_SUBSET_SHIFT) - 1)
-#define VCS_SUBSET_COUNT                                                    4
+#define VCS_SUBSET_COUNT                                                   32


 #define VCS_VCPUID_SUBSET_IDX(v)                    ((v) >> VCS_SUBSET_SHIFT)
@@ -56,11 +56,12 @@ typedef struct VCPUSet {
 } VCPUSet;

 MY_ASSERTS(VCPUSET_ASSERTS,
+           ASSERT_ON_COMPILE(VCS_SUBSET_WIDTH * VCS_SUBSET_COUNT >= MAX_VCPUS);
           /*
            * Catch changes in VCPUSet which need to be reflected in
            * bora/public/iocontrolsMacosTable.h.
            */
-           ASSERT_ON_COMPILE(VCS_SUBSET_COUNT == 4);
+           ASSERT_ON_COMPILE(VCS_SUBSET_COUNT == 32);
           /*
            * There is code that depends on sizeof(VCPUSet) being a power of
            * 2 in at least vcpuHotPlug.c and possible other places.
@@ -144,9 +144,6 @@ VersionedAtomic_BeginWrite(VersionedAtomic *versions) // IN
   /*
    * The write to 'versions->v0' must be observed by all other OS CPUs before
    * the write(s) to the shared data.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine.
    */
   SMP_W_BARRIER_W();
 }
@@ -176,9 +173,6 @@ VersionedAtomic_EndWrite(VersionedAtomic *versions) // IN
   /*
    * The write(s) to the shared data must be observed by all other OS CPUs
    * before the write to 'versions->v1'.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine.
    */
   SMP_W_BARRIER_W();

@@ -223,10 +217,6 @@ VersionedAtomic_BeginTryRead(VersionedAtomic const *versions) // IN
   /*
    * The read from 'versions->v1' must happen on this CPU before the read(s)
    * from the shared data.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine, and maybe
-    * Non-shareable (== current CPU) would work too.
    */
   SMP_R_BARRIER_R();

@@ -262,10 +252,6 @@ VersionedAtomic_EndTryRead(VersionedAtomic const *versions, // IN
   /*
    * The read(s) from the shared data must happen on this CPU before the read
    * from 'versions->v0'.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine, and maybe
-    * Non-shareable (== current CPU) would work too.
    */
   SMP_R_BARRIER_R();

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,52 +41,47 @@

 #include "vm_basic_asm.h"
 #include "x86msr.h"
-
-#ifdef VM_X86_64
-#include "vm_asm_x86_64.h"
-#else
 #include "vm_asm_x86.h"
-#endif


 static INLINE void
 SET_FS64(uint64 fs64)
 {
-   __SET_MSR(MSR_FSBASE, fs64);
+   X86MSR_SetMSR(MSR_FSBASE, fs64);
 }


 static INLINE void
 SET_GS64(uint64 gs64)
 {
-   __SET_MSR(MSR_GSBASE, gs64);
+   X86MSR_SetMSR(MSR_GSBASE, gs64);
 }

 static INLINE void
 SET_KernelGS64(uint64 kgs64)
 {
-   __SET_MSR(MSR_KERNELGSBASE, kgs64);
+   X86MSR_SetMSR(MSR_KERNELGSBASE, kgs64);
 }


 static INLINE uint64
 GET_FS64(void)
 {
-   return __GET_MSR(MSR_FSBASE);
+   return X86MSR_GetMSR(MSR_FSBASE);
 }


 static INLINE uint64
 GET_GS64(void)
 {
-   return __GET_MSR(MSR_GSBASE);
+   return X86MSR_GetMSR(MSR_GSBASE);
 }


 static INLINE uint64
 GET_KernelGS64(void)
 {
-   return __GET_MSR(MSR_KERNELGSBASE);
+   return X86MSR_GetMSR(MSR_KERNELGSBASE);
 }

 #endif // VM_ARM_64
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * vm_asm_x86.h
 *
- *	IA32 asm macros
+ *      IA32 asm macros
 */

 #ifndef _VM_ASM_X86_H_
@@ -49,10 +49,6 @@

 #ifdef __GNUC__

-#if defined __APPLE__
-/* PR 352418: GCC produces error if the non-Apple version is used */
-#define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
-#else
 /* ASSERT_ON_COMPILE_SELECTOR_SIZE:
 *
 *  - Selector must be 16-bits.
@@ -66,26 +62,16 @@
 *                    ((__builtin_constant_p(expr) ? ((expr) >> 16) == 0) \
 *                                                 : sizeof(expr) <= 2)
 */
-#if (__GNUC__ >= 4) && (__GNUC_MINOR__ >= 1)
+#ifndef USE_UBSAN
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)                                \
   ASSERT_ON_COMPILE(sizeof(Selector) == 2 &&                                \
                     ((__builtin_constant_p(expr) && ((expr) >> 16) == 0) || \
                      sizeof(expr) <= 2))
 #else
-/* gcc 3.3.3 is not able to produce a constant expression (PR 356383) */
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)
 #endif
-#endif


-/*
- * [GS]ET_[GI]DT() are defined as macros wrapping a function
- * so we can pass the argument implicitly by reference (requires
- * a macro) and get type checking too (requires a function).
- */
-
-#define SET_GDT(var) _Set_GDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_GDT(_GETSET_DTR_TYPE *dtr)
@@ -97,8 +83,6 @@ _Set_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define SET_IDT(var) _Set_IDT(&(var))
-
 /* Checked against the Intel manual and GCC --hpreg */
 static INLINE void
 _Set_IDT(_GETSET_DTR_TYPE *dtr)
@@ -110,8 +94,6 @@ _Set_IDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_GDT(var) _Get_GDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because there's a hidden input (the [IG]DTR) that can change
@@ -126,8 +108,6 @@ _Get_GDT(_GETSET_DTR_TYPE *dtr)
   );
 }

-#define GET_IDT(var) _Get_IDT(&(var))
-
 /*
 * Checked against the Intel manual and GCC --hpreg
 * volatile because the [IG]DT can change without the compiler knowing it
@@ -218,22 +198,6 @@ _BUILD_GET_R(_GET_CR3, cr3)
 _BUILD_GET_R(_GET_CR4, cr4)
 _BUILD_GET_R(_GET_CR8, cr8)

-#if defined __APPLE__
-/* Mac OS gcc 4 uses DBx instead of DRx register names. */
-_BUILD_SET_R(_SET_DR0, db0)
-_BUILD_SET_R(_SET_DR1, db1)
-_BUILD_SET_R(_SET_DR2, db2)
-_BUILD_SET_R(_SET_DR3, db3)
-_BUILD_SET_R(_SET_DR6, db6)
-_BUILD_SET_R(_SET_DR7, db7)
-
-_BUILD_GET_R(_GET_DR0, db0)
-_BUILD_GET_R(_GET_DR1, db1)
-_BUILD_GET_R(_GET_DR2, db2)
-_BUILD_GET_R(_GET_DR3, db3)
-_BUILD_GET_R(_GET_DR6, db6)
-_BUILD_GET_R(_GET_DR7, db7)
-#else
 _BUILD_SET_R(_SET_DR0, dr0)
 _BUILD_SET_R(_SET_DR1, dr1)
 _BUILD_SET_R(_SET_DR2, dr2)
@@ -247,47 +211,8 @@ _BUILD_GET_R(_GET_DR2, dr2)
 _BUILD_GET_R(_GET_DR3, dr3)
 _BUILD_GET_R(_GET_DR6, dr6)
 _BUILD_GET_R(_GET_DR7, dr7)
-#endif

-#define SET_CR_DR(regType, regNum, expr)                     \
-   do {                                                      \
-      /* Ensure no implicit truncation of 'expr' */          \
-      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
-      _SET_##regType##regNum(expr);                          \
-   } while (0)
-
-#define GET_CR_DR(regType, regNum, var) \
-   do {                                 \
-      _GET_##regType##regNum(&(var));   \
-   } while (0)
-
-#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
 #define SET_CR2(expr) SET_CR_DR(CR, 2, expr)
-#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
-#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
-#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
-
-/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
-#undef GET_CR0
-#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
-#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
-#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
-#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
-#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
-
-#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
-#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
-#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
-#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
-#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
-#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
-
-#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
-#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
-#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
-#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
-#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
-#define GET_DR7(var)  GET_CR_DR(DR, 7, var)

 #define SET_SEGREG(reg, expr)                                          \
   do {                                                                \
@@ -426,440 +351,347 @@ CLTS(void)
 /* Beginning of the section whose correctness has NOT been checked */
 #define FNCLEX()         __asm__("fnclex" ::);

-/* TLB_INVALIDATE_xxx are not checked yet */
+/* TLB_INVALIDATE_PAGE is not checked yet */
 #define TLB_INVALIDATE_PAGE(_addr) do { \
     __asm__ __volatile__("invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
 } while (0)

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) do { \
-     __asm__ __volatile__("fs; invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
-} while (0)
-
 #define RESTORE_FLAGS _Set_flags
 #define ENABLE_INTERRUPTS() __asm__ __volatile__ ("sti": : :"memory")
 #define CLEAR_INTERRUPTS()  __asm__ __volatile__ ("cli": : :"memory")
 #define RAISE_INTERRUPT(_x)  __asm__ __volatile__("int %0" :: "g" (_x))
 #define RETURN_FROM_INT()   __asm__ __volatile__("iret" :: )

-#if ! defined(VMKERNEL)
-#define NO_INTERRUPTS_BEGIN()	do { \
-                                   uintptr_t _flags; \
-                                   SAVE_FLAGS(_flags); \
-                                   CLEAR_INTERRUPTS();
-
-#define NO_INTERRUPTS_END()	   RESTORE_FLAGS(_flags); \
-                                } while(0)
-#endif
-
 /* End of the section whose correctness has NOT been checked */

 #elif defined _MSC_VER  /* !__GNUC__ */

-#define SET_IDT(_idt)    _Set_IDT(&(_idt))
-#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define _BUILD_SET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writedr(reg, r);                              \
+   }
+
+#define _BUILD_GET_DR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t *result)                            \
+   {                                                  \
+      *result = __readdr(reg);                        \
+   }
+
+#define _BUILD_SET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t r)                                  \
+   {                                                  \
+      __writecr##reg(r);                              \
+   }
+
+#define _BUILD_GET_CR(func, reg)                      \
+   static INLINE void                                 \
+   func(uintptr_t *result)                            \
+   {                                                  \
+      *result = __readcr##reg();                      \
+   }
+
+_BUILD_SET_DR(_SET_DR0, 0)
+_BUILD_SET_DR(_SET_DR1, 1)
+_BUILD_SET_DR(_SET_DR2, 2)
+_BUILD_SET_DR(_SET_DR3, 3)
+_BUILD_SET_DR(_SET_DR6, 6)
+_BUILD_SET_DR(_SET_DR7, 7)
+
+_BUILD_GET_DR(_GET_DR0, 0)
+_BUILD_GET_DR(_GET_DR1, 1)
+_BUILD_GET_DR(_GET_DR2, 2)
+_BUILD_GET_DR(_GET_DR3, 3)
+_BUILD_GET_DR(_GET_DR6, 6)
+_BUILD_GET_DR(_GET_DR7, 7)
+
+_BUILD_SET_CR(_SET_CR0, 0);
+_BUILD_SET_CR(_SET_CR3, 3);
+_BUILD_SET_CR(_SET_CR4, 4);
+_BUILD_SET_CR(_SET_CR8, 8);
+
+_BUILD_GET_CR(_GET_CR0, 0);
+_BUILD_GET_CR(_GET_CR2, 2);
+_BUILD_GET_CR(_GET_CR3, 3);
+_BUILD_GET_CR(_GET_CR4, 4);
+_BUILD_GET_CR(_GET_CR8, 8);
+
+static INLINE void
+_Set_GDT(_GETSET_DTR_TYPE *dtr)
+{
+#ifdef VM_X86_64
+   _lgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _lgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm lgdt [eax]
+#endif
+}
+
+static INLINE void
+_Get_GDT(_GETSET_DTR_TYPE *dtr)
+{
+#ifdef VM_X86_64
+   _sgdt(dtr);
+#else
+   /*
+    * timetools 32-bit targets use an old version of MSVC that doesn't support
+    * the _sgdt intrinsic. Luckily, 32-bit still supports inline asm.
+    */
+   __asm mov eax, dtr
+   __asm sgdt [eax]
+#endif
+}
+
+static INLINE void
+_Set_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __lidt(dtr);
+}
+
+static INLINE void
+_Get_IDT(_GETSET_DTR_TYPE *dtr)
+{
+   __sidt(dtr);
+}
+
+
+#define ENABLE_INTERRUPTS() _enable()
+#define CLEAR_INTERRUPTS()  _disable()
+
+#define SAVE_FLAGS(x) do { \
+   (x) = __readeflags();   \
+} while (0)
+
+#define RESTORE_FLAGS(x) __writeeflags(x)
+
+
+#ifdef VM_X86_32
+
 #define SET_TR(_tr)      _Set_TR(_tr)
 #define SET_LDT(_tr)     _Set_LDT(_tr)

-#define GET_IDT(_idt)    _Get_IDT(&(_idt))
-#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
 #define GET_TR(_tr)      do { _tr = _Get_TR();  } while (0)
 #define GET_LDT(_tr)     do { _tr = _Get_LDT(); } while (0)

-#define GET_CR0(_reg)    __asm mov eax, cr0 __asm mov _reg, eax
-#define SET_CR0(_reg)    __asm mov eax, _reg __asm mov cr0, eax
-#define GET_CR2(_reg)    __asm mov eax, cr2 __asm mov _reg, eax
 #define SET_CR2(_reg)    __asm mov eax, _reg __asm mov cr2, eax
-#define GET_CR3(_reg)    __asm mov eax, cr3 __asm mov _reg, eax
-#define SET_CR3(_reg)    __asm mov eax, _reg __asm mov cr3, eax
-/*
- * MSC doesn't seem to like CR4 in __asm statements. We emit
- * the opcode for MOV EAX,CR4 = 0xf020e0 and MOV CR4,EAX = 0xf022e0
- */
-#define GET_CR4(_reg) { \
- __asm _emit 0x0f __asm _emit 0x20 __asm _emit 0xe0 \
- __asm mov _reg, eax \
-}
-#define SET_CR4(_reg)    { \
-  __asm mov eax, _reg \
-  __asm _emit 0x0f __asm _emit 0x22 __asm _emit 0xe0 \
-}
-
-
-#define GET_DR0(_reg) do { __asm mov eax,dr0 __asm mov _reg,eax } while (0)
-#define SET_DR0(_reg) do { __asm mov eax,_reg __asm mov dr0,eax } while (0)
-#define GET_DR1(_reg) do { __asm mov eax,dr1 __asm mov _reg,eax } while (0)
-#define SET_DR1(_reg) do { __asm mov eax,_reg __asm mov dr1,eax } while (0)
-#define GET_DR2(_reg) do { __asm mov eax,dr2 __asm mov _reg,eax } while (0)
-#define SET_DR2(_reg) do { __asm mov eax,_reg __asm mov dr2,eax } while (0)
-#define GET_DR3(_reg) do { __asm mov eax,dr3 __asm mov _reg,eax } while (0)
-#define SET_DR3(_reg) do { __asm mov eax,_reg __asm mov dr3,eax } while (0)
-#define GET_DR6(_reg) do { __asm mov eax,dr6 __asm mov _reg,eax } while (0)
-#define SET_DR6(_reg) do { __asm mov eax,_reg __asm mov dr6,eax } while (0)
-#define GET_DR7(_reg) do { __asm mov eax,dr7 __asm mov _reg,eax } while (0)
-#define SET_DR7(_reg) do { __asm mov eax,_reg __asm mov dr7,eax } while (0)
-
-
 #define CLTS()           __asm clts

 #define FNCLEX()         __asm fnclex

 #define TLB_INVALIDATE_PAGE(_addr) {  \
-	 void *_a = (_addr); \
-     __asm mov eax, _a __asm invlpg [eax] \
+   void *_a = (_addr); \
+   __asm mov eax, _a __asm invlpg [eax] \
 }

-#define TLB_INVALIDATE_PAGE_OFF_FS(_addr) { \
-	uint32 __a = (uint32) (_addr); \
-	__asm mov eax, __a _asm invlpg fs:[eax] \
-}
-
-
-#define ENABLE_INTERRUPTS() { __asm sti }
-#define CLEAR_INTERRUPTS()  { __asm cli }
-
 #define RAISE_INTERRUPT(_x)  {__asm int _x }
 #define RETURN_FROM_INT()   {__asm iretd }


-#define SAVE_FLAGS(x) { \
-     __asm pushfd __asm pop eax __asm mov x, eax \
-}
-
-#define RESTORE_FLAGS(x) { \
-     __asm push x __asm popfd\
-}
-
-
-
-static INLINE void SET_DS(Selector val)
+static INLINE void
+SET_DS(Selector val)
 {
   __asm mov ax, val
   __asm mov ds, ax
 }

-static INLINE void SET_ES(Selector val)
+static INLINE void
+SET_ES(Selector val)
 {
   __asm mov ax, val
   __asm mov es, ax
 }

-static INLINE void SET_FS(Selector val)
+static INLINE void
+SET_FS(Selector val)
 {
   __asm mov ax, val
   __asm mov fs, ax
 }

-static INLINE void SET_GS(Selector val)
+static INLINE void
+SET_GS(Selector val)
 {
   __asm mov ax, val
   __asm mov gs, ax
 }

-static INLINE void SET_SS(Selector val)
+static INLINE void
+SET_SS(Selector val)
 {
   __asm mov ax, val
   __asm mov ss, ax
 }

-static INLINE Selector GET_FS(void)
+static INLINE Selector
+GET_FS(void)
 {
-	Selector _v;
-	__asm mov _v,fs
-	return _v;
+   Selector _v;
+   __asm mov _v,fs
+   return _v;
 }

-static INLINE Selector GET_GS(void)
+static INLINE Selector
+GET_GS(void)
 {
-	Selector _v;
-	__asm mov _v,gs
-	return _v;
+   Selector _v;
+   __asm mov _v,gs
+   return _v;
 }

-
-static INLINE Selector GET_DS(void)
+static INLINE Selector
+GET_DS(void)
 {
-	Selector _v;
-	__asm mov _v,ds
-	return _v;
+   Selector _v;
+   __asm mov _v,ds
+   return _v;
 }

-static INLINE Selector GET_ES(void)
+static INLINE Selector
+GET_ES(void)
 {
-	Selector _v;
-	__asm mov _v,es
-	return _v;
+   Selector _v;
+   __asm mov _v,es
+   return _v;
 }

-static INLINE Selector GET_SS(void)
+static INLINE Selector
+GET_SS(void)
 {
-	Selector _v;
-	__asm mov _v,ss
-	return _v;
+   Selector _v;
+   __asm mov _v,ss
+   return _v;
 }

-static INLINE Selector GET_CS(void)
+static INLINE Selector
+GET_CS(void)
 {
-	Selector _v;
-	__asm mov _v,cs
-	return _v;
+   Selector _v;
+   __asm mov _v,cs
+   return _v;
 }

-#pragma warning( disable : 4035)
-
-static INLINE uint32  GET_WORD_FROM_FS(uint32 *_addr) {
-	__asm mov eax, _addr
-    __asm mov eax, fs:[eax]
-}
-
-static INLINE uint16  GET_SHORT_FROM_FS(uint16 *_addr) {
-	__asm mov eax, _addr
-    __asm mov ax, fs:[eax]
-}
-
-static INLINE uint8  GET_BYTE_FROM_FS(uint8 *_addr) {
-	__asm mov eax, _addr
-     __asm mov al, fs:[eax]
-}
-
-#pragma warning (default: 4035)
-
-static INLINE void  SET_WORD_FS(uint32 *_addr, uint32 _val) {
-    __asm mov eax, _addr
-    __asm mov ebx, _val
-    __asm mov fs:[eax], ebx
-}
-
-static INLINE void  SET_SHORT_FS(uint32 *_addr, uint16 _val) {
-    __asm mov eax, _addr
-    __asm mov bx, _val
-    __asm mov fs:[eax], bx
-}
-
-static INLINE void  SET_BYTE_FS(uint32 *_addr, uint8 _val) {
-    __asm mov eax, _addr
-    __asm mov bl, _val
-    __asm mov fs:[eax], bl
-}
-
-static INLINE void _Set_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lgdt [eax]
-}
-
-static INLINE void _Set_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm lidt [eax]
-}
-
-static INLINE void _Set_LDT(Selector val)
+static INLINE void
+_Set_LDT(Selector val)
 {
   __asm lldt val
 }

-static INLINE void _Set_TR(Selector val)
+static INLINE void
+_Set_TR(Selector val)
 {
   __asm ltr val
 }

-static INLINE void _Get_GDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sgdt [eax]
-}
-
-static INLINE void _Get_IDT(_GETSET_DTR_TYPE *dtr) {
-   __asm mov eax, dtr
-   __asm sidt [eax]
-}
-
-static INLINE Selector _Get_LDT(void) {
+static INLINE Selector
+_Get_LDT(void)
+{
   Selector sel;
   __asm sldt sel
   return sel;
 }

-static INLINE Selector _Get_TR(void) {
+static INLINE Selector
+_Get_TR(void)
+{
   Selector sel;
   __asm str sel
   return sel;
 }

-
-static INLINE void
-MEMCOPY_TO_FS(VA to,
-              char * from,
-              unsigned long n)
-{
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x = *(uint32*) (from + i);
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, x
-	  __asm mov fs:[eax], ebx
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x = from[i];
-	  uint32 _faddr = (uint32) (to+i);
-	  __asm mov eax, _faddr
-	  __asm mov bl, x
-	  __asm mov fs:[eax], bl
-      i++;
-   }
-}
-
-
-
-static INLINE void
-MEMCOPY_FROM_FS(char * to,
-                VA from,
-                unsigned long n)
-{
-   unsigned long i =0;
-   while (i+4  <=n) {
-      uint32 x;
-	  uint32 _faddr = (uint32)(from+i);
-	  __asm mov eax, _faddr
-	  __asm mov ebx, fs:[eax]
-	  __asm mov x,ebx
-      *(uint32*)(to+i)=x;
-      i +=4;
-   }
-   while (i<n) {
-      uint8 x;
-	  uint32 _faddr = (uint32) (from+i);
-	  __asm mov eax, _faddr;
-      __asm mov bl, fs:[eax]
-	  __asm mov x, bl
-      *(uint8*)(to+i)=x;
-      i++;
-   }
-}
-
-#else
-#error No compiler defined for get/set
+#endif /* !VM_X86_32 */
 #endif /* !__GNUC__ && !_MSC_VER */


-#ifdef __GNUC__
-static INLINE unsigned CURRENT_CPL(void)
+#define SET_CR_DR(regType, regNum, expr)                     \
+   do {                                                      \
+      /* Ensure no implicit truncation of 'expr' */          \
+      ASSERT_ON_COMPILE(sizeof(expr) <= sizeof(uintptr_t));  \
+      _SET_##regType##regNum(expr);                          \
+   } while (0)
+
+#define GET_CR_DR(regType, regNum, var) \
+   do {                                 \
+      _GET_##regType##regNum(&(var));   \
+   } while (0)
+
+#define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
+#define SET_DR1(expr) SET_CR_DR(DR, 1, expr)
+#define SET_DR2(expr) SET_CR_DR(DR, 2, expr)
+#define SET_DR3(expr) SET_CR_DR(DR, 3, expr)
+#define SET_DR6(expr) SET_CR_DR(DR, 6, expr)
+#define SET_DR7(expr) SET_CR_DR(DR, 7, expr)
+
+#define GET_DR0(var)  GET_CR_DR(DR, 0, var)
+#define GET_DR1(var)  GET_CR_DR(DR, 1, var)
+#define GET_DR2(var)  GET_CR_DR(DR, 2, var)
+#define GET_DR3(var)  GET_CR_DR(DR, 3, var)
+#define GET_DR6(var)  GET_CR_DR(DR, 6, var)
+#define GET_DR7(var)  GET_CR_DR(DR, 7, var)
+
+/* Undefine GET_CR0; it is defined in mach_asm.h for SLES cross-compile */
+#undef GET_CR0
+#define GET_CR0(var)  GET_CR_DR(CR, 0, var)
+#define GET_CR2(var)  GET_CR_DR(CR, 2, var)
+#define GET_CR3(var)  GET_CR_DR(CR, 3, var)
+#define GET_CR4(var)  GET_CR_DR(CR, 4, var)
+#define GET_CR8(var)  GET_CR_DR(CR, 8, var)
+
+#define SET_CR0(expr) SET_CR_DR(CR, 0, expr)
+#define SET_CR3(expr) SET_CR_DR(CR, 3, expr)
+#define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
+#define SET_CR8(expr) SET_CR_DR(CR, 8, expr)
+
+static INLINE Bool
+INTERRUPTS_ENABLED(void)
+{
+   uintptr_t flags;
+   SAVE_FLAGS(flags);
+   return ((flags & EFLAGS_IF) != 0);
+}
+
+/*
+ * [GS]ET_[GI]DT() are defined as macros wrapping a function
+ * so we can pass the argument implicitly by reference (requires
+ * a macro) and get type checking too (requires a function).
+ */
+
+#define SET_GDT(_gdt)    _Set_GDT(&(_gdt))
+#define GET_GDT(_gdt)    _Get_GDT(&(_gdt))
+
+#define SET_IDT(_idt)    _Set_IDT(&(_idt))
+#define GET_IDT(_idt)    _Get_IDT(&(_idt))
+
+#if !defined(VMKERNEL)
+#define NO_INTERRUPTS_BEGIN() do { \
+                                   uintptr_t _flags; \
+                                   SAVE_FLAGS(_flags); \
+                                   CLEAR_INTERRUPTS();
+
+#define NO_INTERRUPTS_END()        RESTORE_FLAGS(_flags); \
+                                 } while(0)
+#endif
+
+
+#if defined (__GNUC__) || (defined (_MSC_VER) && defined (VM_X86_32))
+static INLINE unsigned
+CURRENT_CPL(void)
 {
   return SELECTOR_RPL(GET_CS());
 }
-#elif defined _MSC_VER
-static INLINE unsigned CURRENT_CPL(void) {
-   volatile Selector _v;
-   __asm mov ax, cs _asm mov _v, ax
-   return SELECTOR_RPL(_v);
-}
-#else
-#error No compiler defined for CURRENT_CPL
 #endif


+static INLINE uint64
+RDPMC(int counter)
+{
 #ifdef __GNUC__
-/* Checked against the Intel manual and GCC --hpreg
-
-   volatile because the msr can change without the compiler knowing it
-   (when we use wrmsr). */
-static INLINE uint64 __GET_MSR(int cx)
-{
-   uint64 msr;
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
-      : "=a" (msr)
-      : "c" (cx)
-      : "%rdx"
-   );
-#else
-   __asm__ __volatile__(
-      "rdmsr"
-      : "=A" (msr)
-      : "c" (cx)
-   );
-#endif
-   return msr;
-}
-#elif defined _MSC_VER
-#pragma warning( disable : 4035)
-static INLINE uint64 __GET_MSR(int input)
-{
-   __asm push ecx
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x32
-   __asm pop ecx
-}
-
-static INLINE void __SET_MSR(int input, uint64 val)
-{
-   uint32 hival = (uint32)(val >> 32);
-   uint32 loval = (uint32)val;
-   __asm push edx
-   __asm push ecx
-   __asm push eax
-   __asm mov  eax, loval
-   __asm mov  edx, hival
-   __asm mov  ecx, input
-   __asm _emit 0x0f __asm _emit 0x30
-   __asm pop eax
-   __asm pop ecx
-   __asm pop edx
-}
-#pragma warning (default: 4035)
-#else
-#error
-#endif
-
-#ifdef __GNUC__
-static INLINE void __SET_MSR(int cx, uint64 val)
-{
-#ifdef VM_X86_64
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "a" ((uint32) val), "d" ((uint32)(val >> 32)), "c" (cx)
-    );
-#else
-   __asm__ __volatile__(
-      "wrmsr"
-      : /* no outputs */
-      : "A" (val),
-        "c" (cx)
-    );
-#endif
-}
-#endif
-
-
-/*
- * RDMSR/WRMSR access the 64bit MSRs as two
- * 32 bit quantities, whereas GET_MSR/SET_MSR
- * above access the MSRs as one 64bit quantity.
- */
-#ifdef __GNUC__
-#undef RDMSR
-#undef WRMSR
-#define RDMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "rdmsr"                         \
-      : "=a" (low), "=d" (high)       \
-      : "c" (msrNum)                  \
-   );                                 \
-} while (0)
-
-#define WRMSR(msrNum, low, high) do { \
-   __asm__ __volatile__(              \
-      "wrmsr"                         \
-      : /* no outputs */              \
-      : "c" (msrNum),                 \
-        "a" (low),                    \
-        "d" (high)                    \
-    );                                \
-} while (0)
-
-static INLINE uint64 RDPMC(int cx)
-{
 #ifdef VM_X86_64
   uint64 pmcLow;
   uint64 pmcHigh;
@@ -867,7 +699,7 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=a" (pmcLow), "=d" (pmcHigh)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmcHigh << 32 | pmcLow;
@@ -877,43 +709,26 @@ static INLINE uint64 RDPMC(int cx)
   __asm__ __volatile__(
      "rdpmc"
      : "=A" (pmc)
-      : "c" (cx)
+      : "c" (counter)
   );

   return pmc;
 #endif
-}
 #elif defined _MSC_VER
-#ifndef VM_X86_64 // XXX Switch to intrinsics with the new 32 and 64-bit compilers.
-
-static INLINE uint64 RDPMC(int counter)
-{
-   __asm mov ecx, counter
-   __asm rdpmc
+   return __readpmc(counter);
+#endif
 }

-static INLINE void WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   __asm mov ecx, msrNum
-   __asm mov eax, lo
-   __asm mov edx, hi
-   __asm wrmsr
-}
-#endif
-#endif

-
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
-static INLINE uint64 __XGETBV(int cx)
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined (ULM)
+static INLINE uint64
+__XGETBV(int cx)
 {
+#ifdef __GNUC__
 #ifdef VM_X86_64
   uint64 lowval, hival;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=a" (lowval), "=d" (hival)
      : "c" (cx)
   );
@@ -921,39 +736,41 @@ static INLINE uint64 __XGETBV(int cx)
 #else
   uint64 val;
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd0"
-#else
      "xgetbv"
-#endif
      : "=A" (val)
      : "c" (cx)
   );
   return val;
 #endif
+#elif defined _MSC_VER
+   return _xgetbv((unsigned)cx);
+#endif
 }

-static INLINE void __XSETBV(int cx, uint64 val)
+static INLINE void
+__XSETBV(int cx, uint64 val)
 {
+#ifdef __GNUC__
   __asm__ __volatile__(
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-      ".byte 0x0f, 0x01, 0xd1"
-#else
      "xsetbv"
-#endif
      : /* no outputs */
      : "a" ((uint32)val), "d" ((uint32)(val >> 32)), "c" (cx)
    );
+#elif defined _MSC_VER
+   _xsetbv((unsigned)cx, val);
+#endif
 }

-static INLINE uint64 GET_XCR0(void)
+static INLINE uint64
+GET_XCR0(void)
 {
   return __XGETBV(0);
 }

 #define SET_XCR0(val) __XSETBV(0, val)

-static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
+static INLINE void
+SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 {
 #ifndef VMM_BOOTSTRAP
   ASSERT(oldVal == GET_XCR0());
@@ -965,33 +782,45 @@ static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 #endif


-#define START_TRACING() { \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags |= EFLAGS_TF;    \
-   RESTORE_FLAGS(flags);  \
-}
-
-#define STOP_TRACING() {  \
-   uintptr_t flags;       \
-   SAVE_FLAGS(flags);     \
-   flags &= ~EFLAGS_TF;   \
-   RESTORE_FLAGS(flags);  \
-}
-
-
-static INLINE Bool
-INTERRUPTS_ENABLED(void)
+static INLINE uint32
+RDTSCP_AuxOnly(void)
 {
-   uintptr_t flags;
-   SAVE_FLAGS(flags);
-   return ((flags & EFLAGS_IF) != 0);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return tscAux;
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   __rdtscp(&tscAux);
+
+   return tscAux;
+#endif
 }

-static INLINE void
-SET_KERNEL_PER_CORE(uint64 val)
+
+static INLINE uint64
+RDTSCP(void)
 {
-   __SET_MSR(MSR_GSBASE, val);
+#ifdef __GNUC__
+   uint32 tscLow, tscHigh, tscAux;
+
+   __asm__ __volatile__(
+      "rdtscp"
+      :"=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
+   );
+
+   return QWORD(tscHigh, tscLow);
+#elif defined _MSC_VER
+   uint32 tscAux;
+
+   return __rdtscp(&tscAux);
+#endif
 }

 #endif
@@ -1,137 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * vm_asm_x86_64.h
- *
- *	x86-64 asm macros
- */
-
-#ifndef _VM_ASM_X86_64_H_
-#define _VM_ASM_X86_64_H_
-
-
-#define INCLUDE_ALLOW_VMX
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMK_MODULE
-#define INCLUDE_ALLOW_VMKERNEL
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_DISTRIBUTE
-#include "includeCheck.h"
-
-#ifdef __GNUC__
-#include "vm_asm_x86.h"
-#endif
-
-/*
- * This file contains inline assembly routines used by x86_64 code.
- */
-
-#ifdef __GNUC__
-
-  /* nop; prevents #error for no compiler definition from firing */
-
-#elif defined _MSC_VER  /* !__GNUC__ */
-
-/*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
- * We define the intrinsic using the basic types corresponding to the
- * Windows typedefs. This avoids having to include windows header files
- * to get to the windows types.
- */
-
-#ifdef _WIN64
-#ifdef __cplusplus
-extern "C" {
-#endif
-unsigned __int64  __readmsr(unsigned long);
-void              __writemsr(unsigned long, unsigned __int64);
-#pragma intrinsic(__readmsr, __writemsr)
-#ifdef __cplusplus
-}
-#endif
-
-
-static INLINE uint64
-RDPMC(int counter)
-{
-   return __readpmc(counter);
-}
-
-
-static INLINE void
-WRMSR(uint32 msrNum, uint32 lo, uint32 hi)
-{
-   uint64 value = QWORD(hi, lo);
-   __writemsr(msrNum, value);
-}
-
-
-static INLINE uint64
-__GET_MSR(int input)
-{
-   return __readmsr((unsigned long)input);
-}
-
-
-static INLINE void
-__SET_MSR(int cx, uint64 val)
-{
-  __writemsr((unsigned long)cx, (unsigned __int64)val);
-}
-
-#endif
-
-#else
-#error No compiler defined for get/set
-#endif /* !__GNUC__ && !_MSC_VER */
-
-
-#ifdef __GNUC__
-static INLINE void
-SWAPGS(void)
-{
-   __asm__ __volatile__("swapgs");
-}
-
-
-static INLINE uint64
-RDTSCP_AuxOnly(void)
-{
-   uint64 tscLow, tscHigh, tscAux;
-
-   __asm__ __volatile__(
-      "rdtscp"
-      : "=a" (tscLow), "=d" (tscHigh), "=c" (tscAux)
-   );
-
-   return tscAux;
-}
-#endif
-
-#endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -67,6 +67,7 @@ extern "C" {
 #if defined (VMKPANIC) 
 #include "vmk_assert.h"
 #else /* !VMKPANIC */
+#include <linux/kernel.h>
 #define _ASSERT_PANIC(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
 #define _ASSERT_PANIC_BUG(bug, name) \
@@ -107,7 +108,7 @@ NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
 } while(0)

 #else
-NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+#define Panic panic
 #endif

 void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
@@ -202,7 +203,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,38 +49,68 @@ extern "C" {
 #endif

 /*
- * In the Atomic_* definitions below, memory ordering and atomicity are somewhat
- * conflated in an inconsistent manner. First, we have Atomic_{Read,Write},
- * which only guarantees single copy atomicity, i.e. that the read/write occurs
- * in an atomic fashion, but have no implication on memory ordering. The second
- * class of Atomics are all the non-unfenced operations excluding
- * Atomic_{Read,Write}*, which both imply atomicity and act as a memory barrier,
- * implying sequentially consistent ordering of the atomic operation with all
- * loads/stores prior to and after it.
+ * There are two concepts involved when dealing with atomic accesses:
+ * 1. Atomicity of the access itself
+ * 2. Ordering of the access with respect to other reads&writes (from the view
+ *    of other processors/devices).
 *
- * Since on x86, the second class of operations are associated with LOCK
- * semantics, assumptions have been made about the ordering these operations
- * imply on surrounding code (see for example the vmkernel's RefCount
- * implementation). As a result, on arm64 we have to provide these same
- * guarantees. We do this by making use of DMB barriers both before and after
- * the atomic ldrx/strx sequences. A barrier before and after is required to
- * avoid having part of the atomic operation reordered with surrounding code,
- * e.g. a store-load reordering of the strx with a following load outside the
- * Atomic_ op. For the first class of operations, Atomic_{Read,Write}, we do not
- * implement a barrier.
+ * Two examples help to clarify #2:
+ * a. Inc: A caller implementing a simple independent global event counter
+ *         might not care if the compiler or processor visibly reorders the
+ *         increment around other memory accesses.
+ * b. Dec: A caller implementing a reference count absolutely *doesn't* want
+ *         the compiler or processor to visibly reordering writes after that
+ *         decrement: if that happened, the program could then end up writing
+ *         to memory that was freed by another processor.
 *
- * This implementation of Atomic operations is suboptimal on arm64, since
- * both atomicity and memory ordering are fused together. Ideally the Atomic
- * operations would only imply atomicity, and an explicit memory barrier in the
- * surrounding code used to enforce ordering where necessary. This would eschew
- * the need for the DMBs. A middle ground can be implemented where we use the
- * arm64 load-acquire/store-release exclusive instructions to implement Atomics.
- * This would imply sequential consistency of the Atomic operations (but not
- * with any of the surrounding non-atomic operations) without the need for a
- * DMB. Using these without a DMB today can still result in problematic
- * reordering by the processor with surrounding non-atomic operations, e.g. a
- * store-load reordering with a stlxr. Future optimization for arm64 should
- * consider the wider change required at the call sites to minimize DMBs.
+ * C11 has standardized a good model for expressing these orderings when doing
+ * atomics. It defines three *tiers* of ordering:
+ * 1. Sequential Consistency (every processor sees the same total order of
+ *    events)
+ *
+ * 2. Acquire/Release ordering (roughly, everybody can agree previous events
+ *    have completed, but they might disagree on the ordering of previous
+ *    independent events).
+ *
+ *    The relative ordering provided by this tier is sufficient for common
+ *    locking and initialization activities, but is insufficient for unusual
+ *    synchronization schemes (e.g. IRIW aka Independent Read Independent
+ *    Write designs such Dekker's algorithm, Peterson's algorithm, etc.)
+ *
+ *    In other words, this tier is close in behavior to Sequential Consistency
+ *    in much the same way a General-Relativity universe is close to a
+ *    Newtonian universe.
+ * 3. Relaxed (i.e unordered/unfenced)
+ *
+ * In C11 standard's terminology for atomic memory ordering,
+ * - in case (a) we want "relaxed" ordering for perf and,
+ * - in case (b) we want "sequentially consistent" ordering (or perhaps the
+ *   only slightly weaker "release" ordering) for correctness.
+ *
+ * There are standardized mappings of operations to orderings for every
+ * processor architecture. See
+ * - https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
+ * - http://preshing.com/20120913/acquire-and-release-semantics/
+ *
+ * In this file:
+ * 1. all RMW (Read/Modify/Write) operations are sequentially consistent.
+ *    This includes operations like Atomic_IncN, Atomic_ReadIfEqualWriteN,
+ *    Atomic_ReadWriteN, etc.
+ * 2. all R and W operations are relaxed. This includes operations like
+ *    Atomic_WriteN, Atomic_ReadN, Atomic_TestBitN, etc.
+ *
+ * The below routines of course ensure both the CPU and compiler honor the
+ * ordering constraint.
+ *
+ * Notes:
+ * 1. Since R-only and W-only operations do not provide ordering, callers
+ *    using them for synchronizing operations like double-checked
+ *    initialization or releasing spinlocks must provide extra barriers.
+ * 2. This implementation of Atomic operations is suboptimal. On x86,simple
+ *    reads and writes have acquire/release semantics at the hardware level.
+ *    On arm64, we have separate instructions for sequentially consistent
+ *    reads and writes (the same instructions are used for acquire/release).
+ *    Neither of these are exposed for R-only or W-only callers.
 *
 * For further details on x86 and ARM memory ordering see
 * https://wiki.eng.vmware.com/ARM/MemoryOrdering.
@@ -118,10 +148,7 @@ typedef struct Atomic_uint128 {
 /*
 * Prototypes for msft atomics.  These are defined & inlined by the
 * compiler so no function definition is needed.  The prototypes are
- * needed for C++.  Since amd64 compiler doesn't support inline asm we
- * have to use these.  Unfortunately, we still have to use some inline asm
- * for the 32 bit code since the and/or/xor implementations didn't show up
- * until XP or 2k3.
+ * needed for C++.
 *
 * The declarations for the intrinsic functions were taken from ntddk.h
 * in the DDK. The declarations must match otherwise the 64-bit C++
@@ -130,7 +157,7 @@ typedef struct Atomic_uint128 {
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
 */
-#if defined _MSC_VER && _MSC_VER >= 1310 && !defined BORA_NO_WIN32_INTRINS
+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -222,38 +249,10 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:


 /*
- * All the assembly code is tricky and written conservatively.
- * For example, to make sure gcc won't introduce copies,
- * we force the addressing mode like this:
- *
- *    "xchgl %0, (%1)"
- *    : "=r" (val)
- *    : "r" (&var->value),
- *      "0" (val)
- *    : "memory"
- *
- * - edward
- *
- * Actually - turns out that gcc never generates memory aliases (it
- * still does generate register aliases though), so we can be a bit
- * more agressive with the memory constraints. The code above can be
- * modified like this:
- *
- *    "xchgl %0, %1"
- *    : "=r" (val),
- *      "=m" (var->value),
- *    : "0" (val),
- *      "1" (var->value)
- *
- * The advantages are that gcc can use whatever addressing mode it
- * likes to access the memory value, and that we dont have to use a
- * way-too-generic "memory" clobber as there is now an explicit
- * declaration that var->value is modified.
- *
- * see also /usr/include/asm/atomic.h to convince yourself this is a
- * valid optimization.
- *
- * - walken
+ * The Read/Modify/Write operations on x86/x64 are all written using the
+ * "memory" constraint. This is to ensure the compiler treats the operation as
+ * a full barrier, flushing any pending/cached state currently residing in
+ * registers.
 */

 #if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
@@ -403,6 +402,7 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "=q" (val),
        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER && _MSC_VER >= 1600
@@ -496,7 +496,7 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
        "+m" (var->value)
      : "q" (newVal),
        "0" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );

   return val;
@@ -1002,26 +1002,15 @@ Atomic_ReadWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchange((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint32.value, eax
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif // _MSC_VER >= 1310
-#else
 #error No compiler defined for Atomic_ReadWrite
 #endif // __GNUC__
 }
@@ -1162,30 +1151,17 @@ Atomic_ReadIfEqualWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgl %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedCompareExchange((long *)&var->value,
-				      (long)newVal,
-				      (long)oldVal);
-#else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, oldVal
-      __asm mov ebx, var
-      __asm mov ecx, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint32.value, ecx
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif
+                                      (long)newVal,
+                                      (long)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite
 #endif
@@ -1267,17 +1243,17 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgq %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif //VM_ARM_V7
 #elif defined _MSC_VER
   return _InterlockedCompareExchange64((__int64 *)&var->value,
-					(__int64)newVal,
-					(__int64)oldVal);
+                                        (__int64)newVal,
+                                        (__int64)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite64
 #endif
@@ -1332,7 +1308,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
      "lock; andl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1341,7 +1317,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
+   __asm lock And [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_And
@@ -1397,7 +1373,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
      "lock; orl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1406,7 +1382,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
+   __asm lock Or [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Or
@@ -1462,7 +1438,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
      "lock; xorl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1471,7 +1447,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Xor
@@ -1510,7 +1486,7 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT
      "lock; xorq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -1569,17 +1545,11 @@ Atomic_Add32(Atomic_uint32 *var, // IN/OUT
      "lock; addl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedExchangeAdd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock add [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Add
 #endif
@@ -1634,17 +1604,11 @@ Atomic_Sub32(Atomic_uint32 *var, // IN/OUT
      "lock; subl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
-   _InterlockedExchangeAdd((long *)&var->value, (long)-val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock sub [ebx]Atomic_uint32.value, eax
-#endif
+   _InterlockedExchangeAdd((long *)&var->value, -(long)val);
 #else
 #error No compiler defined for Atomic_Sub
 #endif
@@ -1680,16 +1644,11 @@ Atomic_Inc32(Atomic_uint32 *var) // IN/OUT
      "lock; incl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedIncrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock inc [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Inc
 #endif
@@ -1725,16 +1684,11 @@ Atomic_Dec32(Atomic_uint32 *var) // IN/OUT
      "lock; decl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedDecrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock dec [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Dec
 #endif
@@ -1939,26 +1893,15 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchangeAdd((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm lock xadd [ebx]Atomic_uint32.value, eax
-   }
-#pragma warning(pop)
-#endif
-#else
 #error No compiler defined for Atomic_ReadAdd32
 #endif
 }
@@ -2044,11 +1987,11 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchgq %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
 #else /* 32-bit version for non-ARM */
   typedef struct {
@@ -2088,9 +2031,9 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b (%3)" "\n\t"
      "xchgl %%ebx, %6"      "\n\t"
      "sete %0"
-      :	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+      : "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : /*
         * See the "Rules for __asm__ statements in __PIC__ code" above: %3
         * must use a register class which does not contain %ebx.
@@ -2111,14 +2054,14 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+        "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : "2" (((S_uint64 *)&oldVal)->lowValue),
        "3" (((S_uint64 *)&oldVal)->highValue),
        "b" (((S_uint64 *)&newVal)->lowValue),
        "c" (((S_uint64 *)&newVal)->highValue)
-      : "cc"
+      : "cc", "memory"
   );
 #   endif
 #endif
@@ -2166,11 +2109,11 @@ Atomic_CMPXCHG32(Atomic_uint32 *var,   // IN/OUT
      "lock; cmpxchgl %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
   return equal;
 #endif /* VM_X86_ANY */
@@ -2247,7 +2190,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
 #   pragma warning(push)
-#   pragma warning(disable : 4035)		// disable no-return warning
+#   pragma warning(disable : 4035)      // disable no-return warning
   {
      __asm mov ecx, var
      __asm mov edx, ecx
@@ -2330,9 +2273,9 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2360,7 +2303,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2378,7 +2324,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -2425,7 +2371,7 @@ Atomic_ReadInc64(Atomic_uint64 *var) // IN/OUT
 static INLINE uint64
 Atomic_ReadDec64(Atomic_uint64 *var) // IN/OUT
 {
-   return Atomic_ReadAdd64(var, CONST64U(-1));
+   return Atomic_ReadAdd64(var, (uint64)CONST64(-1));
 }


@@ -2460,7 +2406,7 @@ Atomic_Add64(Atomic_uint64 *var, // IN/OUT
      "lock; addq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2502,7 +2448,7 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT
      "lock; subq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2542,7 +2488,7 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT
      "lock; incq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedIncrement64((__int64 *)&var->value);
@@ -2581,7 +2527,7 @@ Atomic_Dec64(Atomic_uint64 *var) // IN/OUT
      "lock; decq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedDecrement64((__int64 *)&var->value);
@@ -2617,8 +2563,9 @@ Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2726,7 +2673,7 @@ Atomic_Or64(Atomic_uint64 *var, // IN/OUT
      "lock; orq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedOr64((__int64 *)&var->value, (__int64)val);
@@ -2773,7 +2720,7 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT
      "lock; andq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedAnd64((__int64 *)&var->value, (__int64)val);
@@ -2811,29 +2758,17 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT
-                uint64 bit)         // IN
+                unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; bts %1, %0"
+      "lock; btsq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal | (CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_SetBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2863,29 +2798,17 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT
-                  uint64 bit)         // IN
+                  unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; btr %1, %0"
+      "lock; btrq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal & ~(CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_ClearBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2915,29 +2838,65 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE Bool
 Atomic_TestBit64(Atomic_uint64 *var, // IN
-                 uint64 bit)         // IN
+                 unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
-   Bool out = FALSE;
+   Bool out;
   ASSERT(bit <= 63);
+#if defined __x86_64__ && defined __GNUC__
   __asm__ __volatile__(
-      "bt %2, %1; setc %0"
+      "btq %2, %1; setc %0"
      : "=rm"(out)
      : "m" (var->value),
-        "rJ" (bit)
+        "rJ" ((uint64)bit)
      : "cc"
   );
-   return out;
-#elif defined _MSC_VER
-   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
 #else
-#error No compiler defined for Atomic_TestBit64
+   out = (var->value & (CONST64U(1) << bit)) != 0;
 #endif
-#else // __x86_64__
+   return out;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBit64 --
+ *
+ *      Atomically test and set the bit 'bit' in var.
+ *      Bit must be between 0 and 63.
+ *
+ * Results:
+ *      TRUE if the tested bit was set; else FALSE.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBit64(Atomic_uint64 *var, // IN/OUT
+                    unsigned bit)       // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool out;
   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
+   __asm__ __volatile__(
+      "lock; btsq %2, %1; setc %0"
+      : "=rm" (out), "+m" (var->value)
+      : "rJ" ((uint64)bit)
+      : "cc", "memory"
+   );
+   return out;
+#else
+   uint64 oldVal;
+   uint64 mask;
+   ASSERT(bit <= 63);
+   mask = CONST64U(1) << bit;
+   do {
+      oldVal = var->value;
+   } while (!Atomic_CMPXCHG64(var, oldVal, oldVal | mask));
+   return (oldVal & mask) != 0;
 #endif
 }

@@ -3020,8 +2979,9 @@ Atomic_ReadWrite16(Atomic_uint16 *var,  // IN/OUT:
   __asm__ __volatile__(
      "xchgw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3128,10 +3088,10 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var,   // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgw %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3192,7 +3152,7 @@ Atomic_And16(Atomic_uint16 *var, // IN/OUT
      "lock; andw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3249,7 +3209,7 @@ Atomic_Or16(Atomic_uint16 *var, // IN/OUT
      "lock; orw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3306,7 +3266,7 @@ Atomic_Xor16(Atomic_uint16 *var, // IN/OUT
      "lock; xorw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3363,7 +3323,7 @@ Atomic_Add16(Atomic_uint16 *var, // IN/OUT
      "lock; addw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3420,7 +3380,7 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT
      "lock; subw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3476,7 +3436,7 @@ Atomic_Inc16(Atomic_uint16 *var) // IN/OUT
      "lock; incw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Add16(var, 1);
@@ -3512,7 +3472,7 @@ Atomic_Dec16(Atomic_uint16 *var) // IN/OUT
      "lock; decw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Sub16(var, 1);
@@ -3582,9 +3542,9 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
   __asm__ __volatile__(
      "lock; xaddw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3661,7 +3621,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -3715,7 +3675,7 @@ Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
                            in val)                                           \
   {                                                                          \
      return (out)(cast)Atomic_ReadWrite ## size(var,                         \
-		(uint ## size)(cast)val);                                     \
+                (uint ## size)(cast)val);                                     \
   }                                                                          \
                                                                              \
                                                                              \
@@ -3876,6 +3836,109 @@ MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t)
 MAKE_ATOMIC_TYPE(Int, 32, int, int, int)
 MAKE_ATOMIC_TYPE(Bool, 8, Bool, Bool, Bool)

+/*
+ * Define arbitrary sized bit vector to be used by
+ * Atomic_TestSetBitVector and Atomic_TestClearBitVector.
+ */
+#define ATOMIC_BITVECTOR(varName, capacity) \
+      Atomic_uint8 varName[CEILING(capacity, 8)]
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBitVector --
+ *
+ *      Atomically test and set the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBitVector(Atomic_uint8 *var, // IN/OUT
+                        unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; bts %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadOr8(var + index / 8, bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestClearBitVector --
+ *
+ *      Atomically test and clear the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestClearBitVector(Atomic_uint8 *var, // IN/OUT
+                          unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; btr %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadAnd8(var + index / 8, ~bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestBitVector --
+ *
+ *      Test the bit 'index' (zero-based) in bit vector var.
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestBitVector(const Atomic_uint8 *var, // IN
+                     unsigned index)          // IN
+{
+   uint8 bit = 1 << index % 8;
+   return (Atomic_Read8(var + index / 8) & bit) != 0;
+}
+
+
 #ifdef VM_ARM_64
 #   include "vm_atomic_arm64_end.h"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -221,8 +221,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +263,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +376,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +392,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -768,12 +771,14 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
@@ -888,7 +893,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +907,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1056,7 +989,7 @@ ClearBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1078,7 +1011,7 @@ ComplementBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1170,7 +1103,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1161,101 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,12 +49,29 @@ uint64 _umul128(uint64 multiplier, uint64 multiplicand,
 int64 _mul128(int64 multiplier, int64 multiplicand,
              int64 *highProduct);
 uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
+#ifdef ULM
+void _fxsave64(void *save);
+void _fxsave(void *save);
+void _fxrstor64(const void *load);
+void _fxrstor(const void *load);
+void _xsave64(void *save, uint64 mask);
+void _xsave(void *save, uint64 mask);
+void _xsaveopt64(void *save, uint64 mask);
+void _xsavec(void *save, uint64 mask);
+void _xrstor64(const void *load, uint64 mask);
+void _xrstor(const void *load, uint64 mask);
+#endif /* ULM */
 #ifdef __cplusplus
 }
 #endif

 #pragma intrinsic(_umul128, _mul128, __shiftright128)

+#ifdef ULM
+#pragma intrinsic(_fxsave64, _fxsave, _fxrstor64, _fxrstor, _xsave64, _xsave, \
+                  _xsaveopt64, _xsavec, _xrstor64, _xrstor)
+#endif /* ULM */
+
 #endif // _MSC_VER

 #if defined(__GNUC__)
@@ -113,40 +130,57 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
-#if defined(__GNUC__)

-static INLINE void 
+#if defined(VMM) || defined(VMKERNEL) || defined(FROBOS) || defined(ULM)
+static INLINE void
 FXSAVE_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave64(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXSAVE_COMPAT_ES1(void *save)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
+#elif defined(_MSC_VER)
+   _fxsave(save);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor64(load);
+#endif
 }

-static INLINE void 
+static INLINE void
 FXRSTOR_COMPAT_ES1(const void *load)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
+#elif defined(_MSC_VER)
+   _fxrstor(load);
+#endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 FXRSTOR_AMD_ES0(const void *load)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -170,100 +204,100 @@ FXRSTOR_AMD_ES0(const void *load)
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
-#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

-static INLINE void 
+static INLINE void
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave64(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVE_COMPAT_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsave(save, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
+#ifdef __GNUC__
   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveoptq %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xsaveopt64(save, mask);
+#endif
 }

-static INLINE void 
+static INLINE void
+XSAVEC_COMPAT_ES1(void *save, uint64 mask)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__ (
+        "xsavec %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        : "memory");
+#elif defined(_MSC_VER)
+   _xsavec(save, mask);
+#endif
+}
+
+static INLINE void
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor64(load, mask);
 #endif
 }

-static INLINE void 
+static INLINE void
 XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
+#ifdef __GNUC__
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
+#elif defined(_MSC_VER)
+   _xrstor(load, mask);
 #endif
 }

-static INLINE void 
+#if defined(__GNUC__)
+static INLINE void
 XRSTOR_AMD_ES0(const void *load, uint64 mask)
 {
   uint64 dummy = 0;

-   __asm__ __volatile__ 
+   __asm__ __volatile__
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
@@ -273,36 +307,39 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }

 #endif /* __GNUC__ */
+#endif /* VMM || VMKERNEL || FROBOS || ULM */

 /*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc"); 
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -314,7 +351,7 @@ xtest(void)
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -396,7 +433,7 @@ Mul64x6464(uint64 multiplicand,
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -481,7 +518,7 @@ Muls64x64s64(int64 multiplicand,
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -505,7 +542,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
- * 
+ *
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -244,9 +244,131 @@ GetCallerEFlags(void)

 /* Sequence recommended by Intel for the Pentium 4. */
 #define INTEL_MICROCODE_VERSION() (             \
-   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
+   X86MSR_SetMSR(MSR_BIOS_SIGN_ID, 0),          \
   __GET_EAX_FROM_CPUID(1),                     \
-   __GET_MSR(MSR_BIOS_SIGN_ID))
+   X86MSR_GetMSR(MSR_BIOS_SIGN_ID))
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MFENCE --
+ *
+ *      Wrapper around the MFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See MFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+MFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "mfence"
+      ::: "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_mfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for MFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LFENCE --
+ *
+ *      Wrapper around the LFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See LFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "lfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+   _mm_lfence();
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for LFENCE
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * SFENCE --
+ *
+ *      Wrapper around the SFENCE instruction.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      See SFENCE instruction in Intel SDM or AMD Programmer's Manual.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+SFENCE(void)
+{
+#ifdef __GNUC__
+   __asm__ __volatile__(
+      "sfence"
+      : : : "memory"
+   );
+#elif defined _MSC_VER
+   _ReadWriteBarrier();
+#if defined VM_X86_32
+   __asm sfence;
+#else
+   _mm_sfence();
+#endif
+   _ReadWriteBarrier();
+#else
+#error No compiler defined for SFENCE
+#endif
+}
+

 /*
 *-----------------------------------------------------------------------------
@@ -278,24 +400,63 @@ GetCallerEFlags(void)
 static INLINE void
 RDTSC_BARRIER(void)
 {
-#ifdef __GNUC__
-   __asm__ __volatile__(
-      "mfence \n\t"
-      "lfence \n\t"
-      ::: "memory"
-   );
-#elif defined _MSC_VER
-   /* Prevent compiler from moving code across mfence/lfence. */
-   _ReadWriteBarrier();
-   _mm_mfence();
-   _mm_lfence();
-   _ReadWriteBarrier();
-#else
-#error No compiler defined for RDTSC_BARRIER
-#endif
+   MFENCE();
+   LFENCE();
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * LOCKED_INSN_BARRIER --
+ *
+ *      Implements a full WB load/store barrier using a locked instruction.
+ *
+ *      See PR 1674199 for details. You may choose to use this for
+ *      performance reasons over MFENCE iff you are only dealing with
+ *      WB memory accesses.
+ *
+ *      DANGER! Do not use this barrier instead of MFENCE when dealing
+ *      with non-temporal instructions or UC/WC memory accesses.
+ *
+ *      Caveat Emptor! This function is _NOT_ _PORTABLE_ and most certainly
+ *      not something you should use. Take a look at the SMP_*_BARRIER_*,
+ *      DMA_*_BARRIER_* and MMIO_*_BARRIER_* interfaces instead, when writing
+ *      general OS/VMM code.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      Cause WB loads and stores before the call to be globally visible
+ *      before WB loads and stores after this call.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE void
+LOCKED_INSN_BARRIER(void)
+{
+   volatile long temp = 0;
+
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc", "memory");
+#elif defined _MSC_VER
+   /*
+    * Ignore warning about _InterlockedXor operation on a local variable; we are
+    * using the operation for its side-effects only.
+    */
+   #pragma warning(suppress:28113)
+   _InterlockedXor(&temp, 1);
+#else
+#error LOCKED_INSN_BARRIER not defined for this compiler
+#endif
+}
+
 /*
 * Memory Barriers
 * ===============
@@ -354,7 +515,6 @@ RDTSC_BARRIER(void)
 #   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
 #endif

-
 /*
 * Memory barriers. These take the form of
 *
@@ -373,49 +533,73 @@ RDTSC_BARRIER(void)
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
- * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed. The
- * ST_LD barrier is implemented with a locked xor operation (instead of the
- * mfence instruction) for performance reasons. See PR 1674199 for more
- * details.
+ * This is a simplified version of Table 7-3 (Memory Access Ordering Rules) from
+ * AMD AMD64 Architecture Programmer's Manual Volume 2: System Programming
+ * (September 2018, Publication 24593, Revision 3.30).
 *
- * On x64, special instructions are only provided for load-load (lfence) and
- * store-store (sfence) ordering, and they don't apply to normal memory.
+ * https://www.amd.com/system/files/TechDocs/24593.pdf#page=228
+ *
+ * This table only includes the memory types we care about in the context of
+ * SMP, DMA and MMIO barriers.
+ *
+ * +-------------+------+------+------+------+------+------+
+ * |\ 2nd mem op |      |      |      |      |      |      |
+ * | `---------. | R WB | R UC | R WC | W WB | W UC | W WC |
+ * | 1st mem op \|      |      |      |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WB     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R UC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    R WC     |      |      | LF1  |      |      |      |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WB     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W UC     | MF1  |      | MF1  |      |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ * |    W WC     | MF1  |      | MF1  | SF1  |      | SF2  |
+ * +-------------+------+------+------+------+------+------+
+ *
+ * MF1 - WB or WC load may pass a previous non-conflicting WB, WC or UC store.
+ *       Use MFENCE. This is a combination of rules 'e' and 'i' in the AMD
+ *       diagram.
+ * LF1 - WC load may pass a previous WB, WC or UC load. Use LFENCE. This is
+ *       rule 'b' in the AMD diagram.
+ * SF1 - WB store may pass a previous WC store. Use SFENCE. This is rule 'j' in
+ *       the AMD diagram.
+ * SF2 - WC store may pass a previous UC, WB or non-conflicting WC store. Use
+ *       SFENCE. This is rule 'h' in the AMD diagram.
+ *
+ * To figure out the specific barrier required, pick and collapse the relevant
+ * rows and columns, choosing the strongest barrier.
+ *
+ * SMP barriers only concern with access to "normal memory" (write-back cached
+ * i.e. WB using above terminology), so we only need to worry about store-load
+ * reordering. In other cases a compiler barrier is sufficient. SMP store-load
+ * reordering is handled with a locked XOR (instead of a proper MFENCE
+ * instructon) for performance reasons. See PR 1674199 for more details.
+ *
+ * DMA barriers are equivalent to SMP barriers on x86.
+ *
+ * MMIO barriers are used to mix access to different memory types, so more
+ * reordering is possible, and is handled via LFENCE/SFENCE. Also, a proper
+ * MFENCE must be used instead of the locked XOR trick, due to the latter
+ * not guarding non-temporal/WC accesses.
 */

-
-static INLINE void
-SMP_W_BARRIER_R(void)
-{
-   volatile long temp;
-
-   COMPILER_MEM_BARRIER();
-#if defined __GNUC__
-   __asm__ __volatile__ (
-      "lock xorl $1, %0\n"
-      : "+m" (temp)
-      : /* no additional inputs */
-      : "cc");
-#elif defined _MSC_VER
-   _InterlockedXor(&temp, 1);
-#else
-#error SMP_W_BARRIER_R not defined for this compiler
-#endif
-   COMPILER_MEM_BARRIER();
-}
-
 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
 #define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
 #define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_R()     LOCKED_INSN_BARRIER()
 #define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
-#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
-#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_W_BARRIER_RW()    LOCKED_INSN_BARRIER()
+#define SMP_RW_BARRIER_R()    LOCKED_INSN_BARRIER()
 #define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
-#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_RW()   LOCKED_INSN_BARRIER()

 /*
 * Like the above, only for use with observers other than CPUs,
- * i.e. DMA masters.
+ * i.e. DMA masters. Same as SMP barriers for x86.
 */

 #define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
@@ -429,18 +613,178 @@ SMP_W_BARRIER_R(void)
 #define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

 /*
- * And finally a set for use with MMIO accesses.
+ * And finally a set for use with MMIO accesses. These barriers must be stronger
+ * because they are used when mixing accesses to different memory types.
 */

-#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_R()    LFENCE()
 #define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
-#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
-#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
-#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
-#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
-#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
-#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
-#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+#define MMIO_R_BARRIER_RW()   LFENCE()
+#define MMIO_W_BARRIER_R()    MFENCE()
+#define MMIO_W_BARRIER_W()    SFENCE()
+#define MMIO_W_BARRIER_RW()   MFENCE()
+#define MMIO_RW_BARRIER_R()   MFENCE()
+#define MMIO_RW_BARRIER_W()   SFENCE()
+#define MMIO_RW_BARRIER_RW()  MFENCE()


+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead8 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      8-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint8
+MMIORead8(const volatile void *addr)
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   return *addr8;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite8 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite8(volatile void *addr, // IN
+           uint8 val)           // IN
+{
+   volatile uint8 *addr8 = (volatile uint8 *) addr;
+
+   *addr8 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead16 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      16-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint16
+MMIORead16(const volatile void *addr)
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   return *addr16;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite16 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite16(volatile void *addr,  // IN
+            uint16 val)           // IN
+{
+   volatile uint16 *addr16 = (volatile uint16 *) addr;
+
+   *addr16 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead32 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      32-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint32
+MMIORead32(const volatile void *addr)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   return *addr32;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite32 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite32(volatile void *addr, // OUT
+            uint32 val)
+{
+   volatile uint32 *addr32 = (volatile uint32 *) addr;
+
+   *addr32 = val;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIORead64 --
+ *
+ *      IO read from address "addr".
+ *
+ * Results:
+ *      64-bit value at given location.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE uint64
+MMIORead64(const volatile void *addr)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   return *addr64;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MMIOWrite64 --
+ *
+ *      IO write to address "addr".
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE void
+MMIOWrite64(volatile void *addr, // OUT
+            uint64 val)
+{
+   volatile uint64 *addr64 = (volatile uint64 *) addr;
+
+   *addr64 = val;
+}
+
 #endif // _VM_BASIC_ASM_X86_COMMON_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,15 +37,51 @@
 #include "vm_basic_types.h" // For INLINE.

 /* Checks for FreeBSD, filtering out VMKERNEL. */
-#define __IS_FREEBSD__ (!defined(VMKERNEL) && defined(__FreeBSD__))
+#if !defined(VMKERNEL) && defined(__FreeBSD__)
+#define __IS_FREEBSD__ 1
+#else
+#define __IS_FREEBSD__ 0
+#endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

+/*
+ * <stddef.h> provides definitions for:
+ *   NULL, offsetof
+ * References:
+ *   C90 7.17, C99 7.19, C11 7.19
+ */
+#if !defined(VMKERNEL)
+#  include <stddef.h>
+#else
+   /*
+    * Vmkernel's bogus __FreeBSD__ value causes gcc <stddef.h> to break.
+    * Work around by doing similar things. Bug 2116887 and 2229647.
+    */
+#  ifndef offsetof
+      /*
+       * We use the builtin offset for gcc/clang, except when we're running
+       * under the vmkernel's GDB macro preprocessor, since gdb doesn't
+       * understand __builtin_offsetof.
+       */
+#     if defined VMKERNEL_GDB_MACRO_BUILDER
+#        define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#     else
+#        define offsetof __builtin_offsetof
+#     endif
+#  endif
+
+#  ifndef NULL
+#     ifdef  __cplusplus
+#        define NULL    0
+#     else
+#        define NULL    ((void *)0)
+#     endif
+#  endif
+
+#endif  // VMKERNEL
+
 #if defined _WIN32 && defined USERLEVEL
-   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make 
-                         * sure that it's already defined before we do that.
-                         */
-   #include <windows.h>	// for Sleep() and LOWORD() etc.
+   #include <windows.h> // for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
 #endif

@@ -54,42 +90,8 @@
 * Simple macros
 */

-#ifndef vmw_offsetof
-#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
-
-#if (defined __APPLE__ || defined __FreeBSD__) && \
-    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
-#   include <stddef.h>
-#else
-#ifndef offsetof
-#define VMW_DEFINED_OFFSETOF
-
-/*
- * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
- * in terms of behavior, they need to be separate to match verbatim the
- * definition used by the respective compilers, to avoid a redefinition warning.
- *
- * This is necessary until we eliminate the inclusion of <windows.h> above.
- */
-#ifdef _WIN32
-#define offsetof(s,m)   (size_t)&(((s *)0)->m)
-/*
- * We use the builtin offset for gcc/clang, except when we're running under the
- * vmkernel's GDB macro preprocessor, since gdb doesn't understand
- * __builtin_offsetof.
- */
-#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
-#define offsetof __builtin_offsetof
-#else
-#define offsetof vmw_offsetof
-#endif
-
-#endif // offsetof
-#endif // __APPLE__
-
 #define VMW_CONTAINER_OF(ptr, type, member) \
-   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))
+   ((type *)((char *)(ptr) - offsetof(type, member)))

 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
@@ -101,7 +103,7 @@

 /* The Solaris 9 cross-compiler complains about these not being used */
 #ifndef sun
-static INLINE int 
+static INLINE int
 Min(int a, int b)
 {
   return a < b ? a : b;
@@ -113,7 +115,7 @@ Min(int a, int b)
 #endif

 #ifndef sun
-static INLINE int 
+static INLINE int
 Max(int a, int b)
 {
   return a > b ? a : b;
@@ -155,20 +157,8 @@ Max(int a, int b)

 #define IMPLIES(a,b) (!(a) || (b))

+
 /*
- * Not everybody (e.g., the monitor) has NULL
- */
-
-#ifndef NULL
-#ifdef  __cplusplus
-#define NULL    0
-#else
-#define NULL    ((void *)0)
-#endif
-#endif
-
-
-/* 
 * Token concatenation
 *
 * The C preprocessor doesn't prescan arguments when they are
@@ -205,7 +195,7 @@ Max(int a, int b)
 */

 #ifndef PAGE_SHIFT // {
-#if defined VM_I386
+#if defined __x86_64__ || defined __i386__
   #define PAGE_SHIFT    12
 #elif defined __APPLE__
   #define PAGE_SHIFT    12
@@ -251,26 +241,49 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif

+#ifndef KBYTES_SHIFT
+#define KBYTES_SHIFT 10
+#endif
+
 #ifndef MBYTES_SHIFT
 #define MBYTES_SHIFT 20
 #endif

 #ifndef MBYTES_2_PAGES
-#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
+#define MBYTES_2_PAGES(_nbytes) \
+   ((uint64)(_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
 #endif

 #ifndef PAGES_2_MBYTES
 #define PAGES_2_MBYTES(_npages) ((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
 #endif

+#ifndef ROUNDUP_PAGES_2_MBYTES
+#define ROUNDUP_PAGES_2_MBYTES(_npages) \
+(((_npages) + MASK(MBYTES_SHIFT - PAGE_SHIFT)) >> (MBYTES_SHIFT - PAGE_SHIFT))
+#endif
+
+#ifndef ROUNDDOWN_PAGES_2_MBYTES
+#define ROUNDDOWN_PAGES_2_MBYTES(_npages) \
+((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
+#endif
+
 #ifndef GBYTES_2_PAGES
-#define GBYTES_2_PAGES(_nbytes) ((_nbytes) << (30 - PAGE_SHIFT))
+#define GBYTES_2_PAGES(_nbytes) ((uint64)(_nbytes) << (30 - PAGE_SHIFT))
 #endif

 #ifndef PAGES_2_GBYTES
 #define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
 #endif

+#ifndef BYTES_2_KBYTES
+#define BYTES_2_KBYTES(_nbytes) ((_nbytes) >> KBYTES_SHIFT)
+#endif
+
+#ifndef KBYTES_2_BYTES
+#define KBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << KBYTES_SHIFT)
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
@@ -369,7 +382,7 @@ Max(int a, int b)
 #ifdef _MSC_VER
 #ifdef __cplusplus
 extern "C"
-#endif 
+#endif
 void *_ReturnAddress(void);
 #pragma intrinsic(_ReturnAddress)
 #define GetReturnAddress() _ReturnAddress()
@@ -412,22 +425,6 @@ void *_ReturnAddress(void);

 #ifdef USERLEVEL // {

-/*
- * Note this might be a problem on NT b/c while sched_yield guarantees it
- * moves you to the end of your priority list, Sleep(0) offers no such
- * guarantee.  Bummer.  --Jeremy.
- */
-
-#if defined(_WIN32)
-#      define YIELD()		Sleep(0)
-#elif defined(VMKERNEL)
-/* We don't have a YIELD macro in the vmkernel */
-#else
-#      include <sched.h>        // For sched_yield.  Don't ask.  --Jeremy.
-#      define YIELD()		sched_yield()
-#endif 
-
-
 /*
 * Standardize some Posix names on Windows.
 */
@@ -497,19 +494,8 @@ typedef int pid_t;

 #elif defined(__APPLE__) && defined(KERNEL)

-#include "availabilityMacOS.h"
-
-#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
-// The Mac OS 10.5 kernel SDK defines va_copy in stdarg.h.
+// The macOS kernel SDK defines va_copy in stdarg.h.
 #include <stdarg.h>
-#else
-/*
- * The Mac OS 10.4 kernel SDK needs va_copy. Based on inspection of
- * stdarg.h from the MacOSX10.4u.sdk kernel framework, this should
- * work.
- */
-#define va_copy(dest, src) ((dest) = (src))
-#endif // MAC_OS_X_VERSION_MIN_REQUIRED

 #elif defined(__GNUC__) && (__GNUC__ < 3)

@@ -529,7 +515,7 @@ typedef int pid_t;
 * driver.
 */

-#ifdef _WIN32
+#if defined(_WIN32) && defined(_MSC_VER)
 #define PATH_MAX 256
 #ifndef strcasecmp
 #define strcasecmp(_s1,_s2)   _stricmp((_s1),(_s2))
@@ -539,29 +525,6 @@ typedef int pid_t;
 #endif
 #endif

-#if defined __linux__ && !defined __KERNEL__ && !defined MODULE && \
-                         !defined VMM && !defined FROBOS && !defined __ANDROID__
-#include <features.h>
-#if __GLIBC_PREREQ(2, 1) && !defined GLIBC_VERSION_21
-#define GLIBC_VERSION_21
-#endif
-#if __GLIBC_PREREQ(2, 2) && !defined GLIBC_VERSION_22
-#define GLIBC_VERSION_22
-#endif
-#if __GLIBC_PREREQ(2, 3) && !defined GLIBC_VERSION_23
-#define GLIBC_VERSION_23
-#endif
-#if __GLIBC_PREREQ(2, 4) && !defined GLIBC_VERSION_24
-#define GLIBC_VERSION_24
-#endif
-#if __GLIBC_PREREQ(2, 5) && !defined GLIBC_VERSION_25
-#define GLIBC_VERSION_25
-#endif
-#if __GLIBC_PREREQ(2, 12) && !defined GLIBC_VERSION_212
-#define GLIBC_VERSION_212
-#endif
-#endif
-
 /*
 * Convenience definitions of unicode characters.
 */
@@ -591,10 +554,10 @@ typedef int pid_t;
 #undef DEBUG_ONLY
 #ifdef VMX86_DEBUG
 #define vmx86_debug      1
-#define DEBUG_ONLY(x)    x
+#define DEBUG_ONLY(...)  __VA_ARGS__
 #else
 #define vmx86_debug      0
-#define DEBUG_ONLY(x)
+#define DEBUG_ONLY(...)
 #endif

 #ifdef VMX86_STATS
@@ -634,7 +597,7 @@ typedef int pid_t;
 #define RELEASE_ONLY(x) x
 #else
 #define vmx86_release   0
-#define RELEASE_ONLY(x) 
+#define RELEASE_ONLY(x)
 #endif

 #ifdef VMX86_SERVER
@@ -687,29 +650,38 @@ typedef int pid_t;
 #define vmw_apple_sandbox 0
 #endif

+#if defined(__APPLE__) && defined(VMW_APPLE_APP_STORE)
+#define vmw_apple_app_store 1
+#else
+#define vmw_apple_app_store 0
+#endif
+
 #ifdef VMM
 #define VMM_ONLY(x) x
 #else
 #define VMM_ONLY(x)
 #endif

+#ifdef ULM
+#define vmx86_ulm 1
+#define ULM_ONLY(x) x
+#else
+#define vmx86_ulm 0
+#define ULM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(ULM)
+#define MONITOR_ONLY(x) x
+#else
+#define MONITOR_ONLY(x)
+#endif
+
 #if defined(VMM) || defined(VMKERNEL)
 #define USER_ONLY(x)
 #else
 #define USER_ONLY(x) x
 #endif

-/* VMVISOR ifdef only allowed in the vmkernel */
-#ifdef VMKERNEL
-#ifdef VMVISOR
-#define vmvisor 1
-#define VMVISOR_ONLY(x) x
-#else
-#define vmvisor 0
-#define VMVISOR_ONLY(x)
-#endif
-#endif
-
 #ifdef _WIN32
 #define VMW_INVALID_HANDLE INVALID_HANDLE_VALUE
 #else
@@ -742,12 +714,6 @@ typedef int pid_t;
 #endif
 #endif // _WIN32

-#ifdef HOSTED_LG_PG
-#define hosted_lg_pg 1
-#else
-#define hosted_lg_pg 0
-#endif
-
 /*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
@@ -758,7 +724,7 @@ typedef int pid_t;
 /* This is not intended to be thread-safe. */
 #define DO_ONCE(code)                                                   \
   do {                                                                 \
-      static Bool _doOnceDone = FALSE;                                  \
+      static MONITOR_ONLY(PERVCPU) Bool _doOnceDone = FALSE;            \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,24 +38,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-/* STRICT ANSI means the Xserver build and X defines Bool differently. */
-#if !defined(_XTYPEDEF_BOOL) && \
-    (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || \
-      defined(__MINGW32__) || defined(__APPLE__))
-#define _XTYPEDEF_BOOL
-typedef char           Bool;
-#endif
-
-#ifndef FALSE
-#define FALSE          0
-#endif
-
-#ifndef TRUE
-#define TRUE           1
-#endif
-
-#define IS_BOOL(x)     (((x) & ~1) == 0)
-
 /*
 * Macros __i386__ and __ia64 are intrinsically defined by GCC
 */
@@ -81,17 +63,9 @@ typedef char           Bool;
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
- *
- * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
- * should really be VM_X86_32).
 */

 #ifdef __i386__
-/* VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
- * since it is confused with the __i386__ gcc but defined for both 32- and
- * 64-bit x86. We retain it here for legacy compatibility.
- */
-#define VM_I386
 #define VM_X86_32
 #define VM_X86_ANY
 #define VM_32BIT
@@ -99,12 +73,11 @@ typedef char           Bool;

 #ifdef __x86_64__
 #define VM_X86_64
-#define VM_I386
+#define vm_x86_64 1
 #define VM_X86_ANY
 #define VM_64BIT
-#define vm_x86_64 (1)
 #else
-#define vm_x86_64 (0)
+#define vm_x86_64 0
 #endif

 #ifdef __arm__
@@ -115,13 +88,15 @@ typedef char           Bool;

 #ifdef __aarch64__
 #define VM_ARM_64
+#define vm_arm_64 1
 #define VM_ARM_ANY
 #define VM_64BIT
-#define vm_arm_64 (1)
 #else
-#define vm_arm_64 (0)
+#define vm_arm_64 0
 #endif

+#define vm_64bit (sizeof (void *) == 8)
+
 #ifdef _MSC_VER

 #pragma warning (3 :4505) // unreferenced local function
@@ -135,16 +110,91 @@ typedef char           Bool;

 #endif

-#if defined(__linux__) && defined(__cplusplus) && __cplusplus >= 201103L
+#if defined(__FreeBSD__) && (__FreeBSD__ + 0 < 5)
+#  error FreeBSD detected without major version (PR 2116887)
+#endif

 /*
- * We're using stdint.h instead of cstdint below because of libstdcpp.cpp.
- * It looks like a C++ file. When being preprocessed all the C++ specific
- * defines(e.g. __cplusplus) are set, but the C++ include paths are not.
+ * C99 <stdint.h> or equivalent
+ * Special cases:
+ * - Linux kernel lacks <stdint.h>, preferring <linux/types.h>
+ *   (and defines uintptr_t since 2.6.24, but not intptr_t)
+ * - Solaris collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel + FreeBSD collides with gcc <stdint.h>, but has <sys/stdint.h>
+ * - VMKernel (+DECODERLIB) share macros with Linux kernel
+ * - Windows only added <stdint.h> in vc10/vs2010 (MSC ver 1600),
+ *   and WDKs lack it.
+ *
+ * NB about LLP64 in LP64 environments:
+ * - Apple uses 'long long' uint64_t
+ * - Linux kernel uses 'long long' uint64_t
+ * - Linux userlevel uses 'long' uint64_t
 */
-#include <stdint.h>
+#if !defined(VMKERNEL) && !defined(DECODERLIB) && \
+    defined(__linux__) && defined(__KERNEL__)
+#  include <linux/types.h>
+#  include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+   typedef unsigned long uintptr_t;
+#endif
+   typedef   signed long  intptr_t;
+#elif (defined(__sun__) && defined(_KERNEL)) || \
+      (defined(VMKERNEL) && defined(__FreeBSD__)) || \
+      defined(_SYS_STDINT_H_)
+#  include <sys/stdint.h>
+#elif !defined(_MSC_VER)
+   /* Common case */
+#  include <stdint.h>
+#else
+   /* COMPAT: until pre-vc10 is retired */
+#  include <crtdefs.h>  // uintptr_t
+   typedef unsigned __int64   uint64_t;
+   typedef unsigned int       uint32_t;
+   typedef unsigned short     uint16_t;
+   typedef unsigned char      uint8_t;
+
+   typedef __int64            int64_t;
+   typedef int                int32_t;
+   typedef short              int16_t;
+   typedef signed char        int8_t;
+#endif
+
+/*
+ * size_t and ssize_t, or equivalent
+ * Options:
+ * C90+ <stddef.h> has size_t, but is incompatible with many kernels.
+ * POSIX <sys/types.h> has size_t and ssize_t, is always available at
+ *    userlevel but is missing from some kernels.
+ *
+ * Special cases:
+ * - Linux kernel (again) does everything via <linux/types.h>
+ * - VMKernel may or may not have POSIX headers (tcpip only)
+ * - VMM does not have POSIX headers
+ * - Windows <sys/types.h> does not define ssize_t
+ */
+#if defined(VMKERNEL) || defined(VMM) || defined(DECODERLIB)
+   /* Guard against FreeBSD <sys/types.h> collison. */
+#  if !defined(_SIZE_T_DEFINED) && !defined(_SIZE_T)
+#     define _SIZE_T_DEFINED
+#     define _SIZE_T
+      typedef __SIZE_TYPE__ size_t;
+#endif
+#  if !defined(_SSIZE_T_DEFINED) && !defined(_SSIZE_T)
+#     define _SSIZE_T_DEFINED
+#     define _SSIZE_T
+      typedef int64_t ssize_t;
+#  endif
+#elif defined(__linux__) && defined(__KERNEL__)
+   /* <linux/types.h> provided size_t, ssize_t. */
+#else
+#  include <sys/types.h>
+#  if defined(_WIN64)
+      typedef int64_t ssize_t;
+#  elif defined(_WIN32)
+      typedef int32_t ssize_t;
+#  endif
+#endif

-typedef char          Bool;

 typedef uint64_t    uint64;
 typedef  int64_t     int64;
@@ -155,78 +205,35 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-typedef uint64 BA;
-typedef uint64 MA;
-typedef uint32 MPN32;
-
-#elif defined(__APPLE__) || defined(HAVE_STDINT_H)

 /*
- * TODO: This is a C99 standard header.  We should be able to test for
- * #if __STDC_VERSION__ >= 199901L, but that breaks the Netware build
- * (which doesn't have stdint.h).
+ * The _XTYPEDEF_BOOL guard prevents colliding with:
+ * <X11/Xlib.h> #define Bool int
+ * <X11/Xdefs.h> typedef int Bool;
+ * If using this header AND X11 headers, be sure to #undef Bool and
+ * be careful about the different size.
 */
-
-#include <stdint.h>
-
-typedef uint64_t    uint64;
-typedef  int64_t     int64;
-typedef uint32_t    uint32;
-typedef  int32_t     int32;
-typedef uint16_t    uint16;
-typedef  int16_t     int16;
-typedef  uint8_t    uint8;
-typedef   int8_t     int8;
-
+#if !defined(_XTYPEDEF_BOOL)
+#define _XTYPEDEF_BOOL
 /*
- * Note: C does not specify whether char is signed or unsigned, and
- * both gcc and msvc implement processor-specific signedness.  With
- * three types:
- * typeof(char) != typeof(signed char) != typeof(unsigned char)
- *
- * Be careful here, because gcc (4.0.1 and others) likes to warn about
- * conversions between signed char * and char *.
+ * C does not specify whether char is signed or unsigned, and
+ * both gcc and msvc implement it as a non-signed, non-unsigned type.
+ * Thus, (uint8_t *)&Bool and (int8_t *)&Bool are possible compile errors.
+ * This is intentional.
 */
+typedef char           Bool;
+#endif

-#else /* !HAVE_STDINT_H */
+#ifndef FALSE
+#define FALSE          0
+#endif

-#ifdef _MSC_VER
+#ifndef TRUE
+#define TRUE           1
+#endif

-typedef unsigned __int64 uint64;
-typedef signed __int64 int64;
+#define IS_BOOL(x)     (((x) & ~1) == 0)

-#elif __GNUC__
-/* The Xserver source compiles with -ansi -pendantic */
-#   if !defined(__STRICT_ANSI__) || defined(__FreeBSD__)
-#      if defined(VM_X86_64) || defined(VM_ARM_64)
-typedef unsigned long uint64;
-typedef long int64;
-#      else
-typedef unsigned long long uint64;
-typedef long long int64;
-#      endif
-#   endif
-#else
-#   error - Need compiler define for int64/uint64
-#endif /* _MSC_VER */
-
-typedef unsigned int       uint32;
-typedef unsigned short     uint16;
-typedef unsigned char      uint8;
-
-typedef int                int32;
-typedef short              int16;
-typedef signed char        int8;
-
-#endif /* HAVE_STDINT_H */
-
-/*
- * FreeBSD (for the tools build) unconditionally defines these in
- * sys/inttypes.h so don't redefine them if this file has already
- * been included. [greg]
- *
- * This applies to Solaris as well.
- */

 /*
 * Before trying to do the includes based on OS defines, see if we can use
@@ -236,9 +243,6 @@ typedef signed char        int8;
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
 #ifdef HAVE_SYS_INTTYPES_H
 #include <sys/inttypes.h>
 #endif
@@ -252,14 +256,11 @@ typedef signed char        int8;

 #if !defined(USING_AUTOCONF)
 #   if defined(__FreeBSD__) || defined(sun)
-#      ifdef KLD_MODULE
-#         include <sys/types.h>
-#      else
+#      ifndef KLD_MODULE
 #         if __FreeBSD_version >= 500043
 #            if !defined(VMKERNEL)
 #               include <inttypes.h>
 #            endif
-#            include <sys/types.h>
 #         else
 #            include <sys/inttypes.h>
 #         endif
@@ -267,46 +268,10 @@ typedef signed char        int8;
 #   elif defined __APPLE__
 #      if KERNEL
 #         include <sys/unistd.h>
-#         include <sys/types.h> /* mostly for size_t */
-#         include <stdint.h>
 #      else
 #         include <unistd.h>
 #         include <inttypes.h>
 #         include <stdlib.h>
-#         include <stdint.h>
-#      endif
-#   elif defined __ANDROID__
-#      include <stdint.h>
-#   else
-#      if !defined(__intptr_t_defined) && !defined(intptr_t)
-#         ifdef VM_I386
-#            define __intptr_t_defined
-#            if defined(VM_X86_64)
-typedef int64     intptr_t;
-#            else
-typedef int32     intptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-#            define __intptr_t_defined
-typedef int64     intptr_t;
-#         elif defined(__arm__)
-#            define __intptr_t_defined
-typedef int32     intptr_t;
-#         endif
-#      endif
-
-#      ifndef _STDINT_H
-#         ifdef VM_I386
-#            if defined(VM_X86_64)
-typedef uint64    uintptr_t;
-#            else
-typedef uint32    uintptr_t;
-#            endif
-#         elif defined(VM_ARM_64)
-typedef uint64    uintptr_t;
-#         elif defined(__arm__)
-typedef uint32    uintptr_t;
-#         endif
 #      endif
 #   endif
 #endif
@@ -344,6 +309,7 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
 */

 #ifdef _MSC_VER
+   /* MSVC added C99-compatible formatting in vs2015. */
   #define FMT64      "I64"
   #ifdef VM_X86_64
      #define FMTSZ      "I64"
@@ -354,54 +320,31 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
-#elif defined __APPLE__
-   /* Mac OS hosts use the same formatters for 32- and 64-bit. */
-   #define FMT64 "ll"
-   #if KERNEL
-      #define FMTSZ "l"
+#elif defined __APPLE__ || (!defined VMKERNEL && !defined DECODERLIB && \
+                            defined __linux__ && defined __KERNEL__)
+   /* semi-LLP64 targets; 'long' is 64-bit, but uint64_t is 'long long' */
+   #define FMT64         "ll"
+   #if defined(__APPLE__) && KERNEL
+      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
+      #define FMTSZ      "l"
   #else
-      #define FMTSZ "z"
-   #endif
-   #define FMTPD "l"
-   #define FMTH ""
-#elif __GNUC__
-   #define FMTH ""
-   #if defined(sun)
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTSZ  "l"
-         #define FMTPD  "l"
-      #else
-         #define FMTSZ  ""
-         #define FMTPD  ""
-      #endif
-   #elif defined(__linux__) || \
-        (defined(__FreeBSD__) && (__FreeBSD__ + 0))\
-      || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) \
-      || (defined(_POSIX_VERSION) && _POSIX_VERSION >= 200112L) \
-      || (defined(_POSIX2_VERSION) && _POSIX2_VERSION >= 200112L)
-      /* BSD, Linux */
-      #define FMTSZ     "z"
-
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTPD  "l"
-      #else
-         #define FMTPD  ""
-      #endif
-   #else
-      /* Systems with a pre-C99 libc */
-      #define FMTSZ     "Z"
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTPD  "l"
-      #else
-         #define FMTPD  ""
-      #endif
+      #define FMTSZ      "z"
   #endif
+   #define FMTPD         "l"
+   #define FMTH          ""
+#elif defined __GNUC__
+   /*
+    * Every POSIX system we target has C99-compatible printf
+    * (supports 'z' for size_t and 'll' for long long).
+    */
+   #define FMTH          ""
+   #define FMTSZ         "z"
   #if defined(VM_X86_64) || defined(VM_ARM_64)
-      #define FMT64     "l"
-   #elif defined(sun) || defined(__FreeBSD__)
-      #define FMT64     "ll"
+      #define FMT64      "l"
+      #define FMTPD      "l"
   #else
-      #define FMT64     "L"
+      #define FMT64      "ll"
+      #define FMTPD      ""
   #endif
 #else
   #error - Need compiler define for FMT64 and FMTSZ
@@ -485,26 +428,22 @@ typedef uint8 *TCA;  /* Pointer into TC (usually). */
 * Type big enough to hold an integer between 0..100
 */
 typedef uint8 Percent;
-#define AsPercent(v)	((Percent)(v))
+#define AsPercent(v) ((Percent)(v))


 typedef uintptr_t VA;
 typedef uintptr_t VPN;

 typedef uint64    PA;
-typedef uint32    PPN;
+typedef uint64    PPN;

 typedef uint64    TPA;
-typedef uint32    TPPN;
+typedef uint64    TPPN;

 typedef uint64    PhysMemOff;
 typedef uint64    PhysMemSize;

-/* The Xserver source compiles with -ansi -pendantic */
-#ifndef __STRICT_ANSI__
 typedef uint64    BA;
-#endif
-
 #ifdef VMKERNEL
 typedef void     *BPN;
 #else
@@ -514,9 +453,10 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint32    PageNum;
-typedef unsigned      MemHandle;
-typedef unsigned int  IoHandle;
+typedef uint64    PageCnt;
+typedef uint64    PageNum;
+typedef unsigned  MemHandle;
+typedef unsigned  IoHandle;
 typedef int32     World_ID;

 /* !! do not alter the definition of INVALID_WORLD_ID without ensuring
@@ -556,15 +496,12 @@ typedef uint128 UReg128;
 #endif

 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
-    defined (VMKERNEL) || defined (VMKBOOT)
+    defined (VMKERNEL) || defined (VMKBOOT) || defined (ULM)
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
-/* The Xserver source compiles with -ansi -pendantic */
-#ifndef __STRICT_ANSI__
 typedef uint64 MA;
 typedef uint32 MPN32;
-#endif

 /*
 * This type should be used for variables that contain sector
@@ -615,6 +552,7 @@ typedef uint64 LA64;
 typedef uint64 LPN64;
 typedef uint64 PA64;
 typedef uint64 PPN64;
+typedef uint64 TPPN64;
 typedef uint64 MA64;
 typedef uint64 MPN;

@@ -640,12 +578,15 @@ typedef void * UserVA;
 #endif


-#define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1) /* Maximal observable PPN value. */
-#define INVALID_PPN       ((PPN)0xffffffff)
-#define APIC_INVALID_PPN  ((PPN)0xfffffffe)
+/* Maximal observable PPN value. */
+#define MAX_PPN_BITS      33
+#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1)

-#define INVALID_BPN       ((BPN)0x000000ffffffffffull)
+#define INVALID_PPN       ((PPN)0x000fffffffffffffull)
+#define INVALID_PPN32     ((PPN32)0xffffffff)
+#define APIC_INVALID_PPN  ((PPN)0x000ffffffffffffeull)
+
+#define INVALID_BPN       ((BPN)0x0000ffffffffffffull)

 #define MPN38_MASK        ((1ull << 38) - 1)

@@ -663,7 +604,8 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((PageNum)-1)
+#define INVALID_PAGENUM   ((PageNum)0x000000ffffffffffull)
+#define INVALID_PAGENUM32 ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -755,6 +697,14 @@ typedef void * UserVA;
 #error "gcc version is too old to compile assembly, need gcc-3.3 or better"
 #endif

+/*
+ * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Enforce this here.
+ */
+#if defined _MSC_VER && _MSC_VER < 1400
+#error "cl.exe version is too old, need vc80 or better"
+#endif
+

 /*
 * Consider the following reasons functions are inlined:
@@ -783,7 +733,7 @@ typedef void * UserVA;
 #define INLINE_SINGLE_CALLER INLINE_ALWAYS

 /*
- * Used when a hard guaranteed of no inlining is needed. Very few
+ * Used when a hard guarantee of no inlining is needed. Very few
 * instances need this since the absence of INLINE is a good hint
 * that gcc will not do inlining.
 */
@@ -868,7 +818,7 @@ typedef void * UserVA;
 * will convert any !=0 to a 1.
 */
 #define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
-#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
+#define UNLIKELY(_exp)   __builtin_expect(!!(_exp), 0)
 #else
 #define LIKELY(_exp)      (_exp)
 #define UNLIKELY(_exp)    (_exp)
@@ -965,82 +915,6 @@ typedef void * UserVA;

 #define INFINITE_LOOP()           do { } while (1)

-/*
- * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
- * is defined. Use the same logic here so we don't define it twice. [greg]
- */
-#ifdef __FreeBSD__
-#   ifdef _BSD_SIZE_T_
-#      undef _BSD_SIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#   ifdef _BSD_SSIZE_T_
-#      undef _BSD_SSIZE_T_
-#      ifdef VM_I386
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      endif /* VM_I386 */
-#   endif
-
-#else
-#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
-#      ifdef VM_I386
-#         define _SIZE_T
-#         ifdef VM_X86_64
-             typedef uint64 size_t;
-#         else
-             typedef uint32 size_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SIZE_T
-          typedef uint64 size_t;
-#      elif defined(__arm__)
-#         define _SIZE_T
-          typedef uint32 size_t;
-#      endif
-#   endif
-
-#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
-       !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
-       !defined(_SSIZE_T_DEFINED_)
-#      ifdef VM_I386
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-#         ifdef VM_X86_64
-             typedef int64 ssize_t;
-#         else
-             typedef int32 ssize_t;
-#         endif
-#      elif defined(VM_ARM_64)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-          typedef int64 ssize_t;
-#      elif defined(__arm__)
-#         define _SSIZE_T
-#         define __ssize_t_defined
-#         define _SSIZE_T_DECLARED
-#         define _SSIZE_T_DEFINED_
-             typedef int32 ssize_t;
-#      endif
-#   endif
-
-#endif
-
 /*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
@@ -1086,33 +960,6 @@ typedef void * UserVA;
 # define FMTMODE "o"
 #endif

-/*
- * Format modifier for printing time_t. Most platforms define a time_t to be
- * a long int, but on FreeBSD (as of 5.0, it seems), the time_t is a signed
- * size quantity. Refer to the definition of FMTSZ to see why we need silly
- * preprocessor arithmetic.
- * Use this like this: printf("The mode is %" FMTTIME ".\n", time);
- */
-#if defined(__FreeBSD__) && (__FreeBSD__ + 0) && ((__FreeBSD__ + 0) >= 5)
-#   define FMTTIME FMTSZ"d"
-#else
-#   if defined(_MSC_VER)
-#      ifndef _SAFETIME_H_
-#         if (_MSC_VER < 1400) || defined(_USE_32BIT_TIME_T)
-#             define FMTTIME "ld"
-#         else
-#             define FMTTIME FMT64"d"
-#         endif
-#      else
-#         ifndef FMTTIME
-#            error "safetime.h did not define FMTTIME"
-#         endif
-#      endif
-#   else
-#      define FMTTIME "ld"
-#   endif
-#endif
-
 #ifdef __APPLE__
 /*
 * Format specifier for all these annoying types such as {S,U}Int32
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014,2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,21 +40,25 @@
 /*
 * These definitions suit both the x64 and arm64 architectures. In particular
 * - At all levels, a page table is a 4KB page containing 512 8-byte entries.
- * - Each entry maps 39, 30, 21 and 12 bits at the respective page table
+ * - Each entry maps 48, 39, 30, 21 and 12 bits at the respective page table
 *   levels (with standard page sizes).
 */

-typedef uint64 PT_L4E;
-typedef uint64 PT_L3E;
-typedef uint64 PT_L2E;
-typedef uint64 PT_L1E;
+typedef uint64 PT_Entry;
+typedef PT_Entry PT_L5E;
+typedef PT_Entry PT_L4E;
+typedef PT_Entry PT_L3E;
+typedef PT_Entry PT_L2E;
+typedef PT_Entry PT_L1E;

 typedef enum {
   PT_LEVEL_1 = 1,
   PT_LEVEL_2,
   PT_LEVEL_3,
   PT_LEVEL_4,
-   PT_MAX_LEVELS = PT_LEVEL_4
+   PT_LEVEL_5,
+   PT_LEVEL_STOP = PT_LEVEL_1,
+   PT_MAX_LEVELS = PT_LEVEL_5
 } PT_Level;

 #define PT_PTE_SIZE           8
@@ -64,6 +68,9 @@ typedef enum {
 #define PT_ENTRIES_PER_PT     (1 << PT_LEVEL_SHIFT)
 #define PT_OFF_MASK           (PT_ENTRIES_PER_PT - 1)

+/* Align any page number type to the 2M containing page number. */
+#define PT_2M_ALIGN_PN(x)     ((x) & ~PT_OFF_MASK)
+
 /*
 * log2 of the
 * address space size (in 4KB pages) covered by a level-_l page table entry.
@@ -72,6 +79,7 @@ typedef enum {

 /* Address space size (in 4KB pages) covered by a level-_l page table entry. */
 #define PT_PAGES_PER_LE(_l)   ((uint64)1 << PT_LE_PG_SHIFT(_l))
+#define PT_PAGES_PER_L5E      PT_PAGES_PER_LE(PT_LEVEL_5)
 #define PT_PAGES_PER_L4E      PT_PAGES_PER_LE(PT_LEVEL_4)
 #define PT_PAGES_PER_L3E      PT_PAGES_PER_LE(PT_LEVEL_3)
 #define PT_PAGES_PER_L2E      PT_PAGES_PER_LE(PT_LEVEL_2)
@@ -82,6 +90,7 @@ typedef enum {
 * address space size (in bytes) covered by a level-_l page table entry.
 */
 #define PT_LE_SHIFT(_l)       (PAGE_SHIFT + PT_LE_PG_SHIFT(_l))
+#define PT_L5E_SHIFT          PT_LE_SHIFT(PT_LEVEL_5)
 #define PT_L4E_SHIFT          PT_LE_SHIFT(PT_LEVEL_4)
 #define PT_L3E_SHIFT          PT_LE_SHIFT(PT_LEVEL_3)
 #define PT_L2E_SHIFT          PT_LE_SHIFT(PT_LEVEL_2)
@@ -89,6 +98,7 @@ typedef enum {

 /* Address space size (in bytes) covered by a level-_l page table entry. */
 #define PT_LE_SIZE(_l)        ((uint64)1 << PT_LE_SHIFT(_l))
+#define PT_L5E_SIZE           PT_LE_SIZE(PT_LEVEL_5)
 #define PT_L4E_SIZE           PT_LE_SIZE(PT_LEVEL_4)
 #define PT_L3E_SIZE           PT_LE_SIZE(PT_LEVEL_3)
 #define PT_L2E_SIZE           PT_LE_SIZE(PT_LEVEL_2)
@@ -99,6 +109,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LA_2_LOFF(_a, _l)  (((_a) >> PT_LE_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LA_2_L5OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LA_2_L4OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LA_2_L3OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LA_2_L2OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_2)
@@ -109,6 +120,7 @@ typedef enum {
 * determine the offset (== index) of the level-_l page table entry.
 */
 #define PT_LPN_2_LOFF(_a, _l) (((_a) >> PT_LE_PG_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LPN_2_L5OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_5)
 #define PT_LPN_2_L4OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_4)
 #define PT_LPN_2_L3OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_3)
 #define PT_LPN_2_L2OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_2)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -30,9 +30,6 @@
 #define INCLUDE_ALLOW_VMMON
 #include "includeCheck.h"

-#define INTERRUPT_REDIRECTION_BITMAP_SIZE 32
-#define IO_PERMISSION_BITMAP_SIZE 0
-

 #define VMM_PANIC_NONE         0
 #define VMM_PANIC_MSG_COPYING  1
@@ -0,0 +1,50 @@
+/*********************************************************
+ * Copyright (C) 2017-2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vmmblob.h --
+ *
+ *     VMM blob management.
+ */
+
+#ifndef VMMBLOB_H
+#define VMMBLOB_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#include "includeCheck.h"
+
+struct VMDriver;
+struct MonLoaderHeader;
+
+typedef struct VmmBlobInfo {
+   uint8                  *blobPtr;
+   uint32                  numBytes;
+   MPN                    *mpns;
+   struct MonLoaderHeader *header;
+} VmmBlobInfo;
+
+void   VmmBlob_Cleanup(VmmBlobInfo *bi);
+Bool   VmmBlob_Load(VA64 bsBlobAddr, uint32 numBytes, uint32 headerOffset,
+                    VmmBlobInfo **blobInfo);
+MPN    VmmBlob_GetMpn(struct VMDriver *vm, uint64 blobOffset);
+MPN    VmmBlob_GetHeaderMpn(struct VMDriver *vm);
+uint8 *VmmBlob_GetPtr(VMDriver *vm);
+uint64 VmmBlob_GetSize(VMDriver *vm);
+
+#endif /* VMMBLOB_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -48,16 +48,19 @@

 #define VMMEM_GUEST_WRITEABLE     VMMEM_FLAG_BIT(0)
 #define VMMEM_GUEST_BREAKCOW      VMMEM_FLAG_BIT(1)
-#define VMMEM_GUEST_LARGE_PAGE    VMMEM_FLAG_BIT(2)
-#define VMMEM_GUEST_CAN_FAIL      VMMEM_FLAG_BIT(3)
-#define VMMEM_GUEST_TEST_ZEROCOW  VMMEM_FLAG_BIT(4)
-#define VMMEM_GUEST_TRY_ZEROCOW   VMMEM_FLAG_BIT(5)
-#define VMMEM_GUEST_TRY_POISONCOW VMMEM_FLAG_BIT(6)
-#define VMMEM_GUEST_PREALLOC      VMMEM_FLAG_BIT(7)
-#define VMMEM_GUEST_ALL_FLAGS     MASK(8)
+#define VMMEM_GUEST_2M_PAGE       VMMEM_FLAG_BIT(2)
+#define VMMEM_GUEST_1G_PAGE       VMMEM_FLAG_BIT(3)
+#define VMMEM_GUEST_CAN_FAIL      VMMEM_FLAG_BIT(4)
+#define VMMEM_GUEST_TEST_ZEROCOW  VMMEM_FLAG_BIT(5)
+#define VMMEM_GUEST_TRY_ZEROCOW   VMMEM_FLAG_BIT(6)
+#define VMMEM_GUEST_TRY_POISONCOW VMMEM_FLAG_BIT(7)
+#define VMMEM_GUEST_PREALLOC      VMMEM_FLAG_BIT(8)
+#define VMMEM_GUEST_ALL_FLAGS     MASK(9)
 #define VMMEM_GUEST_TRY_COW       (VMMEM_GUEST_TEST_ZEROCOW | \
                                   VMMEM_GUEST_TRY_ZEROCOW  | \
                                   VMMEM_GUEST_TRY_POISONCOW)
+#define VMMEM_GUEST_LARGE_PAGE    (VMMEM_GUEST_2M_PAGE | \
+                                   VMMEM_GUEST_1G_PAGE)

 #define VMMEM_PLATFORM_CHECK_OK           VMMEM_FLAG_BIT(0)
 #define VMMEM_PLATFORM_KEY_OK             VMMEM_FLAG_BIT(1)
@@ -80,11 +83,10 @@
 * Structure used to query platform about the page state.
 */
 typedef struct PlatformPageInfoList {
-   uint32 numPages;
-   uint32 _pad;
-   BPN    bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
-   MPN    mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
-   uint8  flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
+   PageCnt numPages;
+   BPN     bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
+   MPN     mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
+   uint8   flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
 } PlatformPageInfoList;

 #define VMMEM_SERVICES_TYPE_2_MASK(type)                  \
@@ -94,9 +96,10 @@ typedef struct PlatformPageInfoList {
 #define VMMEM_SERVICES_CLEAR_MASK(typeMask, type)         \
           (typeMask & ~VMMEM_SERVICES_TYPE_2_MASK(type))

-#define VMMEM_SERVICES_DEFS                               \
-   MDEF(VMMEM_SERVICES_TYPE_P2M,  P2MUpdate_FilterPages)  \
-   MDEF(VMMEM_SERVICES_TYPE_SWAP, BusMemSwap_FilterPages)
+#define VMMEM_SERVICES_DEFS                                       \
+   MDEF(VMMEM_SERVICES_TYPE_P2M,         P2MUpdate_FilterPages)   \
+   MDEF(VMMEM_SERVICES_TYPE_SWAP,        BusMemSwap_FilterPages)  \
+   MDEF(VMMEM_SERVICES_TYPE_DIRTY_CLEAR, DirtyDrainer_ClearDirty)

 #define MDEF(_type, _cb) _type,
 typedef enum VmMemServices_Type {
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,8 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#include <asm/processor-flags.h>
+
 #define X86_MAX_INSTR_LEN  15   /* Max byte length of an x86 instruction. */

 #define NUM_IDT_VECTORS 256
@@ -75,7 +77,9 @@
 #define CR3_PDB_MASK   0xfffff000
 #define CR3_IGNORE     0xFFF
 #define PAE_CR3_IGNORE 0x1F
+#ifndef CR3_PCID_MASK
 #define CR3_PCID_MASK  0xFFF
+#endif
 #define CR3_NO_FLUSH   (1ULL << 63)

 #define CR4_VME        0x00000001
@@ -89,6 +93,8 @@
 #define CR4_PCE        0x00000100
 #define CR4_OSFXSR     0x00000200 // CPU/OS supports SIMD insts
 #define CR4_OSXMMEXCPT 0x00000400 // #XF exception enable PIII only
+#define CR4_UMIP       0x00000800
+#define CR4_LA57       0x00001000
 #define CR4_VMXE       0x00002000
 #define CR4_SMXE       0x00004000
 #define CR4_FSGSBASE   0x00010000
@@ -97,7 +103,7 @@
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
 #define CR4_PKE        0x00400000
-#define CR4_RESERVED   CONST64U(0xffffffffff889800)
+#define CR4_RESERVED   CONST64U(0xffffffffff889000)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)

 /*
@@ -201,8 +207,15 @@
 #define EXC_MC           18
 #define EXC_XF           19  // SIMD exception.
 #define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_VC           29  // VMM communication exception (SVM / SEV-ES only).
 #define EXC_SX           30  // Security exception (SVM only).

+/* Bitmap of the exception vectors that have associated error codes. */
+#define EXC_WITH_ERR_CODE_MASK ((1u << EXC_DF) | (1u << EXC_TS) | \
+                                (1u << EXC_NP) | (1u << EXC_SS) | \
+                                (1u << EXC_GP) | (1u << EXC_PF) | \
+                                (1u << EXC_AC))
+
 /*
 * eflag/rflag definitions.
 */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2015-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -121,7 +121,7 @@

 /*
 * Use APIC_MSR_BASEMASK as mask only for reading the MSR_APIC_BASE.
- * Use VMCORESHARED->reservedAPIC as mask when writing MSR_APIC_BASE.
+ * Use guestReservedAPIC as mask when writing MSR_APIC_BASE.
 */
 #define APIC_MSR_BASEMASK     CONST64U(0x0000fffffffff000)
 #define APIC_MSR_X2APIC_ENABLED 0x00000400
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -119,38 +119,45 @@ void __cpuidex(int regs[4], int eax, int ecx);
 #endif

 static INLINE void
-__GET_CPUID(int eax,         // IN
+__GET_CPUID(uint32     eax,  // IN
            CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax)
      : "memory"
   );
 }

 static INLINE void
-__GET_CPUID2(int eax,         // IN
-             int ecx,         // IN
+__GET_CPUID2(uint32 eax,      // IN
+             uint32 ecx,      // IN
             CPUIDRegs *regs) // OUT
 {
   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
+      : "=a" (regs->eax),
+        VM_EBX_OUT(regs->ebx),
+        "=c" (regs->ecx),
+        "=d" (regs->edx)
      : "a" (eax), "c" (ecx)
      : "memory"
   );
 }

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int eax) // IN
+__GET_EAX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx)
      : "a" (eax)
      : "memory", "%ecx", "%edx"
   );
@@ -159,7 +166,7 @@ __GET_EAX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int eax) // IN
+__GET_EBX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ebx;

@@ -174,14 +181,16 @@ __GET_EBX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int eax) // IN
+__GET_ECX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 ecx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=c" (ecx)
      : "a" (eax)
      : "memory", "%edx"
   );
@@ -190,14 +199,16 @@ __GET_ECX_FROM_CPUID(int eax) // IN
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int eax) // IN
+__GET_EDX_FROM_CPUID(uint32 eax) // IN
 {
   uint32 edx;
   uint32 ebx;

   __asm__ __volatile__(
      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=d" (edx)
+      : "=a" (eax),
+        VM_EBX_OUT(ebx),
+        "=d" (edx)
      : "a" (eax)
      : "memory", "%ecx"
   );
@@ -207,7 +218,7 @@ __GET_EDX_FROM_CPUID(int eax) // IN


 static INLINE uint32
-__GET_EAX_FROM_CPUID4(int ecx) // IN
+__GET_EAX_FROM_CPUID4(uint32 ecx) // IN
 {
   uint32 eax;
   uint32 ebx;
@@ -228,7 +239,7 @@ __GET_EAX_FROM_CPUID4(int ecx) // IN
 #elif defined(_MSC_VER) // } {

 static INLINE void
-__GET_CPUID(int input, CPUIDRegs *regs)
+__GET_CPUID(uint32 input, CPUIDRegs *regs)
 {
 #ifdef VM_X86_64
   __cpuid((int *)regs, input);
@@ -262,7 +273,7 @@ __GET_CPUID(int input, CPUIDRegs *regs)
 */

 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __cpuidex((int *)regs, inputEax, inputEcx);
 }
@@ -275,13 +286,13 @@ __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
 */

 extern void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs);
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs);
 #endif // _MSC_VER >= 1500

 #else // VM_X86_64

 static INLINE void
-__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __asm push esi
   __asm push ebx
@@ -305,7 +316,7 @@ __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
 #endif

 static INLINE uint32
-__GET_EAX_FROM_CPUID(int input)
+__GET_EAX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -332,7 +343,7 @@ __GET_EAX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_EBX_FROM_CPUID(int input)
+__GET_EBX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -359,7 +370,7 @@ __GET_EBX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_ECX_FROM_CPUID(int input)
+__GET_ECX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -386,7 +397,7 @@ __GET_ECX_FROM_CPUID(int input)
 }

 static INLINE uint32
-__GET_EDX_FROM_CPUID(int input)
+__GET_EDX_FROM_CPUID(uint32 input)
 {
 #ifdef VM_X86_64
   CPUIDRegs regs;
@@ -420,12 +431,12 @@ __GET_EDX_FROM_CPUID(int input)
 */

 extern uint32
-__GET_EAX_FROM_CPUID4(int inputEcx);
+__GET_EAX_FROM_CPUID4(uint32 inputEcx);

 #else // VM_X86_64

 static INLINE uint32
-__GET_EAX_FROM_CPUID4(int inputEcx)
+__GET_EAX_FROM_CPUID4(uint32 inputEcx)
 {
   uint32 output;

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,43 +97,78 @@ static INLINE uint32 Desc_LongMode(const Descriptor *d) { return d->longmode; }
 static INLINE uint32 Desc_DB(const Descriptor *d)       { return d->DB; }
 static INLINE uint32 Desc_Gran(const Descriptor *d)     { return d->gran; }

-static INLINE uint32 Desc64_Type(const Descriptor64 *d)     { return d->type; }
-static INLINE uint32 Desc64_S(const Descriptor64 *d)        { return d->S; }
-static INLINE uint32 Desc64_DPL(const Descriptor64 *d)      { return d->DPL; }
-static INLINE uint32 Desc64_Present(const Descriptor64 *d)  { return d->present; }
-static INLINE uint32 Desc64_AVL(const Descriptor64 *d)      { return d->AVL; }
-static INLINE uint32 Desc64_Gran(const Descriptor64 *d)     { return d->gran; }
-static INLINE uint32 Desc64_ExtAttrs(const Descriptor64 *d) { return d->ext_attrs; }
+static INLINE uint32
+Desc64_Type(const Descriptor64 *d)
+{
+   return (uint32)d->type;
+}
+
+static INLINE uint32
+Desc64_S(const Descriptor64 *d)
+{
+   return (uint32)d->S;
+}
+
+static INLINE uint32
+Desc64_DPL(const Descriptor64 *d)
+{
+   return (uint32)d->DPL;
+}
+
+static INLINE uint32
+Desc64_Present(const Descriptor64 *d)
+{
+   return (uint32)d->present;
+}
+
+static INLINE uint32
+Desc64_AVL(const Descriptor64 *d)
+{
+   return (uint32)d->AVL;
+}
+
+static INLINE uint32
+Desc64_Gran(const Descriptor64 *d)
+{
+   return (uint32)d->gran;
+}
+
+static INLINE uint32
+Desc64_ExtAttrs(const Descriptor64 *d)
+{
+   return (uint32)d->ext_attrs;
+}

 static INLINE LA32
 Desc_GetBase(const Descriptor *d)
 {
-   return (d->base_hi << 24) | (d->base_mid << 16) | d->base_lo;
+   return ((unsigned)d->base_hi  << 24) |
+          ((unsigned)d->base_mid << 16) | d->base_lo;
 }

 static INLINE LA64
 Desc64_GetBase(const Descriptor64 *d)
 {
-   return ((uint64)d->base_hi << 32)  | 
-          ((uint64)d->base_mid << 24) | (uint32)d->base_lo;
+   return ((uint64)d->base_hi << 32)  |
+          ((uint64)d->base_mid << 24) | (uint64)d->base_lo;
 }

 static INLINE LA32
 Desc64_GetBaseHi(const Descriptor64 *d)
 {
-   return d->base_hi;
+   return (LA32)d->base_hi;
 }

 static INLINE VA32
 Desc_GetLimit(const Descriptor *d)
 {
-   return (d->limit_hi << 16) | d->limit_lo;
+   return ((unsigned)d->limit_hi << 16) | d->limit_lo;
 }

 static INLINE VA32
 Desc64_GetLimit(const Descriptor64 *d)
 {
-   return ((uint32)d->limit_hi << 16) | (uint32)d->limit_lo;
+   return (VA32)(((unsigned)d->limit_hi << 16) | d->limit_lo);
 }

 static INLINE Bool
@@ -310,12 +345,13 @@ static INLINE uint32 DESC_PRESENT(Descriptor d)    { return d.present; }
 *-----------------------------------------------------------------------------
 */

-static INLINE void 
-Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type, 
-                           uint32 DPL, uint32 present, uint32 DB, uint32 gran)
+static INLINE void
+Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit,
+                           uint32 type, uint32 DPL, uint32 present,
+                           uint32 DB, uint32 gran)
 {
-   uint32 baseHi = (uint32) (base >> 32);
-   uint32 baseLo = (uint32) base;
+   uint32 baseHi = (uint32)(base >> 32);
+   uint32 baseLo = (uint32)base;

   /*
    * The first half of a 16-byte descriptor is a valid 8-byte descriptor
@@ -336,15 +372,17 @@ Desc_SetSystemDescriptor64(Descriptor *d, uint64 base, uint32 limit, uint32 type
 *   Given descriptor, return the code/stack size that it specifies.
 *----------------------------------------------------------------------
 */
-static INLINE int
+static INLINE unsigned
 Desc_DBSize(const Descriptor *desc)
 {
   /*
    * Code/stack size is determined by the D/B bit; bit 22 of the 2nd word.
-    * Shift the bit to position 1, mask it out, add 2. Result: 2 or 4.
+    * Shift the bit to position 1, mask it out, add 2.
+    *
+    * Result: 2 or 4.
    */
-   int cSz = ((((const uint32*)desc)[1] >> 21) & 2) + 2;
-   ASSERT(cSz == (Desc_DB(desc) ? 4 : 2));
+   unsigned cSz = ((((const uint32 *)desc)[1] >> 21) & 2) + 2;
+   ASSERT(cSz == (Desc_DB(desc) ? 4U : 2U));
   return cSz;
 }

@@ -526,7 +564,7 @@ typedef struct Gate {
 static INLINE VA
 CallGate_GetOffset(const Gate *cg)
 {
-   return (cg->offset_hi << 16) | cg->offset_lo;
+   return ((unsigned)cg->offset_hi << 16) | cg->offset_lo;
 }

 static INLINE void
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
 *
 *********************************************************/

-/* 
+/*
 * x86msr.h --
 *
 *      MSR number definitions.
@@ -24,6 +24,7 @@

 #ifndef _X86MSR_H_
 #define _X86MSR_H_
+#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX

@@ -42,6 +43,7 @@
 extern "C" {
 #endif

+#define SIZEOF_X86_MSR     8

 /*
 * Results of calling rdmsr(msrNum) on all logical processors.
@@ -58,11 +60,8 @@ struct MSRReply {
    * we use it to correlate the replies of multiple queries.
    */
   uint64 tag;              // OUT
-
   uint64 msrVal;           // OUT
-
   uint8  implemented;      // OUT
-
   uint8  _pad[7];
 }
 #include "vmware_pack_end.h"
@@ -82,6 +81,7 @@ MSRQuery;
 #define MSR_PLATFORM_ID       0x00000017
 #define MSR_APIC_BASE         0x0000001b
 #define MSR_SMI_COUNT         0x00000034 // Intel Nehalem Family
+#define MSR_CORE_THREAD_COUNT 0x00000035 // Intel Nehalem Family +
 #define MSR_FEATCTL           0x0000003a
 #define MSR_TSC_ADJUST        0x0000003b
 #define MSR_SPEC_CTRL         0x00000048
@@ -118,26 +118,36 @@ MSRQuery;
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
 #define MSR_ARCH_CAPABILITIES_RSBA                (1ULL << 2)
+#define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
+#define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)
+#define MSR_ARCH_CAPABILITIES_MDS_NO              (1ULL << 5)

 #define MSR_FLUSH_CMD                        0x10b
-#define MSR_FLUSH_CMD_FLUSH_L1D              1
+#define MSR_FLUSH_CMD_FLUSH_L1D                   (1ULL << 0)

 #define MSR_SPEC_CTRL_IBRS                        (1UL << 0)
 #define MSR_SPEC_CTRL_STIBP                       (1UL << 1)
+#define MSR_SPEC_CTRL_SSBD                        (1UL << 2)
+
 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

+
+#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
+#endif

 /* Intel Core Architecture and later: use only architected counters. */
-#define IA32_MSR_PERF_CAPABILITIES                0x345
-#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT        0
-#define MSR_PERF_CAPABILITIES_LBRFMT_MASK         0x3f
-#define MSR_PERF_CAPABILITIES_PEBSTRAP            (1u << 6)
-#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS    (1u << 7)
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT 8
-#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK  0xf
-#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM    (1u << 12)
-#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES   (1u << 13)
+#define IA32_MSR_PERF_CAPABILITIES                   0x345
+#define MSR_PERF_CAPABILITIES_LBRFMT_SHIFT           0
+#define MSR_PERF_CAPABILITIES_LBRFMT_MASK            0x3f
+#define MSR_PERF_CAPABILITIES_PEBSTRAP               (1u << 6)
+#define MSR_PERF_CAPABILITIES_PEBSSAVEARCHREGS       (1u << 7)
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_SHIFT    8
+#define MSR_PERF_CAPABILITIES_PEBSRECORDFMT_MASK     0xf
+#define MSR_PERF_CAPABILITIES_FREEZE_WHILE_SMM       (1u << 12)
+#define MSR_PERF_CAPABILITIES_FULL_WIDTH_WRITES      (1u << 13)
+#define MSR_PERF_CAPABILITIES_PEBS_ADAPTIVE_DATA     (1u << 14)
+#define MSR_PERF_CAPABILITIES_PERF_METRICS_AVAILABLE (1u << 15)

 #define IA32_MSR_PEBS_ENABLE                      0x3f1

@@ -264,6 +274,8 @@ typedef enum {

 #define MSR_XSS              0x00000da0  // Extended Supervisor State Mask

+#define MSR_MPX_LAX          0x00001000  // MPX Linear Address Extension
+
 /* RTIT MSRs */
 #define MSR_RTIT_CTL              0x00000570
 #define MSR_RTIT_STATUS           0x00000571
@@ -388,6 +400,17 @@ typedef enum {
 #define MSR_SGX_SVN_STATUS_SINIT_SVN     CONST64U(0xff0000)
 #define MSR_SGX_SVN_STATUS_RSVD          CONST64U(0xffffffffff00fffe)

+/*
+ * SGX Flexible Launch Control MSRs.
+ * These MSRs store hash of Launch Enclave's public key.
+ */
+#define MSR_SGXLEPUBKEYHASH0  0x0000008c
+#define MSR_SGXLEPUBKEYHASH1  0x0000008d
+#define MSR_SGXLEPUBKEYHASH2  0x0000008e
+#define MSR_SGXLEPUBKEYHASH3  0x0000008f
+
+#define NUM_SGXLEPUBKEYHASH_MSRs (4)
+
 /* MSR_CR_PAT power-on value */
 #define MSR_CR_PAT_DEFAULT   0x0007040600070406ULL

@@ -426,6 +449,7 @@ typedef enum {
 #define MSR_FEATCTL_VMXE     0x00000004
 #define MSR_FEATCTL_SENTERP  0x00007F00
 #define MSR_FEATCTL_SENTERE  0x00008000
+#define MSR_FEATCTL_FLCE     0x00020000
 #define MSR_FEATCTL_SGXE     0x00040000
 #define MSR_FEATCTL_LMCE     0x00100000

@@ -479,6 +503,8 @@ typedef enum {
 #define MSR_GH_COFVID_CONTROL    0xc0010070  // COFVID Control Register
 #define MSR_GH_COFVID_STATUS     0xc0010071  // COFVID Status Register

+#define MSR_AMD_MCA_INTR_CFG     0xc0000410  // MCA Interrupt Configuration
+
 /* SVM related MSRs */
 #define MSR_VM_CR                  0xc0010114
 #define MSR_IGNNE                  0xc0010115
@@ -492,9 +518,28 @@ typedef enum {
 #define MSR_VM_CR_SVME_DISABLE     0x0000000000000010ULL
 #define MSR_VM_CR_RESERVED         0xffffffffffffffe0ULL

+/* SEV related MSRs. */
+#define MSR_VMPAGE_FLUSH           0xc001011e
+#define MSR_GHCB_PA                0xc0010130
+#define MSR_GHCB_PA_SEVINFO_HV              1
+#define MSR_GHCB_PA_SEVINFO_REQ             2
+#define MSR_GHCB_PA_AP_JUMP_TABLE           3
+#define MSR_SEV_STATUS             0xc0010131
+
+#define MSR_SEV_STATUS_SEV_EN      0x0000000000000001ULL // SEV is enabled
+#define MSR_SEV_STATUS_SEV_ES_EN   0x0000000000000002ULL // SEV-ES is enabled
+
 #define MSR_AMD_DE_CFG           0xc0011029  // Decode configuration
 #define MSR_AMD_DE_CFG_BIT1      (1ULL<<1)

+#define MSR_AMD_LS_CFG           0xc0011020  // load-store configuration
+#define MSR_AMD_LS_CFG_SSBD_BULLDOZER (1ULL<<54)  // BD family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_KYOTO     (1ULL<<33)  // Kyoto family non-arch SSBD
+#define MSR_AMD_LS_CFG_SSBD_ZEN       (1ULL<<10)  // Zen family non-arch SSBD
+
+#define MSR_AMD_VIRT_SPEC_CTRL   0xc001011f  // Virtual speculation control
+#define MSR_AMD_VIRT_SPEC_CTRL_SSBD   (1ULL<<2)   // Virtual SSBD
+
 /* Syscall/Sysret related MSRs (x86_64) */
 #define MSR_STAR             0xc0000081 // Also present on Athlons.
 #define MSR_LSTAR            0xc0000082
@@ -576,6 +621,8 @@ typedef enum {
 #define MSR_HYPERV_HYPERCALL_EN                  1ULL
 #define MSR_HYPERV_REFERENCE_TSC_EN              1ULL
 #define MSR_HYPERV_VP_ASSIST_EN                  1ULL
+#define MSR_HYPERV_SIEF_EN                       1ULL
+#define MSR_HYPERV_SIM_EN                        1ULL

 #define MSR_HYPERV_GUESTOSID_VENDOR_SHIFT        48
 #define MSR_HYPERV_GUESTOSID_VENDOR_MASK         0xfULL
@@ -585,6 +632,9 @@ typedef enum {
 #define MSR_HYPERV_GUESTOSID_OS_MASK             0xfULL
 #define MSR_HYPERV_GUESTOSID_OS_WINNT_DERIVATIVE 4ULL

+/* MSR for forcing RTM abort to recover PMC3 (see PR 2333817) */
+#define MSR_TSX_FORCE_ABORT                      0x0000010f
+#define MSR_TSX_FORCE_ABORT_RTM_BIT_INDEX        0

 /*
 * MTRR bit description
@@ -601,6 +651,8 @@ typedef enum {

 #define MTRR_MASK_VALID       0x800

+typedef unsigned char MTRRType;
+
 #define MTRR_TYPE_UC          0
 #define MTRR_TYPE_WC          1
 #define MTRR_TYPE_WT          4
@@ -610,6 +662,13 @@ typedef enum {
 /* UC- is equivalent to UC, except that the MTRR values take precedence */
 #define MTRR_TYPE_UCM         7

+/*
+ * This value is marked as reserved in the Intel manual. We use it to
+ * specify that type is unknown as it is very unlikely that Intel will
+ * use this value. Note that linux is taking the same liberty.
+ */
+#define MTRR_TYPE_UNKNOW     0xff
+
 /*
 * PERF_STATUS bits
 */
@@ -628,19 +687,28 @@ typedef enum {
 /*
 * MISC_FEATURES_ENABLES bits
 */
+#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
+#endif




+// Platform Quality of Service (PQM) MSRs
+#define MSR_INTEL_PQM_EVTSEL    0xc8d
+#define MSR_INTEL_PQM_CTR       0xc8e
+#define MSR_INTEL_PQM_ASSOC     0xc8f

-/*
- * Platform Quality of Service
- */
+ // Platform Quality Enforcement (PQE) MSRs
+#define MSR_INTEL_PQE_CLOS_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_MASK_MAX      0xd8f

-#define MSR_INTEL_QOS_EVTSEL    0xc8d
-#define MSR_INTEL_QOS_CTR       0xc8e
-#define MSR_INTEL_QOS_ASSOC     0xc8f
+#define MSR_INTEL_PQE_CLOS_L3_MASK_BASE     0xc90
+#define MSR_INTEL_PQE_CLOS_L3_MASK_MAX      0xd0f
+#define MSR_INTEL_PQE_CLOS_L2_MASK_BASE     0xd10
+#define MSR_INTEL_PQE_CLOS_L2_MASK_MAX      0xd4f

 static INLINE uint32
 X86MSR_SysCallEIP(uint64 star)
@@ -663,6 +731,126 @@ X86MSR_SysRetCS(uint64 star)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * X86MSR_{G,S}etMSR --
+ *
+ *      Get or set an MSR in hardware. The interface should remain
+ *      agnostic to the underlying compiler and architecture.
+ *
+ *----------------------------------------------------------------------
+ */
+
+#ifdef __GNUC__
+/*
+ * Checked against the Intel manual and GCC --hpreg
+ * volatile because the msr can change without the compiler knowing it
+ * (when we use wrmsr).
+ */
+#ifdef VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr; shlq $32, %%rdx; orq %%rdx, %%rax"
+      : "=a" (msr)
+      : "c" (cx)
+      : "%rdx"
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "a" ((uint32) value),
+        "d" ((uint32)(value >> 32)),
+        "c" (cx)
+    );
+}
+#else // __GNUC__ && !VM_X86_64
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   uint64 msr;
+   __asm__ __volatile__(
+      "rdmsr"
+      : "=A" (msr)
+      : "c"  (cx)
+   );
+
+   return msr;
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __asm__ __volatile__(
+      "wrmsr"
+      : /* no outputs */
+      : "A" (value),
+        "c" (cx)
+    );
+}
+#endif
+#elif defined _MSC_VER // !__GNUC__ && _MSC_VER
+#ifdef _WIN64
+unsigned __int64  __readmsr(unsigned long);
+void              __writemsr(unsigned long, unsigned __int64);
+#pragma intrinsic(__readmsr, __writemsr)
+static INLINE uint64
+X86MSR_GetMSR(uint32 cx)
+{
+   return __readmsr((unsigned long)cx);
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 cx, uint64 value)
+{
+   __writemsr((unsigned long)(cx), (unsigned __int64)(value));
+}
+#else // !__GNUC__ && _MSC_VER && !_WIN64
+#pragma warning( disable : 4035)
+static INLINE uint64
+X86MSR_GetMSR(uint32 input)
+{
+   __asm push ecx
+   __asm mov  ecx, input
+   /* 0x0f 0x32 -> edx:eax = rdmsr[ecx] */
+   __asm _emit 0x0f __asm _emit 0x32
+   __asm pop ecx
+}
+
+static INLINE void
+X86MSR_SetMSR(uint32 input, uint64 value)
+{
+      uint32 hival = (uint32)((value) >> 32);
+      uint32 loval = (uint32)value;
+      __asm push edx
+      __asm push ecx
+      __asm push eax
+      __asm mov  eax, loval
+      __asm mov  edx, hival
+      __asm mov  ecx, input
+      /* 0x0f 0x30 -> wrmsr[ecx] = edx:eax */
+      __asm _emit 0x0f __asm _emit 0x30
+      __asm pop  eax
+      __asm pop  ecx
+      __asm pop  edx
+}
+#pragma warning (default: 4035)
+#endif
+#else // !__GNUC__ && !_MSC_VER
+#error No compiler defined for RDMSR/WRMSR.
+#endif
+
+
 #if defined __cplusplus
 }
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016,2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,7 +52,6 @@
 #define LM_AVAIL_MASK         (CONST64(0x7) << LM_AVAIL_SHIFT)
 #define LM_FLAGS_MASK         CONST64(0x80000000000001ff)
 #define LM_CR3_FLAGS_MASK     CONST64(0x18)
-#define LM_L4_L3_RSVD_MASK    CONST64(0x180)
 #define LM_L3_1G_RSVD_MASK    CONST64(0x3fffe000)

 #define LM_MAKE_CR3(_mpfn, _flags) \
@@ -65,12 +64,62 @@
                    ((uint64)(_flags) & LM_FLAGS_MASK))

 #define LM_MAKE_PDE(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L5E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L4E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L3E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L2E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
 #define LM_MAKE_L1E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)


+/*
+ *----------------------------------------------------------------------
+ *
+ * LMPTEIsSafe --
+ *
+ *    A shadow PTE is considered "safe" if any of the following conditions are
+ *    met:
+ *
+ *      a) It is not terminal (i.e., present with no reserved bits set)
+ *      b) Terminal, but with MPN and PS fields set to zero
+ *      c) Terminal, but with MPN field specifying an uncachable page
+ *
+ *    In practice, for condition c), we require that bits 45:43 of the EPTE are
+ *    set to b'111.  The position of these bits is undocumented and not
+ *    architectural; they are truly magic.
+ *
+ *----------------------------------------------------------------------
+ */
+#ifdef VMX86_DEBUG
+
+#define LM_SAFE_BITS_MASK (MASK64(3) << 43)
+#define LM_SAFE_BITS(_v)  ((_v) & LM_SAFE_BITS_MASK)
+
+static INLINE Bool
+LMPTEIsTerminal(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   uint64 rsvd2m = (MASK(PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvd1g = (MASK(2 * PT_LEVEL_SHIFT) << PAGE_SHIFT) & ~PTE_LARGE_PAT;
+   uint64 rsvdl4 = PTE_PS;
+   uint64 rsvd = physMask & MASK64(52);
+
+   return (!PTE_PRESENT(pte) ||
+           (pte & LM_PTE_PFN_MASK) & rsvd) != 0 ||
+           ((pte & PTE_PS) != 0 &&
+            ((level == PT_LEVEL_2 && (pte & rsvd2m) != 0) ||
+             (level == PT_LEVEL_3 && (pte & rsvd1g) != 0))) ||
+           (level == PT_LEVEL_4 && (pte & rsvdl4) != 0);
+}
+
+static INLINE Bool
+LMPTEIsSafe(VM_PAE_PTE pte, PT_Level level, uint64 physMask)
+{
+   MPN mpn = (pte & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT;
+   Bool safe = !LMPTEIsTerminal(pte, level, physMask) ||
+               (mpn == 0 && !PTE_LARGEPAGE(pte)) ||
+               LM_SAFE_BITS(pte) == LM_SAFE_BITS_MASK;
+   return safe;
+}
+#endif /* VMX86_DEBUG */


 /*
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,7 +54,6 @@
 #define PTE_PAT         0x00000080
 #define PTE_G           0x00000100
 #define PTE_LARGE_PAT   0x00001000
-#define PTE_RSVD        CONST64(0x0008000000000000)
 #define PTE_NX          CONST64(0x8000000000000000)
 #define PTE_NX_SHIFT    63

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,21 +65,21 @@ typedef struct Descriptor {
 typedef 
 #include "vmware_pack_begin.h"
 struct Descriptor64 {
-   unsigned   limit_lo  : 16;   // Limit bits 15-0.
-   unsigned   base_lo   : 24;   // Base bits  23-0.
-   unsigned   type      : 4;
-   unsigned   S         : 1;
-   unsigned   DPL       : 2;
-   unsigned   present   : 1;
-   unsigned   limit_hi  : 4;    // Limit bits 19-16.
-   unsigned   AVL       : 1;
-   unsigned   reserved0 : 2;
-   unsigned   gran      : 1;
-   unsigned   base_mid  : 8;    // Base bits 31-24.
-   unsigned   base_hi   : 32;   // Base bits 63-32.
-   unsigned   reserved1 : 8;
-   unsigned   ext_attrs : 5;
-   unsigned   reserved2 : 19;
+   uint64   limit_lo  : 16;   // Limit bits 15-0.
+   uint64   base_lo   : 24;   // Base bits  23-0.
+   uint64   type      : 4;
+   uint64   S         : 1;
+   uint64   DPL       : 2;
+   uint64   present   : 1;
+   uint64   limit_hi  : 4;    // Limit bits 19-16.
+   uint64   AVL       : 1;
+   uint64   reserved0 : 2;
+   uint64   gran      : 1;
+   uint64   base_mid  : 8;    // Base bits 31-24.
+   uint64   base_hi   : 32;   // Base bits 63-32.
+   uint64   reserved1 : 8;
+   uint64   ext_attrs : 5;
+   uint64   reserved2 : 19;
 }
 #include "vmware_pack_end.h"
 Descriptor64;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -47,8 +47,10 @@
 #endif
 #include "x86cpuid_asm.h"

-#define SVM_VMCB_IO_BITMAP_SIZE    (3 * PAGE_SIZE)
-#define SVM_VMCB_MSR_BITMAP_SIZE   (2 * PAGE_SIZE)
+#define SVM_VMCB_IO_BITMAP_PAGES   (3)
+#define SVM_VMCB_IO_BITMAP_SIZE    PAGES_2_BYTES(SVM_VMCB_IO_BITMAP_PAGES)
+#define SVM_VMCB_MSR_BITMAP_PAGES  (2)
+#define SVM_VMCB_MSR_BITMAP_SIZE   PAGES_2_BYTES(SVM_VMCB_MSR_BITMAP_PAGES)

 /* Exit controls for the CR/DR access and hardware exceptions */
 #define SVM_CR_RD_CTL(num)         (0x1 << (num))
@@ -63,53 +65,56 @@
 #define SVM_XCP_CTL_ALL            0xffffffff

 /* Execution intercept controls */
-/* VMCB.exitCtl */
-#define SVM_VMCB_EXEC_CTL_INTR        0x0000000000000001ULL
-#define SVM_VMCB_EXEC_CTL_NMI         0x0000000000000002ULL
-#define SVM_VMCB_EXEC_CTL_SMI         0x0000000000000004ULL
-#define SVM_VMCB_EXEC_CTL_INIT        0x0000000000000008ULL
-#define SVM_VMCB_EXEC_CTL_VINTR       0x0000000000000010ULL
-#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR  0x0000000000000020ULL
-#define SVM_VMCB_EXEC_CTL_SIDT        0x0000000000000040ULL
-#define SVM_VMCB_EXEC_CTL_SGDT        0x0000000000000080ULL
-#define SVM_VMCB_EXEC_CTL_SLDT        0x0000000000000100ULL
-#define SVM_VMCB_EXEC_CTL_STR         0x0000000000000200ULL
-#define SVM_VMCB_EXEC_CTL_LIDT        0x0000000000000400ULL
-#define SVM_VMCB_EXEC_CTL_LGDT        0x0000000000000800ULL
-#define SVM_VMCB_EXEC_CTL_LLDT        0x0000000000001000ULL
-#define SVM_VMCB_EXEC_CTL_LTR         0x0000000000002000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSC       0x0000000000004000ULL
-#define SVM_VMCB_EXEC_CTL_RDPMC       0x0000000000008000ULL
-#define SVM_VMCB_EXEC_CTL_PUSHF       0x0000000000010000ULL
-#define SVM_VMCB_EXEC_CTL_POPF        0x0000000000020000ULL
-#define SVM_VMCB_EXEC_CTL_CPUID       0x0000000000040000ULL
-#define SVM_VMCB_EXEC_CTL_RSM         0x0000000000080000ULL
-#define SVM_VMCB_EXEC_CTL_IRET        0x0000000000100000ULL
-#define SVM_VMCB_EXEC_CTL_SWINT       0x0000000000200000ULL
-#define SVM_VMCB_EXEC_CTL_INVD        0x0000000000400000ULL
-#define SVM_VMCB_EXEC_CTL_PAUSE       0x0000000000800000ULL
-#define SVM_VMCB_EXEC_CTL_HLT         0x0000000001000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPG      0x0000000002000000ULL
-#define SVM_VMCB_EXEC_CTL_INVLPGA     0x0000000004000000ULL
-#define SVM_VMCB_EXEC_CTL_IOIO        0x0000000008000000ULL
-#define SVM_VMCB_EXEC_CTL_MSR         0x0000000010000000ULL
-#define SVM_VMCB_EXEC_CTL_TS          0x0000000020000000ULL
-#define SVM_VMCB_EXEC_CTL_FERR_FRZ    0x0000000040000000ULL
-#define SVM_VMCB_EXEC_CTL_SHUTDOWN    0x0000000080000000ULL
-#define SVM_VMCB_EXEC_CTL_VMRUN       0x0000000100000000ULL
-#define SVM_VMCB_EXEC_CTL_VMMCALL     0x0000000200000000ULL
-#define SVM_VMCB_EXEC_CTL_VMLOAD      0x0000000400000000ULL
-#define SVM_VMCB_EXEC_CTL_VMSAVE      0x0000000800000000ULL
-#define SVM_VMCB_EXEC_CTL_STGI        0x0000001000000000ULL
-#define SVM_VMCB_EXEC_CTL_CLGI        0x0000002000000000ULL
-#define SVM_VMCB_EXEC_CTL_SKINIT      0x0000004000000000ULL
-#define SVM_VMCB_EXEC_CTL_RDTSCP      0x0000008000000000ULL
-#define SVM_VMCB_EXEC_CTL_ICEBP       0x0000010000000000ULL
-#define SVM_VMCB_EXEC_CTL_WBINVD      0x0000020000000000ULL
-#define SVM_VMCB_EXEC_CTL_MONITOR     0x0000040000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT       0x0000080000000000ULL
-#define SVM_VMCB_EXEC_CTL_MWAIT_COND  0x0000100000000000ULL
-#define SVM_VMCB_EXEC_CTL_XSETBV      0x0000200000000000ULL
+/* VMCB.execCtl */
+#define SVM_VMCB_EXEC_CTL_INTR           0x0000000000000001ULL
+#define SVM_VMCB_EXEC_CTL_NMI            0x0000000000000002ULL
+#define SVM_VMCB_EXEC_CTL_SMI            0x0000000000000004ULL
+#define SVM_VMCB_EXEC_CTL_INIT           0x0000000000000008ULL
+#define SVM_VMCB_EXEC_CTL_VINTR          0x0000000000000010ULL
+#define SVM_VMCB_EXEC_CTL_CR0_SEL_WR     0x0000000000000020ULL
+#define SVM_VMCB_EXEC_CTL_SIDT           0x0000000000000040ULL
+#define SVM_VMCB_EXEC_CTL_SGDT           0x0000000000000080ULL
+#define SVM_VMCB_EXEC_CTL_SLDT           0x0000000000000100ULL
+#define SVM_VMCB_EXEC_CTL_STR            0x0000000000000200ULL
+#define SVM_VMCB_EXEC_CTL_LIDT           0x0000000000000400ULL
+#define SVM_VMCB_EXEC_CTL_LGDT           0x0000000000000800ULL
+#define SVM_VMCB_EXEC_CTL_LLDT           0x0000000000001000ULL
+#define SVM_VMCB_EXEC_CTL_LTR            0x0000000000002000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSC          0x0000000000004000ULL
+#define SVM_VMCB_EXEC_CTL_RDPMC          0x0000000000008000ULL
+#define SVM_VMCB_EXEC_CTL_PUSHF          0x0000000000010000ULL
+#define SVM_VMCB_EXEC_CTL_POPF           0x0000000000020000ULL
+#define SVM_VMCB_EXEC_CTL_CPUID          0x0000000000040000ULL
+#define SVM_VMCB_EXEC_CTL_RSM            0x0000000000080000ULL
+#define SVM_VMCB_EXEC_CTL_IRET           0x0000000000100000ULL
+#define SVM_VMCB_EXEC_CTL_SWINT          0x0000000000200000ULL
+#define SVM_VMCB_EXEC_CTL_INVD           0x0000000000400000ULL
+#define SVM_VMCB_EXEC_CTL_PAUSE          0x0000000000800000ULL
+#define SVM_VMCB_EXEC_CTL_HLT            0x0000000001000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPG         0x0000000002000000ULL
+#define SVM_VMCB_EXEC_CTL_INVLPGA        0x0000000004000000ULL
+#define SVM_VMCB_EXEC_CTL_IOIO           0x0000000008000000ULL
+#define SVM_VMCB_EXEC_CTL_MSR            0x0000000010000000ULL
+#define SVM_VMCB_EXEC_CTL_TS             0x0000000020000000ULL
+#define SVM_VMCB_EXEC_CTL_FERR_FRZ       0x0000000040000000ULL
+#define SVM_VMCB_EXEC_CTL_SHUTDOWN       0x0000000080000000ULL
+#define SVM_VMCB_EXEC_CTL_VMRUN          0x0000000100000000ULL
+#define SVM_VMCB_EXEC_CTL_VMMCALL        0x0000000200000000ULL
+#define SVM_VMCB_EXEC_CTL_VMLOAD         0x0000000400000000ULL
+#define SVM_VMCB_EXEC_CTL_VMSAVE         0x0000000800000000ULL
+#define SVM_VMCB_EXEC_CTL_STGI           0x0000001000000000ULL
+#define SVM_VMCB_EXEC_CTL_CLGI           0x0000002000000000ULL
+#define SVM_VMCB_EXEC_CTL_SKINIT         0x0000004000000000ULL
+#define SVM_VMCB_EXEC_CTL_RDTSCP         0x0000008000000000ULL
+#define SVM_VMCB_EXEC_CTL_ICEBP          0x0000010000000000ULL
+#define SVM_VMCB_EXEC_CTL_WBINVD         0x0000020000000000ULL
+#define SVM_VMCB_EXEC_CTL_MONITOR        0x0000040000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT          0x0000080000000000ULL
+#define SVM_VMCB_EXEC_CTL_MWAIT_COND     0x0000100000000000ULL
+#define SVM_VMCB_EXEC_CTL_XSETBV         0x0000200000000000ULL
+#define SVM_VMCB_EXEC_CTL_EFER_WR_TRAP   0x0000800000000000ULL
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP(n) (0x0001000000000000ULL << (n))
+#define SVM_VMCB_EXEC_CTL_CR_WR_TRAP_ALL 0xffff000000000000ULL

 /* VMCB.tlbCtl */
 #define SVM_VMCB_TLB_CTL_FLUSH              0x01
@@ -130,12 +135,16 @@

 /* VMCB.intrState */
 #define SVM_VMCB_INTRSTATE_INTR_SHADOW     (1 << 0)
+#define SVM_VMCB_INTRSTATE_GUEST_IF        (1 << 1)

 /* VMCB.nptCtl */
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)
+#define SVM_VMCB_NPTCTL_SEV_ENABLE         (1 << 1)
+#define SVM_VMCB_NPTCTL_SEV_ES_ENABLE      (1 << 2)

-/* VMCB.debugCtl */
-#define SVM_VMCB_DEBUGCTL_LBR_ENABLE       (1 << 0)
+/* VMCB.virtExt */
+#define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
+#define SVM_VMCB_VIRTEXT_V_VMSAVE_VMLOAD   (1 << 1)

 /* VMCB.clean */
 #define SVMCLEAN       \
@@ -233,9 +242,16 @@
 #define SVM_EXITCODE_MWAIT                139
 #define SVM_EXITCODE_MWAIT_COND           140
 #define SVM_EXITCODE_XSETBV               141
+#define SVM_EXITCODE_EFER_WRITE_TRAP      143
+#define SVM_EXITCODE_CR_WRITE_TRAP(n)    (144 + (n))
 #define SVM_EXITCODE_NPF                 1024
 #define SVM_EXITCODE_AVIC_INCOMPLETE_IPI 1025
 #define SVM_EXITCODE_AVIC_NOACCEL        1026
+#define SVM_EXITCODE_VMGEXIT             1027
+#define SVM_EXITCODE_MMIO_READ           0x80000001   // SW only
+#define SVM_EXITCODE_MMIO_WRITE          0x80000002   // SW only
+#define SVM_EXITCODE_NMI_COMPLETE        0x80000003   // SW only
+#define SVM_EXITCODE_UNSUPPORTED         0x8000FFFF   // SW only
 #define SVM_EXITCODE_INVALID             (-1ULL)

 /* ExitInfo1 for I/O exits */
@@ -259,6 +275,9 @@
 #define SVM_IOEXIT_MBZ           0x00000002
 #define SVM_IOEXIT_SBZ           0x0000e000

+/* ExitInfo1 for MSR exits */
+#define SVM_MSREXIT_WRMSR        0x00000001
+
 /* ExitInfo1 for CR exits */
 #define SVM_CREXIT_MOVCR         (1ull << 63)
 #define SVM_CREXIT_GPR_MASK      0xf
@@ -330,7 +349,7 @@ enum {

 #undef CLEANBIT
 };
-   
+
 static INLINE uint64
 SVM_ExecCtlBit(uint32 exitCode)
 {
@@ -389,7 +408,7 @@ SVM_ExecCtlBit(uint32 exitCode)
 *----------------------------------------------------------------------
 *
 * SVM_EnabledFromFeatures --
- * 
+ *
 *  Returns TRUE if SVM is enabled in the given VM control MSR bits.
 *
 *----------------------------------------------------------------------
@@ -405,7 +424,7 @@ SVM_EnabledFromFeatures(uint64 vmCR)
 *----------------------------------------------------------------------
 *
 * SVM_LockedFromFeatures --
- * 
+ *
 *  Returns TRUE if SVM is locked in the given VM control MSR bits.
 *
 *----------------------------------------------------------------------
@@ -429,7 +448,7 @@ SVM_LockedFromFeatures(uint64 vmCR)
 static INLINE Bool
 SVM_EnabledCPU(void)
 {
-   return SVM_EnabledFromFeatures(__GET_MSR(MSR_VM_CR));
+   return SVM_EnabledFromFeatures(X86MSR_GetMSR(MSR_VM_CR));
 }
 #endif /* } !defined(USERLEVEL) */

@@ -461,32 +480,18 @@ SVM_CapableCPU(void)
 *   SVM-enabled monitor.  This function assumes that the processor is
 *   SVM_Capable().  We only support CPUs that populate the exitIntInfo
 *   field of the VMCB when IDT vectoring is interrupted by a task switch
- *   intercept.  That behavior was first introduced with Family 10H.
+ *   intercept.  That behavior was first introduced with AMD Family 10H.
+ *   As Hygon's Dhyana is a descendant of AMD's Zen microarchitecture
+ *   (Family 17H), SVM is always supported.
 *----------------------------------------------------------------------
 */
 static INLINE Bool
-SVM_SupportedVersion(uint32 version)
+SVM_SupportedVersion(CpuidVendor vendor, uint32 version)
 {
-   return CPUID_EFFECTIVE_FAMILY(version) >= CPUID_FAMILY_K8L;
+   return (vendor == CPUID_VENDOR_AMD &&
+           CPUID_EFFECTIVE_FAMILY(version) >= CPUID_FAMILY_K8L) ||
+          vendor == CPUID_VENDOR_HYGON;
 }
-
-
-/*
- *
- *----------------------------------------------------------------------
- * SVM_SupportedCPU --
- *
- *   Wrapper to call SVM_SupportedVersion() with the right
- *   parameter(s) for the current CPU.
- *----------------------------------------------------------------------
- */
-static INLINE Bool
-SVM_SupportedCPU(void)
-{
-   return SVM_SupportedVersion(__GET_EAX_FROM_CPUID(1));
-}
-
-
 #endif /* VMM */

 #endif /* _X86SVM_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -62,13 +62,6 @@ typedef uint64 VM_EPTE;
 * Registers
 */

-typedef union SharedReg64 {
-   Reg8  reg8[2];
-   Reg16 reg16;
-   Reg32 reg32;
-   Reg64 reg64;
-} SharedReg64;
-
 typedef union SharedUReg64 {
   UReg8  ureg8[2];
   UReg16 ureg16;
@@ -86,11 +79,12 @@ typedef uint16 Selector;
 */


+#define RSP_NUM_ENTRIES 3
 #define IST_NUM_ENTRIES 8
 #pragma pack(push, 1)
 typedef struct Task64 {
   uint32     reserved0;
-   uint64     rsp[3];   // Stacks for CPL 0-2.
+   uint64     rsp[RSP_NUM_ENTRIES];   // Stacks for CPL 0-2.
   uint64     ist[IST_NUM_ENTRIES];   // ist[0] is reserved.
   uint64     reserved1;
   uint16     reserved2;
@@ -177,11 +171,7 @@ typedef Task32 Task;

 #pragma pack(push, 1)
 typedef struct {
-#if defined(VMM) || defined(COREQUERY)
-   uint64 va;
-#else
-   uint32 va;
-#endif
+   uint64   va;
   Selector seg;
 } FarPtr;
 #pragma pack(pop)
@@ -467,4 +457,12 @@ typedef struct ExcFrame64ForCopy {
 } ExcFrame64ForCopy;
 #pragma pack(pop)

+/*
+ * Layout of the stack for performing a 64 bit lret instruction.
+ */
+typedef struct LretFrame64 {
+   uint64 rip;
+   uint64 cs;
+} LretFrame64;
+
 #endif // ifndef _X86TYPES_H_
@@ -1,6 +1,5 @@
-
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,6 +30,7 @@ typedef enum {
   CPUID_VENDOR_AMD,
   CPUID_VENDOR_CYRIX,
   CPUID_VENDOR_VIA,
+   CPUID_VENDOR_HYGON,
   CPUID_NUM_VENDORS
 } CpuidVendor;

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -92,6 +92,12 @@ VT_EXIT(RDSEED,               61)
 VT_EXIT(PML_LOGFULL,          62)
 VT_EXIT(XSAVES,               63)
 VT_EXIT(XRSTORS,              64)
-
+VT_EXIT(VMEXIT65,             65)
+VT_EXIT(VMEXIT66,             66)
+VT_EXIT(VMEXIT67,             67)
+VT_EXIT(VMEXIT68,             68)
+VT_EXIT(VMEXIT69,             69)
+VT_EXIT(ENCLV,                70)
+VT_EXIT(SGX_CONFLICT,         71)
 /* Bump this up if you add an exit reason. */
-#define VT_NUM_EXIT_REASONS   65
+#define VT_NUM_EXIT_REASONS   72
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -76,7 +76,8 @@
 #endif

 /*
- * VMCS_FIELD(_name, _val, _size, _type, _index, _cache, _vvt, _access)
+ * VMCS_FIELD(_name, _val, _size, _type, _index, _cache, _vvt, _access,
+ *            _sticky)
 *    _type is one of:
 *       _TC -- control
 *       _TG -- guest-state
@@ -97,16 +98,20 @@
 *       _NA -- no access
 *       _RW -- read/write access
 *       _RO -- read-only access
+ *   _sticky is one of:
+ *       _S  -- sticky
+ *       _NS -- non-sticky
 *
+ * Fields marked sticky in the current VMCS, change only via VMWRITEs.
 */


 VMCS_SET_START(16)
 /* 16-bit control fields. */
 VMCS_GROUP_START(16, CTL)
-VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA)
-VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA)
-VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
 VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
 VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
@@ -176,16 +181,16 @@ VMCS_GROUP_END(16, DATA)

 /* 16-bit guest state. */
 VMCS_GROUP_START(16, GUEST)
-VMCS_FIELD(ES,                      0x0800, _S16, _TG,  0, _NC,  _V, _RW)
-VMCS_FIELD(CS,                      0x0802, _S16, _TG,  1, _NC,  _V, _RW)
-VMCS_FIELD(SS,                      0x0804, _S16, _TG,  2, _NC,  _V, _RW)
-VMCS_FIELD(DS,                      0x0806, _S16, _TG,  3, _NC,  _V, _RW)
-VMCS_FIELD(FS,                      0x0808, _S16, _TG,  4, _NC,  _V, _RW)
-VMCS_FIELD(GS,                      0x080A, _S16, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(LDTR,                    0x080C, _S16, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(TR,                      0x080E, _S16, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(INTR_STATUS,             0x0810, _S16, _TG,  8, _NC, _NV, _NA)
-VMCS_FIELD(PML_INDEX,               0x0812, _S16, _TG,  9, _NC, _NV, _NA)
+VMCS_FIELD(ES,                      0x0800, _S16, _TG,  0, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS,                      0x0802, _S16, _TG,  1, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS,                      0x0804, _S16, _TG,  2, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS,                      0x0806, _S16, _TG,  3, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS,                      0x0808, _S16, _TG,  4, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS,                      0x080A, _S16, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR,                    0x080C, _S16, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR,                      0x080E, _S16, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(INTR_STATUS,             0x0810, _S16, _TG,  8, _NC, _NV, _NA, _NS)
+VMCS_FIELD(PML_INDEX,               0x0812, _S16, _TG,  9, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x0814, _S16, _TG,  10)
 VMCS_UNUSED(                        0x0816, _S16, _TG,  11)
 VMCS_UNUSED(                        0x0818, _S16, _TG,  12)
@@ -212,13 +217,13 @@ VMCS_GROUP_END(16, GUEST)

 /* 16-bit host state. */
 VMCS_GROUP_START(16, HOST)
-VMCS_FIELD(HOST_ES,                 0x0C00, _S16, _TH,  0, _NC,  _V, _NA)
-VMCS_FIELD(HOST_CS,                 0x0C02, _S16, _TH,  1, _NC,  _V, _NA)
-VMCS_FIELD(HOST_SS,                 0x0C04, _S16, _TH,  2, _NC,  _V, _NA)
-VMCS_FIELD(HOST_DS,                 0x0C06, _S16, _TH,  3, _NC,  _V, _NA)
-VMCS_FIELD(HOST_FS,                 0x0C08, _S16, _TH,  4, _NC,  _V, _NA)
-VMCS_FIELD(HOST_GS,                 0x0C0A, _S16, _TH,  5, _NC,  _V, _NA)
-VMCS_FIELD(HOST_TR,                 0x0C0C, _S16, _TH,  6, _NC,  _V, _NA)
+VMCS_FIELD(HOST_ES,                 0x0C00, _S16, _TH,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CS,                 0x0C02, _S16, _TH,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SS,                 0x0C04, _S16, _TH,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_DS,                 0x0C06, _S16, _TH,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_FS,                 0x0C08, _S16, _TH,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GS,                 0x0C0A, _S16, _TH,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_TR,                 0x0C0C, _S16, _TH,  6, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x0C0E, _S16, _TH,  7)
 VMCS_UNUSED(                        0x0C10, _S16, _TH,  8)
 VMCS_UNUSED(                        0x0C12, _S16, _TH,  9)
@@ -251,34 +256,34 @@ VMCS_SET_END(16)
 VMCS_SET_START(64)
 /* 64-bit control fields. */
 VMCS_GROUP_START(64, CTL)
-VMCS_FIELD(IOBITMAPA,               0x2000, _S64, _TC,  0, _NC,  _V, _NA)
-VMCS_FIELD(IOBITMAPB,               0x2002, _S64, _TC,  1, _NC,  _V, _NA)
-VMCS_FIELD(MSRBITMAP,               0x2004, _S64, _TC,  2, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_STORE_ADDR,   0x2006, _S64, _TC,  3, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR,    0x2008, _S64, _TC,  4, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR,   0x200A, _S64, _TC,  5, _NC,  _V, _NA)
-VMCS_FIELD(EXECUTIVE_VMCS_PTR,      0x200C, _S64, _TC,  6, _NC,  _V, _NA)
-VMCS_FIELD(PML_ADDR,                0x200E, _S64, _TC,  7, _NC, _NV, _NA)
-VMCS_FIELD(TSC_OFF,                 0x2010, _S64, _TC,  8,  _C,  _V, _NA)
-VMCS_FIELD(VIRT_APIC_ADDR,          0x2012, _S64, _TC,  9, _NC,  _V, _NA)
-VMCS_FIELD(APIC_ACCESS_ADDR,        0x2014, _S64, _TC, 10, _NC,  _V, _NA)
-VMCS_FIELD(PI_DESC_ADDR,            0x2016, _S64, _TC, 11, _NC, _NV, _NA)
-VMCS_FIELD(VMFUNC_CTLS,             0x2018, _S64, _TC, 12, _NC,  _V, _NA)
-VMCS_FIELD(EPTP,                    0x201A, _S64, _TC, 13, _NC,  _V, _NA)
-VMCS_FIELD(EOI_EXIT0,               0x201C, _S64, _TC, 14, _NC, _NV, _NA)
-VMCS_FIELD(EOI_EXIT1,               0x201E, _S64, _TC, 15, _NC, _NV, _NA)
-VMCS_FIELD(EOI_EXIT2,               0x2020, _S64, _TC, 16, _NC, _NV, _NA)
-VMCS_FIELD(EOI_EXIT3,               0x2022, _S64, _TC, 17, _NC, _NV, _NA)
-VMCS_FIELD(EPTP_LIST_ADDR,          0x2024, _S64, _TC, 18, _NC,  _V, _NA)
-VMCS_FIELD(VMREAD_BITMAP,           0x2026, _S64, _TC, 19, _NC, _NV, _NA)
-VMCS_FIELD(VMWRITE_BITMAP,          0x2028, _S64, _TC, 20, _NC, _NV, _NA)
-VMCS_FIELD(VE_INFO_ADDR,            0x202A, _S64, _TC, 21, _NC,  _V, _NA)
-VMCS_FIELD(XSS_EXITING_BITMAP,      0x202C, _S64, _TC, 22, _NC,  _V, _NA)
-VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC, _NV, _NA)
+VMCS_FIELD(IOBITMAPA,               0x2000, _S64, _TC,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(IOBITMAPB,               0x2002, _S64, _TC,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(MSRBITMAP,               0x2004, _S64, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_ADDR,   0x2006, _S64, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR,    0x2008, _S64, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR,   0x200A, _S64, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(EXECUTIVE_VMCS_PTR,      0x200C, _S64, _TC,  6, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PML_ADDR,                0x200E, _S64, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(TSC_OFF,                 0x2010, _S64, _TC,  8,  _C,  _V, _NA,  _S)
+VMCS_FIELD(VIRT_APIC_ADDR,          0x2012, _S64, _TC,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(APIC_ACCESS_ADDR,        0x2014, _S64, _TC, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PI_DESC_ADDR,            0x2016, _S64, _TC, 11, _NC, _NV, _NA,  _S)
+VMCS_FIELD(VMFUNC_CTLS,             0x2018, _S64, _TC, 12, _NC,  _V, _NA,  _S)
+VMCS_FIELD(EPTP,                    0x201A, _S64, _TC, 13, _NC,  _V, _NA, _NS)
+VMCS_FIELD(EOI_EXIT0,               0x201C, _S64, _TC, 14, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EOI_EXIT1,               0x201E, _S64, _TC, 15, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EOI_EXIT2,               0x2020, _S64, _TC, 16, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EOI_EXIT3,               0x2022, _S64, _TC, 17, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EPTP_LIST_ADDR,          0x2024, _S64, _TC, 18, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMREAD_BITMAP,           0x2026, _S64, _TC, 19, _NC, _NV, _NA,  _S)
+VMCS_FIELD(VMWRITE_BITMAP,          0x2028, _S64, _TC, 20, _NC, _NV, _NA,  _S)
+VMCS_FIELD(VE_INFO_ADDR,            0x202A, _S64, _TC, 21, _NC,  _V, _NA,  _S)
+VMCS_FIELD(XSS_EXITING_BITMAP,      0x202C, _S64, _TC, 22, _NC,  _V, _NA,  _S)
+VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2030, _S64, _TC, 24)
-VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA)
+VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA,  _S)
 VMCS_UNUSED(                        0x2034, _S64, _TC, 26)
-VMCS_UNUSED(                        0x2036, _S64, _TC, 27)
+VMCS_FIELD(ENCLV_EXITING_BITMAP,    0x2036, _S64, _TC, 27, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2038, _S64, _TC, 28)
 VMCS_UNUSED(                        0x203A, _S64, _TC, 29)
 VMCS_UNUSED(                        0x203C, _S64, _TC, 30)
@@ -287,7 +292,7 @@ VMCS_GROUP_END(64, CTL)

 /* 64-bit read-only data field. */
 VMCS_GROUP_START(64, DATA)
-VMCS_FIELD(PHYSADDR,                0x2400, _S64, _TD,  0, _NC,  _V, _RO)
+VMCS_FIELD(PHYSADDR,                0x2400, _S64, _TD,  0, _NC,  _V, _RO, _NS)
 VMCS_UNUSED(                        0x2402, _S64, _TD,  1)
 VMCS_UNUSED(                        0x2404, _S64, _TD,  2)
 VMCS_UNUSED(                        0x2406, _S64, _TD,  3)
@@ -323,16 +328,16 @@ VMCS_GROUP_END(64, DATA)

 /* 64-bit guest state. */
 VMCS_GROUP_START(64, GUEST)
-VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA)
-VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA)
-VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA)
-VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA)
-VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA)
-VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW)
-VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA, _NS)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
 VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
 VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
@@ -359,9 +364,9 @@ VMCS_GROUP_END(64, GUEST)

 /* 64-bit host state. */
 VMCS_GROUP_START(64, HOST)
-VMCS_FIELD(HOST_PAT,                0x2C00, _S64, _TH,  0, _NC,  _V, _NA)
-VMCS_FIELD(HOST_EFER,               0x2C02, _S64, _TH,  1, _NC,  _V, _NA)
-VMCS_FIELD(HOST_PGC,                0x2C04, _S64, _TH,  2, _NC,  _V, _NA)
+VMCS_FIELD(HOST_PAT,                0x2C00, _S64, _TH,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_EFER,               0x2C02, _S64, _TH,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_PGC,                0x2C04, _S64, _TH,  2, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2C06, _S64, _TH,  3)
 VMCS_UNUSED(                        0x2C08, _S64, _TH,  4)
 VMCS_UNUSED(                        0x2C0A, _S64, _TH,  5)
@@ -398,24 +403,24 @@ VMCS_SET_END(64)
 VMCS_SET_START(32)
 /* 32-bit control fields. */
 VMCS_GROUP_START(32, CTL)
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA)
-VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA,  _S)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA, _NS)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA,  _S)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA,  _S)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
 VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
 VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
@@ -434,14 +439,14 @@ VMCS_GROUP_END(32, CTL)

 /* 32-bit read-only data fields. */
 VMCS_GROUP_START(32, DATA)
-VMCS_FIELD(VMINSTR_ERR,             0x4400, _S32, _TD,  0, _NC,  _V, _NA)
-VMCS_FIELD(EXIT_REASON,             0x4402, _S32, _TD,  1, _NC,  _V, _RO)
-VMCS_FIELD(EXIT_INTR_INFO,          0x4404, _S32, _TD,  2,  _C,  _V, _RO)
-VMCS_FIELD(EXIT_INTR_ERR,           0x4406, _S32, _TD,  3, _NC,  _V, _RO)
-VMCS_FIELD(IDTVEC_INFO,             0x4408, _S32, _TD,  4, _NC,  _V, _RO)
-VMCS_FIELD(IDTVEC_ERR,              0x440A, _S32, _TD,  5, _NC,  _V, _RO)
-VMCS_FIELD(INSTRLEN,                0x440C, _S32, _TD,  6, _NC,  _V, _RO)
-VMCS_FIELD(INSTR_INFO,              0x440E, _S32, _TD,  7, _NC,  _V, _RO)
+VMCS_FIELD(VMINSTR_ERR,             0x4400, _S32, _TD,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(EXIT_REASON,             0x4402, _S32, _TD,  1, _NC,  _V, _RO, _NS)
+VMCS_FIELD(EXIT_INTR_INFO,          0x4404, _S32, _TD,  2,  _C,  _V, _RO, _NS)
+VMCS_FIELD(EXIT_INTR_ERR,           0x4406, _S32, _TD,  3, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IDTVEC_INFO,             0x4408, _S32, _TD,  4, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IDTVEC_ERR,              0x440A, _S32, _TD,  5, _NC,  _V, _RO, _NS)
+VMCS_FIELD(INSTRLEN,                0x440C, _S32, _TD,  6, _NC,  _V, _RO, _NS)
+VMCS_FIELD(INSTR_INFO,              0x440E, _S32, _TD,  7, _NC,  _V, _RO, _NS)
 VMCS_UNUSED(                        0x4410, _S32, _TD,  8)
 VMCS_UNUSED(                        0x4412, _S32, _TD,  9)
 VMCS_UNUSED(                        0x4414, _S32, _TD,  10)
@@ -470,30 +475,30 @@ VMCS_GROUP_END(32, DATA)

 /* 32-bit guest state. */
 VMCS_GROUP_START(32, GUEST)
-VMCS_FIELD(ES_LIMIT,                0x4800, _S32, _TG,  0, _NC,  _V, _RW)
-VMCS_FIELD(CS_LIMIT,                0x4802, _S32, _TG,  1, _NC,  _V, _RW)
-VMCS_FIELD(SS_LIMIT,                0x4804, _S32, _TG,  2, _NC,  _V, _RW)
-VMCS_FIELD(DS_LIMIT,                0x4806, _S32, _TG,  3, _NC,  _V, _RW)
-VMCS_FIELD(FS_LIMIT,                0x4808, _S32, _TG,  4, _NC,  _V, _RW)
-VMCS_FIELD(GS_LIMIT,                0x480A, _S32, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(LDTR_LIMIT,              0x480C, _S32, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(TR_LIMIT,                0x480E, _S32, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(GDTR_LIMIT,              0x4810, _S32, _TG,  8, _NC,  _V, _RW)
-VMCS_FIELD(IDTR_LIMIT,              0x4812, _S32, _TG,  9, _NC,  _V, _RW)
-VMCS_FIELD(ES_AR,                   0x4814, _S32, _TG, 10, _NC,  _V, _RW)
-VMCS_FIELD(CS_AR,                   0x4816, _S32, _TG, 11, _NC,  _V, _RW)
-VMCS_FIELD(SS_AR,                   0x4818, _S32, _TG, 12, _NC,  _V, _RW)
-VMCS_FIELD(DS_AR,                   0x481A, _S32, _TG, 13, _NC,  _V, _RW)
-VMCS_FIELD(FS_AR,                   0x481C, _S32, _TG, 14, _NC,  _V, _RW)
-VMCS_FIELD(GS_AR,                   0x481E, _S32, _TG, 15, _NC,  _V, _RW)
-VMCS_FIELD(LDTR_AR,                 0x4820, _S32, _TG, 16, _NC,  _V, _RW)
-VMCS_FIELD(TR_AR,                   0x4822, _S32, _TG, 17, _NC,  _V, _RW)
-VMCS_FIELD(HOLDOFF,                 0x4824, _S32, _TG, 18,  _C,  _V, _RW)
-VMCS_FIELD(ACTSTATE,                0x4826, _S32, _TG, 19, _NC,  _V, _NA)
-VMCS_FIELD(SMBASE,                  0x4828, _S32, _TG, 20, _NC,  _V, _NA)
-VMCS_FIELD(SYSENTER_CS,             0x482A, _S32, _TG, 21, _NC,  _V, _RW)
+VMCS_FIELD(ES_LIMIT,                0x4800, _S32, _TG,  0, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS_LIMIT,                0x4802, _S32, _TG,  1, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS_LIMIT,                0x4804, _S32, _TG,  2, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS_LIMIT,                0x4806, _S32, _TG,  3, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS_LIMIT,                0x4808, _S32, _TG,  4, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS_LIMIT,                0x480A, _S32, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR_LIMIT,              0x480C, _S32, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR_LIMIT,                0x480E, _S32, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GDTR_LIMIT,              0x4810, _S32, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(IDTR_LIMIT,              0x4812, _S32, _TG,  9, _NC,  _V, _RW, _NS)
+VMCS_FIELD(ES_AR,                   0x4814, _S32, _TG, 10, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS_AR,                   0x4816, _S32, _TG, 11, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS_AR,                   0x4818, _S32, _TG, 12, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS_AR,                   0x481A, _S32, _TG, 13, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS_AR,                   0x481C, _S32, _TG, 14, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS_AR,                   0x481E, _S32, _TG, 15, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR_AR,                 0x4820, _S32, _TG, 16, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR_AR,                   0x4822, _S32, _TG, 17, _NC,  _V, _RW, _NS)
+VMCS_FIELD(HOLDOFF,                 0x4824, _S32, _TG, 18,  _C,  _V, _RW, _NS)
+VMCS_FIELD(ACTSTATE,                0x4826, _S32, _TG, 19, _NC,  _V, _NA, _NS)
+VMCS_FIELD(SMBASE,                  0x4828, _S32, _TG, 20, _NC,  _V, _NA, _NS)
+VMCS_FIELD(SYSENTER_CS,             0x482A, _S32, _TG, 21, _NC,  _V, _RW, _NS)
 VMCS_UNUSED(                        0x482C, _S32, _TG, 22)
-VMCS_FIELD(TIMER,                   0x482E, _S32, _TG, 23, _NC, _NV, _NA)
+VMCS_FIELD(TIMER,                   0x482E, _S32, _TG, 23, _NC, _NV, _NA, _NS)
 VMCS_UNUSED(                        0x4830, _S32, _TG, 24)
 VMCS_UNUSED(                        0x4832, _S32, _TG, 25)
 VMCS_UNUSED(                        0x4834, _S32, _TG, 26)
@@ -507,7 +512,7 @@ VMCS_GROUP_END(32, GUEST)

 /* 32-bit host state. */
 VMCS_GROUP_START(32, HOST)
-VMCS_FIELD(HOST_SYSENTER_CS,        0x4C00, _S32, _TH,  0, _NC,  _V, _NA)
+VMCS_FIELD(HOST_SYSENTER_CS,        0x4C00, _S32, _TH,  0, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x4C02, _S32, _TH,  1)
 VMCS_UNUSED(                        0x4C04, _S32, _TH,  2)
 VMCS_UNUSED(                        0x4C06, _S32, _TH,  3)
@@ -546,14 +551,14 @@ VMCS_SET_END(32)
 VMCS_SET_START(NAT)
 /* natural-width control fields. */
 VMCS_GROUP_START(NAT, CTL)
-VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA)
-VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA)
-VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA)
-VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
 VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
 VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
@@ -581,12 +586,12 @@ VMCS_UNUSED(                        0x603E,  _SN, _TC,  31)
 VMCS_GROUP_END(NAT, CTL)

 VMCS_GROUP_START(NAT, DATA)
-VMCS_FIELD(EXIT_QUAL,               0x6400,  _SN, _TD,  0, _NC,  _V, _RO)
-VMCS_FIELD(IO_RCX,                  0x6402,  _SN, _TD,  1, _NC,  _V, _RO)
-VMCS_FIELD(IO_RSI,                  0x6404,  _SN, _TD,  2, _NC,  _V, _RO)
-VMCS_FIELD(IO_RDI,                  0x6406,  _SN, _TD,  3, _NC,  _V, _RO)
-VMCS_FIELD(IO_RIP,                  0x6408,  _SN, _TD,  4, _NC,  _V, _RO)
-VMCS_FIELD(LINEAR_ADDR,             0x640A,  _SN, _TD,  5, _NC,  _V, _RO)
+VMCS_FIELD(EXIT_QUAL,               0x6400,  _SN, _TD,  0, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RCX,                  0x6402,  _SN, _TD,  1, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RSI,                  0x6404,  _SN, _TD,  2, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RDI,                  0x6406,  _SN, _TD,  3, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RIP,                  0x6408,  _SN, _TD,  4, _NC,  _V, _RO, _NS)
+VMCS_FIELD(LINEAR_ADDR,             0x640A,  _SN, _TD,  5, _NC,  _V, _RO, _NS)
 VMCS_UNUSED(                        0x640C,  _SN, _TD,  6)
 VMCS_UNUSED(                        0x640E,  _SN, _TD,  7)
 VMCS_UNUSED(                        0x6410,  _SN, _TD,  8)
@@ -617,26 +622,26 @@ VMCS_GROUP_END(NAT, DATA)

 /* natural-width guest state. */
 VMCS_GROUP_START(NAT, GUEST)
-VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA)
-VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW)
-VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW)
-VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW)
-VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW)
-VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW)
-VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW)
-VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW)
-VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW)
-VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW)
-VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA)
-VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW)
-VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW)
-VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW)
-VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW)
-VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW)
-VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW, _NS)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW, _NS)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA, _NS)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW, _NS)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW, _NS)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW, _NS)
 VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
 VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
 VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
@@ -653,18 +658,18 @@ VMCS_GROUP_END(NAT, GUEST)

 /* natural-width host state. */
 VMCS_GROUP_START(NAT, HOST)
-VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA)
-VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA)
-VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA)
-VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA)
-VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA)
-VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA)
-VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
 VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
 VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2004-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2004-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -159,10 +159,10 @@
 #define VT_ENCODING_RSVD               0xffff9000

 /*
- * The highest index of any currently defined field is 25, for
- * TSC_MULTIPLIER.
+ * The highest index of any currently defined field is 27, for
+ * ENCLV_EXITING_BITMAP.
 */
-#define VT_ENCODING_MAX_INDEX                  25
+#define VT_ENCODING_MAX_INDEX                  27

 enum {
 #define VMCS_FIELD(_name, _val, ...) VT_VMCS_##_name = _val,
@@ -286,14 +286,16 @@ enum {
   VMX_CPU2(INVPCID,            12)                      \
   VMX_CPU2(VMFUNC,             13)                      \
   VMX_CPU2(VMCS_SHADOW,        14)                      \
-   VMX_CPU2(ENCL,               15)                      \
+   VMX_CPU2(ENCLS,              15)                      \
   VMX_CPU2(RDSEED,             16)                      \
   VMX_CPU2(PML,                17)                      \
   VMX_CPU2(EPT_VIOL_VE,        18)                      \
   VMX_CPU2(PT_SUPPRESS_NR_BIT, 19)                      \
   VMX_CPU2(XSAVES,             20)                      \
   VMX_CPU2(EPT_MBX,            22)                      \
-   VMX_CPU2(TSC_SCALING,        25)
+   VMX_CPU2(TSC_SCALING,        25)                      \
+   VMX_CPU2(ENCLV,              28)                      \
+   VMX_CPU2(EPC_VIRT_EXT,       29)

 #define VMX_PROCBASED_CTLS2_CAP                          \
        VMX_PROCBASED_CTLS2_CAP_NDA                      \
@@ -640,7 +642,7 @@ enum {
 #define VT_HOLDOFF_MOVSS       0x00000002
 #define VT_HOLDOFF_SMI         0x00000004
 #define VT_HOLDOFF_NMI         0x00000008
-#define VT_HOLDOFF_ENCLAVE     0x00000010
+#define VT_ENCLAVE_INTR        0x00000010
 #define VT_HOLDOFF_INST        (VT_HOLDOFF_STI | VT_HOLDOFF_MOVSS)
 #define VT_HOLDOFF_RSV         0xFFFFFFE0

@@ -668,6 +670,7 @@ enum {
 #define VT_EPT_QUAL_GUEST_RW           (1 << 10)
 #define VT_EPT_QUAL_GUEST_NX           (1 << 11)
 #define VT_EPT_QUAL_NMIUNMASK          (1 << 12)
+#define VT_EPT_QUAL_SYNTH_PML_FULL     (1 << 31)


 /* IOIO Qualification */
@@ -686,6 +689,12 @@ enum {
 #define VT_GUESTFAIL_QUAL_NMI          3
 #define VT_GUESTFAIL_QUAL_LINK         4

+/* SGX conflict VM-exit Qualification Codes */
+#define VT_SGX_TRACKING_RESOURCE_CONFLICT     0
+#define VT_SGX_TRACKING_REFERENCE_CONFLICT    1
+#define VT_SGX_EPC_PAGE_CONFLICT_EXCEPTION    2
+#define VT_SGX_EPC_PAGE_CONFLICT_ERROR        3
+
 /* VMX abort indicators. */

 #define VT_VMX_ABORT_GUEST_MSRS        1
@@ -709,7 +718,8 @@ enum {
 #define VT_REQUIRED_PINBASED_CTLS                      \
   (VT_PINBASED_CTLS_DEFAULT1                        | \
    VT_VMCS_PIN_VMEXEC_CTL_EXTINT_EXIT               | \
-    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT)
+    VT_VMCS_PIN_VMEXEC_CTL_NMI_EXIT                  | \
+    VT_VMCS_PIN_VMEXEC_CTL_VNMI)

 #define VT_REQUIRED_PROCBASED_CTLS                     \
   (VT_PROCBASED_CTLS_DEFAULT1                       | \
@@ -725,6 +735,7 @@ enum {
    VT_VMCS_CPU_VMEXEC_CTL_LDCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_STCR8                     | \
    VT_VMCS_CPU_VMEXEC_CTL_TPR_SHADOW                | \
+    VT_VMCS_CPU_VMEXEC_CTL_VNMI_WINDOW               | \
    VT_VMCS_CPU_VMEXEC_CTL_MONITOR)

 #define VT_REQUIRED_EXIT_CTLS                          \
@@ -753,6 +764,10 @@ enum {
 #define VT_TSQUAL_JMP   2
 #define VT_TSQUAL_GATE  3

+/* PML Constants */
+#define VT_MAX_PML_INDEX   511
+#define VT_PML_ENTRY_MASK  (~(QWORD(0, PAGE_MASK)))
+
 typedef union {
   struct {
      unsigned selVal:16;
@@ -815,6 +830,23 @@ typedef union {
   uint32 flat;
 } VTIOQualifier;

+#define VT_APICACCESSQUAL_TYPE_LINEAR_READ  0
+#define VT_APICACCESSQUAL_TYPE_LINEAR_WRITE 1
+#define VT_APICACCESSQUAL_TYPE_LINEAR_INSTR 2
+#define VT_APICACCESSQUAL_TYPE_LINEAR_EVENT 3
+#define VT_APICACCESSQUAL_TYPE_PHYS_EVENT   10
+#define VT_APICACCESSQUAL_TYPE_PHYS_INSTR   15
+
+/* APIC Access intercept qualifier */
+typedef union {
+   struct {
+      uint64 offset:12;
+      uint64 type  :4;
+      uint64 rsvd  :48;
+   } bits;
+   uint64 flat;
+} VTAPICAccessQualifier;
+
 #define VT_IINFO_SCALE1    0
 #define VT_IINFO_SCALE2    1
 #define VT_IINFO_SCALE4    2
@@ -1053,7 +1085,7 @@ VT_MBXSupportedFromFeatures(uint64 secondary)
 static INLINE Bool
 VT_EnabledCPU(void)
 {
-   return VT_EnabledFromFeatures(__GET_MSR(MSR_FEATCTL));
+   return VT_EnabledFromFeatures(X86MSR_GetMSR(MSR_FEATCTL));
 }


@@ -1074,19 +1106,14 @@ VT_EnabledCPU(void)
 static INLINE Bool
 VT_SupportedCPU(void)
 {
-   if (__GET_MSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) {
-      return VT_SupportedFromFeatures(__GET_MSR(MSR_VMX_TRUE_PINBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_PROCBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_ENTRY_CTLS),
-                                      __GET_MSR(MSR_VMX_TRUE_EXIT_CTLS),
-                                      __GET_MSR(MSR_VMX_BASIC));
-   } else {
-      return VT_SupportedFromFeatures(__GET_MSR(MSR_VMX_PINBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_PROCBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_ENTRY_CTLS),
-                                      __GET_MSR(MSR_VMX_EXIT_CTLS),
-                                      __GET_MSR(MSR_VMX_BASIC));
-   }
+
+   /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
+   return (X86MSR_GetMSR(MSR_VMX_BASIC) & MSR_VMX_BASIC_TRUE_CTLS) != 0 &&
+          VT_SupportedFromFeatures(X86MSR_GetMSR(MSR_VMX_TRUE_PINBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_PROCBASED_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_ENTRY_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_TRUE_EXIT_CTLS),
+                                   X86MSR_GetMSR(MSR_VMX_BASIC));
 }

 #endif /* } !defined(USERLEVEL) */
@@ -1141,5 +1168,4 @@ VT_ConfigMSRNum(unsigned index)
   return MSR_VMX_BASIC + index;
 }

-
 #endif /* _X86VT_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -48,12 +48,10 @@
 #include "modulecall.h"
 #include "vm_asm.h"
 #include "vmx86.h"
-#include "initblock.h"
 #include "task.h"
 #include "memtrack.h"
 #include "task.h"
 #include "cpuid.h"
-#include "cpuid_info.h"
 #include "circList.h"
 #include "x86msr.h"

@@ -64,9 +62,10 @@
 #include "hostif.h"
 #include "hostif_priv.h"
 #include "vmhost.h"
+#include "sharedAreaVmmon.h"

-static void LinuxDriverQueue(VMLinux *vmLinux);
-static void LinuxDriverDequeue(VMLinux *vmLinux);
+static void LinuxDriverQueue(Device *device);
+static void LinuxDriverDequeue(Device *device);
 static Bool LinuxDriverCheckPadding(void);

 #define VMMON_UNKNOWN_SWAP_SIZE -1ULL
@@ -96,7 +95,9 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd,
                       unsigned long ioarg);

 static int LinuxDriver_Close(struct inode *inode, struct file *filp);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
+static vm_fault_t LinuxDriverFault(struct vm_fault *fault);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
 static int LinuxDriverFault(struct vm_fault *fault);
 #else
 static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault);
@@ -284,6 +285,7 @@ init_module(void)
   }

   CPUID_Init();
+   Vmx86_CacheNXState();

   if (!Task_Initialize()) {
      return -ENOEXEC;
@@ -404,18 +406,19 @@ static int
 LinuxDriver_Open(struct inode *inode, // IN
                 struct file *filp)   // IN
 {
-   VMLinux *vmLinux;
+   Device *device;

-   vmLinux = kmalloc(sizeof *vmLinux, GFP_KERNEL);
-   if (vmLinux == NULL) {
+   device = kmalloc(sizeof *device, GFP_KERNEL);
+   if (device == NULL) {
      return -ENOMEM;
   }
-   memset(vmLinux, 0, sizeof *vmLinux);
+   memset(device, 0, sizeof *device);

-   sema_init(&vmLinux->lock4Gb, 1);
+   sema_init(&device->lock4Gb, 1);
+   init_rwsem(&device->vmDriverRWSema);

-   filp->private_data = vmLinux;
-   LinuxDriverQueue(vmLinux);
+   filp->private_data = device;
+   LinuxDriverQueue(device);

   Vmx86_Open();

@@ -442,17 +445,12 @@ LinuxDriver_Open(struct inode *inode, // IN
 */

 static int
-LinuxDriverAllocPages(unsigned int gfpFlag, // IN
-                      unsigned int order,   // IN
+LinuxDriverAllocPages(unsigned int order,   // IN
                      struct page **pg,     // OUT
-                      unsigned int size)    // IN
+                      PageCnt size)         // IN
 {
-   struct page* page;
-
-   page = alloc_pages(gfpFlag, order);
+   struct page* page = alloc_pages(GFP_HIGHUSER, order);
   if (page) {
-      unsigned int i;
-
      /*
       * Grab an extra reference on all pages except first one - first
       * one was already refcounted by alloc_pages.
@@ -465,9 +463,8 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
       * Note that this depends on Linux VM internals.  It works on all
       * kernels we care about.
       */
-
-      order = 1 << order;
-      for (i = 0; i < order; i++) {
+      PageCnt i, orderToNumPages = 1 << order;
+      for (i = 0; i < orderToNumPages; i++) {
         if (i) {
            /*
             * Debug kernels assert that page->_count is not zero when
@@ -485,17 +482,14 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
            put_page(page);
         } else {
            void *addr = kmap(page);
-
            memset(addr, 0, PAGE_SIZE);
            kunmap(page);
            *pg++ = page;
         }
         page++;
      }
-
      return 0;
   }
-
   return -ENOMEM;
 }

@@ -517,17 +511,16 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
 */

 static void
-LinuxDriverDestructor4Gb(VMLinux *vmLinux) // IN
+LinuxDriverDestructor4Gb(Device *device) // IN
 {
-   unsigned int pg;
-
-   if (!vmLinux->size4Gb) {
+   PageCnt pg;
+   if (!device->size4Gb) {
      return;
   }
-   for (pg = 0; pg < vmLinux->size4Gb; pg++) {
-      put_page(vmLinux->pages4Gb[pg]);
+   for (pg = 0; pg < device->size4Gb; pg++) {
+      put_page(device->pages4Gb[pg]);
   }
-   vmLinux->size4Gb = 0;
+   device->size4Gb = 0;
 }


@@ -547,16 +540,18 @@ static int
 LinuxDriver_Close(struct inode *inode, // IN
                  struct file *filp)   // IN
 {
-   VMLinux *vmLinux;
+   Device *device;

-   vmLinux = (VMLinux *)filp->private_data;
-   ASSERT(vmLinux);
+   device = (Device *)filp->private_data;
+   ASSERT(device);

-   LinuxDriverDequeue(vmLinux);
-   if (vmLinux->vm != NULL) {
-      Vmx86_ReleaseVM(vmLinux->vm);
-      vmLinux->vm = NULL;
+   LinuxDriverDequeue(device);
+   down_write(&device->vmDriverRWSema);
+   if (device->vm != NULL) {
+      Vmx86_ReleaseVM(device->vm);
+      device->vm = NULL;
   }
+   up_write(&device->vmDriverRWSema);

   Vmx86_Close();

@@ -566,9 +561,9 @@ LinuxDriver_Close(struct inode *inode, // IN
    * uses it anymore, and we do not need to hold the semaphore.
    */

-   LinuxDriverDestructor4Gb(vmLinux);
+   LinuxDriverDestructor4Gb(device);

-   kfree(vmLinux);
+   kfree(device);
   filp->private_data = NULL;

   return 0;
@@ -594,7 +589,12 @@ LinuxDriver_Close(struct inode *inode, // IN
 *-----------------------------------------------------------------------------
 */

-static int
+static
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
+vm_fault_t
+#else
+int
+#endif
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
 LinuxDriverFault(struct vm_fault *fault)     //IN/OUT
 #else
@@ -605,16 +605,16 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
   struct vm_area_struct *vma = fault->vma;
 #endif
-   VMLinux *vmLinux = (VMLinux *) vma->vm_file->private_data;
-   unsigned long pg;
+   Device *device = (Device *) vma->vm_file->private_data;
+   PageNum pg;
   struct page* page;

   pg = fault->pgoff;
   pg = VMMON_MAP_OFFSET(pg);
-   if (pg >= vmLinux->size4Gb) {
+   if (pg >= device->size4Gb) {
      return VM_FAULT_SIGBUS;
   }
-   page = vmLinux->pages4Gb[pg];
+   page = device->pages4Gb[pg];
   get_page(page);
   fault->page = page;
   return 0;
@@ -640,15 +640,13 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
 *-----------------------------------------------------------------------------
 */

-static int LinuxDriverAllocContig(VMLinux *vmLinux,
+static int LinuxDriverAllocContig(Device *device,
                                  struct vm_area_struct *vma,
                                  unsigned long off,
-                                  unsigned long size)
+                                  PageCnt size)
 {
   unsigned long vmaOrder = VMMON_MAP_ORDER(off);
-   unsigned long vmaAllocSize;
-   unsigned int gfpFlag;
-   unsigned long i;
+   PageCnt vmaAllocSize, i;

   if (VMMON_MAP_RSVD(off)) {
      /* Reserved bits set... */
@@ -658,24 +656,6 @@ static int LinuxDriverAllocContig(VMLinux *vmLinux,
      /* We do not need non-zero offsets... */
      return -EINVAL;
   }
-   switch (VMMON_MAP_MT(off)) {
-      case VMMON_MAP_MT_LOW4GB:
-#ifdef GFP_DMA32
-         gfpFlag = GFP_USER | GFP_DMA32;
-#else
-         gfpFlag = GFP_USER | GFP_DMA;
-#endif
-         break;
-      case VMMON_MAP_MT_LOW16MB:
-         gfpFlag = GFP_USER | GFP_DMA;
-         break;
-      case VMMON_MAP_MT_ANY:
-         gfpFlag = GFP_HIGHUSER;
-         break;
-      default:
-         /* Invalid memory type */
-         return -EINVAL;
-   }
   if (size > VMMON_MAP_OFFSET_MASK + 1) {
      /* Size is too big to fit to our window. */
      return -ENOMEM;
@@ -686,30 +666,25 @@ static int LinuxDriverAllocContig(VMLinux *vmLinux,
      return -ENOMEM;
   }
   /* Sorry. Only one mmap per one open. */
-   down(&vmLinux->lock4Gb);
-   if (vmLinux->size4Gb) {
-      up(&vmLinux->lock4Gb);
+   down(&device->lock4Gb);
+   if (device->size4Gb) {
+      up(&device->lock4Gb);
      return -EINVAL;
   }
   vmaAllocSize = 1 << vmaOrder;
   for (i = 0; i < size; i += vmaAllocSize) {
-      int err;
-
-      err = LinuxDriverAllocPages(gfpFlag, vmaOrder,
-                                  vmLinux->pages4Gb + i, size - i);
+      int err = LinuxDriverAllocPages(vmaOrder, device->pages4Gb + i, size - i);
      if (err) {
         while (i > 0) {
-            put_page(vmLinux->pages4Gb[--i]);
+            put_page(device->pages4Gb[--i]);
         }
-         up(&vmLinux->lock4Gb);
-
+         up(&device->lock4Gb);
         return err;
      }
   }
-   vmLinux->size4Gb = size;
-   up(&vmLinux->lock4Gb);
+   device->size4Gb = size;
+   up(&device->lock4Gb);
   vma->vm_ops = &vmuser_mops;
-
   return 0;
 }

@@ -737,8 +712,8 @@ static int
 LinuxDriverMmap(struct file *filp,
                struct vm_area_struct *vma)
 {
-   VMLinux *vmLinux = (VMLinux *) filp->private_data;
-   unsigned long size;
+   Device *device = (Device *) filp->private_data;
+   PageCnt size;
   int err;

   /* Only shared mappings */
@@ -752,17 +727,18 @@ LinuxDriverMmap(struct file *filp,
   if (size < 1) {
      return -EINVAL;
   }
-   if (vmLinux->vm) {
+   down_read(&device->vmDriverRWSema);
+   if (device->vm) {
      err = -EINVAL;
   } else {
-      err = LinuxDriverAllocContig(vmLinux, vma, vma->vm_pgoff, size);
+      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
   }
+   up_read(&device->vmDriverRWSema);
   if (err) {
      return err;
   }
   /* Clear VM_IO, otherwise SuSE's kernels refuse to do get_user_pages */
   vma->vm_flags &= ~VM_IO;
-
   return 0;
 }

@@ -1032,27 +1008,31 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
                  u_int iocmd,          // IN:
                  unsigned long ioarg)  // IN:
 {
-   VMLinux *vmLinux = (VMLinux *) filp->private_data;
-   int retval = 0;
+   Device *device = (Device *) filp->private_data;
+   long retval = 0;
   Vcpuid vcpuid;
   VMDriver *vm;
+   Bool needsWriteLock = iocmd == IOCTL_VMX86_CREATE_VM ||
+                         iocmd == IOCTL_VMX86_RELEASE_VM;

-   if (vmLinux == NULL) {
+   ASSERT_ON_COMPILE(sizeof(long) == sizeof(int64));
+   if (device == NULL) {
      return -EINVAL;
   }

-   vm = vmLinux->vm;
-
-   /*
-    * Validate the VM pointer for those IOCTLs that require it.
-    */
+   if (needsWriteLock) {
+      down_write(&device->vmDriverRWSema);
+   } else {
+      down_read(&device->vmDriverRWSema);
+   }

+   vm = device->vm;
+   /* Validate the VM pointer for those IOCTLs that require it. */
   switch (iocmd) {
   case IOCTL_VMX86_VERSION:
   case IOCTL_VMX86_CREATE_VM:
   case IOCTL_VMX86_SET_UID:
   case IOCTL_VMX86_GET_NUM_VMS:
-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
   case IOCTL_VMX86_SET_HARD_LIMIT:
   case IOCTL_VMX86_GET_IPI_VECTORS:
   case IOCTL_VMX86_GET_KHZ_ESTIMATE:
@@ -1093,12 +1073,12 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      if (retval != 0) {
         break;
      }
-      vm = Vmx86_CreateVM(args.bsBlob, args.bsBlobSize);
+      vm = Vmx86_CreateVM(args.bsBlob, args.bsBlobSize, args.numVCPUs);

      if (vm == NULL) {
         retval = -ENOMEM;
      } else {
-         vmLinux->vm = vm;
+         device->vm = vm;
         args.vmid = vm->userID;
         retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args);
      }
@@ -1107,57 +1087,73 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:

   case IOCTL_VMX86_PROCESS_BOOTSTRAP: {
      VMProcessBootstrapBlock *args;
-      Bool res;
+      size_t argsSz = GetVMProcessBootstrapBlockSize(vm->numVCPUs);
+      VA64 uAddr;

-      args = HostIF_AllocKernelMem(sizeof *args, TRUE);
+      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
+      if (retval != 0) {
+         break;
+      }
+      args = HostIF_AllocKernelMem(argsSz, TRUE);
      if (args == NULL) {
         retval = -ENOMEM;
         break;
      }
-      retval = HostIF_CopyFromUser(args, (VA64)ioarg, sizeof *args);
+      retval = HostIF_CopyFromUser(args, uAddr, argsSz);
      if (retval != 0) {
         HostIF_FreeKernelMem(args);
         break;
      }
-      res = Vmx86_ProcessBootstrap(vm,
-                                   args->bsBlobAddr,
-                                   args->numBytes,
-                                   args->headerOffset,
-                                   args->numVCPUs,
-                                   args->ptRootVAs,
-                                   args->shRegions);
-      if (!res) {
+      if (args->numVCPUs != vm->numVCPUs) {
+         retval = -EINVAL;
+      } else if (!Vmx86_ProcessBootstrap(vm,
+                                         args->bsBlobAddr,
+                                         args->numBytes,
+                                         args->headerOffset,
+                                         args->numVCPUs,
+                                         args->perVcpuPages,
+                                         args->shRegions)) {
         retval = -ENOMEM;
      }
      HostIF_FreeKernelMem(args);
      break;
   }

-   case IOCTL_VMX86_RELEASE_VM:
-      vmLinux->vm = NULL;
-      Vmx86_ReleaseVM(vm);
-      break;
+   case IOCTL_VMX86_REGISTER_SHARED: {
+      VMSharedAreaRegistrationBlock args;

-   case IOCTL_VMX86_INIT_VM: {
-      InitBlock *initParams;
-
-      initParams = HostIF_AllocKernelMem(sizeof *initParams, TRUE);
-      if (initParams == NULL) {
-         retval = -ENOMEM;
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
         break;
      }
-      retval = HostIF_CopyFromUser(initParams, ioarg, sizeof *initParams);
-      if (retval == 0) {
-         if (Vmx86_InitVM(vm, initParams) == 0) {
-            retval = HostIF_CopyToUser(ioarg, initParams, sizeof *initParams);
-         } else {
-            retval = -EINVAL;
-         }
+      if (!SharedAreaVmmon_ValidateRegionArgs(vm, &args)) {
+         retval = -EINVAL;
+         break;
+      }
+      if (!SharedAreaVmmon_RegisterRegion(vm, &args)) {
+         retval = -ENOMEM;
      }
-      HostIF_FreeKernelMem(initParams);
      break;
   }

+   case IOCTL_VMX86_REGISTER_STATVARS: {
+      VMStatVarsRegistrationBlock args;
+
+      retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!StatVarsVmmon_RegisterVCPU(vm, &args)) {
+         retval = -ENOMEM;
+      }
+      break;
+   }
+
+   case IOCTL_VMX86_RELEASE_VM:
+      device->vm = NULL;
+      Vmx86_ReleaseVM(vm);
+      break;
+
   case IOCTL_VMX86_RUN_VM:
      vcpuid = ioarg;

@@ -1176,18 +1172,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
 #endif
      break;

-   case IOCTL_VMX86_LOCK_PAGE: {
-      VMLockPage args;
-
-      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
-      if (retval) {
-         break;
-      }
-      args.ret.status = Vmx86_LockPage(vm, args.uAddr, FALSE, &args.ret.mpn);
-      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
-      break;
-   }
-
+   case IOCTL_VMX86_LOCK_PAGE:
   case IOCTL_VMX86_LOCK_PAGE_NEW: {
      VMLockPage args;

@@ -1195,7 +1180,9 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      if (retval) {
         break;
      }
-      args.ret.status = Vmx86_LockPage(vm, args.uAddr, TRUE, &args.ret.mpn);
+      args.ret.status = Vmx86_LockPage(vm, args.uAddr,
+                                       iocmd == IOCTL_VMX86_LOCK_PAGE_NEW,
+                                       &args.ret.mpn);
      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
      break;
   }
@@ -1234,17 +1221,28 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_VMM_PAGE_ROOT: {
+      VcpuPageRoot args;
+
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval) {
+         break;
+      }
+      if (args.vcpuid >= vm->numVCPUs || vm->ptRootMpns == NULL) {
+         retval = -EINVAL;
+      } else {
+         args.pageRoot = vm->ptRootMpns[args.vcpuid];
+         retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      }
+      break;
+   }
+
   case IOCTL_VMX86_GET_NUM_VMS:
      retval = Vmx86_GetNumVMs();
      break;

-   case IOCTL_VMX86_GET_TOTAL_MEM_USAGE:
-      retval = Vmx86_GetTotalMemUsage();
-      break;
-
   case IOCTL_VMX86_SET_HARD_LIMIT: {
-      int32 limit;
-
+      PageCnt limit;
      retval = HostIF_CopyFromUser(&limit, ioarg, sizeof limit);
      if (retval != 0) {
         break;
@@ -1356,6 +1354,16 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      retval = -Vmx86_SetHostClockRate(vm, (unsigned)ioarg);
      break;

+   case IOCTL_VMX86_SEND_ONE_IPI: {
+      Vcpuid v = ioarg;
+      if (v < vm->numVCPUs) {
+         HostIF_OneIPI(vm, v);
+      } else {
+         retval = -EINVAL;
+      }
+      break;
+   }
+
   case IOCTL_VMX86_SEND_IPI: {
      VCPUSet ipiTargets;

@@ -1371,14 +1379,8 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   case IOCTL_VMX86_GET_IPI_VECTORS: {
      IPIVectors ipiVectors;

-      ipiVectors.hostIPIVectors[0] = CALL_FUNCTION_VECTOR;
-#ifdef CALL_FUNCTION_SINGLE_VECTOR
-      ipiVectors.hostIPIVectors[1] = CALL_FUNCTION_SINGLE_VECTOR;
-#else
-      ipiVectors.hostIPIVectors[1] = 0;
-#endif
-      ipiVectors.monitorIPIVector = monitorIPIVector;
-      ipiVectors.hvIPIVector      = hvIPIVector;
+      ipiVectors.monitorIPIVector = HostIF_GetMonitorIPIVector();
+      ipiVectors.hvIPIVector      = HostIF_GetHVIPIVector();

      retval = HostIF_CopyToUser(ioarg, &ipiVectors, sizeof ipiVectors);
      break;
@@ -1484,22 +1486,16 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

-   case IOCTL_VMX86_ALLOC_LOCKED_PAGES:
-   case IOCTL_VMX86_FREE_LOCKED_PAGES: {
+   case IOCTL_VMX86_ALLOC_LOCKED_PAGES: {
         VMMPNList req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
         if (retval) {
-           break;
-         }
-         if (iocmd == IOCTL_VMX86_ALLOC_LOCKED_PAGES) {
-            retval = Vmx86_AllocLockedPages(vm, req.mpnList,
-                                            req.mpnCount, FALSE,
-                                            req.ignoreLimits);
-         } else {
-            retval = Vmx86_FreeLockedPages(vm, req.mpnList,
-                                           req.mpnCount, FALSE);
+            break;
         }
+         retval = Vmx86_AllocLockedPages(vm, req.mpnList,
+                                         req.mpnCount, FALSE,
+                                         req.ignoreLimits);
         break;
      }

@@ -1516,8 +1512,14 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_NUM_ANON_PAGES: {
+      PageCnt numAnonPages;
+      numAnonPages = Vmx86_GetNumAnonPages(vm);
+      retval = HostIF_CopyToUser(ioarg, &numAnonPages, sizeof numAnonPages);
+      break;
+   }
+
   case IOCTL_VMX86_READ_PAGE: {
-         void *tempPage;
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1525,23 +1527,13 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
            break;
         }

-         tempPage = HostIF_AllocPage();
-         if (tempPage == NULL) {
-            retval = -ENOMEM;
-            break;
-         }
-
-         retval = HostIF_ReadPhysical(vm, MPN_2_MA(req.mpn),
-                                      PtrToVA64(tempPage), TRUE, PAGE_SIZE);
-         if (retval == 0) {
-            retval = HostIF_CopyToUser(req.uAddr, tempPage, PAGE_SIZE);
-         }
-
-         HostIF_FreePage(tempPage);
+         retval = HostIF_ReadPhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE,
+                                      PAGE_SIZE);
         break;
      }

   case IOCTL_VMX86_WRITE_PAGE: {
+#ifdef VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1550,6 +1542,9 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
         }
         retval = HostIF_WritePhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE,
                                       PAGE_SIZE);
+#else
+         retval = -EINVAL;
+#endif
         break;
      }

@@ -1608,7 +1603,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      uint64 swapSize;
      retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize);
      if (retval != 0) {
-         Warning("Could not copy swap size from user, status %d\n", retval);
+         Warning("Could not copy swap size from user, status %ld\n", retval);
         break;
      }
      linuxState.swapSize = swapSize;
@@ -1621,12 +1616,31 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_MONITOR_CONTEXT: {
+      VMMonContext args;
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!Vmx86_GetMonitorContext(vm, args.vcpuid, &args.context)) {
+         retval = -EINVAL;
+      } else {
+         retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      }
+      break;
+   }
+
   default:
      Warning("Unknown ioctl %d\n", iocmd);
      retval = -EINVAL;
   }

 exit:
+   if (needsWriteLock) {
+      up_write(&device->vmDriverRWSema);
+   } else {
+      up_read(&device->vmDriverRWSema);
+   }
   return retval;
 }

@@ -1636,7 +1650,7 @@ exit:
 *
 * LinuxDriverQueue --
 *
- *      add the vmLinux to the global queue
+ *      add the device to the global queue
 *
 * Results:
 *
@@ -1648,7 +1662,7 @@ exit:
 */

 static void
-LinuxDriverQueue(VMLinux *vmLinux)  // IN/OUT:
+LinuxDriverQueue(Device *device)  // IN/OUT:
 {
   /*
    * insert in global vm queue
@@ -1656,8 +1670,8 @@ LinuxDriverQueue(VMLinux *vmLinux)  // IN/OUT:

   HostIF_GlobalLock(12);

-   vmLinux->next = linuxState.head;
-   linuxState.head = vmLinux;
+   device->next = linuxState.head;
+   linuxState.head = device;

   HostIF_GlobalUnlock(12);
 }
@@ -1680,16 +1694,16 @@ LinuxDriverQueue(VMLinux *vmLinux)  // IN/OUT:
 */

 static void
-LinuxDriverDequeue(VMLinux *vmLinux)  // IN/OUT:
+LinuxDriverDequeue(Device *device)  // IN/OUT:
 {
-   VMLinux **p;
+   Device **p;

   HostIF_GlobalLock(13);
-   for (p = &linuxState.head; *p != vmLinux; p = &(*p)->next) {
+   for (p = &linuxState.head; *p != device; p = &(*p)->next) {
      ASSERT(*p != NULL);
   }
-   *p = vmLinux->next;
-   vmLinux->next = NULL;
+   *p = device->next;
+   device->next = NULL;
   HostIF_GlobalUnlock(13);
 }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2018-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -27,36 +27,31 @@
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/semaphore.h>
+#include <linux/rwsem.h>
 #include <linux/wait.h>

 #include "vmx86.h"
 #include "driver_vmcore.h"


-/*
- * Used to track a piece of memory that's been mapped into the kernel from
- * userlevel.
- */
-typedef struct VMMappedUserMem {
-   void *addr;
-   size_t numPages;
-   struct page *pages[0];
-} VMMappedUserMem;
-
-
-/*
- * Per-instance driver state
- */
-
+/* Per-instance driver state */
 struct VMDriver;

 /* 16 pages (64KB) looks as a good limit for one allocation */
 #define VMMON_MAX_LOWMEM_PAGES  16

-typedef struct VMLinux {
-   struct VMLinux *next;
+typedef struct Device {
+   struct Device   *next;
   struct VMDriver *vm;
-
+   /*
+    * This RW semaphore protects accesses to the VMDriver to
+    * avoid racing between various ioctls, and the creation
+    * and removal of the VM in question. The lock is read-acquired
+    * by ioctls that reference the VMDriver, and write-acquired by
+    * ioctls or device callbacks that allocate or destory the
+    * VMDriver.
+    */
+   struct rw_semaphore vmDriverRWSema;
   /*
    * The semaphore protect accesses to size4Gb and pages4Gb
    * in mmap(). mmap() may happen only once, and all other
@@ -64,9 +59,9 @@ typedef struct VMLinux {
    * only after successful mmap.
    */
   struct semaphore lock4Gb;
-   unsigned int size4Gb;
+   PageCnt size4Gb;
   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
-} VMLinux;
+} Device;


 /*
@@ -82,7 +77,7 @@ typedef struct VMXLinuxState {
   struct miscdevice misc;
   char deviceName[VM_DEVICE_NAME_SIZE];
   char buf[LINUXLOG_BUFFER_SIZE];
-   VMLinux *head;
+   Device *head;

   struct task_struct *fastClockThread;
   unsigned fastClockRate;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2015,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -25,12 +25,6 @@
 #ifndef _HOSTIF_PRIV_H_
 #define _HOSTIF_PRIV_H_

-/* Functions for mapping and unmapping userspace memory. */
-struct VMMappedUserMem;
-EXTERN void  *HostIF_MapUserMem(VA addr, size_t size,
-                                struct VMMappedUserMem **handle);
-EXTERN void   HostIF_UnmapUserMem(struct VMMappedUserMem *handle);
-
 /* Uptime-related functions. */
 EXTERN void   HostIF_InitUptime(void);
 EXTERN void   HostIF_CleanupUptime(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -79,22 +79,23 @@ typedef struct VMHost {
    */
   Mutex vmMutex;

-   unsigned int       crosspagePagesCount;
-   struct page        *crosspagePages[MAX_INITBLOCK_CPUS];
-   struct task_struct *vcpuSemaTask[MAX_INITBLOCK_CPUS];
+   unsigned int         crosspagePagesCount;
+   struct page        **crosspagePages;      /* ptr to numVCPUs-sized array */
+   struct task_struct **vcpuSemaTask;        /* ptr to numVCPUs-sized array */
+
   /*
    * Pages that were allocated/mapped by VMX and locked by the driver and
    * don't have a particular VA.
    */
-   struct PhysTracker *lockedPages;
+   struct PhysTracker  *lockedPages;
   /*
    * Locked pages that were allocated by the driver and don't have
    * a particular VA. They are used as monitor anonymous pages or
    * as pages for "AWE" guest memory.
    */
-   struct PhysTracker *AWEPages;
+   struct PhysTracker  *AWEPages;
   /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */
-   Bool               hostAPICIsMapped;
+   Bool                 hostAPICIsMapped;
 } VMHost;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,7 +24,7 @@
 *
 */

-#if defined(linux)
+#if defined(__linux__)
 /* Must come before any kernel header file */
 #   include "driver-config.h"
 #   include <linux/kernel.h>
@@ -34,7 +34,6 @@
 #include "modulecall.h"
 #include "vmx86.h"
 #include "task.h"
-#include "initblock.h"
 #include "vm_basic_asm.h"
 #include "iocontrols.h"
 #include "hostif.h"
@@ -42,6 +41,8 @@
 #include "driver_vmcore.h"
 #include "usercalldefs.h"
 #include "cpuid.h"
+#include "vmmblob.h"
+#include "sharedAreaVmmon.h"

 /*
 *----------------------------------------------------------------------
@@ -71,11 +72,15 @@ int
 Vmx86_RunVM(VMDriver *vm,   // IN:
            Vcpuid vcpuid)  // IN:
 {
-   uint32           retval    = MODULECALL_USERRETURN;
-   VMCrossPageData *crosspage = &vm->crosspage[vcpuid]->crosspageData;
+   uint64           retval    = MODULECALL_USERRETURN;
+   VMCrossPageData *crosspage;
   int              bailValue = 0;

-   ASSERT(crosspage && CPUID_HostSupportsHV());
+   ASSERT(vcpuid < vm->numVCPUs);
+   if (vm->crosspage[vcpuid] == NULL) {
+      return USERCALL_VMX86ALLOCERR;
+   }
+   crosspage = &vm->crosspage[vcpuid]->crosspageData;

   /*
    * Check if we were interrupted by signal.
@@ -98,13 +103,13 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
       * Wake up anything that was waiting for this vcpu to run
       */

-      if ((VCPUSet_IsEmpty(&crosspage->yieldVCPUs) &&
+      if ((crosspage->yieldVCPUsIsEmpty &&
           crosspage->moduleCallType != MODULECALL_COSCHED) ||
          crosspage->moduleCallType == MODULECALL_SEMAWAIT) {
         HostIF_WakeUpYielders(vm, vcpuid);
      }

-      if (!VCPUSet_IsEmpty(&crosspage->yieldVCPUs) &&
+      if (!crosspage->yieldVCPUsIsEmpty &&
          crosspage->moduleCallType != MODULECALL_COSCHED &&
          crosspage->moduleCallType != MODULECALL_SEMAWAIT) {
         Vmx86_YieldToSet(vm, vcpuid, &crosspage->yieldVCPUs, 0, TRUE);
@@ -120,7 +125,8 @@ skipTaskSwitch:;
          * from the ioctl (back to the userlevel side of a VCPU thread).
          */
         bailValue = crosspage->userCallType;
-         crosspage->retval = retval;
+         ASSERT(retval == (uint64)((uint32)retval));
+         crosspage->retval = (uint32)retval;
         goto bailOut;
      }

@@ -133,23 +139,30 @@ skipTaskSwitch:;

      case MODULECALL_GET_RECYCLED_PAGES: {
         MPN mpns[MODULECALL_NUM_ARGS];
-         int nPages = MIN((int)crosspage->args[0], MODULECALL_NUM_ARGS);
-
-         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages, TRUE,
-                                         FALSE);
+         PageCnt nPages = MIN(crosspage->args[0], MODULECALL_NUM_ARGS);
+         ASSERT((int64)crosspage->args[0] >= 0);
+         retval = Vmx86_AllocLockedPages(vm, PtrToVA64(mpns), nPages,
+                                         TRUE, FALSE);
         if (retval <= nPages) {
-            int i;
+            PageCnt i;
            for (i = 0; i < retval; i++) {
               crosspage->args[i] = mpns[i];
            }
         } else {
-            // retval is holding an error code
-            Warning("Failed to alloc %u pages: %d\n", nPages, (int)retval);
+            /* retval is holding an error code */
+            Warning("Failed to alloc %"FMT64"u pages: %"FMT64"d\n", nPages,
+                    (int64)retval);
            retval = 0;
         }
         break;
      }

+      case MODULECALL_ALLOC_ANON_LOW_PAGE: {
+         // Return via 64-bit args[0] (may return INVALID_MPN).
+         crosspage->args[0] = Vmx86_AllocLowPage(vm, FALSE);
+         break;
+      }
+
      case MODULECALL_SEMAWAIT: {
         retval = HostIF_SemaphoreWait(vm, vcpuid, crosspage->args);

@@ -177,15 +190,18 @@ skipTaskSwitch:;
         break;
      }

+      case MODULECALL_ONE_IPI: {
+         Vcpuid v = (Vcpuid)crosspage->args[0];
+         HostIF_OneIPI(vm, v);
+         break;
+      }
      case MODULECALL_IPI: {
-         HostIFIPIMode mode;
-         mode = HostIF_IPI(vm, &crosspage->vcpuSet);
-         retval = (mode != IPI_NONE);
+         HostIF_IPI(vm, &crosspage->vcpuSet);
         break;
      }

      case MODULECALL_RELEASE_ANON_PAGES: {
-         unsigned count;
+         PageCnt count;
         MPN mpns[MODULECALL_NUM_ARGS];
         for (count = 0; count < MODULECALL_NUM_ARGS; count++) {
            mpns[count] = (MPN)crosspage->args[count];
@@ -194,15 +210,14 @@ skipTaskSwitch:;
            }
         }
         ASSERT(count > 0);
-         retval = Vmx86_FreeLockedPages(vm, PtrToVA64(mpns), count, TRUE);
+         retval = Vmx86_FreeLockedPages(vm, mpns, count);
         break;
      }

      case MODULECALL_LOOKUP_MPN: {
-         int i;
-         VPN64  vpn    = (VPN64)crosspage->args[0];
-         uint32 nPages = (uint32)crosspage->args[1];
-         VA64   uAddr  = (VA64)VPN_2_VA(vpn);
+         VPN64   vpn        = (VPN64)crosspage->args[0];
+         PageCnt i, nPages  = crosspage->args[1];
+         VA64    uAddr      = (VA64)VPN_2_VA(vpn);
         ASSERT(nPages <= MODULECALL_NUM_ARGS);
         HostIF_VMLock(vm, 38);
         for (i = 0; i < nPages; i++) {
@@ -260,17 +275,56 @@ skipTaskSwitch:;
         crosspage->args[0] = mpn;
      } break;

+      case MODULECALL_GET_MON_IPI_VECTOR: {
+         retval = HostIF_GetMonitorIPIVector();
+      } break;
+
+      case MODULECALL_GET_HV_IPI_VECTOR: {
+         retval = HostIF_GetHVIPIVector();
+      } break;
+
+      case MODULECALL_GET_HOST_TIMER_VECTORS: {
+         uint8 v0, v1;
+         HostIF_GetTimerVectors(&v0, &v1);
+         crosspage->args[0] = v0;
+         crosspage->args[1] = v1;
+      } break;
+
+      case MODULECALL_BOOTSTRAP_CLEANUP: {
+         VmmBlob_Cleanup(vm->blobInfo);
+         vm->blobInfo = NULL;
+      } break;
+
+      case MODULECALL_GET_SHARED_AREA: {
+         SharedAreaVmmonRequest request;
+         request.type       = (SharedAreaType)crosspage->args[0];
+         request.vcpu       = (Vcpuid)crosspage->args[1];
+         request.offset     = (PageCnt)crosspage->args[2];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[3] = SharedAreaVmmon_GetRegionMPN(vm, &request);
+      } break;
+
+      case MODULECALL_GET_STAT_VARS: {
+         Vcpuid  vcpu   = (Vcpuid)crosspage->args[0];
+         PageCnt offset = (PageCnt)crosspage->args[1];
+         /* Store MPN result in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[2] = StatVarsVmmon_GetRegionMPN(vm, vcpu, offset);
+      } break;
+
+      case MODULECALL_GET_NUM_PTP_PAGES: {
+         /* Store PageCnt in crosspage arg as crosspage retval is 32 bit. */
+         crosspage->args[1] = vm->numPTPPages;
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
-
-      crosspage->retval = retval;
-
-#if defined(linux)
+      ASSERT(retval == (uint64)((uint32)retval));
+      crosspage->retval = (uint32)retval;
+#if defined(__linux__)
      cond_resched(); // Other kernels are preemptable
 #endif
   }
-
 bailOut:
   return bailValue;
 }
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -98,6 +98,13 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

+# Use SUBDIRS on 2.x, 3.x, 4.x.  Use M on newer kernels.
+ifeq ($(filter-out 2 3 4,$(firstword $(subst ., ,$(VM_UNAME)))),)
+DIRVAR := SUBDIRS
+else
+DIRVAR := M
+endif
+
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined
@@ -107,7 +114,7 @@ prebuild:: ;
 postbuild:: ;

 $(DRIVER_KO): prebuild
-	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
+	$(MAKE) -C $(BUILD_DIR) $(DIRVAR)=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
 	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
 	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,8 @@

 #define ETHER_MAX_QUEUED_PACKET 1600

+/* Most Ethernet equipment can support jumbo frames up to 9216 bytes. */
+#define ETHER_MAX_JUMBO_FRAME_LEN 9216

 /*
 * State's that a NIC can be in currently we only use this
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -28,6 +28,7 @@

 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
 #include <linux/mm.h>
 #include "compat_skbuff.h"
 #include <linux/sockios.h>
@@ -44,6 +45,22 @@
 #include "vmnetInt.h"


+/*
+ * Default min MTU value as defined by kernel versions >= 4.10.0.
+ * Use the same value for earlier versions of the kernel which do not
+ * enforce a minimum MTU size.
+ */
+#define VMNET_MIN_MTU 68
+#define VMNET_MAX_MTU (ETHER_MAX_JUMBO_FRAME_LEN - \
+                      (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
+
+#ifdef RHEL_RELEASE_CODE
+#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 5) && \
+     RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8, 0))
+#define ndo_change_mtu ndo_change_mtu_rh74
+#endif
+#endif /* RHEL_RELEASE_CODE */
+
 typedef struct VNetNetIF {
   VNetPort                port;
   struct net_device      *dev;
@@ -64,6 +81,11 @@ static int  VNetNetifSetMAC(struct net_device *dev, void *addr);
 static void VNetNetifSetMulticast(struct net_device *dev);
 static int  VNetNetIfProcRead(char *page, char **start, off_t off,
                              int count, int *eof, void *data);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+static int VNetNetifChangeMtu(struct net_device *dev, int new_mtu);
+#endif
+

 /*
 *----------------------------------------------------------------------
@@ -84,7 +106,7 @@ static int  VNetNetIfProcRead(char *page, char **start, off_t off,
 static void
 VNetNetIfSetup(struct net_device *dev)  // IN:
 {
-#if !COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) || defined(HAVE_NET_DEVICE_OPS)
+#ifdef HAVE_NET_DEVICE_OPS
   static const struct net_device_ops vnetNetifOps = {
      .ndo_init = VNetNetifProbe,
      .ndo_open = VNetNetifOpen,
@@ -97,12 +119,23 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
 #else
      .ndo_set_rx_mode = VNetNetifSetMulticast,
 #endif
-
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
+      .ndo_change_mtu = VNetNetifChangeMtu,
+#endif
   };
+#endif /* HAVE_NET_DEVICE_OPS */
+
+   /* Turns on IFF_BROADCAST, IFF_MULTICAST. */
+   ether_setup(dev);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+   /* Set the maximum allowed MTU value. */
+   dev->max_mtu = VMNET_MAX_MTU;
 #endif

-   ether_setup(dev); // turns on IFF_BROADCAST, IFF_MULTICAST
-#if COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0) && !defined(HAVE_NET_DEVICE_OPS) 
+#ifdef HAVE_NET_DEVICE_OPS
+   dev->netdev_ops = &vnetNetifOps;
+#else
   dev->init = VNetNetifProbe;
   dev->open = VNetNetifOpen;
   dev->hard_start_xmit = VNetNetifStartXmit;
@@ -110,9 +143,10 @@ VNetNetIfSetup(struct net_device *dev)  // IN:
   dev->get_stats = VNetNetifGetStats;
   dev->set_mac_address = VNetNetifSetMAC;
   dev->set_multicast_list = VNetNetifSetMulticast;
-#else
-   dev->netdev_ops = &vnetNetifOps;
-   #endif /* HAVE_NET_DEVICE_OPS */
+#ifdef HAVE_CHANGE_MTU
+   dev->change_mtu = VNetNetifChangeMtu;
+#endif /* HAVE_CHANGE_MTU */
+#endif /* HAVE_NET_DEVICE_OPS */

 }

@@ -530,6 +564,38 @@ VNetNetifSetMulticast(struct net_device *dev) // IN: unused
 }


+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+    (defined(HAVE_NET_DEVICE_OPS) || defined(HAVE_CHANGE_MTU))
+/*
+ *----------------------------------------------------------------------
+ *
+ * VNetNetifChangeMtu --
+ *
+ *      Changes the current MTU value of a given vmnet interface.
+ *
+ * Results:
+ *      Returns zero on success, or invalid argument error if an incorrect
+ *      MTU size is being set.
+ *
+ * Side effects:
+ *      None.
+ *----------------------------------------------------------------------
+ */
+
+int
+VNetNetifChangeMtu(struct net_device *dev, // IN/OUT: assigning the mtu member.
+                   int new_mtu)            // IN:     new mtu value.
+{
+   if (new_mtu < VMNET_MIN_MTU || new_mtu > VMNET_MAX_MTU) {
+      return -EINVAL;
+   }
+
+   dev->mtu = new_mtu;
+   return 0;
+}
+#endif
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,8 +41,6 @@

 #else /* _WIN32 */

-#undef __KERNEL__ //To prevent including any kernel specific stuff 
-
 #ifdef VMX86_DEVEL
 #define DBG 1
 #else
@@ -56,6 +56,11 @@

 #ifdef VMX86_DEVEL
 #define DBG 1
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)
+#define read_skb_users(skb) atomic_read(&skb->users)
+#else
+#define read_skb_users(skb) refcount_read(&skb->users)
+#endif
 #else
 #undef DBG
 #endif /* VMX86_DEVEL */
@@ -405,7 +410,7 @@ SMACL_PrintSkb(struct sk_buff *skb,          // IN: sk_buff structure
   LOG(4, (KERN_DEBUG "pkt_type %x truesize %u protocol %u\n",
            skb->pkt_type, skb->truesize, skb->protocol));
   LOG(4, (KERN_DEBUG "users %d, tail %p, end %p\n",
-            atomic_read(&skb->users), compat_skb_tail_pointer(skb),
+            read_skb_users(skb), compat_skb_tail_pointer(skb),
            compat_skb_end_pointer(skb)));
 #if 0
 #define C skb->mac.raw
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -137,16 +137,21 @@ UserifLockPage(VA addr) // IN
 */

 static INLINE int
-VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapPtr(VA64 uAddr,      // IN: pointer to user memory
                 size_t size,     // IN: size of data
                 struct page **p, // OUT: locked page
                 void **ptr)      // OUT: kernel mapped pointer
 {
-   if (!access_ok(VERIFY_WRITE, (void *)uAddr, size) ||
-       (((uAddr + size - 1) & ~(PAGE_SIZE - 1)) !=
-        (uAddr & ~(PAGE_SIZE - 1)))) {
+   uint8 v;
+
+   /* Check area does not straddle two pages. */
+   if ((uAddr & (PAGE_SIZE - 1)) + size > PAGE_SIZE) {
      return -EINVAL;
   }
+   /* Check if it is user's area.  UserifLockPage() checks writability. */
+   if (copy_from_user(&v, (void *)(unsigned long)uAddr, sizeof v) != 0) {
+      return -EFAULT;
+   }

   *p = UserifLockPage(uAddr);
   if (*p == NULL) {
@@ -158,7 +163,7 @@ VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
 }

 static INLINE int
-VNetUserIfMapUint32Ptr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapUint32Ptr(VA64 uAddr,      // IN: pointer to user memory
                       struct page **p, // OUT: locked page
                       uint32 **ptr)    // OUT: kernel mapped pointer
 {
@@ -201,7 +206,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      return -EBUSY;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->pollPtr, &pollPage,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->pollPtr, &pollPage,
                                        &pollPtr)) < 0) {
      return retval;
   }
@@ -213,7 +218,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      goto error_free;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->recvClusterPtr,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->recvClusterPtr,
                                        &recvClusterPage,
                                        &recvClusterCount)) < 0) {
      goto error_free;
@@ -362,12 +367,12 @@ VNetUserIfReceive(VNetJack       *this, // IN
   VNetUserIF *userIf = (VNetUserIF*)this->private;
   uint8 *dest = SKB_2_DESTMAC(skb);
   unsigned long flags;
-   
+
   if (!UP_AND_RUNNING(userIf->port.flags)) {
      userIf->stats.droppedDown++;
      goto drop_packet;
   }
-   
+
   if (!VNetPacketMatch(dest,
                        userIf->port.paddr,
                        (const uint8 *)userIf->port.exactFilter,
@@ -377,13 +382,17 @@ VNetUserIfReceive(VNetJack       *this, // IN
      userIf->stats.droppedMismatch++;
      goto drop_packet;
   }
-   
+
   if (skb_queue_len(&userIf->packetQueue) >= vnet_max_qlen) {
      userIf->stats.droppedOverflow++;
      goto drop_packet;
   }
-   
-   if (skb->len > ETHER_MAX_QUEUED_PACKET) {
+
+   /*
+    * The check would be more accurate if based on the current MTU value
+    * of the corresponding vmnet interface. PR 2267716 is filed to track this.
+    */
+   if (skb->len > ETHER_MAX_JUMBO_FRAME_LEN) {
      userIf->stats.droppedLargePacket++;
      goto drop_packet;
   }
@@ -747,7 +756,7 @@ VNetUserIfRead(VNetPort    *port, // IN
 *----------------------------------------------------------------------
 */

-static int 
+static int
 VNetUserIfWrite(VNetPort    *port, // IN
                struct file *filp, // IN
                const char  *buf,  // IN
@@ -757,11 +766,14 @@ VNetUserIfWrite(VNetPort    *port, // IN
   struct sk_buff *skb;

   /*
-    * Check size
+    * Check size.
+    *
+    * Regarding the upper count limit the check would be more accurate
+    * if based on the current MTU value of the corresponding vmnet interface.
+    * PR 2267716 is filed to track this.
    */
-   
-   if (count < sizeof (struct ethhdr) || 
-       count > ETHER_MAX_QUEUED_PACKET) {
+   if (count < sizeof (struct ethhdr) ||
+       count > ETHER_MAX_JUMBO_FRAME_LEN) {
      return -EINVAL;
   }

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -202,7 +202,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

 #define NOT_TESTED_1024()                                               \
   do {                                                                 \
-      static uint16 count = 0;                                          \
+      static MONITOR_ONLY(PERVCPU) uint16 count = 0;                    \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,38 +49,68 @@ extern "C" {
 #endif

 /*
- * In the Atomic_* definitions below, memory ordering and atomicity are somewhat
- * conflated in an inconsistent manner. First, we have Atomic_{Read,Write},
- * which only guarantees single copy atomicity, i.e. that the read/write occurs
- * in an atomic fashion, but have no implication on memory ordering. The second
- * class of Atomics are all the non-unfenced operations excluding
- * Atomic_{Read,Write}*, which both imply atomicity and act as a memory barrier,
- * implying sequentially consistent ordering of the atomic operation with all
- * loads/stores prior to and after it.
+ * There are two concepts involved when dealing with atomic accesses:
+ * 1. Atomicity of the access itself
+ * 2. Ordering of the access with respect to other reads&writes (from the view
+ *    of other processors/devices).
 *
- * Since on x86, the second class of operations are associated with LOCK
- * semantics, assumptions have been made about the ordering these operations
- * imply on surrounding code (see for example the vmkernel's RefCount
- * implementation). As a result, on arm64 we have to provide these same
- * guarantees. We do this by making use of DMB barriers both before and after
- * the atomic ldrx/strx sequences. A barrier before and after is required to
- * avoid having part of the atomic operation reordered with surrounding code,
- * e.g. a store-load reordering of the strx with a following load outside the
- * Atomic_ op. For the first class of operations, Atomic_{Read,Write}, we do not
- * implement a barrier.
+ * Two examples help to clarify #2:
+ * a. Inc: A caller implementing a simple independent global event counter
+ *         might not care if the compiler or processor visibly reorders the
+ *         increment around other memory accesses.
+ * b. Dec: A caller implementing a reference count absolutely *doesn't* want
+ *         the compiler or processor to visibly reordering writes after that
+ *         decrement: if that happened, the program could then end up writing
+ *         to memory that was freed by another processor.
 *
- * This implementation of Atomic operations is suboptimal on arm64, since
- * both atomicity and memory ordering are fused together. Ideally the Atomic
- * operations would only imply atomicity, and an explicit memory barrier in the
- * surrounding code used to enforce ordering where necessary. This would eschew
- * the need for the DMBs. A middle ground can be implemented where we use the
- * arm64 load-acquire/store-release exclusive instructions to implement Atomics.
- * This would imply sequential consistency of the Atomic operations (but not
- * with any of the surrounding non-atomic operations) without the need for a
- * DMB. Using these without a DMB today can still result in problematic
- * reordering by the processor with surrounding non-atomic operations, e.g. a
- * store-load reordering with a stlxr. Future optimization for arm64 should
- * consider the wider change required at the call sites to minimize DMBs.
+ * C11 has standardized a good model for expressing these orderings when doing
+ * atomics. It defines three *tiers* of ordering:
+ * 1. Sequential Consistency (every processor sees the same total order of
+ *    events)
+ *
+ * 2. Acquire/Release ordering (roughly, everybody can agree previous events
+ *    have completed, but they might disagree on the ordering of previous
+ *    independent events).
+ *
+ *    The relative ordering provided by this tier is sufficient for common
+ *    locking and initialization activities, but is insufficient for unusual
+ *    synchronization schemes (e.g. IRIW aka Independent Read Independent
+ *    Write designs such Dekker's algorithm, Peterson's algorithm, etc.)
+ *
+ *    In other words, this tier is close in behavior to Sequential Consistency
+ *    in much the same way a General-Relativity universe is close to a
+ *    Newtonian universe.
+ * 3. Relaxed (i.e unordered/unfenced)
+ *
+ * In C11 standard's terminology for atomic memory ordering,
+ * - in case (a) we want "relaxed" ordering for perf and,
+ * - in case (b) we want "sequentially consistent" ordering (or perhaps the
+ *   only slightly weaker "release" ordering) for correctness.
+ *
+ * There are standardized mappings of operations to orderings for every
+ * processor architecture. See
+ * - https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
+ * - http://preshing.com/20120913/acquire-and-release-semantics/
+ *
+ * In this file:
+ * 1. all RMW (Read/Modify/Write) operations are sequentially consistent.
+ *    This includes operations like Atomic_IncN, Atomic_ReadIfEqualWriteN,
+ *    Atomic_ReadWriteN, etc.
+ * 2. all R and W operations are relaxed. This includes operations like
+ *    Atomic_WriteN, Atomic_ReadN, Atomic_TestBitN, etc.
+ *
+ * The below routines of course ensure both the CPU and compiler honor the
+ * ordering constraint.
+ *
+ * Notes:
+ * 1. Since R-only and W-only operations do not provide ordering, callers
+ *    using them for synchronizing operations like double-checked
+ *    initialization or releasing spinlocks must provide extra barriers.
+ * 2. This implementation of Atomic operations is suboptimal. On x86,simple
+ *    reads and writes have acquire/release semantics at the hardware level.
+ *    On arm64, we have separate instructions for sequentially consistent
+ *    reads and writes (the same instructions are used for acquire/release).
+ *    Neither of these are exposed for R-only or W-only callers.
 *
 * For further details on x86 and ARM memory ordering see
 * https://wiki.eng.vmware.com/ARM/MemoryOrdering.
@@ -118,10 +148,7 @@ typedef struct Atomic_uint128 {
 /*
 * Prototypes for msft atomics.  These are defined & inlined by the
 * compiler so no function definition is needed.  The prototypes are
- * needed for C++.  Since amd64 compiler doesn't support inline asm we
- * have to use these.  Unfortunately, we still have to use some inline asm
- * for the 32 bit code since the and/or/xor implementations didn't show up
- * until XP or 2k3.
+ * needed for C++.
 *
 * The declarations for the intrinsic functions were taken from ntddk.h
 * in the DDK. The declarations must match otherwise the 64-bit C++
@@ -130,7 +157,7 @@ typedef struct Atomic_uint128 {
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
 */
-#if defined _MSC_VER && _MSC_VER >= 1310 && !defined BORA_NO_WIN32_INTRINS
+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -222,38 +249,10 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:


 /*
- * All the assembly code is tricky and written conservatively.
- * For example, to make sure gcc won't introduce copies,
- * we force the addressing mode like this:
- *
- *    "xchgl %0, (%1)"
- *    : "=r" (val)
- *    : "r" (&var->value),
- *      "0" (val)
- *    : "memory"
- *
- * - edward
- *
- * Actually - turns out that gcc never generates memory aliases (it
- * still does generate register aliases though), so we can be a bit
- * more agressive with the memory constraints. The code above can be
- * modified like this:
- *
- *    "xchgl %0, %1"
- *    : "=r" (val),
- *      "=m" (var->value),
- *    : "0" (val),
- *      "1" (var->value)
- *
- * The advantages are that gcc can use whatever addressing mode it
- * likes to access the memory value, and that we dont have to use a
- * way-too-generic "memory" clobber as there is now an explicit
- * declaration that var->value is modified.
- *
- * see also /usr/include/asm/atomic.h to convince yourself this is a
- * valid optimization.
- *
- * - walken
+ * The Read/Modify/Write operations on x86/x64 are all written using the
+ * "memory" constraint. This is to ensure the compiler treats the operation as
+ * a full barrier, flushing any pending/cached state currently residing in
+ * registers.
 */

 #if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
@@ -403,6 +402,7 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "=q" (val),
        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER && _MSC_VER >= 1600
@@ -496,7 +496,7 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
        "+m" (var->value)
      : "q" (newVal),
        "0" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );

   return val;
@@ -1002,26 +1002,15 @@ Atomic_ReadWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchange((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint32.value, eax
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif // _MSC_VER >= 1310
-#else
 #error No compiler defined for Atomic_ReadWrite
 #endif // __GNUC__
 }
@@ -1162,30 +1151,17 @@ Atomic_ReadIfEqualWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgl %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedCompareExchange((long *)&var->value,
-				      (long)newVal,
-				      (long)oldVal);
-#else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, oldVal
-      __asm mov ebx, var
-      __asm mov ecx, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint32.value, ecx
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif
+                                      (long)newVal,
+                                      (long)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite
 #endif
@@ -1267,17 +1243,17 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgq %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif //VM_ARM_V7
 #elif defined _MSC_VER
   return _InterlockedCompareExchange64((__int64 *)&var->value,
-					(__int64)newVal,
-					(__int64)oldVal);
+                                        (__int64)newVal,
+                                        (__int64)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite64
 #endif
@@ -1332,7 +1308,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
      "lock; andl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1341,7 +1317,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock and [ebx]Atomic_uint32.value, eax
+   __asm lock And [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_And
@@ -1397,7 +1373,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
      "lock; orl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1406,7 +1382,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock or [ebx]Atomic_uint32.value, eax
+   __asm lock Or [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Or
@@ -1462,7 +1438,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
      "lock; xorl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1471,7 +1447,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
 #else
   __asm mov eax, val
   __asm mov ebx, var
-   __asm lock xor [ebx]Atomic_uint32.value, eax
+   __asm lock Xor [ebx]Atomic_uint32.value, eax
 #endif
 #else
 #error No compiler defined for Atomic_Xor
@@ -1510,7 +1486,7 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT
      "lock; xorq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -1569,17 +1545,11 @@ Atomic_Add32(Atomic_uint32 *var, // IN/OUT
      "lock; addl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedExchangeAdd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock add [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Add
 #endif
@@ -1634,17 +1604,11 @@ Atomic_Sub32(Atomic_uint32 *var, // IN/OUT
      "lock; subl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
-   _InterlockedExchangeAdd((long *)&var->value, (long)-val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock sub [ebx]Atomic_uint32.value, eax
-#endif
+   _InterlockedExchangeAdd((long *)&var->value, -(long)val);
 #else
 #error No compiler defined for Atomic_Sub
 #endif
@@ -1680,16 +1644,11 @@ Atomic_Inc32(Atomic_uint32 *var) // IN/OUT
      "lock; incl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedIncrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock inc [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Inc
 #endif
@@ -1725,16 +1684,11 @@ Atomic_Dec32(Atomic_uint32 *var) // IN/OUT
      "lock; decl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedDecrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock dec [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Dec
 #endif
@@ -1939,26 +1893,15 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchangeAdd((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm lock xadd [ebx]Atomic_uint32.value, eax
-   }
-#pragma warning(pop)
-#endif
-#else
 #error No compiler defined for Atomic_ReadAdd32
 #endif
 }
@@ -2044,11 +1987,11 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchgq %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
 #else /* 32-bit version for non-ARM */
   typedef struct {
@@ -2088,9 +2031,9 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b (%3)" "\n\t"
      "xchgl %%ebx, %6"      "\n\t"
      "sete %0"
-      :	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+      : "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : /*
         * See the "Rules for __asm__ statements in __PIC__ code" above: %3
         * must use a register class which does not contain %ebx.
@@ -2111,14 +2054,14 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+        "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : "2" (((S_uint64 *)&oldVal)->lowValue),
        "3" (((S_uint64 *)&oldVal)->highValue),
        "b" (((S_uint64 *)&newVal)->lowValue),
        "c" (((S_uint64 *)&newVal)->highValue)
-      : "cc"
+      : "cc", "memory"
   );
 #   endif
 #endif
@@ -2166,11 +2109,11 @@ Atomic_CMPXCHG32(Atomic_uint32 *var,   // IN/OUT
      "lock; cmpxchgl %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
   return equal;
 #endif /* VM_X86_ANY */
@@ -2247,7 +2190,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
 #   pragma warning(push)
-#   pragma warning(disable : 4035)		// disable no-return warning
+#   pragma warning(disable : 4035)      // disable no-return warning
   {
      __asm mov ecx, var
      __asm mov edx, ecx
@@ -2330,9 +2273,9 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2360,7 +2303,10 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
 *
 * Atomic_ReadSub64 --
 *
- *      Atomically subtracts a 64-bit integer to another
+ *      Atomically subtracts a 64-bit integer from another.
+ *
+ *      Note: It is expected that val <= var.  If untrue, the result
+ *            cannot be represented in an unsigned type.
 *
 * Results:
 *      Returns the old value just prior to the subtraction
@@ -2378,7 +2324,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -val);
+   return Atomic_ReadAdd64(var, (uint64)-(int64)val);
 #endif
 }

@@ -2425,7 +2371,7 @@ Atomic_ReadInc64(Atomic_uint64 *var) // IN/OUT
 static INLINE uint64
 Atomic_ReadDec64(Atomic_uint64 *var) // IN/OUT
 {
-   return Atomic_ReadAdd64(var, CONST64U(-1));
+   return Atomic_ReadAdd64(var, (uint64)CONST64(-1));
 }


@@ -2460,7 +2406,7 @@ Atomic_Add64(Atomic_uint64 *var, // IN/OUT
      "lock; addq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2502,7 +2448,7 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT
      "lock; subq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2542,7 +2488,7 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT
      "lock; incq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedIncrement64((__int64 *)&var->value);
@@ -2581,7 +2527,7 @@ Atomic_Dec64(Atomic_uint64 *var) // IN/OUT
      "lock; decq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedDecrement64((__int64 *)&var->value);
@@ -2617,8 +2563,9 @@ Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2726,7 +2673,7 @@ Atomic_Or64(Atomic_uint64 *var, // IN/OUT
      "lock; orq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedOr64((__int64 *)&var->value, (__int64)val);
@@ -2773,7 +2720,7 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT
      "lock; andq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedAnd64((__int64 *)&var->value, (__int64)val);
@@ -2811,29 +2758,17 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT
-                uint64 bit)         // IN
+                unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; bts %1, %0"
+      "lock; btsq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal | (CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_SetBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2863,29 +2798,17 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT
-                  uint64 bit)         // IN
+                  unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; btr %1, %0"
+      "lock; btrq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal & ~(CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_ClearBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2915,29 +2838,65 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE Bool
 Atomic_TestBit64(Atomic_uint64 *var, // IN
-                 uint64 bit)         // IN
+                 unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
-   Bool out = FALSE;
+   Bool out;
   ASSERT(bit <= 63);
+#if defined __x86_64__ && defined __GNUC__
   __asm__ __volatile__(
-      "bt %2, %1; setc %0"
+      "btq %2, %1; setc %0"
      : "=rm"(out)
      : "m" (var->value),
-        "rJ" (bit)
+        "rJ" ((uint64)bit)
      : "cc"
   );
-   return out;
-#elif defined _MSC_VER
-   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
 #else
-#error No compiler defined for Atomic_TestBit64
+   out = (var->value & (CONST64U(1) << bit)) != 0;
 #endif
-#else // __x86_64__
+   return out;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBit64 --
+ *
+ *      Atomically test and set the bit 'bit' in var.
+ *      Bit must be between 0 and 63.
+ *
+ * Results:
+ *      TRUE if the tested bit was set; else FALSE.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBit64(Atomic_uint64 *var, // IN/OUT
+                    unsigned bit)       // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool out;
   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
+   __asm__ __volatile__(
+      "lock; btsq %2, %1; setc %0"
+      : "=rm" (out), "+m" (var->value)
+      : "rJ" ((uint64)bit)
+      : "cc", "memory"
+   );
+   return out;
+#else
+   uint64 oldVal;
+   uint64 mask;
+   ASSERT(bit <= 63);
+   mask = CONST64U(1) << bit;
+   do {
+      oldVal = var->value;
+   } while (!Atomic_CMPXCHG64(var, oldVal, oldVal | mask));
+   return (oldVal & mask) != 0;
 #endif
 }

@@ -3020,8 +2979,9 @@ Atomic_ReadWrite16(Atomic_uint16 *var,  // IN/OUT:
   __asm__ __volatile__(
      "xchgw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3128,10 +3088,10 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var,   // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgw %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3192,7 +3152,7 @@ Atomic_And16(Atomic_uint16 *var, // IN/OUT
      "lock; andw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3249,7 +3209,7 @@ Atomic_Or16(Atomic_uint16 *var, // IN/OUT
      "lock; orw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3306,7 +3266,7 @@ Atomic_Xor16(Atomic_uint16 *var, // IN/OUT
      "lock; xorw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3363,7 +3323,7 @@ Atomic_Add16(Atomic_uint16 *var, // IN/OUT
      "lock; addw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3420,7 +3380,7 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT
      "lock; subw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3476,7 +3436,7 @@ Atomic_Inc16(Atomic_uint16 *var) // IN/OUT
      "lock; incw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Add16(var, 1);
@@ -3512,7 +3472,7 @@ Atomic_Dec16(Atomic_uint16 *var) // IN/OUT
      "lock; decw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Sub16(var, 1);
@@ -3582,9 +3542,9 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
   __asm__ __volatile__(
      "lock; xaddw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3661,7 +3621,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 static INLINE uint16
 Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
 {
-   return Atomic_ReadAdd16(var, -1);
+   return Atomic_ReadAdd16(var, (uint16)-1);
 }
 #endif

@@ -3715,7 +3675,7 @@ Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
                            in val)                                           \
   {                                                                          \
      return (out)(cast)Atomic_ReadWrite ## size(var,                         \
-		(uint ## size)(cast)val);                                     \
+                (uint ## size)(cast)val);                                     \
   }                                                                          \
                                                                              \
                                                                              \
@@ -3876,6 +3836,109 @@ MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t)
 MAKE_ATOMIC_TYPE(Int, 32, int, int, int)
 MAKE_ATOMIC_TYPE(Bool, 8, Bool, Bool, Bool)

+/*
+ * Define arbitrary sized bit vector to be used by
+ * Atomic_TestSetBitVector and Atomic_TestClearBitVector.
+ */
+#define ATOMIC_BITVECTOR(varName, capacity) \
+      Atomic_uint8 varName[CEILING(capacity, 8)]
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBitVector --
+ *
+ *      Atomically test and set the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBitVector(Atomic_uint8 *var, // IN/OUT
+                        unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; bts %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadOr8(var + index / 8, bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestClearBitVector --
+ *
+ *      Atomically test and clear the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestClearBitVector(Atomic_uint8 *var, // IN/OUT
+                          unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; btr %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadAnd8(var + index / 8, ~bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestBitVector --
+ *
+ *      Test the bit 'index' (zero-based) in bit vector var.
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestBitVector(const Atomic_uint8 *var, // IN
+                     unsigned index)          // IN
+{
+   uint8 bit = 1 << index % 8;
+   return (Atomic_Read8(var + index / 8) & bit) != 0;
+}
+
+
 #ifdef VM_ARM_64
 #   include "vm_atomic_arm64_end.h"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -221,8 +221,9 @@ mssb64_0(const uint64 value)
 #endif

 static INLINE int
-lssb32_0(uint32 value)
+lssb32_0(uint32 v)
 {
+   int value = (int)v;
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -262,8 +263,10 @@ mssb32_0(uint32 value)
 }

 static INLINE int
-lssb64_0(const uint64 value)
+lssb64_0(const uint64 v)
 {
+   int64 value = (int64)v;
+
 #ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
@@ -373,10 +376,10 @@ lssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 lssbPtr(const uintptr_t value)
 {
-   return lssbPtr_0(value) + 1;
+   return (unsigned)lssbPtr_0(value) + 1;
 }

 static INLINE int
@@ -389,34 +392,34 @@ mssbPtr_0(const uintptr_t value)
 #endif
 }

-static INLINE int
+static INLINE unsigned
 mssbPtr(const uintptr_t value)
 {
-   return mssbPtr_0(value) + 1;
+   return (unsigned)mssbPtr_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb32(const uint32 value)
 {
-   return lssb32_0(value) + 1;
+   return (unsigned)lssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb32(const uint32 value)
 {
-   return mssb32_0(value) + 1;
+   return (unsigned)mssb32_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 lssb64(const uint64 value)
 {
-   return lssb64_0(value) + 1;
+   return (unsigned)lssb64_0(value) + 1;
 }

-static INLINE int
+static INLINE unsigned
 mssb64(const uint64 value)
 {
-   return mssb64_0(value) + 1;
+   return (unsigned)mssb64_0(value) + 1;
 }

 #ifdef __GNUC__
@@ -768,12 +771,14 @@ static INLINE void
 PAUSE(void)
 #if defined(__GNUC__) || defined(VM_ARM_32)
 {
-#ifdef VM_ARM_ANY
+#ifdef VM_ARM_64
+   __asm__ __volatile__("yield");
+#elif defined VM_ARM_32
   /*
-    * ARM has no instruction to execute "spin-wait loop", just leave it
-    * empty.
+    * YIELD is available in ARMv6K and above, so we could probably refine this
+    * instead of leaving it empty.
    */
-#else
+#else // x86
   __asm__ __volatile__( "pause" :);
 #endif
 }
@@ -888,7 +893,7 @@ RDTSC(void)
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
- *    Sets or clears a specified single bit in the provided variable.
+ *    Sets tests or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
@@ -902,111 +907,39 @@ RDTSC(void)
 */

 static INLINE void
-SetBit32(uint32 *var, uint32 index)
+SetBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset((long *)var, index);
-#else
-   *var |= (1 << index);
-#endif
+   *var |= 1 << index;
 }

 static INLINE void
-ClearBit32(uint32 *var, uint32 index)
+ClearBit32(uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   __asm__ (
-      "btr %1, %0"
-      : "+mr" (*var)
-      : "rI" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset((long *)var, index);
-#else
   *var &= ~(1 << index);
-#endif
 }

 static INLINE void
-SetBit64(uint64 *var, uint64 index)
+SetBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "bts %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandset64((__int64 *)var, index);
-#endif
-#else
-   *var |= ((uint64)1 << index);
-#endif
+   *var |= CONST64U(1) << index;
 }

 static INLINE void
-ClearBit64(uint64 *var, uint64 index)
+ClearBit64(uint64 *var, unsigned index)
 {
-#if defined(VM_64BIT) && !defined(VM_ARM_64)
-#ifdef __GNUC__
-   __asm__ (
-      "btrq %1, %0"
-      : "+mr" (*var)
-      : "rJ" (index)
-      : "cc"
-   );
-#elif defined(_MSC_VER)
-   _bittestandreset64((__int64 *)var, index);
-#endif
-#else
-   *var &= ~((uint64)1 << index);
-#endif
+   *var &= ~(CONST64U(1) << index);
 }

 static INLINE Bool
-TestBit32(const uint32 *var, uint32 index)
+TestBit32(const uint32 *var, unsigned index)
 {
-#if defined(__GNUC__) && defined(VM_X86_ANY)
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rI" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (1 << index)) != 0;
-#endif
 }

 static INLINE Bool
-TestBit64(const uint64 *var, uint64 index)
+TestBit64(const uint64 *var, unsigned index)
 {
-#if defined __GNUC__ && defined VM_X86_64
-   Bool bit;
-   __asm__ (
-      "bt %[index], %[var] \n"
-      "setc %[bit]"
-      : [bit] "=qQm" (bit)
-      : [index] "rJ" (index), [var] "r" (*var)
-      : "cc"
-   );
-   return bit;
-#else
   return (*var & (CONST64U(1) << index)) != 0;
-#endif
 }

 /*
@@ -1056,7 +989,7 @@ ClearBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1078,7 +1011,7 @@ ComplementBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1170,7 +1103,9 @@ RoundUpPow2C32(uint32 value)
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
-      return (2 << mssb32_0(value - 1));
+      int mssb32 = mssb32_0(value - 1);
+      /* invariant: mssb32 >= 0 */
+      return (2U << (uint32)mssb32);
   }
 }

@@ -1226,6 +1161,101 @@ RoundUpPow2_32(uint32 value)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount32 --
+ *
+ *     Counts "1" bits in a uint32.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount32(uint32 value)
+{
+   /*
+    * Attribution:
+    *     This algorithm was copied from:
+    *         http://www.aggregate.org/MAGIC#Population Count (Ones Count)
+    *
+    *     A virtually identical version (but in assembly) appears in an
+    *     AMD reference manual.
+    *
+    *     No license appears in the original code, but the website
+    *     header states:
+    *
+    *     "None of the following coding tricks came from proprietary
+    *     sources; further, we believe that each of the tricks we did
+    *     not invent is essentially "standard engineering practice" in
+    *     the specialized niche where it applies. Thus, although we
+    *     have not conducted patent searches, etc., to confirm it, we
+    *     believe that these are tricks that freely can be used for
+    *     any purpose. Of course, The Aggregate accepts no
+    *     responsibility for your use of these tricks; you must
+    *     confirm that the trick does what you want and that you can
+    *     use it as you intend. That said, we do intend to maintain
+    *     this page by adding new algorithms and/or correcting
+    *     existing entries. If you have any comments, please contact
+    *     Professor Hank Dietz, http://aggregate.org/hankd/"
+    *
+    *     "This document should be cited using something like the
+    *     following bibtex entry:" (most recent retrieval date added)
+    *
+    *     @techreport{magicalgorithms,
+    *     author={Henry Gordon Dietz},
+    *     title={{The Aggregate Magic Algorithms}},
+    *     institution={University of Kentucky},
+    *     howpublished={Aggregate.Org online technical report},
+    *     URL={http://aggregate.org/MAGIC/},
+    *     urldate={2016-01-27}
+    *     }
+    */
+   value -= ((value >> 1) & 0x55555555);
+   value = (((value >> 2) & 0x33333333) + (value & 0x33333333));
+   value = (((value >> 4) + value) & 0x0f0f0f0f);
+   value += (value >> 8);
+   value += (value >> 16);
+   return value & 0x0000003f;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * PopCount64 --
+ *
+ *     Counts "1" bits in a uint64.
+ *
+ * Results:
+ *     Returns the number of bits set to 1.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE unsigned
+PopCount64(uint64 value)
+{
+   value -= (value >> 1) & 0x5555555555555555ULL;
+   value = ((value >> 2) & 0x3333333333333333ULL) +
+           (value & 0x3333333333333333ULL);
+   value = ((value >> 4) + value) & 0x0f0f0f0f0f0f0f0fULL;
+   value += value >> 8;
+   value += value >> 16;
+   value += value >> 32;
+   return (unsigned) (value & 0xff);
+}
+
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -52,16 +52,25 @@ extern "C" {
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
+ *  Using condition codes as output values (=@ccnz) requires gcc6 or
+ *  above.  Clang does not support condition codes as output
+ *  constraints.
+ *
 */
 #if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
 static INLINE Bool
 xtest(void)
 {
-   uint8 al;
-   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
-                        "setnz %%al\n"
-                        : "=a"(al) : : "cc");
-   return al;
+   Bool result;
+#if defined(__clang__)
+   __asm__ __volatile__("xtest\n"
+                        "setnz %%al"
+                        : "=a" (result) : : "cc");
+#else
+   __asm__ __volatile__("xtest"
+                        : "=@ccnz" (result) : : "cc");
+#endif
+   return result;
 }

 #endif /* __GNUC__ */
@@ -135,49 +144,32 @@ FXRSTOR_AMD_ES0(const void *load)
 static INLINE void 
 XSAVE_ES1(void *save, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x21 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
 XSAVEOPT_ES1(void *save, uint64 mask)
 {
   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x31 \n"
-        :
-        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
+        "xsaveopt %0 \n"
+        : "=m" (*(uint8 *)save)
+        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
 }

 static INLINE void 
 XRSTOR_ES1(const void *load, uint64 mask)
 {
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-   __asm__ __volatile__ (
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "c" ((const uint8 *)load),
-          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
-        : "memory");
-#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
-#endif
 }

 static INLINE void 
@@ -195,17 +187,10 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
-#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
-        ".byte 0x0f, 0xae, 0x29 \n"
-        :
-        : "m" (dummy), "c" ((const uint8 *)load),
-          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
-#endif
        : "eax", "memory");
 }
 #endif /* __GNUC__ */
@@ -415,7 +400,7 @@ Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
      shr  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
@@ -536,7 +521,7 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
      sar  edx, cl
      jmp  SHORT l3
   l2:
-      xor  esi, esi
+      Xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
Author	SHA1	Message	Date
Michal Kubecek	fca70ebea0	vmmon: quick workaround for objtool warnings As discussed in https://bugzilla.suse.com/show_bug.cgi?id=1059674 the reason for multiple objtool warnings is the fact that vmmon module defines its own Panic() function which never returns. While it is marked as such which is used by the compiler for optimization, there is no way to find this from object file. While this seems innocuous, it might result in problems with unwinder later. The quickest way around is to replace vmmon's own Panic() with standard kernel panic() until a cleaner solution is found.	2019-11-14 08:06:57 +01:00
Michal Kubecek	ef8704d97d	vmmon: use standard definition of CR3_PCID_MASK if available The CR3_PCID_MASK is defined in mainline asm/processor-flags.h since commit 6c690ee1039b ("x86/mm: Split read_cr3() into read_cr3_pa() and __read_cr3()") in v4.13-rc1.	2019-11-14 08:06:57 +01:00
Michal Kubecek	38c3889146	vmmon: use standard definition of MSR_MISC_FEATURES_ENABLES if available The MSR_MISC_FEATURES_ENABLES macro is defined in mainline since commit ab6d9468631a ("x86/msr: Rename MISC_FEATURE_ENABLES to MISC_FEATURES_ENABLES") in v4.12-rc1. The MSR_MISC_FEATURES_ENABLES_CPUID_FAULT macro is defined in mainline since commit e9ea1e7f53b8 ("x86/arch_prctl: Add ARCH_[GET\|SET]_CPUID") in v4.12-rc1.	2019-11-14 08:06:54 +01:00
Michal Kubecek	e056593a8b	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE_VMXNET3 if available The PCI_DEVICE_ID_VMWARE_VMXNET3 macro is defined in mainline pci_ids.h since commit b1226c7db1d9 ("vmxnet3: Move PCI Id to pci_ids.h") in v4.10-rc1.	2019-11-14 08:02:08 +01:00
Michal Kubecek	1390fbdb75	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE if available The PCI_VENDOR_ID_VMWARE macro is defined in mainline pci_ids.h since commit 94e57fea6202 ("PCI: Move PCI_VENDOR_ID_VMWARE to pci_ids.h") in v3.18-rc1.	2019-11-14 08:02:08 +01:00
Michal Kubecek	4b2c2a0bca	modules: do not override CC command line options Workstation/Player 15.5.0 tries to guess the right retpoline (Spectre) related compiler command line options, resulting in objtool warnings like warning: objtool: .text.__x86_indirect_thunk_rbp+0x0: unsupported intra-function call warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate e.g. in vmmon-only/common/memtrack.o (but also many others). This seems wrong; out-of-tree modules should be built with the same compiler options that were used to build the kernel they are built against. Distribution kernel devel packages usually make sure this is the way so let them do their job.	2019-11-14 08:02:08 +01:00
Michal Kubecek	5fa25885c8	vmmon: fix wrong VMX86_DEVEL test If VMX86_DEVEL macro is not defined, preprocessor warns when it is tested with "#if" in vmmon-only/linux/driver.c. As VMX86_DEVEL is tested with "#ifdef" everywhere else, change this one test to match the rest.	2019-11-14 08:02:08 +01:00
Michal Kubecek	bf5aeaca48	import Workstation 15.5.1 module sources	2019-11-14 08:00:40 +01:00
Michal Kubecek	51949a3872	import Workstation 15.5.0 module sources	2019-09-21 00:18:33 +02:00
Michal Kubecek	4d36d6a1d2	Merge branch 'master' into workstation	2019-09-21 00:10:22 +02:00
Michal Kubecek	85ede6aae9	misc: fix wrong directory name in INSTALL (fixes #36 ) Fix wrong directory name in one of the examples.	2019-09-09 08:38:27 +02:00
Michal Kubecek	a93f44ee44	import Workstation 15.1.0 module sources	2019-05-15 07:10:51 +02:00
Michal Kubecek	b71c1b9e0b	import Workstation 15.0.1 module sources	2018-11-12 07:53:50 +01:00
Michal Kubecek	d7fe4fe97c	import Workstation 15.0.0 module sources	2018-10-02 19:58:28 +02:00
Michal Kubecek	3f188d0a76	import Workstation 14.1.3 module sources	2018-08-20 07:30:12 +02:00
Michal Kubecek	454cbd7b96	Merge branch 'master' into workstation	2018-05-27 16:13:45 +02:00
Michal Kubecek	2749950c5f	import Workstation 14.1.2 module sources	2018-05-23 18:49:52 +02:00
Michal Kubecek	7fa9cbe769	Merge branch 'master' into workstation	2018-04-02 15:37:06 +02:00
Michal Kubecek	cb66161bb5	Merge branch 'master' into workstation	2018-02-04 21:50:23 +01:00
Michal Kubecek	876363be72	import Workstation 14.1.1 module sources	2018-01-12 13:51:07 +01:00
Michal Kubecek	f78dd5288e	import Workstation 14.1.0 module sources	2017-12-25 10:06:21 +01:00
Michal Kubecek	e4df7e3461	import Workstation 14.0.0 module sources	2017-09-30 21:05:11 +02:00
Michal Kubecek	2bd541e265	import Workstation 12.5.7 module sources	2017-06-23 09:12:56 +02:00
Michal Kubecek	2a214307b9	import Workstation 12.5.6 module sources	2017-05-31 10:17:49 +02:00
Michal Kubecek	755b521dd1	import Workstation 12.5.5 module sources	2017-05-31 09:36:09 +02:00