import Workstation 15.1.0 module sources

import Workstation 15.0.1 module sources
import Workstation 15.0.0 module sources
2019-05-15 07:10:51 +02:00 · 2018-11-12 07:53:50 +01:00 · 2018-10-02 19:58:28 +02:00 · 2018-08-20 07:30:12 +02:00 · 2018-05-27 16:13:45 +02:00 · 2018-05-23 18:49:52 +02:00
79 changed files with 3589 additions and 2888 deletions
@@ -31,7 +31,7 @@ $(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, \
 		$(SRCROOT)/bootstrap/*.c)))

 clean:
-	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions .cache.mk \
+	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \
 		Module.symvers Modules.symvers Module.markers modules.order \
 		$(foreach dir,linux/ common/ vmcore/ bootstrap/ \
 		./,$(addprefix $(dir),.*.cmd .*.o.flags *.o)))
@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998,2015 VMware, Inc. All rights reserved.
+# Copyright (C) 1998,2015,2017 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -72,7 +72,7 @@ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')

 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
-C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o hashFunc.o
+C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o
 C_TARGETS_VMCORE := moduleloop.o
 C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -104,7 +104,7 @@ typedef struct MonPTMPNs {
 } MonPTMPNs;
 
 typedef struct MonLoaderContext {
-   MonLoaderEnvContext *envCtx; /* Environment-specific context */
+   struct MonLoaderEnvContext *envCtx; /* Environment-specific context */
   struct {
      MonPTMPNs    ptMPNs;      /* Mappings into the AS for the current VCPU. */
      VPN          ASFirstVPN;  /* first VPN in the address space */
@@ -405,6 +405,10 @@ MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
         }
         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__,
                 monVPN, pte);
+         if (!PTE_PRESENT(pte)) {
+            ptMPNs->L1MPNs[i] = INVALID_MPN;
+            continue;
+         }
         mpn = LM_PTE_2_PFN(pte);
         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
            return ML_ERROR_PAGE_TABLE_IMPORT;
@@ -490,6 +494,9 @@ MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
         VPN vpn = monVPN + i;
         if (verify) {
            PT_L1E l1e;
+            if (ptMPNs[i] == INVALID_MPN && level == 1) {
+               continue;
+            }
            if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK ||
                LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
               return ML_ERROR_PAGE_TABLE_VERIFY;
@@ -786,7 +793,8 @@ MonLoaderShareFromBlob(MonLoaderContext *ctx,        // IN/OUT
 {
   uint64 endOff = blobOffset + blobSize;

-   if ((blobOffset & PAGE_MASK) != 0 || (blobSize & PAGE_MASK) != 0 ||
+   if ((blobOffset & (PAGE_SIZE - 1)) != 0 ||
+       (blobSize & (PAGE_SIZE - 1)) != 0 ||
       (flags & PTE_RW) != 0) {
      return ML_ERROR_SHARE;
   }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,7 +37,10 @@
 #include "x86paging_common.h"
 #include "x86paging_64.h"
 #include "vmx86.h"
+#include "monLoader.h"
 #include "monLoaderLog.h"
+#include "vmmblob.h"
+#include "memtrack.h"

 typedef struct MonLoaderEnvContext {
   VMDriver *vm;
@@ -101,6 +104,7 @@ void
 MonLoaderCallout_CleanUp(MonLoaderEnvContext *ctx)
 {
   ASSERT(ctx != NULL);
+   Vmx86_CleanupVMMPages(ctx->vm);
   HostIF_FreeKernelMem(ctx);
 }

@@ -179,11 +183,52 @@ MonLoaderCallout_ImportPage(MonLoaderEnvContext *ctx, // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_AllocMPN --
+ *
+ *      Allocates and maps a new VMM page for the specified VCPU.
+ *
+ * Returns:
+ *      An MPN on success, INVALID_MPN on failure.
+ *
+ * Side effects:
+ *      The allocated page is accounted for and its mapping tracked.
+ *
+ *----------------------------------------------------------------------
+ */
+
 MPN
 MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
                          Vcpuid               vcpu) // IN
 {
-   NOT_IMPLEMENTED();
+   VMDriver *vm = ctx->vm;
+   MemTrackEntry *entry;
+   MPN mpn;
+   VPN vpn;
+
+   if (Vmx86_AllocLockedPages(vm, (VA64)&mpn, 1, TRUE, FALSE) != 1) {
+      Log("Failed to allocate page\n");
+      return INVALID_MPN;
+   }
+   vpn = Vmx86_MapPage(mpn);
+   if (vpn == 0) {
+      Log("Failed to map MPN 0x%"FMT64"x\n", mpn);
+      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      return INVALID_MPN;
+   }
+   HostIF_VMLock(vm, 41);
+   entry = MemTrack_Add(vm->vmmTracker, vpn, mpn);
+   HostIF_VMUnlock(vm, 41);
+   if (entry == NULL) {
+      Log("Failed to track mapping from VPN 0x%"FMTVPN"x to MPN 0x%"FMT64"x\n",
+          vpn, mpn);
+      Vmx86_UnmapPage(vpn);
+      Vmx86_FreeLockedPages(vm, (VA64)&mpn, 1, TRUE);
+      return INVALID_MPN;
+   }
+   return mpn;
 }


@@ -227,16 +272,62 @@ MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *ctx, // IN
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_FillPage --
+ *
+ *      Fills a page with a pattern, given the MPN of the page.
+ *
+ * Returns:
+ *      TRUE on success, FALSE if a mapping of the page was not found or
+ *      invalid.
+ *
+ * Side effects:
+ *      The page is filled.
+ *
+ *----------------------------------------------------------------------
+ */
+
 Bool
 MonLoaderCallout_FillPage(MonLoaderEnvContext *ctx,     // IN
                          uint8                pattern, // IN
                          MPN                  mpn,     // IN
                          Vcpuid               vcpu)    // IN
 {
-   NOT_IMPLEMENTED();
+   VMDriver *vm = ctx->vm;
+   MemTrackEntry *entry;
+
+   HostIF_VMLock(vm, 42);
+   entry = MemTrack_LookupMPN(vm->vmmTracker, mpn);
+   HostIF_VMUnlock(vm, 42);
+   if (entry == NULL || entry->mpn != mpn || entry->vpn == 0) {
+      Log("Failed to look up MPN 0x%"FMT64"x\n", mpn);
+      return FALSE;
+   }
+   memset((void *)VPN_2_VA(entry->vpn), pattern, PAGE_SIZE);
+   return TRUE;
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_CopyFromBlob --
+ *
+ *     Copies contents of up to PAGE_SIZE length from the blob at a
+ *     given offset into the page specified by the given MPN. Zero-fills
+ *     the remaining space.
+ *
+ * Returns:
+ *      TRUE on success, FALSE on failure.
+ *
+ * Side effects:
+ *      The page is filled with blob contents.
+ *
+ *----------------------------------------------------------------------
+ */
+
 Bool
 MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *ctx,        // IN
                              uint64               blobOffset, // IN
@@ -244,7 +335,30 @@ MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *ctx,        // IN
                              MPN                  mpn,        // IN
                              Vcpuid               vcpu)       // IN
 {
-   NOT_IMPLEMENTED();
+   MemTrackEntry *entry;
+   VMDriver *vm = ctx->vm;
+   uint64 blobSize = VmmBlob_GetSize(vm);
+   uint8 *blob = VmmBlob_GetPtr(vm);
+   uint8 *buf;
+
+   if (copySize > PAGE_SIZE || copySize == 0 || blobSize < copySize ||
+       blobOffset > blobSize - copySize) {
+      Log("Invalid VMM blob copy parameters: blobOffset 0x%"FMT64"x, "
+          "copySize 0x%"FMTSZ"x, blobSize 0x%"FMT64"x\n", blobOffset, copySize,
+          blobSize);
+      return FALSE;
+   }
+   HostIF_VMLock(vm, 44);
+   entry = MemTrack_LookupMPN(vm->vmmTracker, mpn);
+   HostIF_VMUnlock(vm, 44);
+   if (entry == NULL || entry->vpn == 0) {
+      Log("Failed to look up MPN 0x%"FMT64"x\n", mpn);
+      return FALSE;
+   }
+   buf = (uint8 *)VPN_2_VA(entry->vpn);
+   memcpy(buf, blob + blobOffset, copySize);
+   memset(buf + copySize, 0, PAGE_SIZE - copySize);
+   return TRUE;
 }


@@ -322,15 +436,14 @@ MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
   if (pgOffset >= s->numPages) {
      return INVALID_MPN;
   }
-
   addr = VPN_2_VA(s->baseVpn) + pgOffset * PAGE_SIZE;
-   HostIF_VMLock(ctx->vm, 38);
+   HostIF_VMLock(ctx->vm, 43);
   if (HostIF_LookupUserMPN(ctx->vm, addr, &mpn) != PAGE_LOOKUP_SUCCESS) {
-      HostIF_VMUnlock(ctx->vm, 38);
+      HostIF_VMUnlock(ctx->vm, 43);
      Log("Failed to lookup MPN for shared region VA %"FMTVA"x\n", addr);
      return INVALID_MPN;
   }
-   HostIF_VMUnlock(ctx->vm, 38);
+   HostIF_VMUnlock(ctx->vm, 43);
   return mpn;
 }

@@ -357,7 +470,11 @@ MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx,
                                   unsigned page,   // IN
                                   Vcpuid vcpu)     // IN
 {
-   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   if (subIndex == MONLOADER_HEADER_IDX) {
+      return VmmBlob_GetHeaderMpn(ctx->vm);
+   } else {
+      return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+   }
 }


@@ -393,11 +510,29 @@ MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN
   return TRUE;
 }

+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_GetBlobMpn --
+ *
+ *      Returns the MPN backing the given VMM blob offset.
+ *
+ * Returns:
+ *      An MPN, or INVALID_MPN
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
 MPN
 MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx,    // IN
                            uint64               offset) // IN
 {
-   return INVALID_MPN;
+   ASSERT((offset & (PAGE_SIZE - 1)) == 0);
+   return VmmBlob_GetMpn(ctx->vm, BYTES_2_PAGES(offset));
 }

 Bool
@@ -0,0 +1,230 @@
+/*********************************************************
+ * Copyright (C) 2017-2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vmmblob.c --
+ *
+ *     VMM blob management.
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#endif
+#include "vmware.h"
+#include "vm_assert.h"
+#include "hostif.h"
+#include "vmmblob.h"
+#include "monLoader.h"
+
+#define VMMBLOB_SIZE_MAX (12 * 1024 * 1024)
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetPtr --
+ *
+ *      Returns a pointer to the buffer containing the VMM blob.
+ *
+ * Results:
+ *      A pointer.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint8 *
+VmmBlob_GetPtr(VMDriver *vm)
+{
+   return vm->blobInfo->blobPtr;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetSize --
+ *
+ *      Returns the size of the VMM blob in bytes.
+ *
+ * Results:
+ *      Number of bytes.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint64
+VmmBlob_GetSize(VMDriver *vm)
+{
+   return vm->blobInfo->numBytes;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetMpn --
+ *
+ *      Returns the MPN backing a VMM blob page.
+ *
+ * Results:
+ *      An MPN, or INVALID_MPN.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+VmmBlob_GetMpn(VMDriver *vm, uint64 pgOffset)
+{
+   VmmBlobInfo *bi = vm->blobInfo;
+   uint32 blobNumPages = CEILING(bi->numBytes, PAGE_SIZE);
+   MPN mpn = INVALID_MPN;
+
+   if (pgOffset < blobNumPages) {
+      mpn = bi->mpns[pgOffset];
+   }
+   return mpn;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_GetHeaderMpn --
+ *
+ *      Returns the MPN backing the MonLoader header.
+ *
+ * Results:
+ *      An MPN, or INVALID_MPN.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+VmmBlob_GetHeaderMpn(VMDriver *vm)
+{
+   VmmBlobInfo *bi = vm->blobInfo;
+   uint64 headerOffset = (uint8 *)bi->header - (uint8 *)bi->blobPtr;
+   ASSERT((headerOffset & (PAGE_SIZE - 1)) == 0);
+   return VmmBlob_GetMpn(vm, BYTES_2_PAGES(headerOffset));
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_Cleanup --
+ *
+ *      Cleans up VMM blob state by freeing its memory and associated
+ *      metadata.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+VmmBlob_Cleanup(VmmBlobInfo *bi)
+{
+   const uint32 numPages = CEILING(bi->numBytes, PAGE_SIZE);
+
+   HostIF_FreeKernelPages(numPages, bi->blobPtr);
+   HostIF_FreeKernelMem(bi->mpns);
+   HostIF_FreeKernelMem(bi);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * VmmBlob_Load --
+ *
+ *      Copies and instantiates a VMM bootstrap blob from userspace.
+ *
+ * Results:
+ *      TRUE if successful, FALSE if not.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+VmmBlob_Load(UserVA64 blobAddr,
+             uint32 numBytes,
+             uint32 headerOffset,
+             VmmBlobInfo **blobInfo)
+{
+   uint8 *blob = NULL;
+   MPN *mpns = NULL;
+   MonLoaderHeader *header;
+   VmmBlobInfo *bi = NULL;
+   uint32 numPages = CEILING(numBytes, PAGE_SIZE);
+   size_t fixedHdrSize = MonLoader_GetFixedHeaderSize();
+
+   if (numBytes > VMMBLOB_SIZE_MAX || headerOffset > numBytes ||
+       fixedHdrSize > numBytes - headerOffset) {
+      Warning("Invalid arguments for processing bootstrap. "
+              "Header offset: %u, Fixed header size: %"FMTSZ"u bytes, "
+              "Blob size: %u bytes\n", headerOffset, fixedHdrSize, numBytes);
+      goto error;
+   }
+   mpns = HostIF_AllocKernelMem(numPages * sizeof(*mpns), FALSE);
+   if (mpns == NULL) {
+      Warning("Memory allocation for VMM bootstrap blob failed\n");
+      goto error;
+   }
+   bi = HostIF_AllocKernelMem(sizeof *bi, FALSE);
+   if (bi == NULL) {
+      Warning("Memory allocation for VMM bootstrap blob failed\n");
+      goto error;
+   }
+   blob = HostIF_AllocKernelPages(numPages, mpns);
+   if (blob == NULL) {
+      Warning("Memory allocation for VMM bootstrap blob failed\n");
+      goto error;
+   }
+   if (HostIF_CopyFromUser(blob, blobAddr, numBytes) != 0) {
+      Warning("Error copying VMM bootstrap blob from userspace\n");
+      goto error;
+   }
+   header = (MonLoaderHeader *)(blob + headerOffset);
+   if (header->count > (numBytes - headerOffset - fixedHdrSize) /
+                        sizeof (MonLoaderEntry)) {
+      Warning("Invalid arguments for processing bootstrap. "
+              "Header offset: %u, Number of header entries: %u, "
+              "Blob size: %u bytes\n", headerOffset, header->count, numBytes);
+      goto error;
+   }
+   bi->mpns = mpns;
+   bi->blobPtr = blob;
+   bi->numBytes = numBytes;
+   bi->header = header;
+   *blobInfo = bi;
+
+   return TRUE;
+
+error:
+   if (blob != NULL) {
+      HostIF_FreeKernelPages(numPages, blob);
+   }
+   if (bi != NULL) {
+      HostIF_FreeKernelMem(bi);
+   }
+   if (mpns != NULL) {
+      HostIF_FreeKernelMem(mpns);
+   }
+   return FALSE;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2011 VMware, Inc. All rights reserved.
+ * Copyright (C) 2011, 2016, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -17,7 +17,8 @@
 *********************************************************/

 #include "vmware.h"
-#include "hostif.h"
+#include "x86apic.h"
+#include "x86msr.h"
 #include "x86cpuid_asm.h"
 #include "vm_asm.h"
 #include "cpuid.h"
@@ -56,7 +57,7 @@ APIC_GetMA(void)

   if (cpuVendor != CPUID_VENDOR_INTEL &&
       cpuVendor != CPUID_VENDOR_AMD &&
-       cpuVendor != CPUID_VENDOR_VIA) {
+       cpuVendor != CPUID_VENDOR_HYGON) {
      return (MA)-1;
   }

@@ -78,14 +79,12 @@ APIC_GetMA(void)

   /*
    * On Intel, the high bits are reserved so we mask.
-    * On AMD, high bits are explicitly MBZ, so no need.
-    * Via doesn't specify, so we'll assume reserved.
+    * On AMD and Hygon, high bits are explicitly MBZ, so no need.
    */
-   if (cpuVendor == CPUID_VENDOR_INTEL || cpuVendor == CPUID_VENDOR_VIA) {
+   if (cpuVendor == CPUID_VENDOR_INTEL) {
      /*
       * Intel suggests using CPUID 0x80000008.eax[7-0] (physical
       * address size), with 36 (24 bit MPNs) as a fallback.
-       * Via has that cpuid leaf as well.
       */
      unsigned numPhysicalBits = 36;

@@ -34,10 +34,12 @@
 #include "x86svm.h"
 #include "x86vt.h"

-uint32 cpuidFeatures;
-static CpuidVendor vendor = CPUID_NUM_VENDORS;
-static uint32 version;
-static Bool hasSpecCtrl;
+uint32      cpuidFeatures;
+CpuidVendor cpuidVendor = CPUID_NUM_VENDORS;
+uint32      cpuidVersion;
+Bool        hostSupportsVT;
+Bool        hostSupportsSVM;
+Bool        hostHasSpecCtrl;


 /*
@@ -80,7 +82,7 @@ CPUID_Init(void)
   char name[16];

   __GET_CPUID(1, &regs);
-   version = regs.eax;
+   cpuidVersion = regs.eax;
   cpuidFeatures = regs.edx;

   __GET_CPUID(0, &regs);
@@ -91,33 +93,25 @@ CPUID_Init(void)
   ptr[3] = 0;

   if (strcmp(name, CPUID_INTEL_VENDOR_STRING_FIXED) == 0) {
-      vendor = CPUID_VENDOR_INTEL;
+      cpuidVendor = CPUID_VENDOR_INTEL;
   } else if (strcmp(name, CPUID_AMD_VENDOR_STRING_FIXED) == 0) {
-      vendor = CPUID_VENDOR_AMD;
+      cpuidVendor = CPUID_VENDOR_AMD;
   } else if (strcmp(name, CPUID_CYRIX_VENDOR_STRING_FIXED) == 0) {
-      vendor = CPUID_VENDOR_CYRIX;
+      cpuidVendor = CPUID_VENDOR_CYRIX;
+   } else if (strcmp(name, CPUID_HYGON_VENDOR_STRING_FIXED) == 0) {
+      cpuidVendor = CPUID_VENDOR_HYGON;
   } else {
      Warning("VMMON CPUID: Unrecognized CPU\n");
-      vendor = CPUID_VENDOR_UNKNOWN;
+      cpuidVendor = CPUID_VENDOR_UNKNOWN;
   }

   __GET_CPUID2(7, 0, &regs);
-   hasSpecCtrl = vendor == CPUID_VENDOR_INTEL && ((regs.edx >> 26) & 0x3) != 0;
-}
+   hostHasSpecCtrl = cpuidVendor == CPUID_VENDOR_INTEL &&
+                     (CPUID_ISSET(7, EDX, IBRSIBPB, regs.edx) ||
+                      CPUID_ISSET(7, EDX, STIBP, regs.edx));

-
-CpuidVendor
-CPUID_GetVendor(void)
-{
-   ASSERT(vendor != CPUID_NUM_VENDORS);
-   return vendor;
-}
-
-
-uint32
-CPUID_GetVersion(void)
-{
-   return version;
+   hostSupportsVT = VT_CapableCPU();
+   hostSupportsSVM = SVM_CapableCPU();
 }


@@ -155,51 +149,3 @@ CPUID_AddressSizeSupported(void)

   return result;
 }
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * CPUID_HostSupportsHV --
- *
- *     Determine whether processor supports hardware virtualization.  Two
- *     possibilities are valid: VMX on Intel or SVM on AMD.
- *
- * Results:
- *     True iff the processor supports the required features.
- *
- * Side effects:
- *     None.
- *
- *-----------------------------------------------------------------------------
- */
-
-Bool
-CPUID_HostSupportsHV(void)
-{
-   return (vendor == CPUID_VENDOR_AMD   && SVM_CapableCPU()) ||
-          (vendor == CPUID_VENDOR_INTEL && VT_CapableCPU());
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * CPUID_HostSupportsSpecCtrl --
- *
- *     Determine whether the processor supports MSR_SPEC_CTRL.
- *
- * Results:
- *     True iff the processor supports the required features.
- *
- * Side effects:
- *     None.
- *
- *-----------------------------------------------------------------------------
- */
-
-Bool
-CPUID_HostSupportsSpecCtrl(void)
-{
-   return hasSpecCtrl;
-}
@@ -31,14 +31,15 @@

 #include "x86cpuid.h"

-extern uint32 cpuidFeatures;
+extern uint32      cpuidFeatures;
+extern uint32      cpuidVersion;
+extern CpuidVendor cpuidVendor;
+extern Bool        hostSupportsVT;
+extern Bool        hostSupportsSVM;
+extern Bool        hostHasSpecCtrl;

 void CPUID_Init(void);
-CpuidVendor CPUID_GetVendor(void);
-uint32 CPUID_GetVersion(void);
 Bool CPUID_AddressSizeSupported(void);
-Bool CPUID_HostSupportsHV(void);
-Bool CPUID_HostSupportsSpecCtrl(void);

 static INLINE uint32
 CPUID_GetFeatures(void)
@@ -46,6 +47,43 @@ CPUID_GetFeatures(void)
   return cpuidFeatures;
 }

+static INLINE CpuidVendor
+CPUID_GetVendor(void)
+{
+   ASSERT(cpuidVendor != CPUID_NUM_VENDORS);
+   return cpuidVendor;
+}
+
+static INLINE uint32
+CPUID_GetVersion(void)
+{
+   return cpuidVersion;
+}
+
+static INLINE Bool
+CPUID_HostSupportsVT(void)
+{
+   return hostSupportsVT;
+}
+
+static INLINE Bool
+CPUID_HostSupportsSVM(void)
+{
+   return hostSupportsSVM;
+}
+
+static INLINE Bool
+CPUID_HostSupportsHV(void)
+{
+   return hostSupportsVT || hostSupportsSVM;
+}
+
+static INLINE Bool
+CPUID_HostSupportsSpecCtrl(void)
+{
+   return hostHasSpecCtrl;
+}
+
 static INLINE Bool
 CPUID_SSE2Supported(void)
 {
@@ -1,41 +0,0 @@
-/*********************************************************
- * Copyright (C) 2001 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-
-/*
- * hashFunc.c --
- *
- *      The core implementation lives in lib/shared/hashFunc.h because it
- *      is shared by the vmkernel and vmmon.
- */
-
-#include "hashFunc.h"
-
-/*
- * Wrappers
- */
-
-// arbitrary constant
-#define HASH_INIT_VALUE (42)
-
-// 64-bit hash for one 4K page
-uint64
-HashFunc_HashPage(const void *addr)
-{
-   return hash2((uint64 *)addr, PAGE_SIZE / sizeof (uint64), HASH_INIT_VALUE);
-}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,14 +59,7 @@

 #define CROSSCALL_SLEEP_US 1000

-/* Return value for HostIF_IPI. */
-typedef enum {
-   IPI_NONE,      // No IPI was sent.
-   IPI_UNICAST,   // Unicast IPIs were sent.
-   IPI_BROADCAST  // A broadcast IPI was sent to all PCPUs.
-} HostIFIPIMode;
-
-EXTERN int   HostIF_Init(VMDriver *vm);
+EXTERN int   HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);

@@ -96,6 +89,9 @@ EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 #endif

 EXTERN Bool  HostIF_APICInit(VMDriver *vm, Bool setVMPtr, Bool probe);
+EXTERN uint8 HostIF_GetMonitorIPIVector(void);
+EXTERN uint8 HostIF_GetHVIPIVector(void);
+EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);

 EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
                                  Vcpuid vcpuid,
@@ -104,7 +100,8 @@ EXTERN int   HostIF_SemaphoreWait(VMDriver *vm,
 EXTERN int   HostIF_SemaphoreSignal(uint64 *args);

 EXTERN void  HostIF_SemaphoreForceWakeup(VMDriver *vm, const VCPUSet *vcs);
-EXTERN HostIFIPIMode HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
+EXTERN void  HostIF_IPI(VMDriver *vm, const VCPUSet *vcs);
+EXTERN void  HostIF_OneIPI(VMDriver *vm, Vcpuid v);

 EXTERN uint32 HostIF_GetCurrentPCPU(void);
 EXTERN void HostIF_CallOnEachCPU(void (*func)(void *), void *data);
@@ -119,6 +116,7 @@ EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
 EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr,
                                  unsigned int numPages, Bool kernelMPNBuffer);
 EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn);
+EXTERN MPN HostIF_AllocLowPage(VMDriver *vm);

 EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr,
                               Bool kernelBuffer, size_t len);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2008, 2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2014,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,9 +24,8 @@
 *    Depending on configuration phystracker provides either 2-level or
 *    3-level structure to track whether page (specified by its MPN) is
 *    locked or no.  Linux uses 3-level structures with top limit of
- *    1TB (32bit) or 16TB (64bit).  Windows use 2-level structures
- *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.  On Mac
- *    limit is 2-level 128GB.
+ *    1TB (32bit) or 16TB (64bit).  Windows and Mac use 2-level structures
+ *    ready to hold 128GB (32bit) or 2TB (64bit) of memory.
 *
 *    2-level phystracker is built on top of 3-level one by collapsing
 *    middle level.
@@ -50,9 +49,7 @@
 #define BYTES_PER_ENTRY      (PAGE_SIZE)
 #define PHYSTRACK_L3_ENTRIES (8 * BYTES_PER_ENTRY) /* 128MB */

-#if defined(WINNT_DDK)
-#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
-#elif defined(__linux__)
+#if defined(__linux__)
 #define PHYSTRACK_L2_ENTRIES (BYTES_PER_ENTRY / sizeof(void *)) /* 64GB or 128GB */
 /*
 * Currently MPN is 32 bits.  15 bits are in L3, 9 bits are in L2,
@@ -60,7 +57,7 @@
 */
 #define PHYSTRACK_L1_ENTRIES (256) /* 16TB. */
 #else
-#define PHYSTRACK_L1_ENTRIES ((128 + 4) * 8) /* 128 GB */
+#define PHYSTRACK_L1_ENTRIES (PHYSTRACK_MAX_SUPPORTED_GB * 8)
 #endif

 #ifndef PHYSTRACK_L2_ENTRIES
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -29,7 +29,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#if defined(WINNT_DDK)
+#if !defined (__linux__)
 /* XXX: can be more efficient based on server vs. desktop and version of Windows */
 #define PHYSTRACK_MAX_SUPPORTED_GB (2048 + 4) /* 2 TB 64-bit W2k8 + 4 GB PCI */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -23,7 +23,7 @@
 *     virtual machine monitors.
 */

-#ifdef linux
+#ifdef __linux__
 /* Must come before any kernel header file --hpreg */
 #   include "driver-config.h"

@@ -49,7 +49,6 @@
 #include "cpuid.h"
 #include "vcpuset.h"
 #include "memtrack.h"
-#include "hashFunc.h"
 #if defined(_WIN64)
 #include "x86.h"
 #include "vmmon-asm-x86-64.h"
@@ -64,7 +63,7 @@
 #include "x86vtinstr.h"
 #include "bootstrap_vmm.h"
 #include "monLoader.h"
-
+#include "vmmblob.h"

 PseudoTSC pseudoTSC;

@@ -113,6 +112,12 @@ typedef struct {
   MSRQuery *query;
 } Vmx86GetMSRData;

+static Bool hostUsesNX;
+
+typedef struct NXData {
+   Atomic_uint32 responded;
+   Atomic_uint32 hasNX;
+} NXData;

 /*
 *----------------------------------------------------------------------
@@ -502,6 +507,194 @@ Vmx86DeleteVMFromList(VMDriver *vm)
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86_Free --
+ *
+ *      A wrapper around HostIF_FreeKernelMem that checks if the given
+ *      pointer is NULL before freeing memory.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+void
+Vmx86_Free(void *ptr)
+{
+   if (ptr != NULL) {
+      HostIF_FreeKernelMem(ptr);
+   }
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86_Calloc --
+ *
+ *      A wrapper around HostIF_AllocKernelMem that zeroes memory and
+ *      fails if integer overflow would occur in the computed
+ *      allocation size.
+ *
+ * Results:
+ *      Pointer to allocated memory or NULL on failure. Use
+ *      HostIF_FreeKernelMem or Vmx86_Free to free.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+void *
+Vmx86_Calloc(size_t numElements, // IN
+             size_t elementSize, // IN
+             int nonPageable)    // IN
+{
+   size_t numBytes = numElements * elementSize;
+   void *retval;
+
+   if (UNLIKELY(numBytes / numElements != elementSize)) { // Overflow.
+      return NULL;
+   }
+
+   retval = HostIF_AllocKernelMem(numBytes, nonPageable);
+   if (retval != NULL) {
+      memset(retval, 0, numBytes);
+   }
+   return retval;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86FreeVMDriver --
+ *
+ *      Release kernel memory allocated for the driver structure.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+void
+Vmx86FreeVMDriver(VMDriver *vm)
+{
+   Vmx86_Free(vm->ptRootMpns);
+   Vmx86_Free(vm->crosspage);
+   Vmx86_Free(vm->crosscallWaitSet);
+   Vmx86_Free(vm->ptscOffsets);
+   Vmx86_Free(vm->currentHostCpu);
+   vm->ptRootMpns       = NULL;
+   vm->crosspage        = NULL;
+   vm->crosscallWaitSet = NULL;
+   vm->ptscOffsets      = NULL;
+   vm->currentHostCpu   = NULL;
+   HostIF_FreeKernelMem(vm);
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86AllocVMDriver --
+ *
+ *      Allocate the driver structure for a virtual machine.
+ *
+ * Results:
+ *      Zeroed VMDriver structure or NULL on error.
+ *
+ * Side effects:
+ *      May allocate kernel memory.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+VMDriver *
+Vmx86AllocVMDriver(uint32 numVCPUs)
+{
+   VMDriver *vm = Vmx86_Calloc(1, sizeof *vm, TRUE);
+   if (vm == NULL) {
+      return NULL;
+   }
+   if ((vm->ptRootMpns =
+        Vmx86_Calloc(numVCPUs, sizeof *vm->ptRootMpns, TRUE))       != NULL &&
+       (vm->crosspage =
+        Vmx86_Calloc(numVCPUs, sizeof *vm->crosspage, TRUE))        != NULL &&
+       (vm->crosscallWaitSet =
+        Vmx86_Calloc(numVCPUs, sizeof *vm->crosscallWaitSet, TRUE)) != NULL &&
+       (vm->ptscOffsets =
+        Vmx86_Calloc(numVCPUs, sizeof *vm->ptscOffsets, TRUE))      != NULL &&
+       (vm->currentHostCpu =
+        Vmx86_Calloc(numVCPUs, sizeof *vm->currentHostCpu, TRUE))   != NULL) {
+      return vm;
+   }
+   Vmx86FreeVMDriver(vm);
+   return NULL;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86VMMPageFree --
+ *
+ *     Unmaps the VMM page corresponding to this entry from in the host
+ *     kernel. This function is used as a callback by MemTrack_Cleanup().
+ *
+ * Results:
+ *      None
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+static void
+Vmx86VMMPageFree(void *unused, MemTrackEntry *entry)
+{
+   ASSERT(entry->vpn != 0 && entry->mpn != 0);
+   Vmx86_UnmapPage(entry->vpn);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_CleanupVMMPages --
+ *
+ *     Ummaps all VMM pages from the host kernel address space and frees
+ *     the VMM MemTracker.
+ *
+ * Results:
+ *      None
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+Vmx86_CleanupVMMPages(VMDriver *vm)
+{
+   MemTrack_Cleanup(vm->vmmTracker, Vmx86VMMPageFree, NULL);
+   vm->vmmTracker = NULL;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -529,13 +722,20 @@ Vmx86FreeAllVMResources(VMDriver *vm)

      Vmx86_SetHostClockRate(vm, 0);

-      if (vm->ptpTracker) {
+      if (vm->ptpTracker != NULL) {
         Task_SwitchPTPPageCleanup(vm);
      }
+      if (vm->vmmTracker != NULL) {
+         Vmx86_CleanupVMMPages(vm);
+      }
+      if (vm->blobInfo != NULL) {
+         VmmBlob_Cleanup(vm->blobInfo);
+         vm->blobInfo = NULL;
+      }

      HostIF_FreeAllResources(vm);

-      HostIF_FreeKernelMem(vm);
+      Vmx86FreeVMDriver(vm);
   }
 }

@@ -659,6 +859,64 @@ Vmx86UnreserveFreePages(VMDriver *vm,
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86GetNX --
+ *
+ *       Checks whether NX is enabled on the current CPU.
+ *
+ * Results:
+ *       None.
+ *
+ * Side effects:
+ *       Increments responded-CPU counter, may increment NX CPU counter.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static void
+Vmx86GetNX(void *clientData) // IN/OUT: A NXData *
+{
+   NXData *nxData = (NXData *)clientData;
+   uint64 efer = __GET_MSR(MSR_EFER);
+
+   Atomic_Inc32(&nxData->responded);
+   if ((efer & MSR_EFER_NXE) == MSR_EFER_NXE) {
+      Atomic_Inc32(&nxData->hasNX);
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_CacheNXState --
+ *
+ *       Checks whether every CPU on the host has NX/XD enabled and
+ *       caches this value.
+ *
+ * Results:
+ *       None.
+ *
+ * Side effects:
+ *       Caches host NX value.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+Vmx86_CacheNXState(void)
+{
+   NXData nxData;
+   Atomic_Write32(&nxData.responded, 0);
+   Atomic_Write32(&nxData.hasNX, 0);
+   HostIF_CallOnEachCPU(Vmx86GetNX, &nxData);
+   hostUsesNX = Atomic_Read32(&nxData.hasNX) ==
+                Atomic_Read32(&nxData.responded);
+}
+
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -676,33 +934,43 @@ Vmx86UnreserveFreePages(VMDriver *vm,
 */

 VMDriver *
-Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize)
+Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize, uint32 numVCPUs)
 {
   VMDriver *vm;
   Vcpuid v;
   void *bsBuf = NULL;
   BSVMM_HostParams *bsParams;

+   /* Disallow VM creation if the vmx passes us an invalid number of vcpus. */
+   if (numVCPUs == 0 || numVCPUs > MAX_VCPUS) {
+      return NULL;
+   }
+
   /* Disallow VM creation if HV is not available, as the VM cannot be run. */
   if (!CPUID_HostSupportsHV()) {
      return NULL;
   }

-   vm = HostIF_AllocKernelMem(sizeof *vm, TRUE);
+   /* Disallow VM creation if NX is disabled on the host as VMM requires NX. */
+   if (!hostUsesNX) {
+      Log("NX/XD must be enabled.  Cannot create VM.\n");
+      return NULL;
+   }
+
+   vm = Vmx86AllocVMDriver(numVCPUs);
   if (vm == NULL) {
      return NULL;
   }
-   memset(vm, 0, sizeof *vm);

   vm->userID = 0;
+   vm->numVCPUs = numVCPUs;
   vm->memInfo.admitted = FALSE;
-   for (v = 0; v < MAX_INITBLOCK_CPUS; v++) {
-      vm->currentHostCpu[v] = INVALID_PCPU;
-   }
-   for (v = 0; v < MAX_VCPUS; v++) {
+
+   for (v = 0; v < numVCPUs; v++) {
+      Atomic_Write32(&vm->currentHostCpu[v], INVALID_PCPU);
      vm->ptRootMpns[v] = INVALID_MPN;
   }
-   if (HostIF_Init(vm)) {
+   if (HostIF_Init(vm, numVCPUs)) {
      goto cleanup;
   }
   bsBuf = HostIF_AllocKernelMem(bsBlobSize, FALSE);
@@ -717,7 +985,6 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize)
      Warning("Could not validate the VMM bootstrap blob");
      goto cleanup;
   }
-   memcpy(&vm->bsParams, bsParams, sizeof *bsParams);

   if (!Task_CreateCrossGDT(&bsParams->gdtInit)) {
      goto cleanup;
@@ -726,6 +993,10 @@ Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize)
   if (vm->ptpTracker == NULL) {
      goto cleanup;
   }
+   vm->vmmTracker = MemTrack_Init(vm);
+   if (vm->vmmTracker == NULL) {
+      goto cleanup;
+   }

   HostIF_GlobalLock(0);

@@ -785,8 +1056,6 @@ Vmx86SetPageTableRoots(VMDriver *vm, UserVA64 *ptRootVAs, uint16 numVCPUs)
 {
   uint16 vcpu;

-   ASSERT_ON_COMPILE(MAX_VCPUS == MAX_INITBLOCK_CPUS);
-
   if (numVCPUs > vm->numVCPUs) {
      return FALSE;
   }
@@ -797,11 +1066,14 @@ Vmx86SetPageTableRoots(VMDriver *vm, UserVA64 *ptRootVAs, uint16 numVCPUs)
         return FALSE;
      }
      ASSERT(vm->ptRootMpns[vcpu] == INVALID_MPN);
+      HostIF_VMLock(vm, 38);
      if (HostIF_LookupUserMPN(vm, ptRootVAs[vcpu], &vm->ptRootMpns[vcpu]) !=
          PAGE_LOOKUP_SUCCESS) {
+         HostIF_VMUnlock(vm, 38);
         Warning("Failure looking up page table root MPN for VCPU %d\n", vcpu);
         return FALSE;
      }
+      HostIF_VMUnlock(vm, 38);
   }
   return TRUE;
 }
@@ -859,61 +1131,36 @@ Vmx86_ProcessBootstrap(VMDriver *vm,
                       UserVA64 *ptRootVAs,
                       VMSharedRegion *shRegions)
 {
-   void *blobHdr = NULL;
+   VmmBlobInfo *bi = NULL;
   unsigned errLine;
   Vcpuid errVcpu;
   MonLoaderError ret;
-   MonLoaderHeader *header;
-   Bool res = FALSE;
   MonLoaderArgs args;

-   /*
-    * Only the beginning of the blob may be passed in: the header at
-    * offset 0, which must fit on one page.
-    */
-   if (numBytes > PAGE_SIZE || headerOffset != 0 ||
-       MonLoader_GetFixedHeaderSize() > numBytes) {
-      Warning("Invalid arguments for processing bootstrap. "
-              "Header offset: %d, Fixed header size: %zd bytes, "
-              "Blob size: %d bytes\n",
-              headerOffset, MonLoader_GetFixedHeaderSize(), numBytes);
-      goto cleanup;
-   }
-   blobHdr = HostIF_AllocKernelMem(numBytes, FALSE);
-   if (blobHdr == NULL) {
-      Warning("Failure allocating kernel buffer for bootstrap blob\n");
-      goto cleanup;
-   }
-   if (HostIF_CopyFromUser(blobHdr, bsBlobAddr, numBytes) != 0) {
-      Warning("Failure copying bootstrap blob from userspace\n");
-      goto cleanup;
-   }
-   ASSERT(headerOffset == 0);
-   header = (MonLoaderHeader *)blobHdr;
-   if (MonLoader_GetFullHeaderSize(header) > numBytes) {
-      Warning("Invalid arguments for processing bootstrap. "
-              "Full header size: %zd bytes, Blob size: %d bytes\n",
-              MonLoader_GetFullHeaderSize(header), numBytes);
-      goto cleanup;
+   if (!VmmBlob_Load(bsBlobAddr, numBytes, headerOffset, &bi)) {
+      Warning("Error loading VMM bootstrap blob\n");
+      goto error;
   }
+   vm->blobInfo = bi;
   if (!Vmx86SetPageTableRoots(vm, ptRootVAs, numVCPUs)) {
-      goto cleanup;
+      goto error;
   }
   args.vm = vm;
   args.shRegions = shRegions;
-   ret = MonLoader_Process(header, numVCPUs, &args, &errLine, &errVcpu);
+   ret = MonLoader_Process(bi->header, numVCPUs, &args, &errLine, &errVcpu);
   if (ret != ML_OK) {
      Warning("Error processing bootstrap: error %d at line %u, vcpu %u\n",
               ret, errLine, errVcpu);
-      goto cleanup;
+      goto error;
   }
-   res = TRUE;
+   return TRUE;

-cleanup:
-   if (blobHdr != NULL) {
-      HostIF_FreeKernelMem(blobHdr);
+error:
+   if (bi != NULL) {
+      VmmBlob_Cleanup(bi);
+      vm->blobInfo = NULL;
   }
-   return res;
+   return FALSE;
 }


@@ -1051,12 +1298,11 @@ Vmx86_InitVM(VMDriver *vm,          // IN

      return 1;
   }
-   if (initParams->numVCPUs > MAX_INITBLOCK_CPUS) {
-      Warning("Too many VCPUs for init block %d\n", initParams->numVCPUs);
+   if (vm->numVCPUs == 0 || vm->numVCPUs > MAX_VCPUS) {
+      Warning("Invalid number of VCPUs %d\n", vm->numVCPUs);

      return 1;
   }
-   vm->numVCPUs = initParams->numVCPUs;

   /*
    * Initialize the driver's part of the cross-over page used to
@@ -1398,6 +1644,7 @@ Vmx86_MonTimerIPI(void)
   for (vm = vmDriverList; vm != NULL; vm = vm->nextDriver) {
      Vcpuid v;
      VCPUSet expiredVCPUs;
+      Bool hasWork = FALSE;
      VCPUSet_Empty(&expiredVCPUs);

      for (v = 0; v < vm->numVCPUs; v++) {
@@ -1409,12 +1656,11 @@ Vmx86_MonTimerIPI(void)
         expiry = crosspage->crosspageData.monTimerExpiry;
         if (expiry != 0 && expiry <= pNow) {
            VCPUSet_Include(&expiredVCPUs, v);
+            hasWork = TRUE;
         }
      }
-      if (!VCPUSet_IsEmpty(&expiredVCPUs) &&
-          HostIF_IPI(vm, &expiredVCPUs) == IPI_BROADCAST) {
-         // no point in doing a broadcast for more than one VM.
-         break;
+      if (hasWork) {
+         HostIF_IPI(vm, &expiredVCPUs);
      }
   }
   HostIF_GlobalUnlock(21);
@@ -1854,6 +2100,42 @@ Vmx86_FreeLockedPages(VMDriver *vm,         // IN: VM instance pointer
 }


+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Vmx86_AllocLowPage --
+ *
+ *      Allocate a zeroed locked low page.
+ *
+ * Results:
+ *      Allocated MPN on success. INVALID_MPN on failure.
+ *
+ * Side effects:
+ *      Number of global and per-VM locked pages is increased.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+MPN
+Vmx86_AllocLowPage(VMDriver *vm,      // IN: VMDriver
+                   Bool ignoreLimits) // IN: should limits be ignored?
+{
+   MPN mpn;
+
+   if (!Vmx86ReserveFreePages(vm, 1, ignoreLimits)) {
+      return INVALID_MPN;
+   }
+
+   mpn = HostIF_AllocLowPage(vm);
+
+   if (mpn == INVALID_MPN) {
+      Vmx86UnreserveFreePages(vm, 1);
+   }
+
+   return mpn;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -2259,7 +2541,7 @@ Vmx86VMXEnabled(void)
 static void
 Vmx86EnableHVOnCPU(void)
 {
-   if (CPUID_GetVendor() == CPUID_VENDOR_AMD) {
+   if (CPUID_HostSupportsSVM()) {
      uint64 vmCR = __GET_MSR(MSR_VM_CR);
      if (!SVM_LockedFromFeatures(vmCR)) {
         CPUIDRegs regs;
@@ -2269,13 +2551,11 @@ Vmx86EnableHVOnCPU(void)
                                  MSR_VM_CR_SVM_LOCK);
         }
      }
-   } else if (CPUID_GetVendor() == CPUID_VENDOR_INTEL) {
+   } else if (CPUID_HostSupportsVT()) {
      uint64 featCtl = __GET_MSR(MSR_FEATCTL);
      if (!VT_LockedFromFeatures(featCtl)) {
         __SET_MSR(MSR_FEATCTL, featCtl | MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE);
      }
-   } else {
-      NOT_REACHED();
   }
 }

@@ -2347,6 +2627,10 @@ Vmx86_InitPseudoTSC(PTSCInitParams *params) // IN/OUT
   HostIF_GlobalLock(36);

   if (!pseudoTSC.initialized) {
+      Bool logParams = pseudoTSC.hz != params->tscHz ||
+                       pseudoTSC.hwTSCsSynced != params->hwTSCsSynced ||
+                       pseudoTSC.useRefClock != params->forceRefClock;
+
      pseudoTSC.hz = params->tscHz;
      pseudoTSC.refClockToPTSC.ratio.mult  = params->refClockToPTSC.mult;
      pseudoTSC.refClockToPTSC.ratio.shift = params->refClockToPTSC.shift;
@@ -2364,10 +2648,12 @@ Vmx86_InitPseudoTSC(PTSCInitParams *params) // IN/OUT
      pseudoTSC.useRefClock           = params->forceRefClock;
      pseudoTSC.neverSwitchToRefClock = params->forceTSC;
      pseudoTSC.hwTSCsSynced          = params->hwTSCsSynced;
-      Log("PTSC: initialized at %"FMT64"u Hz using %s, TSCs are %ssynchronized.\n",
-          pseudoTSC.hz, pseudoTSC.useRefClock ? "reference clock" : "TSC",
-          pseudoTSC.hwTSCsSynced ? "" : "not ");
-
+      if (logParams) {
+         Log("PTSC: initialized at %"FMT64"u Hz using %s, TSCs are "
+             "%ssynchronized.\n", pseudoTSC.hz,
+             pseudoTSC.useRefClock ? "reference clock" : "TSC",
+             pseudoTSC.hwTSCsSynced ? "" : "not ");
+      }
      pseudoTSC.initialized = TRUE;
   }
   /*
@@ -2773,6 +3059,8 @@ Vmx86_YieldToSet(VMDriver *vm,       // IN:
 {
   VCPUSet vcpus;

+   ASSERT(currVcpu < vm->numVCPUs);
+   
   if (VCPUSet_IsEmpty(req)) {
      return;
   }
@@ -2796,7 +3084,7 @@ Vmx86_YieldToSet(VMDriver *vm,       // IN:
   }

   VCPUSet_Empty(&vcpus);
-   FOR_EACH_VCPU_IN_SET(req, vcpuid) {
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(req, vcpuid, vm->numVCPUs) {
      if (vcpuid == currVcpu) {
         continue;
      }
@@ -2818,14 +3106,14 @@ Vmx86_YieldToSet(VMDriver *vm,       // IN:
       * at the current vCPU.
       */

-      if (vm->currentHostCpu[vcpuid] != INVALID_PCPU) {
+      if (Atomic_Read32(&vm->currentHostCpu[vcpuid]) != INVALID_PCPU) {
         VCPUSet_AtomicRemove(&vm->crosscallWaitSet[vcpuid], currVcpu);
      } else {
         if (VCPUSet_AtomicIsMember(&vm->crosscallWaitSet[vcpuid], currVcpu)) {
            VCPUSet_Include(&vcpus, vcpuid);
         }
      }
-   } ROF_EACH_VCPU_IN_SET();
+   } ROF_EACH_VCPU_IN_SET_WITH_MAX();

   /*
    * Wake up any threads that had previously yielded the processor to
@@ -2851,9 +3139,9 @@ Vmx86_YieldToSet(VMDriver *vm,       // IN:
    * bits anyway.
    */

-   FOR_EACH_VCPU_IN_SET(&vcpus, vcpuid) {
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(&vcpus, vcpuid, vm->numVCPUs) {
      VCPUSet_AtomicRemove(&vm->crosscallWaitSet[vcpuid], currVcpu);
-   } ROF_EACH_VCPU_IN_SET();
+   } ROF_EACH_VCPU_IN_SET_WITH_MAX();

   HostIF_CancelWaitForThreads(vm, currVcpu);
 }
@@ -2896,7 +3184,7 @@ Vmx86PerfCtrInUse(Bool isGen, unsigned pmcNum, unsigned ctrlMSR,
                PERFCTR_CORE_INST_RETIRED;
      pgcEna = CONST64U(1) << pmcNum;
   } else {
-      ASSERT(pmcNum < 3);
+      ASSERT(pmcNum < PERFCTR_CORE_NUM_FIXED_COUNTERS);
      if ((pmcCtrl & PERFCTR_CORE_FIXED_ENABLE_MASKn(pmcNum)) != 0) {
         return TRUE;
      }
@@ -2968,8 +3256,9 @@ Vmx86GetUnavailPerfCtrsOnCPU(void *data)
      }
      selBase = PERFCTR_CORE_PERFEVTSEL0_ADDR;
      ctrBase = PERFCTR_CORE_PERFCTR0_ADDR;
-   } else if (CPUID_GetVendor() == CPUID_VENDOR_AMD) {
-     if(CPUID_ISSET(0x80000001, ECX, PERFCORE,
+   } else if (CPUID_GetVendor() == CPUID_VENDOR_AMD ||
+              CPUID_GetVendor() == CPUID_VENDOR_HYGON) {
+     if (CPUID_ISSET(0x80000001, ECX, PERFCORE,
        __GET_ECX_FROM_CPUID(0x80000001))) {
         numGen  = 6;
         selBase = PERFCTR_AMD_EXT_BASE_ADDR + PERFCTR_AMD_EXT_EVENTSEL;
@@ -2997,6 +3286,7 @@ Vmx86GetUnavailPerfCtrsOnCPU(void *data)
      }
   }
   if (numFix > 0) {
+      numFix = MIN(numFix, PERFCTR_CORE_NUM_FIXED_COUNTERS);
      for (i = 0; i < numFix; i++) {
         if (Vmx86PerfCtrInUse(FALSE, i, PERFCTR_CORE_FIXED_CTR_CTRL_ADDR,
                               PERFCTR_CORE_FIXED_CTR0_ADDR + i, hasPGC)) {
@@ -3114,3 +3404,52 @@ Vmx86_UnmapPage(VPN vpn) // IN:
 {
   HostIF_UnmapPage(vpn);
 }
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_GetMonitorContext --
+ *
+ *      Gets most of the monitor's saved context (as of the last world switch)
+ *      from a given VCPU's crosspage.  CR3 is omitted as it is privileged,
+ *      while DS/SS/ES are returned due to their potential utility in debugging.
+ *
+ * Results:
+ *      On success, TRUE and context is (partially) populated.  FALSE otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+Vmx86_GetMonitorContext(VMDriver *vm,       // IN: The VM instance.
+                        Vcpuid vcpuid,      // IN: VCPU in question.
+                        Context64 *context) // OUT: context.
+{
+   VMCrossPage *crosspage;
+   VMCrossPageData *cpData;
+   if (vcpuid >= vm->numVCPUs) {
+      return FALSE;
+   }
+   crosspage = vm->crosspage[vcpuid];
+   if (!crosspage) {
+      return FALSE;
+   }
+   cpData = &crosspage->crosspageData;
+   memset(context, 0, sizeof *context);
+   context->es  = cpData->monES;
+   context->ss  = cpData->monSS;
+   context->ds  = cpData->monDS;
+   context->rbx = cpData->monRBX;
+   context->rsp = cpData->monRSP;
+   context->rbp = cpData->monRBP;
+   context->r12 = cpData->monR12;
+   context->r13 = cpData->monR13;
+   context->r14 = cpData->monR14;
+   context->r15 = cpData->monR15;
+   context->rip = cpData->monRIP;
+   return TRUE;
+}
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -40,16 +40,13 @@
 #include "apic.h"
 #include "bootstrap_vmm.h"

-typedef struct PseudoTSCOffsetInfo {
-   uint32 inVMMCnt;                  /* Number of vcpus executing in the VMM. */
-   uint32 vcpuid;                    /* Index into VMDriver.ptscOffsets. */
-} PseudoTSCOffsetInfo;
-
 typedef struct TSCDelta {
   Atomic_uint64 min;
   Atomic_uint64 max;
 } TSCDelta;

+struct VmmBlobInfo;
+
 /*
 * VMDriver - the main data structure for the driver side of a
 *            virtual machine.
@@ -57,32 +54,33 @@ typedef struct TSCDelta {

 typedef struct VMDriver {
   /* Unique (in the driver), strictly positive, VM ID used by userland. */
-   int                          userID;
+   int                     userID;

-   struct VMDriver             *nextDriver;   /* Next on list of all VMDrivers */
+   Vcpuid                  numVCPUs;         /* Number of vcpus in VM. */

-   Vcpuid                       numVCPUs;     /* Number of vcpus in VM. */
-   struct VMHost               *vmhost;       /* Host-specific fields. */
+   struct VMDriver        *nextDriver;       /* Next on list of all VMDrivers */

-   BSVMM_HostParams             bsParams;
-   MPN                          ptRootMpns[MAX_VCPUS];
+   struct VMHost          *vmhost;           /* Host-specific fields. */
+
+   MPN                    *ptRootMpns;       /* numVCPUs-sized array. */
+   struct VmmBlobInfo     *blobInfo;         /* VMM bootstrap blob info. */

   /* Pointers to the crossover pages shared with the monitor. */
-   struct VMCrossPage          *crosspage[MAX_INITBLOCK_CPUS];
-   struct MemTrack             *ptpTracker;
-   volatile uint32              currentHostCpu[MAX_INITBLOCK_CPUS];
-   VCPUSet                      crosscallWaitSet[MAX_INITBLOCK_CPUS];
-   APICDescriptor               hostAPIC;
+   struct VMCrossPage    **crosspage;        /* numVCPUs-sized array. */
+   struct MemTrack        *ptpTracker;       /* Tracks page table patch pages */
+   struct MemTrack        *vmmTracker;       /* Tracks allocated VMM pages */
+   VCPUSet                *crosscallWaitSet; /* numVCPUs-sized array. */
+   APICDescriptor          hostAPIC;

-   struct MemTrack             *memtracker;   /* Memory tracker pointer */
-   Bool                         checkFuncFailed;
-   struct PerfCounter          *perfCounter;
-   VMMemMgmtInfo                memInfo;
-   unsigned                     fastClockRate;/* Protected by FastClockLock. */
-
-   volatile PseudoTSCOffsetInfo ptscOffsetInfo; /* Volatile per PR 699101#29. */
-   Atomic_uint64                ptscLatest;
-   int64                        ptscOffsets[MAX_INITBLOCK_CPUS];
+   struct MemTrack        *memtracker;       /* Memory tracker pointer */
+   Bool                    checkFuncFailed;
+   struct PerfCounter     *perfCounter;
+   VMMemMgmtInfo           memInfo;
+   unsigned                fastClockRate;    /* Protected by FastClockLock. */
+   Atomic_uint64           ptscOffsetInfo;   /* Volatile per PR 699101#29. */
+   Atomic_uint64           ptscLatest;
+   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
+   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -114,7 +112,10 @@ extern PseudoTSC pseudoTSC;

 #define MAX_LOCKED_PAGES (-1)

-extern VMDriver *Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize);
+extern void Vmx86_CacheNXState(void);
+extern VMDriver *Vmx86_CreateVM(VA64 bsBlob,
+                                uint32 bsBlobSize,
+                                uint32 numVCPUs);
 extern Bool Vmx86_ProcessBootstrap(VMDriver *vm,
                                   VA64 bsBlobAddr,
                                   uint32 numBytes,
@@ -147,6 +148,11 @@ extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
 extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr,
                                 unsigned numPages, Bool kernelMPNBuffer);
 extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn);
+extern MPN Vmx86_AllocLowPage(VMDriver *vm, Bool ignoreLimits);
+extern void *Vmx86_Calloc(size_t numElements,
+                          size_t elementSize,
+                          int nonPageable);
+extern void  Vmx86_Free(void *ptr);

 extern int32 Vmx86_GetNumVMs(void);
 extern int32 Vmx86_GetTotalMemUsage(void);
@@ -210,4 +216,11 @@ extern uint64 Vmx86_GetPseudoTSC(void);

 extern uint64 Vmx86_GetUnavailablePerfCtrs(void);

+extern Bool Vmx86_GetMonitorContext(VMDriver *vm, Vcpuid vcpuid,
+                                    Context64 *context);
+extern void Vmx86_CleanupVMMPages(VMDriver *vm);
+
+extern VPN Vmx86_MapPage(MPN mpn);
+extern void Vmx86_UnmapPage(VPN vpn);
+
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006, 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -47,13 +47,11 @@ typedef
 #include "vmware_pack_begin.h"
 struct InitBlock {
   uint32 magicNumber;     /* Magic number (INIT_BLOCK_MAGIC) */
-   Vcpuid numVCPUs;
+   uint32 vmInitFailurePeriod;
   VA64   crosspage[MAX_INITBLOCK_CPUS];
   LPN64  monStartLPN;
   LPN64  monEndLPN;
-   uint32 vmInitFailurePeriod;
-   LA64   crossGDTHKLA;
-   MPN    crossGDTMPNs[5];  // CROSSGDT_NUMPAGES
+   MPN    crossGDTMPN;
   uint16 numPTPPages;
 }
 #include "vmware_pack_end.h"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -267,8 +267,8 @@ CircList_DeleteItem(ListItem *p,         // IN
 {
   ListItem *next;

-   ASSERT(p);
-   ASSERT(headp);
+   ASSERT(p != NULL);
+   ASSERT(headp != NULL);

   next = p->next;
   if (p == next) {
@@ -1,30 +0,0 @@
-#ifndef __COMPAT_POLL_H__
-#define __COMPAT_POLL_H__
-
-#include <linux/poll.h>
-
-#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0))
-
-#ifndef __poll_t
-typedef unsigned int __poll_t;
-#endif
-
-static inline __poll_t compat_vfs_poll(struct file *file,
-				       struct poll_table_struct *pt)
-{
-	if (unlikely(!file->f_op->poll))
-		return DEFAULT_POLLMASK;
-	return file->f_op->poll(file, pt);
-}
-
-#else
-
-static inline __poll_t compat_vfs_poll(struct file *file,
-				       struct poll_table_struct *pt)
-{
-	return vfs_poll(file, pt);
-}
-
-#endif
-
-#endif /* __COMPAT_POLL_H__ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -26,9 +26,11 @@
 #ifndef _CONTEXTINFO_H
 #define _CONTEXTINFO_H

+#define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMX
 #include "includeCheck.h"

 #include "x86desc.h"
@@ -42,7 +42,7 @@
 typedef uint32 PCPU;
 #define INVALID_PCPU		((PCPU) -1)

-#define MAX_PCPUS       576
+#define MAX_PCPUS       1024
 #define MAX_PCPUS_BITS  10  // MAX_PCPUS <= (1 << MAX_PCPUS_BITS)
 #define MAX_PCPUS_MASK  ((1 << MAX_PCPUS_BITS) - 1)

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -70,33 +70,6 @@ typedef struct CPUIDSummary {
 } CPUIDSummary;


-/*
- *----------------------------------------------------------------------
- *
- * CPUIDSummary_RegsFromCpuid0 --
- *
- *      Fills in the given CPUIDRegs struct with the values from the CPUID0 struct.
- *
- * Results:
- *      Returns the CPUIDRegs pointer passed in.
- *
- * Side effects:
- *	None.
- *
- *----------------------------------------------------------------------
- */
-static INLINE CPUIDRegs*
-CPUIDSummary_RegsFromCpuid0(CPUID0* id0In,
-                            CPUIDRegs* id0Out)
-{
-   id0Out->eax = id0In->numEntries;
-   id0Out->ebx = *(uint32 *) (id0In->name + 0);
-   id0Out->edx = *(uint32 *) (id0In->name + 4);
-   id0Out->ecx = *(uint32 *) (id0In->name + 8);
-   return id0Out;
-}
-
-
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,6 +24,19 @@
 *      There is only one in the whole host system, shared by all VMs.
 *      It is allocated when the first VCPU is started and freed when the
 *      driver is unloaded.
+ *
+ *      The crossGDT size is exactly one page.
+ *
+ *      The hosted world switch code is based on the assumption that by
+ *      placing VMM descriptors at the end of the page, they will not
+ *      overlap with host kernel descriptors in use when "crossing over".
+ *
+ *      All necessary host segments must be from first page of GDT.
+ *      In Nov 2006, host GDT limits easily met this constraint:
+ *
+ *          Linux 64 bit:  80 (yes 80, not 7F)
+ *          MacOS 64 bit:  8F
+ *        Windows 64 bit:  6F
 */

 #ifndef _CROSSGDT_H_
@@ -38,31 +51,9 @@
 #include "vm_basic_defs.h"  // PAGE_SIZE
 #include "x86types.h"       // Descriptor

+
 typedef struct CrossGDT {
-   Descriptor gdtes[0x5000 / sizeof (Descriptor)];  // 0x5000 > GDT_LIMIT
+   Descriptor gdtes[PAGE_SIZE / sizeof(Descriptor)];
 } CrossGDT;

-#define CROSSGDT_NUMPAGES CEILING(sizeof (CrossGDT), PAGE_SIZE)
-
-/*
- * Out of the 5 pages, only the first and last are really used.
- *
- * All we need to map are the first and last pages.  This mask tells
- * the setup code which pages it can put stuff in and it tells the
- * mapping and invalidation code which pages are mapped and unmapped.
- */
-#define CROSSGDT_PAGEMASK 0x11
-#define CROSSGDT_GETINDEXMASK(i) (1 << ((i) * sizeof (Descriptor) / PAGE_SIZE))
-#define CROSSGDT_TESTINDEXMASK(i) (CROSSGDT_GETINDEXMASK(i) & CROSSGDT_PAGEMASK)
-
-/*
- * All necessary host segments must be below CROSSGDT_HOSTLIMIT. In Nov 2006,
- * host GDT limits for the various guest OSes were:
- *
- *     Linux 64 bit:  80 (yes 80, not 7F)
- *     MacOS 64 bit:  8F
- *   Windows 64 bit:  6F
- */
-#define CROSSGDT_HOSTLIMIT (PAGE_SIZE / sizeof (Descriptor)) /* 1st page */
-
 #endif
@@ -1,345 +0,0 @@
-/*********************************************************
- * Copyright (C) 2001,2016 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef _HASH_FUNC_H
-#define _HASH_FUNC_H
-
-#define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#define INCLUDE_ALLOW_VMKERNEL
-
-#include "includeCheck.h"
-
-/*
- * hashFunc.h --
- *
- *	Hash functions defined as simple wrappers around code placed
- *	in the public domain by Bob Jenkins.  His original code can
- *	be used freely for any purpose, including commercial (see the
- *	original comments in the enclosed code).
- */
-
-#include "vm_basic_types.h"
-#include "vm_basic_defs.h"
-#include "vm_assert.h"
-
-/*
- * operations
- */
-
-extern uint64 HashFunc_HashBytes(const uint8 *key,  uint32 nBytes);
-extern uint64 HashFunc_HashMoreBytes(const uint8 *key, uint32 nBytes, 
-                                     uint64 initialValue);
-extern uint64 HashFunc_HashBytesSlow(const uint8 *key, uint32 nBytes);
-extern uint64 HashFunc_HashQuads(const uint64 *key, uint32 nQuads);
-extern uint64 HashFunc_HashPage(const void *addr);
-
-/*
--------------------------------------------------------------------
-Hashes 32-bit key and returns a value that can fit into numBits bits.
--------------------------------------------------------------------
-*/
-
-static INLINE uint32
-HashFunc_TruncHash32(uint32 key, unsigned numBits)
-{
-   const uint32 hashConst = 0x9e3779b9; /* the golden ratio */
-   ASSERT(numBits > 0 && numBits <= 32);
-   return (key * hashConst) >> (32 - numBits);
-}
-
-/*
--------------------------------------------------------------------
-Hashes 64-bit key and returns a value that can fit into numBits bits.
--------------------------------------------------------------------
-*/
-
-static INLINE uint32
-HashFunc_TruncHash64(uint64 key, unsigned numBits)
-{
-   return HashFunc_TruncHash32((uint32)(key ^ (key >> 32)), numBits);
-}
-
-
-/*
- * Original Bob Jenkins public-domain code with minor modifications
- * (static functions, 32-bit length args, disabled self-test code).
- */
-
-
-typedef uint64 ub8;   /* unsigned 8-byte quantities */
-typedef uint32 ub4;   /* unsigned 4-byte quantities */
-typedef uint8  ub1;
-
-#define hashsize(n) ((ub8)1<<(n))
-#define hashmask(n) (hashsize(n)-1)
-
-/*
--------------------------------------------------------------------
-mix -- mix 3 64-bit values reversibly.
-mix() takes 48 machine instructions, but only 24 cycles on a superscalar
-  machine (like Intel's new MMX architecture).  It requires 4 64-bit
-  registers for 4::2 parallelism.
-All 1-bit deltas, all 2-bit deltas, all deltas composed of top bits of
-  (a,b,c), and all deltas of bottom bits were tested.  All deltas were
-  tested both on random keys and on keys that were nearly all zero.
-  These deltas all cause every bit of c to change between 1/3 and 2/3
-  of the time (well, only 113/400 to 287/400 of the time for some
-  2-bit delta).  These deltas all cause at least 80 bits to change
-  among (a,b,c) when the mix is run either forward or backward (yes it
-  is reversible).
-This implies that a hash using mix64 has no funnels.  There may be
-  characteristics with 3-bit deltas or bigger, I didn't test for
-  those.
--------------------------------------------------------------------
-*/
-#define mix64(a,b,c) \
-{ \
-  a -= b; a -= c; a ^= (c>>43); \
-  b -= c; b -= a; b ^= (a<<9); \
-  c -= a; c -= b; c ^= (b>>8); \
-  a -= b; a -= c; a ^= (c>>38); \
-  b -= c; b -= a; b ^= (a<<23); \
-  c -= a; c -= b; c ^= (b>>5); \
-  a -= b; a -= c; a ^= (c>>35); \
-  b -= c; b -= a; b ^= (a<<49); \
-  c -= a; c -= b; c ^= (b>>11); \
-  a -= b; a -= c; a ^= (c>>12); \
-  b -= c; b -= a; b ^= (a<<18); \
-  c -= a; c -= b; c ^= (b>>22); \
-}
-
-/*
--------------------------------------------------------------------
-hash() -- hash a variable-length key into a 64-bit value
-  k     : the key (the unaligned variable-length array of bytes)
-  len   : the length of the key, counting by bytes
-  level : can be any 8-byte value
-Returns a 64-bit value.  Every bit of the key affects every bit of
-the return value.  No funnels.  Every 1-bit and 2-bit delta achieves
-avalanche.  About 41+5len instructions.
-
-The best hash table sizes are powers of 2.  There is no need to do
-mod a prime (mod is sooo slow!).  If you need less than 64 bits,
-use a bitmask.  For example, if you need only 10 bits, do
-  h = (h & hashmask(10));
-In which case, the hash table should have hashsize(10) elements.
-
-If you are hashing n strings (ub1 **)k, do it like this:
-  for (i=0, h=0; i<n; ++i) h = hash( k[i], len[i], h);
-
-By Bob Jenkins, Jan 4 1997.  bob_jenkins@burtleburtle.net.  You may
-use this code any way you wish, private, educational, or commercial,
-as long as this whole comment accompanies it.
-
-See http://burtleburtle.net/bob/hash/evahash.html
-Use for hash table lookup, or anything where one collision in 2^^64
-is acceptable.  Do NOT use for cryptographic purposes.
--------------------------------------------------------------------
-*/
-
-static INLINE ub8 hash(register const ub1 *k, /* the key */
-                       register ub4 length,   /* the length of the key */
-                       register ub8 level)    /* the previous hash, or an arbitrary value */
-{
-  register ub8 a,b,c;
-  ub4 len;
-
-  /* Set up the internal state */
-  len = length;
-  a = b = level;                         /* the previous hash value */
-  c = CONST64U(0x9e3779b97f4a7c13); /* the golden ratio; an arbitrary value */
-
-  /*---------------------------------------- handle most of the key */
-  while (len >= 24)
-  {
-    a += (k[0]        +((ub8)k[ 1]<< 8)+((ub8)k[ 2]<<16)+((ub8)k[ 3]<<24)
-     +((ub8)k[4 ]<<32)+((ub8)k[ 5]<<40)+((ub8)k[ 6]<<48)+((ub8)k[ 7]<<56));
-    b += (k[8]        +((ub8)k[ 9]<< 8)+((ub8)k[10]<<16)+((ub8)k[11]<<24)
-     +((ub8)k[12]<<32)+((ub8)k[13]<<40)+((ub8)k[14]<<48)+((ub8)k[15]<<56));
-    c += (k[16]       +((ub8)k[17]<< 8)+((ub8)k[18]<<16)+((ub8)k[19]<<24)
-     +((ub8)k[20]<<32)+((ub8)k[21]<<40)+((ub8)k[22]<<48)+((ub8)k[23]<<56));
-    mix64(a,b,c);
-    k += 24; len -= 24;
-  }
-
-  /*------------------------------------- handle the last 23 bytes */
-  c += length;
-  switch(len)              /* all the case statements fall through */
-  {
-  case 23: c+=((ub8)k[22]<<56);
-  case 22: c+=((ub8)k[21]<<48);
-  case 21: c+=((ub8)k[20]<<40);
-  case 20: c+=((ub8)k[19]<<32);
-  case 19: c+=((ub8)k[18]<<24);
-  case 18: c+=((ub8)k[17]<<16);
-  case 17: c+=((ub8)k[16]<<8);
-    /* the first byte of c is reserved for the length */
-  case 16: b+=((ub8)k[15]<<56);
-  case 15: b+=((ub8)k[14]<<48);
-  case 14: b+=((ub8)k[13]<<40);
-  case 13: b+=((ub8)k[12]<<32);
-  case 12: b+=((ub8)k[11]<<24);
-  case 11: b+=((ub8)k[10]<<16);
-  case 10: b+=((ub8)k[ 9]<<8);
-  case  9: b+=((ub8)k[ 8]);
-  case  8: a+=((ub8)k[ 7]<<56);
-  case  7: a+=((ub8)k[ 6]<<48);
-  case  6: a+=((ub8)k[ 5]<<40);
-  case  5: a+=((ub8)k[ 4]<<32);
-  case  4: a+=((ub8)k[ 3]<<24);
-  case  3: a+=((ub8)k[ 2]<<16);
-  case  2: a+=((ub8)k[ 1]<<8);
-  case  1: a+=((ub8)k[ 0]);
-    /* case 0: nothing left to add */
-  }
-  mix64(a,b,c);
-  /*-------------------------------------------- report the result */
-  return c;
-}
-
-/*
--------------------------------------------------------------------
- This works on all machines, is identical to hash() on little-endian 
- machines, and it is much faster than hash(), but it requires
- -- that the key be an array of ub8's, and
- -- that all your machines have the same endianness, and
- -- that the length be the number of ub8's in the key
--------------------------------------------------------------------
-*/
-static INLINE ub8 hash2(register const ub8 *k, /* the key */
-                        register ub4 length,   /* the length of the key */
-                        register ub8 level)    /* the previous hash, or an arbitrary value */
-{
-  register ub8 a,b,c;
-  ub4 len;
-
-  /* Set up the internal state */
-  len = length;
-  a = b = level;                         /* the previous hash value */
-  c = CONST64U(0x9e3779b97f4a7c13); /* the golden ratio; an arbitrary value */
-
-  /*---------------------------------------- handle most of the key */
-  while (len >= 3)
-  {
-    a += k[0];
-    b += k[1];
-    c += k[2];
-    mix64(a,b,c);
-    k += 3; len -= 3;
-  }
-
-  /*-------------------------------------- handle the last 2 ub8's */
-  c += length;
-  switch(len)              /* all the case statements fall through */
-  {
-    /* c is reserved for the length */
-  case  2: b+=k[1];
-  case  1: a+=k[0];
-    /* case 0: nothing left to add */
-  }
-  mix64(a,b,c);
-  /*-------------------------------------------- report the result */
-  return c;
-}
-
-/*
--------------------------------------------------------------------
- This is identical to hash() on little-endian machines, and it is much
- faster than hash(), but a little slower than hash2(), and it requires
- -- that all your machines be little-endian, for example all Intel x86
-    chips or all VAXen.  It gives wrong results on big-endian machines.
--------------------------------------------------------------------
-*/
-
-static INLINE ub8 hash3(register const ub1 *k, /* the key */
-                        register ub4 length,   /* the length of the key */
-                        register ub8 level)    /* the previous hash, or an arbitrary value */
-{
-  register ub8 a,b,c;
-  ub4 len;
-
-  /* Set up the internal state */
-  len = length;
-  a = b = level;                         /* the previous hash value */
-  c = CONST64U(0x9e3779b97f4a7c13); /* the golden ratio; an arbitrary value */
-
-  /*---------------------------------------- handle most of the key */
-  if (((size_t)k)&7)
-  {
-    while (len >= 24)
-    {
-      a += (k[0]        +((ub8)k[ 1]<< 8)+((ub8)k[ 2]<<16)+((ub8)k[ 3]<<24)
-       +((ub8)k[4 ]<<32)+((ub8)k[ 5]<<40)+((ub8)k[ 6]<<48)+((ub8)k[ 7]<<56));
-      b += (k[8]        +((ub8)k[ 9]<< 8)+((ub8)k[10]<<16)+((ub8)k[11]<<24)
-       +((ub8)k[12]<<32)+((ub8)k[13]<<40)+((ub8)k[14]<<48)+((ub8)k[15]<<56));
-      c += (k[16]       +((ub8)k[17]<< 8)+((ub8)k[18]<<16)+((ub8)k[19]<<24)
-       +((ub8)k[20]<<32)+((ub8)k[21]<<40)+((ub8)k[22]<<48)+((ub8)k[23]<<56));
-      mix64(a,b,c);
-      k += 24; len -= 24;
-    }
-  }
-  else
-  {
-    while (len >= 24)    /* aligned */
-    {
-      a += *(ub8 *)(k+0);
-      b += *(ub8 *)(k+8);
-      c += *(ub8 *)(k+16);
-      mix64(a,b,c);
-      k += 24; len -= 24;
-    }
-  }
-
-  /*------------------------------------- handle the last 23 bytes */
-  c += length;
-  switch(len)              /* all the case statements fall through */
-  {
-  case 23: c+=((ub8)k[22]<<56);
-  case 22: c+=((ub8)k[21]<<48);
-  case 21: c+=((ub8)k[20]<<40);
-  case 20: c+=((ub8)k[19]<<32);
-  case 19: c+=((ub8)k[18]<<24);
-  case 18: c+=((ub8)k[17]<<16);
-  case 17: c+=((ub8)k[16]<<8);
-    /* the first byte of c is reserved for the length */
-  case 16: b+=((ub8)k[15]<<56);
-  case 15: b+=((ub8)k[14]<<48);
-  case 14: b+=((ub8)k[13]<<40);
-  case 13: b+=((ub8)k[12]<<32);
-  case 12: b+=((ub8)k[11]<<24);
-  case 11: b+=((ub8)k[10]<<16);
-  case 10: b+=((ub8)k[ 9]<<8);
-  case  9: b+=((ub8)k[ 8]);
-  case  8: a+=((ub8)k[ 7]<<56);
-  case  7: a+=((ub8)k[ 6]<<48);
-  case  6: a+=((ub8)k[ 5]<<40);
-  case  5: a+=((ub8)k[ 4]<<32);
-  case  4: a+=((ub8)k[ 3]<<24);
-  case  3: a+=((ub8)k[ 2]<<16);
-  case  2: a+=((ub8)k[ 1]<<8);
-  case  1: a+=((ub8)k[ 0]);
-    /* case 0: nothing left to add */
-  }
-  mix64(a,b,c);
-  /*-------------------------------------------- report the result */
-  return c;
-}
-
-#endif //_HASH_FUNC_H
@@ -44,6 +44,7 @@
 #include "pageLock_defs.h"
 #include "numa_defs.h"
 #include "bootstrap_vmm.h"
+#include "contextinfo.h"

 #if defined __cplusplus
 extern "C" {
@@ -138,13 +139,12 @@ PtrToVA64(void const *ptr) // IN
 * vmcore, and rest-of-bora. The vmmon driver is largely outside
 * vmcore and vmcore imports functionality from vmmon. Addition,
 * deletion or modification of an iocontrol used only by rest-of-bora
- * does not break vmcore compatibility. 
+ * does not break vmcore compatibility.
 *
 * See bora/doc/vmcore details.
- *
 */

-#define VMMON_VERSION           (331 << 16 | 0)
+#define VMMON_VERSION           (361 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -158,8 +158,8 @@ PtrToVA64(void const *ptr) // IN
 #else
 #define MAX_VMS 64
 #endif
-/* 
- * MsgWaitForMultipleObjects doesn't scale well enough on Win32. 
+/*
+ * MsgWaitForMultipleObjects doesn't scale well enough on Win32.
 * Allocate with MAX_VMS so static buffers are large, but do
 * admissions control with this value on Win32 until we check
 * scalability (probably in authd).
@@ -231,13 +231,10 @@ enum IOCTLCmd {
   IOCTLCMD(UNLOCK_PAGE_BY_MPN),
    /* AWE calls */
   IOCTLCMD(ALLOC_LOCKED_PAGES),
-   IOCTLCMD(FREE_LOCKED_PAGES),
   IOCTLCMD(GET_NEXT_ANON_PAGE),

   IOCTLCMD(GET_ALL_MSRS),

-   IOCTLCMD(COUNT_PRESENT_PAGES),
-  
   IOCTLCMD(GET_REFERENCE_CLOCK_HZ),
   IOCTLCMD(INIT_PSEUDO_TSC),
   IOCTLCMD(CHECK_PSEUDO_TSC),
@@ -247,6 +244,7 @@ enum IOCTLCmd {

   IOCTLCMD(GET_IPI_VECTORS),
   IOCTLCMD(SEND_IPI),
+   IOCTLCMD(SEND_ONE_IPI),

   /*
    * Keep host-specific calls at the end so they can be undefined
@@ -269,8 +267,8 @@ enum IOCTLCmd {
   IOCTLCMD(FREE_CONTIG_PAGES),
   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows
   IOCTLCMD(CHANGE_HARD_LIMIT),         // used by vmauthd on Windows
-   IOCTLCMD(GET_KERNEL_PROC_ADDRESS),
-   IOCTLCMD(READ_VA64),
+   IOCTLCMD(READ_DISASM_PROC_BINARY),
+   IOCTLCMD(CHECK_CANDIDATE_VA64),
   IOCTLCMD(SET_MEMORY_PARAMS),
   IOCTLCMD(REMEMBER_KHZ_ESTIMATE),
   IOCTLCMD(REMAP_SCATTER_LIST),
@@ -278,6 +276,7 @@ enum IOCTLCmd {
 #endif

 #if defined __APPLE__
+   IOCTLCMD(GET_NUM_RESPONDING_CPUS),
   IOCTLCMD(ALLOC_LOW_PAGES),
   IOCTLCMD(INIT_DRIVER),
   IOCTLCMD(BLUEPILL),
@@ -288,6 +287,7 @@ enum IOCTLCmd {
 #endif

   IOCTLCMD(GET_UNAVAIL_PERF_CTRS),
+   IOCTLCMD(GET_MONITOR_CONTEXT),
   // Must be last.
   IOCTLCMD(LAST)
 };
@@ -322,6 +322,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_INIT_VM             VMIOCTL_BUFFERED(INIT_VM)
 #define IOCTL_VMX86_RUN_VM              VMIOCTL_NEITHER(RUN_VM)
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
+#define IOCTL_VMX86_SEND_ONE_IPI        VMIOCTL_BUFFERED(SEND_ONE_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
 #define IOCTL_VMX86_LOOK_UP_MPN         VMIOCTL_BUFFERED(LOOK_UP_MPN)
 #define IOCTL_VMX86_LOCK_PAGE           VMIOCTL_BUFFERED(LOCK_PAGE)
@@ -346,11 +347,12 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_LOCK_PAGE_NEW       VMIOCTL_BUFFERED(LOCK_PAGE_NEW)
 #define IOCTL_VMX86_UNLOCK_PAGE_BY_MPN  VMIOCTL_BUFFERED(UNLOCK_PAGE_BY_MPN)
 #define IOCTL_VMX86_ALLOC_LOCKED_PAGES  VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES)
-#define IOCTL_VMX86_FREE_LOCKED_PAGES   VMIOCTL_BUFFERED(FREE_LOCKED_PAGES)
 #define IOCTL_VMX86_GET_NEXT_ANON_PAGE  VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE)

-#define IOCTL_VMX86_GET_KERNEL_PROC_ADDRESS  VMIOCTL_BUFFERED(GET_KERNEL_PROC_ADDRESS)
-#define IOCTL_VMX86_READ_VA64           VMIOCTL_BUFFERED(READ_VA64)
+#define IOCTL_VMX86_READ_DISASM_PROC_BINARY \
+                                      VMIOCTL_BUFFERED(READ_DISASM_PROC_BINARY)
+#define IOCTL_VMX86_CHECK_CANDIDATE_VA64 VMIOCTL_BUFFERED(CHECK_CANDIDATE_VA64)
+
 #define IOCTL_VMX86_SET_MEMORY_PARAMS   VMIOCTL_BUFFERED(SET_MEMORY_PARAMS)

 #define IOCTL_VMX86_REMEMBER_KHZ_ESTIMATE VMIOCTL_BUFFERED(REMEMBER_KHZ_ESTIMATE)
@@ -362,6 +364,7 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_CHECK_PSEUDO_TSC         VMIOCTL_BUFFERED(CHECK_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_PSEUDO_TSC           VMIOCTL_NEITHER(GET_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS    VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS)
+#define IOCTL_VMX86_GET_MONITOR_CONTEXT      VMIOCTL_BUFFERED(GET_MONITOR_CONTEXT)
 #define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
 #define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
 #endif
@@ -396,18 +399,18 @@ VMLockPage;

 typedef struct VMAPICInfo {
   uint32 flags;
-} VMAPICInfo; 
+} VMAPICInfo;

-#define VMX86_DRIVER_VCPUID_OFFSET	1000
+#define VMX86_DRIVER_VCPUID_OFFSET 1000


 /*
 * We keep track of 3 different limits on the number of pages we can lock.
 * The host limit is determined at driver load time (in windows only) to
 * make sure we do not starve the host by locking too many pages.
- * The static limit is user defined in the UI and the dynamic limit is 
+ * The static limit is user defined in the UI and the dynamic limit is
 * set by authd's hardLimitMonitor code (windows only), which queries
- * host load and adjusts the limit accordingly.  We lock the minimum of 
+ * host load and adjusts the limit accordingly.  We lock the minimum of
 * all these values at any given time.
 */
 typedef struct LockedPageLimit {
@@ -464,7 +467,7 @@ typedef struct VMMemMgmtInfoPatch {
 */

 typedef struct VMMemInfoArgs {
-   uint64          currentTime;        // Host time in secs of the call. 
+   uint64          currentTime;        // Host time in secs of the call.
   uint32          minVmMemPct;        // % of vm that must fit in memory
   uint32          globalMinAllocation;// pages that must fit in maxLockedPages
   uint32          numLockedPages;     // total locked pages by all vms
@@ -491,13 +494,6 @@ typedef struct VMMPNList {
   VA64      mpnList;    // IN: User VA of an array of 64-bit MPNs.
 } VMMPNList;

-typedef struct VARange {
-   VA64     addr;
-   VA64     bv;
-   unsigned len;
-   uint32   pad;
-} VARange;
-
 typedef struct VMMUnlockPageByMPN {
   MPN       mpn;
   VA64      uAddr;         /* IN: User VA of the page (optional). */
@@ -508,11 +504,6 @@ typedef struct VMMReadWritePage {
   VA64         uAddr; // IN: User VA of a PAGE_SIZE-large buffer.
 } VMMReadWritePage;

-struct passthrough_iorange {
-   unsigned short ioBase;   /* Base of range to pass through. */
-   unsigned short numPorts; /* Length of range. */
-};
-
 /*
 * Data structure for the INIT_PSEUDO_TSC and CHECK_PSEUDO_TSC.
 */
@@ -539,14 +530,9 @@ typedef struct PTSCCheckParams {

 typedef struct IPIVectors {
   /*
-    * Vector(s) the host uses for its own IPIs; we use this as a performance
-    * hint.
-    */
-   uint8 hostIPIVectors[2];
-   /* 
    * Vectors we have allocated or stolen for the monitor interrupts.
    */
-   uint8 monitorIPIVector; 
+   uint8 monitorIPIVector;
   uint8 hvIPIVector;
 } IPIVectors;

@@ -558,6 +544,7 @@ typedef struct IPIVectors {
 typedef struct VMCreateBlock {
   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
+   uint32             numVCPUs;      // IN: Number of VCPUs.
   uint16             vmid;          // OUT: VM ID for the created VM.
 } VMCreateBlock;

@@ -582,6 +569,14 @@ typedef struct VMProcessBootstrapBlock {
   VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions.
 } VMProcessBootstrapBlock;

+/*
+ * Arguments for VMM context retrieval.
+ */
+typedef union {
+   Vcpuid vcpuid;     // IN
+   Context64 context; // OUT
+} VMMonContext;
+
 #if defined __linux__

 /*
@@ -41,14 +41,17 @@
 #include "modulecallstructs.h"
 #include "mon_assert.h"

-#define NUM_EXCEPTIONS 20       /* EXC_DE ... EXC_XF. */
- 
+#define NUM_EXCEPTIONS   20     /* EXC_DE ... EXC_XF. */
+#define CP_STUB_SIZE 16     /* A relative jmp instruction (5 bytes)
+                               padded to the next 16 byte boundary. */
+
 #define MODULECALL_TABLE                                                      \
   MC(INTR)                                                                   \
   MC(SEMAWAIT)                                                               \
   MC(SEMASIGNAL)                                                             \
   MC(SEMAFORCEWAKEUP)                                                        \
-   MC(IPI)          /* Hit thread with IPI. */                                \
+   MC(IPI)          /* Hit multiple threads with IPI. */                      \
+   MC(ONE_IPI)      /* Hit a single thread with IPI. */                       \
   MC(USERRETURN)   /* Return codes for user calls. */                        \
   MC(GET_RECYCLED_PAGES)                                                     \
   MC(RELEASE_ANON_PAGES)                                                     \
@@ -58,7 +61,12 @@
   MC(ALLOC_TMP_GDT)                                                          \
   MC(PIN_MPN)                                                                \
   MC(VMCLEAR_VMCS_ALL_CPUS)                                                  \
-   MC(GET_PAGE_ROOT)
+   MC(GET_PAGE_ROOT)                                                          \
+   MC(ALLOC_ANON_LOW_PAGE)                                                    \
+   MC(GET_MON_IPI_VECTOR)                                                     \
+   MC(GET_HV_IPI_VECTOR)                                                      \
+   MC(GET_HOST_TIMER_VECTORS)                                                 \
+   MC(BOOTSTRAP_CLEANUP)

 /*
 *----------------------------------------------------------------------
@@ -67,9 +75,6 @@
 *
 *      Enumeration of support calls done by the module.
 *
- *      If anything changes in the enum, please update kstatModuleCallPtrs
- *      for stats purposes.
- *
 *----------------------------------------------------------------------
 */

@@ -126,15 +131,6 @@ typedef enum UCCostStamp {
 } UCCostStamp;
 #endif // VMX86_SERVER

-typedef
-#include "vmware_pack_begin.h"
-struct CodeOffsets {
-   uint16 hostToVmm;
-   uint16 vmmToHost;
-}
-#include "vmware_pack_end.h"
-CodeOffsets;
-
 #define SHADOW_DR(cpData, n)    (cpData)->shadowDR[n].ureg64


@@ -143,12 +139,12 @@ CodeOffsets;
 * MAX_SWITCH_PT_PATCHES
 *
 *   This is the maximum number of patches that must be placed into
- *   the monitor page tables so that two pages of the host GDT and the
- *   crosspage can be accessed during worldswitch.
+ *   the monitor page tables so that the host GDT and the crosspage
+ *   can be accessed during worldswitch.
 *
 *----------------------------------------------------------------------
 */
-#define MAX_SWITCH_PT_PATCHES 3
+#define MAX_SWITCH_PT_PATCHES 2

 /*----------------------------------------------------------------------
 *
@@ -214,6 +210,7 @@ struct VMMPageTablePatch {
   uint32   level;              /* [0, 4]  (maximal size: 3 bits) */
   uint32   pteIdx;             /* Index of the PTE in the page table. */
   uint64   pteGlobalIdx;       /* Global index of the PTE in 'level'. */
+   LPN      lpn;                /* Logical page number mapped by patch. */
   VM_PDPTE pte;                /* PTE.                                */
 }
 #include "vmware_pack_end.h"
@@ -221,6 +218,23 @@ VMMPageTablePatch;

 #define MODULECALL_NUM_ARGS  4

+/*
+ * The cross page contains tiny stacks upon which interrupt and exception
+ * handlers in the switch path may temporarily run.  Each stack must be
+ * large enough for the sum of:
+ *
+ * - 1 #DB exception frame (5 * uint64)
+ * - 1 #NMI exception frame (5 * uint64)
+ * - 1 #MCE exception frame (5 * uint64)
+ * - the largest stack use instantaneously possible by #MCE handling code
+ * - the largest stack use instantaneously possible by #NMI handling code
+ * - the largest stack use instantaneously possible by #DB handling code
+ * - one high-water uint32 used to detect stack overflows when debugging
+ *
+ * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+ */
+#define TINY_STACK_SIZE      184
+
 /*
 *----------------------------------------------------------------------
 *
@@ -233,72 +247,71 @@ VMMPageTablePatch;
 typedef
 #include "vmware_pack_begin.h"
 struct VMCrossPageData {
+   uint32   version;           // CROSSPAGE_VERSION
+   uint32   vmmonVersion;      // VMMON_VERSION
+
   /*
-    * A tiny stack upon which interrupt and exception handlers in the switch
-    * path temporarily run.  Keep the end 16-byte aligned.  This stack must
-    * be large enough for the sum of:
-    *
-    * - 1 #DB exception frame (5 * uint64)
-    * - 1 #NMI exception frame (5 * uint64)
-    * - 1 #MCE exception frame (5 * uint64)
-    * - the largest stack use instantaneously possible by #MCE handling code
-    * - the largest stack use instantaneously possible by #NMI handling code
-    * - the largest stack use instantaneously possible by #DB handling code
-    * - one high-water uint32 used to detect stack overflows when debugging
-    * - remaining pad bytes to align to 16 bytes
-    *
-    * 184 bytes is slightly more than enough as of 2015/03/17 -- fjacobs.
+    * The following stacks and contexts are ordered for performance and code
+    * simplicity.  Both HostToVmm and VmmToHost strictly require this ordering.
+    * For any change to this data, both functions must be updated.
+    * Use VMX86_UCCOST to measure performance when changing the layout.
    */
-   uint32   tinyStack[46];

-   uint64   hostCR3;
-   uint32   crosspageMA;
-
-   uint8    hostDRSaved;        // Host DR spilled to hostDR[x].
-   uint8    hostDRInHW;         // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
-                                //   contains host-sized DB,NMI,MCE entries
-   uint16   hostSS;
-   uint64   hostRSP;
-   uint64   hostDR[8];
+   /* A tiny stack and the host context. */
+   uint64   hostTinyStack[TINY_STACK_SIZE / sizeof(uint64)];
+   uint64   hostCR3; /* Edge of context saved/restored in assembly. */
   uint64   hostRBX;
+   uint64   hostRBP;
   uint64   hostRSI;
   uint64   hostRDI;
-   uint64   hostRBP;
   uint64   hostR12;
   uint64   hostR13;
   uint64   hostR14;
   uint64   hostR15;
+   uint64   hostRSP;
+   uint16   hostSS;  /* Edge of context saved/restored in assembly. */
+   uint16   hostDS;  /* Not saved/restored in assembly switch */
+   uint16   hostES;  /* Not saved/restored in assembly switch */
+   uint16   hostPad;
+
+   /* A tiny stack and the monitor context. */
+   uint64   monTinyStack[TINY_STACK_SIZE / sizeof(uint64)];
+   uint64   monCR3; /* Edge of context saved/restored in assembly. */
+   uint64   monRBX;
+   uint64   monRBP;
+   uint64   monR12;
+   uint64   monR13;
+   uint64   monR14;
+   uint64   monR15;
+   uint64   monRSP;
+   uint16   monSS;  /* Edge of context saved/restored in assembly. */
+   uint16   monDS;  /* Not saved/restored in assembly switch */
+   uint16   monES;  /* Not saved/restored in assembly switch */
+   uint16   monPad;
+
+   uint64   crosspageMA;
+
+   uint64   hostDR[8];
   LA64     hostCrossPageLA;   // where host has crosspage mapped
   uint16   hostInitial64CS;
-   uint16   _pad0[3];
+   uint8    hostDRSaved;       // Host DR spilled to hostDR[x].
+   uint8    hostDRInHW;        // 0 -> shadowDR in h/w, 1 -> hostDR in h/w.
+   uint32   _pad0;

   uint64   wsCR0;
   uint64   wsCR4;

   DTR64    crossGDTHKLADesc;   // always uses host kernel linear address
   uint16   _pad1[3];
-   DTR64    mon64GDTR;
-   uint16   mon64ES;
-   uint16   mon64SS;
-   uint16   mon64DS;
-   uint64   mon64CR3;
-   uint64   mon64RBX;
-   uint64   mon64RSP;
-   uint64   mon64RBP;
-   uint64   mon64RSI;
-   uint64   mon64RDI;
-   uint64   mon64R12;
-   uint64   mon64R13;
-   uint64   mon64R14;
-   uint64   mon64R15;
-   uint64   mon64RIP;
-   Task64   monTask64;          /* vmm64's task */
+   DTR64    monGDTR;
+   uint16   _pad2[3];
+   /* A hardcoded value for monitor %rip which facilitates backtraces. */
+   uint64   monRIP;
+   Task64   monTask;          /* vmm's task */

   VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
-   LA64              vmm64CrossPageLA;
-   LA64              vmm64CrossGDTLA;   // where crossGDT mapped by PT patch
-                                        //  64-bit host: host kernel linear
-                                        // address
+   LA64              vmmCrossPageLA;
+   LA64              vmmCrossGDTLA;   // where crossGDT is mapped by PT patch

   /*
    * The monitor may requests up to two actions when returning to the
@@ -318,12 +331,12 @@ struct VMCrossPageData {
   int            userCallType;
   uint32         pcpuNum;   /* Used as extra module call arg within vmmon. */

-   VCPUSet        yieldVCPUs;
-
 #if !defined(VMX86_SERVER)
   uint64 ucTimeStamps[UCCOST_MAX];
 #endif

+   SwitchedMSRState switchedMSRState;
+
   /*
    * The values in the shadow debug registers must match those in the
    * hardware debug register immediately after a task switch in
@@ -333,8 +346,11 @@ struct VMCrossPageData {
   SharedUReg64     shadowDR[8];
   uint8            shadowDRInHW; // bit n set iff %DRn == shadowDR[n]

-   SwitchedMSRState switchedMSRState;
-   uint8            _pad2[7];
+   /* TRUE if no bits are set in yieldVCPUs, FALSE otherwise. */
+   Bool             yieldVCPUsIsEmpty;
+   uint8            _pad3[6];
+
+   VCPUSet          yieldVCPUs;

   /*
    * Adjustment for machines where the hardware TSC does not run
@@ -357,22 +373,22 @@ struct VMCrossPageData {
    * restart RunVM call, nobody else should look at it.
    */
   Bool     moduleCallInterrupted;
-   uint8    _pad3[6];
+   uint8    _pad4[6];

   DTR64    switchHostIDTR;   // baseLA = switchHostIDT's host kernel LA
-   uint16   _pad4[3];
-   DTR64    switchMonIDTR;    // baseLA = switchMonIDT's monitor LA
   uint16   _pad5[3];
+   DTR64    switchMonIDTR;    // baseLA = switchMonIDT's monitor LA
+   uint16   _pad6[3];

   /*
    * Descriptors and interrupt tables for switchNMI handlers.  Each
    * IDT has only enough space for the hardware exceptions; they are
    * sized to accommodate 64-bit descriptors.
    */
-   uint8 switchHostIDT[sizeof(Gate64) * NUM_EXCEPTIONS]; // hostCS:hostVA
-   uint8 switchMonIDT[sizeof(Gate64) * NUM_EXCEPTIONS];  // monCS:monVA
-
+   Gate64 switchHostIDT[NUM_EXCEPTIONS];      // hostCS:hostVA
+   Gate64 switchMonIDT[NUM_EXCEPTIONS];       // monCS:monVA
   volatile Bool wsException[NUM_EXCEPTIONS]; // Tracks faults in worldswitch.
+   uint8         _pad7[4];
   uint64        wsUD2;                       // IP of ud2 instr or 0 if unset.
   uint64        specCtrl; /* host MSR_SPEC_CTRL value before world switch. */
 }
@@ -389,13 +405,19 @@ VMCrossPageData;
 *
 *----------------------------------------------------------------------
 */
+
+#define CODE_BLOCK_SIZE (PAGE_SIZE - sizeof(VMCrossPageData) - \
+                         (2 + NUM_EXCEPTIONS) * CP_STUB_SIZE)
+
 typedef
 #include "vmware_pack_begin.h"
 struct VMCrossPageCode {
-   uint16        size;           // Size of the code module.
-   uint32        vmmonVersion;   // VMMON_VERSION
-   CodeOffsets   offsets;        // Offsets to handlers.
-   uint8         codeBlock[512]; // Code for worldswitch and fault handling.
+   uint8         toVmmFunc[CP_STUB_SIZE];    // Fixed-position stubs to jump
+   uint8         toHostFunc[CP_STUB_SIZE];   // to world switch functions
+   uint8         gateStubs[NUM_EXCEPTIONS *
+                           CP_STUB_SIZE];    // Stubs for crossIDT.
+   uint8         codeBlock[CODE_BLOCK_SIZE]; // Code for worldswitch and
+                                             // fault handling.
 }
 #include "vmware_pack_end.h"
 VMCrossPageCode;
@@ -408,10 +430,10 @@ VMCrossPageCode;
 *
 *      Data structure shared between the monitor and the module
 *      that is used for crossing between the two.
- *      Accessible as vm->cross (kernel module) and CROSS_PAGE
+ *      Accessible as vm->crosspage (kernel module) and CROSS_PAGE
 *      (monitor)
 *
- *      Exactly one page long
+ *      Must be exactly one page long.
 *
 *----------------------------------------------------------------------
 */
@@ -419,19 +441,13 @@ VMCrossPageCode;
 typedef
 #include "vmware_pack_begin.h"
 struct VMCrossPage {
-   uint32          version;         /* 4 bytes. Must be at offset zero. */
-   uint32          crosspage_size;  /* 4 bytes. Must be at offset 4.    */
   VMCrossPageData crosspageData;
   VMCrossPageCode crosspageCode;
-   uint8           _pad[PAGE_SIZE - (sizeof(uint32) /* version */        +
-                                     sizeof(uint32) /* crosspage_size */ +
-                                     sizeof(VMCrossPageData)             +
-                                     sizeof(VMCrossPageCode))];
 }
 #include "vmware_pack_end.h"
 VMCrossPage;

-#define CROSSPAGE_VERSION_BASE 0xbfc /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc09 /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005,2007,2009,2013,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * monAddrLayout.h --
 *
- *    Address layout of the monitor
+ *    Address layout of the monitor.
 */

 #ifndef _MON_ADDR_LAYOUT_H
@@ -33,25 +33,25 @@

 #pragma pack(push, 1)
 typedef struct VMM64_AddrLayout {
-   /* 
+   /*
    *All address are VPNs and all lengths are numPages
    */
   uint64 monBase;            // MONITOR_BASE_VPN
-   uint64 mmuRootStart;       // MMU_ROOT_START
-   uint32 mmuRootLen;         // MMU_ROOT_LEN
-   uint64 mmuL3Start;         // MMU_L3_START
-   uint32 mmuL3Len;           // MMU_L3_LEN
-   uint64 mmuL2Start;         // MMU_L2_START
-   uint32 mmuL2Len;           // MMU_L2_LEN
-   uint64 monPageTableStart;  // MON_PAGE_TABLE_START
-   uint32 monPageTableLen;    // MON_PAGE_TABLE_LEN
+   uint64 monL4Start;         // MON_PAGE_TABLE_L4_START
+   uint32 monL4Len;           // MON_PAGE_TABLE_L4_LEN
+   uint64 monL3Start;         // MON_PAGE_TABLE_L3_START
+   uint32 monL3Len;           // MON_PAGE_TABLE_L3_LEN
+   uint64 monL2Start;         // MON_PAGE_TABLE_L2_START
+   uint32 monL2Len;           // MON_PAGE_TABLE_L2_LEN
+   uint64 monL1Start;         // MON_PAGE_TABLE_L1_START
+   uint32 monL1Len;           // MON_PAGE_TABLE_L1_LEN
 #ifndef VMX86_SERVER
   uint64 monCpStart;         // CROSS_PAGE_START
 #endif
 } VMM64_AddrLayout;
 #pragma pack(pop)

-#define VMM_SCRATCHAS_FIRST_L4OFF   1
-#define VMM_SCRATCHAS_LAST_L4OFF  127
+#define VMM_MONAS_FIRST_L4OFF   1
+#define VMM_MONAS_LAST_L4OFF  127

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,9 +59,10 @@
 * ML_CONTENT_COPY: Allocates new MPNs, maps them and copies from the specified
 * source.
 *
- * ML_CONTENT_SHARE: Memory provided by user or host kernel and mapped into the
- * monitor address space.  The subIndex field specifies which region, as
- * multiple regions may be shared for each source.
+ * ML_CONTENT_SHARE: Memory provided by user, host kernel, or VMM blob,
+ * and mapped into the monitor address space.  The subIndex field
+ * specifies which region, as multiple regions may be shared for each
+ * source.
 *
 * Processing
 * ==========
@@ -188,7 +189,7 @@ typedef struct {

 /* Packed for easy consumption by bootstrap-offsets.pl. */
 #pragma pack(push, 1)
-typedef struct {
+typedef struct MonLoaderHeader {
   uint64         magic;
   uint32         entrySize;
   uint32         count;
@@ -204,29 +205,29 @@ typedef struct {

 /* Environment context structure, defined by the environment. */
 struct MonLoaderEnvContext;
-typedef struct MonLoaderEnvContext MonLoaderEnvContext;

 /* Callout prototypes */
-MPN  MonLoaderCallout_AllocMPN(MonLoaderEnvContext *, Vcpuid);
-void MonLoaderCallout_CleanUp(MonLoaderEnvContext *);
-Bool MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *, uint64, size_t, MPN,
-                                   Vcpuid);
-Bool MonLoaderCallout_FillPage(MonLoaderEnvContext *, uint8, MPN, Vcpuid);
-MPN  MonLoaderCallout_GetPageRoot(MonLoaderEnvContext *, Vcpuid);
-Bool MonLoaderCallout_GetPTE(MonLoaderEnvContext *, MPN, unsigned, Vcpuid,
-                             PT_L1E *);
-Bool MonLoaderCallout_ImportPage(MonLoaderEnvContext *, MPN, Vcpuid);
-Bool MonLoaderCallout_Init(void *, MonLoaderEnvContext **);
-Bool MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *, MPN, unsigned, uint64,
-                                  MPN, Vcpuid);
-MPN  MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *, uint64, unsigned,
-                                        Vcpuid);
-MPN  MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *, uint64, unsigned,
-                                        Vcpuid);
-Bool MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *);
-Bool MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *, uint16, VA64, uint16,
-                                    VA64);
-MPN  MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *, uint64);
+MPN  MonLoaderCallout_AllocMPN(struct MonLoaderEnvContext *, Vcpuid);
+void MonLoaderCallout_CleanUp(struct MonLoaderEnvContext *);
+Bool MonLoaderCallout_CopyFromBlob(struct MonLoaderEnvContext *, uint64,
+                                   size_t, MPN, Vcpuid);
+Bool MonLoaderCallout_FillPage(struct MonLoaderEnvContext *, uint8, MPN,
+                               Vcpuid);
+MPN  MonLoaderCallout_GetPageRoot(struct MonLoaderEnvContext *, Vcpuid);
+Bool MonLoaderCallout_GetPTE(struct MonLoaderEnvContext *, MPN, unsigned,
+                             Vcpuid, PT_L1E *);
+Bool MonLoaderCallout_ImportPage(struct MonLoaderEnvContext *, MPN, Vcpuid);
+Bool MonLoaderCallout_Init(void *, struct MonLoaderEnvContext **);
+Bool MonLoaderCallout_MapMPNInPTE(struct MonLoaderEnvContext *, MPN, unsigned,
+                                  uint64, MPN, Vcpuid);
+MPN  MonLoaderCallout_GetSharedUserPage(struct MonLoaderEnvContext *, uint64,
+                                        unsigned, Vcpuid);
+MPN  MonLoaderCallout_GetSharedHostPage(struct MonLoaderEnvContext *, uint64,
+                                        unsigned, Vcpuid);
+Bool MonLoaderCallout_IsPrivileged(struct MonLoaderEnvContext *);
+Bool MonLoaderCallout_SetEntrypoint(struct MonLoaderEnvContext *, uint16, VA64,
+                                    uint16, VA64);
+MPN  MonLoaderCallout_GetBlobMpn(struct MonLoaderEnvContext *, uint64);

 typedef enum MonLoaderError {
   ML_OK = 0,
@@ -259,7 +260,7 @@ typedef enum MonLoaderError {


 /* A subindex above shared area subindices for sharing of MonLoaderHeader. */
-#define MONLOADER_HEADER_IDX (NUM_SHARED_AREAS + 1)
+#define MONLOADER_HEADER_IDX 6


 MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
@@ -26,137 +26,16 @@
 #define INCLUDE_ALLOW_DISTRIBUTE
 #include "includeCheck.h"

-#include "vm_assert.h"
-#include "vm_atomic.h"
 #include "vm_basic_types.h"

 #if defined __cplusplus
 extern "C" {
 #endif

-
-/*
- * Please bump the version number if your change will break the
- * compatability to the drivers.
- */
-#define ACTION_EXPORTED_VERSION   2
-
-#define ACTION_WORD_SIZE   (sizeof(uint64) * 8)
-#define ACTION_NUM_WORDS   (2)
-#define ACTION_NUM_IDS     (ACTION_NUM_WORDS * ACTION_WORD_SIZE)
-
 #define MONACTION_INVALID  MAX_UINT32

 typedef uint32 MonitorIdemAction;

-/*
- * Representation of a set of actions.
- */
-typedef struct MonitorActionSet {
-   volatile uint64 word[ACTION_NUM_WORDS];
-} MonitorActionSet;
-
-#ifndef __cplusplus
-typedef enum MonitorActionSetName MonitorActionSetName;
-#endif
-
-/*
- * Summary of action and interrupt states.
- */
-typedef struct MonitorActionIntr {
-   MonitorActionSet  pendingSet;
-   volatile Bool     action;
-   Bool              intr;
-   Bool              nmi;
-   Bool              db;
-   uint32            _pad;
-} MonitorActionIntr;
-
-/*
- *------------------------------------------------------------------------
- * MonitorActionSet_AtomicInclude --
- *
- *    This function atomically adds an action to an action set.
- *
- * Results:
- *    TRUE if the action being added did not exist in the action set.
- *    FALSE otherwise.
- *
- * Side effects:
- *    The given action set will be updated.
- *------------------------------------------------------------------------
- */
-static INLINE Bool
-MonitorActionSet_AtomicInclude(MonitorActionSet *set, const uint32 actionID)
-{
-   Atomic_uint64 *atomicSet =
-      Atomic_VolatileToAtomic64(&set->word[actionID / ACTION_WORD_SIZE]);
-   uint64 mask = (uint64)1 << (actionID % ACTION_WORD_SIZE);
-   uint64 oldWord;
-   uint64 newWord;
-
-   ASSERT_ON_COMPILE((ACTION_WORD_SIZE & (ACTION_WORD_SIZE - 1)) == 0);
-#ifdef VMX86_DEBUG
-   /* If ASSERT is not desirable, do explicit check. Please see PR 567811. */
-#ifdef MODULE
-   if (UNLIKELY(actionID / ACTION_WORD_SIZE >= ACTION_NUM_WORDS)) {
-      return FALSE;
-   }
-#else
-   ASSERT(actionID / ACTION_WORD_SIZE < ACTION_NUM_WORDS);
-#endif // MODULE
-#endif // VMX86_DEBUG
-   do {
-      oldWord = Atomic_Read64(atomicSet);
-      newWord = oldWord | mask;
-   } while (!Atomic_CMPXCHG64(atomicSet, oldWord, newWord));
-   return (oldWord & mask) == 0;
-}
-
-
-/*
- *----------------------------------------------------------------------------
- * MonitorAction_SetBits --
- *
- *      The core logic for posting an action. Update the set of pending
- *      actions of the target VCPU in the shared area to mark the action
- *      as present. Make sure the bit is set in the pendingSet first to
- *      avoid a race with the drain loop.
- *
- *      It's the responsibility of the callers to ensure that the change
- *      to actionIntr->action is globally visible before any IPI is sent
- *      (the change to pendingSet is pushed out by the cmpxchg in
- *      MonitorActionSet_AtomicInclude).
- *
- * Results:
- *      TRUE if the action being posted was not pending before.
- *      FALSE otherwise (other threads could have posted the same action).
- *
- * Side effects:
- *      None.
- *----------------------------------------------------------------------------
- */
-static INLINE Bool
-MonitorAction_SetBits(MonitorActionIntr *actionIntr, MonitorIdemAction actionID)
-{
-   /* Careful if optimizing this: see PR70016. */
-   Bool newAction =
-      MonitorActionSet_AtomicInclude(&actionIntr->pendingSet, actionID);
-   actionIntr->action = TRUE;
-   return newAction;
-}
-
-/*
- * C1 states entered by monitor while waiting for an action
- */
-typedef enum {
-   VMM_C1_STATE_INVALID = 0,
-   VMM_C1_STATE_HLT,
-   VMM_C1_STATE_MWAIT,
-   VMM_C1_STATE_PAUSE
-} vmmC1StateType;
-
-
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -48,7 +48,11 @@ typedef uint8  NUMA_MemRangeID;
 */
 #define NUMA_MAX_NODES              64
 #define NUMA_MAX_CPUS_PER_NODE      (vmx86_server ? MAX_PCPUS : 32)
-#define NUMA_MAX_MEM_RANGES         (vm_arm_64 ? 128 : 64)
+#ifdef VMKBOOT
+#define NUMA_MAX_MEM_RANGES         64
+#else
+#define NUMA_MAX_MEM_RANGES         256
+#endif
 #define INVALID_NUMANODE            ((NUMA_Node)-1)
 #define NUMA_NODE_MASK_ANY          ((NUMA_NodeMask)-1)
 #define NUMA_NODE_MASK_NONE         ((NUMA_NodeMask)0)
@@ -70,4 +74,10 @@ typedef struct NUMA_MemRangesList {
   NUMA_MemRange memRange[NUMA_MAX_MEM_RANGES];
 } NUMA_MemRangesList;

+
+typedef struct NUMA_MemRangesListRef {
+   uint64        numMemRanges;
+   const NUMA_MemRange *memRange;
+} NUMA_MemRangesListRef;
+
 #endif // _NUMA_DEFS_H
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,16 +65,6 @@ typedef struct FPTIntrProxyInfo {
   uint32            adapterIndex;
   uint32            vectorIndex;
   MonitorIdemAction actionID;
-   uint32 _pad0;
-
-   /*
-    * These addresses are only needed for hosted platforms, where our kernel
-    * modules lack a SharedArea API.
-    */
-#ifndef VMX86_SERVER
-   VA64              notifyUVA;
-   VA64              vectorIndexUVA;
-#endif
 } FPTIntrProxyInfo;

 typedef struct UPTIntrProxyInfo {
@@ -41,8 +41,8 @@
 */

 #define PSHARE_PAGELIST_MAX             (PAGELIST_MAX)
-#define PSHARE_P2M_BUFFER_MPNS_MAX      (16)
-#define PSHARE_P2M_BUFFER_MPNS_DEFAULT  (4)
+#define PSHARE_P2M_BUFFER_MPNS_MAX      (32)
+#define PSHARE_P2M_BUFFER_MPNS_DEFAULT  (5)
 #define PSHARE_P2M_BUFFER_SLOTS_PER_MPN (PAGE_SIZE / sizeof(PShare_P2MUpdate))
 #define PSHARE_P2M_BUFFER_SLOTS_MIN     (2)

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -97,6 +97,7 @@ extern PTSCInfo ptscInfo;
 Bool PTSC_Init(uint64 tscHz);
 VmAbsoluteTS PTSC_InitialCount(const char *module,
                               const char *option,
+                               Bool stress,
                               VmIntervalTS freq,
                               VmAbsoluteTS defaultCnt);
 Bool PTSC_HasPerfectlySynchronizedTSCs(void);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014, 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -50,9 +50,9 @@ typedef uint32 Vcpuid;                 // VCPU number
 #define BOOT_VCPU_ID     0
 #define IS_BOOT_VCPUID(vcpuid)  ((vcpuid) == BOOT_VCPU_ID)

-#define MAX_VCPUS      192
-
+#define MAX_VCPUS            2048
 #define MAX_CORES_PER_SOCKET   64
+#define MAX_VCPU_SOCKETS      128

 #ifdef VMM
 #include "vcpuset.h"
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
 * vcpuset.h --
 *
- *	ADT for a set of VCPUs.  Implemented as an array of bitmasks.
+ *      ADT for a set of VCPUs.  Implemented as an array of bitmasks.
 *
 */

@@ -80,19 +80,36 @@

 extern VCPUSet  vcpuSetFull;

-#define FOR_EACH_VCPU_IN_SET(_vcpuSet, _v)                                    \
-   do {                                                                       \
-      Vcpuid  _v;                                                             \
-      VCPUSet __vcs;                                                          \
-      VCPUSet_Copy(&__vcs, _vcpuSet);                                         \
-      while ((_v = VCPUSet_FindFirst(&__vcs)) != VCPUID_INVALID) {            \
-         VCPUSet_Remove(&__vcs, _v);
+#define FOR_EACH_VCPU_IN_SET_WITH_MAX(_vcpuSet, _v, _numVcpus)                 \
+   do {                                                                        \
+      Vcpuid _v;                                                               \
+      const VCPUSet *__vcs = (_vcpuSet);                                       \
+      unsigned _subsetIdx = 0;                                                 \
+      uint64 _subset = VCPUSet_Subset(__vcs, _subsetIdx);                      \
+      unsigned _maxSubsets = VCS_VCPUID_SUBSET_IDX((_numVcpus) - 1) + 1;       \
+      ASSERT(_maxSubsets <= VCS_SUBSET_COUNT);                                 \
+      while ((_v = VCPUSet_FindFirstInSubset(__vcs, &_subset, &_subsetIdx,     \
+                                             _maxSubsets)) != VCPUID_INVALID) {\

-#define ROF_EACH_VCPU_IN_SET()                                                \
-      }                                                                       \
+#define ROF_EACH_VCPU_IN_SET_WITH_MAX()                                 \
+      }                                                                 \
   } while (0)


+#define FOR_EACH_VCPU_IN_SET(_vcpuSet, _v)                              \
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(_vcpuSet, _v, MAX_VCPUS)
+
+#define ROF_EACH_VCPU_IN_SET()                                          \
+      }                                                                 \
+   } while (0)
+
+#define FOR_EACH_VCPU_IN_POPULATED_VCPUS(_vcpuSet, _v)                  \
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(_vcpuSet, _v, NumVCPUs())
+
+#define ROF_EACH_VCPU_IN_POPULATED_VCPUS()                              \
+      }                                                                 \
+   } while (0)
+
 #define FOR_EACH_SUBSET_IN_SET(_setIndex)                                     \
   do {                                                                       \
      int _setIndex;                                                          \
@@ -113,10 +130,10 @@ extern VCPUSet  vcpuSetFull;
   } while (0)


-#define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex)                           \
+#define FOR_EACH_POPULATED_SUBSET_IN_SET(_setIndex, _numVcpus)                \
   do {                                                                       \
      int _setIndex;                                                          \
-      int _maxSubsets = VCS_VCPUID_SUBSET_IDX(NumVCPUs() - 1);                \
+      int _maxSubsets = VCS_VCPUID_SUBSET_IDX(_numVcpus - 1);                 \
      for (_setIndex = 0; _setIndex <= _maxSubsets; _setIndex++) {

 #define ROF_EACH_POPULATED_SUBSET_IN_SET()                                    \
@@ -128,7 +145,7 @@ extern VCPUSet  vcpuSetFull;
 *----------------------------------------------------------------------
 *
 * VCPUSet_Empty --
- *      
+ *
 *      Clear all bits in a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -147,7 +164,7 @@ VCPUSet_Empty(VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsEmpty --
- *      
+ *
 *      Return TRUE iff a VCPUSet has no bits set.
 *
 *----------------------------------------------------------------------
@@ -192,7 +209,7 @@ VCPUSet_Full(void)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Copy --
- *      
+ *
 *      Copy one VCPUSet's contents to another.
 *
 *----------------------------------------------------------------------
@@ -211,7 +228,7 @@ VCPUSet_Copy(VCPUSet *dest, const VCPUSet *src)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Equals --
- *      
+ *
 *      Compare two VCPUSets, return TRUE iff their contents match.
 *
 *----------------------------------------------------------------------
@@ -233,7 +250,7 @@ VCPUSet_Equals(const VCPUSet *vcs1, const VCPUSet *vcs2)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsMember --
- *      
+ *
 *      Return TRUE iff the given Vcpuid is present in a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -252,7 +269,7 @@ VCPUSet_IsMember(const VCPUSet *vcs, Vcpuid v)
 *----------------------------------------------------------------------
 *
 * VCPUSet_AtomicIsMember --
- *      
+ *
 *      Return TRUE iff the given Vcpuid is present in a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -273,7 +290,7 @@ VCPUSet_AtomicIsMember(VCPUSet *vcs, Vcpuid v)
 *
 * VCPUSet_FindFirst --
 * VCPUSet_FindLast --
- *      
+ *
 *      Find the first (lowest-numbered) or last (highest-numbered)
 *      Vcpuid in a VCPUSet.
 *
@@ -309,12 +326,56 @@ VCPUSet_FindLast(const VCPUSet *vcs)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * VCPUSet_FindFirstInSubset --
+ *
+ *      Find the first (lowest-numbered) Vcpuid in the given subset of a
+ *      VCPUSet object.  It is required that *subset is initialized to
+ *      a subset of *vcs and *subsetIdx must hold the index of the subset
+ *      stored in *subset.
+ *
+ * Results:
+ *      Vcpuid if at least one is present in a set.
+ *      VCPUID_INVALID if the set is empty.
+ *
+ * Side effects:
+ *      This function is intended to be used for iterating over all bits
+ *      of a VCPUSet, it will modify *subset to clear the bit associated
+ *      with the returned Vcpuid (if any).  The *subsetIdx argument may
+ *      also be updated.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Vcpuid
+VCPUSet_FindFirstInSubset(const VCPUSet *vcs, uint64 *subset,
+                          unsigned *subsetIdx, unsigned maxSubsets)
+{
+   ASSERT(*subsetIdx < maxSubsets && maxSubsets <= VCS_SUBSET_COUNT);
+   do {
+      if (*subset != 0) {
+         int bit;
+         bit = lssb64_0(*subset);
+         *subset &= ~(CONST64U(1) << bit);
+         return bit + (*subsetIdx << VCS_SUBSET_SHIFT);
+      }
+      ++*subsetIdx;
+      if (*subsetIdx < maxSubsets) {
+         *subset = vcs->subset[*subsetIdx];
+      }
+   } while (*subsetIdx < maxSubsets);
+   return VCPUID_INVALID;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
 * VCPUSet_Remove --
 * VCPUSet_AtomicRemove --
- *      
+ *
 *      Remove or atomically remove a single Vcpuid from a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -371,7 +432,7 @@ VCPUSet_RemoveSet(VCPUSet *dest, const VCPUSet *src)
 *
 * VCPUSet_Include --
 * VCPUSet_AtomicInclude --
- *      
+ *
 *      Add or atomically add a single Vcpuid to a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -394,12 +455,33 @@ VCPUSet_AtomicInclude(VCPUSet *vcs, Vcpuid v)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * VCPUSet_AtomicTestInclude  --
+ *
+ *      Atomically add a single Vcpuid to a VCPUSet, and
+ *      return TRUE iff the given Vcpuid was present in the VCPUSet.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Bool
+VCPUSet_AtomicTestInclude(VCPUSet *vcs, Vcpuid v)
+{
+   volatile uint64 *subset = &vcs->subset[VCS_VCPUID_SUBSET_IDX(v)];
+   ASSERT(v < MAX_VCPUS);
+   return Atomic_TestSetBit64(Atomic_VolatileToAtomic64(subset),
+                              v & VCS_SUBSET_MASK);
+}
+
+
 #if defined(VMM) && !defined(MONITOR_APP)
 /*
 *----------------------------------------------------------------------
 *
 * VCPUSet_PackCareful --
- *      
+ *
 *      Pack a VCPUSet into the bytes at "ptr".
 *
 *----------------------------------------------------------------------
@@ -416,7 +498,7 @@ VCPUSet_PackCareful(unsigned numVCPUs, const VCPUSet *vcs, void *ptr)
 *----------------------------------------------------------------------
 *
 * VCPUSet_UnpackCareful --
- *      
+ *
 *      Unpack a VCPUSet from the bytes at "ptr".
 *
 *----------------------------------------------------------------------
@@ -483,7 +565,7 @@ VCPUSet_Populate(VCPUSet *vcs, unsigned numVCPUs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Subset --
- *      
+ *
 *      Return the specified subset of a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -502,7 +584,7 @@ VCPUSet_Subset(const VCPUSet *vcs,
 *----------------------------------------------------------------------
 *
 * VCPUSet_SubsetPtr --
- *      
+ *
 *      Return a pointer to the specified subset of a VCPUSet.
 *
 *----------------------------------------------------------------------
@@ -520,7 +602,7 @@ VCPUSet_SubsetPtr(VCPUSet *vcs, unsigned subset)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsSupersetOrEqual --
- *      
+ *
 *      Return TRUE iff vcs1 contains a superset of the VCPUs in vcs2
 *      or vcs1 and vcs2 contain exactly the same VCPUs.
 *
@@ -543,7 +625,7 @@ VCPUSet_IsSupersetOrEqual(const VCPUSet *vcs1, const VCPUSet *vcs2)
 *----------------------------------------------------------------------
 *
 * VCPUSet_IsSubsetOrEqual --
- *      
+ *
 *      Return TRUE iff vcs1 contains a subset of the VCPUs in vcs2
 *      or vcs1 and vcs2 contain exactly the same VCPUs.
 *
@@ -561,7 +643,7 @@ VCPUSet_IsSubsetOrEqual(const VCPUSet *vcs1, const VCPUSet *vcs2)
 *----------------------------------------------------------------------
 *
 * VCPUSet_MakeSingleton --
- *      
+ *
 *      Add a single Vcpuid to a VCPUSet and remove all others.
 *
 *----------------------------------------------------------------------
@@ -578,27 +660,31 @@ VCPUSet_MakeSingleton(VCPUSet *vcs, Vcpuid v)
 /*
 *----------------------------------------------------------------------
 *
- * VCPUSet_IsSingleton --
- *      
- *      Return TRUE iff vcs contains exactly one VCPU.
+ * VCPUSet_FindSingleton --
+ *
+ *      Return the VCPU in the set iff vcs contains exactly one VCPU.
+ *      Return VCPUID_INVALID otherwise.
 *
 *----------------------------------------------------------------------
 */

-static INLINE Bool
-VCPUSet_IsSingleton(const VCPUSet *vcs)
+static INLINE Vcpuid
+VCPUSet_FindSingleton(const VCPUSet *vcs)
 {
-   Bool foundOnce = FALSE;
+   uint64 foundSub = 0;
+   uint32 foundIdx;
   FOR_EACH_SUBSET_IN_SET(idx) {
      uint64 sub = vcs->subset[idx];
      if (sub != 0) {
-         if (foundOnce || (sub & (sub - 1)) != 0) {
-            return FALSE;
+         if (foundSub != 0 || (sub & (sub - 1)) != 0) {
+            return VCPUID_INVALID;
         }
-         foundOnce = TRUE;
+         foundSub = sub;
+         foundIdx = idx;
      }
   } ROF_EACH_SUBSET_IN_SET();
-   return foundOnce;
+   return foundSub != 0 ? lssb64_0(foundSub) + (foundIdx << VCS_SUBSET_SHIFT) :
+                          VCPUID_INVALID;
 }


@@ -622,7 +708,7 @@ VCPUSet_IsFull(const VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_AtomicReadWriteSubset --
- *      
+ *
 *      For the nth aligned 64-VCPU subset of a VCPU set, atomically
 *      read then write.  Return the contents read.  Set 0 is VCPUs
 *      0-63 and set 1 is VCPUs 64-127.
@@ -667,7 +753,7 @@ VCPUSet_Size(const VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_UnionSubset --
- *      
+ *
 *      Given an 64-bit value and a subset number, add the VCPUs
 *      represented to the set.
 *
@@ -686,7 +772,7 @@ VCPUSet_UnionSubset(VCPUSet *vcs, uint64 vcpus, unsigned n)
 *----------------------------------------------------------------------
 *
 * VCPUSet_SubtractSubset --
- *      
+ *
 *      Given an 64-bit value and a subset number, remove the VCPUs
 *      represented in the subset from the set.
 *
@@ -705,7 +791,7 @@ VCPUSet_SubtractSubset(VCPUSet *vcs, uint64 vcpus, unsigned n)
 *----------------------------------------------------------------------
 *
 * VCPUSet_AtomicUnionSubset --
- *      
+ *
 *      Given a 64-bit value and a subset number, atomically add
 *      the VCPUs represented to the set.
 *
@@ -725,7 +811,7 @@ VCPUSet_AtomicUnionSubset(VCPUSet *vcs, uint64 vcpus, unsigned n)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Invert --
- *      
+ *
 *      Makes all non-present valid VCPUs in a set present and all
 *      VCPUs present non-present.
 *
@@ -746,7 +832,7 @@ VCPUSet_Invert(VCPUSet *vcs)
 *----------------------------------------------------------------------
 *
 * VCPUSet_Intersection
- *      
+ *
 *      Given two VCPUSets, populate the destination set with only the
 *      VCPUs common to both.
 *
@@ -766,7 +852,7 @@ VCPUSet_Intersection(VCPUSet *dest, const VCPUSet *src)
 *----------------------------------------------------------------------
 *
 * VCPUSet_LogFormat --
- *      
+ *
 *      Given a buffer of at least VCS_BUF_SIZE to fill, write into it a
 *      string suitable for use in Log() or LOG().
 *      Returns the buffer which was passed as an argument, after
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2013, 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2013, 2016-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -41,7 +41,7 @@
 #define VCS_SUBSET_WIDTH                                                   64
 #define VCS_SUBSET_SHIFT                                                    6
 #define VCS_SUBSET_MASK               ((CONST64U(1) << VCS_SUBSET_SHIFT) - 1)
-#define VCS_SUBSET_COUNT                                                    4
+#define VCS_SUBSET_COUNT                                                   32


 #define VCS_VCPUID_SUBSET_IDX(v)                    ((v) >> VCS_SUBSET_SHIFT)
@@ -56,11 +56,12 @@ typedef struct VCPUSet {
 } VCPUSet;

 MY_ASSERTS(VCPUSET_ASSERTS,
+           ASSERT_ON_COMPILE(VCS_SUBSET_WIDTH * VCS_SUBSET_COUNT >= MAX_VCPUS);
           /*
            * Catch changes in VCPUSet which need to be reflected in
            * bora/public/iocontrolsMacosTable.h.
            */
-           ASSERT_ON_COMPILE(VCS_SUBSET_COUNT == 4);
+           ASSERT_ON_COMPILE(VCS_SUBSET_COUNT == 32);
           /*
            * There is code that depends on sizeof(VCPUSet) being a power of
            * 2 in at least vcpuHotPlug.c and possible other places.
@@ -144,9 +144,6 @@ VersionedAtomic_BeginWrite(VersionedAtomic *versions) // IN
   /*
    * The write to 'versions->v0' must be observed by all other OS CPUs before
    * the write(s) to the shared data.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine.
    */
   SMP_W_BARRIER_W();
 }
@@ -176,9 +173,6 @@ VersionedAtomic_EndWrite(VersionedAtomic *versions) // IN
   /*
    * The write(s) to the shared data must be observed by all other OS CPUs
    * before the write to 'versions->v1'.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine.
    */
   SMP_W_BARRIER_W();

@@ -223,10 +217,6 @@ VersionedAtomic_BeginTryRead(VersionedAtomic const *versions) // IN
   /*
    * The read from 'versions->v1' must happen on this CPU before the read(s)
    * from the shared data.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine, and maybe
-    * Non-shareable (== current CPU) would work too.
    */
   SMP_R_BARRIER_R();

@@ -262,10 +252,6 @@ VersionedAtomic_EndTryRead(VersionedAtomic const *versions, // IN
   /*
    * The read(s) from the shared data must happen on this CPU before the read
    * from 'versions->v0'.
-    *
-    * On arm64, this barrier is Full System, which is probably too strong:
-    * Inner Shareable (== all OS CPUs) should work just fine, and maybe
-    * Non-shareable (== current CPU) would work too.
    */
   SMP_R_BARRIER_R();

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014,2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,7 +66,7 @@
 *                    ((__builtin_constant_p(expr) ? ((expr) >> 16) == 0) \
 *                                                 : sizeof(expr) <= 2)
 */
-#if (__GNUC__ >= 4) && (__GNUC_MINOR__ >= 1)
+#if (__GNUC__ >= 4) && (__GNUC_MINOR__ >= 1) && !defined(USE_UBSAN)
 #define ASSERT_ON_COMPILE_SELECTOR_SIZE(expr)                                \
   ASSERT_ON_COMPILE(sizeof(Selector) == 2 &&                                \
                     ((__builtin_constant_p(expr) && ((expr) >> 16) == 0) || \
@@ -67,7 +67,6 @@ extern "C" {
 #if defined (VMKPANIC) 
 #include "vmk_assert.h"
 #else /* !VMKPANIC */
-#include <linux/kernel.h>
 #define _ASSERT_PANIC(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
 #define _ASSERT_PANIC_BUG(bug, name) \
@@ -108,7 +107,7 @@ NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
 } while(0)

 #else
-#define Panic panic
+NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
 #endif

 void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,38 +49,68 @@ extern "C" {
 #endif

 /*
- * In the Atomic_* definitions below, memory ordering and atomicity are somewhat
- * conflated in an inconsistent manner. First, we have Atomic_{Read,Write},
- * which only guarantees single copy atomicity, i.e. that the read/write occurs
- * in an atomic fashion, but have no implication on memory ordering. The second
- * class of Atomics are all the non-unfenced operations excluding
- * Atomic_{Read,Write}*, which both imply atomicity and act as a memory barrier,
- * implying sequentially consistent ordering of the atomic operation with all
- * loads/stores prior to and after it.
+ * There are two concepts involved when dealing with atomic accesses:
+ * 1. Atomicity of the access itself
+ * 2. Ordering of the access with respect to other reads&writes (from the view
+ *    of other processors/devices).
 *
- * Since on x86, the second class of operations are associated with LOCK
- * semantics, assumptions have been made about the ordering these operations
- * imply on surrounding code (see for example the vmkernel's RefCount
- * implementation). As a result, on arm64 we have to provide these same
- * guarantees. We do this by making use of DMB barriers both before and after
- * the atomic ldrx/strx sequences. A barrier before and after is required to
- * avoid having part of the atomic operation reordered with surrounding code,
- * e.g. a store-load reordering of the strx with a following load outside the
- * Atomic_ op. For the first class of operations, Atomic_{Read,Write}, we do not
- * implement a barrier.
+ * Two examples help to clarify #2:
+ * a. Inc: A caller implementing a simple independent global event counter
+ *         might not care if the compiler or processor visibly reorders the
+ *         increment around other memory accesses.
+ * b. Dec: A caller implementing a reference count absolutely *doesn't* want
+ *         the compiler or processor to visibly reordering writes after that
+ *         decrement: if that happened, the program could then end up writing
+ *         to memory that was freed by another processor.
 *
- * This implementation of Atomic operations is suboptimal on arm64, since
- * both atomicity and memory ordering are fused together. Ideally the Atomic
- * operations would only imply atomicity, and an explicit memory barrier in the
- * surrounding code used to enforce ordering where necessary. This would eschew
- * the need for the DMBs. A middle ground can be implemented where we use the
- * arm64 load-acquire/store-release exclusive instructions to implement Atomics.
- * This would imply sequential consistency of the Atomic operations (but not
- * with any of the surrounding non-atomic operations) without the need for a
- * DMB. Using these without a DMB today can still result in problematic
- * reordering by the processor with surrounding non-atomic operations, e.g. a
- * store-load reordering with a stlxr. Future optimization for arm64 should
- * consider the wider change required at the call sites to minimize DMBs.
+ * C11 has standardized a good model for expressing these orderings when doing
+ * atomics. It defines three *tiers* of ordering:
+ * 1. Sequential Consistency (every processor sees the same total order of
+ *    events)
+ *
+ * 2. Acquire/Release ordering (roughly, everybody can agree previous events
+ *    have completed, but they might disagree on the ordering of previous
+ *    independent events).
+ *
+ *    The relative ordering provided by this tier is sufficient for common
+ *    locking and initialization activities, but is insufficient for unusual
+ *    synchronization schemes (e.g. IRIW aka Independent Read Independent
+ *    Write designs such Dekker's algorithm, Peterson's algorithm, etc.)
+ *
+ *    In other words, this tier is close in behavior to Sequential Consistency
+ *    in much the same way a General-Relativity universe is close to a
+ *    Newtonian universe.
+ * 3. Relaxed (i.e unordered/unfenced)
+ *
+ * In C11 standard's terminology for atomic memory ordering,
+ * - in case (a) we want "relaxed" ordering for perf and,
+ * - in case (b) we want "sequentially consistent" ordering (or perhaps the
+ *   only slightly weaker "release" ordering) for correctness.
+ *
+ * There are standardized mappings of operations to orderings for every
+ * processor architecture. See
+ * - https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
+ * - http://preshing.com/20120913/acquire-and-release-semantics/
+ *
+ * In this file:
+ * 1. all RMW (Read/Modify/Write) operations are sequentially consistent.
+ *    This includes operations like Atomic_IncN, Atomic_ReadIfEqualWriteN,
+ *    Atomic_ReadWriteN, etc.
+ * 2. all R and W operations are relaxed. This includes operations like
+ *    Atomic_WriteN, Atomic_ReadN, Atomic_TestBitN, etc.
+ *
+ * The below routines of course ensure both the CPU and compiler honor the
+ * ordering constraint.
+ *
+ * Notes:
+ * 1. Since R-only and W-only operations do not provide ordering, callers
+ *    using them for synchronizing operations like double-checked
+ *    initialization or releasing spinlocks must provide extra barriers.
+ * 2. This implementation of Atomic operations is suboptimal. On x86,simple
+ *    reads and writes have acquire/release semantics at the hardware level.
+ *    On arm64, we have separate instructions for sequentially consistent
+ *    reads and writes (the same instructions are used for acquire/release).
+ *    Neither of these are exposed for R-only or W-only callers.
 *
 * For further details on x86 and ARM memory ordering see
 * https://wiki.eng.vmware.com/ARM/MemoryOrdering.
@@ -118,10 +148,7 @@ typedef struct Atomic_uint128 {
 /*
 * Prototypes for msft atomics.  These are defined & inlined by the
 * compiler so no function definition is needed.  The prototypes are
- * needed for C++.  Since amd64 compiler doesn't support inline asm we
- * have to use these.  Unfortunately, we still have to use some inline asm
- * for the 32 bit code since the and/or/xor implementations didn't show up
- * until XP or 2k3.
+ * needed for C++.
 *
 * The declarations for the intrinsic functions were taken from ntddk.h
 * in the DDK. The declarations must match otherwise the 64-bit C++
@@ -130,7 +157,7 @@ typedef struct Atomic_uint128 {
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
 */
-#if defined _MSC_VER && _MSC_VER >= 1310 && !defined BORA_NO_WIN32_INTRINS
+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -222,38 +249,10 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:


 /*
- * All the assembly code is tricky and written conservatively.
- * For example, to make sure gcc won't introduce copies,
- * we force the addressing mode like this:
- *
- *    "xchgl %0, (%1)"
- *    : "=r" (val)
- *    : "r" (&var->value),
- *      "0" (val)
- *    : "memory"
- *
- * - edward
- *
- * Actually - turns out that gcc never generates memory aliases (it
- * still does generate register aliases though), so we can be a bit
- * more agressive with the memory constraints. The code above can be
- * modified like this:
- *
- *    "xchgl %0, %1"
- *    : "=r" (val),
- *      "=m" (var->value),
- *    : "0" (val),
- *      "1" (var->value)
- *
- * The advantages are that gcc can use whatever addressing mode it
- * likes to access the memory value, and that we dont have to use a
- * way-too-generic "memory" clobber as there is now an explicit
- * declaration that var->value is modified.
- *
- * see also /usr/include/asm/atomic.h to convince yourself this is a
- * valid optimization.
- *
- * - walken
+ * The Read/Modify/Write operations on x86/x64 are all written using the
+ * "memory" constraint. This is to ensure the compiler treats the operation as
+ * a full barrier, flushing any pending/cached state currently residing in
+ * registers.
 */

 #if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
@@ -403,6 +402,7 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "=q" (val),
        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER && _MSC_VER >= 1600
@@ -496,7 +496,7 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
        "+m" (var->value)
      : "q" (newVal),
        "0" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );

   return val;
@@ -1002,26 +1002,15 @@ Atomic_ReadWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchange((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint32.value, eax
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif // _MSC_VER >= 1310
-#else
 #error No compiler defined for Atomic_ReadWrite
 #endif // __GNUC__
 }
@@ -1162,30 +1151,17 @@ Atomic_ReadIfEqualWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgl %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedCompareExchange((long *)&var->value,
-				      (long)newVal,
-				      (long)oldVal);
-#else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, oldVal
-      __asm mov ebx, var
-      __asm mov ecx, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint32.value, ecx
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif
+                                      (long)newVal,
+                                      (long)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite
 #endif
@@ -1267,17 +1243,17 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgq %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif //VM_ARM_V7
 #elif defined _MSC_VER
   return _InterlockedCompareExchange64((__int64 *)&var->value,
-					(__int64)newVal,
-					(__int64)oldVal);
+                                        (__int64)newVal,
+                                        (__int64)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite64
 #endif
@@ -1332,7 +1308,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
      "lock; andl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1397,7 +1373,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
      "lock; orl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1462,7 +1438,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
      "lock; xorl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1510,7 +1486,7 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT
      "lock; xorq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -1569,17 +1545,11 @@ Atomic_Add32(Atomic_uint32 *var, // IN/OUT
      "lock; addl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedExchangeAdd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock add [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Add
 #endif
@@ -1634,17 +1604,11 @@ Atomic_Sub32(Atomic_uint32 *var, // IN/OUT
      "lock; subl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
-   _InterlockedExchangeAdd((long *)&var->value, (long)-val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock sub [ebx]Atomic_uint32.value, eax
-#endif
+   _InterlockedExchangeAdd((long *)&var->value, -(long)val);
 #else
 #error No compiler defined for Atomic_Sub
 #endif
@@ -1680,16 +1644,11 @@ Atomic_Inc32(Atomic_uint32 *var) // IN/OUT
      "lock; incl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedIncrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock inc [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Inc
 #endif
@@ -1725,16 +1684,11 @@ Atomic_Dec32(Atomic_uint32 *var) // IN/OUT
      "lock; decl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedDecrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock dec [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Dec
 #endif
@@ -1939,26 +1893,15 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchangeAdd((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm lock xadd [ebx]Atomic_uint32.value, eax
-   }
-#pragma warning(pop)
-#endif
-#else
 #error No compiler defined for Atomic_ReadAdd32
 #endif
 }
@@ -2044,11 +1987,11 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchgq %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
 #else /* 32-bit version for non-ARM */
   typedef struct {
@@ -2088,9 +2031,9 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b (%3)" "\n\t"
      "xchgl %%ebx, %6"      "\n\t"
      "sete %0"
-      :	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+      : "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : /*
         * See the "Rules for __asm__ statements in __PIC__ code" above: %3
         * must use a register class which does not contain %ebx.
@@ -2111,14 +2054,14 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+        "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : "2" (((S_uint64 *)&oldVal)->lowValue),
        "3" (((S_uint64 *)&oldVal)->highValue),
        "b" (((S_uint64 *)&newVal)->lowValue),
        "c" (((S_uint64 *)&newVal)->highValue)
-      : "cc"
+      : "cc", "memory"
   );
 #   endif
 #endif
@@ -2166,11 +2109,11 @@ Atomic_CMPXCHG32(Atomic_uint32 *var,   // IN/OUT
      "lock; cmpxchgl %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
   return equal;
 #endif /* VM_X86_ANY */
@@ -2247,7 +2190,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
 #   pragma warning(push)
-#   pragma warning(disable : 4035)		// disable no-return warning
+#   pragma warning(disable : 4035)      // disable no-return warning
   {
      __asm mov ecx, var
      __asm mov edx, ecx
@@ -2330,9 +2273,9 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2378,7 +2321,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -val);
+   return Atomic_ReadAdd64(var, -(int64)val);
 #endif
 }

@@ -2425,7 +2368,7 @@ Atomic_ReadInc64(Atomic_uint64 *var) // IN/OUT
 static INLINE uint64
 Atomic_ReadDec64(Atomic_uint64 *var) // IN/OUT
 {
-   return Atomic_ReadAdd64(var, CONST64U(-1));
+   return Atomic_ReadAdd64(var, (uint64)CONST64(-1));
 }


@@ -2460,7 +2403,7 @@ Atomic_Add64(Atomic_uint64 *var, // IN/OUT
      "lock; addq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2502,7 +2445,7 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT
      "lock; subq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2542,7 +2485,7 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT
      "lock; incq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedIncrement64((__int64 *)&var->value);
@@ -2581,7 +2524,7 @@ Atomic_Dec64(Atomic_uint64 *var) // IN/OUT
      "lock; decq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedDecrement64((__int64 *)&var->value);
@@ -2617,8 +2560,9 @@ Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2726,7 +2670,7 @@ Atomic_Or64(Atomic_uint64 *var, // IN/OUT
      "lock; orq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedOr64((__int64 *)&var->value, (__int64)val);
@@ -2773,7 +2717,7 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT
      "lock; andq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedAnd64((__int64 *)&var->value, (__int64)val);
@@ -2811,29 +2755,17 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT
-                uint64 bit)         // IN
+                unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; bts %1, %0"
+      "lock; btsq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal | (CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_SetBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2863,29 +2795,17 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT
-                  uint64 bit)         // IN
+                  unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; btr %1, %0"
+      "lock; btrq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal & ~(CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_ClearBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2915,29 +2835,65 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE Bool
 Atomic_TestBit64(Atomic_uint64 *var, // IN
-                 uint64 bit)         // IN
+                 unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
-   Bool out = FALSE;
+   Bool out;
   ASSERT(bit <= 63);
+#if defined __x86_64__ && defined __GNUC__
   __asm__ __volatile__(
-      "bt %2, %1; setc %0"
+      "btq %2, %1; setc %0"
      : "=rm"(out)
      : "m" (var->value),
-        "rJ" (bit)
+        "rJ" ((uint64)bit)
      : "cc"
   );
-   return out;
-#elif defined _MSC_VER
-   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
 #else
-#error No compiler defined for Atomic_TestBit64
+   out = (var->value & (CONST64U(1) << bit)) != 0;
 #endif
-#else // __x86_64__
+   return out;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBit64 --
+ *
+ *      Atomically test and set the bit 'bit' in var.
+ *      Bit must be between 0 and 63.
+ *
+ * Results:
+ *      TRUE if the tested bit was set; else FALSE.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBit64(Atomic_uint64 *var, // IN/OUT
+                    unsigned bit)       // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool out;
   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
+   __asm__ __volatile__(
+      "lock; btsq %2, %1; setc %0"
+      : "=rm" (out), "+m" (var->value)
+      : "rJ" ((uint64)bit)
+      : "cc", "memory"
+   );
+   return out;
+#else
+   uint64 oldVal;
+   uint64 mask;
+   ASSERT(bit <= 63);
+   mask = CONST64U(1) << bit;
+   do {
+      oldVal = var->value;
+   } while (!Atomic_CMPXCHG64(var, oldVal, oldVal | mask));
+   return (oldVal & mask) != 0;
 #endif
 }

@@ -3020,8 +2976,9 @@ Atomic_ReadWrite16(Atomic_uint16 *var,  // IN/OUT:
   __asm__ __volatile__(
      "xchgw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3128,10 +3085,10 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var,   // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgw %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3192,7 +3149,7 @@ Atomic_And16(Atomic_uint16 *var, // IN/OUT
      "lock; andw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3249,7 +3206,7 @@ Atomic_Or16(Atomic_uint16 *var, // IN/OUT
      "lock; orw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3306,7 +3263,7 @@ Atomic_Xor16(Atomic_uint16 *var, // IN/OUT
      "lock; xorw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3363,7 +3320,7 @@ Atomic_Add16(Atomic_uint16 *var, // IN/OUT
      "lock; addw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3420,7 +3377,7 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT
      "lock; subw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3476,7 +3433,7 @@ Atomic_Inc16(Atomic_uint16 *var) // IN/OUT
      "lock; incw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Add16(var, 1);
@@ -3512,7 +3469,7 @@ Atomic_Dec16(Atomic_uint16 *var) // IN/OUT
      "lock; decw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Sub16(var, 1);
@@ -3582,9 +3539,9 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
   __asm__ __volatile__(
      "lock; xaddw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3715,7 +3672,7 @@ Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
                            in val)                                           \
   {                                                                          \
      return (out)(cast)Atomic_ReadWrite ## size(var,                         \
-		(uint ## size)(cast)val);                                     \
+                (uint ## size)(cast)val);                                     \
   }                                                                          \
                                                                              \
                                                                              \
@@ -3876,6 +3833,109 @@ MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t)
 MAKE_ATOMIC_TYPE(Int, 32, int, int, int)
 MAKE_ATOMIC_TYPE(Bool, 8, Bool, Bool, Bool)

+/*
+ * Define arbitrary sized bit vector to be used by
+ * Atomic_TestSetBitVector and Atomic_TestClearBitVector.
+ */
+#define ATOMIC_BITVECTOR(varName, capacity) \
+      Atomic_uint8 varName[CEILING(capacity, 8)]
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBitVector --
+ *
+ *      Atomically test and set the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBitVector(Atomic_uint8 *var, // IN/OUT
+                        unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; bts %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadOr8(var + index / 8, bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestClearBitVector --
+ *
+ *      Atomically test and clear the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestClearBitVector(Atomic_uint8 *var, // IN/OUT
+                          unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; btr %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadAnd8(var + index / 8, ~bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestBitVector --
+ *
+ *      Test the bit 'index' (zero-based) in bit vector var.
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestBitVector(const Atomic_uint8 *var, // IN
+                     unsigned index)          // IN
+{
+   uint8 bit = 1 << index % 8;
+   return (Atomic_Read8(var + index / 8) & bit) != 0;
+}
+
+
 #ifdef VM_ARM_64
 #   include "vm_atomic_arm64_end.h"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1056,7 +1056,7 @@ ClearBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1078,7 +1078,7 @@ ComplementBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -374,8 +374,8 @@ RDTSC_BARRIER(void)
 * Thanks for pasting this whole comment into every architecture header.
 *
 * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed. The
- * ST_LD barrier is implemented with a locked xor operation (instead of the
+ * normal memory types. In other cases, only a compiler barrier is needed.
+ * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
 * mfence instruction) for performance reasons. See PR 1674199 for more
 * details.
 *
@@ -389,19 +389,17 @@ SMP_W_BARRIER_R(void)
 {
   volatile long temp;

-   COMPILER_MEM_BARRIER();
 #if defined __GNUC__
   __asm__ __volatile__ (
-      "lock xorl $1, %0\n"
+      "lock xorl $1, %0"
      : "+m" (temp)
      : /* no additional inputs */
-      : "cc");
+      : "cc", "memory");
 #elif defined _MSC_VER
   _InterlockedXor(&temp, 1);
 #else
 #error SMP_W_BARRIER_R not defined for this compiler
 #endif
-   COMPILER_MEM_BARRIER();
 }

 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,12 +37,16 @@
 #include "vm_basic_types.h" // For INLINE.

 /* Checks for FreeBSD, filtering out VMKERNEL. */
-#define __IS_FREEBSD__ (!defined(VMKERNEL) && defined(__FreeBSD__))
+#if !defined(VMKERNEL) && defined(__FreeBSD__)
+#define __IS_FREEBSD__ 1
+#else
+#define __IS_FREEBSD__ 0
+#endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

 #if defined _WIN32 && defined USERLEVEL
   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make 
+                         * We redefine offsetof macro from stddef; make
                         * sure that it's already defined before we do that.
                         */
   #include <windows.h>	// for Sleep() and LOWORD() etc.
@@ -101,7 +105,7 @@

 /* The Solaris 9 cross-compiler complains about these not being used */
 #ifndef sun
-static INLINE int 
+static INLINE int
 Min(int a, int b)
 {
   return a < b ? a : b;
@@ -113,7 +117,7 @@ Min(int a, int b)
 #endif

 #ifndef sun
-static INLINE int 
+static INLINE int
 Max(int a, int b)
 {
   return a > b ? a : b;
@@ -168,7 +172,7 @@ Max(int a, int b)
 #endif


-/* 
+/*
 * Token concatenation
 *
 * The C preprocessor doesn't prescan arguments when they are
@@ -256,15 +260,26 @@ Max(int a, int b)
 #endif

 #ifndef MBYTES_2_PAGES
-#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
+#define MBYTES_2_PAGES(_nbytes) \
+   ((uint64)(_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
 #endif

 #ifndef PAGES_2_MBYTES
 #define PAGES_2_MBYTES(_npages) ((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
 #endif

+#ifndef ROUNDUP_PAGES_2_MBYTES
+#define ROUNDUP_PAGES_2_MBYTES(_npages) \
+(((_npages) + MASK(MBYTES_SHIFT - PAGE_SHIFT)) >> (MBYTES_SHIFT - PAGE_SHIFT))
+#endif
+
+#ifndef ROUNDDOWN_PAGES_2_MBYTES
+#define ROUNDDOWN_PAGES_2_MBYTES(_npages) \
+((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
+#endif
+
 #ifndef GBYTES_2_PAGES
-#define GBYTES_2_PAGES(_nbytes) ((_nbytes) << (30 - PAGE_SHIFT))
+#define GBYTES_2_PAGES(_nbytes) ((uint64)(_nbytes) << (30 - PAGE_SHIFT))
 #endif

 #ifndef PAGES_2_GBYTES
@@ -369,7 +384,7 @@ Max(int a, int b)
 #ifdef _MSC_VER
 #ifdef __cplusplus
 extern "C"
-#endif 
+#endif
 void *_ReturnAddress(void);
 #pragma intrinsic(_ReturnAddress)
 #define GetReturnAddress() _ReturnAddress()
@@ -412,22 +427,6 @@ void *_ReturnAddress(void);

 #ifdef USERLEVEL // {

-/*
- * Note this might be a problem on NT b/c while sched_yield guarantees it
- * moves you to the end of your priority list, Sleep(0) offers no such
- * guarantee.  Bummer.  --Jeremy.
- */
-
-#if defined(_WIN32)
-#      define YIELD()		Sleep(0)
-#elif defined(VMKERNEL)
-/* We don't have a YIELD macro in the vmkernel */
-#else
-#      include <sched.h>        // For sched_yield.  Don't ask.  --Jeremy.
-#      define YIELD()		sched_yield()
-#endif 
-
-
 /*
 * Standardize some Posix names on Windows.
 */
@@ -497,19 +496,8 @@ typedef int pid_t;

 #elif defined(__APPLE__) && defined(KERNEL)

-#include "availabilityMacOS.h"
-
-#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
-// The Mac OS 10.5 kernel SDK defines va_copy in stdarg.h.
+// The macOS kernel SDK defines va_copy in stdarg.h.
 #include <stdarg.h>
-#else
-/*
- * The Mac OS 10.4 kernel SDK needs va_copy. Based on inspection of
- * stdarg.h from the MacOSX10.4u.sdk kernel framework, this should
- * work.
- */
-#define va_copy(dest, src) ((dest) = (src))
-#endif // MAC_OS_X_VERSION_MIN_REQUIRED

 #elif defined(__GNUC__) && (__GNUC__ < 3)

@@ -529,7 +517,7 @@ typedef int pid_t;
 * driver.
 */

-#ifdef _WIN32
+#if defined(_WIN32) && defined(_MSC_VER)
 #define PATH_MAX 256
 #ifndef strcasecmp
 #define strcasecmp(_s1,_s2)   _stricmp((_s1),(_s2))
@@ -539,29 +527,6 @@ typedef int pid_t;
 #endif
 #endif

-#if defined __linux__ && !defined __KERNEL__ && !defined MODULE && \
-                         !defined VMM && !defined FROBOS && !defined __ANDROID__
-#include <features.h>
-#if __GLIBC_PREREQ(2, 1) && !defined GLIBC_VERSION_21
-#define GLIBC_VERSION_21
-#endif
-#if __GLIBC_PREREQ(2, 2) && !defined GLIBC_VERSION_22
-#define GLIBC_VERSION_22
-#endif
-#if __GLIBC_PREREQ(2, 3) && !defined GLIBC_VERSION_23
-#define GLIBC_VERSION_23
-#endif
-#if __GLIBC_PREREQ(2, 4) && !defined GLIBC_VERSION_24
-#define GLIBC_VERSION_24
-#endif
-#if __GLIBC_PREREQ(2, 5) && !defined GLIBC_VERSION_25
-#define GLIBC_VERSION_25
-#endif
-#if __GLIBC_PREREQ(2, 12) && !defined GLIBC_VERSION_212
-#define GLIBC_VERSION_212
-#endif
-#endif
-
 /*
 * Convenience definitions of unicode characters.
 */
@@ -634,7 +599,7 @@ typedef int pid_t;
 #define RELEASE_ONLY(x) x
 #else
 #define vmx86_release   0
-#define RELEASE_ONLY(x) 
+#define RELEASE_ONLY(x)
 #endif

 #ifdef VMX86_SERVER
@@ -687,6 +652,12 @@ typedef int pid_t;
 #define vmw_apple_sandbox 0
 #endif

+#if defined(__APPLE__) && defined(VMW_APPLE_APP_STORE)
+#define vmw_apple_app_store 1
+#else
+#define vmw_apple_app_store 0
+#endif
+
 #ifdef VMM
 #define VMM_ONLY(x) x
 #else
@@ -699,17 +670,6 @@ typedef int pid_t;
 #define USER_ONLY(x) x
 #endif

-/* VMVISOR ifdef only allowed in the vmkernel */
-#ifdef VMKERNEL
-#ifdef VMVISOR
-#define vmvisor 1
-#define VMVISOR_ONLY(x) x
-#else
-#define vmvisor 0
-#define VMVISOR_ONLY(x)
-#endif
-#endif
-
 #ifdef _WIN32
 #define VMW_INVALID_HANDLE INVALID_HANDLE_VALUE
 #else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,24 +38,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-/* STRICT ANSI means the Xserver build and X defines Bool differently. */
-#if !defined(_XTYPEDEF_BOOL) && \
-    (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || \
-      defined(__MINGW32__) || defined(__APPLE__))
-#define _XTYPEDEF_BOOL
-typedef char           Bool;
-#endif
-
-#ifndef FALSE
-#define FALSE          0
-#endif
-
-#ifndef TRUE
-#define TRUE           1
-#endif
-
-#define IS_BOOL(x)     (((x) & ~1) == 0)
-
 /*
 * Macros __i386__ and __ia64 are intrinsically defined by GCC
 */
@@ -87,7 +69,8 @@ typedef char           Bool;
 */

 #ifdef __i386__
-/* VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
+/*
+ * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
 * since it is confused with the __i386__ gcc but defined for both 32- and
 * 64-bit x86. We retain it here for legacy compatibility.
 */
@@ -99,12 +82,12 @@ typedef char           Bool;

 #ifdef __x86_64__
 #define VM_X86_64
+#define vm_x86_64 1
 #define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
-#define vm_x86_64 (1)
 #else
-#define vm_x86_64 (0)
+#define vm_x86_64 0
 #endif

 #ifdef __arm__
@@ -115,13 +98,15 @@ typedef char           Bool;

 #ifdef __aarch64__
 #define VM_ARM_64
+#define vm_arm_64 1
 #define VM_ARM_ANY
 #define VM_64BIT
-#define vm_arm_64 (1)
 #else
-#define vm_arm_64 (0)
+#define vm_arm_64 0
 #endif

+#define vm_64bit (sizeof (void *) == 8)
+
 #ifdef _MSC_VER

 #pragma warning (3 :4505) // unreferenced local function
@@ -135,17 +120,15 @@ typedef char           Bool;

 #endif

-#if defined(__linux__) && defined(__cplusplus) && __cplusplus >= 201103L
-
+#if defined(__cplusplus) && __cplusplus >= 201103L || \
+    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
+    defined(__APPLE__) || defined(HAVE_STDINT_H)
 /*
- * We're using stdint.h instead of cstdint below because of libstdcpp.cpp.
- * It looks like a C++ file. When being preprocessed all the C++ specific
- * defines(e.g. __cplusplus) are set, but the C++ include paths are not.
+ * We're using <stdint.h> instead of <cstdint> below because some C++ code
+ * deliberately compiles without C++ include paths.
 */
 #include <stdint.h>

-typedef char          Bool;
-
 typedef uint64_t    uint64;
 typedef  int64_t     int64;
 typedef uint32_t    uint32;
@@ -155,56 +138,25 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-typedef uint64 BA;
-typedef uint64 MA;
-typedef uint32 MPN32;
-
-#elif defined(__APPLE__) || defined(HAVE_STDINT_H)
-
-/*
- * TODO: This is a C99 standard header.  We should be able to test for
- * #if __STDC_VERSION__ >= 199901L, but that breaks the Netware build
- * (which doesn't have stdint.h).
- */
-
-#include <stdint.h>
-
-typedef uint64_t    uint64;
-typedef  int64_t     int64;
-typedef uint32_t    uint32;
-typedef  int32_t     int32;
-typedef uint16_t    uint16;
-typedef  int16_t     int16;
-typedef  uint8_t    uint8;
-typedef   int8_t     int8;
-
-/*
- * Note: C does not specify whether char is signed or unsigned, and
- * both gcc and msvc implement processor-specific signedness.  With
- * three types:
- * typeof(char) != typeof(signed char) != typeof(unsigned char)
- *
- * Be careful here, because gcc (4.0.1 and others) likes to warn about
- * conversions between signed char * and char *.
- */
-
 #else /* !HAVE_STDINT_H */

+/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
 #ifdef _MSC_VER

 typedef unsigned __int64 uint64;
 typedef signed __int64 int64;

 #elif __GNUC__
-/* The Xserver source compiles with -ansi -pendantic */
-#   if !defined(__STRICT_ANSI__) || defined(__FreeBSD__)
-#      if defined(VM_X86_64) || defined(VM_ARM_64)
+#   if defined(VM_X86_64) || defined(VM_ARM_64)
 typedef unsigned long uint64;
 typedef long int64;
-#      else
+#   else
+/*
+ * Only strict c90 (without extensions) lacks a 'long long' type.
+ * If this declaration fails ... use -std=c99 or -std=gnu90.
+ */
 typedef unsigned long long uint64;
 typedef long long int64;
-#      endif
 #   endif
 #else
 #   error - Need compiler define for int64/uint64
@@ -220,6 +172,36 @@ typedef signed char        int8;

 #endif /* HAVE_STDINT_H */

+
+/*
+ * The _XTYPEDEF_BOOL guard prevents colliding with:
+ * <X11/Xlib.h> #define Bool int
+ * <X11/Xdefs.h> typedef int Bool;
+ * If using this header AND X11 headers, be sure to #undef Bool and
+ * be careful about the different size.
+ */
+#if !defined(_XTYPEDEF_BOOL)
+#define _XTYPEDEF_BOOL
+/*
+ * C does not specify whether char is signed or unsigned, and
+ * both gcc and msvc implement it as a non-signed, non-unsigned type.
+ * Thus, (uint8_t *)&Bool and (int8_t *)&Bool are possible compile errors.
+ * This is intentional.
+ */
+typedef char           Bool;
+#endif
+
+#ifndef FALSE
+#define FALSE          0
+#endif
+
+#ifndef TRUE
+#define TRUE           1
+#endif
+
+#define IS_BOOL(x)     (((x) & ~1) == 0)
+
+
 /*
 * FreeBSD (for the tools build) unconditionally defines these in
 * sys/inttypes.h so don't redefine them if this file has already
@@ -344,6 +326,7 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
 */

 #ifdef _MSC_VER
+   /* MSVC added C99-compatible formatting in vs2015. */
   #define FMT64      "I64"
   #ifdef VM_X86_64
      #define FMTSZ      "I64"
@@ -355,53 +338,29 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTH       "I"
   #endif
 #elif defined __APPLE__
-   /* Mac OS hosts use the same formatters for 32- and 64-bit. */
-   #define FMT64 "ll"
+   /* macOS hosts use the same formatters for 32- and 64-bit. */
+   #define FMT64         "ll"
   #if KERNEL
-      #define FMTSZ "l"
+      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
+      #define FMTSZ      "l"
   #else
-      #define FMTSZ "z"
-   #endif
-   #define FMTPD "l"
-   #define FMTH ""
-#elif __GNUC__
-   #define FMTH ""
-   #if defined(sun)
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTSZ  "l"
-         #define FMTPD  "l"
-      #else
-         #define FMTSZ  ""
-         #define FMTPD  ""
-      #endif
-   #elif defined(__linux__) || \
-        (defined(__FreeBSD__) && (__FreeBSD__ + 0))\
-      || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) \
-      || (defined(_POSIX_VERSION) && _POSIX_VERSION >= 200112L) \
-      || (defined(_POSIX2_VERSION) && _POSIX2_VERSION >= 200112L)
-      /* BSD, Linux */
-      #define FMTSZ     "z"
-
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTPD  "l"
-      #else
-         #define FMTPD  ""
-      #endif
-   #else
-      /* Systems with a pre-C99 libc */
-      #define FMTSZ     "Z"
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTPD  "l"
-      #else
-         #define FMTPD  ""
-      #endif
+      #define FMTSZ      "z"
   #endif
+   #define FMTPD         "l"
+   #define FMTH          ""
+#elif defined __GNUC__
+   /*
+    * Every POSIX system we target has C99-compatible printf
+    * (supports 'z' for size_t and 'll' for long long).
+    */
+   #define FMTH          ""
+   #define FMTSZ         "z"
   #if defined(VM_X86_64) || defined(VM_ARM_64)
-      #define FMT64     "l"
-   #elif defined(sun) || defined(__FreeBSD__)
-      #define FMT64     "ll"
+      #define FMT64      "l"
+      #define FMTPD      "l"
   #else
-      #define FMT64     "L"
+      #define FMT64      "ll"
+      #define FMTPD      ""
   #endif
 #else
   #error - Need compiler define for FMT64 and FMTSZ
@@ -485,26 +444,23 @@ typedef uint8 *TCA;  /* Pointer into TC (usually). */
 * Type big enough to hold an integer between 0..100
 */
 typedef uint8 Percent;
-#define AsPercent(v)	((Percent)(v))
+#define AsPercent(v) ((Percent)(v))


 typedef uintptr_t VA;
 typedef uintptr_t VPN;

 typedef uint64    PA;
-typedef uint32    PPN;
+typedef uint64    PPN;
+typedef uint32    PPNTMP;

 typedef uint64    TPA;
-typedef uint32    TPPN;
+typedef uint64    TPPN;

 typedef uint64    PhysMemOff;
 typedef uint64    PhysMemSize;

-/* The Xserver source compiles with -ansi -pendantic */
-#ifndef __STRICT_ANSI__
 typedef uint64    BA;
-#endif
-
 #ifdef VMKERNEL
 typedef void     *BPN;
 #else
@@ -514,9 +470,12 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint32    PageNum;
-typedef unsigned      MemHandle;
-typedef unsigned int  IoHandle;
+typedef uint64    PgCnt64;
+typedef uint64    PageCnt;
+typedef uint64    PgNum64;
+typedef uint64    PageNum;
+typedef unsigned  MemHandle;
+typedef unsigned  IoHandle;
 typedef int32     World_ID;

 /* !! do not alter the definition of INVALID_WORLD_ID without ensuring
@@ -560,11 +519,8 @@ typedef uint128 UReg128;
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
-/* The Xserver source compiles with -ansi -pendantic */
-#ifndef __STRICT_ANSI__
 typedef uint64 MA;
 typedef uint32 MPN32;
-#endif

 /*
 * This type should be used for variables that contain sector
@@ -615,6 +571,7 @@ typedef uint64 LA64;
 typedef uint64 LPN64;
 typedef uint64 PA64;
 typedef uint64 PPN64;
+typedef uint64 TPPN64;
 typedef uint64 MA64;
 typedef uint64 MPN;

@@ -640,10 +597,14 @@ typedef void * UserVA;
 #endif


+/* Maximal observable PPN value. */
 #define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1) /* Maximal observable PPN value. */
-#define INVALID_PPN       ((PPN)0xffffffff)
-#define APIC_INVALID_PPN  ((PPN)0xfffffffe)
+#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+
+#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN32     ((PPN32)0xffffffff)
+#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
+#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)

 #define INVALID_BPN       ((BPN)0x000000ffffffffffull)

@@ -663,7 +624,7 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((PageNum)-1)
+#define INVALID_PAGENUM   ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -755,6 +716,14 @@ typedef void * UserVA;
 #error "gcc version is too old to compile assembly, need gcc-3.3 or better"
 #endif

+/*
+ * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Enforce this here.
+ */
+#if defined _MSC_VER && _MSC_VER < 1400
+#error "cl.exe version is too old, need vc80 or better"
+#endif
+

 /*
 * Consider the following reasons functions are inlined:
@@ -783,7 +752,7 @@ typedef void * UserVA;
 #define INLINE_SINGLE_CALLER INLINE_ALWAYS

 /*
- * Used when a hard guaranteed of no inlining is needed. Very few
+ * Used when a hard guarantee of no inlining is needed. Very few
 * instances need this since the absence of INLINE is a good hint
 * that gcc will not do inlining.
 */
@@ -1086,33 +1055,6 @@ typedef void * UserVA;
 # define FMTMODE "o"
 #endif

-/*
- * Format modifier for printing time_t. Most platforms define a time_t to be
- * a long int, but on FreeBSD (as of 5.0, it seems), the time_t is a signed
- * size quantity. Refer to the definition of FMTSZ to see why we need silly
- * preprocessor arithmetic.
- * Use this like this: printf("The mode is %" FMTTIME ".\n", time);
- */
-#if defined(__FreeBSD__) && (__FreeBSD__ + 0) && ((__FreeBSD__ + 0) >= 5)
-#   define FMTTIME FMTSZ"d"
-#else
-#   if defined(_MSC_VER)
-#      ifndef _SAFETIME_H_
-#         if (_MSC_VER < 1400) || defined(_USE_32BIT_TIME_T)
-#             define FMTTIME "ld"
-#         else
-#             define FMTTIME FMT64"d"
-#         endif
-#      else
-#         ifndef FMTTIME
-#            error "safetime.h did not define FMTTIME"
-#         endif
-#      endif
-#   else
-#      define FMTTIME "ld"
-#   endif
-#endif
-
 #ifdef __APPLE__
 /*
 * Format specifier for all these annoying types such as {S,U}Int32
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2014,2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2014-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,6 +54,7 @@ typedef enum {
   PT_LEVEL_2,
   PT_LEVEL_3,
   PT_LEVEL_4,
+   PT_LEVEL_STOP = PT_LEVEL_1,
   PT_MAX_LEVELS = PT_LEVEL_4
 } PT_Level;

@@ -64,6 +65,9 @@ typedef enum {
 #define PT_ENTRIES_PER_PT     (1 << PT_LEVEL_SHIFT)
 #define PT_OFF_MASK           (PT_ENTRIES_PER_PT - 1)

+/* Align any page number type to the 2M containing page number. */
+#define PT_2M_ALIGN_PN(x)     ((x) & ~PT_OFF_MASK)
+
 /*
 * log2 of the
 * address space size (in 4KB pages) covered by a level-_l page table entry.
@@ -0,0 +1,50 @@
+/*********************************************************
+ * Copyright (C) 2017-2018 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vmmblob.h --
+ *
+ *     VMM blob management.
+ */
+
+#ifndef VMMBLOB_H
+#define VMMBLOB_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#include "includeCheck.h"
+
+struct VMDriver;
+struct MonLoaderHeader;
+
+typedef struct VmmBlobInfo {
+   uint8                  *blobPtr;
+   uint32                  numBytes;
+   MPN                    *mpns;
+   struct MonLoaderHeader *header;
+} VmmBlobInfo;
+
+void   VmmBlob_Cleanup(VmmBlobInfo *bi);
+Bool   VmmBlob_Load(VA64 bsBlobAddr, uint32 numBytes, uint32 headerOffset,
+                    VmmBlobInfo **blobInfo);
+MPN    VmmBlob_GetMpn(struct VMDriver *vm, uint64 blobOffset);
+MPN    VmmBlob_GetHeaderMpn(struct VMDriver *vm);
+uint8 *VmmBlob_GetPtr(VMDriver *vm);
+uint64 VmmBlob_GetSize(VMDriver *vm);
+
+#endif /* VMMBLOB_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -48,16 +48,19 @@

 #define VMMEM_GUEST_WRITEABLE     VMMEM_FLAG_BIT(0)
 #define VMMEM_GUEST_BREAKCOW      VMMEM_FLAG_BIT(1)
-#define VMMEM_GUEST_LARGE_PAGE    VMMEM_FLAG_BIT(2)
-#define VMMEM_GUEST_CAN_FAIL      VMMEM_FLAG_BIT(3)
-#define VMMEM_GUEST_TEST_ZEROCOW  VMMEM_FLAG_BIT(4)
-#define VMMEM_GUEST_TRY_ZEROCOW   VMMEM_FLAG_BIT(5)
-#define VMMEM_GUEST_TRY_POISONCOW VMMEM_FLAG_BIT(6)
-#define VMMEM_GUEST_PREALLOC      VMMEM_FLAG_BIT(7)
-#define VMMEM_GUEST_ALL_FLAGS     MASK(8)
+#define VMMEM_GUEST_2M_PAGE       VMMEM_FLAG_BIT(2)
+#define VMMEM_GUEST_1G_PAGE       VMMEM_FLAG_BIT(3)
+#define VMMEM_GUEST_CAN_FAIL      VMMEM_FLAG_BIT(4)
+#define VMMEM_GUEST_TEST_ZEROCOW  VMMEM_FLAG_BIT(5)
+#define VMMEM_GUEST_TRY_ZEROCOW   VMMEM_FLAG_BIT(6)
+#define VMMEM_GUEST_TRY_POISONCOW VMMEM_FLAG_BIT(7)
+#define VMMEM_GUEST_PREALLOC      VMMEM_FLAG_BIT(8)
+#define VMMEM_GUEST_ALL_FLAGS     MASK(9)
 #define VMMEM_GUEST_TRY_COW       (VMMEM_GUEST_TEST_ZEROCOW | \
                                   VMMEM_GUEST_TRY_ZEROCOW  | \
                                   VMMEM_GUEST_TRY_POISONCOW)
+#define VMMEM_GUEST_LARGE_PAGE    (VMMEM_GUEST_2M_PAGE | \
+                                   VMMEM_GUEST_1G_PAGE)

 #define VMMEM_PLATFORM_CHECK_OK           VMMEM_FLAG_BIT(0)
 #define VMMEM_PLATFORM_KEY_OK             VMMEM_FLAG_BIT(1)
@@ -80,11 +83,10 @@
 * Structure used to query platform about the page state.
 */
 typedef struct PlatformPageInfoList {
-   uint32 numPages;
-   uint32 _pad;
-   BPN    bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
-   MPN    mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
-   uint8  flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
+   PgCnt64 numPages;
+   BPN     bpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // bpns to check
+   MPN     mpn[MAX_PLATFORM_PAGE_INFO_PAGES];    // filled in by host
+   uint8   flags[MAX_PLATFORM_PAGE_INFO_PAGES];  // filled in by host
 } PlatformPageInfoList;

 #define VMMEM_SERVICES_TYPE_2_MASK(type)                  \
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,8 +35,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-#include <asm/processor-flags.h>
-
 #define X86_MAX_INSTR_LEN  15   /* Max byte length of an x86 instruction. */

 #define NUM_IDT_VECTORS 256
@@ -77,9 +75,7 @@
 #define CR3_PDB_MASK   0xfffff000
 #define CR3_IGNORE     0xFFF
 #define PAE_CR3_IGNORE 0x1F
-#ifndef CR3_PCID_MASK
 #define CR3_PCID_MASK  0xFFF
-#endif
 #define CR3_NO_FLUSH   (1ULL << 63)

 #define CR4_VME        0x00000001
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2015-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -121,7 +121,7 @@

 /*
 * Use APIC_MSR_BASEMASK as mask only for reading the MSR_APIC_BASE.
- * Use VMCORESHARED->reservedAPIC as mask when writing MSR_APIC_BASE.
+ * Use guestReservedAPIC as mask when writing MSR_APIC_BASE.
 */
 #define APIC_MSR_BASEMASK     CONST64U(0x0000fffffffff000)
 #define APIC_MSR_X2APIC_ENABLED 0x00000400
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -211,6 +211,7 @@ typedef enum {
 #define CPUID_AMD_VENDOR_STRING         "AuthcAMDenti"
 #define CPUID_CYRIX_VENDOR_STRING       "CyriteadxIns"
 #define CPUID_VIA_VENDOR_STRING         "CentaulsaurH"
+#define CPUID_HYGON_VENDOR_STRING       "HygouinenGen"

 #define CPUID_HYPERV_HYPERVISOR_VENDOR_STRING  "Microsoft Hv"
 #define CPUID_KVM_HYPERVISOR_VENDOR_STRING     "KVMKVMKVM\0\0\0"
@@ -221,6 +222,7 @@ typedef enum {
 #define CPUID_AMD_VENDOR_STRING_FIXED   "AuthenticAMD"
 #define CPUID_CYRIX_VENDOR_STRING_FIXED "CyrixInstead"
 #define CPUID_VIA_VENDOR_STRING_FIXED   "CentaurHauls"
+#define CPUID_HYGON_VENDOR_STRING_FIXED "HygonGenuine"

 /*
 * FIELD can be defined to process the CPUID information provided in the
@@ -252,7 +254,7 @@ typedef enum {
 * defaultMasks (cpuidcompat.c) if defined there, and must be less than or
 * equal to the version of the cpuid leaf it's in.
 *
- * The eighth parameter describes whether the feature is capable of being used
+ * The ninth parameter describes whether the feature is capable of being used
 * by usermode code (TRUE), or just CPL0 kernel code (FALSE).
 *
 * FLAG is defined identically to FIELD, but its accessors are more appropriate
@@ -338,7 +340,7 @@ FLAG(   1,  0, ECX,  2,  1, DTES64,                            NO,   0, FALSE) \
 FLAG(   1,  0, ECX,  3,  1, MWAIT,                             YES,  4, FALSE) \
 FLAG(   1,  0, ECX,  4,  1, DSCPL,                             NO,   0, FALSE) \
 FLAG(   1,  0, ECX,  5,  1, VMX,                               YES,  4, FALSE) \
-FLAG(   1,  0, ECX,  6,  1, SMX,                               NO,   0, FALSE) \
+FLAG(   1,  0, ECX,  6,  1, SMX,                               YES, 17, FALSE) \
 FLAG(   1,  0, ECX,  7,  1, EIST,                              NO,   0, FALSE) \
 FLAG(   1,  0, ECX,  8,  1, TM2,                               NO,   0, FALSE) \
 FLAG(   1,  0, ECX,  9,  1, SSSE3,                             YES,  4, TRUE)  \
@@ -463,6 +465,9 @@ FLAG(   6,  0, ECX,  3,  1, ENERGY_PERF_BIAS,                  NO,   0, FALSE)
 #define CPUID_7_EDX_28 \
 FLAG(   7,  0, EDX, 28,  1, LEVEL7EDX_RSVD1,                    NO,  0, FALSE)

+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
+#define CPUID_7_EDX_10 \
+FLAG(   7,  0, EDX, 10,  1, LEAF7_RSVD,                         NO,  0, FALSE)
 #define CPUID_7_EDX_31 \
 FLAG(   7,  0, EDX, 31,  1, LEVEL7EDX_RSVD2,                    NO,  0, FALSE)

@@ -470,7 +475,7 @@ FLAG(   7,  0, EDX, 31,  1, LEVEL7EDX_RSVD2,                    NO,  0, FALSE)
 #define CPUID_FIELD_DATA_LEVEL_7                                               \
 FLAG(   7,  0, EBX,  0,  1, FSGSBASE,                          YES,  9, FALSE) \
 FLAG(   7,  0, EBX,  1,  1, TSC_ADJUST,                        ANY, 11, FALSE) \
-FLAG(   7,  0, EBX,  2,  1, SGX,                               NO,   0, FALSE) \
+FLAG(   7,  0, EBX,  2,  1, SGX,                               YES, 17, FALSE) \
 FLAG(   7,  0, EBX,  3,  1, BMI1,                              YES,  9, TRUE)  \
 FLAG(   7,  0, EBX,  4,  1, HLE,                               YES, 11, TRUE)  \
 FLAG(   7,  0, EBX,  5,  1, AVX2,                              YES, 11, TRUE)  \
@@ -489,10 +494,10 @@ FLAG(   7,  0, EBX, 17,  1, AVX512DQ,                          YES, 13, TRUE)  \
 FLAG(   7,  0, EBX, 18,  1, RDSEED,                            YES, 11, TRUE)  \
 FLAG(   7,  0, EBX, 19,  1, ADX,                               YES, 11, TRUE)  \
 FLAG(   7,  0, EBX, 20,  1, SMAP,                              YES, 11, FALSE) \
-FLAG(   7,  0, EBX, 21,  1, AVX512IFMA,                        YES, 15, TRUE)  \
+FLAG(   7,  0, EBX, 21,  1, AVX512IFMA,                        YES, 17, TRUE)  \
 FLAG(   7,  0, EBX, 23,  1, CLFLUSHOPT,                        YES, 13, TRUE)  \
 FLAG(   7,  0, EBX, 24,  1, CLWB,                              YES, 13, TRUE)  \
-FLAG(   7,  0, EBX, 25,  1, PT,                                NO,   0, FALSE) \
+FLAG(   7,  0, EBX, 25,  1, PT,                                YES, 17, FALSE) \
 FLAG(   7,  0, EBX, 26,  1, AVX512PF,                          YES, 13, TRUE)  \
 FLAG(   7,  0, EBX, 27,  1, AVX512ER,                          YES, 13, TRUE)  \
 FLAG(   7,  0, EBX, 28,  1, AVX512CD,                          YES, 13, TRUE)  \
@@ -500,10 +505,24 @@ FLAG(   7,  0, EBX, 29,  1, SHA,                               YES, 14, TRUE)  \
 FLAG(   7,  0, EBX, 30,  1, AVX512BW,                          YES, 13, TRUE)  \
 FLAG(   7,  0, EBX, 31,  1, AVX512VL,                          YES, 13, TRUE)  \
 FLAG(   7,  0, ECX,  0,  1, PREFETCHWT1,                       YES, 13, TRUE)  \
-FLAG(   7,  0, ECX,  1,  1, AVX512VBMI,                        YES, 15, TRUE)  \
+FLAG(   7,  0, ECX,  1,  1, AVX512VBMI,                        YES, 17, TRUE)  \
+FLAG(   7,  0, ECX,  2,  1, UMIP,                              NO,   0, FALSE) \
 FLAG(   7,  0, ECX,  3,  1, PKU,                               YES, 13, TRUE)  \
 FLAG(   7,  0, ECX,  4,  1, OSPKE,                             ANY, 13, TRUE)  \
-FLAG(   7,  0, ECX, 30,  1, SGX_LC,                            NO,   0, FALSE) \
+FLAG(   7,  0, ECX,  6,  1, AVX512VBMI2,                       NO,   0, TRUE)  \
+FLAG(   7,  0, ECX,  8,  1, GFNI,                              NO,   0, TRUE)  \
+FLAG(   7,  0, ECX,  9,  1, VAES,                              NO,   0, TRUE)  \
+FLAG(   7,  0, ECX, 10,  1, VPCLMULQDQ,                        NO,   0, TRUE)  \
+FLAG(   7,  0, ECX, 11,  1, AVX512VNNI,                        NO,   0, TRUE)  \
+FLAG(   7,  0, ECX, 12,  1, AVX512BITALG,                      NO,   0, TRUE)  \
+FLAG(   7,  0, ECX, 14,  1, AVX512VPOPCNTDQ,                   YES, 16, TRUE)  \
+FLAG(   7,  0, ECX, 16,  1, VA57,                              NO,   0, TRUE)  \
+FIELD(  7,  0, ECX, 17,  5, MAWA,                              NO,   0, TRUE)  \
+FLAG(   7,  0, ECX, 22,  1, RDPID,                             NO,   0, TRUE)  \
+FLAG(   7,  0, ECX, 30,  1, SGX_LC,                            YES, 17, FALSE) \
+FLAG(   7,  0, EDX,  2,  1, AVX512QVNNIW,                      YES, 16, TRUE)  \
+FLAG(   7,  0, EDX,  3,  1, AVX512QFMAPS,                      YES, 16, TRUE)  \
+CPUID_7_EDX_10                                                                 \
 FLAG(   7,  0, EDX, 26,  1, IBRSIBPB,                          ANY,  9, FALSE) \
 FLAG(   7,  0, EDX, 27,  1, STIBP,                             YES,  9, FALSE) \
 CPUID_7_EDX_28                                                                 \
@@ -634,22 +653,22 @@ FIELD( 10,  1, EDX,  0, 16, MAX_COS_NUMBER,                    NO,   0, FALSE)

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
 #define CPUID_FIELD_DATA_LEVEL_12                                              \
-FLAG(  12,  0, EAX,  0,  1, SGX1,                              NO,   0, FALSE) \
+FLAG(  12,  0, EAX,  0,  1, SGX1,                              YES, 17, FALSE) \
 FLAG(  12,  0, EAX,  1,  1, SGX2,                              NO,   0, FALSE) \
-FLAG(  12,  0, EBX,  0,  1, SGX_MISCSELECT_EXINFO,             NO,   0, FALSE) \
+FLAG(  12,  0, EBX,  0,  1, SGX_MISCSELECT_EXINFO,             YES, 17, FALSE) \
 FIELD( 12,  0, EBX,  1, 31, SGX_MISCSELECT_RSVD,               NO,   0, FALSE) \
-FIELD( 12,  0, EDX,  0,  8, MAX_ENCLAVE_SIZE_NOT64,            NO,   0, FALSE) \
-FIELD( 12,  0, EDX,  8,  8, MAX_ENCLAVE_SIZE_64,               NO,   0, FALSE) \
-FIELD( 12,  1, EAX,  0, 32, SECS_ATTRIBUTES0,                  NO,   0, FALSE) \
-FIELD( 12,  1, EBX,  0, 32, SECS_ATTRIBUTES1,                  NO,   0, FALSE) \
-FIELD( 12,  1, ECX,  0, 32, SECS_ATTRIBUTES2,                  NO,   0, FALSE) \
-FIELD( 12,  1, EDX,  0, 32, SECS_ATTRIBUTES3,                  NO,   0, FALSE) \
-FIELD( 12,  2, EAX,  0, 15, EPC00_VALID,                       NO,   0, FALSE) \
-FIELD( 12,  2, EAX, 12, 20, EPC00_BASE_LOW,                    NO,   0, FALSE) \
-FIELD( 12,  2, EBX,  0, 20, EPC00_BASE_HIGH,                   NO,   0, FALSE) \
-FIELD( 12,  2, ECX,  0, 15, EPC00_PROTECTED,                   NO,   0, FALSE) \
-FIELD( 12,  2, ECX, 12, 20, EPC00_SIZE_LOW,                    NO,   0, FALSE) \
-FIELD( 12,  2, EDX,  0, 20, EPC00_SIZE_HIGH,                   NO,   0, FALSE) \
+FIELD( 12,  0, EDX,  0,  8, MAX_ENCLAVE_SIZE_NOT64,            YES, 17, FALSE) \
+FIELD( 12,  0, EDX,  8,  8, MAX_ENCLAVE_SIZE_64,               YES, 17, FALSE) \
+FIELD( 12,  1, EAX,  0, 32, SECS_ATTRIBUTES0,                  YES, 17, FALSE) \
+FIELD( 12,  1, EBX,  0, 32, SECS_ATTRIBUTES1,                  YES, 17, FALSE) \
+FIELD( 12,  1, ECX,  0, 32, SECS_ATTRIBUTES2,                  YES, 17, FALSE) \
+FIELD( 12,  1, EDX,  0, 32, SECS_ATTRIBUTES3,                  YES, 17, FALSE) \
+FIELD( 12,  2, EAX,  0, 15, EPC00_VALID,                       YES, 17, FALSE) \
+FIELD( 12,  2, EAX, 12, 20, EPC00_BASE_LOW,                    YES, 17, FALSE) \
+FIELD( 12,  2, EBX,  0, 20, EPC00_BASE_HIGH,                   YES, 17, FALSE) \
+FIELD( 12,  2, ECX,  0, 15, EPC00_PROTECTED,                   YES, 17, FALSE) \
+FIELD( 12,  2, ECX, 12, 20, EPC00_SIZE_LOW,                    YES, 17, FALSE) \
+FIELD( 12,  2, EDX,  0, 20, EPC00_SIZE_HIGH,                   YES, 17, FALSE) \
 FIELD( 12,  3, EAX,  0,  4, EPC01_VALID,                       NO,   0, FALSE) \
 FIELD( 12,  3, EAX, 12, 20, EPC01_BASE_LOW,                    NO,   0, FALSE) \
 FIELD( 12,  3, EBX,  0, 20, EPC01_BASE_HIGH,                   NO,   0, FALSE) \
@@ -659,20 +678,20 @@ FIELD( 12,  3, EDX,  0, 20, EPC01_SIZE_HIGH,                   NO,   0, FALSE)

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
 #define CPUID_FIELD_DATA_LEVEL_14                                              \
-FIELD( 14,  0, EAX,  0, 32, MAX_SUB_LEAF,                        NO, 0, FALSE) \
-FLAG(  14,  0, EBX,  0,  1, CR3FTR_AND_MATCHMSR_AVAILABLE,       NO, 0, FALSE) \
-FLAG(  14,  0, EBX,  1,  1, PSB_AND_CYCLE_ACCURATE_MODE,         NO, 0, FALSE) \
-FLAG(  14,  0, EBX,  2,  1, IP_TRACESTOP_FTR_PTMSR_PERSIST,      NO, 0, FALSE) \
-FLAG(  14,  0, EBX,  3,  1, MTC_PKT_GENERATION_SUPPORTED,        NO, 0, FALSE) \
-FLAG(  14,  0, ECX,  0,  1, TOPA_OUTPUT_SUPPORTED,               NO, 0, FALSE) \
-FLAG(  14,  0, ECX,  1,  1, TOPA_ALLOW_MULTIPLE_ENTRIES,         NO, 0, FALSE) \
-FLAG(  14,  0, ECX,  2,  1, SINGLE_RANGE_OUTPUT_SCHEME,          NO, 0, FALSE) \
-FLAG(  14,  0, ECX,  3,  1, TRACE_TRANSPORT_SUBSYSTEM,           NO, 0, FALSE) \
-FLAG(  14,  0, ECX, 31,  1, LIP_PRESENT_FOR_IP_PAYLOADS,         NO, 0, FALSE) \
-FIELD( 14,  1, EAX,  0,  2, NUM_ADDR_RANGE_FOR_FILTERING,        NO, 0, FALSE) \
-FIELD( 14,  1, EAX, 16, 16, SUPPORTED_MTC_ENCODINGS,             NO, 0, FALSE) \
-FIELD( 14,  1, EBX,  0, 16, SUPPORTED_CYCLE_THRESHOLD_ENCODINGS, NO, 0, FALSE) \
-FIELD( 14,  1, EBX, 16, 16, SUPPORTED_PSB_FREQ_ENCODINGS,        NO, 0, FALSE) \
+FIELD( 14,  0, EAX,  0, 32, MAX_PT_SUB_LEAF,                   YES, 17, FALSE) \
+FLAG(  14,  0, EBX,  0,  1, CR3FTR_AND_MATCHMSR_AVAILABLE,     YES, 17, FALSE) \
+FLAG(  14,  0, EBX,  1,  1, PSB_AND_CYCLE_ACCURATE_MODE,       YES, 17, FALSE) \
+FLAG(  14,  0, EBX,  2,  1, IP_TRACESTOP_FTR_PTMSR_PERSIST,    YES, 17, FALSE) \
+FLAG(  14,  0, EBX,  3,  1, MTC_PKT_GENERATION_SUPPORTED,      YES, 17, FALSE) \
+FLAG(  14,  0, ECX,  0,  1, TOPA_OUTPUT_SUPPORTED,             YES, 17, FALSE) \
+FLAG(  14,  0, ECX,  1,  1, TOPA_ALLOW_MULTIPLE_ENTRIES,       YES, 17, FALSE) \
+FLAG(  14,  0, ECX,  2,  1, SINGLE_RANGE_OUTPUT_SCHEME,        YES, 17, FALSE) \
+FLAG(  14,  0, ECX,  3,  1, TRACE_TRANSPORT_SUBSYSTEM,         NO,   0, FALSE) \
+FLAG(  14,  0, ECX, 31,  1, LIP_PRESENT_FOR_IP_PAYLOADS,       YES, 17, FALSE) \
+FIELD( 14,  1, EAX,  0,  2, NUM_ADDR_RANGE_FOR_FILTERING,      YES, 17, FALSE) \
+FIELD( 14,  1, EAX, 16, 16, SUPPORTED_MTC_ENCODINGS,           YES, 17, FALSE) \
+FIELD( 14,  1, EBX,  0, 16, SUPPORTED_CYCLE_THRESHOLD_ENCODINGS, YES,17,FALSE) \
+FIELD( 14,  1, EBX, 16, 16, SUPPORTED_PSB_FREQ_ENCODINGS,      YES, 17, FALSE) \

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
 #define CPUID_FIELD_DATA_LEVEL_15                                              \
@@ -939,6 +958,11 @@ FIELD( 86,  0, EDX, 18, 14, L3CACHE_SIZE,                      NA,   0, FALSE)

 /*    LEVEL, REG, POS, SIZE, NAME,                        MON SUPP, HWV, CPL3 */
 #define CPUID_FIELD_DATA_LEVEL_87                                              \
+FLAG(  87,  0, EBX,  0,  1, MCA_OVERFLOW_RECOV,                NA,   0, FALSE) \
+FLAG(  87,  0, EBX,  1,  1, SUCCOR,                            NA,   0, FALSE) \
+FLAG(  87,  0, EBX,  2,  1, HWA,                               NA,   0, FALSE) \
+FLAG(  87,  0, EBX,  3,  1, SCALABLE_MCA,                      NA,   0, FALSE) \
+FLAG(  87,  0, EBX,  4,  1, PFEH_SUPPORT_PRESENT,              NA,   0, FALSE) \
 FLAG(  87,  0, EDX,  0,  1, TS,                                NA,   0, FALSE) \
 FLAG(  87,  0, EDX,  1,  1, FID,                               NA,   0, FALSE) \
 FLAG(  87,  0, EDX,  2,  1, VID,                               NA,   0, FALSE) \
@@ -950,6 +974,17 @@ FLAG(  87,  0, EDX,  7,  1, HWPSTATE,                          NA,   0, FALSE) \
 FLAG(  87,  0, EDX,  8,  1, TSC_INVARIANT,                     NA,   0, FALSE) \
 FLAG(  87,  0, EDX,  9,  1, CORE_PERF_BOOST,                   NA,   0, FALSE)

+#define CPUID_88_EBX_14 \
+FLAG(  88,  0, EBX, 14,  1, LEAF88_RSVD1,                       NO,  0, FALSE)
+#define CPUID_88_EBX_15 \
+FLAG(  88,  0, EBX, 15,  1, LEAF88_RSVD2,                       NO,  0, FALSE)
+#define CPUID_88_EBX_16 \
+FLAG(  88,  0, EBX, 16,  1, LEAF88_RSVD3,                       NO,  0, FALSE)
+#define CPUID_88_EBX_17 \
+FLAG(  88,  0, EBX, 17,  1, LEAF88_RSVD4,                       NO,  0, FALSE)
+#define CPUID_88_EBX_18 \
+FLAG(  88,  0, EBX, 18,  1, LEAF88_RSVD5,                       NO,  0, FALSE)
+
 /*    LEVEL, REG, POS, SIZE, NAME,                        MON SUPP, HWV, CPL3 */
 #define CPUID_FIELD_DATA_LEVEL_88                                              \
 FIELD( 88,  0, EAX,  0,  8, PHYS_BITS,                         YES,  4, FALSE) \
@@ -958,7 +993,12 @@ FIELD( 88,  0, EAX, 16,  8, GUEST_PHYS_ADDR_SZ,                YES,  8, FALSE) \
 FLAG(  88,  0, EBX,  0,  1, CLZERO,                            YES, 14, TRUE)  \
 FLAG(  88,  0, EBX,  1,  1, IRPERF,                            NO,   0, FALSE) \
 FLAG(  88,  0, EBX,  2,  1, XSAVE_ERR_PTR,                     NO,   0, FALSE) \
-FLAG(  88,  0, EBX, 12,  1, IBPBONLY,                          ANY,  9, FALSE) \
+FLAG(  88,  0, EBX, 12,  1, LEAF88_IBPB,                       ANY,  9, FALSE) \
+CPUID_88_EBX_14 \
+CPUID_88_EBX_15 \
+CPUID_88_EBX_16 \
+CPUID_88_EBX_17 \
+CPUID_88_EBX_18 \
 FIELD( 88,  0, ECX,  0,  8, LEAF88_CORE_COUNT,                 YES,  4, FALSE) \
 FIELD( 88,  0, ECX, 12,  4, APICID_COREID_SIZE,                YES,  7, FALSE) \
 FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                      NO,   0, FALSE)
@@ -989,7 +1029,7 @@ CPUID_8A_EDX_11 \
 FLAG(  8A,  0, EDX, 12,  1, SVM_PAUSE_THRESHOLD,               NO,   0, FALSE) \
 FLAG(  8A,  0, EDX, 13,  1, SVM_AVIC,                          NO,   0, FALSE) \
 CPUID_8A_EDX_14 \
-FLAG(  8A,  0, EDX, 15,  1, SVM_VIRT_VMSAVE_VMLOAD,            NO,   0, FALSE) \
+FLAG(  8A,  0, EDX, 15,  1, SVM_V_VMSAVE_VMLOAD,               NO,   0, FALSE) \
 FLAG(  8A,  0, EDX, 16,  1, SVM_VGIF,                          NO,   0, FALSE) \
 FIELD( 8A,  0, EDX, 17, 15, SVMEDX_RSVD,                       NO,   0, FALSE)

@@ -1176,48 +1216,6 @@ enum {
 #undef FIELD
 #undef FLAG

-/*
- * Legal CPUID config file mask characters.  For a description of the
- * cpuid masking system, please see:
- *
- * http://vmweb.vmware.com/~mts/cgi-bin/view.cgi/Apps/CpuMigrationChecks
- */
-
-#define CPUID_MASK_HIDE_CHR    '0'
-#define CPUID_MASK_HIDE_STR    "0"
-#define CPUID_MASK_FORCE_CHR   '1'
-#define CPUID_MASK_FORCE_STR   "1"
-#define CPUID_MASK_PASS_CHR    '-'
-#define CPUID_MASK_PASS_STR    "-"
-#define CPUID_MASK_TRUE_CHR    'T'
-#define CPUID_MASK_TRUE_STR    "T"
-#define CPUID_MASK_FALSE_CHR   'F'
-#define CPUID_MASK_FALSE_STR   "F"
-#define CPUID_MASK_IGNORE_CHR  'X'
-#define CPUID_MASK_IGNORE_STR  "X"
-#define CPUID_MASK_HOST_CHR    'H'
-#define CPUID_MASK_HOST_STR    "H"
-#define CPUID_MASK_RSVD_CHR    'R'
-#define CPUID_MASK_RSVD_STR    "R"
-#define CPUID_MASK_INSTALL_CHR 'I'
-#define CPUID_MASK_INSTALL_STR "I"
-
-/*
- * When LM is disabled, we overlay the following masks onto the
- * guest's default masks.  Any level that is not defined below should
- * be treated as all "-"s
- */
-
-#define CPT_ID1ECX_LM_DISABLED  "----:----:----:----:--0-:----:----:----"
-#define CPT_ID81EDX_LM_DISABLED "--0-:----:----:----:----:----:----:----"
-#define CPT_ID81ECX_LM_DISABLED "----:----:----:----:----:----:----:---0"
-
-#define CPT_GET_LM_DISABLED_MASK(lvl, reg)                                  \
-   ((lvl == 1 && reg == CPUID_REG_ECX) ? CPT_ID1ECX_LM_DISABLED :           \
-    (lvl == 0x80000001 && reg == CPUID_REG_ECX) ? CPT_ID81ECX_LM_DISABLED : \
-    (lvl == 0x80000001 && reg == CPUID_REG_EDX) ? CPT_ID81EDX_LM_DISABLED : \
-    NULL)
-
 /*
 * CPUID_MASK --
 * CPUID_SHIFT --
@@ -1381,6 +1379,9 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 /* Effective VIA CPU Families */
 #define CPUID_FAMILY_C7               6

+/* Effective Hygon CPU Families. */
+#define CPUID_FAMILY_DHYANA          0x18
+
 /* Intel model information */
 #define CPUID_MODEL_PPRO              1
 #define CPUID_MODEL_PII_03            3
@@ -1420,12 +1421,14 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_MODEL_ATOM_4D        0x4d  // Avoton
 #define CPUID_MODEL_SKYLAKE_4E     0x4e  // Skylake-Y / Kabylake U/Y ES
 #define CPUID_MODEL_BROADWELL_4F   0x4f  // Broadwell EP/EN/EX
+#define CPUID_MODEL_SKYLAKE_55     0x55  // Skylake EP/EN/EX
 #define CPUID_MODEL_BROADWELL_56   0x56  // Broadwell DE
 #define CPUID_MODEL_KNL_57         0x57  // Knights Landing
 #define CPUID_MODEL_ATOM_5A        0x5a  // Future Silvermont
 #define CPUID_MODEL_ATOM_5D        0x5d  // Future Silvermont
 #define CPUID_MODEL_SKYLAKE_5E     0x5e  // Skylake-S / Kabylake S/H ES
 #define CPUID_MODEL_ATOM_5F        0x5f  // Denverton
+#define CPUID_MODEL_KNM_85         0x85  // Knights Mill
 #define CPUID_MODEL_KABYLAKE_8E    0x8e  // Kabylake U/Y QS
 #define CPUID_MODEL_KABYLAKE_9E    0x9e  // Kabylake S/H QS

@@ -1433,6 +1436,8 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_STEPPING_KABYLAKE_ES     0x8  // Kabylake S/H/U/Y ES
 #define CPUID_STEPPING_COFFEELAKE_A    0xA  // Coffeelake U/S/H
 #define CPUID_STEPPING_COFFEELAKE_B    0xB  // Coffeelake S/H
+#define CPUID_STEPPING_CASCADELAKE_A   0x5  // Cascade Lake A-step
+#define CPUID_STEPPING_CASCADELAKE_B1  0x7  // Cascade Lake B1-step

 #define CPUID_MODEL_PIII_07    7
 #define CPUID_MODEL_PIII_08    8
@@ -1459,17 +1464,21 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_MODEL_ZEN_1F            0x1F // Max Zen model defined in BKDG

 /* VIA model information */
-#define CPUID_MODEL_NANO       15     // Isaiah
+#define CPUID_MODEL_NANO                15 // Isaiah
+
+/* Hygon model information. */
+#define CPUID_MODEL_DHYANA_A             0 // Dhyana A

 /*
 *----------------------------------------------------------------------
 *
- * CPUID_IsVendor{AMD,Intel,VIA} --
+ * CPUID_IsVendor{AMD,Intel,VIA,Hygon} --
 *
- *      Determines if the vendor string in cpuid id0 is from {AMD,Intel,VIA}.
+ *      Determines if the vendor string in cpuid id0 is from
+ *      {AMD,Intel,VIA,Hygon}.
 *
 * Results:
- *      True iff vendor string is CPUID_{AMD,INTEL,VIA}_VENDOR_STRING
+ *      True iff vendor string is CPUID_{AMD,INTEL,VIA,HYGON}_VENDOR_STRING
 *
 * Side effects:
 *      None.
@@ -1503,6 +1512,12 @@ CPUID_IsVendorVIA(CPUIDRegs *id0)
   return CPUID_IsRawVendor(id0, CPUID_VIA_VENDOR_STRING);
 }

+static INLINE Bool
+CPUID_IsVendorHygon(CPUIDRegs *id0)
+{
+   return CPUID_IsRawVendor(id0, CPUID_HYGON_VENDOR_STRING);
+}
+
 static INLINE uint32
 CPUID_EFFECTIVE_FAMILY(uint32 v) /* %eax from CPUID with %eax=1. */
 {
@@ -1647,6 +1662,15 @@ CPUID_MODEL_IS_HASWELL(uint32 v) // IN: %eax from CPUID with %eax=1.
           effectiveModel == CPUID_MODEL_HASWELL_46);
 }

+static INLINE Bool
+CPUID_MODEL_IS_CASCADELAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_55 &&
+          CPUID_EFFECTIVE_STEPPING(v) >= CPUID_STEPPING_CASCADELAKE_A &&
+          CPUID_EFFECTIVE_STEPPING(v) <= CPUID_STEPPING_CASCADELAKE_B1;
+}

 static INLINE Bool
 CPUID_MODEL_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
@@ -1655,6 +1679,7 @@ CPUID_MODEL_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
   return CPUID_FAMILY_IS_P6(v) &&
          ((CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_5E &&
            CPUID_EFFECTIVE_STEPPING(v) != CPUID_STEPPING_KABYLAKE_ES) ||
+            CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_55 ||
           (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_4E &&
            CPUID_EFFECTIVE_STEPPING(v) != CPUID_STEPPING_KABYLAKE_ES));
 }
@@ -1695,6 +1720,7 @@ CPUID_UARCH_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
           CPUID_MODEL_IS_SKYLAKE(v));
 }

+
 static INLINE Bool
 CPUID_UARCH_IS_HASWELL(uint32 v) // IN: %eax from CPUID with %eax=1.
 {
@@ -1789,6 +1815,14 @@ CPUID_MODEL_IS_KNIGHTS_LANDING(uint32 v) // IN: %eax from CPUID with %eax=1.
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KNL_57;
 }

+static INLINE Bool
+CPUID_MODEL_IS_KNIGHTS_MILL(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KNM_85;
+}
+

 static INLINE Bool
 CPUID_FAMILY_IS_K7(uint32 eax)
@@ -1974,6 +2008,22 @@ CPUID_MODEL_IS_ZEN(uint32 eax)
          CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN_1F;
 }

+
+static INLINE Bool
+CPUID_FAMILY_IS_DHYANA(uint32 eax)
+{
+   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_DHYANA;
+}
+
+
+static INLINE Bool
+CPUID_MODEL_IS_DHYANA_A(uint32 eax)
+{
+   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_DHYANA &&
+          CPUID_EFFECTIVE_MODEL(eax)  == CPUID_MODEL_DHYANA_A;
+}
+
+
 #define CPUID_TYPE_PRIMARY     0
 #define CPUID_TYPE_OVERDRIVE   1
 #define CPUID_TYPE_SECONDARY   2
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,7 +24,6 @@

 #ifndef _X86MSR_H_
 #define _X86MSR_H_
-#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX

@@ -43,6 +42,7 @@
 extern "C" {
 #endif

+#define SIZEOF_X86_MSR     8

 /*
 * Results of calling rdmsr(msrNum) on all logical processors.
@@ -83,6 +83,7 @@ MSRQuery;
 #define MSR_PLATFORM_ID       0x00000017
 #define MSR_APIC_BASE         0x0000001b
 #define MSR_SMI_COUNT         0x00000034 // Intel Nehalem Family
+#define MSR_CORE_THREAD_COUNT 0x00000035 // Intel Nehalem Family +
 #define MSR_FEATCTL           0x0000003a
 #define MSR_TSC_ADJUST        0x0000003b
 #define MSR_SPEC_CTRL         0x00000048
@@ -119,17 +120,16 @@ MSRQuery;
 #define MSR_ARCH_CAPABILITIES_RDCL_NO             (1ULL << 0)
 #define MSR_ARCH_CAPABILITIES_IBRS_ALL            (1ULL << 1)
 #define MSR_ARCH_CAPABILITIES_RSBA                (1ULL << 2)
+#define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
+#define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)

-#define MSR_FLUSH_CMD                        0x10b
-#define MSR_FLUSH_CMD_FLUSH_L1D              1

 #define MSR_SPEC_CTRL_IBRS                        (1UL << 0)
 #define MSR_SPEC_CTRL_STIBP                       (1UL << 1)
+
 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

-#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
-#endif

 /* Intel Core Architecture and later: use only architected counters. */
 #define IA32_MSR_PERF_CAPABILITIES                0x345
@@ -391,6 +391,17 @@ typedef enum {
 #define MSR_SGX_SVN_STATUS_SINIT_SVN     CONST64U(0xff0000)
 #define MSR_SGX_SVN_STATUS_RSVD          CONST64U(0xffffffffff00fffe)

+/*
+ * SGX Flexible Launch Control MSRs.
+ * These MSRs store hash of Launch Enclave's public key.
+ */
+#define MSR_SGXLEPUBKEYHASH0  0x0000008c
+#define MSR_SGXLEPUBKEYHASH1  0x0000008d
+#define MSR_SGXLEPUBKEYHASH2  0x0000008e
+#define MSR_SGXLEPUBKEYHASH3  0x0000008f
+
+#define NUM_SGXLEPUBKEYHASH_MSRs (4)
+
 /* MSR_CR_PAT power-on value */
 #define MSR_CR_PAT_DEFAULT   0x0007040600070406ULL

@@ -429,6 +440,7 @@ typedef enum {
 #define MSR_FEATCTL_VMXE     0x00000004
 #define MSR_FEATCTL_SENTERP  0x00007F00
 #define MSR_FEATCTL_SENTERE  0x00008000
+#define MSR_FEATCTL_FLCE     0x00020000
 #define MSR_FEATCTL_SGXE     0x00040000
 #define MSR_FEATCTL_LMCE     0x00100000

@@ -453,9 +465,7 @@ typedef enum {
 #define MSR_K7_HWCR_SSEDIS         0x00008000ULL // Disable SSE bit
 #define MSR_K7_HWCR_MONMWAITUSEREN 0x00000400ULL // Enable MONITOR/MWAIT CPL>0
 #define MSR_K7_HWCR_TLBFFDIS       0x00000040ULL // Disable TLB Flush Filter
-#ifndef MSR_K7_HWCR_SMMLOCK
 #define MSR_K7_HWCR_SMMLOCK        0x00000001ULL // Lock SMM environment
-#endif

 #ifndef MSR_K8_SYSCFG
 #define MSR_K8_SYSCFG        0xc0010010
@@ -484,6 +494,8 @@ typedef enum {
 #define MSR_GH_COFVID_CONTROL    0xc0010070  // COFVID Control Register
 #define MSR_GH_COFVID_STATUS     0xc0010071  // COFVID Status Register

+#define MSR_AMD_MCA_INTR_CFG     0xc0000410  // MCA Interrupt Configuration
+
 /* SVM related MSRs */
 #define MSR_VM_CR                  0xc0010114
 #define MSR_IGNNE                  0xc0010115
@@ -606,6 +618,8 @@ typedef enum {

 #define MTRR_MASK_VALID       0x800

+typedef unsigned char MTRRType;
+
 #define MTRR_TYPE_UC          0
 #define MTRR_TYPE_WC          1
 #define MTRR_TYPE_WT          4
@@ -615,6 +629,13 @@ typedef enum {
 /* UC- is equivalent to UC, except that the MTRR values take precedence */
 #define MTRR_TYPE_UCM         7

+/*
+ * This value is marked as reserved in the Intel manual. We use it to
+ * specify that type is unknown as it is very unlikely that Intel will
+ * use this value. Note that linux is taking the same liberty.
+ */
+#define MTRR_TYPE_UNKNOW     0xff
+
 /*
 * PERF_STATUS bits
 */
@@ -633,11 +654,7 @@ typedef enum {
 /*
 * MISC_FEATURES_ENABLES bits
 */
-#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
-#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
-#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
-#endif



@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -54,7 +54,6 @@
 #define PTE_PAT         0x00000080
 #define PTE_G           0x00000100
 #define PTE_LARGE_PAT   0x00001000
-#define PTE_RSVD        CONST64(0x0008000000000000)
 #define PTE_NX          CONST64(0x8000000000000000)
 #define PTE_NX_SHIFT    63

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012,2014-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -45,6 +45,7 @@
 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS       3
 #define PERFCTR_SANDYBRIDGE_NUM_GEN_COUNTERS     8 /* When HT is disabled */
 #define PERFCTR_CORE_NUM_ARCH_EVENTS             7
+#define PERFCTR_CORE_NUM_FIXED_COUNTERS          3
 #define PERFCTR_PENTIUM4_VAL_MASK                0xffffffffffLL
 #define PERFCTR_AMD_VAL_MASK                     0xffffffffffffLL
 #define PERF_EVENT_NAME_LEN                      64
@@ -109,10 +110,15 @@
 /* AMD Performance Counter MSR Definitions */
 #define PERFCTR_AMD_PERFEVTSEL0_ADDR             0xC0010000
 #define PERFCTR_AMD_PERFCTR0_ADDR                0xC0010004
+/* AMD with PerfCtrExtCore (PERFCORE) support MSR Definitions */
 #define PERFCTR_AMD_EXT_BASE_ADDR                0xC0010200
 #define PERFCTR_AMD_EXT_EVENTSEL                 0
 #define PERFCTR_AMD_EXT_CTR                      1
 #define PERFCTR_AMD_EXT_MSR_STRIDE               2
+#define PERFCTR_AMD_AMD_EXT_CNTR_BASE_ADDR \
+        (PERFCTR_AMD_EXT_BASE_ADDR + PERFCTR_AMD_EXT_CTR)
+#define PERFCTR_AMD_AMD_EXT_EVSL_BASE_ADDR \
+        (PERFCTR_AMD_EXT_BASE_ADDR + PERFCTR_AMD_EXT_EVENTSEL)

 /* AMD Clocks */
 #define PERFCTR_AMD_CPU_CLK_UNHALTED                           0x76
@@ -847,7 +853,7 @@ PerfCtr_PTAvailable(void)
   __GET_CPUID(0, &regs);
   if (CPUID_IsVendorIntel(&regs)) {
      __GET_CPUID2(7, 0, &regs);
-      return (regs.ecx & CPUID_INTERNAL_MASK_PT) != 0;
+      return (regs.ebx & CPUID_INTERNAL_MASK_PT) != 0;
   }
   return FALSE;
 }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -134,8 +134,9 @@
 /* VMCB.nptCtl */
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)

-/* VMCB.debugCtl */
-#define SVM_VMCB_DEBUGCTL_LBR_ENABLE       (1 << 0)
+/* VMCB.virtExt */
+#define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
+#define SVM_VMCB_VIRTEXT_V_VMSAVE_VMLOAD   (1 << 1)

 /* VMCB.clean */
 #define SVMCLEAN       \
@@ -330,7 +331,7 @@ enum {

 #undef CLEANBIT
 };
-   
+
 static INLINE uint64
 SVM_ExecCtlBit(uint32 exitCode)
 {
@@ -389,7 +390,7 @@ SVM_ExecCtlBit(uint32 exitCode)
 *----------------------------------------------------------------------
 *
 * SVM_EnabledFromFeatures --
- * 
+ *
 *  Returns TRUE if SVM is enabled in the given VM control MSR bits.
 *
 *----------------------------------------------------------------------
@@ -405,7 +406,7 @@ SVM_EnabledFromFeatures(uint64 vmCR)
 *----------------------------------------------------------------------
 *
 * SVM_LockedFromFeatures --
- * 
+ *
 *  Returns TRUE if SVM is locked in the given VM control MSR bits.
 *
 *----------------------------------------------------------------------
@@ -461,32 +462,18 @@ SVM_CapableCPU(void)
 *   SVM-enabled monitor.  This function assumes that the processor is
 *   SVM_Capable().  We only support CPUs that populate the exitIntInfo
 *   field of the VMCB when IDT vectoring is interrupted by a task switch
- *   intercept.  That behavior was first introduced with Family 10H.
+ *   intercept.  That behavior was first introduced with AMD Family 10H.
+ *   As Hygon's Dhyana is a descendant of AMD's Zen microarchitecture
+ *   (Family 17H), SVM is always supported.
 *----------------------------------------------------------------------
 */
 static INLINE Bool
-SVM_SupportedVersion(uint32 version)
+SVM_SupportedVersion(CpuidVendor vendor, uint32 version)
 {
-   return CPUID_EFFECTIVE_FAMILY(version) >= CPUID_FAMILY_K8L;
+   return (vendor == CPUID_VENDOR_AMD &&
+           CPUID_EFFECTIVE_FAMILY(version) >= CPUID_FAMILY_K8L) ||
+          vendor == CPUID_VENDOR_HYGON;
 }
-
-
-/*
- *
- *----------------------------------------------------------------------
- * SVM_SupportedCPU --
- *
- *   Wrapper to call SVM_SupportedVersion() with the right
- *   parameter(s) for the current CPU.
- *----------------------------------------------------------------------
- */
-static INLINE Bool
-SVM_SupportedCPU(void)
-{
-   return SVM_SupportedVersion(__GET_EAX_FROM_CPUID(1));
-}
-
-
 #endif /* VMM */

 #endif /* _X86SVM_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -86,11 +86,12 @@ typedef uint16 Selector;
 */


+#define RSP_NUM_ENTRIES 3
 #define IST_NUM_ENTRIES 8
 #pragma pack(push, 1)
 typedef struct Task64 {
   uint32     reserved0;
-   uint64     rsp[3];   // Stacks for CPL 0-2.
+   uint64     rsp[RSP_NUM_ENTRIES];   // Stacks for CPL 0-2.
   uint64     ist[IST_NUM_ENTRIES];   // ist[0] is reserved.
   uint64     reserved1;
   uint16     reserved2;
@@ -1,6 +1,5 @@
-
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -31,6 +30,7 @@ typedef enum {
   CPUID_VENDOR_AMD,
   CPUID_VENDOR_CYRIX,
   CPUID_VENDOR_VIA,
+   CPUID_VENDOR_HYGON,
   CPUID_NUM_VENDORS
 } CpuidVendor;

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -76,7 +76,8 @@
 #endif

 /*
- * VMCS_FIELD(_name, _val, _size, _type, _index, _cache, _vvt, _access)
+ * VMCS_FIELD(_name, _val, _size, _type, _index, _cache, _vvt, _access,
+ *            _sticky)
 *    _type is one of:
 *       _TC -- control
 *       _TG -- guest-state
@@ -97,16 +98,20 @@
 *       _NA -- no access
 *       _RW -- read/write access
 *       _RO -- read-only access
+ *   _sticky is one of:
+ *       _S  -- sticky
+ *       _NS -- non-sticky
 *
+ * Fields marked sticky in the current VMCS, change only via VMWRITEs.
 */


 VMCS_SET_START(16)
 /* 16-bit control fields. */
 VMCS_GROUP_START(16, CTL)
-VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA)
-VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA)
-VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
 VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
 VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
@@ -176,16 +181,16 @@ VMCS_GROUP_END(16, DATA)

 /* 16-bit guest state. */
 VMCS_GROUP_START(16, GUEST)
-VMCS_FIELD(ES,                      0x0800, _S16, _TG,  0, _NC,  _V, _RW)
-VMCS_FIELD(CS,                      0x0802, _S16, _TG,  1, _NC,  _V, _RW)
-VMCS_FIELD(SS,                      0x0804, _S16, _TG,  2, _NC,  _V, _RW)
-VMCS_FIELD(DS,                      0x0806, _S16, _TG,  3, _NC,  _V, _RW)
-VMCS_FIELD(FS,                      0x0808, _S16, _TG,  4, _NC,  _V, _RW)
-VMCS_FIELD(GS,                      0x080A, _S16, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(LDTR,                    0x080C, _S16, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(TR,                      0x080E, _S16, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(INTR_STATUS,             0x0810, _S16, _TG,  8, _NC, _NV, _NA)
-VMCS_FIELD(PML_INDEX,               0x0812, _S16, _TG,  9, _NC, _NV, _NA)
+VMCS_FIELD(ES,                      0x0800, _S16, _TG,  0, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS,                      0x0802, _S16, _TG,  1, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS,                      0x0804, _S16, _TG,  2, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS,                      0x0806, _S16, _TG,  3, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS,                      0x0808, _S16, _TG,  4, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS,                      0x080A, _S16, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR,                    0x080C, _S16, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR,                      0x080E, _S16, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(INTR_STATUS,             0x0810, _S16, _TG,  8, _NC, _NV, _NA, _NS)
+VMCS_FIELD(PML_INDEX,               0x0812, _S16, _TG,  9, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x0814, _S16, _TG,  10)
 VMCS_UNUSED(                        0x0816, _S16, _TG,  11)
 VMCS_UNUSED(                        0x0818, _S16, _TG,  12)
@@ -212,13 +217,13 @@ VMCS_GROUP_END(16, GUEST)

 /* 16-bit host state. */
 VMCS_GROUP_START(16, HOST)
-VMCS_FIELD(HOST_ES,                 0x0C00, _S16, _TH,  0, _NC,  _V, _NA)
-VMCS_FIELD(HOST_CS,                 0x0C02, _S16, _TH,  1, _NC,  _V, _NA)
-VMCS_FIELD(HOST_SS,                 0x0C04, _S16, _TH,  2, _NC,  _V, _NA)
-VMCS_FIELD(HOST_DS,                 0x0C06, _S16, _TH,  3, _NC,  _V, _NA)
-VMCS_FIELD(HOST_FS,                 0x0C08, _S16, _TH,  4, _NC,  _V, _NA)
-VMCS_FIELD(HOST_GS,                 0x0C0A, _S16, _TH,  5, _NC,  _V, _NA)
-VMCS_FIELD(HOST_TR,                 0x0C0C, _S16, _TH,  6, _NC,  _V, _NA)
+VMCS_FIELD(HOST_ES,                 0x0C00, _S16, _TH,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CS,                 0x0C02, _S16, _TH,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SS,                 0x0C04, _S16, _TH,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_DS,                 0x0C06, _S16, _TH,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_FS,                 0x0C08, _S16, _TH,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GS,                 0x0C0A, _S16, _TH,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_TR,                 0x0C0C, _S16, _TH,  6, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x0C0E, _S16, _TH,  7)
 VMCS_UNUSED(                        0x0C10, _S16, _TH,  8)
 VMCS_UNUSED(                        0x0C12, _S16, _TH,  9)
@@ -251,32 +256,32 @@ VMCS_SET_END(16)
 VMCS_SET_START(64)
 /* 64-bit control fields. */
 VMCS_GROUP_START(64, CTL)
-VMCS_FIELD(IOBITMAPA,               0x2000, _S64, _TC,  0, _NC,  _V, _NA)
-VMCS_FIELD(IOBITMAPB,               0x2002, _S64, _TC,  1, _NC,  _V, _NA)
-VMCS_FIELD(MSRBITMAP,               0x2004, _S64, _TC,  2, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_STORE_ADDR,   0x2006, _S64, _TC,  3, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR,    0x2008, _S64, _TC,  4, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR,   0x200A, _S64, _TC,  5, _NC,  _V, _NA)
-VMCS_FIELD(EXECUTIVE_VMCS_PTR,      0x200C, _S64, _TC,  6, _NC,  _V, _NA)
-VMCS_FIELD(PML_ADDR,                0x200E, _S64, _TC,  7, _NC, _NV, _NA)
-VMCS_FIELD(TSC_OFF,                 0x2010, _S64, _TC,  8,  _C,  _V, _NA)
-VMCS_FIELD(VIRT_APIC_ADDR,          0x2012, _S64, _TC,  9, _NC,  _V, _NA)
-VMCS_FIELD(APIC_ACCESS_ADDR,        0x2014, _S64, _TC, 10, _NC,  _V, _NA)
-VMCS_FIELD(PI_DESC_ADDR,            0x2016, _S64, _TC, 11, _NC, _NV, _NA)
-VMCS_FIELD(VMFUNC_CTLS,             0x2018, _S64, _TC, 12, _NC,  _V, _NA)
-VMCS_FIELD(EPTP,                    0x201A, _S64, _TC, 13, _NC,  _V, _NA)
-VMCS_FIELD(EOI_EXIT0,               0x201C, _S64, _TC, 14, _NC, _NV, _NA)
-VMCS_FIELD(EOI_EXIT1,               0x201E, _S64, _TC, 15, _NC, _NV, _NA)
-VMCS_FIELD(EOI_EXIT2,               0x2020, _S64, _TC, 16, _NC, _NV, _NA)
-VMCS_FIELD(EOI_EXIT3,               0x2022, _S64, _TC, 17, _NC, _NV, _NA)
-VMCS_FIELD(EPTP_LIST_ADDR,          0x2024, _S64, _TC, 18, _NC,  _V, _NA)
-VMCS_FIELD(VMREAD_BITMAP,           0x2026, _S64, _TC, 19, _NC, _NV, _NA)
-VMCS_FIELD(VMWRITE_BITMAP,          0x2028, _S64, _TC, 20, _NC, _NV, _NA)
-VMCS_FIELD(VE_INFO_ADDR,            0x202A, _S64, _TC, 21, _NC,  _V, _NA)
-VMCS_FIELD(XSS_EXITING_BITMAP,      0x202C, _S64, _TC, 22, _NC,  _V, _NA)
-VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC, _NV, _NA)
+VMCS_FIELD(IOBITMAPA,               0x2000, _S64, _TC,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(IOBITMAPB,               0x2002, _S64, _TC,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(MSRBITMAP,               0x2004, _S64, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_ADDR,   0x2006, _S64, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR,    0x2008, _S64, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR,   0x200A, _S64, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(EXECUTIVE_VMCS_PTR,      0x200C, _S64, _TC,  6, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PML_ADDR,                0x200E, _S64, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(TSC_OFF,                 0x2010, _S64, _TC,  8,  _C,  _V, _NA,  _S)
+VMCS_FIELD(VIRT_APIC_ADDR,          0x2012, _S64, _TC,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(APIC_ACCESS_ADDR,        0x2014, _S64, _TC, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PI_DESC_ADDR,            0x2016, _S64, _TC, 11, _NC, _NV, _NA,  _S)
+VMCS_FIELD(VMFUNC_CTLS,             0x2018, _S64, _TC, 12, _NC,  _V, _NA,  _S)
+VMCS_FIELD(EPTP,                    0x201A, _S64, _TC, 13, _NC,  _V, _NA, _NS)
+VMCS_FIELD(EOI_EXIT0,               0x201C, _S64, _TC, 14, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EOI_EXIT1,               0x201E, _S64, _TC, 15, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EOI_EXIT2,               0x2020, _S64, _TC, 16, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EOI_EXIT3,               0x2022, _S64, _TC, 17, _NC, _NV, _NA,  _S)
+VMCS_FIELD(EPTP_LIST_ADDR,          0x2024, _S64, _TC, 18, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMREAD_BITMAP,           0x2026, _S64, _TC, 19, _NC, _NV, _NA,  _S)
+VMCS_FIELD(VMWRITE_BITMAP,          0x2028, _S64, _TC, 20, _NC, _NV, _NA,  _S)
+VMCS_FIELD(VE_INFO_ADDR,            0x202A, _S64, _TC, 21, _NC,  _V, _NA,  _S)
+VMCS_FIELD(XSS_EXITING_BITMAP,      0x202C, _S64, _TC, 22, _NC,  _V, _NA,  _S)
+VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2030, _S64, _TC, 24)
-VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA)
+VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA,  _S)
 VMCS_UNUSED(                        0x2034, _S64, _TC, 26)
 VMCS_UNUSED(                        0x2036, _S64, _TC, 27)
 VMCS_UNUSED(                        0x2038, _S64, _TC, 28)
@@ -287,7 +292,7 @@ VMCS_GROUP_END(64, CTL)

 /* 64-bit read-only data field. */
 VMCS_GROUP_START(64, DATA)
-VMCS_FIELD(PHYSADDR,                0x2400, _S64, _TD,  0, _NC,  _V, _RO)
+VMCS_FIELD(PHYSADDR,                0x2400, _S64, _TD,  0, _NC,  _V, _RO, _NS)
 VMCS_UNUSED(                        0x2402, _S64, _TD,  1)
 VMCS_UNUSED(                        0x2404, _S64, _TD,  2)
 VMCS_UNUSED(                        0x2406, _S64, _TD,  3)
@@ -323,16 +328,16 @@ VMCS_GROUP_END(64, DATA)

 /* 64-bit guest state. */
 VMCS_GROUP_START(64, GUEST)
-VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA)
-VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA)
-VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA)
-VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA)
-VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA)
-VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW)
-VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA, _NS)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA, _NS)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA, _NS)
 VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
 VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
 VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
@@ -359,9 +364,9 @@ VMCS_GROUP_END(64, GUEST)

 /* 64-bit host state. */
 VMCS_GROUP_START(64, HOST)
-VMCS_FIELD(HOST_PAT,                0x2C00, _S64, _TH,  0, _NC,  _V, _NA)
-VMCS_FIELD(HOST_EFER,               0x2C02, _S64, _TH,  1, _NC,  _V, _NA)
-VMCS_FIELD(HOST_PGC,                0x2C04, _S64, _TH,  2, _NC,  _V, _NA)
+VMCS_FIELD(HOST_PAT,                0x2C00, _S64, _TH,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_EFER,               0x2C02, _S64, _TH,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_PGC,                0x2C04, _S64, _TH,  2, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x2C06, _S64, _TH,  3)
 VMCS_UNUSED(                        0x2C08, _S64, _TH,  4)
 VMCS_UNUSED(                        0x2C0A, _S64, _TH,  5)
@@ -398,24 +403,24 @@ VMCS_SET_END(64)
 VMCS_SET_START(32)
 /* 32-bit control fields. */
 VMCS_GROUP_START(32, CTL)
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA)
-VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA,  _S)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA, _NS)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA,  _S)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA,  _S)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA,  _S)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA,  _S)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
 VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
 VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
@@ -434,14 +439,14 @@ VMCS_GROUP_END(32, CTL)

 /* 32-bit read-only data fields. */
 VMCS_GROUP_START(32, DATA)
-VMCS_FIELD(VMINSTR_ERR,             0x4400, _S32, _TD,  0, _NC,  _V, _NA)
-VMCS_FIELD(EXIT_REASON,             0x4402, _S32, _TD,  1, _NC,  _V, _RO)
-VMCS_FIELD(EXIT_INTR_INFO,          0x4404, _S32, _TD,  2,  _C,  _V, _RO)
-VMCS_FIELD(EXIT_INTR_ERR,           0x4406, _S32, _TD,  3, _NC,  _V, _RO)
-VMCS_FIELD(IDTVEC_INFO,             0x4408, _S32, _TD,  4, _NC,  _V, _RO)
-VMCS_FIELD(IDTVEC_ERR,              0x440A, _S32, _TD,  5, _NC,  _V, _RO)
-VMCS_FIELD(INSTRLEN,                0x440C, _S32, _TD,  6, _NC,  _V, _RO)
-VMCS_FIELD(INSTR_INFO,              0x440E, _S32, _TD,  7, _NC,  _V, _RO)
+VMCS_FIELD(VMINSTR_ERR,             0x4400, _S32, _TD,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(EXIT_REASON,             0x4402, _S32, _TD,  1, _NC,  _V, _RO, _NS)
+VMCS_FIELD(EXIT_INTR_INFO,          0x4404, _S32, _TD,  2,  _C,  _V, _RO, _NS)
+VMCS_FIELD(EXIT_INTR_ERR,           0x4406, _S32, _TD,  3, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IDTVEC_INFO,             0x4408, _S32, _TD,  4, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IDTVEC_ERR,              0x440A, _S32, _TD,  5, _NC,  _V, _RO, _NS)
+VMCS_FIELD(INSTRLEN,                0x440C, _S32, _TD,  6, _NC,  _V, _RO, _NS)
+VMCS_FIELD(INSTR_INFO,              0x440E, _S32, _TD,  7, _NC,  _V, _RO, _NS)
 VMCS_UNUSED(                        0x4410, _S32, _TD,  8)
 VMCS_UNUSED(                        0x4412, _S32, _TD,  9)
 VMCS_UNUSED(                        0x4414, _S32, _TD,  10)
@@ -470,30 +475,30 @@ VMCS_GROUP_END(32, DATA)

 /* 32-bit guest state. */
 VMCS_GROUP_START(32, GUEST)
-VMCS_FIELD(ES_LIMIT,                0x4800, _S32, _TG,  0, _NC,  _V, _RW)
-VMCS_FIELD(CS_LIMIT,                0x4802, _S32, _TG,  1, _NC,  _V, _RW)
-VMCS_FIELD(SS_LIMIT,                0x4804, _S32, _TG,  2, _NC,  _V, _RW)
-VMCS_FIELD(DS_LIMIT,                0x4806, _S32, _TG,  3, _NC,  _V, _RW)
-VMCS_FIELD(FS_LIMIT,                0x4808, _S32, _TG,  4, _NC,  _V, _RW)
-VMCS_FIELD(GS_LIMIT,                0x480A, _S32, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(LDTR_LIMIT,              0x480C, _S32, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(TR_LIMIT,                0x480E, _S32, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(GDTR_LIMIT,              0x4810, _S32, _TG,  8, _NC,  _V, _RW)
-VMCS_FIELD(IDTR_LIMIT,              0x4812, _S32, _TG,  9, _NC,  _V, _RW)
-VMCS_FIELD(ES_AR,                   0x4814, _S32, _TG, 10, _NC,  _V, _RW)
-VMCS_FIELD(CS_AR,                   0x4816, _S32, _TG, 11, _NC,  _V, _RW)
-VMCS_FIELD(SS_AR,                   0x4818, _S32, _TG, 12, _NC,  _V, _RW)
-VMCS_FIELD(DS_AR,                   0x481A, _S32, _TG, 13, _NC,  _V, _RW)
-VMCS_FIELD(FS_AR,                   0x481C, _S32, _TG, 14, _NC,  _V, _RW)
-VMCS_FIELD(GS_AR,                   0x481E, _S32, _TG, 15, _NC,  _V, _RW)
-VMCS_FIELD(LDTR_AR,                 0x4820, _S32, _TG, 16, _NC,  _V, _RW)
-VMCS_FIELD(TR_AR,                   0x4822, _S32, _TG, 17, _NC,  _V, _RW)
-VMCS_FIELD(HOLDOFF,                 0x4824, _S32, _TG, 18,  _C,  _V, _RW)
-VMCS_FIELD(ACTSTATE,                0x4826, _S32, _TG, 19, _NC,  _V, _NA)
-VMCS_FIELD(SMBASE,                  0x4828, _S32, _TG, 20, _NC,  _V, _NA)
-VMCS_FIELD(SYSENTER_CS,             0x482A, _S32, _TG, 21, _NC,  _V, _RW)
+VMCS_FIELD(ES_LIMIT,                0x4800, _S32, _TG,  0, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS_LIMIT,                0x4802, _S32, _TG,  1, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS_LIMIT,                0x4804, _S32, _TG,  2, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS_LIMIT,                0x4806, _S32, _TG,  3, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS_LIMIT,                0x4808, _S32, _TG,  4, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS_LIMIT,                0x480A, _S32, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR_LIMIT,              0x480C, _S32, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR_LIMIT,                0x480E, _S32, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GDTR_LIMIT,              0x4810, _S32, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(IDTR_LIMIT,              0x4812, _S32, _TG,  9, _NC,  _V, _RW, _NS)
+VMCS_FIELD(ES_AR,                   0x4814, _S32, _TG, 10, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS_AR,                   0x4816, _S32, _TG, 11, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS_AR,                   0x4818, _S32, _TG, 12, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS_AR,                   0x481A, _S32, _TG, 13, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS_AR,                   0x481C, _S32, _TG, 14, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS_AR,                   0x481E, _S32, _TG, 15, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR_AR,                 0x4820, _S32, _TG, 16, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR_AR,                   0x4822, _S32, _TG, 17, _NC,  _V, _RW, _NS)
+VMCS_FIELD(HOLDOFF,                 0x4824, _S32, _TG, 18,  _C,  _V, _RW, _NS)
+VMCS_FIELD(ACTSTATE,                0x4826, _S32, _TG, 19, _NC,  _V, _NA, _NS)
+VMCS_FIELD(SMBASE,                  0x4828, _S32, _TG, 20, _NC,  _V, _NA, _NS)
+VMCS_FIELD(SYSENTER_CS,             0x482A, _S32, _TG, 21, _NC,  _V, _RW, _NS)
 VMCS_UNUSED(                        0x482C, _S32, _TG, 22)
-VMCS_FIELD(TIMER,                   0x482E, _S32, _TG, 23, _NC, _NV, _NA)
+VMCS_FIELD(TIMER,                   0x482E, _S32, _TG, 23, _NC, _NV, _NA, _NS)
 VMCS_UNUSED(                        0x4830, _S32, _TG, 24)
 VMCS_UNUSED(                        0x4832, _S32, _TG, 25)
 VMCS_UNUSED(                        0x4834, _S32, _TG, 26)
@@ -507,7 +512,7 @@ VMCS_GROUP_END(32, GUEST)

 /* 32-bit host state. */
 VMCS_GROUP_START(32, HOST)
-VMCS_FIELD(HOST_SYSENTER_CS,        0x4C00, _S32, _TH,  0, _NC,  _V, _NA)
+VMCS_FIELD(HOST_SYSENTER_CS,        0x4C00, _S32, _TH,  0, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x4C02, _S32, _TH,  1)
 VMCS_UNUSED(                        0x4C04, _S32, _TH,  2)
 VMCS_UNUSED(                        0x4C06, _S32, _TH,  3)
@@ -546,14 +551,14 @@ VMCS_SET_END(32)
 VMCS_SET_START(NAT)
 /* natural-width control fields. */
 VMCS_GROUP_START(NAT, CTL)
-VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA)
-VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA)
-VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA)
-VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA,  _S)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
 VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
 VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
@@ -581,12 +586,12 @@ VMCS_UNUSED(                        0x603E,  _SN, _TC,  31)
 VMCS_GROUP_END(NAT, CTL)

 VMCS_GROUP_START(NAT, DATA)
-VMCS_FIELD(EXIT_QUAL,               0x6400,  _SN, _TD,  0, _NC,  _V, _RO)
-VMCS_FIELD(IO_RCX,                  0x6402,  _SN, _TD,  1, _NC,  _V, _RO)
-VMCS_FIELD(IO_RSI,                  0x6404,  _SN, _TD,  2, _NC,  _V, _RO)
-VMCS_FIELD(IO_RDI,                  0x6406,  _SN, _TD,  3, _NC,  _V, _RO)
-VMCS_FIELD(IO_RIP,                  0x6408,  _SN, _TD,  4, _NC,  _V, _RO)
-VMCS_FIELD(LINEAR_ADDR,             0x640A,  _SN, _TD,  5, _NC,  _V, _RO)
+VMCS_FIELD(EXIT_QUAL,               0x6400,  _SN, _TD,  0, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RCX,                  0x6402,  _SN, _TD,  1, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RSI,                  0x6404,  _SN, _TD,  2, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RDI,                  0x6406,  _SN, _TD,  3, _NC,  _V, _RO, _NS)
+VMCS_FIELD(IO_RIP,                  0x6408,  _SN, _TD,  4, _NC,  _V, _RO, _NS)
+VMCS_FIELD(LINEAR_ADDR,             0x640A,  _SN, _TD,  5, _NC,  _V, _RO, _NS)
 VMCS_UNUSED(                        0x640C,  _SN, _TD,  6)
 VMCS_UNUSED(                        0x640E,  _SN, _TD,  7)
 VMCS_UNUSED(                        0x6410,  _SN, _TD,  8)
@@ -617,26 +622,26 @@ VMCS_GROUP_END(NAT, DATA)

 /* natural-width guest state. */
 VMCS_GROUP_START(NAT, GUEST)
-VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA)
-VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW)
-VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW)
-VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW)
-VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW)
-VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW)
-VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW)
-VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW)
-VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW)
-VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW)
-VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW)
-VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW)
-VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW)
-VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA)
-VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW)
-VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW)
-VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW)
-VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW)
-VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW)
-VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA, _NS)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW, _NS)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW, _NS)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW, _NS)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW, _NS)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW, _NS)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW, _NS)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW, _NS)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW, _NS)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA, _NS)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW, _NS)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW, _NS)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW, _NS)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW, _NS)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW, _NS)
 VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
 VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
 VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
@@ -653,18 +658,18 @@ VMCS_GROUP_END(NAT, GUEST)

 /* natural-width host state. */
 VMCS_GROUP_START(NAT, HOST)
-VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA)
-VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA)
-VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA)
-VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA)
-VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA)
-VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA)
-VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA,  _S)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA,  _S)
 VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
 VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
 VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2004-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2004-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -286,7 +286,7 @@ enum {
   VMX_CPU2(INVPCID,            12)                      \
   VMX_CPU2(VMFUNC,             13)                      \
   VMX_CPU2(VMCS_SHADOW,        14)                      \
-   VMX_CPU2(ENCL,               15)                      \
+   VMX_CPU2(ENCLS,              15)                      \
   VMX_CPU2(RDSEED,             16)                      \
   VMX_CPU2(PML,                17)                      \
   VMX_CPU2(EPT_VIOL_VE,        18)                      \
@@ -640,7 +640,7 @@ enum {
 #define VT_HOLDOFF_MOVSS       0x00000002
 #define VT_HOLDOFF_SMI         0x00000004
 #define VT_HOLDOFF_NMI         0x00000008
-#define VT_HOLDOFF_ENCLAVE     0x00000010
+#define VT_ENCLAVE_INTR        0x00000010
 #define VT_HOLDOFF_INST        (VT_HOLDOFF_STI | VT_HOLDOFF_MOVSS)
 #define VT_HOLDOFF_RSV         0xFFFFFFE0

@@ -668,6 +668,7 @@ enum {
 #define VT_EPT_QUAL_GUEST_RW           (1 << 10)
 #define VT_EPT_QUAL_GUEST_NX           (1 << 11)
 #define VT_EPT_QUAL_NMIUNMASK          (1 << 12)
+#define VT_EPT_QUAL_SYNTH_PML_FULL     (1 << 31)


 /* IOIO Qualification */
@@ -753,6 +754,10 @@ enum {
 #define VT_TSQUAL_JMP   2
 #define VT_TSQUAL_GATE  3

+/* PML Constants */
+#define VT_MAX_PML_INDEX   511
+#define VT_PML_ENTRY_MASK  (~(QWORD(0, PAGE_MASK)))
+
 typedef union {
   struct {
      unsigned selVal:16;
@@ -815,6 +820,23 @@ typedef union {
   uint32 flat;
 } VTIOQualifier;

+#define VT_APICACCESSQUAL_TYPE_LINEAR_READ  0
+#define VT_APICACCESSQUAL_TYPE_LINEAR_WRITE 1
+#define VT_APICACCESSQUAL_TYPE_LINEAR_INSTR 2
+#define VT_APICACCESSQUAL_TYPE_LINEAR_EVENT 3
+#define VT_APICACCESSQUAL_TYPE_PHYS_EVENT   10
+#define VT_APICACCESSQUAL_TYPE_PHYS_INSTR   15
+
+/* APIC Access intercept qualifier */
+typedef union {
+   struct {
+      uint64 offset:12;
+      uint64 type  :4;
+      uint64 rsvd  :48;
+   } bits;
+   uint64 flat;
+} VTAPICAccessQualifier;
+
 #define VT_IINFO_SCALE1    0
 #define VT_IINFO_SCALE2    1
 #define VT_IINFO_SCALE4    2
@@ -1081,11 +1103,8 @@ VT_SupportedCPU(void)
                                      __GET_MSR(MSR_VMX_TRUE_EXIT_CTLS),
                                      __GET_MSR(MSR_VMX_BASIC));
   } else {
-      return VT_SupportedFromFeatures(__GET_MSR(MSR_VMX_PINBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_PROCBASED_CTLS),
-                                      __GET_MSR(MSR_VMX_ENTRY_CTLS),
-                                      __GET_MSR(MSR_VMX_EXIT_CTLS),
-                                      __GET_MSR(MSR_VMX_BASIC));
+      /* Bug 1914425 - VMM no longer supports CPUs without TRUE_xxx_CTLS */
+      return FALSE;
   }
 }

@@ -1141,5 +1160,4 @@ VT_ConfigMSRNum(unsigned index)
   return MSR_VMX_BASIC + index;
 }

-
 #endif /* _X86VT_H_ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -65,8 +65,8 @@
 #include "hostif_priv.h"
 #include "vmhost.h"

-static void LinuxDriverQueue(VMLinux *vmLinux);
-static void LinuxDriverDequeue(VMLinux *vmLinux);
+static void LinuxDriverQueue(Device *device);
+static void LinuxDriverDequeue(Device *device);
 static Bool LinuxDriverCheckPadding(void);

 #define VMMON_UNKNOWN_SWAP_SIZE -1ULL
@@ -284,6 +284,7 @@ init_module(void)
   }

   CPUID_Init();
+   Vmx86_CacheNXState();

   if (!Task_Initialize()) {
      return -ENOEXEC;
@@ -404,18 +405,18 @@ static int
 LinuxDriver_Open(struct inode *inode, // IN
                 struct file *filp)   // IN
 {
-   VMLinux *vmLinux;
+   Device *device;

-   vmLinux = kmalloc(sizeof *vmLinux, GFP_KERNEL);
-   if (vmLinux == NULL) {
+   device = kmalloc(sizeof *device, GFP_KERNEL);
+   if (device == NULL) {
      return -ENOMEM;
   }
-   memset(vmLinux, 0, sizeof *vmLinux);
+   memset(device, 0, sizeof *device);

-   sema_init(&vmLinux->lock4Gb, 1);
+   sema_init(&device->lock4Gb, 1);

-   filp->private_data = vmLinux;
-   LinuxDriverQueue(vmLinux);
+   filp->private_data = device;
+   LinuxDriverQueue(device);

   Vmx86_Open();

@@ -517,17 +518,17 @@ LinuxDriverAllocPages(unsigned int gfpFlag, // IN
 */

 static void
-LinuxDriverDestructor4Gb(VMLinux *vmLinux) // IN
+LinuxDriverDestructor4Gb(Device *device) // IN
 {
   unsigned int pg;

-   if (!vmLinux->size4Gb) {
+   if (!device->size4Gb) {
      return;
   }
-   for (pg = 0; pg < vmLinux->size4Gb; pg++) {
-      put_page(vmLinux->pages4Gb[pg]);
+   for (pg = 0; pg < device->size4Gb; pg++) {
+      put_page(device->pages4Gb[pg]);
   }
-   vmLinux->size4Gb = 0;
+   device->size4Gb = 0;
 }


@@ -547,15 +548,15 @@ static int
 LinuxDriver_Close(struct inode *inode, // IN
                  struct file *filp)   // IN
 {
-   VMLinux *vmLinux;
+   Device *device;

-   vmLinux = (VMLinux *)filp->private_data;
-   ASSERT(vmLinux);
+   device = (Device *)filp->private_data;
+   ASSERT(device);

-   LinuxDriverDequeue(vmLinux);
-   if (vmLinux->vm != NULL) {
-      Vmx86_ReleaseVM(vmLinux->vm);
-      vmLinux->vm = NULL;
+   LinuxDriverDequeue(device);
+   if (device->vm != NULL) {
+      Vmx86_ReleaseVM(device->vm);
+      device->vm = NULL;
   }

   Vmx86_Close();
@@ -566,9 +567,9 @@ LinuxDriver_Close(struct inode *inode, // IN
    * uses it anymore, and we do not need to hold the semaphore.
    */

-   LinuxDriverDestructor4Gb(vmLinux);
+   LinuxDriverDestructor4Gb(device);

-   kfree(vmLinux);
+   kfree(device);
   filp->private_data = NULL;

   return 0;
@@ -605,16 +606,16 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
   struct vm_area_struct *vma = fault->vma;
 #endif
-   VMLinux *vmLinux = (VMLinux *) vma->vm_file->private_data;
+   Device *device = (Device *) vma->vm_file->private_data;
   unsigned long pg;
   struct page* page;

   pg = fault->pgoff;
   pg = VMMON_MAP_OFFSET(pg);
-   if (pg >= vmLinux->size4Gb) {
+   if (pg >= device->size4Gb) {
      return VM_FAULT_SIGBUS;
   }
-   page = vmLinux->pages4Gb[pg];
+   page = device->pages4Gb[pg];
   get_page(page);
   fault->page = page;
   return 0;
@@ -640,7 +641,7 @@ LinuxDriverFault(struct vm_area_struct *vma, //IN
 *-----------------------------------------------------------------------------
 */

-static int LinuxDriverAllocContig(VMLinux *vmLinux,
+static int LinuxDriverAllocContig(Device *device,
                                  struct vm_area_struct *vma,
                                  unsigned long off,
                                  unsigned long size)
@@ -686,9 +687,9 @@ static int LinuxDriverAllocContig(VMLinux *vmLinux,
      return -ENOMEM;
   }
   /* Sorry. Only one mmap per one open. */
-   down(&vmLinux->lock4Gb);
-   if (vmLinux->size4Gb) {
-      up(&vmLinux->lock4Gb);
+   down(&device->lock4Gb);
+   if (device->size4Gb) {
+      up(&device->lock4Gb);
      return -EINVAL;
   }
   vmaAllocSize = 1 << vmaOrder;
@@ -696,18 +697,18 @@ static int LinuxDriverAllocContig(VMLinux *vmLinux,
      int err;

      err = LinuxDriverAllocPages(gfpFlag, vmaOrder,
-                                  vmLinux->pages4Gb + i, size - i);
+                                  device->pages4Gb + i, size - i);
      if (err) {
         while (i > 0) {
-            put_page(vmLinux->pages4Gb[--i]);
+            put_page(device->pages4Gb[--i]);
         }
-         up(&vmLinux->lock4Gb);
+         up(&device->lock4Gb);

         return err;
      }
   }
-   vmLinux->size4Gb = size;
-   up(&vmLinux->lock4Gb);
+   device->size4Gb = size;
+   up(&device->lock4Gb);
   vma->vm_ops = &vmuser_mops;

   return 0;
@@ -737,7 +738,7 @@ static int
 LinuxDriverMmap(struct file *filp,
                struct vm_area_struct *vma)
 {
-   VMLinux *vmLinux = (VMLinux *) filp->private_data;
+   Device *device = (Device *) filp->private_data;
   unsigned long size;
   int err;

@@ -752,10 +753,10 @@ LinuxDriverMmap(struct file *filp,
   if (size < 1) {
      return -EINVAL;
   }
-   if (vmLinux->vm) {
+   if (device->vm) {
      err = -EINVAL;
   } else {
-      err = LinuxDriverAllocContig(vmLinux, vma, vma->vm_pgoff, size);
+      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
   }
   if (err) {
      return err;
@@ -981,7 +982,7 @@ LinuxDriverReadTSC(void *data,   // OUT: TSC values
 *-----------------------------------------------------------------------------
 */

-__always_inline static Bool
+__attribute__((always_inline)) static Bool
 LinuxDriverSyncReadTSCs(uint64 *delta) // OUT: TSC max - TSC min
 {
   TSCDelta tscDelta;
@@ -1032,16 +1033,16 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
                  u_int iocmd,          // IN:
                  unsigned long ioarg)  // IN:
 {
-   VMLinux *vmLinux = (VMLinux *) filp->private_data;
+   Device *device = (Device *) filp->private_data;
   int retval = 0;
   Vcpuid vcpuid;
   VMDriver *vm;

-   if (vmLinux == NULL) {
+   if (device == NULL) {
      return -EINVAL;
   }

-   vm = vmLinux->vm;
+   vm = device->vm;

   /*
    * Validate the VM pointer for those IOCTLs that require it.
@@ -1093,12 +1094,12 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      if (retval != 0) {
         break;
      }
-      vm = Vmx86_CreateVM(args.bsBlob, args.bsBlobSize);
+      vm = Vmx86_CreateVM(args.bsBlob, args.bsBlobSize, args.numVCPUs);

      if (vm == NULL) {
         retval = -ENOMEM;
      } else {
-         vmLinux->vm = vm;
+         device->vm = vm;
         args.vmid = vm->userID;
         retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args);
      }
@@ -1107,14 +1108,19 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:

   case IOCTL_VMX86_PROCESS_BOOTSTRAP: {
      VMProcessBootstrapBlock *args;
+      VA64 uAddr;
      Bool res;

+      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
+      if (retval != 0) {
+         break;
+      }
      args = HostIF_AllocKernelMem(sizeof *args, TRUE);
      if (args == NULL) {
         retval = -ENOMEM;
         break;
      }
-      retval = HostIF_CopyFromUser(args, (VA64)ioarg, sizeof *args);
+      retval = HostIF_CopyFromUser(args, uAddr, sizeof *args);
      if (retval != 0) {
         HostIF_FreeKernelMem(args);
         break;
@@ -1134,22 +1140,27 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

   case IOCTL_VMX86_RELEASE_VM:
-      vmLinux->vm = NULL;
+      device->vm = NULL;
      Vmx86_ReleaseVM(vm);
      break;

   case IOCTL_VMX86_INIT_VM: {
      InitBlock *initParams;
+      VA64 uAddr;

+      retval = HostIF_CopyFromUser(&uAddr, (VA64)ioarg, sizeof uAddr);
+      if (retval != 0) {
+         break;
+      }
      initParams = HostIF_AllocKernelMem(sizeof *initParams, TRUE);
      if (initParams == NULL) {
         retval = -ENOMEM;
         break;
      }
-      retval = HostIF_CopyFromUser(initParams, ioarg, sizeof *initParams);
+      retval = HostIF_CopyFromUser(initParams, uAddr, sizeof *initParams);
      if (retval == 0) {
         if (Vmx86_InitVM(vm, initParams) == 0) {
-            retval = HostIF_CopyToUser(ioarg, initParams, sizeof *initParams);
+            retval = HostIF_CopyToUser(uAddr, initParams, sizeof *initParams);
         } else {
            retval = -EINVAL;
         }
@@ -1176,18 +1187,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
 #endif
      break;

-   case IOCTL_VMX86_LOCK_PAGE: {
-      VMLockPage args;
-
-      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
-      if (retval) {
-         break;
-      }
-      args.ret.status = Vmx86_LockPage(vm, args.uAddr, FALSE, &args.ret.mpn);
-      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
-      break;
-   }
-
+   case IOCTL_VMX86_LOCK_PAGE:
   case IOCTL_VMX86_LOCK_PAGE_NEW: {
      VMLockPage args;

@@ -1195,7 +1195,9 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      if (retval) {
         break;
      }
-      args.ret.status = Vmx86_LockPage(vm, args.uAddr, TRUE, &args.ret.mpn);
+      args.ret.status = Vmx86_LockPage(vm, args.uAddr,
+                                       iocmd == IOCTL_VMX86_LOCK_PAGE_NEW,
+                                       &args.ret.mpn);
      retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
      break;
   }
@@ -1356,6 +1358,16 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      retval = -Vmx86_SetHostClockRate(vm, (unsigned)ioarg);
      break;

+   case IOCTL_VMX86_SEND_ONE_IPI: {
+      Vcpuid v = ioarg;
+      if (v < vm->numVCPUs) {
+         HostIF_OneIPI(vm, v);
+      } else {
+         retval = -EINVAL;
+      }
+      break;
+   }
+
   case IOCTL_VMX86_SEND_IPI: {
      VCPUSet ipiTargets;

@@ -1371,14 +1383,8 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   case IOCTL_VMX86_GET_IPI_VECTORS: {
      IPIVectors ipiVectors;

-      ipiVectors.hostIPIVectors[0] = CALL_FUNCTION_VECTOR;
-#ifdef CALL_FUNCTION_SINGLE_VECTOR
-      ipiVectors.hostIPIVectors[1] = CALL_FUNCTION_SINGLE_VECTOR;
-#else
-      ipiVectors.hostIPIVectors[1] = 0;
-#endif
-      ipiVectors.monitorIPIVector = monitorIPIVector;
-      ipiVectors.hvIPIVector      = hvIPIVector;
+      ipiVectors.monitorIPIVector = HostIF_GetMonitorIPIVector();
+      ipiVectors.hvIPIVector      = HostIF_GetHVIPIVector();

      retval = HostIF_CopyToUser(ioarg, &ipiVectors, sizeof ipiVectors);
      break;
@@ -1484,22 +1490,16 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

-   case IOCTL_VMX86_ALLOC_LOCKED_PAGES:
-   case IOCTL_VMX86_FREE_LOCKED_PAGES: {
+   case IOCTL_VMX86_ALLOC_LOCKED_PAGES: {
         VMMPNList req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
         if (retval) {
-           break;
-         }
-         if (iocmd == IOCTL_VMX86_ALLOC_LOCKED_PAGES) {
-            retval = Vmx86_AllocLockedPages(vm, req.mpnList,
-                                            req.mpnCount, FALSE,
-                                            req.ignoreLimits);
-         } else {
-            retval = Vmx86_FreeLockedPages(vm, req.mpnList,
-                                           req.mpnCount, FALSE);
+            break;
         }
+         retval = Vmx86_AllocLockedPages(vm, req.mpnList,
+                                         req.mpnCount, FALSE,
+                                         req.ignoreLimits);
         break;
      }

@@ -1517,7 +1517,6 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

   case IOCTL_VMX86_READ_PAGE: {
-         void *tempPage;
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1525,19 +1524,8 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
            break;
         }

-         tempPage = HostIF_AllocPage();
-         if (tempPage == NULL) {
-            retval = -ENOMEM;
-            break;
-         }
-
-         retval = HostIF_ReadPhysical(vm, MPN_2_MA(req.mpn),
-                                      PtrToVA64(tempPage), TRUE, PAGE_SIZE);
-         if (retval == 0) {
-            retval = HostIF_CopyToUser(req.uAddr, tempPage, PAGE_SIZE);
-         }
-
-         HostIF_FreePage(tempPage);
+         retval = HostIF_ReadPhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE,
+                                      PAGE_SIZE);
         break;
      }

@@ -1621,6 +1609,20 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

+   case IOCTL_VMX86_GET_MONITOR_CONTEXT: {
+      VMMonContext args;
+      retval = HostIF_CopyFromUser(&args, ioarg, sizeof args);
+      if (retval != 0) {
+         break;
+      }
+      if (!Vmx86_GetMonitorContext(vm, args.vcpuid, &args.context)) {
+         retval = -EINVAL;
+      } else {
+         retval = HostIF_CopyToUser(ioarg, &args, sizeof args);
+      }
+      break;
+   }
+
   default:
      Warning("Unknown ioctl %d\n", iocmd);
      retval = -EINVAL;
@@ -1636,7 +1638,7 @@ exit:
 *
 * LinuxDriverQueue --
 *
- *      add the vmLinux to the global queue
+ *      add the device to the global queue
 *
 * Results:
 *
@@ -1648,7 +1650,7 @@ exit:
 */

 static void
-LinuxDriverQueue(VMLinux *vmLinux)  // IN/OUT:
+LinuxDriverQueue(Device *device)  // IN/OUT:
 {
   /*
    * insert in global vm queue
@@ -1656,8 +1658,8 @@ LinuxDriverQueue(VMLinux *vmLinux)  // IN/OUT:

   HostIF_GlobalLock(12);

-   vmLinux->next = linuxState.head;
-   linuxState.head = vmLinux;
+   device->next = linuxState.head;
+   linuxState.head = device;

   HostIF_GlobalUnlock(12);
 }
@@ -1680,16 +1682,16 @@ LinuxDriverQueue(VMLinux *vmLinux)  // IN/OUT:
 */

 static void
-LinuxDriverDequeue(VMLinux *vmLinux)  // IN/OUT:
+LinuxDriverDequeue(Device *device)  // IN/OUT:
 {
-   VMLinux **p;
+   Device **p;

   HostIF_GlobalLock(13);
-   for (p = &linuxState.head; *p != vmLinux; p = &(*p)->next) {
+   for (p = &linuxState.head; *p != device; p = &(*p)->next) {
      ASSERT(*p != NULL);
   }
-   *p = vmLinux->next;
-   vmLinux->next = NULL;
+   *p = device->next;
+   device->next = NULL;
   HostIF_GlobalUnlock(13);
 }

@@ -53,8 +53,8 @@ struct VMDriver;
 /* 16 pages (64KB) looks as a good limit for one allocation */
 #define VMMON_MAX_LOWMEM_PAGES  16

-typedef struct VMLinux {
-   struct VMLinux *next;
+typedef struct Device {
+   struct Device   *next;
   struct VMDriver *vm;

   /*
@@ -66,7 +66,7 @@ typedef struct VMLinux {
   struct semaphore lock4Gb;
   unsigned int size4Gb;
   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
-} VMLinux;
+} Device;


 /*
@@ -82,7 +82,7 @@ typedef struct VMXLinuxState {
   struct miscdevice misc;
   char deviceName[VM_DEVICE_NAME_SIZE];
   char buf[LINUXLOG_BUFFER_SIZE];
-   VMLinux *head;
+   Device *head;

   struct task_struct *fastClockThread;
   unsigned fastClockRate;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -74,7 +74,6 @@

 #include "pgtbl.h"
 #include "versioned_atomic.h"
-#include "compat_poll.h"

 #if !defined(CONFIG_HIGH_RES_TIMERS)
 #error CONFIG_HIGH_RES_TIMERS required for acceptable performance
@@ -178,6 +177,8 @@ static struct {
   put_page(pfn_to_page(_pfn));                          \
 } while (0)

+static void UnlockEntry(void *clientData, MemTrackEntry *entryPtr);
+
 uint8 monitorIPIVector;
 uint8 hvIPIVector;

@@ -341,6 +342,7 @@ Bool
 HostIF_PrepareWaitForThreads(VMDriver *vm,     // IN:
                             Vcpuid currVcpu)  // IN:
 {
+   ASSERT(currVcpu < vm->numVCPUs);
   set_current_state(TASK_INTERRUPTIBLE);
   vm->vmhost->vcpuSemaTask[currVcpu] = current;
   return FALSE;
@@ -393,6 +395,7 @@ void
 HostIF_CancelWaitForThreads(VMDriver *vm,     // IN:
                            Vcpuid currVcpu)  // IN:
 {
+   ASSERT(currVcpu < vm->numVCPUs);
   vm->vmhost->vcpuSemaTask[currVcpu] = NULL;
   set_current_state(TASK_RUNNING);
 }
@@ -431,17 +434,19 @@ HostIF_WakeUpYielders(VMDriver *vm,     // IN:
    * fully coherent snapshot, as long as the subset copy-and-remove is atomic
    * so no VCPU added is lost entirely.
    */
-
+   ASSERT(currVcpu < vm->numVCPUs);
   VCPUSet_Empty(&req);
-   FOR_EACH_SUBSET_IN_SET(subIdx) {
+   FOR_EACH_POPULATED_SUBSET_IN_SET(subIdx, vm->numVCPUs) {
      subset = VCPUSet_AtomicReadWriteSubset(&vm->crosscallWaitSet[currVcpu],
                                             0, subIdx);
      VCPUSet_UnionSubset(&req, subset, subIdx);
-   } ROF_EACH_SUBSET_IN_SET();
+   } ROF_EACH_POPULATED_SUBSET_IN_SET();

   preempt_disable();
   while ((vcpuid = VCPUSet_FindFirst(&req)) != VCPUID_INVALID) {
-      struct task_struct *t = vm->vmhost->vcpuSemaTask[vcpuid];
+      struct task_struct *t;
+      ASSERT(vcpuid < vm->numVCPUs);
+      t = vm->vmhost->vcpuSemaTask[vcpuid];
      VCPUSet_Remove(&req, vcpuid);
      if (t && (t->state & TASK_INTERRUPTIBLE)) {
         wake_up_process(t);
@@ -907,6 +912,128 @@ HostIF_FreeLockedPages(VMDriver *vm,         // IN: VM instance pointer
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_AllocLowPage --
+ *
+ *      Allocate a zeroed locked low page.
+ *
+ * Results:
+ *      Allocated MPN on success. INVALID_MPN on failure.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+HostIF_AllocLowPage(VMDriver *vm) //  IN: VM instance pointer
+{
+   unsigned int gfpFlag;
+   struct page* pg;
+   MPN mpn = INVALID_MPN;
+
+#ifdef GFP_DMA32
+   gfpFlag = GFP_USER | GFP_DMA32;
+#else
+   gfpFlag = GFP_USER | GFP_DMA;
+#endif
+
+   pg = alloc_page(gfpFlag);
+
+   if (pg != NULL) {
+      void *addr;
+      VMHost *vmh;
+
+      // zero page
+      addr = kmap(pg);
+      memset(addr, 0, PAGE_SIZE);
+      kunmap(pg);
+
+      mpn = (MPN)page_to_pfn(pg);
+
+      HostIF_VMLock(vm, 40);
+      vmh = vm->vmhost;
+      if (vmh->AWEPages != NULL) {
+         if (PhysTrack_Test(vmh->AWEPages, mpn)) {
+            Warning("%s: duplicate MPN %016" FMT64 "x\n", __func__, mpn);
+         }
+         PhysTrack_Add(vmh->AWEPages, mpn);
+      } else {
+         __free_page(pg);
+         mpn = INVALID_MPN;
+      }
+      HostIF_VMUnlock(vm, 40);
+   }
+   return mpn;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIFFreeVMHost --
+ *
+ *      Releases the memory allocated for the host-dependent part of
+ *      the driver.
+ *
+ * Results:
+ *      None
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+static void
+HostIFFreeVMHost(VMHost *vmhost) // IN:
+{
+   ASSERT(vmhost->lockedPages == NULL &&
+          vmhost->AWEPages    == NULL &&
+          vmhost->crosspagePagesCount == 0 &&
+          !vmhost->hostAPICIsMapped);
+   Vmx86_Free(vmhost->crosspagePages);
+   Vmx86_Free(vmhost->vcpuSemaTask);
+   vmhost->crosspagePages = NULL;
+   vmhost->vcpuSemaTask   = NULL;
+   HostIF_FreeKernelMem(vmhost);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIFAllocVMHost --
+ *
+ *      Allocate memory for the host-dependent part of the driver.
+ *
+ * Results:
+ *      Pointer to allocated structure on success, NULL on error.
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+static VMHost *
+HostIFAllocVMHost(uint32 numVCPUs) // IN:
+{
+   VMHost *vmhost = Vmx86_Calloc(1, sizeof *vmhost, TRUE);
+   if (vmhost == NULL) {
+      return NULL;
+   }
+   if ((vmhost->crosspagePages =
+        Vmx86_Calloc(numVCPUs, sizeof *vmhost->crosspagePages, TRUE)) != NULL &&
+       (vmhost->vcpuSemaTask =
+        Vmx86_Calloc(numVCPUs, sizeof *vmhost->vcpuSemaTask, TRUE))   != NULL) {
+      return vmhost;
+   }
+   HostIFFreeVMHost(vmhost);
+   return NULL;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -915,34 +1042,44 @@ HostIF_FreeLockedPages(VMDriver *vm,         // IN: VM instance pointer
 *      Initialize the host-dependent part of the driver.
 *
 * Results:
- *     zero on success, non-zero on error.
+ *      zero on success, non-zero on error.
 *
 * Side effects:
- *     None
+ *      None
 *
 *----------------------------------------------------------------------
 */

 int
-HostIF_Init(VMDriver *vm)  // IN:
+HostIF_Init(VMDriver *vm, uint32 numVCPUs)
 {
   vm->memtracker = MemTrack_Init(vm);
   if (vm->memtracker == NULL) {
      return -1;
   }

-   vm->vmhost = (VMHost *) HostIF_AllocKernelMem(sizeof *vm->vmhost, TRUE);
+   vm->vmhost = HostIFAllocVMHost(numVCPUs);
   if (vm->vmhost == NULL) {
-      return -1;
+      goto error;
   }
-   memset(vm->vmhost, 0, sizeof *vm->vmhost);

-   if (HostIFHostMemInit(vm)) {
-      return -1;
+   if (HostIFHostMemInit(vm) != 0) {
+      goto error;
   }
   MutexInit(&vm->vmhost->vmMutex, "vm");

   return 0;
+error:
+   if (vm->vmhost != NULL) {
+      HostIFHostMemCleanup(vm);
+      HostIFFreeVMHost(vm->vmhost);
+      vm->vmhost = NULL;
+   }
+   if (vm->memtracker != NULL) {
+      MemTrack_Cleanup(vm->memtracker, UnlockEntry, vm);
+      vm->memtracker = NULL;
+   }
+   return -1;
 }


@@ -1311,6 +1448,7 @@ HostIF_FreeAllResources(VMDriver *vm) // IN
      vm->memtracker = NULL;
   }
   if (vm->vmhost) {
+      ASSERT(vm->vmhost->crosspagePagesCount <= vm->numVCPUs);
      for (cnt = vm->vmhost->crosspagePagesCount; cnt > 0; ) {
         struct page* p = vm->vmhost->crosspagePages[--cnt];
         UnmapCrossPage(p, vm->crosspage[cnt]);
@@ -1322,7 +1460,7 @@ HostIF_FreeAllResources(VMDriver *vm) // IN
         vm->hostAPIC.base = NULL;
         vm->vmhost->hostAPICIsMapped = FALSE;
      }
-      HostIF_FreeKernelMem(vm->vmhost);
+      HostIFFreeVMHost(vm->vmhost);
      vm->vmhost = NULL;
   }
 }
@@ -1500,9 +1638,13 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
    * since at least 2.6.0.
    */

+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
   extern unsigned long totalram_pages;

   unsigned int totalPhysicalPages = totalram_pages;
+#else
+   unsigned int totalPhysicalPages = totalram_pages();
+#endif

   /*
    * Use the memory information linux exports as of late for a more
@@ -1600,6 +1742,49 @@ HostIF_WaitForFreePages(unsigned int timeoutMs)  // IN:
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIFGetTime --
+ *
+ *      Reads the current time in UPTIME_FREQ units.
+ *
+ * Results:
+ *      The uptime, in units of UPTIME_FREQ.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static uint64
+HostIFGetTime(void)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+   struct timeval tv;
+
+   do_gettimeofday(&tv);
+   return tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ;
+#else
+   struct timespec64 now;
+
+   /*
+    * Use raw time used by Posix timers.  This time is not affected by
+    * NTP adjustments, so it may drift from real time and monotonic time,
+    * but it will stay in sync with other timers.
+    */
+   ktime_get_raw_ts64(&now);
+   /*
+    * UPTIME_FREQ resolution is lower than tv_nsec,
+    * so we have to do division...
+    */
+   ASSERT_ON_COMPILE(1000000000 % UPTIME_FREQ == 0);
+   return now.tv_nsec / (1000000000 / UPTIME_FREQ) + now.tv_sec * UPTIME_FREQ;
+#endif
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1631,7 +1816,6 @@ HostIF_WaitForFreePages(unsigned int timeoutMs)  // IN:
 static uint64
 HostIFReadUptimeWork(unsigned long *j)  // OUT: current jiffies
 {
-   struct timeval tv;
   uint64 monotime, uptime, upBase, monoBase;
   int64 diff;
   uint32 version;
@@ -1646,13 +1830,12 @@ HostIFReadUptimeWork(unsigned long *j)  // OUT: current jiffies
      monoBase = uptimeState.monotimeBase;
   } while (!VersionedAtomic_EndTryRead(&uptimeState.version, version));

-   do_gettimeofday(&tv);
+   uptime = HostIFGetTime();
   upBase = Atomic_Read64(&uptimeState.uptimeBase);

   monotime = (uint64)(jifs - jifBase) * (UPTIME_FREQ / HZ);
   monotime += monoBase;

-   uptime = tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ;
   uptime += upBase;

   /*
@@ -1757,13 +1940,11 @@ HostIFUptimeResyncMono(struct timer_list *timer)  // IN: ignored
 void
 HostIF_InitUptime(void)
 {
-   struct timeval tv;
+   uint64 tm;

   uptimeState.jiffiesBase = jiffies;
-   do_gettimeofday(&tv);
-   Atomic_Write64(&uptimeState.uptimeBase,
-                  -(tv.tv_usec * (UPTIME_FREQ / 1000000) +
-                    tv.tv_sec * UPTIME_FREQ));
+   tm = HostIFGetTime();
+   Atomic_Write64(&uptimeState.uptimeBase, -tm);

 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) && !defined(timer_setup)
   init_timer(&uptimeState.timer);
@@ -1909,15 +2090,10 @@ HostIF_CopyToUser(VA64 dst,         // OUT
 *
 * HostIF_MapCrossPage --
 *
- *    Obtain kernel pointer to crosspage.
- *
- *    We must return a VA that is obtained through a kernel mapping, so that
- *    the mapping never goes away (see bug 29753).
- *
- *    However, the LA corresponding to that VA must not overlap with the
- *    monitor (see bug 32922). The userland code ensures that by only
- *    allocating cross pages from low memory. For those pages, the kernel
- *    uses a permanent mapping, instead of a temporary one with a high LA.
+ *    Map the cross page in the kernel address space given a user VA.  The
+ *    kernel mapping must not overlap the monitor's 64MB VA space (bug 32922).
+ *    In practice, the host kernel does not return mappings in that range.
+ *    This is checked in TaskCreatePTPatch() which gracefully fails if it must.
 *
 * Results:
 *    The kernel virtual address on success
@@ -1943,7 +2119,7 @@ HostIF_MapCrossPage(VMDriver *vm, // IN
   }
   vPgAddr = (VA) MapCrossPage(page);
   HostIF_VMLock(vm, 27);
-   if (vm->vmhost->crosspagePagesCount >= MAX_INITBLOCK_CPUS) {
+   if (vm->vmhost->crosspagePagesCount >= vm->numVCPUs) {
      HostIF_VMUnlock(vm, 27);
      UnmapCrossPage(page, (void*)vPgAddr);

@@ -1963,7 +2139,8 @@ HostIF_MapCrossPage(VMDriver *vm, // IN
 *
 * HostIF_AllocKernelPages --
 *
- *      Allocates and maps a set of locked pages.
+ *      Allocates and maps a set of locked pages. The pages might not
+ *      be physically contiguous.
 *
 * Results:
 *      On success: Host kernel virtual address of the first page.
@@ -1982,27 +2159,16 @@ void *
 HostIF_AllocKernelPages(unsigned numPages, // IN: Number of pages
                        MPN     *mpns)     // OUT: Array of MPNs
 {
-   MPN startMPN;
-   struct page *pages;
   unsigned i;
-   void *ptr;
+   void *ptr = vmalloc(numPages * PAGE_SIZE);

-   for (i = 0; (1 << i) < numPages; i++) { }
-   /* Allocates physically contiguous pages. */
-   pages = alloc_pages(GFP_KERNEL, i);
-
-   if (pages == NULL) {
+   if (ptr == NULL) {
      return NULL;
   }
-
-   startMPN = page_to_pfn(pages);
-
+   ASSERT((PtrToVA64(ptr) & (PAGE_SIZE - 1)) == 0);
   for (i = 0; i < numPages; i++) {
-      mpns[i] = startMPN + i;
+      mpns[i] = vmalloc_to_pfn((uint8 *)ptr + i * PAGE_SIZE);
   }
-   ptr = (void *)page_address(pages);
-   ASSERT(!(PtrToVA64(ptr) & (PAGE_SIZE - 1))); /* Page-aligned */
-
   return ptr;
 }

@@ -2027,10 +2193,7 @@ void
 HostIF_FreeKernelPages(unsigned numPages, // IN: Number of pages
                       void    *ptr)      // IN: Kernel VA of first page
 {
-   unsigned i;
-
-   for (i = 0; (1 << i) < numPages; i++) { }
-   free_pages((VA)ptr, i);
+   vfree(ptr);
 }


@@ -2331,6 +2494,80 @@ HostIF_APICInit(VMDriver *vm,   // IN:
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_GetTimerVectors --
+ *
+ *      Determine which vectors the host might be using for timer
+ *      interrupts.
+ *
+ * Results:
+ *      The host-specific timer interrupt vectors.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+HostIF_GetTimerVectors(uint8 *v0, // OUT:
+                       uint8 *v1) // OUT:
+{
+   *v0 = 0xef; // APIC
+   *v1 = 0x20; // PIC
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_GetMonitorIPIVector --
+ *
+ *      Return a dedicated vector the monitor can use for sending
+ *      IPIs.
+ *
+ * Results:
+ *     The vector determined by HostIF_APICInit.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint8
+HostIF_GetMonitorIPIVector(void)
+{
+   return monitorIPIVector;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_GetHVIPIVector --
+ *
+ *      Return a dedicated vector the monitor can use for posted
+ *      interrupts.
+ *
+ * Results:
+ *     The vector determined by HostIF_APICInit.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+uint8
+HostIF_GetHVIPIVector(void)
+{
+   return hvIPIVector;
+}
+
+
 /*
 *-----------------------------------------------------------------------------
 *
@@ -2360,6 +2597,8 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:
   int timeoutms = args[2];
   uint64 value;

+   ASSERT(vcpuid < vm->numVCPUs);
+
   file = vmware_fget(waitFD);
   if (file == NULL) {
      return MX_WAITERROR;
@@ -2374,7 +2613,7 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:

      poll_initwait(&table);
      current->state = TASK_INTERRUPTIBLE;
-      mask = compat_vfs_poll(file, &table.pt);
+      mask = file->f_op->poll(file, &table.pt);
      if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
         vm->vmhost->vcpuSemaTask[vcpuid] = current;
         schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
@@ -2450,13 +2689,13 @@ void
 HostIF_SemaphoreForceWakeup(VMDriver *vm,       // IN:
                            const VCPUSet *vcs) // IN:
 {
-   FOR_EACH_VCPU_IN_SET(vcs, vcpuid) {
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(vcs, vcpuid, vm->numVCPUs) {
      struct task_struct *t = vm->vmhost->vcpuSemaTask[vcpuid];
      vm->vmhost->vcpuSemaTask[vcpuid] = NULL;
      if (t && (t->state & TASK_INTERRUPTIBLE)) {
         wake_up_process(t);
      }
-   } ROF_EACH_VCPU_IN_SET();
+   } ROF_EACH_VCPU_IN_SET_WITH_MAX();
 }


@@ -2535,6 +2774,35 @@ HostIF_SemaphoreSignal(uint64 *args)  // IN:
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_OneIPI --
+ *
+ *    If the passed VCPU thread is on some CPU in the system,
+ *    attempt to hit it with an IPI.
+ *
+ * Result:
+ *    None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+HostIF_OneIPI(VMDriver *vm,                // IN:
+              Vcpuid v)                    // IN:
+{
+   uint32 targetHostCpu;
+
+   ASSERT(v < vm->numVCPUs);
+   targetHostCpu = Atomic_Read32(&vm->currentHostCpu[v]);
+   if (targetHostCpu != INVALID_PCPU) {
+      ASSERT(targetHostCpu < MAX_PCPUS);
+      arch_send_call_function_single_ipi(targetHostCpu);
+   }
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -2543,32 +2811,19 @@ HostIF_SemaphoreSignal(uint64 *args)  // IN:
 *    If the passed VCPU threads are on some CPUs in the system,
 *    attempt to hit them with an IPI.
 *
- *    On older Linux systems we do a broadcast.
- *
 * Result:
- *    The mode used to send IPIs.
+ *    None.
 *
 *----------------------------------------------------------------------
 */

-HostIFIPIMode
+void
 HostIF_IPI(VMDriver *vm,                // IN:
           const VCPUSet *ipiTargets)   // IN:
 {
-   HostIFIPIMode mode = IPI_NONE;
-
-   ASSERT(vm);
-
-   FOR_EACH_VCPU_IN_SET(ipiTargets, v) {
-      uint32 targetHostCpu = vm->currentHostCpu[v];
-      if (targetHostCpu != INVALID_PCPU) {
-         ASSERT(targetHostCpu < MAX_PCPUS);
-         arch_send_call_function_single_ipi(targetHostCpu);
-         mode = IPI_UNICAST;
-      }
-   } ROF_EACH_VCPU_IN_SET();
-
-   return mode;
+   FOR_EACH_VCPU_IN_SET_WITH_MAX(ipiTargets, v, vm->numVCPUs) {
+      HostIF_OneIPI(vm, v);
+   } ROF_EACH_VCPU_IN_SET_WITH_MAX();
 }


@@ -2750,14 +3005,6 @@ HostIFCheckTrackedMPN(VMDriver *vm, // IN: The VM instance
      }
   }
   HostIF_VMUnlock(vm, 32);
-
-   if (vmx86_debug) {
-      /*
-       * The monitor may have old KSeg mappings to pages which it no longer
-       * owns.  Minimize customer noise by only logging this for developers.
-       */
-      Log("%s: MPN %" FMT64 "x not owned by this VM\n", __FUNCTION__, mpn);
-   }
   return FALSE;
 }

@@ -3201,7 +3448,12 @@ HostIF_MapUserMem(VA addr,                  // IN: User memory virtual address

   ASSERT(handle);

-   if (!access_ok(VERIFY_WRITE, p, size)) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+   if (!access_ok(VERIFY_WRITE, p, size))
+#else
+   if (!access_ok(p, size))
+#endif
+   {
      printk(KERN_ERR "%s: Couldn't verify write to uva 0x%p with size %"
             FMTSZ"u\n", __func__, p, size);

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -79,22 +79,23 @@ typedef struct VMHost {
    */
   Mutex vmMutex;

-   unsigned int       crosspagePagesCount;
-   struct page        *crosspagePages[MAX_INITBLOCK_CPUS];
-   struct task_struct *vcpuSemaTask[MAX_INITBLOCK_CPUS];
+   unsigned int         crosspagePagesCount;
+   struct page        **crosspagePages;      /* ptr to numVCPUs-sized array */
+   struct task_struct **vcpuSemaTask;        /* ptr to numVCPUs-sized array */
+
   /*
    * Pages that were allocated/mapped by VMX and locked by the driver and
    * don't have a particular VA.
    */
-   struct PhysTracker *lockedPages;
+   struct PhysTracker  *lockedPages;
   /*
    * Locked pages that were allocated by the driver and don't have
    * a particular VA. They are used as monitor anonymous pages or
    * as pages for "AWE" guest memory.
    */
-   struct PhysTracker *AWEPages;
+   struct PhysTracker  *AWEPages;
   /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */
-   Bool               hostAPICIsMapped;
+   Bool                 hostAPICIsMapped;
 } VMHost;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -42,6 +42,7 @@
 #include "driver_vmcore.h"
 #include "usercalldefs.h"
 #include "cpuid.h"
+#include "vmmblob.h"

 /*
 *----------------------------------------------------------------------
@@ -72,10 +73,12 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
            Vcpuid vcpuid)  // IN:
 {
   uint32           retval    = MODULECALL_USERRETURN;
-   VMCrossPageData *crosspage = &vm->crosspage[vcpuid]->crosspageData;
+   VMCrossPageData *crosspage;
   int              bailValue = 0;

-   ASSERT(crosspage && CPUID_HostSupportsHV());
+   ASSERT(vcpuid < vm->numVCPUs && CPUID_HostSupportsHV());
+   crosspage = &vm->crosspage[vcpuid]->crosspageData;
+   ASSERT(crosspage);

   /*
    * Check if we were interrupted by signal.
@@ -98,13 +101,13 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
       * Wake up anything that was waiting for this vcpu to run
       */

-      if ((VCPUSet_IsEmpty(&crosspage->yieldVCPUs) &&
+      if ((crosspage->yieldVCPUsIsEmpty &&
           crosspage->moduleCallType != MODULECALL_COSCHED) ||
          crosspage->moduleCallType == MODULECALL_SEMAWAIT) {
         HostIF_WakeUpYielders(vm, vcpuid);
      }

-      if (!VCPUSet_IsEmpty(&crosspage->yieldVCPUs) &&
+      if (!crosspage->yieldVCPUsIsEmpty &&
          crosspage->moduleCallType != MODULECALL_COSCHED &&
          crosspage->moduleCallType != MODULECALL_SEMAWAIT) {
         Vmx86_YieldToSet(vm, vcpuid, &crosspage->yieldVCPUs, 0, TRUE);
@@ -150,6 +153,12 @@ skipTaskSwitch:;
         break;
      }

+      case MODULECALL_ALLOC_ANON_LOW_PAGE: {
+         // Return via 64-bit args[0] (may return INVALID_MPN).
+         crosspage->args[0] = Vmx86_AllocLowPage(vm, FALSE);
+         break;
+      }
+
      case MODULECALL_SEMAWAIT: {
         retval = HostIF_SemaphoreWait(vm, vcpuid, crosspage->args);

@@ -177,10 +186,13 @@ skipTaskSwitch:;
         break;
      }

+      case MODULECALL_ONE_IPI: {
+         Vcpuid v = (Vcpuid)crosspage->args[0];
+         HostIF_OneIPI(vm, v);
+         break;
+      }
      case MODULECALL_IPI: {
-         HostIFIPIMode mode;
-         mode = HostIF_IPI(vm, &crosspage->vcpuSet);
-         retval = (mode != IPI_NONE);
+         HostIF_IPI(vm, &crosspage->vcpuSet);
         break;
      }

@@ -260,6 +272,26 @@ skipTaskSwitch:;
         crosspage->args[0] = mpn;
      } break;

+      case MODULECALL_GET_MON_IPI_VECTOR: {
+         retval = HostIF_GetMonitorIPIVector();
+      } break;
+
+      case MODULECALL_GET_HV_IPI_VECTOR: {
+         retval = HostIF_GetHVIPIVector();
+      } break;
+
+      case MODULECALL_GET_HOST_TIMER_VECTORS: {
+         uint8 v0, v1;
+         HostIF_GetTimerVectors(&v0, &v1);
+         crosspage->args[0] = v0;
+         crosspage->args[1] = v1;
+      } break;
+
+      case MODULECALL_BOOTSTRAP_CLEANUP: {
+         VmmBlob_Cleanup(vm->blobInfo);
+         vm->blobInfo = NULL;
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
@@ -32,6 +32,6 @@ $(DRIVER)-y := driver.o hub.o userif.o netif.o bridge.o procfs.o smac_compat.o \
 ####

 clean:
-	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko .cache.mk \
+	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko \
 	       .tmp_versions Module.symvers Modules.symvers Module.markers \
 	       modules.order $($(DRIVER)-y) .*.cmd .*.o.flags
@@ -56,6 +56,11 @@

 #ifdef VMX86_DEVEL
 #define DBG 1
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)
+#define read_skb_users(skb) atomic_read(&skb->users)
+#else
+#define read_skb_users(skb) refcount_read(&skb->users)
+#endif
 #else
 #undef DBG
 #endif /* VMX86_DEVEL */
@@ -405,7 +410,7 @@ SMACL_PrintSkb(struct sk_buff *skb,          // IN: sk_buff structure
   LOG(4, (KERN_DEBUG "pkt_type %x truesize %u protocol %u\n",
            skb->pkt_type, skb->truesize, skb->protocol));
   LOG(4, (KERN_DEBUG "users %d, tail %p, end %p\n",
-            atomic_read(&skb->users), compat_skb_tail_pointer(skb),
+            read_skb_users(skb), compat_skb_tail_pointer(skb),
            compat_skb_end_pointer(skb)));
 #if 0
 #define C skb->mac.raw
@@ -137,16 +137,21 @@ UserifLockPage(VA addr) // IN
 */

 static INLINE int
-VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapPtr(VA64 uAddr,      // IN: pointer to user memory
                 size_t size,     // IN: size of data
                 struct page **p, // OUT: locked page
                 void **ptr)      // OUT: kernel mapped pointer
 {
-   if (!access_ok(VERIFY_WRITE, (void *)uAddr, size) ||
-       (((uAddr + size - 1) & ~(PAGE_SIZE - 1)) !=
-        (uAddr & ~(PAGE_SIZE - 1)))) {
+   uint8 v;
+
+   /* Check area does not straddle two pages. */
+   if ((uAddr & (PAGE_SIZE - 1)) + size > PAGE_SIZE) {
      return -EINVAL;
   }
+   /* Check if it is user's area.  UserifLockPage() checks writability. */
+   if (copy_from_user(&v, (void *)(unsigned long)uAddr, sizeof v) != 0) {
+      return -EFAULT;
+   }

   *p = UserifLockPage(uAddr);
   if (*p == NULL) {
@@ -158,7 +163,7 @@ VNetUserIfMapPtr(VA uAddr,        // IN: pointer to user memory
 }

 static INLINE int
-VNetUserIfMapUint32Ptr(VA uAddr,        // IN: pointer to user memory
+VNetUserIfMapUint32Ptr(VA64 uAddr,      // IN: pointer to user memory
                       struct page **p, // OUT: locked page
                       uint32 **ptr)    // OUT: kernel mapped pointer
 {
@@ -201,7 +206,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      return -EBUSY;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->pollPtr, &pollPage,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->pollPtr, &pollPage,
                                        &pollPtr)) < 0) {
      return retval;
   }
@@ -213,7 +218,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN
      goto error_free;
   }

-   if ((retval = VNetUserIfMapUint32Ptr((VA)vn->recvClusterPtr,
+   if ((retval = VNetUserIfMapUint32Ptr(vn->recvClusterPtr,
                                        &recvClusterPage,
                                        &recvClusterCount)) < 0) {
      goto error_free;
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -49,38 +49,68 @@ extern "C" {
 #endif

 /*
- * In the Atomic_* definitions below, memory ordering and atomicity are somewhat
- * conflated in an inconsistent manner. First, we have Atomic_{Read,Write},
- * which only guarantees single copy atomicity, i.e. that the read/write occurs
- * in an atomic fashion, but have no implication on memory ordering. The second
- * class of Atomics are all the non-unfenced operations excluding
- * Atomic_{Read,Write}*, which both imply atomicity and act as a memory barrier,
- * implying sequentially consistent ordering of the atomic operation with all
- * loads/stores prior to and after it.
+ * There are two concepts involved when dealing with atomic accesses:
+ * 1. Atomicity of the access itself
+ * 2. Ordering of the access with respect to other reads&writes (from the view
+ *    of other processors/devices).
 *
- * Since on x86, the second class of operations are associated with LOCK
- * semantics, assumptions have been made about the ordering these operations
- * imply on surrounding code (see for example the vmkernel's RefCount
- * implementation). As a result, on arm64 we have to provide these same
- * guarantees. We do this by making use of DMB barriers both before and after
- * the atomic ldrx/strx sequences. A barrier before and after is required to
- * avoid having part of the atomic operation reordered with surrounding code,
- * e.g. a store-load reordering of the strx with a following load outside the
- * Atomic_ op. For the first class of operations, Atomic_{Read,Write}, we do not
- * implement a barrier.
+ * Two examples help to clarify #2:
+ * a. Inc: A caller implementing a simple independent global event counter
+ *         might not care if the compiler or processor visibly reorders the
+ *         increment around other memory accesses.
+ * b. Dec: A caller implementing a reference count absolutely *doesn't* want
+ *         the compiler or processor to visibly reordering writes after that
+ *         decrement: if that happened, the program could then end up writing
+ *         to memory that was freed by another processor.
 *
- * This implementation of Atomic operations is suboptimal on arm64, since
- * both atomicity and memory ordering are fused together. Ideally the Atomic
- * operations would only imply atomicity, and an explicit memory barrier in the
- * surrounding code used to enforce ordering where necessary. This would eschew
- * the need for the DMBs. A middle ground can be implemented where we use the
- * arm64 load-acquire/store-release exclusive instructions to implement Atomics.
- * This would imply sequential consistency of the Atomic operations (but not
- * with any of the surrounding non-atomic operations) without the need for a
- * DMB. Using these without a DMB today can still result in problematic
- * reordering by the processor with surrounding non-atomic operations, e.g. a
- * store-load reordering with a stlxr. Future optimization for arm64 should
- * consider the wider change required at the call sites to minimize DMBs.
+ * C11 has standardized a good model for expressing these orderings when doing
+ * atomics. It defines three *tiers* of ordering:
+ * 1. Sequential Consistency (every processor sees the same total order of
+ *    events)
+ *
+ * 2. Acquire/Release ordering (roughly, everybody can agree previous events
+ *    have completed, but they might disagree on the ordering of previous
+ *    independent events).
+ *
+ *    The relative ordering provided by this tier is sufficient for common
+ *    locking and initialization activities, but is insufficient for unusual
+ *    synchronization schemes (e.g. IRIW aka Independent Read Independent
+ *    Write designs such Dekker's algorithm, Peterson's algorithm, etc.)
+ *
+ *    In other words, this tier is close in behavior to Sequential Consistency
+ *    in much the same way a General-Relativity universe is close to a
+ *    Newtonian universe.
+ * 3. Relaxed (i.e unordered/unfenced)
+ *
+ * In C11 standard's terminology for atomic memory ordering,
+ * - in case (a) we want "relaxed" ordering for perf and,
+ * - in case (b) we want "sequentially consistent" ordering (or perhaps the
+ *   only slightly weaker "release" ordering) for correctness.
+ *
+ * There are standardized mappings of operations to orderings for every
+ * processor architecture. See
+ * - https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
+ * - http://preshing.com/20120913/acquire-and-release-semantics/
+ *
+ * In this file:
+ * 1. all RMW (Read/Modify/Write) operations are sequentially consistent.
+ *    This includes operations like Atomic_IncN, Atomic_ReadIfEqualWriteN,
+ *    Atomic_ReadWriteN, etc.
+ * 2. all R and W operations are relaxed. This includes operations like
+ *    Atomic_WriteN, Atomic_ReadN, Atomic_TestBitN, etc.
+ *
+ * The below routines of course ensure both the CPU and compiler honor the
+ * ordering constraint.
+ *
+ * Notes:
+ * 1. Since R-only and W-only operations do not provide ordering, callers
+ *    using them for synchronizing operations like double-checked
+ *    initialization or releasing spinlocks must provide extra barriers.
+ * 2. This implementation of Atomic operations is suboptimal. On x86,simple
+ *    reads and writes have acquire/release semantics at the hardware level.
+ *    On arm64, we have separate instructions for sequentially consistent
+ *    reads and writes (the same instructions are used for acquire/release).
+ *    Neither of these are exposed for R-only or W-only callers.
 *
 * For further details on x86 and ARM memory ordering see
 * https://wiki.eng.vmware.com/ARM/MemoryOrdering.
@@ -118,10 +148,7 @@ typedef struct Atomic_uint128 {
 /*
 * Prototypes for msft atomics.  These are defined & inlined by the
 * compiler so no function definition is needed.  The prototypes are
- * needed for C++.  Since amd64 compiler doesn't support inline asm we
- * have to use these.  Unfortunately, we still have to use some inline asm
- * for the 32 bit code since the and/or/xor implementations didn't show up
- * until XP or 2k3.
+ * needed for C++.
 *
 * The declarations for the intrinsic functions were taken from ntddk.h
 * in the DDK. The declarations must match otherwise the 64-bit C++
@@ -130,7 +157,7 @@ typedef struct Atomic_uint128 {
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
 */
-#if defined _MSC_VER && _MSC_VER >= 1310 && !defined BORA_NO_WIN32_INTRINS
+#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -222,38 +249,10 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:


 /*
- * All the assembly code is tricky and written conservatively.
- * For example, to make sure gcc won't introduce copies,
- * we force the addressing mode like this:
- *
- *    "xchgl %0, (%1)"
- *    : "=r" (val)
- *    : "r" (&var->value),
- *      "0" (val)
- *    : "memory"
- *
- * - edward
- *
- * Actually - turns out that gcc never generates memory aliases (it
- * still does generate register aliases though), so we can be a bit
- * more agressive with the memory constraints. The code above can be
- * modified like this:
- *
- *    "xchgl %0, %1"
- *    : "=r" (val),
- *      "=m" (var->value),
- *    : "0" (val),
- *      "1" (var->value)
- *
- * The advantages are that gcc can use whatever addressing mode it
- * likes to access the memory value, and that we dont have to use a
- * way-too-generic "memory" clobber as there is now an explicit
- * declaration that var->value is modified.
- *
- * see also /usr/include/asm/atomic.h to convince yourself this is a
- * valid optimization.
- *
- * - walken
+ * The Read/Modify/Write operations on x86/x64 are all written using the
+ * "memory" constraint. This is to ensure the compiler treats the operation as
+ * a full barrier, flushing any pending/cached state currently residing in
+ * registers.
 */

 #if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
@@ -403,6 +402,7 @@ Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
      : "=q" (val),
        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER && _MSC_VER >= 1600
@@ -496,7 +496,7 @@ Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
        "+m" (var->value)
      : "q" (newVal),
        "0" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );

   return val;
@@ -1002,26 +1002,15 @@ Atomic_ReadWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchange((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm xchg [ebx]Atomic_uint32.value, eax
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif // _MSC_VER >= 1310
-#else
 #error No compiler defined for Atomic_ReadWrite
 #endif // __GNUC__
 }
@@ -1162,30 +1151,17 @@ Atomic_ReadIfEqualWrite32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgl %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedCompareExchange((long *)&var->value,
-				      (long)newVal,
-				      (long)oldVal);
-#else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, oldVal
-      __asm mov ebx, var
-      __asm mov ecx, newVal
-      __asm lock cmpxchg [ebx]Atomic_uint32.value, ecx
-      // eax is the return value, this is documented to work - edward
-   }
-#pragma warning(pop)
-#endif
+                                      (long)newVal,
+                                      (long)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite
 #endif
@@ -1267,17 +1243,17 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgq %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #endif //VM_ARM_V7
 #elif defined _MSC_VER
   return _InterlockedCompareExchange64((__int64 *)&var->value,
-					(__int64)newVal,
-					(__int64)oldVal);
+                                        (__int64)newVal,
+                                        (__int64)oldVal);
 #else
 #error No compiler defined for Atomic_ReadIfEqualWrite64
 #endif
@@ -1332,7 +1308,7 @@ Atomic_And32(Atomic_uint32 *var, // IN/OUT
      "lock; andl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1397,7 +1373,7 @@ Atomic_Or32(Atomic_uint32 *var, // IN/OUT
      "lock; orl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1462,7 +1438,7 @@ Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
      "lock; xorl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
@@ -1510,7 +1486,7 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT
      "lock; xorq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -1569,17 +1545,11 @@ Atomic_Add32(Atomic_uint32 *var, // IN/OUT
      "lock; addl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedExchangeAdd((long *)&var->value, (long)val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock add [ebx]Atomic_uint32.value, eax
-#endif
 #else
 #error No compiler defined for Atomic_Add
 #endif
@@ -1634,17 +1604,11 @@ Atomic_Sub32(Atomic_uint32 *var, // IN/OUT
      "lock; subl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
-   _InterlockedExchangeAdd((long *)&var->value, (long)-val);
-#else
-   __asm mov eax, val
-   __asm mov ebx, var
-   __asm lock sub [ebx]Atomic_uint32.value, eax
-#endif
+   _InterlockedExchangeAdd((long *)&var->value, -(long)val);
 #else
 #error No compiler defined for Atomic_Sub
 #endif
@@ -1680,16 +1644,11 @@ Atomic_Inc32(Atomic_uint32 *var) // IN/OUT
      "lock; incl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedIncrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock inc [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Inc
 #endif
@@ -1725,16 +1684,11 @@ Atomic_Dec32(Atomic_uint32 *var) // IN/OUT
      "lock; decl %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   _InterlockedDecrement((long *)&var->value);
-#else
-   __asm mov ebx, var
-   __asm lock dec [ebx]Atomic_uint32.value
-#endif
 #else
 #error No compiler defined for Atomic_Dec
 #endif
@@ -1939,26 +1893,15 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddl %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #endif /* VM_X86_ANY */
 #elif defined _MSC_VER
-#if _MSC_VER >= 1310
   return _InterlockedExchangeAdd((long *)&var->value, (long)val);
 #else
-#pragma warning(push)
-#pragma warning(disable : 4035)         // disable no-return warning
-   {
-      __asm mov eax, val
-      __asm mov ebx, var
-      __asm lock xadd [ebx]Atomic_uint32.value, eax
-   }
-#pragma warning(pop)
-#endif
-#else
 #error No compiler defined for Atomic_ReadAdd32
 #endif
 }
@@ -2044,11 +1987,11 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchgq %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
 #else /* 32-bit version for non-ARM */
   typedef struct {
@@ -2088,9 +2031,9 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b (%3)" "\n\t"
      "xchgl %%ebx, %6"      "\n\t"
      "sete %0"
-      :	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+      : "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : /*
         * See the "Rules for __asm__ statements in __PIC__ code" above: %3
         * must use a register class which does not contain %ebx.
@@ -2111,14 +2054,14 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
      "lock; cmpxchg8b %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy1),
-	"=d" (dummy2)
+        "=qm" (equal),
+        "=a" (dummy1),
+        "=d" (dummy2)
      : "2" (((S_uint64 *)&oldVal)->lowValue),
        "3" (((S_uint64 *)&oldVal)->highValue),
        "b" (((S_uint64 *)&newVal)->lowValue),
        "c" (((S_uint64 *)&newVal)->highValue)
-      : "cc"
+      : "cc", "memory"
   );
 #   endif
 #endif
@@ -2166,11 +2109,11 @@ Atomic_CMPXCHG32(Atomic_uint32 *var,   // IN/OUT
      "lock; cmpxchgl %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
-	"=qm" (equal),
-	"=a" (dummy)
+        "=qm" (equal),
+        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
-      : "cc"
+      : "cc", "memory"
   );
   return equal;
 #endif /* VM_X86_ANY */
@@ -2247,7 +2190,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN
   return _InterlockedAdd64((__int64 *)&var->value, 0);
 #elif defined _MSC_VER && defined __i386__
 #   pragma warning(push)
-#   pragma warning(disable : 4035)		// disable no-return warning
+#   pragma warning(disable : 4035)      // disable no-return warning
   {
      __asm mov ecx, var
      __asm mov edx, ecx
@@ -2330,9 +2273,9 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "lock; xaddq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2378,7 +2321,7 @@ Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
 #if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
 #else
-   return Atomic_ReadAdd64(var, -val);
+   return Atomic_ReadAdd64(var, -(int64)val);
 #endif
 }

@@ -2425,7 +2368,7 @@ Atomic_ReadInc64(Atomic_uint64 *var) // IN/OUT
 static INLINE uint64
 Atomic_ReadDec64(Atomic_uint64 *var) // IN/OUT
 {
-   return Atomic_ReadAdd64(var, CONST64U(-1));
+   return Atomic_ReadAdd64(var, (uint64)CONST64(-1));
 }


@@ -2460,7 +2403,7 @@ Atomic_Add64(Atomic_uint64 *var, // IN/OUT
      "lock; addq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2502,7 +2445,7 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT
      "lock; subq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #endif
 #elif defined _MSC_VER
@@ -2542,7 +2485,7 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT
      "lock; incq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedIncrement64((__int64 *)&var->value);
@@ -2581,7 +2524,7 @@ Atomic_Dec64(Atomic_uint64 *var) // IN/OUT
      "lock; decq %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedDecrement64((__int64 *)&var->value);
@@ -2617,8 +2560,9 @@ Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT
   __asm__ __volatile__(
      "xchgq %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined _MSC_VER
@@ -2726,7 +2670,7 @@ Atomic_Or64(Atomic_uint64 *var, // IN/OUT
      "lock; orq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedOr64((__int64 *)&var->value, (__int64)val);
@@ -2773,7 +2717,7 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT
      "lock; andq %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined _MSC_VER
   _InterlockedAnd64((__int64 *)&var->value, (__int64)val);
@@ -2811,29 +2755,17 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT
-                uint64 bit)         // IN
+                unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; bts %1, %0"
+      "lock; btsq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal | (CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_SetBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2863,29 +2795,17 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE void
 Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT
-                  uint64 bit)         // IN
+                  unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
+#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
-      "lock; btr %1, %0"
+      "lock; btrq %1, %0"
      : "+m" (var->value)
-      : "ri" (bit)
-      : "cc"
+      : "ri" ((uint64)bit)
+      : "cc", "memory"
   );
-#elif defined _MSC_VER
-   uint64 oldVal;
-   uint64 newVal;
-   ASSERT(bit <= 63);
-   do {
-      oldVal = var->value;
-      newVal = oldVal & ~(CONST64U(1) << bit);
-   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
 #else
-#error No compiler defined for Atomic_ClearBit64
-#endif
-#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
@@ -2915,29 +2835,65 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT

 static INLINE Bool
 Atomic_TestBit64(Atomic_uint64 *var, // IN
-                 uint64 bit)         // IN
+                 unsigned bit)       // IN
 {
-#if defined __x86_64__
-#if defined __GNUC__
-   Bool out = FALSE;
+   Bool out;
   ASSERT(bit <= 63);
+#if defined __x86_64__ && defined __GNUC__
   __asm__ __volatile__(
-      "bt %2, %1; setc %0"
+      "btq %2, %1; setc %0"
      : "=rm"(out)
      : "m" (var->value),
-        "rJ" (bit)
+        "rJ" ((uint64)bit)
      : "cc"
   );
-   return out;
-#elif defined _MSC_VER
-   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
 #else
-#error No compiler defined for Atomic_TestBit64
+   out = (var->value & (CONST64U(1) << bit)) != 0;
 #endif
-#else // __x86_64__
+   return out;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBit64 --
+ *
+ *      Atomically test and set the bit 'bit' in var.
+ *      Bit must be between 0 and 63.
+ *
+ * Results:
+ *      TRUE if the tested bit was set; else FALSE.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBit64(Atomic_uint64 *var, // IN/OUT
+                    unsigned bit)       // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool out;
   ASSERT(bit <= 63);
-   return (var->value & (CONST64U(1) << bit)) != 0;
+   __asm__ __volatile__(
+      "lock; btsq %2, %1; setc %0"
+      : "=rm" (out), "+m" (var->value)
+      : "rJ" ((uint64)bit)
+      : "cc", "memory"
+   );
+   return out;
+#else
+   uint64 oldVal;
+   uint64 mask;
+   ASSERT(bit <= 63);
+   mask = CONST64U(1) << bit;
+   do {
+      oldVal = var->value;
+   } while (!Atomic_CMPXCHG64(var, oldVal, oldVal | mask));
+   return (oldVal & mask) != 0;
 #endif
 }

@@ -3020,8 +2976,9 @@ Atomic_ReadWrite16(Atomic_uint16 *var,  // IN/OUT:
   __asm__ __volatile__(
      "xchgw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
+      : "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3128,10 +3085,10 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var,   // IN/OUT
   __asm__ __volatile__(
      "lock; cmpxchgw %2, %1"
      : "=a" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "r" (newVal),
-	"0" (oldVal)
-      : "cc"
+        "0" (oldVal)
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3192,7 +3149,7 @@ Atomic_And16(Atomic_uint16 *var, // IN/OUT
      "lock; andw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3249,7 +3206,7 @@ Atomic_Or16(Atomic_uint16 *var, // IN/OUT
      "lock; orw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3306,7 +3263,7 @@ Atomic_Xor16(Atomic_uint16 *var, // IN/OUT
      "lock; xorw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3363,7 +3320,7 @@ Atomic_Add16(Atomic_uint16 *var, // IN/OUT
      "lock; addw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3420,7 +3377,7 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT
      "lock; subw %1, %0"
      : "+m" (var->value)
      : "re" (val)
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_V7
   register volatile uint16 res;
@@ -3476,7 +3433,7 @@ Atomic_Inc16(Atomic_uint16 *var) // IN/OUT
      "lock; incw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Add16(var, 1);
@@ -3512,7 +3469,7 @@ Atomic_Dec16(Atomic_uint16 *var) // IN/OUT
      "lock; decw %0"
      : "+m" (var->value)
      :
-      : "cc"
+      : "cc", "memory"
   );
 #elif defined VM_ARM_ANY
   Atomic_Sub16(var, 1);
@@ -3582,9 +3539,9 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
   __asm__ __volatile__(
      "lock; xaddw %0, %1"
      : "=r" (val),
-	"+m" (var->value)
+        "+m" (var->value)
      : "0" (val)
-      : "cc"
+      : "cc", "memory"
   );
   return val;
 #elif defined VM_ARM_V7
@@ -3715,7 +3672,7 @@ Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
                            in val)                                           \
   {                                                                          \
      return (out)(cast)Atomic_ReadWrite ## size(var,                         \
-		(uint ## size)(cast)val);                                     \
+                (uint ## size)(cast)val);                                     \
   }                                                                          \
                                                                              \
                                                                              \
@@ -3876,6 +3833,109 @@ MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t)
 MAKE_ATOMIC_TYPE(Int, 32, int, int, int)
 MAKE_ATOMIC_TYPE(Bool, 8, Bool, Bool, Bool)

+/*
+ * Define arbitrary sized bit vector to be used by
+ * Atomic_TestSetBitVector and Atomic_TestClearBitVector.
+ */
+#define ATOMIC_BITVECTOR(varName, capacity) \
+      Atomic_uint8 varName[CEILING(capacity, 8)]
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestSetBitVector --
+ *
+ *      Atomically test and set the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestSetBitVector(Atomic_uint8 *var, // IN/OUT
+                        unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; bts %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadOr8(var + index / 8, bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestClearBitVector --
+ *
+ *      Atomically test and clear the bit 'index' in bit vector var.
+ *
+ *      The index input value specifies which bit to modify and is 0-based.
+ *
+ * Results:
+ *      Returns the value of the bit before modification.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestClearBitVector(Atomic_uint8 *var, // IN/OUT
+                          unsigned index)    // IN
+{
+#if defined __x86_64__ && defined __GNUC__
+   Bool bit;
+   __asm__ __volatile__(
+      "lock; btr %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (var->value)
+      : "rI" (index)
+      : "cc", "memory"
+   );
+   return bit;
+#else
+   uint8 bit = 1 << index % 8;
+   return (Atomic_ReadAnd8(var + index / 8, ~bit) & bit) != 0;
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestBitVector --
+ *
+ *      Test the bit 'index' (zero-based) in bit vector var.
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+Atomic_TestBitVector(const Atomic_uint8 *var, // IN
+                     unsigned index)          // IN
+{
+   uint8 bit = 1 << index % 8;
+   return (Atomic_Read8(var + index / 8) & bit) != 0;
+}
+
+
 #ifdef VM_ARM_64
 #   include "vm_atomic_arm64_end.h"
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1056,7 +1056,7 @@ ClearBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1078,7 +1078,7 @@ ComplementBitVector(void *var, int32 index)
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
-      : "cc"
+      : "memory", "cc"
   );
   return bit;
 #elif defined(_MSC_VER)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -374,8 +374,8 @@ RDTSC_BARRIER(void)
 * Thanks for pasting this whole comment into every architecture header.
 *
 * On x86, we only need to care specifically about store-load reordering on
- * normal memory types. In other cases, only a compiler barrier is needed. The
- * ST_LD barrier is implemented with a locked xor operation (instead of the
+ * normal memory types. In other cases, only a compiler barrier is needed.
+ * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
 * mfence instruction) for performance reasons. See PR 1674199 for more
 * details.
 *
@@ -389,19 +389,17 @@ SMP_W_BARRIER_R(void)
 {
   volatile long temp;

-   COMPILER_MEM_BARRIER();
 #if defined __GNUC__
   __asm__ __volatile__ (
-      "lock xorl $1, %0\n"
+      "lock xorl $1, %0"
      : "+m" (temp)
      : /* no additional inputs */
-      : "cc");
+      : "cc", "memory");
 #elif defined _MSC_VER
   _InterlockedXor(&temp, 1);
 #else
 #error SMP_W_BARRIER_R not defined for this compiler
 #endif
-   COMPILER_MEM_BARRIER();
 }

 #define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,12 +37,16 @@
 #include "vm_basic_types.h" // For INLINE.

 /* Checks for FreeBSD, filtering out VMKERNEL. */
-#define __IS_FREEBSD__ (!defined(VMKERNEL) && defined(__FreeBSD__))
+#if !defined(VMKERNEL) && defined(__FreeBSD__)
+#define __IS_FREEBSD__ 1
+#else
+#define __IS_FREEBSD__ 0
+#endif
 #define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

 #if defined _WIN32 && defined USERLEVEL
   #include <stddef.h>  /*
-                         * We redefine offsetof macro from stddef; make 
+                         * We redefine offsetof macro from stddef; make
                         * sure that it's already defined before we do that.
                         */
   #include <windows.h>	// for Sleep() and LOWORD() etc.
@@ -101,7 +105,7 @@

 /* The Solaris 9 cross-compiler complains about these not being used */
 #ifndef sun
-static INLINE int 
+static INLINE int
 Min(int a, int b)
 {
   return a < b ? a : b;
@@ -113,7 +117,7 @@ Min(int a, int b)
 #endif

 #ifndef sun
-static INLINE int 
+static INLINE int
 Max(int a, int b)
 {
   return a > b ? a : b;
@@ -168,7 +172,7 @@ Max(int a, int b)
 #endif


-/* 
+/*
 * Token concatenation
 *
 * The C preprocessor doesn't prescan arguments when they are
@@ -256,15 +260,26 @@ Max(int a, int b)
 #endif

 #ifndef MBYTES_2_PAGES
-#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
+#define MBYTES_2_PAGES(_nbytes) \
+   ((uint64)(_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
 #endif

 #ifndef PAGES_2_MBYTES
 #define PAGES_2_MBYTES(_npages) ((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
 #endif

+#ifndef ROUNDUP_PAGES_2_MBYTES
+#define ROUNDUP_PAGES_2_MBYTES(_npages) \
+(((_npages) + MASK(MBYTES_SHIFT - PAGE_SHIFT)) >> (MBYTES_SHIFT - PAGE_SHIFT))
+#endif
+
+#ifndef ROUNDDOWN_PAGES_2_MBYTES
+#define ROUNDDOWN_PAGES_2_MBYTES(_npages) \
+((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
+#endif
+
 #ifndef GBYTES_2_PAGES
-#define GBYTES_2_PAGES(_nbytes) ((_nbytes) << (30 - PAGE_SHIFT))
+#define GBYTES_2_PAGES(_nbytes) ((uint64)(_nbytes) << (30 - PAGE_SHIFT))
 #endif

 #ifndef PAGES_2_GBYTES
@@ -369,7 +384,7 @@ Max(int a, int b)
 #ifdef _MSC_VER
 #ifdef __cplusplus
 extern "C"
-#endif 
+#endif
 void *_ReturnAddress(void);
 #pragma intrinsic(_ReturnAddress)
 #define GetReturnAddress() _ReturnAddress()
@@ -412,22 +427,6 @@ void *_ReturnAddress(void);

 #ifdef USERLEVEL // {

-/*
- * Note this might be a problem on NT b/c while sched_yield guarantees it
- * moves you to the end of your priority list, Sleep(0) offers no such
- * guarantee.  Bummer.  --Jeremy.
- */
-
-#if defined(_WIN32)
-#      define YIELD()		Sleep(0)
-#elif defined(VMKERNEL)
-/* We don't have a YIELD macro in the vmkernel */
-#else
-#      include <sched.h>        // For sched_yield.  Don't ask.  --Jeremy.
-#      define YIELD()		sched_yield()
-#endif 
-
-
 /*
 * Standardize some Posix names on Windows.
 */
@@ -497,19 +496,8 @@ typedef int pid_t;

 #elif defined(__APPLE__) && defined(KERNEL)

-#include "availabilityMacOS.h"
-
-#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
-// The Mac OS 10.5 kernel SDK defines va_copy in stdarg.h.
+// The macOS kernel SDK defines va_copy in stdarg.h.
 #include <stdarg.h>
-#else
-/*
- * The Mac OS 10.4 kernel SDK needs va_copy. Based on inspection of
- * stdarg.h from the MacOSX10.4u.sdk kernel framework, this should
- * work.
- */
-#define va_copy(dest, src) ((dest) = (src))
-#endif // MAC_OS_X_VERSION_MIN_REQUIRED

 #elif defined(__GNUC__) && (__GNUC__ < 3)

@@ -529,7 +517,7 @@ typedef int pid_t;
 * driver.
 */

-#ifdef _WIN32
+#if defined(_WIN32) && defined(_MSC_VER)
 #define PATH_MAX 256
 #ifndef strcasecmp
 #define strcasecmp(_s1,_s2)   _stricmp((_s1),(_s2))
@@ -539,29 +527,6 @@ typedef int pid_t;
 #endif
 #endif

-#if defined __linux__ && !defined __KERNEL__ && !defined MODULE && \
-                         !defined VMM && !defined FROBOS && !defined __ANDROID__
-#include <features.h>
-#if __GLIBC_PREREQ(2, 1) && !defined GLIBC_VERSION_21
-#define GLIBC_VERSION_21
-#endif
-#if __GLIBC_PREREQ(2, 2) && !defined GLIBC_VERSION_22
-#define GLIBC_VERSION_22
-#endif
-#if __GLIBC_PREREQ(2, 3) && !defined GLIBC_VERSION_23
-#define GLIBC_VERSION_23
-#endif
-#if __GLIBC_PREREQ(2, 4) && !defined GLIBC_VERSION_24
-#define GLIBC_VERSION_24
-#endif
-#if __GLIBC_PREREQ(2, 5) && !defined GLIBC_VERSION_25
-#define GLIBC_VERSION_25
-#endif
-#if __GLIBC_PREREQ(2, 12) && !defined GLIBC_VERSION_212
-#define GLIBC_VERSION_212
-#endif
-#endif
-
 /*
 * Convenience definitions of unicode characters.
 */
@@ -634,7 +599,7 @@ typedef int pid_t;
 #define RELEASE_ONLY(x) x
 #else
 #define vmx86_release   0
-#define RELEASE_ONLY(x) 
+#define RELEASE_ONLY(x)
 #endif

 #ifdef VMX86_SERVER
@@ -687,6 +652,12 @@ typedef int pid_t;
 #define vmw_apple_sandbox 0
 #endif

+#if defined(__APPLE__) && defined(VMW_APPLE_APP_STORE)
+#define vmw_apple_app_store 1
+#else
+#define vmw_apple_app_store 0
+#endif
+
 #ifdef VMM
 #define VMM_ONLY(x) x
 #else
@@ -699,17 +670,6 @@ typedef int pid_t;
 #define USER_ONLY(x) x
 #endif

-/* VMVISOR ifdef only allowed in the vmkernel */
-#ifdef VMKERNEL
-#ifdef VMVISOR
-#define vmvisor 1
-#define VMVISOR_ONLY(x) x
-#else
-#define vmvisor 0
-#define VMVISOR_ONLY(x)
-#endif
-#endif
-
 #ifdef _WIN32
 #define VMW_INVALID_HANDLE INVALID_HANDLE_VALUE
 #else
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -38,24 +38,6 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

-/* STRICT ANSI means the Xserver build and X defines Bool differently. */
-#if !defined(_XTYPEDEF_BOOL) && \
-    (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || \
-      defined(__MINGW32__) || defined(__APPLE__))
-#define _XTYPEDEF_BOOL
-typedef char           Bool;
-#endif
-
-#ifndef FALSE
-#define FALSE          0
-#endif
-
-#ifndef TRUE
-#define TRUE           1
-#endif
-
-#define IS_BOOL(x)     (((x) & ~1) == 0)
-
 /*
 * Macros __i386__ and __ia64 are intrinsically defined by GCC
 */
@@ -87,7 +69,8 @@ typedef char           Bool;
 */

 #ifdef __i386__
-/* VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
+/*
+ * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
 * since it is confused with the __i386__ gcc but defined for both 32- and
 * 64-bit x86. We retain it here for legacy compatibility.
 */
@@ -99,12 +82,12 @@ typedef char           Bool;

 #ifdef __x86_64__
 #define VM_X86_64
+#define vm_x86_64 1
 #define VM_I386
 #define VM_X86_ANY
 #define VM_64BIT
-#define vm_x86_64 (1)
 #else
-#define vm_x86_64 (0)
+#define vm_x86_64 0
 #endif

 #ifdef __arm__
@@ -115,13 +98,15 @@ typedef char           Bool;

 #ifdef __aarch64__
 #define VM_ARM_64
+#define vm_arm_64 1
 #define VM_ARM_ANY
 #define VM_64BIT
-#define vm_arm_64 (1)
 #else
-#define vm_arm_64 (0)
+#define vm_arm_64 0
 #endif

+#define vm_64bit (sizeof (void *) == 8)
+
 #ifdef _MSC_VER

 #pragma warning (3 :4505) // unreferenced local function
@@ -135,17 +120,15 @@ typedef char           Bool;

 #endif

-#if defined(__linux__) && defined(__cplusplus) && __cplusplus >= 201103L
-
+#if defined(__cplusplus) && __cplusplus >= 201103L || \
+    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
+    defined(__APPLE__) || defined(HAVE_STDINT_H)
 /*
- * We're using stdint.h instead of cstdint below because of libstdcpp.cpp.
- * It looks like a C++ file. When being preprocessed all the C++ specific
- * defines(e.g. __cplusplus) are set, but the C++ include paths are not.
+ * We're using <stdint.h> instead of <cstdint> below because some C++ code
+ * deliberately compiles without C++ include paths.
 */
 #include <stdint.h>

-typedef char          Bool;
-
 typedef uint64_t    uint64;
 typedef  int64_t     int64;
 typedef uint32_t    uint32;
@@ -155,56 +138,25 @@ typedef  int16_t     int16;
 typedef  uint8_t     uint8;
 typedef   int8_t      int8;

-typedef uint64 BA;
-typedef uint64 MA;
-typedef uint32 MPN32;
-
-#elif defined(__APPLE__) || defined(HAVE_STDINT_H)
-
-/*
- * TODO: This is a C99 standard header.  We should be able to test for
- * #if __STDC_VERSION__ >= 199901L, but that breaks the Netware build
- * (which doesn't have stdint.h).
- */
-
-#include <stdint.h>
-
-typedef uint64_t    uint64;
-typedef  int64_t     int64;
-typedef uint32_t    uint32;
-typedef  int32_t     int32;
-typedef uint16_t    uint16;
-typedef  int16_t     int16;
-typedef  uint8_t    uint8;
-typedef   int8_t     int8;
-
-/*
- * Note: C does not specify whether char is signed or unsigned, and
- * both gcc and msvc implement processor-specific signedness.  With
- * three types:
- * typeof(char) != typeof(signed char) != typeof(unsigned char)
- *
- * Be careful here, because gcc (4.0.1 and others) likes to warn about
- * conversions between signed char * and char *.
- */
-
 #else /* !HAVE_STDINT_H */

+/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
 #ifdef _MSC_VER

 typedef unsigned __int64 uint64;
 typedef signed __int64 int64;

 #elif __GNUC__
-/* The Xserver source compiles with -ansi -pendantic */
-#   if !defined(__STRICT_ANSI__) || defined(__FreeBSD__)
-#      if defined(VM_X86_64) || defined(VM_ARM_64)
+#   if defined(VM_X86_64) || defined(VM_ARM_64)
 typedef unsigned long uint64;
 typedef long int64;
-#      else
+#   else
+/*
+ * Only strict c90 (without extensions) lacks a 'long long' type.
+ * If this declaration fails ... use -std=c99 or -std=gnu90.
+ */
 typedef unsigned long long uint64;
 typedef long long int64;
-#      endif
 #   endif
 #else
 #   error - Need compiler define for int64/uint64
@@ -220,6 +172,36 @@ typedef signed char        int8;

 #endif /* HAVE_STDINT_H */

+
+/*
+ * The _XTYPEDEF_BOOL guard prevents colliding with:
+ * <X11/Xlib.h> #define Bool int
+ * <X11/Xdefs.h> typedef int Bool;
+ * If using this header AND X11 headers, be sure to #undef Bool and
+ * be careful about the different size.
+ */
+#if !defined(_XTYPEDEF_BOOL)
+#define _XTYPEDEF_BOOL
+/*
+ * C does not specify whether char is signed or unsigned, and
+ * both gcc and msvc implement it as a non-signed, non-unsigned type.
+ * Thus, (uint8_t *)&Bool and (int8_t *)&Bool are possible compile errors.
+ * This is intentional.
+ */
+typedef char           Bool;
+#endif
+
+#ifndef FALSE
+#define FALSE          0
+#endif
+
+#ifndef TRUE
+#define TRUE           1
+#endif
+
+#define IS_BOOL(x)     (((x) & ~1) == 0)
+
+
 /*
 * FreeBSD (for the tools build) unconditionally defines these in
 * sys/inttypes.h so don't redefine them if this file has already
@@ -344,6 +326,7 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
 */

 #ifdef _MSC_VER
+   /* MSVC added C99-compatible formatting in vs2015. */
   #define FMT64      "I64"
   #ifdef VM_X86_64
      #define FMTSZ      "I64"
@@ -355,53 +338,29 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
      #define FMTH       "I"
   #endif
 #elif defined __APPLE__
-   /* Mac OS hosts use the same formatters for 32- and 64-bit. */
-   #define FMT64 "ll"
+   /* macOS hosts use the same formatters for 32- and 64-bit. */
+   #define FMT64         "ll"
   #if KERNEL
-      #define FMTSZ "l"
+      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
+      #define FMTSZ      "l"
   #else
-      #define FMTSZ "z"
-   #endif
-   #define FMTPD "l"
-   #define FMTH ""
-#elif __GNUC__
-   #define FMTH ""
-   #if defined(sun)
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTSZ  "l"
-         #define FMTPD  "l"
-      #else
-         #define FMTSZ  ""
-         #define FMTPD  ""
-      #endif
-   #elif defined(__linux__) || \
-        (defined(__FreeBSD__) && (__FreeBSD__ + 0))\
-      || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) \
-      || (defined(_POSIX_VERSION) && _POSIX_VERSION >= 200112L) \
-      || (defined(_POSIX2_VERSION) && _POSIX2_VERSION >= 200112L)
-      /* BSD, Linux */
-      #define FMTSZ     "z"
-
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTPD  "l"
-      #else
-         #define FMTPD  ""
-      #endif
-   #else
-      /* Systems with a pre-C99 libc */
-      #define FMTSZ     "Z"
-      #if defined(VM_X86_64) || defined(VM_ARM_64)
-         #define FMTPD  "l"
-      #else
-         #define FMTPD  ""
-      #endif
+      #define FMTSZ      "z"
   #endif
+   #define FMTPD         "l"
+   #define FMTH          ""
+#elif defined __GNUC__
+   /*
+    * Every POSIX system we target has C99-compatible printf
+    * (supports 'z' for size_t and 'll' for long long).
+    */
+   #define FMTH          ""
+   #define FMTSZ         "z"
   #if defined(VM_X86_64) || defined(VM_ARM_64)
-      #define FMT64     "l"
-   #elif defined(sun) || defined(__FreeBSD__)
-      #define FMT64     "ll"
+      #define FMT64      "l"
+      #define FMTPD      "l"
   #else
-      #define FMT64     "L"
+      #define FMT64      "ll"
+      #define FMTPD      ""
   #endif
 #else
   #error - Need compiler define for FMT64 and FMTSZ
@@ -485,26 +444,23 @@ typedef uint8 *TCA;  /* Pointer into TC (usually). */
 * Type big enough to hold an integer between 0..100
 */
 typedef uint8 Percent;
-#define AsPercent(v)	((Percent)(v))
+#define AsPercent(v) ((Percent)(v))


 typedef uintptr_t VA;
 typedef uintptr_t VPN;

 typedef uint64    PA;
-typedef uint32    PPN;
+typedef uint64    PPN;
+typedef uint32    PPNTMP;

 typedef uint64    TPA;
-typedef uint32    TPPN;
+typedef uint64    TPPN;

 typedef uint64    PhysMemOff;
 typedef uint64    PhysMemSize;

-/* The Xserver source compiles with -ansi -pendantic */
-#ifndef __STRICT_ANSI__
 typedef uint64    BA;
-#endif
-
 #ifdef VMKERNEL
 typedef void     *BPN;
 #else
@@ -514,9 +470,12 @@ typedef uint64    BPN;
 #define UINT64_2_BPN(u) ((BPN)(u))
 #define BPN_2_UINT64(b) ((uint64)(b))

-typedef uint32    PageNum;
-typedef unsigned      MemHandle;
-typedef unsigned int  IoHandle;
+typedef uint64    PgCnt64;
+typedef uint64    PageCnt;
+typedef uint64    PgNum64;
+typedef uint64    PageNum;
+typedef unsigned  MemHandle;
+typedef unsigned  IoHandle;
 typedef int32     World_ID;

 /* !! do not alter the definition of INVALID_WORLD_ID without ensuring
@@ -560,11 +519,8 @@ typedef uint128 UReg128;
 typedef  Reg64  Reg;
 typedef UReg64 UReg;
 #endif
-/* The Xserver source compiles with -ansi -pendantic */
-#ifndef __STRICT_ANSI__
 typedef uint64 MA;
 typedef uint32 MPN32;
-#endif

 /*
 * This type should be used for variables that contain sector
@@ -615,6 +571,7 @@ typedef uint64 LA64;
 typedef uint64 LPN64;
 typedef uint64 PA64;
 typedef uint64 PPN64;
+typedef uint64 TPPN64;
 typedef uint64 MA64;
 typedef uint64 MPN;

@@ -640,10 +597,14 @@ typedef void * UserVA;
 #endif


+/* Maximal observable PPN value. */
 #define MAX_PPN_BITS      31
-#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1) /* Maximal observable PPN value. */
-#define INVALID_PPN       ((PPN)0xffffffff)
-#define APIC_INVALID_PPN  ((PPN)0xfffffffe)
+#define MAX_PPN           (((PPN64)1 << MAX_PPN_BITS) - 1)
+
+#define INVALID_PPN       ((PPN64)0xffffffff)
+#define INVALID_PPN32     ((PPN32)0xffffffff)
+#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
+#define APIC_INVALID_PPN  ((PPN64)0xfffffffe)

 #define INVALID_BPN       ((BPN)0x000000ffffffffffull)

@@ -663,7 +624,7 @@ typedef void * UserVA;
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
-#define INVALID_PAGENUM   ((PageNum)-1)
+#define INVALID_PAGENUM   ((uint32)-1)

 /*
 * Format modifier for printing VA, LA, and VPN.
@@ -755,6 +716,14 @@ typedef void * UserVA;
 #error "gcc version is too old to compile assembly, need gcc-3.3 or better"
 #endif

+/*
+ * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Enforce this here.
+ */
+#if defined _MSC_VER && _MSC_VER < 1400
+#error "cl.exe version is too old, need vc80 or better"
+#endif
+

 /*
 * Consider the following reasons functions are inlined:
@@ -783,7 +752,7 @@ typedef void * UserVA;
 #define INLINE_SINGLE_CALLER INLINE_ALWAYS

 /*
- * Used when a hard guaranteed of no inlining is needed. Very few
+ * Used when a hard guarantee of no inlining is needed. Very few
 * instances need this since the absence of INLINE is a good hint
 * that gcc will not do inlining.
 */
@@ -1086,33 +1055,6 @@ typedef void * UserVA;
 # define FMTMODE "o"
 #endif

-/*
- * Format modifier for printing time_t. Most platforms define a time_t to be
- * a long int, but on FreeBSD (as of 5.0, it seems), the time_t is a signed
- * size quantity. Refer to the definition of FMTSZ to see why we need silly
- * preprocessor arithmetic.
- * Use this like this: printf("The mode is %" FMTTIME ".\n", time);
- */
-#if defined(__FreeBSD__) && (__FreeBSD__ + 0) && ((__FreeBSD__ + 0) >= 5)
-#   define FMTTIME FMTSZ"d"
-#else
-#   if defined(_MSC_VER)
-#      ifndef _SAFETIME_H_
-#         if (_MSC_VER < 1400) || defined(_USE_32BIT_TIME_T)
-#             define FMTTIME "ld"
-#         else
-#             define FMTTIME FMT64"d"
-#         endif
-#      else
-#         ifndef FMTTIME
-#            error "safetime.h did not define FMTTIME"
-#         endif
-#      endif
-#   else
-#      define FMTTIME "ld"
-#   endif
-#endif
-
 #ifdef __APPLE__
 /*
 * Format specifier for all these annoying types such as {S,U}Int32
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2005-2012,2014-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2005-2012,2014-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -35,8 +35,6 @@
 #endif
 #endif

-#include <linux/pci_ids.h>
-
 /* LSILogic 53C1030 Parallel SCSI controller
 * LSILogic SAS1068 SAS controller
 */
@@ -55,10 +53,7 @@
 *    VMware HD Audio codec
 *    VMware HD Audio controller
 */
-#ifndef PCI_VENDOR_ID_VMWARE
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
-#endif
-
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
 #define PCI_DEVICE_ID_VMWARE_VGA                0x0711
@@ -75,11 +70,7 @@
 #define PCI_DEVICE_ID_VMWARE_1394               0x0780
 #define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
 #define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
-
-#ifndef PCI_DEVICE_ID_VMWARE_VMXNET3
 #define PCI_DEVICE_ID_VMWARE_VMXNET3            0x07B0
-#endif
-
 #define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
 #define PCI_DEVICE_ID_VMWARE_82574              0x07D0
 #define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
@@ -107,7 +98,16 @@
 #define PCI_DEVICE_VMI_INTERFACE        0x00
 #define PCI_DEVICE_VMI_REVISION         0x01

-#define PCI_DEVICE_ID_VMWARE_DUMMY      0x0809
+/*
+ * Device IDs for the PCI passthru test device:
+ *
+ * 0x0809 is for old fashioned PCI with MSI.
+ * 0x080A is for PCI express with MSI-X.
+ * 0x080B is for PCI express with configurable BARs.
+ */
+#define PCI_DEVICE_ID_VMWARE_PCI_TEST   0x0809
+#define PCI_DEVICE_ID_VMWARE_PCIE_TEST1 0x080A
+#define PCI_DEVICE_ID_VMWARE_PCIE_TEST2 0x080B

 #define PCI_DEVICE_ID_VMWARE_VRDMA      0x0820
 #define PCI_DEVICE_ID_VMWARE_VTPM       0x0830
@@ -124,6 +124,7 @@
 */
 #define PCI_VENDOR_ID_AMD               0x1022
 #define PCI_DEVICE_ID_AMD_VLANCE        0x2000
+#define PCI_DEVICE_ID_AMD_IOMMU         0x1577
 #define PCI_VENDOR_ID_BUSLOGIC			0x104B
 #define PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER_NC	0x0140
 #define PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER	0x1040
@@ -162,16 +163,18 @@
 #define PCI_DEVICE_ID_INTEL_LYNXPOINT_XHCI 0x8c31

 /*
- * Intel Quickassist (QAT) devices:
- * - Gen 1: Cave Creek (CVC) and Coleto Creek (CLC)
- * - Gen 2: Lewisburg (LBG)
+ * Intel Volume Management Device (VMD)
 */
-#define PCI_DEVICE_ID_INTEL_QAT_CVC     0x0434
-#define PCI_DEVICE_ID_INTEL_QAT_CLC     0x0435
-#define PCI_DEVICE_ID_INTEL_QAT_LBG     0x37c8
-#define PCI_DEVICE_ID_INTEL_QAT_CVC_VF  0x0442
-#define PCI_DEVICE_ID_INTEL_QAT_CLC_VF  0x0443
-#define PCI_DEVICE_ID_INTEL_QAT_LBG_VF  0x37c9
+#define PCI_DEVICE_ID_INTEL_VMD_V1           0x201d
+
+/*
+ * Intel Quickassist (QAT) devices.
+ */
+#define PCI_DEVICE_ID_INTEL_QAT_DH895XCC     0x0435
+#define PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF  0x0443
+
+#define PCI_DEVICE_ID_INTEL_QAT_C62X         0x37c8
+#define PCI_DEVICE_ID_INTEL_QAT_C62X_VF      0x37c9

 /*
 * Intel FPGAs
@@ -246,8 +249,12 @@

 /************* SCSI implementation limits ********************************/
 #define SCSI_MAX_CONTROLLERS	 4	  // Need more than 1 for MSCS clustering
-#define	SCSI_MAX_DEVICES	 16	  // BT-958 emulates only 16
-#define PVSCSI_MAX_DEVICES       255      // 255 (including the controller)
+#define	SCSI_MAX_DEVICES         16	  // BT-958 emulates only 16
+#define PVSCSI_HWV14_MAX_DEVICES 65	  /* HWv14 And Later Supports 64 
+					   * + controller at ID 7 
+					   */
+#define PVSCSI_MAX_DEVICES       255	  // 255 (including the controller)
+#define PVSCSI_MAX_NUM_DISKS     (PVSCSI_HWV14_MAX_DEVICES - 1)

 /************* SATA implementation limits ********************************/
 #define SATA_MAX_CONTROLLERS   4
@@ -256,10 +263,7 @@
 #define AHCI_MAX_PORTS SATA_MAX_DEVICES

 /*
- * Maximum number of supported disk in a VM.
- *
- * Note: With some config options for PVSCSI, maximum number of disks could
- * be ~1K but that number is not publicly supported yet.
+ * Publicly supported maximum number of disks per VM.
 */
 #define MAX_NUM_DISKS \
   ((SATA_MAX_CONTROLLERS * SATA_MAX_DEVICES) + \
@@ -267,6 +271,19 @@
    (NVME_MAX_CONTROLLERS * NVME_MAX_NAMESPACES) + \
    (IDE_NUM_INTERFACES * IDE_DRIVES_PER_IF))

+/*
+ * Maximum number of supported disks in a VM from HWV14 or later, using PVSCSI updated max
+ * devices.  The note above still holds true, but instead of publicly supporting
+ * all devices, HWv14 simply extends the maximum support to 256 devices,
+ * instead ~244 calculated above.
+ *
+ * PVSCSI_HW_MAX_DEVICES is 65 - allowing 64 disks + controller (at ID 7)
+ * 4 * 64 = 256 devices.
+ *
+ */
+#define MAX_NUM_DISKS_HWV14 MAX(MAX_NUM_DISKS, \
+   (SCSI_MAX_CONTROLLERS * PVSCSI_MAX_NUM_DISKS))
+
 /*
 * VSCSI_BV_INTS is the number of uint32's needed for a bit vector
 * to cover all scsi devices per target.
@@ -374,6 +391,6 @@ DEFINE_GUID(GUID_CLASS_VMWARE_USB_DEVICES,
 *
 */
 #define JEDEC_VENDOR_ID_VMWARE          0x289
-#define JEDEC_DEVICE_ID_VMWARE_NVDIMM   0x0
+#define JEDEC_DEVICE_ID_VMWARE_NVDIMM   0x1

 #endif /* VM_DEVICE_VERSION_H */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,15 +37,8 @@
 * Logging
 */

-#ifdef notdef
-#ifdef VMX86_RELEASE
 #define LOGLEVEL 0
-#else
-#define LOGLEVEL 1
-#endif
-#endif
-     
-#define LOGLEVEL 1
+

 #if LOGLEVEL >= 0
 #define LOG(level, args) ((void) (LOGLEVEL >= (level) ? (printk args) : 0))
Author	SHA1	Message	Date
Michal Kubecek	a93f44ee44	import Workstation 15.1.0 module sources	2019-05-15 07:10:51 +02:00
Michal Kubecek	b71c1b9e0b	import Workstation 15.0.1 module sources	2018-11-12 07:53:50 +01:00
Michal Kubecek	d7fe4fe97c	import Workstation 15.0.0 module sources	2018-10-02 19:58:28 +02:00
Michal Kubecek	3f188d0a76	import Workstation 14.1.3 module sources	2018-08-20 07:30:12 +02:00
Michal Kubecek	454cbd7b96	Merge branch 'master' into workstation	2018-05-27 16:13:45 +02:00
Michal Kubecek	2749950c5f	import Workstation 14.1.2 module sources	2018-05-23 18:49:52 +02:00
Michal Kubecek	7fa9cbe769	Merge branch 'master' into workstation	2018-04-02 15:37:06 +02:00
Michal Kubecek	cb66161bb5	Merge branch 'master' into workstation	2018-02-04 21:50:23 +01:00
Michal Kubecek	876363be72	import Workstation 14.1.1 module sources	2018-01-12 13:51:07 +01:00
Michal Kubecek	f78dd5288e	import Workstation 14.1.0 module sources	2017-12-25 10:06:21 +01:00
Michal Kubecek	e4df7e3461	import Workstation 14.0.0 module sources	2017-09-30 21:05:11 +02:00
Michal Kubecek	2bd541e265	import Workstation 12.5.7 module sources	2017-06-23 09:12:56 +02:00
Michal Kubecek	2a214307b9	import Workstation 12.5.6 module sources	2017-05-31 10:17:49 +02:00
Michal Kubecek	755b521dd1	import Workstation 12.5.5 module sources	2017-05-31 09:36:09 +02:00