vmmon: quick workaround for objtool warnings

As discussed in

  https://bugzilla.suse.com/show_bug.cgi?id=1059674

the reason for multiple objtool warnings is the fact that vmmon module
defines its own Panic() function which never returns. While it is marked as
such which is used by the compiler for optimization, there is no way to
find this from object file.

While this seems innocuous, it might result in problems with unwinder
later. The quickest way around is to replace vmmon's own Panic() with
standard kernel panic() until a cleaner solution is found.

vmmon-only/Makefile

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -84,8 +84,6 @@ endif
 
 ifneq ($(VM_KBUILD), no)
 
-VMCCVER := $(shell $(CC) -dumpversion)
-
 # If there is no version defined, we are in toplevel pass, not yet in kernel makefiles...
 ifeq ($(VERSION),)
 
@@ -100,14 +98,6 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi
 
-# Pass gcc version down the chain, so we can detect if kernel attempts to use unapproved compiler
-VM_CCVER := $(VMCCVER)
-export VM_CCVER
-VM_CC := $(CC)
-export VM_CC
-
-MAKEOVERRIDES := $(filter-out CC=%,$(MAKEOVERRIDES))
-
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined

vmmon-only/Makefile.kernel

@@ -24,16 +24,14 @@ INCLUDE := -I$(SRCROOT)/include -I$(SRCROOT)/common -I$(SRCROOT)/linux \
 
 EXTRA_CFLAGS := $(CC_OPTS) $(INCLUDE)
 
-EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/autoconf/smpcall.c, -DVMW_HAVE_SMP_CALL_3ARG, )
-EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/autoconf/tsc_khz.c, -DVMW_HAVE_TSC_KHZ, )
-
 obj-m += $(DRIVER).o
 
 $(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, \
-		$(wildcard $(SRCROOT)/linux/*.c $(SRCROOT)/common/*.c $(SRCROOT)/vmcore/*.c)))
+		$(wildcard $(SRCROOT)/linux/*.c $(SRCROOT)/common/*.c $(SRCROOT)/vmcore/*.c \
+		$(SRCROOT)/bootstrap/*.c)))
 
 clean:
 	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \
 		Module.symvers Modules.symvers Module.markers modules.order \
-		$(foreach dir,linux/ common/ vmcore/ \
+		$(foreach dir,linux/ common/ vmcore/ bootstrap/ \
 		./,$(addprefix $(dir),.*.cmd .*.o.flags *.o)))

vmmon-only/Makefile.normal

@@ -26,7 +26,7 @@ vm_check_build = $(shell if $(CC) $(CC_OPTS) $(INCLUDE) -Werror -S -o /dev/null
 DESTDIR := driver-$(VM_UNAME)
 
 ####
-####  DRIVERNAME should be untouched unless you have a good reason to change 
+####  DRIVERNAME should be untouched unless you have a good reason to change
 #### it.  The form below is how the scripts expect it.
 ####
 DRIVERNAME := $(DRIVER)-xxx-$(VM_UNAME)
@@ -49,7 +49,7 @@ endif
 #### If desparate, I think that bringing in <linux/bitops.h> might
 #### suffice.
 ####
-CC_WARNINGS := -Wall -Wstrict-prototypes 
+CC_WARNINGS := -Wall -Wstrict-prototypes
 # Don't use -pipe or egcs-2.91.66 (shipped with RedHat) will die
 CC_KFLAGS := -D__KERNEL__ -fno-strength-reduce -fno-omit-frame-pointer \
 	     -fno-common -DKBUILD_MODNAME=$(DRIVER)
@@ -63,7 +63,7 @@ else
 CC_KFLAGS += -DCPU=586 $(call check_gcc,-march=i586,-m486)
 endif
 
-CC_OPTS := -O2 -DMODULE -DVMMON -DVMCORE $(GLOBAL_DEFS) $(CC_KFLAGS) $(CC_WARNINGS) 
+CC_OPTS := -O2 -DMODULE -DVMMON -DVMCORE $(GLOBAL_DEFS) $(CC_KFLAGS) $(CC_WARNINGS)
 
 INCLUDE := -I$(SRCROOT)/include -I$(SRCROOT)/common -I$(SRCROOT)/linux \
 	   -I$(SRCROOT)/vmcore -I$(HEADER_DIR)
@@ -71,16 +71,16 @@ INCLUDE := -I$(SRCROOT)/include -I$(SRCROOT)/common -I$(SRCROOT)/linux \
 INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')
 
-CC_OPTS += $(call vm_check_build, $(SRCROOT)/autoconf/smpcall.c, -DVMW_HAVE_SMP_CALL_3ARG, )
-CC_OPTS += $(call vm_check_build, $(SRCROOT)/autoconf/tsc_khz.c, -DVMW_HAVE_TSC_KHZ, )
-
 C_TARGETS_LINUX  := driver.o hostif.o driverLog.o
 C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o hashFunc.o
 C_TARGETS_VMCORE := moduleloop.o
+C_TARGETS_BOOTSTRAP := bootstrap.o
 C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d}
 C_TARGETS_COMMON_D := ${C_TARGETS_COMMON:.o=.d}
 C_TARGETS_VMCORE_D := ${C_TARGETS_VMCORE:.o=.d}
-C_TARGETS := $(C_TARGETS_LINUX) $(C_TARGETS_COMMON) $(C_TARGETS_VMCORE)
+C_TARGETS_BOOTSTRAP_D := ${C_TARGETS_BOOTSTRAP:.o=.d}
+C_TARGETS := $(C_TARGETS_LINUX) $(C_TARGETS_COMMON) $(C_TARGETS_VMCORE) \
+             $(C_TARGETS_BOOTSTRAP)
 
 ####
 #### Make Targets are beneath here.
@@ -116,6 +116,10 @@ $(C_TARGETS_VMCORE): %.o: $(SRCROOT)/vmcore/%.c
 	$(ECHO) "Compiling  vmcore/$(<F)"
 	$(CC) $(CC_OPTS) $(INCLUDE) -c $<
 
+$(C_TARGETS_BOOTSTRAP): %.o: $(SRCROOT)/bootstrap/%.c
+	$(ECHO) "Compiling  bootstrap/$(<F)"
+	$(CC) $(CC_OPTS) $(INCLUDE) -c $<
+
 clean:
 	rm -rf $(DESTDIR)/
 
@@ -131,8 +135,12 @@ $(C_TARGETS_VMCORE_D): %.d: $(SRCROOT)/vmcore/%.c
 	$(ECHO) "Dependencies for $(<F)"
 	$(CC) -MM $(CC_OPTS) $(INCLUDE) $< > $@
 
-deps: setup 
-	$(MAKE) -C $(DESTDIR) -f ../Makefile SRCROOT=../$(SRCROOT) driver_deps 
+$(C_TARGETS_BOOTSTRAP_D): %.d: $(SRCROOT)/bootstrap/%.c
+	$(ECHO) "Dependencies for $(<F)"
+	$(CC) -MM $(CC_OPTS) $(INCLUDE) $< > $@
+
+deps: setup
+	$(MAKE) -C $(DESTDIR) -f ../Makefile SRCROOT=../$(SRCROOT) driver_deps
 
 driver_deps: ${C_TARGETS:.o=.d}
 

vmmon-only/README

@@ -3,11 +3,11 @@ portion of the VMware Virtual Machine Monitor.  In order to build, make
 certain the Makefile is correct, especially in whether or not your system
 is multi-processor and then just type
 
-	make
+        make
 
 from this directory.  A copy of the module will be left in
 
-	driver-<kernel version>/vmmon.o
+        driver-<kernel version>/vmmon.o
 
 (e.g. driver-up-2.0.32/vmmon.o).
 

vmmon-only/autoconf/smpcall.c

@@ -1,37 +0,0 @@
-/*********************************************************
- * Copyright (C) 2008 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * Detect whether smp_call_function has 4 or 3 arguments.
- * Change happened between 2.6.26 and 2.6.27-rc1.
- */
-
-#include "compat_version.h"
-#include "compat_autoconf.h"
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
-#   error This compile test intentionally fails.
-#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
-#   include <linux/smp.h>
-
-int
-vmware_smp_call_function(void (*func)(void *info), void *info, int wait)
-{
-   return smp_call_function(func, info, wait);
-}
-#endif

vmmon-only/bootstrap/bootstrap.c

@@ -17,16 +17,37 @@
  *********************************************************/
 
 /*
- * Detect if tsc_khz is available.
+ * bootstrap.c --
+ *
+ *    Implements the early VMM bootstraping code that is executed
+ *    by the host (vmmon/VMKernel) to create the VMM context.
  */
 
-#include "compat_version.h"
-#include "compat_autoconf.h"
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "bootstrap_vmm.h"
 
-#include <linux/cpu.h>
-
-int
-vmw_tsc_khz(void)
+/*
+ *---------------------------------------------------------------------
+ *
+ * BSVMM_Validate --
+ *
+ *    Validates the VMM bootstrap blob. For now, we do it by checking
+ *    the magic number. Returns the bootstrap parameter table if
+ *    successful, NULL otherwise.
+ *
+ *---------------------------------------------------------------------
+ */
+BSVMM_HostParams *
+BSVMM_Validate(void *buf, uint32 nbytes)
 {
-    return tsc_khz;
+   BSVMM_HostParams *bsParams = buf;
+
+   if (nbytes < sizeof *bsParams) {
+      return NULL;
+   }
+   if (bsParams->magic != BOOTSTRAP_MAGIC) {
+      return NULL;
+   }
+   return bsParams;
 }

vmmon-only/bootstrap/monLoader.c

@@ -0,0 +1,997 @@
+/*********************************************************
+ * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * monLoader.c --
+ *
+ *      Processes the monitor loader header.
+ *
+ *      See monLoader.h for a full description.
+ */
+
+#ifdef VMX86_VMX
+
+#include <string.h>
+#include <limits.h>
+#include "vmx.h"
+#include "vm_assert.h"
+
+#elif defined VMKERNEL
+
+#include "vmkernel.h"
+#include "libc.h"
+#include "vm_libc.h"
+
+#define UINT_MAX (~0U)
+
+#elif defined VMMON
+
+#include "hostKernel.h"
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h>
+#   include <linux/kernel.h>
+#elif defined __APPLE__
+#   include <machine/limits.h>
+#   include <string.h>
+#else
+#   include <limits.h>
+#   include <string.h>
+#endif
+
+#include "vm_assert.h"
+
+#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#error MonLoader cannot be built as part of this environment
+#endif
+
+#include "vm_basic_types.h"
+#include "monLoader.h"
+#include "vcpuid.h"
+#include "x86types.h"
+#include "x86paging_64.h"
+#include "monLoaderLog.h"
+
+#define CANONICAL_MASK MASK64(36)
+/* The index of VPN v within an array of LxEs where index 0 maps VPN b. */
+#define L4EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 36))
+#define L3EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 27))
+#define L2EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 18))
+#define L1EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 9))
+
+/*
+ * Import address space from VMX rather than building it.  Once monitor loading
+ * is removed from VMX entirely, code to import will be deleted.
+ */
+#define IMPORT_AS_FROM_VMX (TRUE)
+
+/* The maximum (canonical-address) VPN */
+#define VPN_MAX MASK64(52)
+
+/* Sufficient MPN counts to accommodate the monitor's top 64MB. */
+#define L4MPNCOUNTMAX  1
+#define L3MPNCOUNTMAX  1
+#define L2MPNCOUNTMAX  1
+#define L1MPNCOUNTMAX 32
+
+#define LINE_INVALID  ((unsigned)-1)
+
+typedef struct MonPTMPNs {
+   MPN L4MPNs[L4MPNCOUNTMAX]; /* The page root.  Definitely only one page. */
+   MPN L3MPNs[L3MPNCOUNTMAX];
+   MPN L2MPNs[L2MPNCOUNTMAX];
+   MPN L1MPNs[L1MPNCOUNTMAX];
+   unsigned L4MPNCount;
+   unsigned L3MPNCount;
+   unsigned L2MPNCount;
+   unsigned L1MPNCount;
+} MonPTMPNs;
+ 
+typedef struct MonLoaderContext {
+   MonLoaderEnvContext *envCtx; /* Environment-specific context */
+   struct {
+      MonPTMPNs    ptMPNs;      /* Mappings into the AS for the current VCPU. */
+      VPN          ASFirstVPN;  /* first VPN in the address space */
+      VPN          ASLastVPN;   /* last VPN in the address space (inclusive) */
+      uint64       ASPTEFlags;  /* PTE flags for L4->L1 connection. */
+      Vcpuid       currentVCPU;
+      Bool         ASImported;  /* Was this AS imported or created anew? */
+      Bool         hasAddrSpace;
+   } vcpu;
+} MonLoaderContext;
+
+
+static const Bool importFromVMX = IMPORT_AS_FROM_VMX;
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderTranslateMonVPNToL1E --
+ *
+ *      Translates a monitor VPN to a level 1 page table entry, for the current
+ *      VCPU.  Assumes preallocated and connected page tables, so only the L1
+ *      table is examined.
+ *
+ * Result:
+ *      ML_OK and a PTE if successful, some other value on error.  Fails if a
+ *      page table page is not already allocated or the VPN is not in the
+ *      context's address space's defined range.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderTranslateMonVPNToL1E(MonLoaderContext *ctx,    // IN/OUT
+                              VPN               monVPN, // IN
+                              PT_L1E           *pte)    // OUT
+{
+   Vcpuid vcpu = ctx->vcpu.currentVCPU;
+   /* Only modify the L1E as L4->L1 are guaranteed connected already. */
+   unsigned L1Off = PT_LPN_2_L1OFF(monVPN);
+   unsigned L1Page = L1EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
+
+   if (monVPN < ctx->vcpu.ASFirstVPN || monVPN > ctx->vcpu.ASLastVPN) {
+      return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
+   }
+   if (ctx->vcpu.ptMPNs.L1MPNs[L1Page] == INVALID_MPN) {
+      return ML_ERROR_MAP; /* Page table not sufficiently preallocated. */
+   }
+   if (!MonLoaderCallout_GetPTE(ctx->envCtx, ctx->vcpu.ptMPNs.L1MPNs[L1Page],
+                                L1Off, vcpu, pte)) {
+      return ML_ERROR_CALLOUT_GETPTE;
+   }
+   return ML_OK;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderIsMapped --
+ *
+ *      Checks whether a VPN has an existing mapping on the current VCPU.
+ *      Assumes preallocated and connected page tables.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.  Fails if a page table
+ *      page is not already allocated or the VPN is not in the context's
+ *      address space's defined range.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderIsMapped(MonLoaderContext *ctx,    // IN/OUT
+                  VPN               monVPN, // IN
+                  Bool             *mapped) // OUT
+{
+   PT_L1E pte;
+   MonLoaderError ret = MonLoaderTranslateMonVPNToL1E(ctx, monVPN, &pte);
+
+   if (ret != ML_OK) {
+      return ret;
+   }
+   *mapped = PTE_PRESENT(pte);
+   return ML_OK;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderAllocMPN --
+ *
+ *      Allocates a new MPN.
+ *
+ * Result:
+ *      ML_OK and an MPN, on success.  An error and no MPN on failure.
+ *
+ * Side effects:
+ *      An MPN may be allocated.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderAllocMPN(MonLoaderContext *ctx, // IN
+                  MPN *mpn)              // OUT
+{
+   *mpn = MonLoaderCallout_AllocMPN(ctx->envCtx, ctx->vcpu.currentVCPU);
+   if (*mpn == INVALID_MPN) {
+      return ML_ERROR_ALLOC;
+   }
+   return ML_OK;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderMapMPN --
+ *
+ *      Maps an MPN into the current VCPU's address space at a VPN.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.  Fails if a page table
+ *      page is not already allocated or the VPN is not in the context's
+ *      address space's defined range.
+ *
+ * Side effects:
+ *      An MPN is mapped on success.  Nothing on failure.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderMapMPN(MonLoaderContext *ctx,    // IN/OUT
+                MPN               mpn,    // IN
+                uint64            flags,  // IN
+                VPN               monVPN) // IN
+{
+   MonPTMPNs *ptMPNs = &ctx->vcpu.ptMPNs;
+   /* Only modify the L1E, as L4->L1 are guaranteed connected already. */
+   unsigned L1Off = PT_LPN_2_L1OFF(monVPN);
+   unsigned L1Page = L1EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
+
+   if (vmx86_debug) {
+      MonLoaderError ret;
+      Bool mapped;
+      if (monVPN < ctx->vcpu.ASFirstVPN || monVPN > ctx->vcpu.ASLastVPN) {
+         LOG(5, "%s: monVPN=0x%"FMTVPN"x, AS=0x%"FMTVPN"x-0x%"FMTVPN"x\n",
+                 __FUNCTION__, monVPN, ctx->vcpu.ASFirstVPN,
+                 ctx->vcpu.ASLastVPN);
+         return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
+      }
+      if (ptMPNs->L1MPNs[L1Page] == INVALID_MPN) {
+         return ML_ERROR_MAP; /* Page table not sufficiently preallocated. */
+      }
+      ret = MonLoaderIsMapped(ctx, monVPN, &mapped);
+      if (ret != ML_OK) {
+         return ret;
+      }
+      if (mapped) {
+         return ML_ERROR_ALREADY_MAPPED;
+      }
+   }
+   if (!MonLoaderCallout_MapMPNInPTE(ctx->envCtx, ptMPNs->L1MPNs[L1Page], L1Off,
+                                     flags, mpn, ctx->vcpu.currentVCPU)) {
+      return ML_ERROR_CALLOUT_MAPINPTE;
+   }
+   return ML_OK;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCreateAddressSpace --
+ *
+ *      Creates or (if importAS is set) verifies an address space.  The VPN
+ *      range specified by firstVPN and size is used to determine page counts
+ *      at each page table level to map every page.  Sufficient pages are then
+ *      allocated or verified.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.
+ *
+ * Side effects:
+ *      MPNs are allocated if not importing.  Sets ctx->vcpu.hasAddrSpace.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderCreateAddressSpace(MonLoaderContext *ctx,      // IN/OUT
+                            VPN               firstVPN, // IN
+                            uint64            flags,    // IN
+                            uint64            monPages, // IN
+                            Bool              importAS) // IN
+{
+   Vcpuid vcpu = ctx->vcpu.currentVCPU;
+   VPN lastVPN = firstVPN + monPages - 1;
+   unsigned L3MPNsNeeded = L3EArrayIdx(0, lastVPN) -
+                           L3EArrayIdx(0, firstVPN) + 1;
+   unsigned L2MPNsNeeded = L2EArrayIdx(0, lastVPN) -
+                           L2EArrayIdx(0, firstVPN) + 1;
+   unsigned L1MPNsNeeded = L1EArrayIdx(0, lastVPN) -
+                           L1EArrayIdx(0, firstVPN) + 1;
+   unsigned i;
+   MPN mpn;
+   MonPTMPNs *ptMPNs;
+
+   if (vcpu == VCPUID_INVALID) {
+      return ML_ERROR_ARGS;
+   }
+
+   /* Verify MonPTMPNs is large enough for this address space. */
+   if (L3MPNsNeeded > L3MPNCOUNTMAX || L2MPNsNeeded > L2MPNCOUNTMAX ||
+       L1MPNsNeeded > L1MPNCOUNTMAX) {
+      return ML_ERROR_ADDRSPACE_TOO_LARGE;
+   }
+   ctx->vcpu.ASFirstVPN = firstVPN;
+   ctx->vcpu.ASLastVPN = lastVPN;
+   ctx->vcpu.ASPTEFlags = flags;
+   ctx->vcpu.ASImported = importAS;
+
+   ptMPNs = &ctx->vcpu.ptMPNs;
+
+   if (importAS) {
+      mpn = MonLoaderCallout_GetPageRoot(ctx->envCtx, vcpu);
+      if (mpn == INVALID_MPN) {
+         return ML_ERROR_CALLOUT_PAGEROOT_GET;
+      }
+      LOG(5, "%s: vcpu %u page root=0x%"FMT64"x\n", __FUNCTION__, vcpu, mpn);
+      ptMPNs->L4MPNs[0] = mpn;
+      MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
+   } else {
+      NOT_IMPLEMENTED();
+   }
+   ptMPNs->L4MPNCount = 1;
+
+   for (i = 0; i < L3MPNsNeeded; i++) {
+      if (importAS) {
+         VPN monVPN = firstVPN + i * PT_PAGES_PER_L4E;
+         unsigned L4Off = PT_LPN_2_L4OFF(monVPN);
+         PT_L1E pte;
+         if (!MonLoaderCallout_GetPTE(ctx->envCtx, ptMPNs->L4MPNs[0], L4Off,
+                                      vcpu, &pte)) {
+            return ML_ERROR_CALLOUT_GETPTE;
+         }
+         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L4E=0x%"FMT64"x\n", __FUNCTION__,
+                 monVPN, pte);
+         mpn = LM_PTE_2_PFN(pte);
+         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
+            return ML_ERROR_PAGE_TABLE_IMPORT;
+         }
+         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
+      } else {
+         NOT_IMPLEMENTED();
+      }
+      ptMPNs->L3MPNs[i] = mpn;
+   }
+   ptMPNs->L3MPNCount = L3MPNsNeeded;
+      
+   for (i = 0; i < L2MPNsNeeded; i++) {
+      if (importAS) {
+         VPN monVPN = firstVPN + i * PT_PAGES_PER_L3E;
+         unsigned L3Off = PT_LPN_2_L3OFF(monVPN);
+         unsigned L3Page = L3EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
+         PT_L1E pte;
+         if (!MonLoaderCallout_GetPTE(ctx->envCtx, ptMPNs->L3MPNs[L3Page],
+                                      L3Off, vcpu, &pte)) {
+            return ML_ERROR_CALLOUT_GETPTE;
+         }
+         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L3E=0x%"FMT64"x\n", __FUNCTION__,
+                 monVPN, pte);
+         mpn = LM_PTE_2_PFN(pte);
+         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
+            return ML_ERROR_PAGE_TABLE_IMPORT;
+         }
+         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
+      } else {
+         NOT_IMPLEMENTED();
+      }
+      ptMPNs->L2MPNs[i] = mpn;
+   }
+   ptMPNs->L2MPNCount = L2MPNsNeeded;
+
+   for (i = 0; i < L1MPNsNeeded; i++) {
+      if (importAS) {
+         VPN monVPN = firstVPN + i * PT_PAGES_PER_L2E;
+         unsigned L2Off = PT_LPN_2_L2OFF(monVPN);
+         unsigned L2Page = L2EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN);
+         PT_L1E pte;
+         if (!MonLoaderCallout_GetPTE(ctx->envCtx, ptMPNs->L2MPNs[L2Page],
+                                      L2Off, vcpu, &pte)) {
+            return ML_ERROR_CALLOUT_GETPTE;
+         }
+         LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__,
+                 monVPN, pte);
+         mpn = LM_PTE_2_PFN(pte);
+         if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) {
+            return ML_ERROR_PAGE_TABLE_IMPORT;
+         }
+         MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu);
+      } else {
+         NOT_IMPLEMENTED();
+      }
+      ptMPNs->L1MPNs[i] = mpn;
+   }
+   ptMPNs->L1MPNCount = L1MPNsNeeded;
+
+   ctx->vcpu.hasAddrSpace = TRUE;
+   return ML_OK;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderMapPageTables --
+ *
+ *      Maps in the page tables for the specified level, or verifies that the
+ *      existing mappings match internal state and flags if the address space
+ *      was imported.  Assumes pre-allocated memory at all levels and
+ *      pre-connected L4->L1 page tables.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.
+ *
+ * Side effects:
+ *      Page tables are updated if mappings are created.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderMapPageTables(MonLoaderContext *ctx,      // IN/OUT
+                       unsigned          level,    // IN
+                       uint64            flags,    // IN
+                       VPN               monVPN,   // IN
+                       uint64            monPages) // IN
+{
+   Bool verify = ctx->vcpu.ASImported;
+   uint64    i;
+   unsigned  count;
+   MPN      *ptMPNs;
+
+   if (vmx86_debug) {
+      if (!ctx->vcpu.hasAddrSpace) {
+         return ML_ERROR_NO_ADDRSPACE;
+      }
+      if (level < 1 || level > 4) {
+         return ML_ERROR_ARGS;
+      }
+   }
+
+   switch (level) {
+      case 4:
+         count = ctx->vcpu.ptMPNs.L4MPNCount;
+         ptMPNs = ctx->vcpu.ptMPNs.L4MPNs;
+         break;
+      case 3:
+         count = ctx->vcpu.ptMPNs.L3MPNCount;
+         ptMPNs = ctx->vcpu.ptMPNs.L3MPNs;
+         break;
+      case 2:
+         count = ctx->vcpu.ptMPNs.L2MPNCount;
+         ptMPNs = ctx->vcpu.ptMPNs.L2MPNs;
+         break;
+      case 1:
+         count = ctx->vcpu.ptMPNs.L1MPNCount;
+         ptMPNs = ctx->vcpu.ptMPNs.L1MPNs;
+         break;
+      default:
+         return ML_ERROR_ARGS;
+         break;
+
+      if (monPages != count) {
+         return ML_ERROR_PAGE_TABLE_MAP_SIZE;
+      }
+      
+      for (i = 0; i < monPages; i++) {
+         VPN vpn = monVPN + i;
+         if (verify) {
+            PT_L1E l1e;
+            if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK ||
+                LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) {
+               return ML_ERROR_PAGE_TABLE_VERIFY;
+            }
+         } else {
+            NOT_IMPLEMENTED();
+         }
+      }
+   }
+   return ML_OK;
+} 
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderZero --
+ *
+ *      Allocates new pages, zeroes them and maps them on the current VCPU.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.  The number of
+ *      MPNs allocated in *allocs is incremented.
+ *
+ * Side effects:
+ *      Allocations, zeroing, mapping and accounting are performed.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderZero(MonLoaderContext *ctx,      // IN/OUT
+              uint64            flags,    // IN
+              VPN               monVPN,   // IN
+              uint64            numPages, // IN
+              uint64            monPages, // IN
+              unsigned         *allocs)   // IN/OUT
+{
+   uint64 i;
+
+   if (numPages == 0 || numPages > monPages) {
+      return ML_ERROR_SIZE;
+   }
+   for (i = 0; i < numPages; i++) {
+      MPN mpn;
+      MonLoaderError ret = MonLoaderAllocMPN(ctx, &mpn);
+      if (ret != ML_OK) {
+         return ret;
+      }
+      if (!MonLoaderCallout_FillPage(ctx->envCtx, 0, mpn,
+                                     ctx->vcpu.currentVCPU)) {
+         return ML_ERROR_CALLOUT_ZERO;
+      }
+      ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN + i);
+      if (ret != ML_OK) {
+         return ret;
+      }
+      *allocs += 1;
+   }
+   return ML_OK;
+}
+   
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCopyFromBlob --
+ *
+ *      Allocates new pages, copies blob contents and maps the pages.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.  The number of
+ *      MPNs allocated in *allocs is incremented.
+ *
+ * Side effects:
+ *      Allocations, copying, mapping and accounting are performed.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderCopyFromBlob(MonLoaderContext *ctx,        // IN/OUT
+                      uint64            flags,      // IN
+                      VPN               monVPN,     // IN
+                      uint64            monBytes,   // IN
+                      uint64            blobOffset, // IN
+                      uint64            blobSize,   // IN
+                      unsigned         *allocs)     // IN/OUT
+{
+   uint64 bytesLeft = blobSize;
+
+   if (blobSize > monBytes || blobSize == 0) {
+      return ML_ERROR_SIZE; /* Entry corrupt: size mismatch. */
+   }
+   while (bytesLeft) {
+      MPN mpn;
+      uint64 toCopy = MIN(bytesLeft, PAGE_SIZE);
+
+      MonLoaderError ret = MonLoaderAllocMPN(ctx, &mpn);
+      if (ret != ML_OK) {
+         return ret;
+      }
+
+      ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN);
+      if (ret != ML_OK) {
+         return ret;
+      }
+
+      if (!MonLoaderCallout_CopyFromBlob(ctx->envCtx, blobOffset, toCopy, mpn,
+                                         ctx->vcpu.currentVCPU)) {
+         return ML_ERROR_CALLOUT_COPY;
+      }
+
+      monVPN++;
+      blobOffset += toCopy;
+      bytesLeft -= toCopy;
+      *allocs += 1;
+   }
+   return ML_OK;
+}
+ 
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderShareWork --
+ *
+ *      Maps user or host pages in.  monPages is a maximum, so partial sharing
+ *      is considered successful and remaining monitor pages will not be mapped.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.
+ *
+ * Side effects:
+ *      User pages may be mapped in.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderShareWork(MonLoaderContext *ctx,        // IN/OUT
+                   uint64            flags,      // IN
+                   VPN               monVPN,     // IN
+                   uint64            monPages,   // IN
+                   uint64            subIndex,   // IN
+                   Bool              user)       // IN
+{
+   Vcpuid vcpu = ctx->vcpu.currentVCPU;
+   uint64 i;
+
+   if (monPages == 0 || monPages > UINT_MAX) {
+      return ML_ERROR_SIZE; /* Entry corrupt: size mismatch. */
+   }
+   for (i = 0; i < monPages; i++) {
+      MPN mpn;
+      MonLoaderError ret;
+      unsigned pgNum = (unsigned)i;
+
+      if (user) {
+         mpn = MonLoaderCallout_GetSharedUserPage(ctx->envCtx, subIndex, pgNum,
+                                                  vcpu);
+      } else {
+         mpn = MonLoaderCallout_GetSharedHostPage(ctx->envCtx, subIndex, pgNum,
+                                                  vcpu);
+      }
+      if (mpn == INVALID_MPN) {
+         /* Partial sharing is allowed.  Return success if any occurred. */
+         return i != 0 ? ML_OK : ML_ERROR_SHARE;
+      }
+      ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN + i);
+      if (ret != ML_OK) {
+         return ret;
+      }
+   }
+   return ML_OK;
+}
+
+
+#ifndef VMMON
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderSetEntrypoint --
+ *
+ *      Calls out to set the code and stack entrypoint.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.
+ *
+ * Side effects:
+ *      The monitor entrypoint for the current VCPU may be set.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderSetEntrypoint(MonLoaderContext *ctx,           // IN
+                       uint16            codeSelector,  // IN
+                       VA64              code,          // IN
+                       uint16            stackSelector, // IN
+                       VA64              stack)         // IN
+{
+   if (VA_2_VPN(code) < ctx->vcpu.ASFirstVPN  ||
+       VA_2_VPN(code) > ctx->vcpu.ASLastVPN   ||
+       VA_2_VPN(stack) < ctx->vcpu.ASFirstVPN ||
+       VA_2_VPN(stack) > ctx->vcpu.ASLastVPN) {
+      return ML_ERROR_INVALID_VPN; /* Address is outside address space. */
+   }
+   if (MonLoaderCallout_SetEntrypoint(ctx->envCtx, codeSelector, code,
+                                      stackSelector, stack)) {
+      return ML_OK;
+   } else {
+      return ML_ERROR_CALLOUT_ENTRYPOINT;
+   }
+}
+#endif
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderShareFromUser --
+ *
+ *      Maps user pages in.  monPages is a maximum, so partial sharing is
+ *      considered successful and remaining monitor pages will not be mapped.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.
+ *
+ * Side effects:
+ *      User pages may be mapped in.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderShareFromUser(MonLoaderContext *ctx,        // IN/OUT
+                       uint64            flags,      // IN
+                       VPN               monVPN,     // IN
+                       uint64            monPages,   // IN
+                       uint64            subIndex)   // IN
+{
+   return MonLoaderShareWork(ctx, flags, monVPN, monPages, subIndex, TRUE);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderShareFromHost --
+ *
+ *      Maps host pages in.  monPages is a maximum, so partial sharing is
+ *      considered successful and remaining monitor pages will not be mapped.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.
+ *
+ * Side effects:
+ *      Host pages may be mapped in.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderShareFromHost(MonLoaderContext *ctx,        // IN/OUT
+                       uint64            flags,      // IN
+                       VPN               monVPN,     // IN
+                       uint64            monPages,   // IN
+                       uint64            subIndex)   // IN
+{
+   return MonLoaderShareWork(ctx, flags, monVPN, monPages, subIndex, FALSE);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderShareFromBlob --
+ *
+ *      Maps blob pages in. monPages is a maximum, so partial sharing is
+ *      considered successful and remaining monitor pages will not be
+ *      mapped. blobOffset must be page-aligned and blobSize must be a
+ *      multiple of PAGE_SIZE. Writeable mappings are not allowed.
+ *
+ *     Result:
+ *          ML_OK if successful, some other value on error.
+ *
+ *     Side effects:
+ *          Blob pages may be mapped in.
+ *
+ *----------------------------------------------------------------------
+ */
+static MonLoaderError
+MonLoaderShareFromBlob(MonLoaderContext *ctx,        // IN/OUT
+                       uint64            flags,      // IN
+                       VPN               monVPN,     // IN
+                       uint64            monBytes,   // IN
+                       uint64            blobOffset, // IN
+                       uint64            blobSize)   // IN
+{
+   uint64 endOff = blobOffset + blobSize;
+
+   if ((blobOffset & PAGE_MASK) != 0 || (blobSize & PAGE_MASK) != 0 ||
+       (flags & PTE_RW) != 0) {
+      return ML_ERROR_SHARE;
+   }
+   if (blobSize > monBytes || blobSize == 0) {
+      return ML_ERROR_SIZE;
+   }
+   while (blobOffset < endOff) {
+      MonLoaderError ret;
+      MPN mpn = MonLoaderCallout_GetBlobMpn(ctx->envCtx, blobOffset);
+      if (mpn == INVALID_MPN) {
+         return ML_ERROR_INVALID_VPN;
+      }
+      ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN);
+      if (ret != ML_OK) {
+         return ret;
+      }
+      blobOffset += PAGE_SIZE;
+      monVPN++;
+   }
+   ASSERT(blobOffset == endOff);
+   return ML_OK;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoader_Process --
+ *
+ *      Builds and populates a monitor address space per the given a header and
+ *      VCPU count.  Calls out to support functions.
+ *
+ * Result:
+ *      ML_OK if successful, some other value on error.  If an error occurs,
+ *      *line contains the line in the table at which the failure occurred and
+ *      *vcpu contains the VCPU for which processing was running.
+ *
+ * Side effects:
+ *      An address space and page table mappings are created or verified, for
+ *      all VCPUs.  Callouts are called.
+ *
+ *----------------------------------------------------------------------
+ */
+MonLoaderError
+MonLoader_Process(MonLoaderHeader  *header,   // IN/OUT
+                  unsigned          numVCPUs, // IN
+                  void             *args,     // IN
+                  unsigned         *line,     // OUT
+                  Vcpuid           *vcpu)     // OUT
+{
+   unsigned i;
+   MonLoaderError ret;
+   MonLoaderContext ctx;
+
+   if (numVCPUs > MAX_VCPUS) {
+      return ML_ERROR_ARGS;
+   }
+
+   if (header->magic != LOADER_HEADER_MAGIC) {
+      return ML_ERROR_MAGIC;
+   }
+
+   if (vmx86_debug && !line) {
+      return ML_ERROR_ARGS;
+   }
+   *line = LINE_INVALID;
+
+   if (header->entries == 0 || header->count == 0) {
+      return ML_ERROR_TABLE_MISSING;
+   }
+
+   if (!MonLoaderCallout_Init(args, &ctx.envCtx)) {
+      return ML_ERROR_CALLOUT_INIT;
+   }
+   ret = ML_OK;
+   for (*vcpu = 0; *vcpu < numVCPUs && ret == ML_OK; (*vcpu) += 1) {
+      /* Reset VCPU-specific part of context. */
+      memset(&ctx.vcpu, 0, sizeof ctx.vcpu);
+      ctx.vcpu.currentVCPU = *vcpu;
+      ctx.vcpu.hasAddrSpace = FALSE;
+      for (i = 0; i < header->count && ret == ML_OK; i++) {
+         MonLoaderEntry *entry = &header->entries[i];
+         MonLoaderContentType content    = entry->content;
+         MonLoaderSourceType  source     = entry->source;
+         VPN                  monVPN     = entry->monVPN;
+         uint64               blobOffset = entry->blobSrc.offset;
+         uint64               blobSize   = entry->blobSrc.size;
+         uint64               flags      = entry->flags;
+         uint64               monPages   = entry->monPages;
+         uint64               monBytes   = PAGES_2_BYTES(monPages);
+         uint64               subIndex   = entry->subIndex;
+         uint64               bspOnly    = entry->bspOnly;
+         uint64               isPriv     = vmx86_server || entry->procVmmon;
+         *line = i;
+
+         /*
+          * The entry is specific to the bootstrap, which only runs in VCPU 0.
+          */
+         if (bspOnly && !IS_BOOT_VCPUID(*vcpu)) {
+            continue;
+         }
+         /*
+          * Entries marked as privileged (isPriv == TRUE) are processed
+          * by the privileged layer (VMKernel/vmmon). Others are
+          * processed by the VMX. One exception is the entry of type
+          * ML_CONTENT_ADDRSPACE: it is always processed by the VMX and,
+          * if marked as privileged, also by the privileged layer.
+          */
+         if (isPriv != MonLoaderCallout_IsPrivileged(ctx.envCtx) &&
+             (content != ML_CONTENT_ADDRSPACE || isPriv == 0)) {
+            continue;
+         }
+         if (monPages == 0 || monVPN + monPages - 1 > VPN_MAX) {
+            ret = ML_ERROR_SIZE;
+            continue;
+         }
+
+         switch (content) {
+            case ML_CONTENT_ADDRSPACE:
+               if (ctx.vcpu.hasAddrSpace) {
+                  ret = ML_ERROR_DUPLICATE;
+                  continue;
+               }
+               /* Create or verify the address space and page table MPNs. */
+               ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages,
+                                                 importFromVMX);
+               break;
+            case ML_CONTENT_PAGETABLE_L4:
+            case ML_CONTENT_PAGETABLE_L3:
+            case ML_CONTENT_PAGETABLE_L2:
+            case ML_CONTENT_PAGETABLE_L1:
+               /* Create or verify page table mappings. */
+               ret = MonLoaderMapPageTables(&ctx, CONTENT_TO_PTLEVEL(content),
+                                            flags, monVPN, monPages);
+               break;
+            case ML_CONTENT_ALLOCZERO: {
+               uint64 numPages;
+               if (source == ML_SOURCE_NONE) {
+                  numPages = monPages;
+               } else if (source == ML_SOURCE_BLOB) {
+                  /*
+                   * Used for loading the uninitialized data section
+                   * (.bss), which has a blob size but no blob image
+                   * associated with it.
+                   */
+                  numPages = CEILING(blobSize, PAGE_SIZE);
+               } else {
+                  ret = ML_ERROR_SOURCE_INVALID;
+                  break;
+               }
+               ret = MonLoaderZero(&ctx, flags, monVPN, numPages, monPages,
+                                   &entry->allocs);
+               break;
+            }
+            case ML_CONTENT_COPY:
+               switch (source) {
+                  case ML_SOURCE_BLOB:
+                     ret = MonLoaderCopyFromBlob(&ctx, flags, monVPN, monBytes,
+                                                 blobOffset, blobSize,
+                                                 &entry->allocs);
+                     break;
+                  default:
+                     ret = ML_ERROR_SOURCE_INVALID;
+                     break;
+               }
+               break;
+            case ML_CONTENT_SHARE:
+               switch (source) {
+                  case ML_SOURCE_USER:
+                     ret = MonLoaderShareFromUser(&ctx, flags, monVPN,
+                                                  monPages, subIndex);
+                     break;
+                  case ML_SOURCE_HOST:
+                     ret = MonLoaderShareFromHost(&ctx, flags, monVPN,
+                                                  monPages, subIndex);
+                     break;
+                  case ML_SOURCE_BLOB:
+                     ret = MonLoaderShareFromBlob(&ctx, flags, monVPN,
+                                                  monBytes, blobOffset,
+                                                  blobSize);
+                     break;
+                  default:
+                     ret = ML_ERROR_SOURCE_INVALID;
+                     break;
+               }
+               break;
+            case ML_CONTENT_INVALID:
+            default:
+               ret = ML_ERROR_CONTENT_INVALID;
+               break;
+         }
+      }
+   }
+#ifndef VMMON
+   if (ret == ML_OK) {
+      if (ctx.vcpu.ASLastVPN == 0) {
+         ret = ML_ERROR_NO_ADDRSPACE;
+      } else {
+         ret = MonLoaderSetEntrypoint(&ctx, header->codeSelector,
+                                      header->codeEntrypoint,
+                                      header->stackSelector,
+                                      header->stackEntrypoint);
+      }
+   }
+#endif
+   MonLoaderCallout_CleanUp(ctx.envCtx);
+   return ret;
+}

vmmon-only/bootstrap/monLoaderVmmon.c

@@ -0,0 +1,411 @@
+/*********************************************************
+ * Copyright (C) 2016-2017 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * monLoaderVmmon.c --
+ *
+ *      vmmon implementation of the MonLoader callouts.
+ *
+ */
+
+#ifdef __linux__
+#   include "driver-config.h"
+#   include <linux/string.h>
+#else
+#   include <string.h>
+#endif
+
+#include "vm_assert.h"
+#include "vcpuid.h"
+#include "x86types.h"
+#include "hostif.h"
+#include "x86paging_common.h"
+#include "x86paging_64.h"
+#include "vmx86.h"
+#include "monLoaderLog.h"
+
+typedef struct MonLoaderEnvContext {
+   VMDriver *vm;
+   VMSharedRegion *shRegions;
+} MonLoaderEnvContext;
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_Init --
+ *
+ *      Initializes platform-specific MonLoader context.
+ *
+ * Returns:
+ *      TRUE if successful, FALSE if not.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx)
+{
+   MonLoaderEnvContext *c;
+   MonLoaderArgs *mlArgs = (MonLoaderArgs *)args;
+
+   c = HostIF_AllocKernelMem(sizeof *c, FALSE);
+   if (c == NULL) {
+      Log("Failed to allocate memory for MonLoader context\n");
+      return FALSE;
+   }
+   memset(c, 0, sizeof *c);
+   c->vm = mlArgs->vm;
+   c->shRegions = mlArgs->shRegions;
+
+   *ctx = c;
+   return TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_Cleanup --
+ *
+ *      Cleans up platform-specific MonLoader context.
+ *
+ * Returns:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+MonLoaderCallout_CleanUp(MonLoaderEnvContext *ctx)
+{
+   ASSERT(ctx != NULL);
+   HostIF_FreeKernelMem(ctx);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_GetPageRoot --
+ *
+ *      Gets the page root MPN for a VCPU.
+ *
+ * Returns:
+ *      An MPN, or INVALID_MPN in case of error.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+
+MPN
+MonLoaderCallout_GetPageRoot(MonLoaderEnvContext *ctx,  // IN
+                             Vcpuid               vcpu) // IN
+{
+   VMDriver *vm = ctx->vm;
+
+   if (vcpu >= vm->numVCPUs) {
+      return INVALID_MPN;
+   }
+   return vm->ptRootMpns[vcpu];
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_GetPTE --
+ *
+ *      Gets the page table entry at index idx in a page table MPN.
+ *
+ * Returns:
+ *      TRUE and sets *pte on success.  FALSE otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+MonLoaderCallout_GetPTE(MonLoaderEnvContext *ctx,   // IN
+                        MPN                  ptMPN, // IN
+                        unsigned             idx,   // IN
+                        Vcpuid               vcpu,  // IN
+                        PT_L1E              *pte)   // OUT
+{
+   VMDriver *vm = ctx->vm;
+   MA pteMA = MPN_2_MA(ptMPN) + idx * sizeof *pte;
+
+   ASSERT(idx < (PAGE_SIZE / sizeof *pte));
+   if (HostIF_ReadPhysical(vm, pteMA, PtrToVA64(pte), TRUE, sizeof *pte) != 0) {
+      Log("Failed to read PTE %d from MPN %"FMT64"x\n", idx, ptMPN);
+      return FALSE;
+   }
+   return TRUE;
+}
+
+
+Bool
+MonLoaderCallout_ImportPage(MonLoaderEnvContext *ctx, // IN
+                            MPN mpn,                  // IN
+                            Vcpuid vcpu)              // IN
+{
+   return TRUE;
+}
+
+
+MPN
+MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx,  // IN
+                          Vcpuid               vcpu) // IN
+{
+   NOT_IMPLEMENTED();
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_MapMPNInPTE --
+ *
+ *      Maps an MPN into a page table at the specified index.
+ *
+ * Returns:
+ *      TRUE on success, FALSE on failure.
+ *
+ * Side effects:
+ *      A mapping is created.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *ctx, // IN
+                             MPN ptMPN,                // IN
+                             unsigned idx,             // IN
+                             uint64 flags,             // IN
+                             MPN mpn,                  // IN
+                             Vcpuid vcpu)              // IN
+{
+   VMDriver *vm = ctx->vm;
+   PT_L1E pte = LM_MAKE_PTE(mpn, 0, flags);
+   MA pteMA = MPN_2_MA(ptMPN) + idx * sizeof pte;
+
+   ASSERT(idx < (PAGE_SIZE / sizeof pte));
+   if (HostIF_WritePhysical(vm, pteMA, PtrToVA64(&pte), TRUE, sizeof pte) !=
+       0) {
+      Warning("Failed to map MPN %"FMT64"x\n", mpn);
+      return FALSE;
+   }
+   LOG(5, "MonLoader mapped MPN %"FMT64"x at PT index %u for VCPU %u\n",
+       mpn, idx, vcpu);
+   return TRUE;
+}
+
+
+Bool
+MonLoaderCallout_FillPage(MonLoaderEnvContext *ctx,     // IN
+                          uint8                pattern, // IN
+                          MPN                  mpn,     // IN
+                          Vcpuid               vcpu)    // IN
+{
+   NOT_IMPLEMENTED();
+}
+
+
+Bool
+MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *ctx,        // IN
+                              uint64               blobOffset, // IN
+                              size_t               copySize,   // IN
+                              MPN                  mpn,        // IN
+                              Vcpuid               vcpu)       // IN
+{
+   NOT_IMPLEMENTED();
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderFindSharedRegion --
+ *
+ *      Searches the shared region table for the given index and returns
+ *      the corresponding descriptor.
+ *
+ * Returns:
+ *      A pointer to the shared region descriptor or NULL if not found.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static VMSharedRegion *
+MonLoaderFindSharedRegion(MonLoaderEnvContext *ctx,
+                          uint64 index)
+{
+   int i;
+
+   for (i = 0; i < ML_SHARED_REGIONS_MAX; i++) {
+      if (ctx->shRegions[i].baseVpn == INVALID_VPN) {
+         /* Not found. */
+         return NULL;
+      } else if (ctx->shRegions[i].index == index) {
+         return &ctx->shRegions[i];
+      }
+   }
+   return NULL;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderGetSharedRegionMPN --
+ *
+ *      Gets shared page's MPN. Assumes vcpuid == 0 for the bootstrap.
+ *
+ * Returns:
+ *      An MPN or INVALID_MPN.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static MPN
+MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx,
+                            uint64 index,
+                            Vcpuid vcpuid,
+                            unsigned pgOffset)
+{
+   VMSharedRegion *s;
+   VA addr;
+   MPN mpn;
+
+   ASSERT(IS_BOOT_VCPUID(vcpuid));
+
+   if (index >= ML_SHARED_REGIONS_MAX) {
+      Log("Invalid shared region %"FMT64"x", index);
+      return INVALID_MPN;
+   }
+   s = MonLoaderFindSharedRegion(ctx, index);
+   if (s == NULL) {
+      return INVALID_MPN;
+   }
+   if (pgOffset >= s->numPages) {
+      return INVALID_MPN;
+   }
+
+   addr = VPN_2_VA(s->baseVpn) + pgOffset * PAGE_SIZE;
+   HostIF_VMLock(ctx->vm, 38);
+   if (HostIF_LookupUserMPN(ctx->vm, addr, &mpn) != PAGE_LOOKUP_SUCCESS) {
+      HostIF_VMUnlock(ctx->vm, 38);
+      Log("Failed to lookup MPN for shared region VA %"FMTVA"x\n", addr);
+      return INVALID_MPN;
+   }
+   HostIF_VMUnlock(ctx->vm, 38);
+   return mpn;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_GetSharedUserPage --
+ *
+ *      Gets a user-provided shared page's MPN for a VCPU.
+ *
+ * Returns:
+ *      An MPN or INVALID_MPN.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+MPN
+MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx,
+                                   uint64 subIndex, // IN
+                                   unsigned page,   // IN
+                                   Vcpuid vcpu)     // IN
+{
+   return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page);
+}
+
+
+MPN
+MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx,      // IN
+                                   uint64               subIndex, // IN
+                                   unsigned             page,     // IN
+                                   Vcpuid               vcpu)     // IN
+{
+   NOT_IMPLEMENTED();
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoaderCallout_IsPrivileged --
+ *
+ *      Returns whether or not this is a privileged environment.
+ *
+ * Returns:
+ *      TRUE if privileged, FALSE if not.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN
+{
+   return TRUE;
+}
+
+MPN
+MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx,    // IN
+                            uint64               offset) // IN
+{
+   return INVALID_MPN;
+}
+
+Bool
+MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *ctx,           // IN
+                               uint16               codeSelector,  // IN
+                               VA64                 code,          // IN
+                               uint16               stackSelector, // IN
+                               VA64                 stack)         // IN
+{
+   NOT_IMPLEMENTED();
+}

vmmon-only/common/apic.c

@@ -165,7 +165,7 @@ APIC_ReadID(const APICDescriptor *desc) // IN
  *
  * APIC_Write --
  *
- *      Writes the given value to the given APIC register using the proper 
+ *      Writes the given value to the given APIC register using the proper
  *      interface.  Does not check to see if the register number is valid.
  *
  * Side effects:
@@ -195,7 +195,7 @@ APIC_Write(const APICDescriptor *desc, // IN
  *
  * APIC_ReadICR --
  *
- *      Reads the APIC ICR using the proper interface.  The semantics of the 
+ *      Reads the APIC ICR using the proper interface.  The semantics of the
  *      ICR are different in X2APIC mode so APIC_Read() should not be used.
  *
  * Side effects:
@@ -207,7 +207,7 @@ APIC_Write(const APICDescriptor *desc, // IN
  *----------------------------------------------------------------------
  */
 
-uint64 
+uint64
 APIC_ReadICR(const APICDescriptor *desc) // IN
 {
    if (desc->isX2) {
@@ -225,7 +225,7 @@ APIC_ReadICR(const APICDescriptor *desc) // IN
  *
  * APIC_WriteICR --
  *
- *      Writes the given value to the APIC ICR using the proper interface.  
+ *      Writes the given value to the APIC ICR using the proper interface.
  *      The semantics of the ICR are different in X2APIC mode so APIC_Write()
  *      should not be used.
  *

vmmon-only/common/comport.c

@@ -19,20 +19,20 @@
 /*
  * comport.c --
  *
- *      Simple COM1 port routines for debugging code that can't do any real 
+ *      Simple COM1 port routines for debugging code that can't do any real
  *      host IO, such as the worldswitch and related.
  *
- *      They all wait for the last character to complete transmission so if the 
- *      system crashes immediately on return, the last character will be seen 
+ *      They all wait for the last character to complete transmission so if the
+ *      system crashes immediately on return, the last character will be seen
  *      by the remote end.
  *
- *      These routines do not have any external dependencies so can be called 
+ *      These routines do not have any external dependencies so can be called
  *      by any code that has privilege to access IO ports.
  *
- *      Under Windows, they can be made to forward output to DbgPrint for 
- *      printing via the debugger.  Just have USE_DBGPRINT set to 1.  If you 
- *      let USE_DBGPRINT be 0 with Windows, make sure the comport hardware is 
- *      powered-up by leaving Hyperterm running with the comport open, else 
+ *      Under Windows, they can be made to forward output to DbgPrint for
+ *      printing via the debugger.  Just have USE_DBGPRINT set to 1.  If you
+ *      let USE_DBGPRINT be 0 with Windows, make sure the comport hardware is
+ *      powered-up by leaving Hyperterm running with the comport open, else
  *      Windows will power the chip down.
  */
 

vmmon-only/common/cpuid.c

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -31,6 +31,8 @@
 #include "hostif.h"
 #include "cpuid.h"
 #include "x86cpuid_asm.h"
+#include "x86svm.h"
+#include "x86vt.h"
 
 uint32 cpuidFeatures;
 static CpuidVendor vendor = CPUID_NUM_VENDORS;
@@ -111,68 +113,10 @@ CPUID_GetVendor(void)
 uint32
 CPUID_GetVersion(void)
 {
-   ASSERT(vendor != CPUID_NUM_VENDORS);
    return version;
 }
 
 
-/*
- *-----------------------------------------------------------------------------
- *
- * CPUID_SyscallSupported --
- *
- *     Determine whether processor supports syscall opcode and MSRs.
- *
- * Results:
- *     FALSE     if processor does not support syscall
- *     TRUE      if processor supports syscall
- *
- * Side effects:
- *     It determines value only on first call, caching it for future.
- *
- *-----------------------------------------------------------------------------
- */
-
-Bool
-CPUID_SyscallSupported(void)
-{
-   /*
-    * It is OK to use local static variables here as 'result' does not depend
-    * on any work done in CPUID_Init(). It purely depends on the CPU.
-    */
-   static Bool initialized = FALSE;
-   static Bool result;
-
-   if (UNLIKELY(!initialized)) {
-      result =    CPUIDExtendedSupported() >= 0x80000001
-               && (__GET_EDX_FROM_CPUID(0x80000001) & (1 << 11));
-      initialized = TRUE;
-   }
-
-   return result;
-}
-
-
-Bool
-CPUID_LongModeSupported(void)
-{
-   /*
-    * It is OK to use local static variables here as 'result' does not depend
-    * on any work done in CPUID_Init(). It purely depends on the CPU.
-    */
-   static Bool initialized = FALSE;
-   static Bool result;
-
-   if (UNLIKELY(!initialized)) {
-      result =       CPUIDExtendedSupported() >= 0x80000001
-                  && (__GET_EDX_FROM_CPUID(0x80000001) & (1 << 29));
-      initialized = TRUE;
-   }
-
-   return result;
-}
-
-
 /*
  *-----------------------------------------------------------------------------
  *
@@ -207,3 +151,28 @@ CPUID_AddressSizeSupported(void)
 
    return result;
 }
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * CPUID_HostSupportsHV --
+ *
+ *     Determine whether processor supports hardware virtualization.  Two
+ *     possibilities are valid: VMX on Intel or SVM on AMD.
+ *
+ * Results:
+ *     True iff the processor supports the required features.
+ *
+ * Side effects:
+ *     None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+Bool
+CPUID_HostSupportsHV(void)
+{
+   return (vendor == CPUID_VENDOR_AMD   && SVM_CapableCPU()) ||
+          (vendor == CPUID_VENDOR_INTEL && VT_CapableCPU());
+}

vmmon-only/common/cpuid.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998, 2016-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -36,11 +36,10 @@ extern uint32 cpuidFeatures;
 void CPUID_Init(void);
 CpuidVendor CPUID_GetVendor(void);
 uint32 CPUID_GetVersion(void);
-Bool CPUID_SyscallSupported(void);
-Bool CPUID_LongModeSupported(void);
 Bool CPUID_AddressSizeSupported(void);
+Bool CPUID_HostSupportsHV(void);
 
-static INLINE uint32 
+static INLINE uint32
 CPUID_GetFeatures(void)
 {
    return cpuidFeatures;
@@ -53,4 +52,3 @@ CPUID_SSE2Supported(void)
 }
 
 #endif
-

vmmon-only/common/hashFunc.c

@@ -20,7 +20,7 @@
 /*
  * hashFunc.c --
  *
- *	The core implementation lives in lib/shared/hashFunc.h because it 
+ *      The core implementation lives in lib/shared/hashFunc.h because it
  *      is shared by the vmkernel and vmmon.
  */
 
@@ -31,10 +31,10 @@
  */
 
 // arbitrary constant
-#define	HASH_INIT_VALUE	(42)
+#define HASH_INIT_VALUE (42)
 
 // 64-bit hash for one 4K page
-uint64 
+uint64
 HashFunc_HashPage(const void *addr)
 {
    return hash2((uint64 *)addr, PAGE_SIZE / sizeof (uint64), HASH_INIT_VALUE);

vmmon-only/common/hostif.h

@@ -17,8 +17,8 @@
  *********************************************************/
 
 /*
- *  hostif.h - Platform dependent interface for supporting 
- *             the vmx86 device driver. 
+ *  hostif.h - Platform dependent interface for supporting
+ *             the vmx86 device driver.
  */
 
 
@@ -36,7 +36,7 @@
 #include "hostifGlobalLock.h"
 
 /*
- * Host-specific definitions. 
+ * Host-specific definitions.
  */
 #if !__linux__ && !defined(WINNT_DDK) && !defined __APPLE__
 #error "Only Linux or NT or Mac OS defined for now."
@@ -69,11 +69,12 @@ typedef enum {
 EXTERN int   HostIF_Init(VMDriver *vm);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);
-EXTERN void  HostIF_InitFP(VMDriver *vm);
 
 EXTERN void *HostIF_AllocPage(void);
 EXTERN void  HostIF_FreePage(void *ptr);
 
+EXTERN VPN   HostIF_MapPage(MPN mpn);
+EXTERN void  HostIF_UnmapPage(VPN vpn);
 EXTERN int   HostIF_LockPage(VMDriver *vm, VA64 uAddr,
                              Bool allowMultipleMPNsPerVA, MPN *mpn);
 EXTERN int   HostIF_UnlockPage(VMDriver *vm, VA64 uAddr);
@@ -86,9 +87,8 @@ EXTERN unsigned int HostIF_EstimateLockedPageLimit(const VMDriver *vm,
                                                    unsigned int lockedPages);
 EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
-EXTERN void *HostIF_AllocCrossGDT(uint32 numPages, MPN maxValidFirst,
-                                  MPN *crossGDTMPNs);
-EXTERN void  HostIF_FreeCrossGDT(uint32 numPages, void *crossGDT);
+EXTERN void *HostIF_AllocKernelPages(unsigned numPages, MPN *mpns);
+EXTERN void  HostIF_FreeKernelPages(unsigned numPages, void *ptr);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
@@ -119,12 +119,11 @@ EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr,
 EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr,
                                   unsigned int numPages, Bool kernelMPNBuffer);
 EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn);
-EXTERN int HostIF_GetLockedPageList(VMDriver *vm, VA64 uAddr,
-                                    unsigned int numPages);
 
-EXTERN int HostIF_ReadPage(VMDriver *vm, MPN mpn, VA64 addr, Bool kernelBuffer);
-EXTERN int HostIF_WritePage(VMDriver *vm, MPN mpn, VA64 addr,
-                            Bool kernelBuffer);
+EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr,
+                               Bool kernelBuffer, size_t len);
+EXTERN int HostIF_WritePhysical(VMDriver *vm, MA ma, VA64 addr,
+                                Bool kernelBuffer, size_t len);
 EXTERN int HostIF_WriteMachinePage(MPN mpn, VA64 addr);
 #if defined __APPLE__
 // There is no need for a fast clock lock on Mac OS.
@@ -141,6 +140,9 @@ EXTERN void HostIF_FreeMachinePage(MPN mpn);
 
 EXTERN int HostIF_SafeRDMSR(uint32 msr, uint64 *val);
 
+EXTERN int HostIF_CopyFromUser(void *dst, VA64 src, size_t len);
+EXTERN int HostIF_CopyToUser(VA64 dst, const void *src, size_t len);
+
 #if defined __APPLE__
 EXTERN void HostIF_PageUnitTest(void);
 #endif

vmmon-only/common/memtrack.c

@@ -151,7 +151,7 @@ typedef struct MemTrack {
    unsigned          numPages;      /* Number of pages tracked. */
    MemTrackDir1      dir1;          /* First level directory. */
    MemTrackHT        vpnHashTable;  /* VPN to entry hashtable. */
-   MemTrackHT        mpnHashTable;  /* MPN to entry hashtable. */
+   MemTrackHT       *mpnHashTable;  /* MPN to entry hashtable. */
 } MemTrack;
 
 /*
@@ -276,6 +276,7 @@ MemTrackCleanup(MemTrack *mt)    // IN
    unsigned idx;
    unsigned p1;
    MemTrackDir1 *dir1;
+   Bool freeBackMap = mt != NULL && mt->mpnHashTable != NULL;
 
    if (mt == NULL) {
       return;
@@ -304,10 +305,15 @@ MemTrackCleanup(MemTrack *mt)    // IN
       if (mt->vpnHashTable.pages[idx] != NULL) {
          HostIF_FreePage(mt->vpnHashTable.pages[idx]);
       }
-      if (mt->mpnHashTable.pages[idx] != NULL) {
-         HostIF_FreePage(mt->mpnHashTable.pages[idx]);
+      if (freeBackMap) {
+         if (mt->mpnHashTable->pages[idx] != NULL) {
+            HostIF_FreePage(mt->mpnHashTable->pages[idx]);
+         }
       }
    }
+   if (freeBackMap) {
+      HostIF_FreeKernelMem(mt->mpnHashTable);
+   }
 
    HostIF_FreeKernelMem(mt);
 }
@@ -359,14 +365,19 @@ MemTrack_Init(VMDriver *vm) // IN:
       mt->vpnHashTable.pages[idx] = htPage;
    }
 
+   mt->mpnHashTable = HostIF_AllocKernelMem(sizeof *mt->mpnHashTable, FALSE);
+   if (mt->mpnHashTable == NULL) {
+      Warning("MemTrack failed to allocate MPN hash table.\n");
+      goto error;
+   }
+   memset(mt->mpnHashTable, 0, sizeof *mt->mpnHashTable);
    for (idx = 0; idx < MEMTRACK_HT_PAGES; idx++) {
       MemTrackHTPage *htPage = MemTrackAllocPage();
-
       if (htPage == NULL) {
          Warning("MemTrack failed to allocate MPN hash table (%d).\n", idx);
          goto error;
       }
-      mt->mpnHashTable.pages[idx] = htPage;
+      mt->mpnHashTable->pages[idx] = htPage;
    }
 
    return mt;
@@ -406,6 +417,7 @@ MemTrack_Add(MemTrack *mt,    // IN
    MemTrackDir3 *dir3;
    MEMTRACK_IDX2DIR(idx, p1, p2, p3);
 
+
    ASSERT(HostIF_VMLockIsHeld(mt->vm));
 
    if (p1 >= MEMTRACK_DIR1_ENTRIES ||
@@ -429,7 +441,7 @@ MemTrack_Add(MemTrack *mt,    // IN
    ent->mpn = mpn;
 
    MemTrackHTInsert(&mt->vpnHashTable, ent, &ent->vpnChain, ent->vpn);
-   MemTrackHTInsert(&mt->mpnHashTable, ent, &ent->mpnChain, ent->mpn);
+   MemTrackHTInsert(mt->mpnHashTable, ent, &ent->mpnChain, ent->mpn);
 
    mt->numPages++;
 
@@ -492,7 +504,7 @@ MemTrack_LookupMPN(MemTrack *mt, // IN
 {
    MemTrackEntry *next;
    ASSERT(HostIF_VMLockIsHeld(mt->vm));
-   next = *MemTrackHTLookup(&mt->mpnHashTable, mpn);
+   next = *MemTrackHTLookup(mt->mpnHashTable, mpn);
 
    while (next != NULL) {
       if (next->mpn == mpn) {

vmmon-only/common/memtrack.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -20,7 +20,7 @@
  * memtrack.h --
  *
  *    Utility module for tracking pinned memory, which allows later
- *    lookup by VPN.
+ *    lookup by VPN and optionally by MPN.
  */
 
 

vmmon-only/common/phystrack.c

@@ -226,7 +226,7 @@ PhysTrack_Alloc(VMDriver *vm)
    }
 
    return tracker;
-} 
+}
 
 
 /*
@@ -238,7 +238,7 @@ PhysTrack_Alloc(VMDriver *vm)
  *
  * Results:
  *      reallocates all structures, including 'tracker'
- *     
+ *
  * Side effects:
  *      tracker deallocated
  *
@@ -246,7 +246,7 @@ PhysTrack_Alloc(VMDriver *vm)
  */
 
 void
-PhysTrack_Free(PhysTracker *tracker) 
+PhysTrack_Free(PhysTracker *tracker)
 {
    unsigned int p1;
 

vmmon-only/common/task.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2013,2015 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -16,26 +16,24 @@
  *
  *********************************************************/
 
-
-
 #ifndef TASK_H
 #define TASK_H
 
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
+#include "bootstrap_vmm.h"
 
 struct InitBlock;
-struct InitCrossGDT;
 
-extern Bool Task_AllocCrossGDT(struct InitBlock *initBlock);
+extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt);
 extern int Task_InitCrosspage(VMDriver *vm, struct InitBlock *params);
-extern int Task_InitCrossGDT(struct InitCrossGDT *initCrossGDT);
 extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid);
 extern Bool Task_Initialize(void);
 extern void Task_Terminate(void);
 extern MPN Task_GetHVRootPageForPCPU(uint32 pCPU);
 extern Descriptor *Task_GetTmpGDT(uint32 pCPU);
+extern void Task_SwitchPTPPageCleanup(VMDriver *vm);
 
 #endif
 

vmmon-only/common/vmx86.h

@@ -38,6 +38,7 @@
 #include "rateconv.h"
 #include "vmmem_shared.h"
 #include "apic.h"
+#include "bootstrap_vmm.h"
 
 typedef struct PseudoTSCOffsetInfo {
    uint32 inVMMCnt;                  /* Number of vcpus executing in the VMM. */
@@ -63,8 +64,12 @@ typedef struct VMDriver {
    Vcpuid                       numVCPUs;     /* Number of vcpus in VM. */
    struct VMHost               *vmhost;       /* Host-specific fields. */
 
+   BSVMM_HostParams             bsParams;
+   MPN                          ptRootMpns[MAX_VCPUS];
+
    /* Pointers to the crossover pages shared with the monitor. */
    struct VMCrossPage          *crosspage[MAX_INITBLOCK_CPUS];
+   struct MemTrack             *ptpTracker;
    volatile uint32              currentHostCpu[MAX_INITBLOCK_CPUS];
    VCPUSet                      crosscallWaitSet[MAX_INITBLOCK_CPUS];
    APICDescriptor               hostAPIC;
@@ -74,13 +79,17 @@ typedef struct VMDriver {
    struct PerfCounter          *perfCounter;
    VMMemMgmtInfo                memInfo;
    unsigned                     fastClockRate;/* Protected by FastClockLock. */
-   int                          fastSuspResFlag;
 
    volatile PseudoTSCOffsetInfo ptscOffsetInfo; /* Volatile per PR 699101#29. */
    Atomic_uint64                ptscLatest;
    int64                        ptscOffsets[MAX_INITBLOCK_CPUS];
 } VMDriver;
 
+typedef struct MonLoaderArgs {
+   VMDriver *vm;
+   VMSharedRegion *shRegions;
+} MonLoaderArgs;
+
 typedef struct VmTimeStart {
    uint64 count;
    uint64 time;
@@ -105,7 +114,14 @@ extern PseudoTSC pseudoTSC;
 
 #define MAX_LOCKED_PAGES (-1)
 
-extern VMDriver *Vmx86_CreateVM(void);
+extern VMDriver *Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize);
+extern Bool Vmx86_ProcessBootstrap(VMDriver *vm,
+                                   VA64 bsBlobAddr,
+                                   uint32 numBytes,
+                                   uint32 headerOffset,
+                                   uint16 numVCPUs,
+                                   UserVA64 *ptRootVAs,
+                                   VMSharedRegion *shRegions);
 extern int Vmx86_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 extern int Vmx86_ReleaseVM(VMDriver *vm);
 extern int Vmx86_InitVM(VMDriver *vm, InitBlock *initParams);
@@ -126,13 +142,11 @@ extern int Vmx86_UnlockPageByMPN(VMDriver *vm, MPN mpn, VA64 uAddr);
 extern MPN Vmx86_GetRecycledPage(VMDriver *vm);
 extern int Vmx86_ReleaseAnonPage(VMDriver *vm, MPN mpn);
 extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr,
-				  unsigned numPages, Bool kernelMPNBuffer,
+                                  unsigned numPages, Bool kernelMPNBuffer,
                                   Bool ignoreLimits);
 extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr,
-				 unsigned numPages, Bool kernelMPNBuffer);
+                                 unsigned numPages, Bool kernelMPNBuffer);
 extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn);
-extern int Vmx86_GetLockedPageList(VMDriver *vm, VA64 uAddr,
-				   unsigned int numPages);
 
 extern int32 Vmx86_GetNumVMs(void);
 extern int32 Vmx86_GetTotalMemUsage(void);
@@ -147,15 +161,12 @@ extern Bool Vmx86_Readmit(VMDriver *curVM, OvhdMem_Deltas *delta);
 extern void Vmx86_UpdateMemInfo(VMDriver *curVM,
                                 const VMMemMgmtInfoPatch *patch);
 extern void Vmx86_Add2MonPageTable(VMDriver *vm, VPN vpn, MPN mpn,
-				   Bool readOnly);
-extern Bool Vmx86_PAEEnabled(void);
-extern Bool Vmx86_VMXEnabled(void);
+                                   Bool readOnly);
 extern Bool Vmx86_GetAllMSRs(MSRQuery *query);
+extern void Vmx86_FlushVMCSAllCPUs(MA vmcs);
 extern void Vmx86_MonTimerIPI(void);
 extern void Vmx86_InitIDList(void);
-extern VMDriver *Vmx86_LookupVMByUserID(int userID);
-extern Bool Vmx86_FastSuspResSetOtherFlag(VMDriver *vm, int otherVmUserId);
-extern int  Vmx86_FastSuspResGetMyFlag(VMDriver *vm, Bool blockWait);
+extern Bool Vmx86_GetPageRoot(VMDriver *vm, Vcpuid vcpuid, MPN *mpn);
 extern void Vmx86_Open(void);
 extern void Vmx86_Close(void);
 
@@ -199,4 +210,4 @@ extern uint64 Vmx86_GetPseudoTSC(void);
 
 extern uint64 Vmx86_GetUnavailablePerfCtrs(void);
 
-#endif 
+#endif

vmmon-only/include/address_defs.h

@@ -36,6 +36,11 @@
 
 #include "vm_basic_defs.h" // For PAGE_SHIFT
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 /*
  * Virtual, physical, machine address and page conversion macros
  */
@@ -92,4 +97,9 @@ IsGoodMPNOrMemref(MPN mpn)
    return IsGoodMPN(mpn) || mpn == MEMREF_MPN;
 }
 
+
+#if defined __cplusplus
+} // extern "C"
 #endif
+
+#endif // _ADDRESS_DEFS_H_

vmmon-only/include/basic_initblock.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006, 2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -40,7 +40,7 @@
 #include "vcpuid.h"
 
 
-#define MAX_INITBLOCK_CPUS     128
+#define MAX_INITBLOCK_CPUS     MAX_VCPUS
 
 
 typedef
@@ -49,9 +49,12 @@ struct InitBlock {
    uint32 magicNumber;     /* Magic number (INIT_BLOCK_MAGIC) */
    Vcpuid numVCPUs;
    VA64   crosspage[MAX_INITBLOCK_CPUS];
+   LPN64  monStartLPN;
+   LPN64  monEndLPN;
    uint32 vmInitFailurePeriod;
    LA64   crossGDTHKLA;
    MPN    crossGDTMPNs[5];  // CROSSGDT_NUMPAGES
+   uint16 numPTPPages;
 }
 #include "vmware_pack_end.h"
 InitBlock;

vmmon-only/include/bootstrap_vmm.h

@@ -0,0 +1,65 @@
+/*********************************************************
+ * Copyright (C) 2015,2017 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * bootstrap_vmm.h --
+ *
+ *    Public VMM bootstrap declarations.
+ */
+
+#ifndef _BOOTSTRAP_VMM_H
+#define _BOOTSTRAP_VMM_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_USERLEVEL
+
+#include "includeCheck.h"
+#include "vm_basic_types.h"
+#include "monAddrLayout.h"
+
+#define BOOTSTRAP_MAGIC 42
+#define BOOTSTRAP_MAX_GDT_DESCS 2
+
+typedef struct BSVMM_GDTInitEntry {
+   uint16 index;
+   LA32   base;
+   VA32   limit;
+   uint32 type;
+   uint32 S;
+   uint32 DPL;
+   uint32 present;
+   uint32 longmode;
+   uint32 DB;
+   uint32 gran;
+} BSVMM_GDTInitEntry;
+
+typedef struct BSVMM_GDTInit {
+   BSVMM_GDTInitEntry entries[BOOTSTRAP_MAX_GDT_DESCS];
+} BSVMM_GDTInit;
+
+typedef struct BSVMM_HostParams {
+   uint32           magic;
+   VMM64_AddrLayout addrLayout;
+   BSVMM_GDTInit    gdtInit;
+} BSVMM_HostParams;
+
+BSVMM_HostParams* BSVMM_Validate(void *buf, uint32 nbytes);
+
+#endif

vmmon-only/include/circList.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -32,75 +32,126 @@
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMKERNEL
 #include "includeCheck.h"
+
 #include "vmware.h"
 
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
 typedef struct ListItem {
    struct ListItem *prev;
    struct ListItem *next;
 } ListItem;
 
-/* A list with no elements is a null pointer. */
-#define   LIST_ITEM_DEF(name)   \
-   ListItem * name = NULL
-
-#define   LIST_EMPTY(l)      ((l) == NULL)
-
-/* initialize list item */
-#define   INIT_LIST_ITEM(p)   \
-   do {   \
-      (p)->prev = (p)->next = (p);   \
-   } while (0)
-
-/* check if initialized */
-#define   IS_LIST_ITEM_INITIALIZED(li)   \
-   (((li) == (li)->prev) && ((li) == (li)->next))
-
-/* return first element in the list */
-#define   LIST_FIRST(l)      (l)
-#define   LIST_FIRST_CHK(l)   (l)
-
-/* return last element in the list */
-#define   LIST_LAST(l)      ((l)->prev)
-#define   LIST_LAST_CHK(l)   (LIST_EMPTY(l) ? NULL : LIST_LAST(l))
 
 /*
- * LIST_CONTAINER - get the struct for this entry (like list_entry)
+ *----------------------------------------------------------------------
+ *
+ * CircList_IsEmpty --
+ *
+ *      A NULL list is an empty list.
+ *
+ * Result:
+ *      TRUE if list is empty, FALSE otherwise.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Bool
+CircList_IsEmpty(const ListItem *item)  // IN
+{
+   return item == NULL;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_InitItem --
+ *
+ *      Initialize item as a single-element circular list.
+ *
+ * Result:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE void
+CircList_InitItem(ListItem *item)  // OUT
+{
+   item->prev = item->next = item;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_First --
+ *
+ *      Return first item in the list.
+ *
+ * Result:
+ *      First item.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE ListItem *
+CircList_First(ListItem *item)  // IN
+{
+   return item;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * CircList_Last --
+ *
+ *      Return last item in the list.
+ *
+ * Result:
+ *      Last item.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE ListItem *
+CircList_Last(ListItem *item)
+{
+   return item->prev;
+}
+
+
+/*
+ * CIRC_LIST_CONTAINER - get the struct for this entry (like list_entry)
  * @ptr: the &struct ListItem pointer.
  * @type:   the type of the struct this is embedded in.
  * @member: the name of the list struct within the struct.
  */
-#define LIST_CONTAINER(ptr, type, member) \
+#define CIRC_LIST_CONTAINER(ptr, type, member) \
    VMW_CONTAINER_OF(ptr, type, member)
-
-/*
- * delete item from the list
+/* 
+ * Historical name, left here to reduce churn.
+ * TODO: remove, all LIST_CONTAINER uses should be
+ * VMW_CONTAINER_OF and stop depending on circList.h
+ * to provide the definition.
  */
-#define   LIST_DEL            DelListItem
-
-/*
- * link two lists together
- */
-#define   LIST_SPLICE         SpliceLists
-
-/*
- * Split a list into two lists
- */
-#define   LIST_SPLIT          SplitLists
-
-/*
- * Add item to front of stack. List pointer points to new head.
- */
-#define   LIST_PUSH           PushListItem
-
-/*
- * Add item at back of queue. List pointer only changes if list was empty.
- */
-#define   LIST_QUEUE          QueueListItem
-
-/*
- * Get the list size.
- */
-#define   LIST_SIZE           GetListSize
+#define LIST_CONTAINER(ptr, type, member) VMW_CONTAINER_OF(ptr, type, member)
 
 /*
  * LIST_SCAN_FROM scans the list from "from" up until "until".
@@ -109,38 +160,32 @@ typedef struct ListItem {
  * "until" is the element where search should stop.
  * member is the field to use for the search - either "next" or "prev".
  */
-#define   LIST_SCAN_FROM(p, from, until, member)   \
+#define CIRC_LIST_SCAN_FROM(p, from, until, member)   \
    for (p = (from); (p) != NULL;   \
       (p) = (((p)->member == (until)) ? NULL : (p)->member))
 
 /* scan the entire list (non-destructively) */
-#define   LIST_SCAN(p, l)   \
-   LIST_SCAN_FROM(p, LIST_FIRST(l), LIST_FIRST(l), next)
+#define CIRC_LIST_SCAN(p, l)   \
+   CIRC_LIST_SCAN_FROM(p, CircList_First(l), CircList_First(l), next)
 
 
-/* scan a list backward from last element to first (non-destructively) */
-#define   LIST_SCAN_BACK(p, l)   \
-   LIST_SCAN_FROM(p, LIST_LAST_CHK(l), LIST_LAST(l), prev)
-
 /* scan the entire list where loop element may be destroyed */
-#define   LIST_SCAN_SAFE(p, pn, l)   \
-   if (!LIST_EMPTY(l))  \
-      for (p = (l), (pn) = NextListItem(p, l); (p) != NULL;   \
-           (p) = (pn), (pn) = NextListItem(p, l))
+#define CIRC_LIST_SCAN_SAFE(p, pn, l)   \
+   if (!CircList_IsEmpty(l))  \
+      for (p = (l), (pn) = CircList_Next(p, l); (p) != NULL;   \
+           (p) = (pn), (pn) = CircList_Next(p, l))
 
 /* scan the entire list backwards where loop element may be destroyed */
-#define   LIST_SCAN_BACK_SAFE(p, pn, l)   \
-   if (!LIST_EMPTY(l))  \
-      for (p = LIST_LAST(l), (pn) = PrevListItem(p, l); (p) != NULL;   \
-           (p) = (pn), (pn) = PrevListItem(p, l))
+#define CIRC_LIST_SCAN_BACK_SAFE(p, pn, l)   \
+   if (!CircList_IsEmpty(l))  \
+      for (p = CircList_Last(l), (pn) = CircList_Prev(p, l); (p) != NULL;   \
+           (p) = (pn), (pn) = CircList_Prev(p, l))
 
 
-/* function definitions */
-
 /*
  *----------------------------------------------------------------------
  *
- * NextListItem --
+ * CircList_Next --
  *
  *      Returns the next member of a doubly linked list, or NULL if last.
  *      Assumes: p is member of the list headed by head.
@@ -156,8 +201,8 @@ typedef struct ListItem {
  */
 
 static INLINE ListItem *
-NextListItem(ListItem *p,        // IN
-             ListItem *head)     // IN
+CircList_Next(ListItem *p,        // IN
+              ListItem *head)     // IN
 {
    if (head == NULL || p == NULL) {
       return NULL;
@@ -171,7 +216,7 @@ NextListItem(ListItem *p,        // IN
 /*
  *----------------------------------------------------------------------
  *
- * PrevListItem --
+ * CircList_Prev --
  *
  *      Returns the prev member of a doubly linked list, or NULL if first.
  *      Assumes: p is member of the list headed by head.
@@ -187,8 +232,8 @@ NextListItem(ListItem *p,        // IN
  */
 
 static INLINE ListItem *
-PrevListItem(ListItem *p,        // IN
-             ListItem *head)     // IN
+CircList_Prev(ListItem *p,        // IN
+              ListItem *head)     // IN
 {
    if (head == NULL || p == NULL) {
       return NULL;
@@ -201,7 +246,7 @@ PrevListItem(ListItem *p,        // IN
 /*
  *----------------------------------------------------------------------
  *
- * DelListItem --
+ * CircList_DeleteItem --
  *
  *      Deletes a member of a doubly linked list, possibly modifies the
  *      list header itself.
@@ -217,8 +262,8 @@ PrevListItem(ListItem *p,        // IN
  */
 
 static INLINE void
-DelListItem(ListItem *p,         // IN
-            ListItem **headp)    // IN/OUT
+CircList_DeleteItem(ListItem *p,         // IN
+                    ListItem **headp)    // IN/OUT
 {
    ListItem *next;
 
@@ -241,7 +286,7 @@ DelListItem(ListItem *p,         // IN
 /*
  *----------------------------------------------------------------------
  *
- * QueueListItem --
+ * CircList_Queue --
  *
  *      Adds a new member to the back of a doubly linked list (queue)
  *      Assumes neither p nor headp is null and p is not a member of *headp.
@@ -256,14 +301,14 @@ DelListItem(ListItem *p,         // IN
  */
 
 static INLINE void
-QueueListItem(ListItem *p,              // IN
-              ListItem **headp)         // IN/OUT
+CircList_Queue(ListItem *p,              // IN
+               ListItem **headp)         // IN/OUT
 {
    ListItem *head;
 
    head = *headp;
-   if (LIST_EMPTY(head)) {
-      INIT_LIST_ITEM(p);
+   if (CircList_IsEmpty(head)) {
+      CircList_InitItem(p);
       *headp = p;
    } else {
       p->prev = head->prev;
@@ -277,7 +322,7 @@ QueueListItem(ListItem *p,              // IN
 /*
  *----------------------------------------------------------------------
  *
- * PushListItem --
+ * CircList_Push --
  *
  *      Adds a new member to the front of a doubly linked list (stack)
  *      Assumes neither p nor headp is null and p is not a member of *headp.
@@ -292,10 +337,10 @@ QueueListItem(ListItem *p,              // IN
  */
 
 static INLINE void
-PushListItem(ListItem *p,               // IN
-             ListItem **headp)          // IN/OUT
+CircList_Push(ListItem *p,               // IN
+              ListItem **headp)          // IN/OUT
 {
-   QueueListItem(p, headp);
+   CircList_Queue(p, headp);
    *headp = p;
 }
 
@@ -303,7 +348,7 @@ PushListItem(ListItem *p,               // IN
 /*
  *----------------------------------------------------------------------
  *
- * SpliceLists --
+ * CircList_Splice --
  *
  *      Make a single list {l1 l2} from {l1} and {l2} and return it.
  *      It is okay for one or both lists to be NULL.
@@ -320,16 +365,16 @@ PushListItem(ListItem *p,               // IN
  */
 
 static INLINE ListItem *
-SpliceLists(ListItem *l1,      // IN
-            ListItem *l2)      // IN
+CircList_Splice(ListItem *l1,      // IN
+                ListItem *l2)      // IN
 {
    ListItem *l1Last, *l2Last;
 
-   if (LIST_EMPTY(l1)) {
+   if (CircList_IsEmpty(l1)) {
       return l2;
    }
 
-   if (LIST_EMPTY(l2)) {
+   if (CircList_IsEmpty(l2)) {
       return l1;
    }
 
@@ -349,10 +394,11 @@ SpliceLists(ListItem *l1,      // IN
 }
 
 
+#if 0  /* Presently unused, enable if a use is found */
 /*
  *----------------------------------------------------------------------
  *
- * SplitLists --
+ * CircList_Split --
  *
  *      Make a list l = {l1 l2} into two separate lists {l1} and {l2}, where:
  *      l = { ... x -> p -> ... } split into:
@@ -372,14 +418,14 @@ SpliceLists(ListItem *l1,      // IN
  */
 
 static INLINE void
-SplitLists(ListItem *p,         // IN
-           ListItem *l,         // IN
-           ListItem **l1p,      // OUT
-           ListItem **l2p)      // OUT
+CircList_Split(ListItem *p,         // IN
+               ListItem *l,         // IN
+               ListItem **l1p,      // OUT
+               ListItem **l2p)      // OUT
 {
    ListItem *last;
 
-   if (p == LIST_FIRST(l)) {   /* first element */
+   if (p == CircList_First(l)) {   /* first element */
       *l1p = NULL;
       *l2p = l;
       return;
@@ -395,12 +441,13 @@ SplitLists(ListItem *p,         // IN
    p->prev = last;
    last->next = p;
 }
+#endif
 
 
 /*
  *----------------------------------------------------------------------
  *
- * GetListSize --
+ * CircList_Size --
  *
  *	Return the number of items in the list.
  *
@@ -414,15 +461,19 @@ SplitLists(ListItem *p,         // IN
  */
 
 static INLINE int
-GetListSize(ListItem *head)     // IN
+CircList_Size(ListItem *head)     // IN
 {
    ListItem *li;
    int ret = 0;
 
-   LIST_SCAN(li, head) {
+   CIRC_LIST_SCAN(li, head) {
       ret++;
    }
    return ret;
 }
 
+#if defined(__cplusplus)
+}  // extern "C"
+#endif
+
 #endif /* _CIRCLIST_H_ */

vmmon-only/include/community_source.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the

vmmon-only/include/compat_cred.h

@@ -1,51 +0,0 @@
-/*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef __COMPAT_CRED_H__
-#   define __COMPAT_CRED_H__
-
-
-/*
- * Include linux/cred.h via linux/sched.h - it is not nice, but
- * as cpp does not have #ifexist...
- */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
-#include <linux/sched.h>
-#else
-#include <linux/cred.h>
-#endif
-
-#if !defined(current_fsuid) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)
-#define current_uid() (current->uid)
-#define current_euid() (current->euid)
-#define current_fsuid() (current->fsuid)
-#define current_gid() (current->gid)
-#define current_egid() (current->egid)
-#define current_fsgid() (current->fsgid)
-#endif
-
-#if !defined(cap_set_full)
-/* cap_set_full was removed in kernel version 3.0-rc4. */
-#define cap_set_full(_c) do { (_c) = CAP_FULL_SET; } while (0)
-#endif
-
-#if !defined(GLOBAL_ROOT_UID)
-#define GLOBAL_ROOT_UID (0)
-#endif
-
-#endif /* __COMPAT_CRED_H__ */

vmmon-only/include/compat_highmem.h

@@ -1,32 +0,0 @@
-/*********************************************************
- * Copyright (C) 2012 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef __COMPAT_HIGHMEM_H__
-#   define __COMPAT_HIGHMEM_H__
-
-#include <linux/highmem.h>
-
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
-#   define compat_kmap_atomic(_page)   kmap_atomic(_page)
-#   define compat_kunmap_atomic(_page) kunmap_atomic(_page)
-#else
-#   define compat_kmap_atomic(_page)   kmap_atomic((_page), KM_USER0)
-#   define compat_kunmap_atomic(_page) kunmap_atomic((_page), KM_USER0)
-#endif
-
-#endif /* __COMPAT_HIGHMEM_H__ */

vmmon-only/include/compat_interrupt.h

@@ -1,55 +0,0 @@
-/*********************************************************
- * Copyright (C) 2003 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef __COMPAT_INTERRUPT_H__
-#   define __COMPAT_INTERRUPT_H__
-
-
-#include <linux/interrupt.h>
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 69)
-/*
- * We cannot just define irqreturn_t, as some 2.4.x kernels have
- * typedef void irqreturn_t; for "increasing" backward compatibility.
- */
-typedef void compat_irqreturn_t;
-#define COMPAT_IRQ_NONE
-#define COMPAT_IRQ_HANDLED
-#define COMPAT_IRQ_RETVAL(x)
-#else
-typedef irqreturn_t compat_irqreturn_t;
-#define COMPAT_IRQ_NONE		IRQ_NONE
-#define COMPAT_IRQ_HANDLED	IRQ_HANDLED
-#define COMPAT_IRQ_RETVAL(x)	IRQ_RETVAL(x)
-#endif
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
-#define COMPAT_IRQF_DISABLED    SA_INTERRUPT
-#define COMPAT_IRQF_SHARED      SA_SHIRQ
-#else
-#define COMPAT_IRQF_DISABLED    IRQF_DISABLED
-#define COMPAT_IRQF_SHARED      IRQF_SHARED
-#endif
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
-#define COMPAT_IRQ_HANDLER_ARGS(irq, devp) (int irq, void *devp, struct pt_regs *regs)
-#else
-#define COMPAT_IRQ_HANDLER_ARGS(irq, devp) (int irq, void *devp)
-#endif
-
-#endif /* __COMPAT_INTERRUPT_H__ */

vmmon-only/include/compat_ioport.h

@@ -1,63 +0,0 @@
-/*********************************************************
- * Copyright (C) 2003 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef __COMPAT_IOPORT_H__
-#   define __COMPAT_IOPORT_H__
-
-
-#include <linux/ioport.h>
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0)
-static inline void *
-compat_request_region(unsigned long start, unsigned long len, const char *name)
-{
-   if (check_region(start, len)) {
-      return NULL;
-   }
-   request_region(start, len, name);
-   return (void*)1;
-}
-#else
-#define compat_request_region(start, len, name) request_region(start, len, name)
-#endif
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 7)
-/* mmap io support starts from 2.3.7, fail the call for kernel prior to that */
-static inline void *
-compat_request_mem_region(unsigned long start, unsigned long len, const char *name)
-{
-   return NULL;
-}
-
-static inline void
-compat_release_mem_region(unsigned long start, unsigned long len)
-{
-   return;
-}
-#else
-#define compat_request_mem_region(start, len, name) request_mem_region(start, len, name)
-#define compat_release_mem_region(start, len)       release_mem_region(start, len)
-#endif
-
-/* these two macro defs are needed by compat_pci_request_region */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 15)
-#   define IORESOURCE_IO    0x00000100
-#   define IORESOURCE_MEM   0x00000200
-#endif
-
-#endif /* __COMPAT_IOPORT_H__ */

vmmon-only/include/compat_pci.h

@@ -1,72 +0,0 @@
-/*********************************************************
- * Copyright (C) 1999 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-/*
- * compat_pci.h: PCI compatibility wrappers.
- */
-
-#ifndef __COMPAT_PCI_H__
-#define __COMPAT_PCI_H__
-
-#include "compat_ioport.h"
-#include <linux/pci.h>
-
-#ifndef DMA_BIT_MASK
-#  define DMA_BIT_MASK(n) DMA_##n##BIT_MASK
-#endif
-
-/*
- * Power Management related compat wrappers.
- */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 10)
-#   define compat_pci_save_state(pdev)      pci_save_state((pdev), NULL)
-#   define compat_pci_restore_state(pdev)   pci_restore_state((pdev), NULL)
-#else
-#   define compat_pci_save_state(pdev)      pci_save_state((pdev))
-#   define compat_pci_restore_state(pdev)   pci_restore_state((pdev))
-#endif
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)
-#   define pm_message_t          u32
-#   define compat_pci_choose_state(pdev, state)  (state)
-#   define PCI_D0               0
-#   define PCI_D3hot            3
-#else
-#   define compat_pci_choose_state(pdev, state)  pci_choose_state((pdev), (state))
-#endif
-
-/* 2.6.14 changed the PCI shutdown callback */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14)
-#   define COMPAT_PCI_SHUTDOWN(func)               .driver = { .shutdown = (func), }
-#   define COMPAT_PCI_DECLARE_SHUTDOWN(func, var)  (func)(struct device *(var))
-#   define COMPAT_PCI_TO_DEV(dev)                  (to_pci_dev(dev))
-#else
-#   define COMPAT_PCI_SHUTDOWN(func)               .shutdown = (func)
-#   define COMPAT_PCI_DECLARE_SHUTDOWN(func, var)  (func)(struct pci_dev *(var))
-#   define COMPAT_PCI_TO_DEV(dev)                  (dev)
-#endif
-
-/* 2.6.26 introduced the device_set_wakeup_enable() function */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
-#   define compat_device_set_wakeup_enable(dev, val) do {} while(0)
-#else
-#   define compat_device_set_wakeup_enable(dev, val) \
-       device_set_wakeup_enable(dev, val)
-#endif
-
-#endif /* __COMPAT_PCI_H__ */

vmmon-only/include/compat_pgtable.h

@@ -48,11 +48,6 @@ typedef p4d_t compat_p4d_t;
 #   define COMPAT_P4D_MASK                  0            /* Not used */
 typedef pgd_t compat_p4d_t;
 #endif
-/* p[gu]d_large did not exist before 2.6.25 */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)
-#   define pud_large(pud) 0
-#   define pgd_large(pgd) 0
-#endif
 /* pud_pfn did not exist before 3.8. */
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
 #   define pud_pfn(pud)  INVALID_MPN

vmmon-only/include/compat_semaphore.h

@@ -1,49 +0,0 @@
-/*********************************************************
- * Copyright (C) 2002 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef __COMPAT_SEMAPHORE_H__
-#   define __COMPAT_SEMAPHORE_H__
-
-
-/* <= 2.6.25 have asm only, 2.6.26 has both, and 2.6.27-rc2+ has linux only. */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
-#   include <asm/semaphore.h>
-#else
-#   include <linux/semaphore.h>
-#endif
-
-
-/*
-* The init_MUTEX_LOCKED() API appeared in 2.2.18, and is also in
-* 2.2.17-21mdk --hpreg
-*/
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 18)
-   #ifndef init_MUTEX_LOCKED
-      #define init_MUTEX_LOCKED(_sem) *(_sem) = MUTEX_LOCKED
-   #endif
-   #ifndef DECLARE_MUTEX
-      #define DECLARE_MUTEX(name) struct semaphore name = MUTEX
-   #endif
-   #ifndef DECLARE_MUTEX_LOCKED
-      #define DECLARE_MUTEX_LOCKED(name) struct semaphore name = MUTEX_LOCKED
-   #endif
-#endif
-
-
-#endif /* __COMPAT_SEMAPHORE_H__ */

vmmon-only/include/cpuid_info.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -29,6 +29,10 @@
 #include "vm_basic_asm.h"
 #include "x86cpuid_asm.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
 
 typedef struct CPUID0 {
    int numEntries;
@@ -92,4 +96,9 @@ CPUIDSummary_RegsFromCpuid0(CPUID0* id0In,
    return id0Out;
 }
 
+
+#if defined __cplusplus
+} // extern "C"
 #endif
+
+#endif // _CPUID_INFO_H

vmmon-only/include/crossgdt.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2015 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -42,7 +42,7 @@ typedef struct CrossGDT {
    Descriptor gdtes[0x5000 / sizeof (Descriptor)];  // 0x5000 > GDT_LIMIT
 } CrossGDT;
 
-#define CROSSGDT_NUMPAGES ((sizeof (CrossGDT) + PAGE_SIZE - 1) / PAGE_SIZE)
+#define CROSSGDT_NUMPAGES CEILING(sizeof (CrossGDT), PAGE_SIZE)
 
 /*
  * Out of the 5 pages, only the first and last are really used.
@@ -52,28 +52,17 @@ typedef struct CrossGDT {
  * mapping and invalidation code which pages are mapped and unmapped.
  */
 #define CROSSGDT_PAGEMASK 0x11
+#define CROSSGDT_GETINDEXMASK(i) (1 << ((i) * sizeof (Descriptor) / PAGE_SIZE))
+#define CROSSGDT_TESTINDEXMASK(i) (CROSSGDT_GETINDEXMASK(i) & CROSSGDT_PAGEMASK)
 
 /*
- * These segments are placed in the first crossGDT page.  We assume
- * they do not overlap any host segments (checked by
- * Task_InitCrossGDT).  The only real requirement is that they (and
- * the host CS/SS) be physically contiguous with the start of the
- * crossGDT so they will remain valid when paging is turned off.
+ * All necessary host segments must be below CROSSGDT_HOSTLIMIT. In Nov 2006,
+ * host GDT limits for the various guest OSes were:
  *
- * As of this writing (Nov 2006), host GDT limits:
  *     Linux 64 bit:  80 (yes 80, not 7F)
- *           32 bit:  FF
  *     MacOS 64 bit:  8F
- *           32 bit:  8F
  *   Windows 64 bit:  6F
- *           32 bit: 3FF
  */
-#define CROSSGDT_LOWSEG   (0x0FD0 / 8)  // all host segs must be below this
-#define CROSSGDT_64BITCS  (0x0FD0 / 8)  // 64-bit code segment
-#define CROSSGDT_64BITSS  (0x0FD8 / 8)  // 64-bit data segment
-#define CROSSGDT_FLAT32CS (0x0FE0 / 8)  // 32-bit flat code seg
-#define CROSSGDT_FLAT32SS (0x0FE8 / 8)  // 32-bit flat data seg
-#define CROSSGDT_MON32CS  (0x0FF0 / 8)  // 32-bit FFC00000 base code seg
-#define CROSSGDT_MON32SS  (0x0FF8 / 8)  // 32-bit FFC00000 base data seg
+#define CROSSGDT_HOSTLIMIT (PAGE_SIZE / sizeof (Descriptor)) /* 1st page */
 
 #endif

vmmon-only/include/iocontrols.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -43,6 +43,17 @@
 #include "overheadmem_types.h"
 #include "pageLock_defs.h"
 #include "numa_defs.h"
+#include "bootstrap_vmm.h"
+
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
+/*
+ * Maximum number of shared regions that can be passed to MonLoader.
+ */
+#define ML_SHARED_REGIONS_MAX 3
 
 /*
  *-----------------------------------------------------------------------------
@@ -133,7 +144,7 @@ PtrToVA64(void const *ptr) // IN
  *
  */
 
-#define VMMON_VERSION           (308 << 16 | 0)
+#define VMMON_VERSION           (329 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))
 
@@ -197,11 +208,10 @@ enum IOCTLCmd {
 #endif
    IOCTLCMD(VERSION) = IOCTLCMD(FIRST),
    IOCTLCMD(CREATE_VM),
+   IOCTLCMD(PROCESS_BOOTSTRAP),
    IOCTLCMD(RELEASE_VM),
    IOCTLCMD(GET_NUM_VMS),
-   IOCTLCMD(ALLOC_CROSSGDT),
    IOCTLCMD(INIT_VM),
-   IOCTLCMD(INIT_CROSSGDT),
    IOCTLCMD(RUN_VM),
    IOCTLCMD(LOOK_UP_MPN),
    IOCTLCMD(LOCK_PAGE),
@@ -212,7 +222,6 @@ enum IOCTLCmd {
    IOCTLCMD(ADMIT),
    IOCTLCMD(UPDATE_MEM_INFO),
    IOCTLCMD(READMIT),
-   IOCTLCMD(PAE_ENABLED),
    IOCTLCMD(GET_TOTAL_MEM_USAGE),
    IOCTLCMD(GET_KHZ_ESTIMATE),
    IOCTLCMD(SET_HOST_CLOCK_RATE),
@@ -224,7 +233,6 @@ enum IOCTLCmd {
    IOCTLCMD(ALLOC_LOCKED_PAGES),
    IOCTLCMD(FREE_LOCKED_PAGES),
    IOCTLCMD(GET_NEXT_ANON_PAGE),
-   IOCTLCMD(GET_LOCKED_PAGES_LIST),
 
    IOCTLCMD(GET_ALL_MSRS),
 
@@ -251,7 +259,6 @@ enum IOCTLCmd {
 
 #if defined __linux__ || defined __APPLE__
    IOCTLCMD(GET_ALL_CPUID),
-   IOCTLCMD(GET_KERNEL_CLOCK_RATE),
 #endif
 
 #if defined _WIN32 || defined __APPLE__
@@ -260,9 +267,8 @@ enum IOCTLCmd {
 
 #if defined _WIN32
    IOCTLCMD(FREE_CONTIG_PAGES),
-   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS),	// Windows 2000 only
-   IOCTLCMD(BLUE_SCREEN),	// USE_BLUE_SCREEN only
-   IOCTLCMD(CHANGE_HARD_LIMIT),
+   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows
+   IOCTLCMD(CHANGE_HARD_LIMIT),         // used by vmauthd on Windows
    IOCTLCMD(GET_KERNEL_PROC_ADDRESS),
    IOCTLCMD(READ_VA64),
    IOCTLCMD(SET_MEMORY_PARAMS),
@@ -277,14 +283,7 @@ enum IOCTLCmd {
    IOCTLCMD(BLUEPILL),
 #endif
 
-   IOCTLCMD(SET_POLL_TIMEOUT_PTR),
-
-   IOCTLCMD(FAST_SUSP_RES_SET_OTHER_FLAG),
-   IOCTLCMD(FAST_SUSP_RES_GET_MY_FLAG),
-
 #if defined __linux__
-   IOCTLCMD(SET_HOST_CLOCK_PRIORITY),
-   IOCTLCMD(VMX_ENABLED),
    IOCTLCMD(SET_HOST_SWAP_SIZE),
 #endif
 
@@ -318,10 +317,9 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_VERSION             VMIOCTL_BUFFERED(VERSION)
 #define IOCTL_VMX86_CREATE_VM           VMIOCTL_BUFFERED(CREATE_VM)
 #define IOCTL_VMX86_RELEASE_VM          VMIOCTL_BUFFERED(RELEASE_VM)
+#define IOCTL_VMX86_PROCESS_BOOTSTRAP   VMIOCTL_BUFFERED(PROCESS_BOOTSTRAP)
 #define IOCTL_VMX86_GET_NUM_VMS         VMIOCTL_BUFFERED(GET_NUM_VMS)
-#define IOCTL_VMX86_ALLOC_CROSSGDT      VMIOCTL_BUFFERED(ALLOC_CROSSGDT)
 #define IOCTL_VMX86_INIT_VM             VMIOCTL_BUFFERED(INIT_VM)
-#define IOCTL_VMX86_INIT_CROSSGDT       VMIOCTL_BUFFERED(INIT_CROSSGDT)
 #define IOCTL_VMX86_RUN_VM              VMIOCTL_NEITHER(RUN_VM)
 #define IOCTL_VMX86_SEND_IPI            VMIOCTL_NEITHER(SEND_IPI)
 #define IOCTL_VMX86_GET_IPI_VECTORS     VMIOCTL_BUFFERED(GET_IPI_VECTORS)
@@ -334,8 +332,6 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_ADMIT               VMIOCTL_BUFFERED(ADMIT)
 #define IOCTL_VMX86_READMIT             VMIOCTL_BUFFERED(READMIT)
 #define IOCTL_VMX86_UPDATE_MEM_INFO     VMIOCTL_BUFFERED(UPDATE_MEM_INFO)
-#define IOCTL_VMX86_PAE_ENABLED         VMIOCTL_BUFFERED(PAE_ENABLED)
-#define IOCTL_VMX86_BEEP                VMIOCTL_BUFFERED(BEEP)
 #define IOCTL_VMX86_HARD_LIMIT_MONITOR_STATUS   VMIOCTL_BUFFERED(HARD_LIMIT_MONITOR_STATUS)
 #define IOCTL_VMX86_CHANGE_HARD_LIMIT   VMIOCTL_BUFFERED(CHANGE_HARD_LIMIT)
 #define IOCTL_VMX86_ALLOC_CONTIG_PAGES  VMIOCTL_BUFFERED(ALLOC_CONTIG_PAGES)
@@ -352,7 +348,6 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_ALLOC_LOCKED_PAGES  VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES)
 #define IOCTL_VMX86_FREE_LOCKED_PAGES   VMIOCTL_BUFFERED(FREE_LOCKED_PAGES)
 #define IOCTL_VMX86_GET_NEXT_ANON_PAGE  VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE)
-#define IOCTL_VMX86_GET_LOCKED_PAGES_LIST VMIOCTL_BUFFERED(GET_LOCKED_PAGES_LIST)
 
 #define IOCTL_VMX86_GET_KERNEL_PROC_ADDRESS  VMIOCTL_BUFFERED(GET_KERNEL_PROC_ADDRESS)
 #define IOCTL_VMX86_READ_VA64           VMIOCTL_BUFFERED(READ_VA64)
@@ -361,16 +356,11 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_REMEMBER_KHZ_ESTIMATE VMIOCTL_BUFFERED(REMEMBER_KHZ_ESTIMATE)
 
 #define IOCTL_VMX86_GET_ALL_MSRS        VMIOCTL_BUFFERED(GET_ALL_MSRS)
-#define IOCTL_VMX86_COUNT_PRESENT_PAGES	VMIOCTL_BUFFERED(COUNT_PRESENT_PAGES)
-
-#define IOCTL_VMX86_FAST_SUSP_RES_SET_OTHER_FLAG VMIOCTL_BUFFERED(FAST_SUSP_RES_SET_OTHER_FLAG)
-#define IOCTL_VMX86_FAST_SUSP_RES_GET_MY_FLAG    VMIOCTL_BUFFERED(FAST_SUSP_RES_GET_MY_FLAG)
 
 #define IOCTL_VMX86_GET_REFERENCE_CLOCK_HZ   VMIOCTL_BUFFERED(GET_REFERENCE_CLOCK_HZ)
 #define IOCTL_VMX86_INIT_PSEUDO_TSC          VMIOCTL_BUFFERED(INIT_PSEUDO_TSC)
 #define IOCTL_VMX86_CHECK_PSEUDO_TSC         VMIOCTL_BUFFERED(CHECK_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_PSEUDO_TSC           VMIOCTL_NEITHER(GET_PSEUDO_TSC)
-#define IOCTL_VMX86_SET_HOST_CLOCK_PRIORITY  VMIOCTL_BUFFERED(SET_HOST_CLOCK_PRIORITY)
 #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS    VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS)
 #define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
 #define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
@@ -563,13 +553,34 @@ typedef struct IPIVectors {
 #endif
 
 /*
- * This struct is passed to IOCTL_VMX86_INIT_CROSSGDT to fill in a crossGDT
- * entry.
+ * Arguments and return value for VM creation.
  */
-typedef struct InitCrossGDT {
-   uint32 index;      // index in crossGDT to update (offset / 8)
-   Descriptor value;  // value to set the crossGDT entry to
-} InitCrossGDT;
+typedef struct VMCreateBlock {
+   VA64               bsBlob;        // IN: User VA of the VMM bootstrap blob.
+   uint32             bsBlobSize;    // IN: Size of VMM bootstrap blob.
+   uint16             vmid;          // OUT: VM ID for the created VM.
+} VMCreateBlock;
+
+/*
+ * Information about a shared region.
+ */
+typedef struct VMSharedRegion {
+   uint32 index;
+   VPN    baseVpn;
+   uint32 numPages;
+} VMSharedRegion;
+
+/*
+ * Arguments for VMM bootstrap processing.
+ */
+typedef struct VMProcessBootstrapBlock {
+   VA64           bsBlobAddr;    // IN: User VA of the VMM bootstrap blob.
+   uint32         numBytes;      // IN: Size of VMM bootstrap blob.
+   uint32         headerOffset;  // IN: Offset of header in blob.
+   uint16         numVCPUs;      // IN: Number of VCPUs.
+   VA64           ptRootVAs[MAX_VCPUS];  // IN: User VA of PT roots.
+   VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions.
+} VMProcessBootstrapBlock;
 
 #if defined __linux__
 
@@ -620,4 +631,9 @@ typedef struct VMAllocContiguousMem {
 /* Clean up helper macros */
 #undef IOCTLCMD
 
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif // ifndef _IOCONTROLS_H_

vmmon-only/include/memDefaults.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -24,12 +24,15 @@
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_MODULE
-
 #include "includeCheck.h"
 
 #include "vm_basic_math.h"
 #include "vm_basic_defs.h"
 
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
 #define MEMDEFAULTS_MIN_HOST_PAGES   MBYTES_2_PAGES(128)
 
 
@@ -151,4 +154,8 @@ MemDefaults_CalcMinReservedMBs(uint32 hostMem)  // IN:
 void MemDefaults_GetReservedMemory(uint32 *host, uint32 *min,
                                    uint32 *max, uint32 *recommended);
 
+#if defined(__cplusplus)
+}  // extern "C"
+#endif
+
 #endif

vmmon-only/include/modulecall.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -56,7 +56,9 @@
    MC(COSCHED)                                                                \
    MC(ALLOC_VMX_PAGE)                                                         \
    MC(ALLOC_TMP_GDT)                                                          \
-   MC(PIN_MPN)
+   MC(PIN_MPN)                                                                \
+   MC(VMCLEAR_VMCS_ALL_CPUS)                                                  \
+   MC(GET_PAGE_ROOT)
 
 /*
  *----------------------------------------------------------------------
@@ -124,55 +126,14 @@ typedef enum UCCostStamp {
 } UCCostStamp;
 #endif // VMX86_SERVER
 
-/*
- * Header for the wsBody64.S worldswitch code file.
- */
-typedef struct WSModule {
-   uint32 vmmonVersion;  // VMMON_VERSION when assembled as part of monitor
-   uint16 moduleSize;    // size of whole wsBody64 module
-   uint16 hostToVmm;     // offset from beginning of header to hostToVmm
-   uint16 vmm64ToHost;
-   uint16 _pad[3];
-
-   uint8 code[1024];      // big enough for MAX('.wsBody64', 'wsBody64Log')
-} WSModule;
-
 typedef
 #include "vmware_pack_begin.h"
-struct SwitchNMIOffsets {
-   uint16         db;    // offset to start of  #DB handler
-   uint16         nmi;   // offset to start of #NMI handler
-   uint16         df;    // offset to start of  #DF handler
-   uint16         ud;    // offset to start of  #UD handler
-   uint16         gp;    // offset to start of  #GP handler
-   uint16         pf;    // offset to start of  #PF handler
-   uint16         mce;   // offset to start of #MCE handler
+struct CodeOffsets {
+   uint16 hostToVmm;
+   uint16 vmmToHost;
 }
 #include "vmware_pack_end.h"
-SwitchNMIOffsets;
-
-/*
- * This is a header for the switchNMI.S module.  It contains code for
- * exceptions occurring during worldswitch.  The code gets copied to
- * the crosspage by initialization.
- */
-typedef
-#include "vmware_pack_begin.h"
-struct SwitchNMI {                                // see switchNMI.S
-   uint16           switchNMISize;
-   SwitchNMIOffsets host;                         // offsets to handlers
-   volatile Bool    wsException[NUM_EXCEPTIONS];  // EXC_DE ... EXC_XF
-                                                  // TRUE -> fault occurred in
-                                                  //   worldswitch
-   uint64           wsUD2;                        // IP of ud2 instruction
-                                                  //    0ULL == unset
-                                                  //   other == worldswitch IP
-   uint8            codeBlock[768];               // Enough for
-                                                  //   max('.switchNMI',
-                                                  //       '.switchNMILog').
-}
-#include "vmware_pack_end.h"
-SwitchNMI;
+CodeOffsets;
 
 #define SHADOW_DR(cpData, n)    (cpData)->shadowDR[n].ureg64
 
@@ -211,7 +172,7 @@ SwitchNMI;
 
 /*----------------------------------------------------------------------
  *
- * VMM64PageTablePatch
+ * VMMPageTablePatch
  *
  *    Describes an entry in the monitor page table which needs to be
  *    patched during the back-to-host worldswitch.
@@ -224,28 +185,16 @@ SwitchNMI;
  *        This is the level in the page table to which the patch must
  *        be applied: L4, L3, L2, L1.  This information is used to
  *        determine the base of the region of memory which must be
- *        patched.  The level value corresponds to the following
- *        regions in monitor memory:
+ *        patched. The value zero is reserved to indicate an empty spot
+ *        in the array of patches.
  *
- *          MMU_ROOT_64
- *          MMU_L3_64
- *          MMU_L2_64
- *          MON_PAGE_TABLE_64
+ *      o index
  *
- *        The value zero is reserved to indicate an empty spot in the
- *        array of patches.
+ *        The index of the PTE at the given page table.
  *
- *      o level offset
+ *      o ptIdx
  *
- *        The monitor memory regions corresponding to the page table
- *        levels may be more than one page in length, so a 'page
- *        offset' is required to know the starting address of the page
- *        table page which must be patched in 'level'.
- *
- *      o page index
- *
- *        The 'index' value specifies the element in the page which
- *        should be patched.
+ *        The index of the page table at the given level.
  *
  *      o pte
  *
@@ -256,19 +205,19 @@ SwitchNMI;
  */
 typedef
 #include "vmware_pack_begin.h"
-struct VMM64PageTablePatch {
+struct VMMPageTablePatch {
 #define PTP_EMPTY    (0U) /* Unused array entry. (must be 0) */
-#define PTP_LEVEL_L1 (1U)       /* leaf level */
+#define PTP_LEVEL_L1 (1U)
 #define PTP_LEVEL_L2 (2U)
 #define PTP_LEVEL_L3 (3U)
 #define PTP_LEVEL_L4 (4U)       /* root level */
-   uint16   level;              /* [0, 4]  (maximal size: 3 bits) */
-   uint16   page;               /* Index of 'page' in 'level'.    */
-   uint32   index;              /* Index of 'pte' in 'page'.      */
-   VM_PDPTE pte;                /* PTE.                           */
+   uint32   level;              /* [0, 4]  (maximal size: 3 bits) */
+   uint32   pteIdx;             /* Index of the PTE in the page table. */
+   uint64   pteGlobalIdx;       /* Global index of the PTE in 'level'. */
+   VM_PDPTE pte;                /* PTE.                                */
 }
 #include "vmware_pack_end.h"
-VMM64PageTablePatch;
+VMMPageTablePatch;
 
 #define MODULECALL_NUM_ARGS  4
 
@@ -345,16 +294,17 @@ struct VMCrossPageData {
    uint64   mon64RIP;
    Task64   monTask64;          /* vmm64's task */
 
-   VMM64PageTablePatch vmm64PTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
-   LA64                vmm64CrossPageLA;
-   LA64                vmm64CrossGDTLA;   // where crossGDT mapped by PT patch
-                                          //  64-bit host: host kernel linear
-                                          // address
+   VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */
+   LA64              vmm64CrossPageLA;
+   LA64              vmm64CrossGDTLA;   // where crossGDT mapped by PT patch
+                                        //  64-bit host: host kernel linear
+                                        // address
 
    /*
     * The monitor may requests up to two actions when returning to the
     * host.  The moduleCallType field and args encode a request for
-    * some action in the driver.  The userCallType field (together
+    * some action in the driver.  The vcpuSet field is an additional
+    * argument used in some calls.  The userCallType field (together
     * with the RPC block) encodes a user call request.  The two
     * requests are independent.  The user call is executed first, with
     * the exception of MODULECALL_INTR which has a special effect.
@@ -362,6 +312,9 @@ struct VMCrossPageData {
    ModuleCallType moduleCallType;
    uint32         retval;
    uint64         args[MODULECALL_NUM_ARGS];
+#if !defined(VMX86_SERVER)
+   VCPUSet        vcpuSet;
+#endif
    int            userCallType;
    uint32         pcpuNum;   /* Used as extra module call arg within vmmon. */
 
@@ -397,8 +350,6 @@ struct VMCrossPageData {
    VmAbsoluteTS hstTimerExpiry; // PTSC of host timer interrupt
    VmAbsoluteTS monTimerExpiry; // PTSC of next MonTimer callback
 
-   Bool     activateVMX;        // TRUE -> activate Intel VMX extensions
-   Bool     activateSVM;        // TRUE -> activate AMD SVM extensions
    Bool     retryWorldSwitch;   // TRUE -> return to host on host->vmm switch
    /*
     * TRUE if moduleCall was interrupted by signal. Only
@@ -406,12 +357,11 @@ struct VMCrossPageData {
     * restart RunVM call, nobody else should look at it.
     */
    Bool     moduleCallInterrupted;
-   uint8    _pad3[4];
+   uint8    _pad3[6];
 
-   DTR64    switchHostIDTR;     // baseLA = switchHostIDT's host knl LA
+   DTR64    switchHostIDTR;   // baseLA = switchHostIDT's host kernel LA
    uint16   _pad4[3];
-   DTR64    switchMon64IDTR;    // has baseLA = switchMon64IDT's monitor LA
-                                //   contains 64-bit DB,NMI,MCE entries
+   DTR64    switchMonIDTR;    // baseLA = switchMonIDT's monitor LA
    uint16   _pad5[3];
 
    /*
@@ -419,12 +369,16 @@ struct VMCrossPageData {
     * IDT has only enough space for the hardware exceptions; they are
     * sized to accommodate 64-bit descriptors.
     */
-   uint8 switchHostIDT [sizeof(Gate64) * NUM_EXCEPTIONS]; // hostCS:hostVA
-   uint8 switchMon64IDT[sizeof(Gate64) * NUM_EXCEPTIONS]; // 64-bit monCS:monVA
+   uint8 switchHostIDT[sizeof(Gate64) * NUM_EXCEPTIONS]; // hostCS:hostVA
+   uint8 switchMonIDT[sizeof(Gate64) * NUM_EXCEPTIONS];  // monCS:monVA
+
+   volatile Bool wsException[NUM_EXCEPTIONS]; // Tracks faults in worldswitch.
+   uint64        wsUD2;                       // IP of ud2 instr or 0 if unset.
 }
 #include "vmware_pack_end.h"
 VMCrossPageData;
 
+
 /*
  *----------------------------------------------------------------------
  *
@@ -437,8 +391,10 @@ VMCrossPageData;
 typedef
 #include "vmware_pack_begin.h"
 struct VMCrossPageCode {
-   WSModule   worldswitch;
-   SwitchNMI  faultHandler;
+   uint16        size;           // Size of the code module.
+   uint32        vmmonVersion;   // VMMON_VERSION
+   CodeOffsets   offsets;        // Offsets to handlers.
+   uint8         codeBlock[512]; // Code for worldswitch and fault handling.
 }
 #include "vmware_pack_end.h"
 VMCrossPageCode;
@@ -465,16 +421,16 @@ struct VMCrossPage {
    uint32          version;         /* 4 bytes. Must be at offset zero. */
    uint32          crosspage_size;  /* 4 bytes. Must be at offset 4.    */
    VMCrossPageData crosspageData;
+   VMCrossPageCode crosspageCode;
    uint8           _pad[PAGE_SIZE - (sizeof(uint32) /* version */        +
                                      sizeof(uint32) /* crosspage_size */ +
                                      sizeof(VMCrossPageData)             +
                                      sizeof(VMCrossPageCode))];
-   VMCrossPageCode crosspageCode;
 }
 #include "vmware_pack_end.h"
 VMCrossPage;
 
-#define CROSSPAGE_VERSION_BASE 0xbf1 /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xbfb /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)
 
 #if !defined(VMX86_SERVER) && defined(VMM)

vmmon-only/include/modulecallstructs.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006,2009-2011,2013-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006,2009-2011,2013-2015 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -66,7 +66,8 @@
    SWMSR(MSR_CSTAR)         \
    SWMSR(MSR_SFMASK)        \
    SWMSR(MSR_TSC_AUX)       \
-   SWMSR(MSR_BD_TSC_RATIO)
+   SWMSR(MSR_BD_TSC_RATIO)  \
+   SWMSR(MSR_BNDCFGS)
 
 /*
  *      Data structures for dealing with the context-switched MSRs that need

vmmon-only/include/monAddrLayout.h

@@ -0,0 +1,57 @@
+/*********************************************************
+ * Copyright (C) 2005,2007,2009,2013,2015 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * monAddrLayout.h --
+ *
+ *    Address layout of the monitor
+ */
+
+#ifndef _MON_ADDR_LAYOUT_H
+#define _MON_ADDR_LAYOUT_H
+
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_USERLEVEL
+#include "includeCheck.h"
+
+#pragma pack(push, 1)
+typedef struct VMM64_AddrLayout {
+   /* 
+    *All address are VPNs and all lengths are numPages
+    */
+   uint64 monBase;            // MONITOR_BASE_VPN
+   uint64 mmuRootStart;       // MMU_ROOT_START
+   uint32 mmuRootLen;         // MMU_ROOT_LEN
+   uint64 mmuL3Start;         // MMU_L3_START
+   uint32 mmuL3Len;           // MMU_L3_LEN
+   uint64 mmuL2Start;         // MMU_L2_START
+   uint32 mmuL2Len;           // MMU_L2_LEN
+   uint64 monPageTableStart;  // MON_PAGE_TABLE_START
+   uint32 monPageTableLen;    // MON_PAGE_TABLE_LEN
+#ifndef VMX86_SERVER
+   uint64 monCpStart;         // CROSS_PAGE_START
+#endif
+} VMM64_AddrLayout;
+#pragma pack(pop)
+
+#define VMM_SCRATCHAS_FIRST_L4OFF   1
+#define VMM_SCRATCHAS_LAST_L4OFF  127
+
+#endif

vmmon-only/include/monLoader.h

@@ -0,0 +1,302 @@
+/*********************************************************
+ * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * monLoader.h --
+ *
+ *      Describes the monitor loader, its header and support code for
+ *      processing this header.
+ *
+ *
+ * Overview
+ * ========
+ *
+ * Before the monitor or its in-context bootstrap code can run, a monitor
+ * address space must be created and partially populated.  This address space
+ * contains code and data statically built, empty but allocated space, shared
+ * and run-time initialized content.  The monitor loader header regularizes
+ * encoding of address space information, allowing a common representation and
+ * common code to be re-used for different contexts (vmmon for hosted, vmkernel
+ * for ESX, and the VMX while deprivileging of bootstrap is ongoing).
+ *
+ * The Header
+ * ==========
+ *
+ * The monitor loader header contains a sequence of entries describing content.
+ * Each entry has a start and end VPN, an optional content source, permission
+ * flags to be applied when mapping, an optional subIndex specifying a shared
+ * region (for shared content types) and a flag specifying whether to process
+ * this entry for all VCPUs or just the bootstrap processor.
+ *
+ * Content types
+ * =============
+ *
+ * ADDRSPACE: Must be first.  Describes the address space in which all other
+ * entries reside.  Page tables from L4 to L1 will be allocated, in order,
+ * eagerly.  These page tables will be wired with the permissions of this entry
+ * from L4 to L1.  This allows easy access to the L1E for any VPN in the space.
+ *
+ * ML_CONTENT_PAGETABLE_Lx: Maps the page tables for a given level.  It must be
+ * exactly the size of the tables preallocated by ADDRSPACE for the level.
+ *
+ * ML_CONTENT_ALLOCZERO: Allocates new MPNs, zeroes the pages and maps them.
+ *
+ * ML_CONTENT_COPY: Allocates new MPNs, maps them and copies from the specified
+ * source.
+ *
+ * ML_CONTENT_SHARE: Memory provided by user or host kernel and mapped into the
+ * monitor address space.  The subIndex field specifies which region, as
+ * multiple regions may be shared for each source.
+ *
+ * Processing
+ * ==========
+ *
+ * The monitor loader can be built in different contexts.  Each context must
+ * provide implementations of callout functions.  To fully build the context,
+ * callouts for memory allocation, access and reporting of resources will be
+ * added, as will accounting functionality.
+ *
+ * Callouts
+ * ========
+ * 
+ * Memory-accessing callouts take a Vcpuid as memory is tracked per-VCPU.
+ *
+ * MonLoaderCallout_AllocMPN(MPN): Allocate a new MPN.
+ * MonLoaderCallout_CleanUp(): Release temporary MonLoader callout resources.
+ * MonLoaderCallout_CopyFromBlob(blobOffset, size, MPN): Copy blob contents.
+ * MonLoaderCallout_FillPage(pattern, mpn): fill a page with a pattern.
+ * MonLoaderCallout_GetPTE(MPN, index, *pte): Get indexth PTE in a PT MPN.
+ * MonLoaderCallout_GetPageRoot(vcpu): Get a VCPU's page root.
+ * MonLoaderCallout_ImportPage(MPN): Import an MPN for tracking and future use.
+ * MonLoaderCallout_Init(): Initialize MonLoader callouts.
+ * MonLoaderCallout_GetSharedHostPage(subIdx, page): Get shared host page MPN.
+ * MonLoaderCallout_GetSharedUserPage(subIdx, page): Get shared user page MPN.
+ * MonLoaderCallout_SetEntrypoint(): Sets the entry code/stack for execution.
+ *
+ * Building vs Importing
+ * =====================
+ *
+ * While legacy VMX is still responsible for building part of the monitor
+ * context, the monitor loader must cooperate.  Rather than building the
+ * context itself, the monitor loader verifies its environment matches its
+ * header's expectations.  This includes verifying sufficient allocations,
+ * permissions of page table wiring and page table self-mapping.
+ *
+ * Errors
+ * ======
+ *
+ * To avoid ASSERTs across contexts, errors are returned and clean-up is done
+ * carefully if processing fails at any stage.  Some errors are debug-only and
+ * essentially verify invariants for debug builds.  The line in the table at
+ * which the error was encountered is returned for logging and debugging.
+ *
+ * Compatibility
+ * =============
+ *
+ * bootstrap-offsets.pl relies on MonLoaderContentType, MonLoaderSourceType and
+ * the structure of MonLoaderEntry to determine and set blob offsets for
+ * ML_CONTENT_COPY + ML_SOURCE_BLOB.  If these are changed, the script should
+ * be updated accordingly.
+ */
+
+#ifndef _MON_LOADER
+#define _MON_LOADER
+
+#include "vm_basic_types.h"
+#include "x86paging_common.h"
+#include "x86types.h" /* PT_L1E */
+#include "vcpuid.h"   /* Vcpuid */
+
+#define ML_NAME_MAX 16
+
+/* ML perms are simple and abbreviated. */
+
+#define ML_PERM_RWX (PTE_P | PTE_RW)
+#define ML_PERM_RW  (PTE_P | PTE_RW | PTE_NX)
+#define ML_PERM_RO  (PTE_P |          PTE_NX)
+#define ML_PERM_RX   PTE_P
+
+#define ML_PERM_MASK (PTE_P| PTE_RW | PTE_NX | PTE_US)
+
+#define ML_PERMS_MATCH(x,p) (((x) & ML_PERM_MASK) == ((p) & ML_PERM_MASK))
+
+#define LOADER_HEADER_MAGIC 0x8675309E98675309
+
+typedef enum {
+   ML_CONTENT_INVALID = 0,
+   ML_CONTENT_ADDRSPACE,     /* The static address space. */
+   ML_CONTENT_ALLOCZERO,     /* Allocate, zero-fill and map. */
+   ML_CONTENT_COPY,          /* Copy data from external source. */
+   ML_CONTENT_PAGETABLE_L4,  /* Mappings for level 4 page tables in AS. */
+   ML_CONTENT_PAGETABLE_L3,  /* Mappings for level 3 page tables in AS. */
+   ML_CONTENT_PAGETABLE_L2,  /* Mappings for level 2 page tables in AS. */
+   ML_CONTENT_PAGETABLE_L1,  /* Mappings for level 1 page tables in AS. */
+   ML_CONTENT_SHARE,         /* Share data from external source. */
+} MonLoaderContentType;
+
+#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? 4 : \
+                               x == ML_CONTENT_PAGETABLE_L3 ? 3 : \
+                               x == ML_CONTENT_PAGETABLE_L2 ? 2 : \
+                               x == ML_CONTENT_PAGETABLE_L1 ? 1 : \
+                               0)
+
+/* Sources of content, for pages not zeroed, unmapped or otherwise special. */
+typedef enum MonLoaderSourceType {
+   ML_SOURCE_INVALID  = 0x0,
+   ML_SOURCE_NONE,
+   ML_SOURCE_BLOB,
+   ML_SOURCE_USER,
+   ML_SOURCE_HOST,
+} MonLoaderSourceType;
+
+/*
+ * NOTE: When modifying MonLoaderEntry, corresponding changes must be
+ * made to vmcore/make/misc/bootstrap-offsets.pl.
+ */
+typedef struct {
+   MonLoaderContentType content;           /* Content type. */
+   MonLoaderSourceType  source;            /* Source, if any, for content. */
+   VPN                  monVPN;            /* Destination in monitor AS. */
+   uint64               monPages;          /* Size in pages. */
+   uint64               flags;             /* PTE permissions */
+   unsigned             allocs;            /* MPNs allocated (all VCPUs). */
+   /* Set for source ML_SOURCE_BLOB: */
+   struct blobSrc {
+      uint64 offset; /* offset within the blob, in bytes. */
+      uint64 size;   /* size of content, in bytes. */
+   } blobSrc;
+   uint64               bspOnly;           /* Process only on BSP. */
+   uint64               subIndex;          /* Region ID for ML_CONTENT_COPY. */
+   uint64               procVmmon;         /* Processed by the vmmon on hosted. */
+} MonLoaderEntry;
+
+
+/* Packed for easy consumption by bootstrap-offsets.pl. */
+#pragma pack(push, 1)
+typedef struct {
+   uint64         magic;
+   uint32         entrySize;
+   uint32         count;
+   /* cs:rip */
+   uint16         codeSelector;
+   VA64           codeEntrypoint;
+   /* ss:rip */
+   uint16         stackSelector;
+   VA64           stackEntrypoint;
+   MonLoaderEntry entries[];
+} MonLoaderHeader;
+#pragma pack(pop)
+
+/* Environment context structure, defined by the environment. */
+struct MonLoaderEnvContext;
+typedef struct MonLoaderEnvContext MonLoaderEnvContext;
+
+/* Callout prototypes */
+MPN  MonLoaderCallout_AllocMPN(MonLoaderEnvContext *, Vcpuid);
+void MonLoaderCallout_CleanUp(MonLoaderEnvContext *);
+Bool MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *, uint64, size_t, MPN,
+                                   Vcpuid);
+Bool MonLoaderCallout_FillPage(MonLoaderEnvContext *, uint8, MPN, Vcpuid);
+MPN  MonLoaderCallout_GetPageRoot(MonLoaderEnvContext *, Vcpuid);
+Bool MonLoaderCallout_GetPTE(MonLoaderEnvContext *, MPN, unsigned, Vcpuid,
+                             PT_L1E *);
+Bool MonLoaderCallout_ImportPage(MonLoaderEnvContext *, MPN, Vcpuid);
+Bool MonLoaderCallout_Init(void *, MonLoaderEnvContext **);
+Bool MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *, MPN, unsigned, uint64,
+                                  MPN, Vcpuid);
+MPN  MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *, uint64, unsigned,
+                                        Vcpuid);
+MPN  MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *, uint64, unsigned,
+                                        Vcpuid);
+Bool MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *);
+Bool MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *, uint16, VA64, uint16,
+                                    VA64);
+MPN  MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *, uint64);
+
+typedef enum MonLoaderError {
+   ML_OK = 0,
+   ML_ERROR_ADDRSPACE_TOO_LARGE,
+   ML_ERROR_ALLOC,
+   ML_ERROR_ALREADY_MAPPED,
+   ML_ERROR_ARGS,
+   ML_ERROR_CALLOUT_INIT,
+   ML_ERROR_CALLOUT_COPY,
+   ML_ERROR_CALLOUT_ENTRYPOINT,
+   ML_ERROR_CALLOUT_GETPTE,
+   ML_ERROR_CALLOUT_MAPINPTE,
+   ML_ERROR_CALLOUT_PAGEROOT_GET,
+   ML_ERROR_CALLOUT_ZERO,
+   ML_ERROR_CONTENT_INVALID,
+   ML_ERROR_CONTEXT_INIT,
+   ML_ERROR_DUPLICATE,
+   ML_ERROR_INVALID_VPN,
+   ML_ERROR_MAGIC,
+   ML_ERROR_MAP,
+   ML_ERROR_NO_ADDRSPACE,
+   ML_ERROR_PAGE_TABLE_IMPORT,
+   ML_ERROR_PAGE_TABLE_MAP_SIZE,
+   ML_ERROR_PAGE_TABLE_VERIFY,
+   ML_ERROR_SHARE,
+   ML_ERROR_SIZE,
+   ML_ERROR_SOURCE_INVALID,
+   ML_ERROR_TABLE_MISSING,
+} MonLoaderError;
+
+
+/* A subindex above shared area subindices for sharing of MonLoaderHeader. */
+#define MONLOADER_HEADER_IDX (NUM_SHARED_AREAS + 1)
+
+
+MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs,
+                                 void *args, unsigned *line, Vcpuid *vcpu);
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoader_GetFixedHeaderSize --
+ *
+ *      Returns the size of the fixed portion of MonLoaderHeader.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE size_t
+MonLoader_GetFixedHeaderSize(void)
+{
+   return sizeof(MonLoaderHeader);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * MonLoader_GetFullHeaderSize --
+ *
+ *      Returns the size of the full MonLoaderHeader, including the
+ *      variable size portion.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE size_t
+MonLoader_GetFullHeaderSize(MonLoaderHeader *header)
+{
+   return MonLoader_GetFixedHeaderSize() +
+          header->count * sizeof(MonLoaderEntry);
+}
+
+#endif /* !_MON_LOADER */

vmmon-only/include/monLoaderLog.h

@@ -0,0 +1,66 @@
+/*********************************************************
+ * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * monLoaderLog.h --
+ *
+ *      Logging definitions for MonLoader.
+ */
+
+#ifndef _MONLOADER_LOG_H_
+#define _MONLOADER_LOG_H_
+
+#ifdef VMX86_VMX
+
+#define LOGLEVEL_MODULE main
+#include "loglevel_vmcore.h"
+/*
+ * Redefine LOG() to match the vmkernel calling convention.
+ */
+#undef LOG_BYNAME
+#undef LOG
+
+#define LOG_BYNAME(_mod, _min, _fmt, ...) \
+        (DOLOG_BYNAME(_mod, _min) ? Log(_fmt "\n", ## __VA_ARGS__) : (void) 0)
+
+#define LOG(_min, _fmt, ...) \
+        LOG_BYNAME(LOGLEVEL_MODULE, _min, _fmt, ## __VA_ARGS__)
+
+#elif defined VMKERNEL
+
+#define LOGLEVEL_MODULE MonLoader
+#include "log.h"
+
+#elif defined VMMON
+
+#include "vm_assert.h"
+#define ML_LOGLEVEL_VMMON 0 /* MonLoader/vmmon loglevel. */
+
+#undef LOG
+#define LOG(_min, _fmt, ...)                         \
+do {                                                 \
+   if (vmx86_log && ML_LOGLEVEL_VMMON >= (_min)) {   \
+      Log(_fmt "\n", ## __VA_ARGS__);                \
+   }                                                 \
+} while (0)
+
+#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */
+#error MonLoader cannot be built as part of this environment
+#endif
+
+#endif /* _MONLOADER_LOG_H_ */

vmmon-only/include/mon_assert.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2007-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2007-2015 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -169,8 +169,13 @@ extern const char __vmm_pathnames_start;
  * seen to hoist code into the failing arm of the assertion, where it
  * can then tell that, because of the assertion failure, the code ends
  * up accessing an array out of bounds.
+ *
+ * Assertion failures for the monitor's bootstrap are reduced to panics
+ * logging the current %rip.  As such, no .assert_info section is created
+ * and VMX merely processes the panic.
  */
 
+#ifndef VMM_BOOTSTRAP
 #define ASSERT_RECORDINFO(assembly, assertType, bugNr)                   \
    __asm__ __volatile__(".pushsection .assert_info;"                     \
                         ".quad 0f;"                                      \
@@ -182,6 +187,16 @@ extern const char __vmm_pathnames_start;
                         [file] "i" (__VMM__FILE__),                      \
                         [type] "i" (assertType),                         \
                         [bug]  "i" (bugNr))
+#else
+
+extern uint64 bsAssertRIP;
+#define ASSERT_RECORDINFO(assembly, assertType, bugNr)                   \
+   __asm__ __volatile__("lea 0(%%rip), %0\n\t"                           \
+                        : "=r"(bsAssertRIP));                            \
+   Panic("Bootstrap: %s failure at rip=0x%lx",                           \
+         assertType == AssertType_AssertVerify ? "VERIFY" : "ASSERT",    \
+         bsAssertRIP);
+#endif /* VMM_BOOTSTRAP */
 
 #define _ASSERT_PANIC(name)                                              \
    ({COMPILER_MEM_BARRIER();                                             \

vmmon-only/include/monitorAction_exported.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2010-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 2010-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -22,8 +22,6 @@
 #define INCLUDE_ALLOW_VMX
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_USERLEVEL
-#define INCLUDE_ALLOW_MODULE
-#define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_DISTRIBUTE
 #include "includeCheck.h"
@@ -32,6 +30,11 @@
 #include "vm_atomic.h"
 #include "vm_basic_types.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 /*
  * Please bump the version number if your change will break the
  * compatability to the drivers.
@@ -106,7 +109,7 @@ MonitorActionSet_AtomicInclude(MonitorActionSet *set, const uint32 actionID)
    do {
       oldWord = Atomic_Read64(atomicSet);
       newWord = oldWord | mask;
-   } while (!Atomic_CMPXCHG64(atomicSet, &oldWord, &newWord));
+   } while (!Atomic_CMPXCHG64(atomicSet, oldWord, newWord));
    return (oldWord & mask) == 0;
 }
 
@@ -153,4 +156,9 @@ typedef enum {
    VMM_C1_STATE_PAUSE
 } vmmC1StateType;
 
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif // _MONITORACTION_EXPORTED_H_

vmmon-only/include/numa_defs.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006,2009-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -39,15 +39,16 @@
 
 /* Machine NUMA nodes */
 typedef uint32 NUMA_Node;
-typedef uint32 NUMA_NodeMask;
+typedef uint32 NUMA_PxmID;
+typedef uint64 NUMA_NodeMask;
 typedef uint8  NUMA_MemRangeID;
 
 /*
  * Constants
  */
-#define NUMA_MAX_NODES              32
+#define NUMA_MAX_NODES              64
 #define NUMA_MAX_CPUS_PER_NODE      (vmx86_server ? MAX_PCPUS : 32)
-#define NUMA_MAX_MEM_RANGES         64
+#define NUMA_MAX_MEM_RANGES         (vm_arm_64 ? 128 : 64)
 #define INVALID_NUMANODE            ((NUMA_Node)-1)
 #define NUMA_NODE_MASK_ANY          ((NUMA_NodeMask)-1)
 #define NUMA_NODE_MASK_NONE         ((NUMA_NodeMask)0)
@@ -59,7 +60,7 @@ typedef uint8  NUMA_MemRangeID;
 typedef struct {
    MPN          startMPN;
    MPN          endMPN;
-   NUMA_Node    id;
+   NUMA_PxmID   id;
    Bool         isReliable;
    Bool         isVolatile;
 } NUMA_MemRange;

vmmon-only/include/pagelist.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2010-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2010-2014,2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -36,6 +36,11 @@
 #include "vm_assert.h"
 #include "vmcore_types.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 /*
  * Sets of pages are passed between the monitor and the platform to be 
  * shared, invalidated, remapped, or swapped.
@@ -53,13 +58,11 @@ typedef struct PageListEntry {
 
 #define PAGELIST_MAX     (PAGE_SIZE / sizeof(PageListEntry))
 
-static INLINE PageListEntry
-PageList_CreateEntry(BPN bpn)
+static INLINE void
+PageList_SetEntry(PageListEntry *ple, BPN bpn)
 {
-   PageListEntry ple;
-   CompressedBPN_Write(&ple.cbpn, bpn);
-   ple.voided = FALSE;
-   return ple;
+   CompressedBPN_Write(&ple->cbpn, bpn);
+   ple->voided = FALSE;
 }   
 
 static INLINE BPN
@@ -98,4 +101,9 @@ PageList_IsBPNDup(const PageListEntry *pageList, unsigned i, BPN bpn)
    return FALSE;
 }
 
+
+#if defined __cplusplus
+} // extern "C"
 #endif
+
+#endif // _PAGELIST_H

vmmon-only/include/pshare_ext.h

@@ -44,6 +44,7 @@
 #define PSHARE_P2M_BUFFER_MPNS_MAX      (16)
 #define PSHARE_P2M_BUFFER_MPNS_DEFAULT  (4)
 #define PSHARE_P2M_BUFFER_SLOTS_PER_MPN (PAGE_SIZE / sizeof(PShare_P2MUpdate))
+#define PSHARE_P2M_BUFFER_SLOTS_MIN     (2)
 
 #define PSHARE_POISON_MARKER            (CONST64U(0xAAAAAAAAAAAAAAAA))
 
@@ -57,8 +58,16 @@ MY_ASSERTS(PSHARE_EXT,
  * types
  */
 
+typedef union {
+   MPN mpn;
+   uint64 vpmemRef;
+} PShare_P2MUpdateReference;
+
+#define PSHARE_SET_INVALID_P2MUPDATE_REFERENCE(ref) ((ref).vpmemRef = CONST64U(-1))
+#define PSHARE_IS_INVALID_P2MUPDATE_REFERENCE(ref) ((ref).vpmemRef == CONST64U(-1))
+
 typedef struct PShare_P2MUpdate {
-   BPN     bpn;
-   MPN     mpn;
+   BPN bpn;
+   PShare_P2MUpdateReference reference;
 } PShare_P2MUpdate;
 #endif

vmmon-only/include/ptsc.h

@@ -170,7 +170,6 @@ int64 PTSC_CyclesToUS(VmRelativeTS ts);
 /*
  * ESX with userworld VMX
  */
-#include "x86types.h"
 #include "user_layout.h"
 
 #ifdef VM_ARM_64

vmmon-only/include/rateconv.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -31,7 +31,6 @@
 #define _VM_RATECONV_H_
 
 #define INCLUDE_ALLOW_USERLEVEL
-
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMK_MODULE
@@ -44,6 +43,12 @@
 #include "vm_basic_asm.h"
 #include "vm_assert.h"
 #include "vm_atomic.h"
+#include "versioned_atomic.h"
+
+#if defined __cplusplus
+extern "C" {
+#endif
+
 
 /* RateConv_Params is part of vmx<->vmmon interface (INIT_PSEUDO_TSC ioctl) */
 typedef struct RateConv_Params {
@@ -54,8 +59,7 @@ typedef struct RateConv_Params {
 
 typedef struct RateConv_ParamsVolatile {
    RateConv_Params p;
-   Bool            changed;
-   uint8           pad[7];
+   VersionedAtomic vers;
 } RateConv_ParamsVolatile;
 
 typedef struct RateConv_Ratio {
@@ -115,4 +119,8 @@ RateConv_Signed(const RateConv_Params *conv, int64 x)
 }
 
 
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif // _VM_RATECONV_H_

vmmon-only/include/vcpuid.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014, 2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -38,15 +38,19 @@
 
 #include "vm_basic_types.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
 
-typedef uint32 Vcpuid;			// VCPU number
 
-#define VCPUID_INVALID	(~0U)
+typedef uint32 Vcpuid;                 // VCPU number
+
+#define VCPUID_INVALID  (~0U)
 
 #define BOOT_VCPU_ID     0
 #define IS_BOOT_VCPUID(vcpuid)  ((vcpuid) == BOOT_VCPU_ID)
 
-#define MAX_VCPUS      128
+#define MAX_VCPUS      192
 
 #define MAX_CORES_PER_SOCKET   64
 
@@ -62,4 +66,8 @@ extern const VCPUSet     curVcpuidSet;
 
 #endif  /* VMM */
 
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif // ifndef _VCPUID_H_

vmmon-only/include/vcpuset.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -417,13 +417,13 @@ VCPUSet_PackCareful(unsigned numVCPUs, const VCPUSet *vcs, void *ptr)
  *
  * VCPUSet_UnpackCareful --
  *      
- *      Unpack a VCPUSet from the bytes at "src".
+ *      Unpack a VCPUSet from the bytes at "ptr".
  *
  *----------------------------------------------------------------------
  */
 
 static INLINE void
-VCPUSet_UnpackCareful(unsigned numVCPUs, VCPUSet *vcs, void *ptr)
+VCPUSet_UnpackCareful(unsigned numVCPUs, VCPUSet *vcs, const void *ptr)
 {
    memcpy(vcs->subset, ptr, (numVCPUs + 7) / 8);
 }

vmmon-only/include/vcpuset_types.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2013, 2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -19,7 +19,7 @@
 /*
  * vcpuset_types.h --
  *
- *	ADT for a set of VCPUs.  Implemented as an array of bitmasks.
+ *    ADT for a set of VCPUs.  Implemented as an array of bitmasks.
  *
  */
 
@@ -35,14 +35,14 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
 
-#include "vm_basic_asm.h"
-#include "vm_atomic.h"
 #include "vcpuid.h"
+#include "vm_assert.h"
 
 #define VCS_SUBSET_WIDTH                                                   64
 #define VCS_SUBSET_SHIFT                                                    6
 #define VCS_SUBSET_MASK               ((CONST64U(1) << VCS_SUBSET_SHIFT) - 1)
-#define VCS_SUBSET_COUNT                                                    2
+#define VCS_SUBSET_COUNT                                                    4
+
 
 #define VCS_VCPUID_SUBSET_IDX(v)                    ((v) >> VCS_SUBSET_SHIFT)
 #define VCS_VCPUID_SUBSET_BIT(v)     (CONST64U(1) << ((v) & VCS_SUBSET_MASK))
@@ -55,4 +55,16 @@ typedef struct VCPUSet {
    uint64 subset[VCS_SUBSET_COUNT];
 } VCPUSet;
 
+MY_ASSERTS(VCPUSET_ASSERTS,
+           /*
+            * Catch changes in VCPUSet which need to be reflected in
+            * bora/public/iocontrolsMacosTable.h.
+            */
+           ASSERT_ON_COMPILE(VCS_SUBSET_COUNT == 4);
+           /*
+            * There is code that depends on sizeof(VCPUSet) being a power of
+            * 2 in at least vcpuHotPlug.c and possible other places.
+            */
+            ASSERT_ON_COMPILE((sizeof(VCPUSet) & (sizeof(VCPUSet) - 1)) == 0);
+)
 #endif

vmmon-only/include/versioned_atomic.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2015-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -17,28 +17,62 @@
  *********************************************************/
 
 /*
- *  Versioned atomic synchronization:
+ * versioned_atomic.h --
  *
- *    These synchronization macros allow single-writer/many-reader
- *    access to data, based on Leslie Lamport's paper "Concurrent
- *    Reading and Writing", Communications of the ACM, November 1977.
+ *      This file implements versioned atomic synchronization, which allows
+ *      concurrent accesses to shared data by a single writer and multiple
+ *      readers. The algorithm is lock-free (it does not use a mutual exclusion
+ *      mechanism), and the writer is wait-free (it never waits for readers).
  *
- *    many-writer/many-reader can be implemented on top of versioned
- *    atomics by using an additional spin lock to synchronize
- *    writers. This is preferable for cases where readers are expected to
- *    greatly outnumber writers.
+ *      Many-writer/many-reader can be implemented on top of versioned atomics
+ *      by adding an external mutual exclusion mechanism to serialize all
+ *      writer threads into a single logical writer thread. This is preferable
+ *      for cases where readers are expected to greatly outnumber writers.
  *
- *    Multiple concurrent writers to the version variables are not
- *    allowed.  Even if writers are working on lock-free or disjoint
- *    data, the version counters are not interlocked for read-modify-write.
+ *      Implementation
+ *      --------------
+ *      This implementation is based on Leslie Lamport's paper "Concurrent
+ *      Reading and Writing", Communications of the ACM, November 1977
+ *      (http://url.eng.vmware.com/udcc ).
  *
- *    Recursive use of versioned atomics in writers is currently not supported.
+ *      Lamport's algorithm was designed for systems in which the reader and
+ *      the writers run on separate computers and where the version variables
+ *      (v0 and v1 in this implementation) are stored on a shared hard disk. So
+ *      it did not assume that the version variables could be accessed
+ *      atomically. Instead, it assumed that the version variables were made of
+ *      multiple basic units of data, called "digits", and that each digit
+ *      could be accessed atomically. The upper-left corner of page 4 of the
+ *      paper precisely and concisely describes the algorithm, and in
+ *      particular the order in which digits of the version variables must be
+ *      accessed (left to right, right to left, or no particular order).
+ *
+ *      This implementation is designed for systems in which the reader and the
+ *      writers run on the same computer and where the version variables are
+ *      stored in shared memory. So it assumes that the version variables can
+ *      be accessed atomically. As a result, this implementation uses a
+ *      simplified version of Lamport's algorithm, where:
+ *      o Each left to right or right to left access in Lamport's algorithm is
+ *        replaced with a single atomic access in this implementation.
+ *      o Each access in no particular order in Lamport's algorithm is replaced
+ *        with a single non-atomic access in this implementation.
+ *      Note: The writer is the only thread which writes to version variables,
+ *            so it is allowed to read them non-atomically.
+ *
+ *      Multiple concurrent writers to the version variables are not allowed.
+ *      Even if writers are working on lock-free or disjoint data, the version
+ *      variables are not interlocked for read-modify-write. See PR514764.
+ *
+ *      Recursive use of versioned atomics in writers is currently not
+ *      supported. See PR514764.
  */
 
 #ifndef _VERSIONED_ATOMIC_H
 #define _VERSIONED_ATOMIC_H
- 
+
+#define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_DISTRIBUTE
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMCORE
 #define INCLUDE_ALLOW_VMMON
@@ -46,6 +80,12 @@
 
 #include "vm_basic_asm.h"
 #include "vm_assert.h"
+#include "vm_atomic.h"
+
+#if defined __cplusplus
+extern "C" {
+#endif
+
 
 /*
  * Users with attribute(packed) structs must ensure any
@@ -56,115 +96,193 @@
  */
 
 typedef struct {
-   volatile uint32 v0;
-   volatile uint32 v1;
+   uint32 v0;
+   uint32 v1;
 } ALIGNED(4) VersionedAtomic;
 
+
 /*
  *-----------------------------------------------------------------------------
  *
  * VersionedAtomic_BeginWrite --
- *    Called by a writer to indicate that the data protected by
- *    a given atomic version is about to change. Effectively locks out
- *    all readers until EndWrite is called.
- * 
+ *
+ *      Called by the writer to indicate that it is about to write to the
+ *      shared data protected by 'versions'. Effectively locks out all readers
+ *      until VersionedAtomic_EndWrite() is called.
+ *
  * Results:
- *      .
+ *      None
  *
  * Side effects:
- *
+ *      None
  *
  *-----------------------------------------------------------------------------
  */
+
 static INLINE void
-VersionedAtomic_BeginWrite(VersionedAtomic *versions)
+VersionedAtomic_BeginWrite(VersionedAtomic *versions) // IN
 {
-   /*
-    * As long as the variable is on natural boundary it is guaranteed
-    * to be atomic, so we don't need to write from left to right as in
-    * Lamport's original algorithm.
-    */
-   ASSERT(((size_t)(&versions->v0) & (sizeof(versions->v0) - 1)) == 0);
-   /* If recursive use is needed demand for support in PR514764 */
+   /* Catch improper initialization or unsupported recursive use. */
    ASSERT(versions->v1 == versions->v0);
-   versions->v0++;
-   COMPILER_MEM_BARRIER();
+
+   /*
+    * Do not use Atomic_Inc32() below: it is more expensive than
+    * Atomic_Write32() and unnecessary: only the write needs to be atomic, not
+    * the whole read + inc + write.
+    */
+#if !defined VMM
+   /*
+    * Atomic_Write32() below requires the address to be naturally aligned, but
+    * currently only checks that condition when VMM is defined (tracked by
+    * PR1055836). So until that PR is fixed, we must check that condition here
+    * when VMM is not defined.
+    */
+   ASSERT(((uintptr_t)&versions->v0 % sizeof versions->v0) == 0);
+#endif
+   Atomic_Write32((Atomic_uint32 *)&versions->v0, versions->v0 + 1);
+
+   /*
+    * The write to 'versions->v0' must be observed by all other OS CPUs before
+    * the write(s) to the shared data.
+    *
+    * On arm64, this barrier is Full System, which is probably too strong:
+    * Inner Shareable (== all OS CPUs) should work just fine.
+    */
+   SMP_W_BARRIER_W();
 }
 
+
 /*
  *-----------------------------------------------------------------------------
  *
  * VersionedAtomic_EndWrite --
- *    Called by a writer after it is done updating shared data. Lets
- *    pending and new readers proceed on shared data.
- * 
+ *
+ *      Called by the writer to indicate that it has finished writing to the
+ *      shared data protected by 'versions'. Lets pending and new readers
+ *      proceed.
+ *
  * Results:
- *      .
+ *      None
  *
  * Side effects:
- *
+ *      None
  *
  *-----------------------------------------------------------------------------
  */
+
 static INLINE void
-VersionedAtomic_EndWrite(VersionedAtomic *versions)
+VersionedAtomic_EndWrite(VersionedAtomic *versions) // IN
 {
-   ASSERT(((size_t)(&versions->v1) & (sizeof(versions->v1) - 1)) == 0);
+   /*
+    * The write(s) to the shared data must be observed by all other OS CPUs
+    * before the write to 'versions->v1'.
+    *
+    * On arm64, this barrier is Full System, which is probably too strong:
+    * Inner Shareable (== all OS CPUs) should work just fine.
+    */
+   SMP_W_BARRIER_W();
+
    ASSERT(versions->v1 + 1 == versions->v0);
-   COMPILER_MEM_BARRIER();
-   versions->v1 = versions->v0;
+
+#if !defined VMM
+   /*
+    * Atomic_Write32() below requires the address to be naturally aligned, but
+    * currently only checks that condition when VMM is defined (tracked by
+    * PR1055836). So until that PR is fixed, we must check that condition here
+    * when VMM is not defined.
+    */
+   ASSERT(((uintptr_t)&versions->v1 % sizeof versions->v1) == 0);
+#endif
+   Atomic_Write32((Atomic_uint32 *)&versions->v1, versions->v0);
 }
 
+
 /*
  *-----------------------------------------------------------------------------
  *
  * VersionedAtomic_BeginTryRead --
- *    Called by a reader before it tried to read shared data.
- * 
+ *
+ *      Called by a reader before it starts reading the shared data protected
+ *      by 'versions'.
+ *
  * Results:
- *    Returns a version number to the reader. This version number
- *    is required to confirm validity of the read operation when reader
- *    calls EndTryRead.
+ *      A version number for the caller to pass to the matching call to
+ *      VersionedAtomic_EndTryRead(), if any.
  *
  * Side effects:
- *
+ *      None
  *
  *-----------------------------------------------------------------------------
  */
-static INLINE uint32
-VersionedAtomic_BeginTryRead(const VersionedAtomic *versions)
-{
-   uint32 readVersion;
 
-   readVersion = versions->v1;
-   COMPILER_MEM_BARRIER();
+static INLINE uint32
+VersionedAtomic_BeginTryRead(VersionedAtomic const *versions) // IN
+{
+   uint32 readVersion = Atomic_Read32((Atomic_uint32 *)&versions->v1);
+
+   /*
+    * The read from 'versions->v1' must happen on this CPU before the read(s)
+    * from the shared data.
+    *
+    * On arm64, this barrier is Full System, which is probably too strong:
+    * Inner Shareable (== all OS CPUs) should work just fine, and maybe
+    * Non-shareable (== current CPU) would work too.
+    */
+   SMP_R_BARRIER_R();
 
    return readVersion;
 }
 
+
 /*
  *-----------------------------------------------------------------------------
  *
  * VersionedAtomic_EndTryRead --
- *    Called by a reader after it finishes reading shared data, to confirm
- *    validity of the data that was just read (IOW, to make sure that a
- *    writer did not intervene while the read was in progress).
- * 
+ *
+ *      Called by a reader after it has finished reading the shared data
+ *      protected by 'versions', to confirm that the data is consistent, i.e.
+ *      that the writer has not modified the data while the reader was reading
+ *      it.
+ *
  * Results:
- *    TRUE if the data read between BeginTryRead() and this call is
- *    valid. FALSE otherwise.
+ *      TRUE if the shared data read between VersionedAtomic_BeginTryRead() and
+ *      this call is consistent.
+ *      FALSE otherwise.
  *
  * Side effects:
- *
+ *      None
  *
  *-----------------------------------------------------------------------------
  */
+
 static INLINE Bool
-VersionedAtomic_EndTryRead(const VersionedAtomic *versions,
-                           uint32 readVersion)
+VersionedAtomic_EndTryRead(VersionedAtomic const *versions, // IN
+                           uint32 readVersion)              // IN
 {
-   COMPILER_MEM_BARRIER();
-   return LIKELY(versions->v0 == readVersion);
+   /*
+    * The read(s) from the shared data must happen on this CPU before the read
+    * from 'versions->v0'.
+    *
+    * On arm64, this barrier is Full System, which is probably too strong:
+    * Inner Shareable (== all OS CPUs) should work just fine, and maybe
+    * Non-shareable (== current CPU) would work too.
+    */
+   SMP_R_BARRIER_R();
+
+   /*
+    * There is a theoretical ABA issue here: if the writer updates the shared
+    * data exactly 2^32 times while a reader reads it, the read will
+    * incorrectly be considered consistent. In practice, this issue is
+    * unlikely, so we ignore it. But should we need it, we could make the issue
+    * even less likely by using 64-bit version variables.
+    */
+   return LIKELY(   Atomic_Read32((Atomic_uint32 *)&versions->v0)
+                 == readVersion);
 }
 
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif //_VERSIONED_ATOMIC_H

vmmon-only/include/vm_asm_x86.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2014,2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -435,16 +435,13 @@ CLTS(void)
      __asm__ __volatile__("fs; invlpg %0": :"m" (*(char *) (_addr)):"memory"); \
 } while (0)
 
-#if ! defined(VMKBOOT)
 #define RESTORE_FLAGS _Set_flags
 #define ENABLE_INTERRUPTS() __asm__ __volatile__ ("sti": : :"memory")
 #define CLEAR_INTERRUPTS()  __asm__ __volatile__ ("cli": : :"memory")
-#endif
-
 #define RAISE_INTERRUPT(_x)  __asm__ __volatile__("int %0" :: "g" (_x))
 #define RETURN_FROM_INT()   __asm__ __volatile__("iret" :: )
 
-#if ! defined(VMKERNEL) && ! defined(VMKBOOT)
+#if ! defined(VMKERNEL)
 #define NO_INTERRUPTS_BEGIN()	do { \
                                    uintptr_t _flags; \
                                    SAVE_FLAGS(_flags); \
@@ -958,10 +955,12 @@ static INLINE uint64 GET_XCR0(void)
 
 static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal)
 {
+#ifndef VMM_BOOTSTRAP
    ASSERT(oldVal == GET_XCR0());
    if (newVal != oldVal) {
       SET_XCR0(newVal);
    }
+#endif
 }
 #endif
 

vmmon-only/include/vm_asm_x86_64.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -32,6 +32,7 @@
 #define INCLUDE_ALLOW_VMMON
 #define INCLUDE_ALLOW_VMK_MODULE
 #define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_DISTRIBUTE
 #include "includeCheck.h"
 

vmmon-only/include/vm_assert.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -67,6 +67,7 @@ extern "C" {
 #if defined (VMKPANIC) 
 #include "vmk_assert.h"
 #else /* !VMKPANIC */
+#include <linux/kernel.h>
 #define _ASSERT_PANIC(name) \
            Panic(_##name##Fmt "\n", __FILE__, __LINE__)
 #define _ASSERT_PANIC_BUG(bug, name) \
@@ -99,26 +100,15 @@ void Warning(const char *fmt, ...) PRINTF_DECL(1, 2);
 #if defined VMKPANIC
 void Panic_SaveRegs(void);
 
-#ifdef VMX86_DEBUG
-void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
-#else
 NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
-#endif
-
-NORETURN void Panic_NoSaveNoReturn(const char *fmt, ...) PRINTF_DECL(1, 2);
 
 #define Panic(fmt...) do { \
    Panic_SaveRegs();       \
    Panic_NoSave(fmt);      \
 } while(0)
 
-#define Panic_NoReturn(fmt...) do { \
-   Panic_SaveRegs();                \
-   Panic_NoSaveNoReturn(fmt);       \
-} while(0)
-
 #else
-NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+#define Panic panic
 #endif
 
 void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
@@ -176,8 +166,6 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
 
 #define ASSERT_NOT_IMPLEMENTED(cond) \
            ASSERT_IFNOT(cond, NOT_IMPLEMENTED())
-#define ASSERT_NOT_IMPLEMENTED_BUG(bug, cond) \
-           ASSERT_IFNOT(cond, NOT_IMPLEMENTED_BUG(bug))
 
 #if defined VMKPANIC || defined VMM
 #define NOT_IMPLEMENTED()        _ASSERT_PANIC_NORETURN(AssertNotImplemented)
@@ -202,10 +190,8 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
            ASSERT_IFNOT(cond, _ASSERT_PANIC(AssertMemAlloc))
 
 #ifdef VMX86_DEVEL
-#define ASSERT_DEVEL(cond) ASSERT(cond)
 #define NOT_TESTED()       Warning(_AssertNotTestedFmt "\n", __FILE__, __LINE__)
 #else
-#define ASSERT_DEVEL(cond) ((void)0)
 #define NOT_TESTED()       Log(_AssertNotTestedFmt "\n", __FILE__, __LINE__)
 #endif
 
@@ -305,7 +291,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
 #define ASSERT_ON_COMPILE(e) \
    do {                      \
       _Static_assert(e, #e); \
-   } while (0);
+   } while (0)
 #endif
 
 /*

vmmon-only/include/vm_basic_asm.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -43,6 +43,10 @@
 #elif defined VM_X86_32
 #include "vm_basic_asm_x86_common.h"
 #include "vm_basic_asm_x86.h"
+#elif defined VM_ARM_32
+#include "vm_basic_asm_arm32.h"
+#define MUL64_NO_ASM 1
+#include "mul64.h"
 #elif defined VM_ARM_64
 #include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
@@ -51,6 +55,11 @@
 #include "mul64.h"
 #endif
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 /*
  * Locate most and least significant bit set functions. Use our own name
  * space to avoid namespace collisions. The new names follow a pattern,
@@ -623,6 +632,11 @@ uint16set(void *dst, uint16 val, size_t count)
 {
 #ifdef VM_X86_64
    __stosw((uint16*)dst, val, count);
+#elif defined(VM_ARM_32)
+   size_t i;
+   for (i = 0; i < count; i++) {
+      ((uint16 *)dst)[i] = val;
+   }
 #else
    __asm { pushf;
            mov ax, val;
@@ -641,6 +655,11 @@ uint32set(void *dst, uint32 val, size_t count)
 {
 #ifdef VM_X86_64
    __stosd((unsigned long*)dst, (unsigned long)val, count);
+#elif defined(VM_ARM_32)
+   size_t i;
+   for (i = 0; i < count; i++) {
+      ((uint32 *)dst)[i] = val;
+   }
 #else
    __asm { pushf;
            mov eax, val;
@@ -673,7 +692,7 @@ static INLINE uint16
 Bswap16(uint16 v)
 {
 #if defined(VM_ARM_64)
-   __asm__("rev16 %0, %0" : "+r"(v));
+   __asm__("rev16 %w0, %w0" : "+r"(v));
    return v;
 #else
    return ((v >> 8) & 0x00ff) | ((v << 8) & 0xff00);
@@ -702,12 +721,12 @@ Bswap32(uint32 v) // IN
       : "0" (v)
    );
    return v;
-#elif defined(VM_ARM_32) && !defined(__ANDROID__)
+#elif defined(VM_ARM_32) && !defined(__ANDROID__) && !defined(_MSC_VER)
     __asm__("rev %0, %0" : "+r"(v));
     return v;
 #elif defined(VM_ARM_64)
-   __asm__("rev32 %0, %0" : "+r"(v));
-    return v;
+   __asm__("rev32 %x0, %x0" : "+r"(v));
+   return v;
 #else
    return    (v >> 24)
           | ((v >>  8) & 0xFF00)
@@ -740,63 +759,6 @@ Bswap64(uint64 v) // IN
 }
 
 
-/*
- * COMPILER_MEM_BARRIER prevents the compiler from re-ordering memory
- * references accross the barrier.  NOTE: It does not generate any
- * instruction, so the CPU is free to do whatever it wants to...
- */
-#ifdef __GNUC__
-#define COMPILER_MEM_BARRIER()   __asm__ __volatile__ ("": : :"memory")
-#define COMPILER_READ_BARRIER()  COMPILER_MEM_BARRIER()
-#define COMPILER_WRITE_BARRIER() COMPILER_MEM_BARRIER()
-#elif defined(_MSC_VER)
-#define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
-#define COMPILER_READ_BARRIER()  _ReadBarrier()
-#define COMPILER_WRITE_BARRIER() _WriteBarrier()
-#endif
-
-
-/*
- *----------------------------------------------------------------------
- *
- * COMPILER_FORCED_LOAD_AND_MEM_BARRIER --
- *
- *        This macro prevents the compiler from re-ordering memory references
- *        across the barrier. In addition it emits a forced load from the given
- *        memory reference. The memory reference has to be either 1, 2, 4 or 8
- *        bytes wide.
- *        The forced load of a memory reference can be used exploit details of a
- *        given CPUs memory model. For example x86 CPUs won't reorder stores to
- *        a memory location x with loads from a memory location x.
- *        NOTE: It does not generate any fencing instruction, so the CPU is free
- *              to reorder instructions according to its memory model.
- *
- * Results:
- *        None
- *
- * Side Effects:
- *        None.
- *
- *----------------------------------------------------------------------
- */
-
-#ifdef VM_X86_64
-#ifdef __GNUC__
-
-#define COMPILER_FORCED_LOAD_AND_MEM_BARRIER(_memory_reference)               \
-   do {                                                                       \
-      typeof(_memory_reference) _dummy;                                       \
-                                                                              \
-      asm volatile("mov %1, %0\n\t"                                           \
-                   : "=r" (_dummy) /* Let compiler choose reg for _dummy */   \
-                   : "m" (_memory_reference)                                  \
-                   : "memory");                                               \
-   } while(0)
-
-#endif /* __GNUC__ */
-#endif /* VM_X86_64 */
-
-
 /*
  * PAUSE is a P4 instruction that improves spinlock power+performance;
  * on non-P4 IA32 systems, the encoding is interpreted as a REPZ-NOP.
@@ -804,7 +766,7 @@ Bswap64(uint64 v) // IN
  */
 static INLINE void
 PAUSE(void)
-#ifdef __GNUC__
+#if defined(__GNUC__) || defined(VM_ARM_32)
 {
 #ifdef VM_ARM_ANY
    /*
@@ -878,6 +840,15 @@ RDTSC(void)
 {
    return __rdtsc();
 }
+#elif defined(VM_ARM_32)
+{
+   /*
+    * We need to do more inverstagetion here to find
+    * a microsoft equivalent of that code
+    */
+   NOT_IMPLEMENTED();
+   return 0;
+}
 #else
 #pragma warning( disable : 4035)
 {
@@ -1254,5 +1225,10 @@ RoundUpPow2_32(uint32 value)
 #endif
 }
 
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif // _VM_BASIC_ASM_H_
 

vmmon-only/include/vm_basic_asm_x86.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,11 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 #ifdef VM_X86_64
 /*
  * The gcc inline asm uses the "A" constraint which differs in 32 & 64
@@ -545,4 +550,9 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
 #pragma warning(default: 4035)
 #endif
 
+
+#if defined __cplusplus
+} // extern "C"
 #endif
+
+#endif // _VM_BASIC_ASM_X86_H_

vmmon-only/include/vm_basic_asm_x86_64.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -59,24 +59,23 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
 
 #if defined(__GNUC__)
 /*
- * GET_CURRENT_RIP
+ * GET_CURRENT_PC
  *
- * Returns the current instruction pointer. In the example below:
+ * Returns the current program counter (i.e. instruction pointer i.e. rip
+ * register on x86_64). In the example below:
  *
  *   foo.c
- *   L123: Foo(GET_CURRENT_RIP())
+ *   L123: Foo(GET_CURRENT_PC())
  *
- * the return value from GET_CURRENT_RIP will point a debugger to L123.
+ * the return value from GET_CURRENT_PC will point a debugger to L123.
  */
-#define GET_CURRENT_RIP() ({                                          \
+#define GET_CURRENT_PC() ({                                           \
       void *__rip;                                                    \
       asm("lea 0(%%rip), %0;\n\t"                                     \
          : "=r" (__rip));                                             \
       __rip;                                                          \
 })
 
-#define GET_CURRENT_PC() GET_CURRENT_RIP()
-
 /*
  * GET_CURRENT_LOCATION
  *

vmmon-only/include/vm_basic_asm_x86_common.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 2013-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -89,6 +89,9 @@ void _mm_mfence(void);
 void _mm_lfence(void);
 #pragma intrinsic(_mm_mfence, _mm_lfence)
 
+long _InterlockedXor(long volatile *, long);
+#pragma intrinsic(_InterlockedXor)
+
 unsigned int __getcallerseflags(void);
 #pragma intrinsic(__getcallerseflags)
 
@@ -294,20 +297,150 @@ RDTSC_BARRIER(void)
 
 
 /*
- * Compiler/CPU barriers. These take the form of <mem access type>_<mem access
- * type>_MEM_BARRIER, where <mem access type> is either LD (load), ST (store) or
- * LDST (any). On x86 we only need to care specifically about store-load
- * reordering on normal memory types and mfence, otherwise only a compiler
- * barrier is needed.
+ * Memory Barriers
+ * ===============
+ *
+ *    Terminology
+ *    -----------
+ *
+ * A compiler memory barrier prevents the compiler from re-ordering memory
+ * accesses accross the barrier. It is not a CPU instruction, it is a compiler
+ * directive (i.e. it does not emit any code).
+ *
+ * A CPU memory barrier prevents the CPU from re-ordering memory accesses
+ * accross the barrier. It is a CPU instruction.
+ *
+ * A memory barrier is the union of a compiler memory barrier and a CPU memory
+ * barrier. A compiler memory barrier is a useless construct by itself. It is
+ * only useful when combined with a CPU memory barrier, to implement a memory
+ * barrier.
+ *
+ *    Semantics
+ *    ---------
+ *
+ * At the time COMPILER_*_BARRIER were created (and references to them were
+ * added to the code), the code was only targetting x86. The intent of the code
+ * was really to use a memory barrier, but because x86 uses a strongly ordered
+ * memory model, the CPU would not re-order memory accesses, and the code could
+ * get away with using just a compiler memory barrier. So COMPILER_*_BARRIER
+ * were born and were implemented as compiler memory barriers _on x86_. But
+ * make no mistake, _the semantics that the code expects from
+ * COMPILER_*_BARRIER are that of a memory barrier_!
+ *
+ *    DO NOT USE!
+ *    -----------
+ *
+ * On at least one non-x86 architecture, COMPILER_*_BARRIER are
+ * 1) Misnomers
+ * 2) Not fine-grained enough to provide the best performance.
+ * For the above two reasons, usage of COMPILER_*_BARRIER is now deprecated.
+ * _Do not add new references to COMPILER_*_BARRIER._ Instead, precisely
+ * document the intent of your code by using
+ * <mem_type/purpose>_<before_access_type>_BARRIER_<after_access_type>.
+ * Existing references to COMPILER_*_BARRIER are being slowly but surely
+ * converted, and when no references are left, COMPILER_*_BARRIER will be
+ * retired.
+ *
+ * Thanks for pasting this whole comment into every architecture header.
  */
-#define LD_LD_MEM_BARRIER()      COMPILER_MEM_BARRIER()
-#define LD_ST_MEM_BARRIER()      COMPILER_MEM_BARRIER()
-#define LD_LDST_MEM_BARRIER()    COMPILER_MEM_BARRIER()
-#define ST_LD_MEM_BARRIER()      asm volatile ("mfence" ::: "memory")
-#define ST_ST_MEM_BARRIER()      COMPILER_MEM_BARRIER()
-#define ST_LDST_MEM_BARRIER()    ST_LD_MEM_BARRIER()
-#define LDST_LD_MEM_BARRIER()    ST_LD_MEM_BARRIER()
-#define LDST_ST_MEM_BARRIER()    COMPILER_MEM_BARRIER()
-#define LDST_LDST_MEM_BARRIER()  ST_LD_MEM_BARRIER()
+
+#if defined __GNUC__
+#   define COMPILER_READ_BARRIER()  COMPILER_MEM_BARRIER()
+#   define COMPILER_WRITE_BARRIER() COMPILER_MEM_BARRIER()
+#   define COMPILER_MEM_BARRIER()   __asm__ __volatile__("" ::: "memory")
+#elif defined _MSC_VER
+#   define COMPILER_READ_BARRIER()  _ReadBarrier()
+#   define COMPILER_WRITE_BARRIER() _WriteBarrier()
+#   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
+#endif
+
+
+/*
+ * Memory barriers. These take the form of
+ *
+ * <mem_type/purpose>_<before_access_type>_BARRIER_<after_access_type>
+ *
+ * where:
+ *   <mem_type/purpose> is either SMP, DMA, or MMIO.
+ *   <*_access type> is either R(load), W(store) or RW(any).
+ *
+ * Above every use of these memory barriers in the code, there _must_ be a
+ * comment to justify the use, i.e. a comment which:
+ *
+ * 1) Precisely identifies which memory accesses must not be re-ordered across
+ *    the memory barrier.
+ * 2) Explains why it is important that the memory accesses not be re-ordered.
+ *
+ * Thanks for pasting this whole comment into every architecture header.
+ *
+ * On x86, we only need to care specifically about store-load reordering on
+ * normal memory types. In other cases, only a compiler barrier is needed. The
+ * ST_LD barrier is implemented with a locked xor operation (instead of the
+ * mfence instruction) for performance reasons. See PR 1674199 for more
+ * details.
+ *
+ * On x64, special instructions are only provided for load-load (lfence) and
+ * store-store (sfence) ordering, and they don't apply to normal memory.
+ */
+
+
+static INLINE void
+SMP_W_BARRIER_R(void)
+{
+   volatile long temp;
+
+   COMPILER_MEM_BARRIER();
+#if defined __GNUC__
+   __asm__ __volatile__ (
+      "lock xorl $1, %0\n"
+      : "+m" (temp)
+      : /* no additional inputs */
+      : "cc");
+#elif defined _MSC_VER
+   _InterlockedXor(&temp, 1);
+#else
+#error SMP_W_BARRIER_R not defined for this compiler
+#endif
+   COMPILER_MEM_BARRIER();
+}
+
+#define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
+#define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
+#define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
+#define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
+#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
+#define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
+#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()
+
+/*
+ * Like the above, only for use with observers other than CPUs,
+ * i.e. DMA masters.
+ */
+
+#define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
+#define DMA_R_BARRIER_W()     SMP_R_BARRIER_W()
+#define DMA_R_BARRIER_RW()    SMP_R_BARRIER_RW()
+#define DMA_W_BARRIER_R()     SMP_W_BARRIER_R()
+#define DMA_W_BARRIER_W()     SMP_W_BARRIER_W()
+#define DMA_W_BARRIER_RW()    SMP_W_BARRIER_RW()
+#define DMA_RW_BARRIER_R()    SMP_RW_BARRIER_R()
+#define DMA_RW_BARRIER_W()    SMP_RW_BARRIER_W()
+#define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()
+
+/*
+ * And finally a set for use with MMIO accesses.
+ */
+
+#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
+#define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
+#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
+#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
+#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
+#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
+#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
+#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
+#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()
+
 
 #endif // _VM_BASIC_ASM_X86_COMMON_H_

vmmon-only/include/vm_basic_defs.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -251,12 +251,16 @@ Max(int a, int b)
 #define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
 #endif
 
+#ifndef MBYTES_SHIFT
+#define MBYTES_SHIFT 20
+#endif
+
 #ifndef MBYTES_2_PAGES
-#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (20 - PAGE_SHIFT))
+#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
 #endif
 
 #ifndef PAGES_2_MBYTES
-#define PAGES_2_MBYTES(_npages) ((_npages) >> (20 - PAGE_SHIFT))
+#define PAGES_2_MBYTES(_npages) ((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
 #endif
 
 #ifndef GBYTES_2_PAGES
@@ -268,11 +272,11 @@ Max(int a, int b)
 #endif
 
 #ifndef BYTES_2_MBYTES
-#define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> 20)
+#define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
 #endif
 
 #ifndef MBYTES_2_BYTES
-#define MBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << 20)
+#define MBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << MBYTES_SHIFT)
 #endif
 
 #ifndef BYTES_2_GBYTES
@@ -299,6 +303,26 @@ Max(int a, int b)
 #define VM_PAE_LARGE_2_SMALL_PAGES (BYTES_2_PAGES(VM_PAE_LARGE_PAGE_SIZE))
 #endif
 
+#ifndef VM_PAE_LARGE_2_BYTES
+#define VM_PAE_LARGE_2_BYTES(_2mbytes) ((_2mbytes) << VM_PAE_LARGE_PAGE_SHIFT)
+#endif
+
+#ifndef VM_1GB_PAGE_SHIFT
+#define VM_1GB_PAGE_SHIFT 30
+#endif
+
+#ifndef VM_1GB_PAGE_SIZE
+#define VM_1GB_PAGE_SIZE (1 << VM_1GB_PAGE_SHIFT)
+#endif
+
+#ifndef VM_1GB_2_PAGES
+#define VM_1GB_2_PAGES (BYTES_2_PAGES(VM_1GB_PAGE_SIZE))
+#endif
+
+#ifndef VM_1GB_2_PDIRS
+#define VM_1GB_2_PDIRS (VM_1GB_PAGE_SIZE / VM_PAE_LARGE_PAGE_SIZE)
+#endif
+
 /*
  * Word operations
  */
@@ -394,9 +418,7 @@ void *_ReturnAddress(void);
  * guarantee.  Bummer.  --Jeremy.
  */
 
-#if defined(N_PLAT_NLM)
-/* We do not have YIELD() as we do not need it yet... */
-#elif defined(_WIN32)
+#if defined(_WIN32)
 #      define YIELD()		Sleep(0)
 #elif defined(VMKERNEL)
 /* We don't have a YIELD macro in the vmkernel */
@@ -412,7 +434,11 @@ void *_ReturnAddress(void);
 
 #ifdef _WIN32 // {
 
+/* Conflict with definition of Visual Studio 2015 */
+#if (_MSC_VER < 1900)
 #define snprintf  _snprintf
+#endif
+
 #define strtok_r  strtok_s
 
 #if (_MSC_VER < 1500)
@@ -652,14 +678,24 @@ typedef int pid_t;
 #define APPLE_ONLY(x) x
 #else
 #define vmx86_apple 0
-#define APPLE_ONLY(x) 
+#define APPLE_ONLY(x)
+#endif
+
+#if defined(__APPLE__) && defined(VMW_APPLE_SANDBOX)
+#define vmw_apple_sandbox 1
+#else
+#define vmw_apple_sandbox 0
 #endif
 
 #ifdef VMM
 #define VMM_ONLY(x) x
-#define USER_ONLY(x)
 #else
 #define VMM_ONLY(x)
+#endif
+
+#if defined(VMM) || defined(VMKERNEL)
+#define USER_ONLY(x)
+#else
 #define USER_ONLY(x) x
 #endif
 
@@ -781,6 +817,22 @@ typedef int pid_t;
 #define SIZE_80BIT  10
 #define SIZE_128BIT 16
 #define SIZE_256BIT 32
+#define SIZE_512BIT 64
 
+/*
+ * Allocate a variable of type _type, aligned to _align bytes, returning a
+ * pointer to the variable in _var.  Potentially _align - 1 bytes may be
+ * wasted.  On x86, GCC 6.3.0 behaves sub-optimally when variables are declared
+ * on the stack using the aligned attribute, so this pattern is preferred.
+ * See PRs 1795155, 1819963.
+ */
+#define WITH_PTR_TO_ALIGNED_VAR(_type, _align, _var)                     \
+   do {                                                                  \
+      uint8 _buf_##_var[sizeof(_type) + (_align) - 1];                   \
+      _type *_var = (_type *) ((uintptr_t)(_buf_##_var + (_align) - 1) & \
+                               ~((uintptr_t) ((_align) - 1)));
+
+#define END_PTR_TO_ALIGNED_VAR \
+   } while (0)
 
 #endif // ifndef _VM_BASIC_DEFS_H_

vmmon-only/include/vm_basic_math.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2008-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2008-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -38,6 +38,10 @@
 #include "vm_basic_types.h" // For INLINE.
 #include "vm_basic_asm.h"   // For Div64...
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
 
 static INLINE uint32
 RatioOf(uint32 numer1, uint32 numer2, uint32 denom)
@@ -62,13 +66,36 @@ ExponentialAvg(uint32 avg, uint32 value, uint32 gainNumer, uint32 gainDenom)
    return (term1 + term2) / gainDenom;
 }
 
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * IsZeroOrPowerOfTwo --
+ * IsZeroOrPowerOfTwo64 --
+ *
+ * Results:
+ *      TRUE iff the value is 0 or a power of two.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
 static INLINE Bool
-IsPowerOfTwo(uint32 x)
+IsZeroOrPowerOfTwo64(uint64 x)
 {
-   /* Does not check for zero. Callers depend on this. */
    return !(x & (x - 1));
 }
 
+
+static INLINE Bool
+IsZeroOrPowerOfTwo(uint32 x)     // IN
+{
+   return !(x & (x - 1));
+}
+
+
 static INLINE uint32
 GetPowerOfTwo(uint32 x)
 {
@@ -163,4 +190,8 @@ RotateRight64(uint64 value, uint8 shift)
 #endif // if !defined(_WIN32) && !defined(_WIN64)
 
 
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif // ifndef _VM_BASIC_MATH_H_

vmmon-only/include/vm_basic_types.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -40,7 +40,8 @@
 
 /* STRICT ANSI means the Xserver build and X defines Bool differently. */
 #if !defined(_XTYPEDEF_BOOL) && \
-    (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || defined(__MINGW32__))
+    (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || \
+      defined(__MINGW32__) || defined(__APPLE__))
 #define _XTYPEDEF_BOOL
 typedef char           Bool;
 #endif
@@ -53,8 +54,7 @@ typedef char           Bool;
 #define TRUE           1
 #endif
 
-#define IsBool(x)      (((x) & ~1) == 0)
-#define IsBool2(x, y)  ((((x) | (y)) & ~1) == 0)
+#define IS_BOOL(x)     (((x) & ~1) == 0)
 
 /*
  * Macros __i386__ and __ia64 are intrinsically defined by GCC
@@ -312,6 +312,19 @@ typedef uint32    uintptr_t;
 #endif
 
 
+#if defined(__GNUC__) && defined(__SIZEOF_INT128__)
+
+typedef unsigned __int128 uint128;
+typedef          __int128  int128;
+
+#define MIN_INT128   ((int128)1 << 127)
+#define MAX_INT128   (~MIN_INT128)
+#define MIN_UINT128  ((uint128)0)
+#define MAX_UINT128  (~MIN_UINT128)
+
+#endif
+
+
 /*
  * Time
  * XXX These should be cleaned up.  -- edward
@@ -353,14 +366,7 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
    #define FMTH ""
 #elif __GNUC__
    #define FMTH ""
-   #if defined(N_PLAT_NLM) || defined(sun) || \
-       (defined(__FreeBSD__) && (__FreeBSD__ + 0) && ((__FreeBSD__ + 0) < 5))
-      /*
-       * Why (__FreeBSD__ + 0)?  See bug 141008.
-       * Yes, we really need to test both (__FreeBSD__ + 0) and
-       * ((__FreeBSD__ + 0) < 5).  No, we can't remove "+ 0" from
-       * ((__FreeBSD__ + 0) < 5).
-       */
+   #if defined(sun)
       #if defined(VM_X86_64) || defined(VM_ARM_64)
          #define FMTSZ  "l"
          #define FMTPD  "l"
@@ -368,7 +374,8 @@ typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */
          #define FMTSZ  ""
          #define FMTPD  ""
       #endif
-   #elif defined(__linux__) \
+   #elif defined(__linux__) || \
+        (defined(__FreeBSD__) && (__FreeBSD__ + 0))\
       || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) \
       || (defined(_POSIX_VERSION) && _POSIX_VERSION >= 200112L) \
       || (defined(_POSIX2_VERSION) && _POSIX2_VERSION >= 200112L)
@@ -543,6 +550,11 @@ typedef uint16  UReg16;
 typedef uint32  UReg32;
 typedef uint64  UReg64;
 
+#if defined(__GNUC__) && defined(__SIZEOF_INT128__)
+typedef  int128  Reg128;
+typedef uint128 UReg128;
+#endif
+
 #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
     defined (VMKERNEL) || defined (VMKBOOT)
 typedef  Reg64  Reg;
@@ -673,10 +685,17 @@ typedef void * UserVA;
 #endif
 #define CONST         const
 
-
 #ifndef INLINE
 #   ifdef _MSC_VER
-#      define INLINE        __inline
+       /*
+        * On UWP(Universal Windows Platform),
+        * Only X86 32bit support '__inline'
+        */
+#      if defined(VM_WIN_UWP) && !defined(VM_X86_32)
+#            define INLINE
+#      else
+#            define INLINE        __inline
+#      endif
 #   else
 #      define INLINE        inline
 #   endif
@@ -887,6 +906,8 @@ typedef void * UserVA;
 #ifndef UNUSED_PARAM
 # if defined(__GNUC__)
 #  define UNUSED_PARAM(_parm) _parm  __attribute__((__unused__))
+# elif defined _MSC_VER
+#  define UNUSED_PARAM(_parm) __pragma(warning(suppress:4100)) _parm
 # else
 #  define UNUSED_PARAM(_parm) _parm
 # endif
@@ -921,6 +942,21 @@ typedef void * UserVA;
 #define ALIGNED(n)
 #endif
 
+
+/*
+ * Encapsulate the syntactic differences between gcc and msvc alignment control.
+ * BOUNDARY must match in the prefix and suffix.
+ */
+
+#ifdef _WIN32
+#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
+#define ALIGN_SUFFIX(BOUNDRY)
+#else
+#define ALIGN_PREFIX(BOUNDRY)
+#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
+#endif
+
+
 /*
  * Once upon a time, this was used to silence compiler warnings that
  * get generated when the compiler thinks that a function returns
@@ -1102,16 +1138,6 @@ typedef void * UserVA;
 #endif
 
 
-/*
- * Define MXSemaHandle here so both vmmon and vmx see this definition.
- */
-
-#ifdef _WIN32
-typedef uintptr_t MXSemaHandle;
-#else
-typedef int MXSemaHandle;
-#endif
-
 /*
  * Define type for poll device handles.
  */

vmmon-only/include/vm_pagetable.h

@@ -0,0 +1,128 @@
+/*********************************************************
+ * Copyright (C) 2014,2016 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * vm_pagetable.h --
+ *
+ *      Virtual memory page table definitions
+ */
+
+#ifndef _VM_PAGETABLE_H_
+#define _VM_PAGETABLE_H_
+
+#define INCLUDE_ALLOW_MODULE
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_DISTRIBUTE
+#define INCLUDE_ALLOW_VMCORE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_USERLEVEL
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+
+/*
+ * These definitions suit both the x64 and arm64 architectures. In particular
+ * - At all levels, a page table is a 4KB page containing 512 8-byte entries.
+ * - Each entry maps 39, 30, 21 and 12 bits at the respective page table
+ *   levels (with standard page sizes).
+ */
+
+typedef uint64 PT_L4E;
+typedef uint64 PT_L3E;
+typedef uint64 PT_L2E;
+typedef uint64 PT_L1E;
+
+typedef enum {
+   PT_LEVEL_1 = 1,
+   PT_LEVEL_2,
+   PT_LEVEL_3,
+   PT_LEVEL_4,
+   PT_MAX_LEVELS = PT_LEVEL_4
+} PT_Level;
+
+#define PT_PTE_SIZE           8
+#define PT_LEVEL_SHIFT        9
+
+#define PT_PTE_PFN_SHIFT      PAGE_SHIFT
+#define PT_ENTRIES_PER_PT     (1 << PT_LEVEL_SHIFT)
+#define PT_OFF_MASK           (PT_ENTRIES_PER_PT - 1)
+
+/*
+ * log2 of the
+ * address space size (in 4KB pages) covered by a level-_l page table entry.
+ */
+#define PT_LE_PG_SHIFT(_l)    (PT_LEVEL_SHIFT * ((_l) - PT_LEVEL_1))
+
+/* Address space size (in 4KB pages) covered by a level-_l page table entry. */
+#define PT_PAGES_PER_LE(_l)   ((uint64)1 << PT_LE_PG_SHIFT(_l))
+#define PT_PAGES_PER_L4E      PT_PAGES_PER_LE(PT_LEVEL_4)
+#define PT_PAGES_PER_L3E      PT_PAGES_PER_LE(PT_LEVEL_3)
+#define PT_PAGES_PER_L2E      PT_PAGES_PER_LE(PT_LEVEL_2)
+#define PT_PAGES_PER_L1E      PT_PAGES_PER_LE(PT_LEVEL_1)
+
+/*
+ * log2 of the
+ * address space size (in bytes) covered by a level-_l page table entry.
+ */
+#define PT_LE_SHIFT(_l)       (PAGE_SHIFT + PT_LE_PG_SHIFT(_l))
+#define PT_L4E_SHIFT          PT_LE_SHIFT(PT_LEVEL_4)
+#define PT_L3E_SHIFT          PT_LE_SHIFT(PT_LEVEL_3)
+#define PT_L2E_SHIFT          PT_LE_SHIFT(PT_LEVEL_2)
+#define PT_L1E_SHIFT          PT_LE_SHIFT(PT_LEVEL_1)
+
+/* Address space size (in bytes) covered by a level-_l page table entry. */
+#define PT_LE_SIZE(_l)        ((uint64)1 << PT_LE_SHIFT(_l))
+#define PT_L4E_SIZE           PT_LE_SIZE(PT_LEVEL_4)
+#define PT_L3E_SIZE           PT_LE_SIZE(PT_LEVEL_3)
+#define PT_L2E_SIZE           PT_LE_SIZE(PT_LEVEL_2)
+#define PT_L1E_SIZE           PT_LE_SIZE(PT_LEVEL_1)
+
+/*
+ * For a given LA,
+ * determine the offset (== index) of the level-_l page table entry.
+ */
+#define PT_LA_2_LOFF(_a, _l)  (((_a) >> PT_LE_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LA_2_L4OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_4)
+#define PT_LA_2_L3OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_3)
+#define PT_LA_2_L2OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_2)
+#define PT_LA_2_L1OFF(_a)     PT_LA_2_LOFF(_a, PT_LEVEL_1)
+
+/*
+ * For a given LPN,
+ * determine the offset (== index) of the level-_l page table entry.
+ */
+#define PT_LPN_2_LOFF(_a, _l) (((_a) >> PT_LE_PG_SHIFT(_l)) & PT_OFF_MASK)
+#define PT_LPN_2_L4OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_4)
+#define PT_LPN_2_L3OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_3)
+#define PT_LPN_2_L2OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_2)
+#define PT_LPN_2_L1OFF(_a)    PT_LPN_2_LOFF(_a, PT_LEVEL_1)
+
+/* Size (in bytes) of a 1GB page. */
+#define PT_1G_PAGE_SIZE       PT_L3E_SIZE
+#define PT_1G_PAGE_MASK       (PT_L3E_SIZE - 1)
+
+/* Size (in bytes) of a 2MB page. */
+#define PT_2M_PAGE_SIZE       PT_L2E_SIZE
+#define PT_2M_PAGE_MASK       (PT_L2E_SIZE - 1)
+
+#define PT_LE_NXT_PG(_a, _l)  ((((_a) >> PT_LE_PG_SHIFT(_l)) + 1) \
+                               << PT_LE_PG_SHIFT(_l))
+
+#endif

vmmon-only/include/vmm_constants.h

@@ -42,7 +42,4 @@
 
 #define VMM_PANIC_MSG_SIZE     256
 
-#define VMM_CPL0_STACK 1
-#define VMM_CPL1_STACK 2
-
 #endif

vmmon-only/include/vmmem_shared.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -40,7 +40,7 @@
 #define VMMEM_FLAG_BIT(x) (1 << (x))
 
 #define VMMEM_ANON_LOW_MEM        VMMEM_FLAG_BIT(0)
-#define VMMEM_ANON_LARGE_PAGE     VMMEM_FLAG_BIT(1)
+#define VMMEM_ANON_CONTIG         VMMEM_FLAG_BIT(1)
 #define VMMEM_ANON_CAN_FAIL       VMMEM_FLAG_BIT(2)
 #define VMMEM_ANON_USE_PREALLOC   VMMEM_FLAG_BIT(3)
 #define VMMEM_ANON_IOABLE_PAGE    VMMEM_FLAG_BIT(4)
@@ -64,10 +64,14 @@
 #define VMMEM_PLATFORM_COW                VMMEM_FLAG_BIT(2)
 #define VMMEM_PLATFORM_EXPOSED_TO_VMM     VMMEM_FLAG_BIT(3)
 #define VMMEM_PLATFORM_P2M_UPDATE_PENDING VMMEM_FLAG_BIT(4)
-#define VMMEM_PLATFORM_TRY_COW_SUCCESS    VMMEM_FLAG_BIT(5)
-#define VMMEM_PLATFORM_DIRTY              VMMEM_FLAG_BIT(6)
-#define VMMEM_PLATFORM_BACKED_LARGE       VMMEM_FLAG_BIT(7)
+#define VMMEM_PLATFORM_DIRTY              VMMEM_FLAG_BIT(5)
+#define VMMEM_PLATFORM_IS_2M_PAGE         VMMEM_FLAG_BIT(6)
+#define VMMEM_PLATFORM_IS_1G_PAGE         VMMEM_FLAG_BIT(7)
 #define VMMEM_PLATFORM_LARGE_RETRY        VMMEM_FLAG_BIT(8)
+#define VMMEM_PLATFORM_TRY_COW_SUCCESS    VMMEM_FLAG_BIT(9)
+
+#define VMMEM_PLATFORM_BACKED_LARGE      (VMMEM_PLATFORM_IS_2M_PAGE |   \
+                                          VMMEM_PLATFORM_IS_1G_PAGE)
 
 
 #define MAX_PLATFORM_PAGE_INFO_PAGES  240

vmmon-only/include/vmware.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the

vmmon-only/include/vmware_pack_begin.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the

vmmon-only/include/vmware_pack_end.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the

vmmon-only/include/vmware_pack_init.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the

vmmon-only/include/x86_basic_defs.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006-2013 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -35,6 +35,8 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
 
+#include <asm/processor-flags.h>
+
 #define X86_MAX_INSTR_LEN  15   /* Max byte length of an x86 instruction. */
 
 #define NUM_IDT_VECTORS 256
@@ -54,7 +56,20 @@
 #define CR0_NW         0x20000000
 #define CR0_CD         0x40000000
 #define CR0_PG         0x80000000
+
+#define CR0_CACHE_CONTROL (CR0_CD | CR0_NW)
+
 #define CR0_RESERVED   CONST64U(0xffffffff1ffaffc0)
+/*
+ * Note: The "Not Reserved" bits in CR0 are:
+ *   PG, CD, NW, AM, WP, NE, ET, TS, EM, MP, PE
+ *        |   |   |               |   |   |
+ *        |   |   +---------------+---+---+---> CR0_MUTABLE
+ *        |   |
+ *        +---+--> CR0_CACHE_CONTROL
+ *
+ * (CR0_MUTABLE is defined in vmkernel/private/x86/cpu.h)
+ */
 
 #define CR3_PWT        0x00000008
 #define CR3_PCD        0x00000010
@@ -62,7 +77,9 @@
 #define CR3_PDB_MASK   0xfffff000
 #define CR3_IGNORE     0xFFF
 #define PAE_CR3_IGNORE 0x1F
+#ifndef CR3_PCID_MASK
 #define CR3_PCID_MASK  0xFFF
+#endif
 #define CR3_NO_FLUSH   (1ULL << 63)
 
 #define CR4_VME        0x00000001
@@ -83,8 +100,8 @@
 #define CR4_OSXSAVE    0x00040000
 #define CR4_SMEP       0x00100000
 #define CR4_SMAP       0x00200000
-/* Removing a bit from CR4_RESERVED causes Task_Switch to leave the bit set. */
-#define CR4_RESERVED   CONST64U(0xffffffffffc89800) 
+#define CR4_PKE        0x00400000
+#define CR4_RESERVED   CONST64U(0xffffffffff889800)
 #define CR8_RESERVED   CONST64U(0xfffffffffffffff0)
 
 /*
@@ -186,8 +203,9 @@
 #define EXC_MF           16
 #define EXC_AC           17
 #define EXC_MC           18
-#define EXC_XF           19  /* SIMD exception.                */
-#define EXC_SX           30  /* Security exception (SVM only). */
+#define EXC_XF           19  // SIMD exception.
+#define EXC_VE           20  // Virtualization exception - VT only.
+#define EXC_SX           30  // Security exception (SVM only).
 
 /*
  * eflag/rflag definitions.
@@ -238,5 +256,12 @@ typedef enum x86_FLAGS {
    EFLAGS__4           = 0x7fffffff    /* ensure 4 byte encoding */
 } x86_FLAGS;
 
+/*
+ *   MPX bound configuration registers
+ */
+#define BNDCFG_EN        0x00000001
+#define BNDCFG_BNDPRSV   0x00000002
+#define BNDCFG_RSVD      0x00000ffc
+#define BNDCFG_BDBASE    CONST64U(0xfffffffffffff000)
 
 #endif // ifndef _VM_BASIC_DEFS_H_

vmmon-only/include/x86apic.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998,2015 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -45,6 +45,7 @@
 #define X2APIC_VERSION_0x15       0x15
 #define APIC_MAXLVT_0x4           0x4
 #define XAPIC_MAXLVT_0x5          0x5
+#define XAPIC_MAXLVT_0x6          0x6 // Intel only: Nehalem (1A) and onward
 #define APIC_VERSION_MASK         0xff
 #define APIC_MAX_LVT_MASK         0xff
 #define APIC_MAX_LVT_SHIFT        16

vmmon-only/include/x86cpuid_asm.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -63,6 +63,14 @@ extern "C" {
  */
 void __cpuid(int regs[4], int eax);
 #pragma intrinsic(__cpuid)
+
+/*
+ * __cpuidex has been supported since VS2008
+ */
+#if _MSC_VER >= 1500
+void __cpuidex(int regs[4], int eax, int ecx);
+#pragma intrinsic(__cpuidex)
+#endif /* _MSC_VER >= 1500 */
 #endif /* VM_X86_64 */
 
 #ifdef __cplusplus
@@ -247,6 +255,20 @@ __GET_CPUID(int input, CPUIDRegs *regs)
 
 #ifdef VM_X86_64
 
+#if _MSC_VER >= 1500
+
+/*
+ * __cpuidex has been supported since VS2008
+ */
+
+static INLINE void
+__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
+{
+   __cpuidex((int *)regs, inputEax, inputEcx);
+}
+
+#else // _MSC_VER >= 1500
+
 /*
  * No inline assembly in Win64. Implemented in bora/lib/misc in
  * cpuidMasm64.asm.
@@ -254,6 +276,7 @@ __GET_CPUID(int input, CPUIDRegs *regs)
 
 extern void
 __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs);
+#endif // _MSC_VER >= 1500
 
 #else // VM_X86_64
 
@@ -441,5 +464,13 @@ __GET_EAX_FROM_CPUID4(int inputEcx)
    _dx = regs.edx;                      \
 }
 
+#define GET_CPUID2(_ax, _bx, _cx, _dx) {\
+   CPUIDRegs regs;                      \
+   __GET_CPUID2(_ax, _cx, &regs);       \
+   _ax = regs.eax;                      \
+   _bx = regs.ebx;                      \
+   _cx = regs.ecx;                      \
+   _dx = regs.edx;                      \
+}
 
 #endif

vmmon-only/include/x86msr.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -24,6 +24,7 @@
 
 #ifndef _X86MSR_H_
 #define _X86MSR_H_
+#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX
 
@@ -38,6 +39,11 @@
 #include "vm_basic_types.h"
 #include "community_source.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 /*
  * Results of calling rdmsr(msrNum) on all logical processors.
  */
@@ -92,6 +98,7 @@ MSRQuery;
 #define MSR_MCG_CAP           0x00000179
 #define MSR_MCG_STATUS        0x0000017a
 #define MSR_MCG_CTL           0x0000017b
+#define MSR_MCG_EXT_CTL       0x000004d0
 #define MSR_EVNTSEL0          0x00000186
 #define MSR_EVNTSEL1          0x00000187
 #define MSR_FLEX_RATIO        0x00000194 // Intel Nehalem Family
@@ -106,7 +113,9 @@ MSRQuery;
 #define MSR_TSC_AUX           0xc0000103
 #define MSR_BD_TSC_RATIO      0xc0000104
 
+#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
+#endif
 
 /* Intel Core Architecture and later: use only architected counters. */
 #define IA32_MSR_PERF_CAPABILITIES                0x345
@@ -240,6 +249,133 @@ typedef enum {
 #define MSR_X2APIC_DIVIDER   0x0000083e
 #define MSR_X2APIC_SELFIPI   0x0000083f
 
+#define MSR_BNDCFGS          0x00000d90  // Sup. mode bounds configuration
+
+#define MSR_XSS              0x00000da0  // Extended Supervisor State Mask
+
+/* RTIT MSRs */
+#define MSR_RTIT_CTL              0x00000570
+#define MSR_RTIT_STATUS           0x00000571
+#define MSR_RTIT_OUTPUT_BASE      0x00000560
+#define MSR_RTIT_OUTPUT_MASK_PTRS 0x00000561
+#define MSR_RTIT_CR3_MATCH        0x00000572
+#define MSR_RTIT_ADDR0_A          0x00000580
+#define MSR_RTIT_ADDR0_B          0x00000581
+#define MSR_RTIT_ADDR1_A          0x00000582
+#define MSR_RTIT_ADDR1_B          0x00000583
+#define MSR_RTIT_ADDR2_A          0x00000584
+#define MSR_RTIT_ADDR2_B          0x00000585
+#define MSR_RTIT_ADDR3_A          0x00000586
+#define MSR_RTIT_ADDR3_B          0x00000587
+
+#define MSR_RTIT_NUM_OF_ADDR_REG  8
+
+/* RTIT control MSR bits */
+#define MSR_RTIT_CTL_TRACE_EN      (1LL<<0)  // Enable tracing
+#define MSR_RTIT_CTL_CYC_EN        (1LL<<1)  // Enable CYC Packet
+#define MSR_RTIT_CTL_OS            (1LL<<2)  // CPL0 filter
+#define MSR_RTIT_CTL_USER          (1LL<<3)  // CPL > 0 filter
+#define MSR_RTIT_CTL_FABRIC_EN     (1LL<<6)  // Trace output direction
+#define MSR_RTIT_CTL_CR3_FILTER_EN (1LL<<7)  // Enable CR3 filter
+#define MSR_RTIT_CTL_TOPA          (1LL<<8)  // Enable ToPA output scheme
+#define MSR_RTIT_CTL_MTC_EN        (1LL<<9)  // Enable MTC packet
+#define MSR_RTIT_CTL_TSC_EN        (1LL<<10) // Enable TSC packet
+#define MSR_RTIT_CTL_DIS_RETC      (1LL<<11) // Disable RET compression
+#define MSR_RTIT_CTL_BRANCH_EN     (1LL<<13) // Disable COFI-based packets
+#define MSR_RTIT_CTL_RSVD          CONST64U(0xffff0000f0841030) // Reserved bits
+
+#define MSR_RTIT_CTL_MTCFREQ_MASK CONST64U(0xf)
+#define MSR_RTIT_CTL_MTCFREQ_SHIFT 14
+#define MSR_RTIT_CTL_MTCFREQ(_msr)  \
+       (((_msr) >> MSR_RTIT_CTL_MTCFREQ_SHIFT) & MSR_RTIT_CTL_MTCFREQ_MASK)
+
+#define MSR_RTIT_CTL_CYCTHRESH_MASK CONST64U(0xf)
+#define MSR_RTIT_CTL_CYCTHRESH_SHIFT 19
+#define MSR_RTIT_CTL_CYCTHRESH(_msr)  \
+       (((_msr) >> MSR_RTIT_CTL_CYCTHRESH_SHIFT) & MSR_RTIT_CTL_CYCTHRESH_MASK)
+
+#define MSR_RTIT_CTL_PSBFREQ_MASK CONST64U(0xf)
+#define MSR_RTIT_CTL_PSBFREQ_SHIFT 24
+#define MSR_RTIT_CTL_PSBFREQ(_msr)  \
+       (((_msr) >> MSR_RTIT_CTL_PSBFREQ_SHIFT) & MSR_RTIT_CTL_PSBFREQ_MASK)
+
+#define MSR_RTIT_CTL_ADDR0_CFG_MASK  CONST64U(0xf)
+#define MSR_RTIT_CTL_ADDR0_CFG_SHIFT 32
+#define MSR_RTIT_CTL_ADDR0_CFG(_msr)  \
+       (((_msr) >> MSR_RTIT_CTL_ADDR0_CFG_SHIFT) & MSR_RTIT_CTL_ADDR0_CFG_MASK)
+
+#define MSR_RTIT_CTL_ADDR1_CFG_MASK  CONST64U(0xf)
+#define MSR_RTIT_CTL_ADDR1_CFG_SHIFT 36
+#define MSR_RTIT_CTL_ADDR1_CFG(_msr)  \
+       (((_msr) >> MSR_RTIT_CTL_ADDR1_CFG_SHIFT) & MSR_RTIT_CTL_ADDR1_CFG_MASK)
+
+#define MSR_RTIT_CTL_ADDR2_CFG_MASK  CONST64U(0xf)
+#define MSR_RTIT_CTL_ADDR2_CFG_SHIFT 40
+#define MSR_RTIT_CTL_ADDR2_CFG(_msr)  \
+       (((_msr) >> MSR_RTIT_CTL_ADDR2_CFG_SHIFT) & MSR_RTIT_CTL_ADDR2_CFG_MASK)
+
+#define MSR_RTIT_CTL_ADDR3_CFG_MASK  CONST64U(0xf)
+#define MSR_RTIT_CTL_ADDR3_CFG_SHIFT 44
+#define MSR_RTIT_CTL_ADDR3_CFG(_msr)  \
+       (((_msr) >> MSR_RTIT_CTL_ADDR3_CFG_SHIFT) & MSR_RTIT_CTL_ADDR3_CFG_MASK)
+
+/* RTIT status MSR bits */
+#define MSR_RTIT_STATUS_FILTER_EN    (1LL<<0)
+#define MSR_RTIT_STATUS_CONTEXT_EN   (1LL<<1)
+#define MSR_RTIT_STATUS_TRIGGER_EN   (1LL<<2)
+#define MSR_RTIT_STATUS_ERROR        (1LL<<4)
+#define MSR_RTIT_STATUS_STOPPED      (1LL<<5)
+#define MSR_RTIT_STATUS_RSVD         CONST64U(0xfffe0000ffffffc8)
+
+#define MSR_RTIT_STATUS_PKT_BYTES_CNT_MASK  CONST64U(0x1ffff)
+#define MSR_RTIT_STATUS_PKT_BYTES_CNT_SHIFT 32
+#define MSR_RTIT_STATUS_PKT_BYTES_CNT(_msr)  \
+        (((_msr) >> MSR_RTIT_STATUS_PKT_BYTES_CNT_SHIFT) & \
+             MSR_RTIT_STATUS_PKT_BYTES_CNT_MASK)
+
+/* RTIT CR3 MSR bits */
+#define MSR_RTIT_CR3_MATCH_MASK CONST64U(~0x1f)
+
+/* RTIT output base MSR bits */
+#define MSR_RTIT_OUTPUT_BASE_MASK CONST64U(~0x7f)
+#define MSR_RTIT_OUTPUT_BASE_RSVD CONST64U(0x7f)
+
+/* RTIT output mask ptrs MSR bits */
+#define MSR_RTIT_OUTPUT_MASK_LOWERMASK CONST64U(0x7f)
+
+/*
+ * Get the mask value for the single contiguous output region
+ * when MSR_RTIT_CTL.ToPA is masked
+ */
+#define MSR_RTIT_OUTPUT_MASK_MASK  CONST64U(0xffffffff)
+#define MSR_RTIT_OUTPUT_MASK(_msr) \
+        ((_msr) & MSR_RTIT_OUTPUT_MASK_MASK)
+
+/* Get the offset pointer of the ToPA table when MSR_RTIT_CTL.ToPA is set */
+#define MSR_RTIT_OUTPUT_TABLE_OFFSET_MASK  CONST64U(0x1ffffff)
+#define MSR_RTIT_OUTPUT_TABLE_OFFSET_SHIFT 7
+#define MSR_RTIT_OUTPUT_TABLE_OFFSET(_msr) \
+        ((((_msr) >> MSR_RTIT_OUTPUT_TABLE_OFFSET_SHIFT) & \
+              MSR_RTIT_OUTPUT_TABLE_OFFSET_MASK) << 3)
+#define MSR_RTIT_OUTPUT_TABLE_ENTRY(_msr) \
+        (((_msr) >> MSR_RTIT_OUTPUT_TABLE_OFFSET_SHIFT) & \
+              MSR_RTIT_OUTPUT_TABLE_OFFSET_MASK)
+
+#define MSR_RTIT_OUTPUT_OFFSET_MASK  CONST64U(0xffffffff)
+#define MSR_RTIT_OUTPUT_OFFSET_SHIFT 32
+#define MSR_RTIT_OUTPUT_OFFSET(_msr) \
+        (((_msr) >> MSR_RTIT_OUTPUT_OFFSET_SHIFT) & \
+             MSR_RTIT_OUTPUT_OFFSET_MASK)
+
+#define MSR_RTIT_CR3_MATCH_RSVD CONST64U(0x1f)
+
+/* SGX MSRs */
+#define MSR_SGX_SVN_STATUS               0x00000500
+
+/* SGX SVN status MSR fields */
+#define MSR_SGX_SVN_STATUS_LOCK          0x1
+#define MSR_SGX_SVN_STATUS_SINIT_SVN     CONST64U(0xff0000)
+#define MSR_SGX_SVN_STATUS_RSVD          CONST64U(0xffffffffff00fffe)
 
 /* MSR_CR_PAT power-on value */
 #define MSR_CR_PAT_DEFAULT   0x0007040600070406ULL
@@ -277,7 +413,10 @@ typedef enum {
 #define MSR_FEATCTL_LOCK     0x00000001
 #define MSR_FEATCTL_SMXE     0x00000002
 #define MSR_FEATCTL_VMXE     0x00000004
+#define MSR_FEATCTL_SENTERP  0x00007F00
+#define MSR_FEATCTL_SENTERE  0x00008000
 #define MSR_FEATCTL_SGXE     0x00040000
+#define MSR_FEATCTL_LMCE     0x00100000
 
 /* MSR_EFER bits. */
 #define MSR_EFER_SCE         0x0000000000000001ULL  /* Sys call ext'ns:  r/w */
@@ -423,6 +562,7 @@ typedef enum {
 
 #define MSR_HYPERV_HYPERCALL_EN                  1ULL
 #define MSR_HYPERV_REFERENCE_TSC_EN              1ULL
+#define MSR_HYPERV_VP_ASSIST_EN                  1ULL
 
 #define MSR_HYPERV_GUESTOSID_VENDOR_SHIFT        48
 #define MSR_HYPERV_GUESTOSID_VENDOR_MASK         0xfULL
@@ -475,12 +615,24 @@ typedef enum {
 /*
  * MISC_FEATURES_ENABLES bits
  */
+#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
+#endif
 
 
 
 
 
+/*
+ * Platform Quality of Service
+ */
+
+#define MSR_INTEL_QOS_EVTSEL    0xc8d
+#define MSR_INTEL_QOS_CTR       0xc8e
+#define MSR_INTEL_QOS_ASSOC     0xc8f
+
 static INLINE uint32
 X86MSR_SysCallEIP(uint64 star)
 {
@@ -502,4 +654,8 @@ X86MSR_SysRetCS(uint64 star)
 }
 
 
+#if defined __cplusplus
+}
+#endif
+
 #endif /* _X86MSR_H_ */

vmmon-only/include/x86paging_64.h

@@ -0,0 +1,87 @@
+/*********************************************************
+ * Copyright (C) 1998-2014,2016 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * x86paging_64.h --
+ *
+ *      Contains definitions for the x86 page table layout specific to
+ *      long mode.
+ */
+
+#ifndef _X86PAGING_64_H_
+#define _X86PAGING_64_H_
+
+#define INCLUDE_ALLOW_USERLEVEL
+
+#define INCLUDE_ALLOW_MODULE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_DISTRIBUTE
+#define INCLUDE_ALLOW_VMCORE
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+#include "x86types.h"
+#include "vm_pagetable.h"
+
+#define LM_PTE_PFN_MASK      CONST64U(0xffffffffff000)
+#define LM_PTE_2_PFN(_pte)   (((_pte) & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT)
+
+#define LM_PDE_PFN_MASK      0xfffffffe00000LL
+#define LM_PDPTE_PFN_MASK    0xfffffc0000000LL
+
+#define LM_AVAIL_SHIFT         9
+
+#define LM_AVAIL_MASK         (CONST64(0x7) << LM_AVAIL_SHIFT)
+#define LM_FLAGS_MASK         CONST64(0x80000000000001ff)
+#define LM_CR3_FLAGS_MASK     CONST64(0x18)
+#define LM_L4_L3_RSVD_MASK    CONST64(0x180)
+#define LM_L3_1G_RSVD_MASK    CONST64(0x3fffe000)
+
+#define LM_MAKE_CR3(_mpfn, _flags) \
+                   (((uint64)(_mpfn) << PT_PTE_PFN_SHIFT) | \
+                   ((_flags) & LM_CR3_FLAGS_MASK))
+
+#define LM_MAKE_PTE(_mpfn, _avail, _flags) \
+                   (((uint64)(_mpfn) << PT_PTE_PFN_SHIFT) | \
+                    (((_avail) << LM_AVAIL_SHIFT) & LM_AVAIL_MASK) | \
+                    ((uint64)(_flags) & LM_FLAGS_MASK))
+
+#define LM_MAKE_PDE(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L4E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L3E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L2E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+#define LM_MAKE_L1E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags)
+
+
+/*
+ * x86-64 architecture requires implementations supporting less than
+ * full 64-bit VAs to ensure that all virtual addresses are in canonical
+ * form. An address is in canonical form if the address bits from the
+ * most significant implemented bit up to bit 63 are all ones or all
+ * zeros. If this is not the case, the processor generates #GP/#SS. Our
+ * VCPU implements 48 bits of virtual address space.
+ */
+
+#define VA64_IMPL_BITS             48
+#define VA64_CANONICAL_MASK        ~((CONST64U(1) << (VA64_IMPL_BITS - 1)) - 1)
+#define VA64_CANONICAL_HOLE_START   (CONST64U(1) << (VA64_IMPL_BITS - 1))
+#define VA64_CANONICAL_HOLE_LEN  VA64_CANONICAL_MASK - VA64_CANONICAL_HOLE_START
+
+#endif /* _X86PAGING_64_H_ */

vmmon-only/include/x86paging_common.h

@@ -0,0 +1,113 @@
+/*********************************************************
+ * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * x86paging_common.h --
+ *
+ *      Contains definitions for the x86 page table layout that are common
+ *      to legacy and long modes.
+ */
+
+#ifndef _X86PAGING_COMMON_H_
+#define _X86PAGING_COMMON_H_
+
+#define INCLUDE_ALLOW_USERLEVEL
+
+#define INCLUDE_ALLOW_MODULE
+#define INCLUDE_ALLOW_VMMON
+#define INCLUDE_ALLOW_VMK_MODULE
+#define INCLUDE_ALLOW_VMKERNEL
+#define INCLUDE_ALLOW_DISTRIBUTE
+#define INCLUDE_ALLOW_VMCORE
+#include "includeCheck.h"
+
+#include "vm_basic_types.h"
+
+
+/*
+ *   page tables
+ */
+
+#define PTE_P           0x00000001
+#define PTE_RW          0x00000002
+#define PTE_US          0x00000004
+#define PTE_PWT         0x00000008
+#define PTE_PCD         0x00000010 // careful: MMU_PTE_EX has same value.
+#define PTE_A           0x00000020
+#define PTE_D           0x00000040
+#define PTE_PS          0x00000080
+#define PTE_PAT         0x00000080
+#define PTE_G           0x00000100
+#define PTE_LARGE_PAT   0x00001000
+#define PTE_RSVD        CONST64(0x0008000000000000)
+#define PTE_NX          CONST64(0x8000000000000000)
+#define PTE_NX_SHIFT    63
+
+#define PTE_PK_MASK     CONST64(0x7800000000000000)
+#define PTE_PK_SHIFT    59
+
+#define PTE_LARGE_PAT_2_PS(_pte) (((_pte) & PTE_LARGE_PAT) >> 5)
+
+#define PTE_PROT_FLAGS  (PTE_P|PTE_RW|PTE_US)
+#define PTE_FLAGS       (PTE_PROT_FLAGS|PTE_G)
+
+#define PTE_KERNEL              (PTE_P|PTE_RW)
+#define PTE_KERNEL_NX           (PTE_P|PTE_RW|PTE_NX)
+
+#define PTE_PRESENT(_pte)   (((_pte) & PTE_P) != 0)
+#define PTE_WRITEABLE(_pte) (((_pte) & PTE_RW) != 0)
+#define PTE_ACCESS(_pte)    (((_pte) & PTE_A) != 0)
+#define PTE_DIRTY(_pte)     (((_pte) & PTE_D) != 0)
+#define PTE_USER(_pte)      (((_pte) & PTE_US) != 0)
+#define PTE_GLOBAL(_pte)    (((_pte) & PTE_G) != 0)
+#define PTE_LARGEPAGE(_pte) (((_pte) & PTE_PS) != 0)
+#define PTE_UNCACHED(_pte)  (((_pte) & PTE_PCD) != 0)
+#define PTE_NOEXECUTE(_pte) (((_pte) & PTE_NX) != 0)
+#define PTE_PK(_pte)        (((_pte) & PTE_PK_MASK) >> PTE_PK_SHIFT)
+
+#define PK_AD         1 /* Access disable bit */
+#define PK_WD         2 /* Write disable bit */
+#define PKR_WIDTH     2
+#define PKR_MASK      3
+#define PK_RIGHTS(_pkru, _key) (((_pkru) >> ((_key) * PKR_WIDTH)) & PKR_MASK);
+
+/* Error code flags */
+#define PF_P            0x0001
+#define PF_RW           0x0002
+#define PF_US           0x0004
+#define PF_RSVD         0x0008
+#define PF_ID           0x0010
+#define PF_PK           0x0020
+#define PF_SGX          0x8000
+
+/*
+ * Operand definitions for the INVPCID instruction. See SDM Vol. 2A.
+ */
+#define INVPCID_EXTENT_ADDR               0
+#define INVPCID_EXTENT_PCID_CTX           1
+#define INVPCID_EXTENT_ALL_CTX            2
+#define INVPCID_EXTENT_ALL_CTX_LOCAL      3
+#define INVPCID_EXTENT_MAX                3
+#define INVPCID_MAX_PCID              0xfff
+
+typedef struct {
+   uint64   pcid;    /* Bits 63:12 reserved.                   */
+   uint64   la;      /* Used for single address invalidation.  */
+} InvpcidArg;
+
+#endif /* _X86PAGING_COMMON_H_ */

vmmon-only/include/x86perfctr.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2012,2014-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012,2014-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -39,7 +39,7 @@
 #define PERFCTR_PENTIUM4_NUM_COUNTERS            18
 #define PERFCTR_PENTIUM4_NUM_COUNTERS_WITH_L3    26
 #define PERFCTR_AMD_NUM_COUNTERS                 4 
-#define PERFCTR_BD_NUM_COUNTERS                  6
+#define PERFCTR_AMD_EXT_NUM_COUNTERS             6
 #define PERFCTR_P6_NUM_COUNTERS                  2
 #define PERFCTR_NEHALEM_NUM_GEN_COUNTERS         4
 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS       3
@@ -109,10 +109,10 @@
 /* AMD Performance Counter MSR Definitions */
 #define PERFCTR_AMD_PERFEVTSEL0_ADDR             0xC0010000
 #define PERFCTR_AMD_PERFCTR0_ADDR                0xC0010004
-#define PERFCTR_BD_BASE_ADDR                     0xC0010200
-#define PERFCTR_BD_EVENTSEL                      0
-#define PERFCTR_BD_CTR                           1
-#define PERFCTR_BD_MSR_STRIDE                    2
+#define PERFCTR_AMD_EXT_BASE_ADDR                0xC0010200
+#define PERFCTR_AMD_EXT_EVENTSEL                 0
+#define PERFCTR_AMD_EXT_CTR                      1
+#define PERFCTR_AMD_EXT_MSR_STRIDE               2
 
 /* AMD Clocks */
 #define PERFCTR_AMD_CPU_CLK_UNHALTED                           0x76
@@ -144,10 +144,6 @@
 #define PERFCTR_AMD_L2_MISS                                    0x7e
 #define PERFCTR_AMD_L2_FILL_WRITEBACK                          0x7f
 
-/* AMD L3 Cache */
-#define PERFCTR_AMD_REQUESTS_TO_L3                             (0x4e0 | 0xf000)
-#define PERFCTR_AMD_L3_MISS                                    (0x4e1 | 0xf000)
-
 /* AMD Instruction Cache Events */
 
 #define PERFCTR_AMD_INSTR_FETCHES                              0x80
@@ -395,7 +391,7 @@
  *    Use CPUID 0xa to get perf capabilities.  Some (7) events are
  *    architectural; most are version-specific.
  *
- *    V1: Yonah, V2: Merom, V2+: Penryn
+ *    V1: Yonah, V2: Merom, V2+: Penryn, V3: Nehalem, V4: Skylake
  *
  *    V1 is similar to P6, with some additions:
  *       - Global control MSR
@@ -404,6 +400,14 @@
  *       - Ability to freeze a counter before PMI delivery.
  *       - Freeze during SMI (Penryn+ only)
  *       - VMCS global enable (Penryn+ only)
+ *    V3 introduces:
+ *       - (nothing we virtualize)
+ *    V4 introduces:
+ *       - Global unavailable (in-use) MSR
+ *       - Freeze on PMI bit
+ *       - ASCI bit
+ *       - Global status set MSR
+ *       - Global status reset MSR (previously called Ovf ctrl)
  *
  * ----------------------------------------------------------------------
  */
@@ -443,6 +447,7 @@
 #define PERFCTR_CORE_PMI_UNAVAILABLE_IN_USE         (CONST64U(1) << 63)
 // XXX serebrin/dhecht: 1-10-11: Make ANYTHREAD depend on number of fixed PMCs
 
+#define PERFCTR_CORE_GLOBAL_STATUS_TOPA_PMI      (1ULL << 55)
 #define PERFCTR_CORE_GLOBAL_STATUS_CTR_FRZ       (1ULL << 59)
 #define PERFCTR_CORE_GLOBAL_STATUS_ASCI          (1ULL << 60)
 #define PERFCTR_CORE_GLOBAL_STATUS_OVFBUFFER     (1ULL << 62)
@@ -632,7 +637,7 @@
  *      escrAddr:     MSR # of the Perf Event Selector (0xc0010000 + index).
  *      escrVal:      Value placed in PerfEvtSel MSR; what to measure.
  *
- *      On AMD BD:
+ *      On AMD with PerfCtrExtCore support:
  *      index:        Which perf ctr, 0 to 5.  RDPMC argument
  *      addr:         MSR of raw perf ctr              (0xc0010201 + 2 * index).
  *                                  aliased PMCs 0 - 3 (0xc0010004 + index).
@@ -731,13 +736,17 @@ PerfCtr_PgcToOvfValidBits(uint64 pgcValBits)
 static INLINE uint64
 PerfCtr_PgcToStsRstValidBits(uint64 pgcValBits)
 {
-   return pgcValBits | (1ULL << 55) | MASKRANGE64(63, 58);
+   return pgcValBits |
+          PERFCTR_CORE_GLOBAL_STATUS_TOPA_PMI |
+          MASKRANGE64(63, 58);
 }
 
 static INLINE uint64
 PerfCtr_PgcToGssValidBits(uint64 pgcValBits)
 {
-   return pgcValBits | (1ULL << 55) | MASKRANGE64(62, 58);
+   return pgcValBits |
+          PERFCTR_CORE_GLOBAL_STATUS_TOPA_PMI |
+          MASKRANGE64(62, 58);
 }
 
 /*
@@ -815,6 +824,34 @@ PerfCtr_PEBSAvailable(void)
    return FALSE; 
 }
 
+/*
+ *----------------------------------------------------------------------
+ *
+ *  PerfCtr_PTAvailable --
+ *
+ *      Checks if this CPU is capable of PT(Intel processor trace).
+ *
+ * Results:
+ *      TRUE if PT is supported, FALSE otherwise.
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Bool
+PerfCtr_PTAvailable(void)
+{
+   CPUIDRegs regs;
+   __GET_CPUID(0, &regs);
+   if (CPUID_IsVendorIntel(&regs)) {
+      __GET_CPUID2(7, 0, &regs);
+      return (regs.ecx & CPUID_INTERNAL_MASK_PT) != 0;
+   }
+   return FALSE;
+}
+
 /* The following are taken from the Intel Architecture Manual,
  * Book 3, Table 14-4. 
  */ 

vmmon-only/include/x86sel.h

@@ -35,6 +35,11 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 #define SELECTOR_GDT             0
 #define SELECTOR_LDT             1
 #define SELECTOR_RPL_SHIFT       0
@@ -64,4 +69,9 @@ MAKE_SELECTOR(unsigned index, unsigned ti, unsigned rpl)
    return MAKE_SELECTOR_UNCHECKED(index, ti, rpl);
 }
 
+
+#if defined __cplusplus
+} // extern "C"
+#endif
+
 #endif /* !defined _X86SEL_H_ */

vmmon-only/include/x86svm.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -47,10 +47,6 @@
 #endif
 #include "x86cpuid_asm.h"
 
-#ifdef VMM
-#include "cpuidMonitor.h"
-#endif
-
 #define SVM_VMCB_IO_BITMAP_SIZE    (3 * PAGE_SIZE)
 #define SVM_VMCB_MSR_BITMAP_SIZE   (2 * PAGE_SIZE)
 

vmmon-only/include/x86types.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2003-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -39,16 +39,7 @@
 #include "vm_basic_defs.h"
 #include "address_defs.h"
 #include "x86segdescrs.h"
-
-/*
- * Types used for PL4 page table in x86_64 and arm64
- */
-
-typedef uint64 VM_L4E;
-typedef uint64 VM_L3E;
-typedef uint64 VM_L2E;
-typedef uint64 VM_L1E;
-
+#include "vm_pagetable.h"
 
 /*
  * Page table

vmmon-only/include/x86vendor.h

@@ -1,6 +1,6 @@
 
 /*********************************************************
- * Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the

vmmon-only/include/x86vt-exit-reasons.h

@@ -0,0 +1,97 @@
+/*********************************************************
+ * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * x86vt-exit-reasons.h --
+ *
+ * VT exit reasons.
+ *
+ */
+
+#include "community_source.h"
+
+#define VT_EXITREASON_VMENTRYFAIL           (1U << 31)
+
+VT_EXIT(EXC_OR_NMI,            0)
+VT_EXIT(EXTINT,                1)
+VT_EXIT(TRIPLEFAULT,           2)
+VT_EXIT(INIT,                  3)
+VT_EXIT(SIPI,                  4)
+VT_EXIT(IOSMI,                 5)
+VT_EXIT(OTHERSMI,              6)
+VT_EXIT(VINTR_WINDOW,          7)
+VT_EXIT(VNMI_WINDOW,           8)
+VT_EXIT(TS,                    9)
+VT_EXIT(CPUID,                10)
+VT_EXIT(GETSEC,               11)
+VT_EXIT(HLT,                  12)
+VT_EXIT(INVD,                 13)
+VT_EXIT(INVLPG,               14)
+VT_EXIT(RDPMC,                15)
+VT_EXIT(RDTSC,                16)
+VT_EXIT(RSM,                  17)
+VT_EXIT(VMCALL,               18)
+VT_EXIT(VMCLEAR,              19)
+VT_EXIT(VMLAUNCH,             20)
+VT_EXIT(VMPTRLD,              21)
+VT_EXIT(VMPTRST,              22)
+VT_EXIT(VMREAD,               23)
+VT_EXIT(VMRESUME,             24)
+VT_EXIT(VMWRITE,              25)
+VT_EXIT(VMXOFF,               26)
+VT_EXIT(VMXON,                27)
+VT_EXIT(CR,                   28)
+VT_EXIT(DR,                   29)
+VT_EXIT(IO,                   30)
+VT_EXIT(RDMSR,                31)
+VT_EXIT(WRMSR,                32)
+VT_EXIT(VMENTRYFAIL_GUEST,   (33 | VT_EXITREASON_VMENTRYFAIL))
+VT_EXIT(VMENTRYFAIL_MSR,     (34 | VT_EXITREASON_VMENTRYFAIL))
+VT_EXIT(VMEXIT35,             35)
+VT_EXIT(MWAIT,                36)
+VT_EXIT(MTF,                  37)
+VT_EXIT(VMEXIT38,             38)
+VT_EXIT(MONITOR,              39)
+VT_EXIT(PAUSE,                40)
+VT_EXIT(VMENTRYFAIL_MC,      (41 | VT_EXITREASON_VMENTRYFAIL))
+VT_EXIT(VMEXIT42,             42)
+VT_EXIT(TPR,                  43)
+VT_EXIT(APIC,                 44)
+VT_EXIT(EOI,                  45)
+VT_EXIT(GDTR_IDTR,            46)
+VT_EXIT(LDTR_TR,              47)
+VT_EXIT(EPT_VIOLATION,        48)
+VT_EXIT(EPT_MISCONFIG,        49)
+VT_EXIT(INVEPT,               50)
+VT_EXIT(RDTSCP,               51)
+VT_EXIT(TIMER,                52)
+VT_EXIT(INVVPID,              53)
+VT_EXIT(WBINVD,               54)
+VT_EXIT(XSETBV,               55)
+VT_EXIT(APIC_WRITE,           56)
+VT_EXIT(RDRAND,               57)
+VT_EXIT(INVPCID,              58)
+VT_EXIT(VMFUNC,               59)
+VT_EXIT(ENCLS,                60)
+VT_EXIT(RDSEED,               61)
+VT_EXIT(PML_LOGFULL,          62)
+VT_EXIT(XSAVES,               63)
+VT_EXIT(XRSTORS,              64)
+
+/* Bump this up if you add an exit reason. */
+#define VT_NUM_EXIT_REASONS   65

vmmon-only/include/x86vt-vmcs-fields.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2015-2017 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -19,195 +19,697 @@
 /*
  * x86vt-vmcs-fields.h --
  *
- * VMCS encodings; SDM volume 3 Appendix B.
- * These are the values passed to VMWRITE and VMREAD.
+ * VMCS fields (SDM volume 3 Appendix B).
  *
- * VMCS_FIELD(name, encoding)
  */
 
 #include "community_source.h"
 
 /*
- * VMCS_FIELD(name,                    encoding, vVT support)
+ * There are 16 groups of VMCS fields: 4 sizes crossed with 4 types.
+ * The 4 sizes are 16-bit, 64-bit, 32-bit, and "natural", in that order,
+ * per the enumeration of the size in the field encoding.
+ * The 4 types are control, read-only data, guest-state, and host-state,
+ * in that order, per the enumeration of the type in the field encoding.
  */
 
+/*
+ * Use of this table:
+ * A typical consumer will define a VMCS_FIELD macro to extract and
+ * collate the desired information for each VMCS field, and then it will
+ * include this file to invoke that macro with the property list for each
+ * VMCS field.  Some properties are represented mnemonically (e.g. _S16 for
+ * "16-bit size," so that the extracted property can be defined by another
+ * macro.  For example, some consumers may choose to interpret the size
+ * property "_S16" as "uint16", while others may wish to interpret it as "s16."
+ * For convenience, each of the 16 groups is preceded by a VMCS_GROUP_START
+ * macro and followed by a VMCS_GROUP_END macro.  Similarly, each set of four
+ * groups with the same size is preceded by a VMCS_SET_START macro and followed
+ * by a VMCS_SET_END macro.  This file provides empty definitions of
+ * VMCS_GROUP_START, VMCS_GROUP_END, VMCS_SET_START, and VMCS_SET_END if the
+ * consumer has not defined them.
+ */
+
+#ifndef VMCS_GROUP_START
+#define VMCS_GROUP_START(size, type)
+#define LOCAL_GROUP_START
+#endif
+
+#ifndef VMCS_GROUP_END
+#define VMCS_GROUP_END(size, type)
+#define LOCAL_GROUP_END
+#endif
+
+#ifndef VMCS_SET_START
+#define VMCS_SET_START(size)
+#define LOCAL_SET_START
+#endif
+
+#ifndef VMCS_SET_END
+#define VMCS_SET_END(size)
+#define LOCAL_SET_END
+#endif
+
+#ifndef VMCS_UNUSED
+#define VMCS_UNUSED(val, size, type, index)
+#define LOCAL_UNUSED
+#endif
+
+/*
+ * VMCS_FIELD(_name, _val, _size, _type, _index, _cache, _vvt, _access)
+ *    _type is one of:
+ *       _TC -- control
+ *       _TG -- guest-state
+ *       _TH -- host-state
+ *       _TD -- read-only data
+ *    _size is one of:
+ *       _S16 -- 16-bits
+ *       _S32 -- 32-bits
+ *       _S64 -- 64-bits
+ *       _SN  -- natural width
+ *    _cache is one of:
+ *       _C  -- cached
+ *       _NC -- not cached
+ *    _vvt is one of:
+ *       _V  -- virtualized
+ *       _NV -- not virtualized
+ *   _access is one of:
+ *       _NA -- no access
+ *       _RW -- read/write access
+ *       _RO -- read-only access
+ *
+ */
+
+
+VMCS_SET_START(16)
 /* 16-bit control fields. */
-VMCS_FIELD(VPID,                    0x0000, TRUE)
-VMCS_FIELD(PI_NOTIFY,               0x0002, FALSE)
-VMCS_FIELD(EPTP_INDEX,              0x0004, FALSE)
+VMCS_GROUP_START(16, CTL)
+VMCS_FIELD(VPID,                    0x0000, _S16, _TC,  0,  _C,  _V, _NA)
+VMCS_FIELD(PI_NOTIFY,               0x0002, _S16, _TC,  1, _NC, _NV, _NA)
+VMCS_FIELD(EPTP_INDEX,              0x0004, _S16, _TC,  2, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x0006, _S16, _TC,  3)
+VMCS_UNUSED(                        0x0008, _S16, _TC,  4)
+VMCS_UNUSED(                        0x000A, _S16, _TC,  5)
+VMCS_UNUSED(                        0x000C, _S16, _TC,  6)
+VMCS_UNUSED(                        0x000E, _S16, _TC,  7)
+VMCS_UNUSED(                        0x0010, _S16, _TC,  8)
+VMCS_UNUSED(                        0x0012, _S16, _TC,  9)
+VMCS_UNUSED(                        0x0014, _S16, _TC,  10)
+VMCS_UNUSED(                        0x0016, _S16, _TC,  11)
+VMCS_UNUSED(                        0x0018, _S16, _TC,  12)
+VMCS_UNUSED(                        0x001A, _S16, _TC,  13)
+VMCS_UNUSED(                        0x001C, _S16, _TC,  14)
+VMCS_UNUSED(                        0x001E, _S16, _TC,  15)
+VMCS_UNUSED(                        0x0020, _S16, _TC,  16)
+VMCS_UNUSED(                        0x0022, _S16, _TC,  17)
+VMCS_UNUSED(                        0x0024, _S16, _TC,  18)
+VMCS_UNUSED(                        0x0026, _S16, _TC,  19)
+VMCS_UNUSED(                        0x0028, _S16, _TC,  20)
+VMCS_UNUSED(                        0x002A, _S16, _TC,  21)
+VMCS_UNUSED(                        0x002C, _S16, _TC,  22)
+VMCS_UNUSED(                        0x002E, _S16, _TC,  23)
+VMCS_UNUSED(                        0x0030, _S16, _TC,  24)
+VMCS_UNUSED(                        0x0032, _S16, _TC,  25)
+VMCS_UNUSED(                        0x0034, _S16, _TC,  26)
+VMCS_UNUSED(                        0x0036, _S16, _TC,  27)
+VMCS_UNUSED(                        0x0038, _S16, _TC,  28)
+VMCS_UNUSED(                        0x003A, _S16, _TC,  29)
+VMCS_UNUSED(                        0x003C, _S16, _TC,  30)
+VMCS_UNUSED(                        0x003E, _S16, _TC,  31)
+VMCS_GROUP_END(16, CTL)
+
+/* 16-bit read-only data fields. */
+VMCS_GROUP_START(16, DATA)
+VMCS_UNUSED(                        0x0400, _S16, _TD,  0)
+VMCS_UNUSED(                        0x0402, _S16, _TD,  1)
+VMCS_UNUSED(                        0x0404, _S16, _TD,  2)
+VMCS_UNUSED(                        0x0406, _S16, _TD,  3)
+VMCS_UNUSED(                        0x0408, _S16, _TD,  4)
+VMCS_UNUSED(                        0x040A, _S16, _TD,  5)
+VMCS_UNUSED(                        0x040C, _S16, _TD,  6)
+VMCS_UNUSED(                        0x040E, _S16, _TD,  7)
+VMCS_UNUSED(                        0x0410, _S16, _TD,  8)
+VMCS_UNUSED(                        0x0412, _S16, _TD,  9)
+VMCS_UNUSED(                        0x0414, _S16, _TD,  10)
+VMCS_UNUSED(                        0x0416, _S16, _TD,  11)
+VMCS_UNUSED(                        0x0418, _S16, _TD,  12)
+VMCS_UNUSED(                        0x041A, _S16, _TD,  13)
+VMCS_UNUSED(                        0x041C, _S16, _TD,  14)
+VMCS_UNUSED(                        0x041E, _S16, _TD,  15)
+VMCS_UNUSED(                        0x0420, _S16, _TD,  16)
+VMCS_UNUSED(                        0x0422, _S16, _TD,  17)
+VMCS_UNUSED(                        0x0424, _S16, _TD,  18)
+VMCS_UNUSED(                        0x0426, _S16, _TD,  19)
+VMCS_UNUSED(                        0x0428, _S16, _TD,  20)
+VMCS_UNUSED(                        0x042A, _S16, _TD,  21)
+VMCS_UNUSED(                        0x042C, _S16, _TD,  22)
+VMCS_UNUSED(                        0x042E, _S16, _TD,  23)
+VMCS_UNUSED(                        0x0430, _S16, _TD,  24)
+VMCS_UNUSED(                        0x0432, _S16, _TD,  25)
+VMCS_UNUSED(                        0x0434, _S16, _TD,  26)
+VMCS_UNUSED(                        0x0436, _S16, _TD,  27)
+VMCS_UNUSED(                        0x0438, _S16, _TD,  28)
+VMCS_UNUSED(                        0x043A, _S16, _TD,  29)
+VMCS_UNUSED(                        0x043C, _S16, _TD,  30)
+VMCS_UNUSED(                        0x043E, _S16, _TD,  31)
+VMCS_GROUP_END(16, DATA)
 
 /* 16-bit guest state. */
-VMCS_FIELD(ES,                      0x0800, TRUE)
-VMCS_FIELD(CS,                      0x0802, TRUE)
-VMCS_FIELD(SS,                      0x0804, TRUE)
-VMCS_FIELD(DS,                      0x0806, TRUE)
-VMCS_FIELD(FS,                      0x0808, TRUE)
-VMCS_FIELD(GS,                      0x080A, TRUE)
-VMCS_FIELD(LDTR,                    0x080C, TRUE)
-VMCS_FIELD(TR,                      0x080E, TRUE)
-VMCS_FIELD(INTR_STATUS,             0x0810, FALSE)
+VMCS_GROUP_START(16, GUEST)
+VMCS_FIELD(ES,                      0x0800, _S16, _TG,  0, _NC,  _V, _RW)
+VMCS_FIELD(CS,                      0x0802, _S16, _TG,  1, _NC,  _V, _RW)
+VMCS_FIELD(SS,                      0x0804, _S16, _TG,  2, _NC,  _V, _RW)
+VMCS_FIELD(DS,                      0x0806, _S16, _TG,  3, _NC,  _V, _RW)
+VMCS_FIELD(FS,                      0x0808, _S16, _TG,  4, _NC,  _V, _RW)
+VMCS_FIELD(GS,                      0x080A, _S16, _TG,  5, _NC,  _V, _RW)
+VMCS_FIELD(LDTR,                    0x080C, _S16, _TG,  6, _NC,  _V, _RW)
+VMCS_FIELD(TR,                      0x080E, _S16, _TG,  7, _NC,  _V, _RW)
+VMCS_FIELD(INTR_STATUS,             0x0810, _S16, _TG,  8, _NC, _NV, _NA)
+VMCS_FIELD(PML_INDEX,               0x0812, _S16, _TG,  9, _NC, _NV, _NA)
+VMCS_UNUSED(                        0x0814, _S16, _TG,  10)
+VMCS_UNUSED(                        0x0816, _S16, _TG,  11)
+VMCS_UNUSED(                        0x0818, _S16, _TG,  12)
+VMCS_UNUSED(                        0x081A, _S16, _TG,  13)
+VMCS_UNUSED(                        0x081C, _S16, _TG,  14)
+VMCS_UNUSED(                        0x081E, _S16, _TG,  15)
+VMCS_UNUSED(                        0x0820, _S16, _TG,  16)
+VMCS_UNUSED(                        0x0822, _S16, _TG,  17)
+VMCS_UNUSED(                        0x0824, _S16, _TG,  18)
+VMCS_UNUSED(                        0x0826, _S16, _TG,  19)
+VMCS_UNUSED(                        0x0828, _S16, _TG,  20)
+VMCS_UNUSED(                        0x082A, _S16, _TG,  21)
+VMCS_UNUSED(                        0x082C, _S16, _TG,  22)
+VMCS_UNUSED(                        0x082E, _S16, _TG,  23)
+VMCS_UNUSED(                        0x0830, _S16, _TG,  24)
+VMCS_UNUSED(                        0x0832, _S16, _TG,  25)
+VMCS_UNUSED(                        0x0834, _S16, _TG,  26)
+VMCS_UNUSED(                        0x0836, _S16, _TG,  27)
+VMCS_UNUSED(                        0x0838, _S16, _TG,  28)
+VMCS_UNUSED(                        0x083A, _S16, _TG,  29)
+VMCS_UNUSED(                        0x083C, _S16, _TG,  30)
+VMCS_UNUSED(                        0x083E, _S16, _TG,  31)
+VMCS_GROUP_END(16, GUEST)
 
 /* 16-bit host state. */
-VMCS_FIELD(HOST_ES,                 0x0C00, TRUE)
-VMCS_FIELD(HOST_CS,                 0x0C02, TRUE)
-VMCS_FIELD(HOST_SS,                 0x0C04, TRUE)
-VMCS_FIELD(HOST_DS,                 0x0C06, TRUE)
-VMCS_FIELD(HOST_FS,                 0x0C08, TRUE)
-VMCS_FIELD(HOST_GS,                 0x0C0A, TRUE)
-VMCS_FIELD(HOST_TR,                 0x0C0C, TRUE)
+VMCS_GROUP_START(16, HOST)
+VMCS_FIELD(HOST_ES,                 0x0C00, _S16, _TH,  0, _NC,  _V, _NA)
+VMCS_FIELD(HOST_CS,                 0x0C02, _S16, _TH,  1, _NC,  _V, _NA)
+VMCS_FIELD(HOST_SS,                 0x0C04, _S16, _TH,  2, _NC,  _V, _NA)
+VMCS_FIELD(HOST_DS,                 0x0C06, _S16, _TH,  3, _NC,  _V, _NA)
+VMCS_FIELD(HOST_FS,                 0x0C08, _S16, _TH,  4, _NC,  _V, _NA)
+VMCS_FIELD(HOST_GS,                 0x0C0A, _S16, _TH,  5, _NC,  _V, _NA)
+VMCS_FIELD(HOST_TR,                 0x0C0C, _S16, _TH,  6, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x0C0E, _S16, _TH,  7)
+VMCS_UNUSED(                        0x0C10, _S16, _TH,  8)
+VMCS_UNUSED(                        0x0C12, _S16, _TH,  9)
+VMCS_UNUSED(                        0x0C14, _S16, _TH,  10)
+VMCS_UNUSED(                        0x0C16, _S16, _TH,  11)
+VMCS_UNUSED(                        0x0C18, _S16, _TH,  12)
+VMCS_UNUSED(                        0x0C1A, _S16, _TH,  13)
+VMCS_UNUSED(                        0x0C1C, _S16, _TH,  14)
+VMCS_UNUSED(                        0x0C1E, _S16, _TH,  15)
+VMCS_UNUSED(                        0x0C20, _S16, _TH,  16)
+VMCS_UNUSED(                        0x0C22, _S16, _TH,  17)
+VMCS_UNUSED(                        0x0C24, _S16, _TH,  18)
+VMCS_UNUSED(                        0x0C26, _S16, _TH,  19)
+VMCS_UNUSED(                        0x0C28, _S16, _TH,  20)
+VMCS_UNUSED(                        0x0C2A, _S16, _TH,  21)
+VMCS_UNUSED(                        0x0C2C, _S16, _TH,  22)
+VMCS_UNUSED(                        0x0C2E, _S16, _TH,  23)
+VMCS_UNUSED(                        0x0C30, _S16, _TH,  24)
+VMCS_UNUSED(                        0x0C32, _S16, _TH,  25)
+VMCS_UNUSED(                        0x0C34, _S16, _TH,  26)
+VMCS_UNUSED(                        0x0C36, _S16, _TH,  27)
+VMCS_UNUSED(                        0x0C38, _S16, _TH,  28)
+VMCS_UNUSED(                        0x0C3A, _S16, _TH,  29)
+VMCS_UNUSED(                        0x0C3C, _S16, _TH,  30)
+VMCS_UNUSED(                        0x0C3E, _S16, _TH,  31)
+VMCS_GROUP_END(16, HOST)
+VMCS_SET_END(16)
 
+
+VMCS_SET_START(64)
 /* 64-bit control fields. */
-VMCS_FIELD(IOBITMAPA,               0x2000, TRUE)
-VMCS_FIELD(IOBITMAPB,               0x2002, TRUE)
-VMCS_FIELD(MSRBITMAP,               0x2004, TRUE)
-VMCS_FIELD(VMEXIT_MSR_STORE_ADDR,   0x2006, TRUE)
-VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR,    0x2008, TRUE)
-VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR,   0x200A, TRUE)
-VMCS_FIELD(EXECUTIVE_VMCS_PTR,      0x200C, TRUE)
-VMCS_FIELD(TSC_OFF,                 0x2010, TRUE)
-VMCS_FIELD(VIRT_APIC_ADDR,          0x2012, TRUE)
-VMCS_FIELD(APIC_ACCESS_ADDR,        0x2014, FALSE)
-VMCS_FIELD(PI_DESC_ADDR,            0x2016, FALSE)
-VMCS_FIELD(VMFUNC_CTLS,             0x2018, FALSE)
-VMCS_FIELD(EPTP,                    0x201A, TRUE)
-VMCS_FIELD(EOI_EXIT0,               0x201C, FALSE)
-VMCS_FIELD(EOI_EXIT1,               0x201E, FALSE)
-VMCS_FIELD(EOI_EXIT2,               0x2020, FALSE)
-VMCS_FIELD(EOI_EXIT3,               0x2022, FALSE)
-VMCS_FIELD(EPTP_LIST_ADDR,          0x2024, FALSE)
-VMCS_FIELD(VMREAD_BITMAP,           0x2026, FALSE)
-VMCS_FIELD(VMWRITE_BITMAP,          0x2028, FALSE)
-VMCS_FIELD(VE_INFO_ADDR,            0x202A, FALSE)
-VMCS_FIELD(XSS_EXITING_BITMAP,      0x202C, FALSE)
-VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, FALSE)
+VMCS_GROUP_START(64, CTL)
+VMCS_FIELD(IOBITMAPA,               0x2000, _S64, _TC,  0, _NC,  _V, _NA)
+VMCS_FIELD(IOBITMAPB,               0x2002, _S64, _TC,  1, _NC,  _V, _NA)
+VMCS_FIELD(MSRBITMAP,               0x2004, _S64, _TC,  2, _NC,  _V, _NA)
+VMCS_FIELD(VMEXIT_MSR_STORE_ADDR,   0x2006, _S64, _TC,  3, _NC,  _V, _NA)
+VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR,    0x2008, _S64, _TC,  4, _NC,  _V, _NA)
+VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR,   0x200A, _S64, _TC,  5, _NC,  _V, _NA)
+VMCS_FIELD(EXECUTIVE_VMCS_PTR,      0x200C, _S64, _TC,  6, _NC,  _V, _NA)
+VMCS_FIELD(PML_ADDR,                0x200E, _S64, _TC,  7, _NC, _NV, _NA)
+VMCS_FIELD(TSC_OFF,                 0x2010, _S64, _TC,  8,  _C,  _V, _NA)
+VMCS_FIELD(VIRT_APIC_ADDR,          0x2012, _S64, _TC,  9, _NC,  _V, _NA)
+VMCS_FIELD(APIC_ACCESS_ADDR,        0x2014, _S64, _TC, 10, _NC,  _V, _NA)
+VMCS_FIELD(PI_DESC_ADDR,            0x2016, _S64, _TC, 11, _NC, _NV, _NA)
+VMCS_FIELD(VMFUNC_CTLS,             0x2018, _S64, _TC, 12, _NC,  _V, _NA)
+VMCS_FIELD(EPTP,                    0x201A, _S64, _TC, 13, _NC,  _V, _NA)
+VMCS_FIELD(EOI_EXIT0,               0x201C, _S64, _TC, 14, _NC, _NV, _NA)
+VMCS_FIELD(EOI_EXIT1,               0x201E, _S64, _TC, 15, _NC, _NV, _NA)
+VMCS_FIELD(EOI_EXIT2,               0x2020, _S64, _TC, 16, _NC, _NV, _NA)
+VMCS_FIELD(EOI_EXIT3,               0x2022, _S64, _TC, 17, _NC, _NV, _NA)
+VMCS_FIELD(EPTP_LIST_ADDR,          0x2024, _S64, _TC, 18, _NC,  _V, _NA)
+VMCS_FIELD(VMREAD_BITMAP,           0x2026, _S64, _TC, 19, _NC, _NV, _NA)
+VMCS_FIELD(VMWRITE_BITMAP,          0x2028, _S64, _TC, 20, _NC, _NV, _NA)
+VMCS_FIELD(VE_INFO_ADDR,            0x202A, _S64, _TC, 21, _NC,  _V, _NA)
+VMCS_FIELD(XSS_EXITING_BITMAP,      0x202C, _S64, _TC, 22, _NC,  _V, _NA)
+VMCS_FIELD(ENCLS_EXITING_BITMAP,    0x202E, _S64, _TC, 23, _NC, _NV, _NA)
+VMCS_UNUSED(                        0x2030, _S64, _TC, 24)
+VMCS_FIELD(TSC_MULTIPLIER,          0x2032, _S64, _TC, 25, _NC, _NV, _NA)
+VMCS_UNUSED(                        0x2034, _S64, _TC, 26)
+VMCS_UNUSED(                        0x2036, _S64, _TC, 27)
+VMCS_UNUSED(                        0x2038, _S64, _TC, 28)
+VMCS_UNUSED(                        0x203A, _S64, _TC, 29)
+VMCS_UNUSED(                        0x203C, _S64, _TC, 30)
+VMCS_UNUSED(                        0x203E, _S64, _TC, 31)
+VMCS_GROUP_END(64, CTL)
 
 /* 64-bit read-only data field. */
-VMCS_FIELD(PHYSADDR,                0x2400, TRUE)
+VMCS_GROUP_START(64, DATA)
+VMCS_FIELD(PHYSADDR,                0x2400, _S64, _TD,  0, _NC,  _V, _RO)
+VMCS_UNUSED(                        0x2402, _S64, _TD,  1)
+VMCS_UNUSED(                        0x2404, _S64, _TD,  2)
+VMCS_UNUSED(                        0x2406, _S64, _TD,  3)
+VMCS_UNUSED(                        0x2408, _S64, _TD,  4)
+VMCS_UNUSED(                        0x240A, _S64, _TD,  5)
+VMCS_UNUSED(                        0x240C, _S64, _TD,  6)
+VMCS_UNUSED(                        0x240E, _S64, _TD,  7)
+VMCS_UNUSED(                        0x2410, _S64, _TD,  8)
+VMCS_UNUSED(                        0x2412, _S64, _TD,  9)
+VMCS_UNUSED(                        0x2414, _S64, _TD,  10)
+VMCS_UNUSED(                        0x2416, _S64, _TD,  11)
+VMCS_UNUSED(                        0x2418, _S64, _TD,  12)
+VMCS_UNUSED(                        0x241A, _S64, _TD,  13)
+VMCS_UNUSED(                        0x241C, _S64, _TD,  14)
+VMCS_UNUSED(                        0x241E, _S64, _TD,  15)
+VMCS_UNUSED(                        0x2420, _S64, _TD,  16)
+VMCS_UNUSED(                        0x2422, _S64, _TD,  17)
+VMCS_UNUSED(                        0x2424, _S64, _TD,  18)
+VMCS_UNUSED(                        0x2426, _S64, _TD,  19)
+VMCS_UNUSED(                        0x2428, _S64, _TD,  20)
+VMCS_UNUSED(                        0x242A, _S64, _TD,  21)
+VMCS_UNUSED(                        0x242C, _S64, _TD,  22)
+VMCS_UNUSED(                        0x242E, _S64, _TD,  23)
+VMCS_UNUSED(                        0x2430, _S64, _TD,  24)
+VMCS_UNUSED(                        0x2432, _S64, _TD,  25)
+VMCS_UNUSED(                        0x2434, _S64, _TD,  26)
+VMCS_UNUSED(                        0x2436, _S64, _TD,  27)
+VMCS_UNUSED(                        0x2438, _S64, _TD,  28)
+VMCS_UNUSED(                        0x243A, _S64, _TD,  29)
+VMCS_UNUSED(                        0x243C, _S64, _TD,  30)
+VMCS_UNUSED(                        0x243E, _S64, _TD,  31)
+VMCS_GROUP_END(64, DATA)
 
 /* 64-bit guest state. */
-VMCS_FIELD(LINK_PTR,                0x2800, TRUE)
-VMCS_FIELD(DEBUGCTL,                0x2802, TRUE)
-VMCS_FIELD(PAT,                     0x2804, FALSE)
-VMCS_FIELD(EFER,                    0x2806, TRUE)
-VMCS_FIELD(PGC,                     0x2808, TRUE)
-VMCS_FIELD(PDPTE0,                  0x280A, TRUE)
-VMCS_FIELD(PDPTE1,                  0x280C, TRUE)
-VMCS_FIELD(PDPTE2,                  0x280E, TRUE)
-VMCS_FIELD(PDPTE3,                  0x2810, TRUE)
+VMCS_GROUP_START(64, GUEST)
+VMCS_FIELD(LINK_PTR,                0x2800, _S64, _TG,  0, _NC,  _V, _NA)
+VMCS_FIELD(DEBUGCTL,                0x2802, _S64, _TG,  1, _NC,  _V, _NA)
+VMCS_FIELD(PAT,                     0x2804, _S64, _TG,  2, _NC,  _V, _NA)
+VMCS_FIELD(EFER,                    0x2806, _S64, _TG,  3, _NC,  _V, _NA)
+VMCS_FIELD(PGC,                     0x2808, _S64, _TG,  4, _NC,  _V, _NA)
+VMCS_FIELD(PDPTE0,                  0x280A, _S64, _TG,  5, _NC,  _V, _RW)
+VMCS_FIELD(PDPTE1,                  0x280C, _S64, _TG,  6, _NC,  _V, _RW)
+VMCS_FIELD(PDPTE2,                  0x280E, _S64, _TG,  7, _NC,  _V, _RW)
+VMCS_FIELD(PDPTE3,                  0x2810, _S64, _TG,  8, _NC,  _V, _RW)
+VMCS_FIELD(BNDCFGS,                 0x2812, _S64, _TG,  9, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x2814, _S64, _TG,  10)
+VMCS_UNUSED(                        0x2816, _S64, _TG,  11)
+VMCS_UNUSED(                        0x2818, _S64, _TG,  12)
+VMCS_UNUSED(                        0x281A, _S64, _TG,  13)
+VMCS_UNUSED(                        0x281C, _S64, _TG,  14)
+VMCS_UNUSED(                        0x281E, _S64, _TG,  15)
+VMCS_UNUSED(                        0x2820, _S64, _TG,  16)
+VMCS_UNUSED(                        0x2822, _S64, _TG,  17)
+VMCS_UNUSED(                        0x2824, _S64, _TG,  18)
+VMCS_UNUSED(                        0x2826, _S64, _TG,  19)
+VMCS_UNUSED(                        0x2828, _S64, _TG,  20)
+VMCS_UNUSED(                        0x282A, _S64, _TG,  21)
+VMCS_UNUSED(                        0x282C, _S64, _TG,  22)
+VMCS_UNUSED(                        0x282E, _S64, _TG,  23)
+VMCS_UNUSED(                        0x2830, _S64, _TG,  24)
+VMCS_UNUSED(                        0x2832, _S64, _TG,  25)
+VMCS_UNUSED(                        0x2834, _S64, _TG,  26)
+VMCS_UNUSED(                        0x2836, _S64, _TG,  27)
+VMCS_UNUSED(                        0x2838, _S64, _TG,  28)
+VMCS_UNUSED(                        0x283A, _S64, _TG,  29)
+VMCS_UNUSED(                        0x283C, _S64, _TG,  30)
+VMCS_UNUSED(                        0x283E, _S64, _TG,  31)
+VMCS_GROUP_END(64, GUEST)
 
 /* 64-bit host state. */
-VMCS_FIELD(HOST_PAT,                0x2C00, FALSE)
-VMCS_FIELD(HOST_EFER,               0x2C02, TRUE)
-VMCS_FIELD(HOST_PGC,                0x2C04, TRUE)
+VMCS_GROUP_START(64, HOST)
+VMCS_FIELD(HOST_PAT,                0x2C00, _S64, _TH,  0, _NC,  _V, _NA)
+VMCS_FIELD(HOST_EFER,               0x2C02, _S64, _TH,  1, _NC,  _V, _NA)
+VMCS_FIELD(HOST_PGC,                0x2C04, _S64, _TH,  2, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x2C06, _S64, _TH,  3)
+VMCS_UNUSED(                        0x2C08, _S64, _TH,  4)
+VMCS_UNUSED(                        0x2C0A, _S64, _TH,  5)
+VMCS_UNUSED(                        0x2C0C, _S64, _TH,  6)
+VMCS_UNUSED(                        0x2C0E, _S64, _TH,  7)
+VMCS_UNUSED(                        0x2C10, _S64, _TH,  8)
+VMCS_UNUSED(                        0x2C12, _S64, _TH,  9)
+VMCS_UNUSED(                        0x2C14, _S64, _TH,  10)
+VMCS_UNUSED(                        0x2C16, _S64, _TH,  11)
+VMCS_UNUSED(                        0x2C18, _S64, _TH,  12)
+VMCS_UNUSED(                        0x2C1A, _S64, _TH,  13)
+VMCS_UNUSED(                        0x2C1C, _S64, _TH,  14)
+VMCS_UNUSED(                        0x2C1E, _S64, _TH,  15)
+VMCS_UNUSED(                        0x2C20, _S64, _TH,  16)
+VMCS_UNUSED(                        0x2C22, _S64, _TH,  17)
+VMCS_UNUSED(                        0x2C24, _S64, _TH,  18)
+VMCS_UNUSED(                        0x2C26, _S64, _TH,  19)
+VMCS_UNUSED(                        0x2C28, _S64, _TH,  20)
+VMCS_UNUSED(                        0x2C2A, _S64, _TH,  21)
+VMCS_UNUSED(                        0x2C2C, _S64, _TH,  22)
+VMCS_UNUSED(                        0x2C2E, _S64, _TH,  23)
+VMCS_UNUSED(                        0x2C30, _S64, _TH,  24)
+VMCS_UNUSED(                        0x2C32, _S64, _TH,  25)
+VMCS_UNUSED(                        0x2C34, _S64, _TH,  26)
+VMCS_UNUSED(                        0x2C36, _S64, _TH,  27)
+VMCS_UNUSED(                        0x2C38, _S64, _TH,  28)
+VMCS_UNUSED(                        0x2C3A, _S64, _TH,  29)
+VMCS_UNUSED(                        0x2C3C, _S64, _TH,  30)
+VMCS_UNUSED(                        0x2C3E, _S64, _TH,  31)
+VMCS_GROUP_END(64, HOST)
+VMCS_SET_END(64)
 
+
+VMCS_SET_START(32)
 /* 32-bit control fields. */
-VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, TRUE)
-VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, TRUE)
-VMCS_FIELD(XCP_BITMAP,              0x4004, TRUE)
-VMCS_FIELD(PF_ERR_MASK,             0x4006, TRUE)
-VMCS_FIELD(PF_ERR_MATCH,            0x4008, TRUE)
-VMCS_FIELD(CR3_TARG_COUNT,          0x400A, TRUE)
-VMCS_FIELD(VMEXIT_CTL,              0x400C, TRUE)
-VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, TRUE)
-VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, TRUE)
-VMCS_FIELD(VMENTRY_CTL,             0x4012, TRUE)
-VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, TRUE)
-VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, TRUE)
-VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, TRUE)
-VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, TRUE)
-VMCS_FIELD(TPR_THRESHOLD,           0x401C, TRUE)
-VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, TRUE)
-VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, FALSE)
-VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, FALSE)
+VMCS_GROUP_START(32, CTL)
+VMCS_FIELD(PIN_VMEXEC_CTL,          0x4000, _S32, _TC,  0,  _C,  _V, _NA)
+VMCS_FIELD(CPU_VMEXEC_CTL,          0x4002, _S32, _TC,  1,  _C,  _V, _NA)
+VMCS_FIELD(XCP_BITMAP,              0x4004, _S32, _TC,  2, _NC,  _V, _NA)
+VMCS_FIELD(PF_ERR_MASK,             0x4006, _S32, _TC,  3, _NC,  _V, _NA)
+VMCS_FIELD(PF_ERR_MATCH,            0x4008, _S32, _TC,  4, _NC,  _V, _NA)
+VMCS_FIELD(CR3_TARG_COUNT,          0x400A, _S32, _TC,  5, _NC,  _V, _NA)
+VMCS_FIELD(VMEXIT_CTL,              0x400C, _S32, _TC,  6, _NC,  _V, _NA)
+VMCS_FIELD(VMEXIT_MSR_STORE_COUNT,  0x400E, _S32, _TC,  7, _NC,  _V, _NA)
+VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT,   0x4010, _S32, _TC,  8, _NC,  _V, _NA)
+VMCS_FIELD(VMENTRY_CTL,             0x4012, _S32, _TC,  9, _NC,  _V, _NA)
+VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT,  0x4014, _S32, _TC, 10, _NC,  _V, _NA)
+VMCS_FIELD(VMENTRY_INTR_INFO,       0x4016, _S32, _TC, 11, _NC,  _V, _NA)
+VMCS_FIELD(VMENTRY_XCP_ERR,         0x4018, _S32, _TC, 12, _NC,  _V, _NA)
+VMCS_FIELD(VMENTRY_INSTR_LEN,       0x401A, _S32, _TC, 13, _NC,  _V, _NA)
+VMCS_FIELD(TPR_THRESHOLD,           0x401C, _S32, _TC, 14,  _C,  _V, _NA)
+VMCS_FIELD(2ND_VMEXEC_CTL,          0x401E, _S32, _TC, 15,  _C,  _V, _NA)
+VMCS_FIELD(PAUSE_LOOP_GAP,          0x4020, _S32, _TC, 16, _NC,  _V, _NA)
+VMCS_FIELD(PAUSE_LOOP_WINDOW,       0x4022, _S32, _TC, 17, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x4024, _S32, _TC, 18)
+VMCS_UNUSED(                        0x4026, _S32, _TC, 19)
+VMCS_UNUSED(                        0x4028, _S32, _TC, 20)
+VMCS_UNUSED(                        0x402A, _S32, _TC, 21)
+VMCS_UNUSED(                        0x402C, _S32, _TC, 22)
+VMCS_UNUSED(                        0x402E, _S32, _TC, 23)
+VMCS_UNUSED(                        0x4030, _S32, _TC, 24)
+VMCS_UNUSED(                        0x4032, _S32, _TC, 25)
+VMCS_UNUSED(                        0x4034, _S32, _TC, 26)
+VMCS_UNUSED(                        0x4036, _S32, _TC, 27)
+VMCS_UNUSED(                        0x4038, _S32, _TC, 28)
+VMCS_UNUSED(                        0x403A, _S32, _TC, 29)
+VMCS_UNUSED(                        0x403C, _S32, _TC, 30)
+VMCS_UNUSED(                        0x403E, _S32, _TC, 31)
+VMCS_GROUP_END(32, CTL)
 
 /* 32-bit read-only data fields. */
-VMCS_FIELD(VMINSTR_ERR,             0x4400, TRUE)
-VMCS_FIELD(EXIT_REASON,             0x4402, TRUE)
-VMCS_FIELD(EXIT_INTR_INFO,          0x4404, TRUE)
-VMCS_FIELD(EXIT_INTR_ERR,           0x4406, TRUE)
-VMCS_FIELD(IDTVEC_INFO,             0x4408, TRUE)
-VMCS_FIELD(IDTVEC_ERR,              0x440A, TRUE)
-VMCS_FIELD(INSTRLEN,                0x440C, TRUE)
-VMCS_FIELD(INSTR_INFO,              0x440E, TRUE)
+VMCS_GROUP_START(32, DATA)
+VMCS_FIELD(VMINSTR_ERR,             0x4400, _S32, _TD,  0, _NC,  _V, _NA)
+VMCS_FIELD(EXIT_REASON,             0x4402, _S32, _TD,  1, _NC,  _V, _RO)
+VMCS_FIELD(EXIT_INTR_INFO,          0x4404, _S32, _TD,  2,  _C,  _V, _RO)
+VMCS_FIELD(EXIT_INTR_ERR,           0x4406, _S32, _TD,  3, _NC,  _V, _RO)
+VMCS_FIELD(IDTVEC_INFO,             0x4408, _S32, _TD,  4, _NC,  _V, _RO)
+VMCS_FIELD(IDTVEC_ERR,              0x440A, _S32, _TD,  5, _NC,  _V, _RO)
+VMCS_FIELD(INSTRLEN,                0x440C, _S32, _TD,  6, _NC,  _V, _RO)
+VMCS_FIELD(INSTR_INFO,              0x440E, _S32, _TD,  7, _NC,  _V, _RO)
+VMCS_UNUSED(                        0x4410, _S32, _TD,  8)
+VMCS_UNUSED(                        0x4412, _S32, _TD,  9)
+VMCS_UNUSED(                        0x4414, _S32, _TD,  10)
+VMCS_UNUSED(                        0x4416, _S32, _TD,  11)
+VMCS_UNUSED(                        0x4418, _S32, _TD,  12)
+VMCS_UNUSED(                        0x441A, _S32, _TD,  13)
+VMCS_UNUSED(                        0x441C, _S32, _TD,  14)
+VMCS_UNUSED(                        0x441E, _S32, _TD,  15)
+VMCS_UNUSED(                        0x4420, _S32, _TD,  16)
+VMCS_UNUSED(                        0x4422, _S32, _TD,  17)
+VMCS_UNUSED(                        0x4424, _S32, _TD,  18)
+VMCS_UNUSED(                        0x4426, _S32, _TD,  19)
+VMCS_UNUSED(                        0x4428, _S32, _TD,  20)
+VMCS_UNUSED(                        0x442A, _S32, _TD,  21)
+VMCS_UNUSED(                        0x442C, _S32, _TD,  22)
+VMCS_UNUSED(                        0x442E, _S32, _TD,  23)
+VMCS_UNUSED(                        0x4430, _S32, _TD,  24)
+VMCS_UNUSED(                        0x4432, _S32, _TD,  25)
+VMCS_UNUSED(                        0x4434, _S32, _TD,  26)
+VMCS_UNUSED(                        0x4436, _S32, _TD,  27)
+VMCS_UNUSED(                        0x4438, _S32, _TD,  28)
+VMCS_UNUSED(                        0x443A, _S32, _TD,  29)
+VMCS_UNUSED(                        0x443C, _S32, _TD,  30)
+VMCS_UNUSED(                        0x443E, _S32, _TD,  31)
+VMCS_GROUP_END(32, DATA)
 
 /* 32-bit guest state. */
-VMCS_FIELD(ES_LIMIT,                0x4800, TRUE)
-VMCS_FIELD(CS_LIMIT,                0x4802, TRUE)
-VMCS_FIELD(SS_LIMIT,                0x4804, TRUE)
-VMCS_FIELD(DS_LIMIT,                0x4806, TRUE)
-VMCS_FIELD(FS_LIMIT,                0x4808, TRUE)
-VMCS_FIELD(GS_LIMIT,                0x480A, TRUE)
-VMCS_FIELD(LDTR_LIMIT,              0x480C, TRUE)
-VMCS_FIELD(TR_LIMIT,                0x480E, TRUE)
-VMCS_FIELD(GDTR_LIMIT,              0x4810, TRUE)
-VMCS_FIELD(IDTR_LIMIT,              0x4812, TRUE)
-VMCS_FIELD(ES_AR,                   0x4814, TRUE)
-VMCS_FIELD(CS_AR,                   0x4816, TRUE)
-VMCS_FIELD(SS_AR,                   0x4818, TRUE)
-VMCS_FIELD(DS_AR,                   0x481A, TRUE)
-VMCS_FIELD(FS_AR,                   0x481C, TRUE)
-VMCS_FIELD(GS_AR,                   0x481E, TRUE)
-VMCS_FIELD(LDTR_AR,                 0x4820, TRUE)
-VMCS_FIELD(TR_AR,                   0x4822, TRUE)
-VMCS_FIELD(HOLDOFF,                 0x4824, TRUE)
-VMCS_FIELD(ACTSTATE,                0x4826, TRUE)
-VMCS_FIELD(SMBASE,                  0x4828, TRUE)
-VMCS_FIELD(SYSENTER_CS,             0x482A, TRUE)
-VMCS_FIELD(TIMER,                   0x482E, FALSE)
+VMCS_GROUP_START(32, GUEST)
+VMCS_FIELD(ES_LIMIT,                0x4800, _S32, _TG,  0, _NC,  _V, _RW)
+VMCS_FIELD(CS_LIMIT,                0x4802, _S32, _TG,  1, _NC,  _V, _RW)
+VMCS_FIELD(SS_LIMIT,                0x4804, _S32, _TG,  2, _NC,  _V, _RW)
+VMCS_FIELD(DS_LIMIT,                0x4806, _S32, _TG,  3, _NC,  _V, _RW)
+VMCS_FIELD(FS_LIMIT,                0x4808, _S32, _TG,  4, _NC,  _V, _RW)
+VMCS_FIELD(GS_LIMIT,                0x480A, _S32, _TG,  5, _NC,  _V, _RW)
+VMCS_FIELD(LDTR_LIMIT,              0x480C, _S32, _TG,  6, _NC,  _V, _RW)
+VMCS_FIELD(TR_LIMIT,                0x480E, _S32, _TG,  7, _NC,  _V, _RW)
+VMCS_FIELD(GDTR_LIMIT,              0x4810, _S32, _TG,  8, _NC,  _V, _RW)
+VMCS_FIELD(IDTR_LIMIT,              0x4812, _S32, _TG,  9, _NC,  _V, _RW)
+VMCS_FIELD(ES_AR,                   0x4814, _S32, _TG, 10, _NC,  _V, _RW)
+VMCS_FIELD(CS_AR,                   0x4816, _S32, _TG, 11, _NC,  _V, _RW)
+VMCS_FIELD(SS_AR,                   0x4818, _S32, _TG, 12, _NC,  _V, _RW)
+VMCS_FIELD(DS_AR,                   0x481A, _S32, _TG, 13, _NC,  _V, _RW)
+VMCS_FIELD(FS_AR,                   0x481C, _S32, _TG, 14, _NC,  _V, _RW)
+VMCS_FIELD(GS_AR,                   0x481E, _S32, _TG, 15, _NC,  _V, _RW)
+VMCS_FIELD(LDTR_AR,                 0x4820, _S32, _TG, 16, _NC,  _V, _RW)
+VMCS_FIELD(TR_AR,                   0x4822, _S32, _TG, 17, _NC,  _V, _RW)
+VMCS_FIELD(HOLDOFF,                 0x4824, _S32, _TG, 18,  _C,  _V, _RW)
+VMCS_FIELD(ACTSTATE,                0x4826, _S32, _TG, 19, _NC,  _V, _NA)
+VMCS_FIELD(SMBASE,                  0x4828, _S32, _TG, 20, _NC,  _V, _NA)
+VMCS_FIELD(SYSENTER_CS,             0x482A, _S32, _TG, 21, _NC,  _V, _RW)
+VMCS_UNUSED(                        0x482C, _S32, _TG, 22)
+VMCS_FIELD(TIMER,                   0x482E, _S32, _TG, 23, _NC, _NV, _NA)
+VMCS_UNUSED(                        0x4830, _S32, _TG, 24)
+VMCS_UNUSED(                        0x4832, _S32, _TG, 25)
+VMCS_UNUSED(                        0x4834, _S32, _TG, 26)
+VMCS_UNUSED(                        0x4836, _S32, _TG, 27)
+VMCS_UNUSED(                        0x4838, _S32, _TG, 28)
+VMCS_UNUSED(                        0x483A, _S32, _TG, 29)
+VMCS_UNUSED(                        0x483C, _S32, _TG, 30)
+VMCS_UNUSED(                        0x483E, _S32, _TG, 31)
+
+VMCS_GROUP_END(32, GUEST)
 
 /* 32-bit host state. */
-VMCS_FIELD(HOST_SYSENTER_CS,        0x4C00, TRUE)
+VMCS_GROUP_START(32, HOST)
+VMCS_FIELD(HOST_SYSENTER_CS,        0x4C00, _S32, _TH,  0, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x4C02, _S32, _TH,  1)
+VMCS_UNUSED(                        0x4C04, _S32, _TH,  2)
+VMCS_UNUSED(                        0x4C06, _S32, _TH,  3)
+VMCS_UNUSED(                        0x4C08, _S32, _TH,  4)
+VMCS_UNUSED(                        0x4C0A, _S32, _TH,  5)
+VMCS_UNUSED(                        0x4C0C, _S32, _TH,  6)
+VMCS_UNUSED(                        0x4C0E, _S32, _TH,  7)
+VMCS_UNUSED(                        0x4C10, _S32, _TH,  8)
+VMCS_UNUSED(                        0x4C12, _S32, _TH,  9)
+VMCS_UNUSED(                        0x4C14, _S32, _TH,  10)
+VMCS_UNUSED(                        0x4C16, _S32, _TH,  11)
+VMCS_UNUSED(                        0x4C18, _S32, _TH,  12)
+VMCS_UNUSED(                        0x4C1A, _S32, _TH,  13)
+VMCS_UNUSED(                        0x4C1C, _S32, _TH,  14)
+VMCS_UNUSED(                        0x4C1E, _S32, _TH,  15)
+VMCS_UNUSED(                        0x4C20, _S32, _TH,  16)
+VMCS_UNUSED(                        0x4C22, _S32, _TH,  17)
+VMCS_UNUSED(                        0x4C24, _S32, _TH,  18)
+VMCS_UNUSED(                        0x4C26, _S32, _TH,  19)
+VMCS_UNUSED(                        0x4C28, _S32, _TH,  20)
+VMCS_UNUSED(                        0x4C2A, _S32, _TH,  21)
+VMCS_UNUSED(                        0x4C2C, _S32, _TH,  22)
+VMCS_UNUSED(                        0x4C2E, _S32, _TH,  23)
+VMCS_UNUSED(                        0x4C30, _S32, _TH,  24)
+VMCS_UNUSED(                        0x4C32, _S32, _TH,  25)
+VMCS_UNUSED(                        0x4C34, _S32, _TH,  26)
+VMCS_UNUSED(                        0x4C36, _S32, _TH,  27)
+VMCS_UNUSED(                        0x4C38, _S32, _TH,  28)
+VMCS_UNUSED(                        0x4C3A, _S32, _TH,  29)
+VMCS_UNUSED(                        0x4C3C, _S32, _TH,  30)
+VMCS_UNUSED(                        0x4C3E, _S32, _TH,  31)
+VMCS_GROUP_END(32, HOST)
+VMCS_SET_END(32)
 
+
+VMCS_SET_START(NAT)
 /* natural-width control fields. */
-VMCS_FIELD(CR0_GHMASK,              0x6000, TRUE)
-VMCS_FIELD(CR4_GHMASK,              0x6002, TRUE)
-VMCS_FIELD(CR0_SHADOW,              0x6004, TRUE)
-VMCS_FIELD(CR4_SHADOW,              0x6006, TRUE)
-VMCS_FIELD(CR3_TARGVAL0,            0x6008, TRUE)
-VMCS_FIELD(CR3_TARGVAL1,            0x600A, TRUE)
-VMCS_FIELD(CR3_TARGVAL2,            0x600C, TRUE)
-VMCS_FIELD(CR3_TARGVAL3,            0x600E, TRUE)
+VMCS_GROUP_START(NAT, CTL)
+VMCS_FIELD(CR0_GHMASK,              0x6000,  _SN, _TC,  0,  _C,  _V, _NA)
+VMCS_FIELD(CR4_GHMASK,              0x6002,  _SN, _TC,  1,  _C,  _V, _NA)
+VMCS_FIELD(CR0_SHADOW,              0x6004,  _SN, _TC,  2, _NC,  _V, _NA)
+VMCS_FIELD(CR4_SHADOW,              0x6006,  _SN, _TC,  3, _NC,  _V, _NA)
+VMCS_FIELD(CR3_TARGVAL0,            0x6008,  _SN, _TC,  4, _NC,  _V, _NA)
+VMCS_FIELD(CR3_TARGVAL1,            0x600A,  _SN, _TC,  5, _NC,  _V, _NA)
+VMCS_FIELD(CR3_TARGVAL2,            0x600C,  _SN, _TC,  6, _NC,  _V, _NA)
+VMCS_FIELD(CR3_TARGVAL3,            0x600E,  _SN, _TC,  7, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x6010,  _SN, _TC,  8)
+VMCS_UNUSED(                        0x6012,  _SN, _TC,  9)
+VMCS_UNUSED(                        0x6014,  _SN, _TC,  10)
+VMCS_UNUSED(                        0x6016,  _SN, _TC,  11)
+VMCS_UNUSED(                        0x6018,  _SN, _TC,  12)
+VMCS_UNUSED(                        0x601A,  _SN, _TC,  13)
+VMCS_UNUSED(                        0x601C,  _SN, _TC,  14)
+VMCS_UNUSED(                        0x601E,  _SN, _TC,  15)
+VMCS_UNUSED(                        0x6020,  _SN, _TC,  16)
+VMCS_UNUSED(                        0x6022,  _SN, _TC,  17)
+VMCS_UNUSED(                        0x6024,  _SN, _TC,  18)
+VMCS_UNUSED(                        0x6026,  _SN, _TC,  19)
+VMCS_UNUSED(                        0x6028,  _SN, _TC,  20)
+VMCS_UNUSED(                        0x602A,  _SN, _TC,  21)
+VMCS_UNUSED(                        0x602C,  _SN, _TC,  22)
+VMCS_UNUSED(                        0x602E,  _SN, _TC,  23)
+VMCS_UNUSED(                        0x6030,  _SN, _TC,  24)
+VMCS_UNUSED(                        0x6032,  _SN, _TC,  25)
+VMCS_UNUSED(                        0x6034,  _SN, _TC,  26)
+VMCS_UNUSED(                        0x6036,  _SN, _TC,  27)
+VMCS_UNUSED(                        0x6038,  _SN, _TC,  28)
+VMCS_UNUSED(                        0x603A,  _SN, _TC,  29)
+VMCS_UNUSED(                        0x603C,  _SN, _TC,  30)
+VMCS_UNUSED(                        0x603E,  _SN, _TC,  31)
+VMCS_GROUP_END(NAT, CTL)
 
-/* natural-width read-only data fields. */
-VMCS_FIELD(EXIT_QUAL,               0x6400, TRUE)
-VMCS_FIELD(IO_ECX,                  0x6402, TRUE)
-VMCS_FIELD(IO_ESI,                  0x6404, TRUE)
-VMCS_FIELD(IO_EDI,                  0x6406, TRUE)
-VMCS_FIELD(IO_EIP,                  0x6408, TRUE)
-VMCS_FIELD(LINEAR_ADDR,             0x640A, TRUE)
+VMCS_GROUP_START(NAT, DATA)
+VMCS_FIELD(EXIT_QUAL,               0x6400,  _SN, _TD,  0, _NC,  _V, _RO)
+VMCS_FIELD(IO_RCX,                  0x6402,  _SN, _TD,  1, _NC,  _V, _RO)
+VMCS_FIELD(IO_RSI,                  0x6404,  _SN, _TD,  2, _NC,  _V, _RO)
+VMCS_FIELD(IO_RDI,                  0x6406,  _SN, _TD,  3, _NC,  _V, _RO)
+VMCS_FIELD(IO_RIP,                  0x6408,  _SN, _TD,  4, _NC,  _V, _RO)
+VMCS_FIELD(LINEAR_ADDR,             0x640A,  _SN, _TD,  5, _NC,  _V, _RO)
+VMCS_UNUSED(                        0x640C,  _SN, _TD,  6)
+VMCS_UNUSED(                        0x640E,  _SN, _TD,  7)
+VMCS_UNUSED(                        0x6410,  _SN, _TD,  8)
+VMCS_UNUSED(                        0x6412,  _SN, _TD,  9)
+VMCS_UNUSED(                        0x6414,  _SN, _TD,  10)
+VMCS_UNUSED(                        0x6416,  _SN, _TD,  11)
+VMCS_UNUSED(                        0x6418,  _SN, _TD,  12)
+VMCS_UNUSED(                        0x641A,  _SN, _TD,  13)
+VMCS_UNUSED(                        0x641C,  _SN, _TD,  14)
+VMCS_UNUSED(                        0x641E,  _SN, _TD,  15)
+VMCS_UNUSED(                        0x6420,  _SN, _TD,  16)
+VMCS_UNUSED(                        0x6422,  _SN, _TD,  17)
+VMCS_UNUSED(                        0x6424,  _SN, _TD,  18)
+VMCS_UNUSED(                        0x6426,  _SN, _TD,  19)
+VMCS_UNUSED(                        0x6428,  _SN, _TD,  20)
+VMCS_UNUSED(                        0x642A,  _SN, _TD,  21)
+VMCS_UNUSED(                        0x642C,  _SN, _TD,  22)
+VMCS_UNUSED(                        0x642E,  _SN, _TD,  23)
+VMCS_UNUSED(                        0x6430,  _SN, _TD,  24)
+VMCS_UNUSED(                        0x6432,  _SN, _TD,  25)
+VMCS_UNUSED(                        0x6434,  _SN, _TD,  26)
+VMCS_UNUSED(                        0x6436,  _SN, _TD,  27)
+VMCS_UNUSED(                        0x6438,  _SN, _TD,  28)
+VMCS_UNUSED(                        0x643A,  _SN, _TD,  29)
+VMCS_UNUSED(                        0x643C,  _SN, _TD,  30)
+VMCS_UNUSED(                        0x643E,  _SN, _TD,  31)
+VMCS_GROUP_END(NAT, DATA)
 
 /* natural-width guest state. */
-VMCS_FIELD(CR0,                     0x6800, TRUE)
-VMCS_FIELD(CR3,                     0x6802, TRUE)
-VMCS_FIELD(CR4,                     0x6804, TRUE)
-VMCS_FIELD(ES_BASE,                 0x6806, TRUE)
-VMCS_FIELD(CS_BASE,                 0x6808, TRUE)
-VMCS_FIELD(SS_BASE,                 0x680A, TRUE)
-VMCS_FIELD(DS_BASE,                 0x680C, TRUE)
-VMCS_FIELD(FS_BASE,                 0x680E, TRUE)
-VMCS_FIELD(GS_BASE,                 0x6810, TRUE)
-VMCS_FIELD(LDTR_BASE,               0x6812, TRUE)
-VMCS_FIELD(TR_BASE,                 0x6814, TRUE)
-VMCS_FIELD(GDTR_BASE,               0x6816, TRUE)
-VMCS_FIELD(IDTR_BASE,               0x6818, TRUE)
-VMCS_FIELD(DR7,                     0x681A, TRUE)
-VMCS_FIELD(ESP,                     0x681C, TRUE)
-VMCS_FIELD(EIP,                     0x681E, TRUE)
-VMCS_FIELD(EFLAGS,                  0x6820, TRUE)
-VMCS_FIELD(PENDDBG,                 0x6822, TRUE)
-VMCS_FIELD(SYSENTER_ESP,            0x6824, TRUE)
-VMCS_FIELD(SYSENTER_EIP,            0x6826, TRUE)
+VMCS_GROUP_START(NAT, GUEST)
+VMCS_FIELD(CR0,                     0x6800,  _SN, _TG,  0, _NC,  _V, _NA)
+VMCS_FIELD(CR3,                     0x6802,  _SN, _TG,  1, _NC,  _V, _RW)
+VMCS_FIELD(CR4,                     0x6804,  _SN, _TG,  2, _NC,  _V, _RW)
+VMCS_FIELD(ES_BASE,                 0x6806,  _SN, _TG,  3, _NC,  _V, _RW)
+VMCS_FIELD(CS_BASE,                 0x6808,  _SN, _TG,  4, _NC,  _V, _RW)
+VMCS_FIELD(SS_BASE,                 0x680A,  _SN, _TG,  5, _NC,  _V, _RW)
+VMCS_FIELD(DS_BASE,                 0x680C,  _SN, _TG,  6, _NC,  _V, _RW)
+VMCS_FIELD(FS_BASE,                 0x680E,  _SN, _TG,  7, _NC,  _V, _RW)
+VMCS_FIELD(GS_BASE,                 0x6810,  _SN, _TG,  8, _NC,  _V, _RW)
+VMCS_FIELD(LDTR_BASE,               0x6812,  _SN, _TG,  9, _NC,  _V, _RW)
+VMCS_FIELD(TR_BASE,                 0x6814,  _SN, _TG, 10, _NC,  _V, _RW)
+VMCS_FIELD(GDTR_BASE,               0x6816,  _SN, _TG, 11, _NC,  _V, _RW)
+VMCS_FIELD(IDTR_BASE,               0x6818,  _SN, _TG, 12, _NC,  _V, _RW)
+VMCS_FIELD(DR7,                     0x681A,  _SN, _TG, 13, _NC,  _V, _NA)
+VMCS_FIELD(RSP,                     0x681C,  _SN, _TG, 14, _NC,  _V, _RW)
+VMCS_FIELD(RIP,                     0x681E,  _SN, _TG, 15, _NC,  _V, _RW)
+VMCS_FIELD(RFLAGS,                  0x6820,  _SN, _TG, 16, _NC,  _V, _RW)
+VMCS_FIELD(PENDDBG,                 0x6822,  _SN, _TG, 17, _NC,  _V, _RW)
+VMCS_FIELD(SYSENTER_ESP,            0x6824,  _SN, _TG, 18, _NC,  _V, _RW)
+VMCS_FIELD(SYSENTER_EIP,            0x6826,  _SN, _TG, 19, _NC,  _V, _RW)
+VMCS_UNUSED(                        0x6828,  _SN, _TG, 20)
+VMCS_UNUSED(                        0x682A,  _SN, _TG, 21)
+VMCS_UNUSED(                        0x682C,  _SN, _TG, 22)
+VMCS_UNUSED(                        0x682E,  _SN, _TG, 23)
+VMCS_UNUSED(                        0x6830,  _SN, _TG, 24)
+VMCS_UNUSED(                        0x6832,  _SN, _TG, 25)
+VMCS_UNUSED(                        0x6834,  _SN, _TG, 26)
+VMCS_UNUSED(                        0x6836,  _SN, _TG, 27)
+VMCS_UNUSED(                        0x6838,  _SN, _TG, 28)
+VMCS_UNUSED(                        0x683A,  _SN, _TG, 29)
+VMCS_UNUSED(                        0x683C,  _SN, _TG, 30)
+VMCS_UNUSED(                        0x683E,  _SN, _TG, 31)
+VMCS_GROUP_END(NAT, GUEST)
 
 /* natural-width host state. */
-VMCS_FIELD(HOST_CR0,                0x6C00, TRUE)
-VMCS_FIELD(HOST_CR3,                0x6C02, TRUE)
-VMCS_FIELD(HOST_CR4,                0x6C04, TRUE)
-VMCS_FIELD(HOST_FSBASE,             0x6C06, TRUE)
-VMCS_FIELD(HOST_GSBASE,             0x6C08, TRUE)
-VMCS_FIELD(HOST_TRBASE,             0x6C0A, TRUE)
-VMCS_FIELD(HOST_GDTRBASE,           0x6C0C, TRUE)
-VMCS_FIELD(HOST_IDTRBASE,           0x6C0E, TRUE)
-VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10, TRUE)
-VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12, TRUE)
-VMCS_FIELD(HOST_ESP,                0x6C14, TRUE)
-VMCS_FIELD(HOST_EIP,                0x6C16, TRUE)
+VMCS_GROUP_START(NAT, HOST)
+VMCS_FIELD(HOST_CR0,                0x6C00,  _SN, _TH,  0, _NC,  _V, _NA)
+VMCS_FIELD(HOST_CR3,                0x6C02,  _SN, _TH,  1, _NC,  _V, _NA)
+VMCS_FIELD(HOST_CR4,                0x6C04,  _SN, _TH,  2, _NC,  _V, _NA)
+VMCS_FIELD(HOST_FSBASE,             0x6C06,  _SN, _TH,  3, _NC,  _V, _NA)
+VMCS_FIELD(HOST_GSBASE,             0x6C08,  _SN, _TH,  4, _NC,  _V, _NA)
+VMCS_FIELD(HOST_TRBASE,             0x6C0A,  _SN, _TH,  5, _NC,  _V, _NA)
+VMCS_FIELD(HOST_GDTRBASE,           0x6C0C,  _SN, _TH,  6, _NC,  _V, _NA)
+VMCS_FIELD(HOST_IDTRBASE,           0x6C0E,  _SN, _TH,  7, _NC,  _V, _NA)
+VMCS_FIELD(HOST_SYSENTER_ESP,       0x6C10,  _SN, _TH,  8, _NC,  _V, _NA)
+VMCS_FIELD(HOST_SYSENTER_EIP,       0x6C12,  _SN, _TH,  9, _NC,  _V, _NA)
+VMCS_FIELD(HOST_RSP,                0x6C14,  _SN, _TH, 10, _NC,  _V, _NA)
+VMCS_FIELD(HOST_RIP,                0x6C16,  _SN, _TH, 11, _NC,  _V, _NA)
+VMCS_UNUSED(                        0x6C18,  _SN, _TH, 12)
+VMCS_UNUSED(                        0x6C1A,  _SN, _TH, 13)
+VMCS_UNUSED(                        0x6C1C,  _SN, _TH, 14)
+VMCS_UNUSED(                        0x6C1E,  _SN, _TH, 15)
+VMCS_UNUSED(                        0x6C20,  _SN, _TH, 16)
+VMCS_UNUSED(                        0x6C22,  _SN, _TH, 17)
+VMCS_UNUSED(                        0x6C24,  _SN, _TH, 18)
+VMCS_UNUSED(                        0x6C26,  _SN, _TH, 19)
+VMCS_UNUSED(                        0x6C28,  _SN, _TH, 20)
+VMCS_UNUSED(                        0x6C2A,  _SN, _TH, 21)
+VMCS_UNUSED(                        0x6C2C,  _SN, _TH, 22)
+VMCS_UNUSED(                        0x6C2E,  _SN, _TH, 23)
+VMCS_UNUSED(                        0x6C30,  _SN, _TH, 24)
+VMCS_UNUSED(                        0x6C32,  _SN, _TH, 25)
+VMCS_UNUSED(                        0x6C34,  _SN, _TH, 26)
+VMCS_UNUSED(                        0x6C36,  _SN, _TH, 27)
+VMCS_UNUSED(                        0x6C38,  _SN, _TH, 28)
+VMCS_UNUSED(                        0x6C3A,  _SN, _TH, 29)
+VMCS_UNUSED(                        0x6C3C,  _SN, _TH, 30)
+VMCS_UNUSED(                        0x6C3E,  _SN, _TH, 31)
+VMCS_GROUP_END(NAT, HOST)
+VMCS_SET_END(NAT)
+
+
+#ifdef LOCAL_GROUP_START
+#undef VMCS_GROUP_START
+#undef LOCAL_GROUP_START
+#endif
+
+#ifdef LOCAL_GROUP_END
+#undef VMCS_GROUP_END
+#undef LOCAL_GROUP_END
+#endif
+
+#ifdef LOCAL_SET_START
+#undef VMCS_SET_START
+#undef LOCAL_SET_START
+#endif
+
+#ifdef LOCAL_SET_END
+#undef VMCS_SET_END
+#undef LOCAL_SET_END
+#endif
+
+#ifdef LOCAL_UNUSED
+#undef VMCS_UNUSED
+#undef LOCAL_UNUSED
+#endif

vmmon-only/include/x86vt.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2004-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2004-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -90,6 +90,8 @@
    (CONST64U(1) << MSR_VMX_MISC_ACTSTATE_SHUTDOWN_SHIFT)
 #define MSR_VMX_MISC_ACTSTATE_SIPI              \
    (CONST64U(1) << MSR_VMX_MISC_ACTSTATE_SIPI_SHIFT)
+#define MSR_VMX_MISC_PROCESSOR_TRACE_IN_VMX     \
+   (CONST64U(1) << MSR_VMX_MISC_PROCESSOR_TRACE_IN_VMX_SHIFT)
 #define MSR_VMX_MISC_RDMSR_SMBASE_IN_SMM        \
    (CONST64U(1) << MSR_VMX_MISC_RDMSR_SMBASE_IN_SMM_SHIFT)
 #define MSR_VMX_MISC_ALLOW_ALL_VMWRITES         \
@@ -128,6 +130,8 @@
    (CONST64U(1) << MSR_VMX_EPT_VPID_INVVPID_ALL_CTX_SHIFT)
 #define MSR_VMX_EPT_VPID_INVVPID_VPID_CTX_LOCAL \
    (CONST64U(1) << MSR_VMX_EPT_VPID_INVVPID_VPID_CTX_LOCAL_SHIFT)
+#define MSR_VMX_EPT_VPID_ADV_EXIT_INFO          \
+   (CONST64U(1) << MSR_VMX_EPT_VPID_ADV_EXIT_INFO_SHIFT)
 
 #define VT_VMCS_STANDARD_TAG           0x00000000
 #define VT_VMCS_SHADOW_TAG             0x80000000
@@ -154,8 +158,14 @@
 #define VT_ENCODING_NUM_SIZES                   4
 #define VT_ENCODING_RSVD               0xffff9000
 
+/*
+ * The highest index of any currently defined field is 25, for
+ * TSC_MULTIPLIER.
+ */
+#define VT_ENCODING_MAX_INDEX                  25
+
 enum {
-#define VMCS_FIELD(name, encoding, vvt) VT_VMCS_##name = encoding,
+#define VMCS_FIELD(_name, _val, ...) VT_VMCS_##_name = _val,
 #include "x86vt-vmcs-fields.h"
 #undef VMCS_FIELD
 };
@@ -278,8 +288,12 @@ enum {
    VMX_CPU2(VMCS_SHADOW,        14)                      \
    VMX_CPU2(ENCL,               15)                      \
    VMX_CPU2(RDSEED,             16)                      \
+   VMX_CPU2(PML,                17)                      \
    VMX_CPU2(EPT_VIOL_VE,        18)                      \
-   VMX_CPU2(XSAVES,             20)
+   VMX_CPU2(PT_SUPPRESS_NR_BIT, 19)                      \
+   VMX_CPU2(XSAVES,             20)                      \
+   VMX_CPU2(EPT_MBX,            22)                      \
+   VMX_CPU2(TSC_SCALING,        25)
 
 #define VMX_PROCBASED_CTLS2_CAP                          \
         VMX_PROCBASED_CTLS2_CAP_NDA                      \
@@ -292,15 +306,17 @@ enum {
    VMXCTL(_EXIT_CTLS, _field, _pos)
 #define VMX_EXIT_CTLS_CAP_NDA
 #define VMX_EXIT_CTLS_CAP_PUB                            \
-   VMX_EXIT(SAVE_DEBUGCTL,       2)                      \
-   VMX_EXIT(LONGMODE,            9)                      \
-   VMX_EXIT(LOAD_PGC,           12)                      \
-   VMX_EXIT(INTRACK,            15)                      \
-   VMX_EXIT(SAVE_PAT,           18)                      \
-   VMX_EXIT(LOAD_PAT,           19)                      \
-   VMX_EXIT(SAVE_EFER,          20)                      \
-   VMX_EXIT(LOAD_EFER,          21)                      \
-   VMX_EXIT(SAVE_TIMER,         22)
+   VMX_EXIT(SAVE_DEBUGCTL,        2)                     \
+   VMX_EXIT(LONGMODE,             9)                     \
+   VMX_EXIT(LOAD_PGC,            12)                     \
+   VMX_EXIT(INTRACK,             15)                     \
+   VMX_EXIT(SAVE_PAT,            18)                     \
+   VMX_EXIT(LOAD_PAT,            19)                     \
+   VMX_EXIT(SAVE_EFER,           20)                     \
+   VMX_EXIT(LOAD_EFER,           21)                     \
+   VMX_EXIT(SAVE_TIMER,          22)                     \
+   VMX_EXIT(CLEAR_BNDCFGS,       23)                     \
+   VMX_EXIT(PT_SUPPRESS_VMX_PKT, 24)
 
 #define VMX_EXIT_CTLS_CAP                                \
         VMX_EXIT_CTLS_CAP_NDA                            \
@@ -313,13 +329,15 @@ enum {
    VMXCTL(_ENTRY_CTLS, _field, _pos)
 #define VMX_ENTRY_CTLS_CAP_NDA
 #define VMX_ENTRY_CTLS_CAP_PUB                           \
-   VMX_ENTRY(LOAD_DEBUGCTL,      2)                      \
-   VMX_ENTRY(LONGMODE,           9)                      \
-   VMX_ENTRY(ENTRY_TO_SMM,      10)                      \
-   VMX_ENTRY(SMM_TEARDOWN,      11)                      \
-   VMX_ENTRY(LOAD_PGC,          13)                      \
-   VMX_ENTRY(LOAD_PAT,          14)                      \
-   VMX_ENTRY(LOAD_EFER,         15)
+   VMX_ENTRY(LOAD_DEBUGCTL,        2)                    \
+   VMX_ENTRY(LONGMODE,             9)                    \
+   VMX_ENTRY(ENTRY_TO_SMM,        10)                    \
+   VMX_ENTRY(SMM_TEARDOWN,        11)                    \
+   VMX_ENTRY(LOAD_PGC,            13)                    \
+   VMX_ENTRY(LOAD_PAT,            14)                    \
+   VMX_ENTRY(LOAD_EFER,           15)                    \
+   VMX_ENTRY(LOAD_BNDCFGS,        16)                    \
+   VMX_ENTRY(PT_SUPPRESS_VMX_PKT, 17)
 
 #define VMX_ENTRY_CTLS_CAP                               \
         VMX_ENTRY_CTLS_CAP_NDA                           \
@@ -332,17 +350,18 @@ enum {
    VMXCAP(_MISC, _field, _pos, _len)
 #define VMX_MISC_CAP_NDA
 #define VMX_MISC_CAP_PUB                                 \
-   VMX_MISC(TMR_RATIO,              0,  5)               \
-   VMX_MISC(VMEXIT_SAVES_LMA,       5,  1)               \
-   VMX_MISC(ACTSTATE_HLT,           6,  1)               \
-   VMX_MISC(ACTSTATE_SHUTDOWN,      7,  1)               \
-   VMX_MISC(ACTSTATE_SIPI,          8,  1)               \
-   VMX_MISC(RDMSR_SMBASE_IN_SMM,   15,  1)               \
-   VMX_MISC(CR3_TARGETS,           16,  9)               \
-   VMX_MISC(MAX_MSRS,              25,  3)               \
-   VMX_MISC(ALLOW_ALL_VMWRITES,    29,  1)               \
-   VMX_MISC(ZERO_VMENTRY_INSTLEN,  30,  1)               \
-   VMX_MISC(MSEG_ID,               32, 32)               \
+   VMX_MISC(TMR_RATIO,               0,  5)              \
+   VMX_MISC(VMEXIT_SAVES_LMA,        5,  1)              \
+   VMX_MISC(ACTSTATE_HLT,            6,  1)              \
+   VMX_MISC(ACTSTATE_SHUTDOWN,       7,  1)              \
+   VMX_MISC(ACTSTATE_SIPI,           8,  1)              \
+   VMX_MISC(PROCESSOR_TRACE_IN_VMX, 14,  1)              \
+   VMX_MISC(RDMSR_SMBASE_IN_SMM,    15,  1)              \
+   VMX_MISC(CR3_TARGETS,            16,  9)              \
+   VMX_MISC(MAX_MSRS,               25,  3)              \
+   VMX_MISC(ALLOW_ALL_VMWRITES,     29,  1)              \
+   VMX_MISC(ZERO_VMENTRY_INSTLEN,   30,  1)              \
+   VMX_MISC(MSEG_ID,                32, 32)              \
 
 #define VMX_MISC_CAP                                     \
         VMX_MISC_CAP_NDA                                 \
@@ -427,6 +446,7 @@ enum {
    VMX_EPT(SP_1GB,                 17,  1)              \
    VMX_EPT(INVEPT,                 20,  1)              \
    VMX_EPT(ACCESS_DIRTY,           21,  1)              \
+   VMX_EPT(ADV_EXIT_INFO,          22,  1)              \
    VMX_EPT(INVEPT_EPT_CTX,         25,  1)              \
    VMX_EPT(INVEPT_GLOBAL,          26,  1)              \
    VMX_EPT(INVVPID,                32,  1)              \
@@ -532,68 +552,14 @@ enum {
 /* Exception error must-be-zero bits for VMEntry */
 #define VT_XCP_ERR_MBZ             0xffff8000
 
+
 /* Exit reasons. */
-#define VT_EXITREASON_EXC_OR_NMI            0
-#define VT_EXITREASON_EXTINT                1
-#define VT_EXITREASON_TRIPLEFAULT           2
-#define VT_EXITREASON_INIT                  3
-#define VT_EXITREASON_SIPI                  4
-#define VT_EXITREASON_IOSMI                 5
-#define VT_EXITREASON_OTHERSMI              6
-#define VT_EXITREASON_VINTR_WINDOW          7
-#define VT_EXITREASON_VNMI_WINDOW           8
-#define VT_EXITREASON_TS                    9
-#define VT_EXITREASON_CPUID                10
-#define VT_EXITREASON_GETSEC               11
-#define VT_EXITREASON_HLT                  12
-#define VT_EXITREASON_INVD                 13
-#define VT_EXITREASON_INVLPG               14
-#define VT_EXITREASON_RDPMC                15
-#define VT_EXITREASON_RDTSC                16
-#define VT_EXITREASON_RSM                  17
-#define VT_EXITREASON_VMCALL               18
-#define VT_EXITREASON_VMCLEAR              19
-#define VT_EXITREASON_VMLAUNCH             20
-#define VT_EXITREASON_VMPTRLD              21
-#define VT_EXITREASON_VMPTRST              22
-#define VT_EXITREASON_VMREAD               23
-#define VT_EXITREASON_VMRESUME             24
-#define VT_EXITREASON_VMWRITE              25
-#define VT_EXITREASON_VMXOFF               26
-#define VT_EXITREASON_VMXON                27
-#define VT_EXITREASON_CR                   28
-#define VT_EXITREASON_DR                   29
-#define VT_EXITREASON_IO                   30
-#define VT_EXITREASON_RDMSR                31
-#define VT_EXITREASON_WRMSR                32
-#define VT_EXITREASON_VMENTRYFAIL_GUEST   (33 | VT_EXITREASON_VMENTRYFAIL)
-#define VT_EXITREASON_VMENTRYFAIL_MSR     (34 | VT_EXITREASON_VMENTRYFAIL)
-#define VT_EXITREASON_MWAIT                36
-#define VT_EXITREASON_MTF                  37
-#define VT_EXITREASON_MONITOR              39
-#define VT_EXITREASON_PAUSE                40
-#define VT_EXITREASON_VMENTRYFAIL_MC      (41 | VT_EXITREASON_VMENTRYFAIL)
-#define VT_EXITREASON_TPR                  43
-#define VT_EXITREASON_APIC                 44
-#define VT_EXITREASON_EOI                  45
-#define VT_EXITREASON_GDTR_IDTR            46
-#define VT_EXITREASON_LDTR_TR              47
-#define VT_EXITREASON_EPT_VIOLATION        48
-#define VT_EXITREASON_EPT_MISCONFIG        49
-#define VT_EXITREASON_INVEPT               50
-#define VT_EXITREASON_RDTSCP               51
-#define VT_EXITREASON_TIMER                52
-#define VT_EXITREASON_INVVPID              53
-#define VT_EXITREASON_WBINVD               54
-#define VT_EXITREASON_XSETBV               55
-#define VT_EXITREASON_APIC_WRITE           56
-#define VT_EXITREASON_RDRAND               57
-#define VT_EXITREASON_INVPCID              58
-#define VT_EXITREASON_VMFUNC               59
-#define VT_EXITREASON_ENCLS                60
-#define VT_EXITREASON_RDSEED               61
-#define VT_EXITREASON_XSAVES               63
-#define VT_EXITREASON_XRSTORS              64
+enum {
+#define VT_EXIT(_name, _val) VT_EXITREASON_##_name = _val,
+#include "x86vt-exit-reasons.h"
+#undef VT_EXIT
+};
+
 
 /*
  * VT synthesized exit reasons:
@@ -618,7 +584,6 @@ enum {
         (VT_EXITREASON_SYNTH_EXC_BASE + gatenum) /* 0-31 */
 
 #define VT_EXITREASON_INSIDE_ENCLAVE        (1U << 27)
-#define VT_EXITREASON_VMENTRYFAIL           (1U << 31)
 
 /* Instruction error codes. */
 #define VT_ERROR_VMCALL_VMX_ROOT            1
@@ -690,12 +655,18 @@ enum {
 #define VT_EPT_QUAL_ACCESS_W           (1 << 1)
 #define VT_EPT_QUAL_ACCESS_X           (1 << 2)
 #define VT_EPT_QUAL_PROT_SHIFT         3
-#define VT_EPT_QUAL_PROT_MASK          (0x7 << VT_EPT_QUAL_PROT_SHIFT)
+#define VT_EPT_QUAL_PROT_MASK(_mbx)    (((_mbx) ? 0xf : 0x7) << \
+                                                  VT_EPT_QUAL_PROT_SHIFT)
 #define VT_EPT_QUAL_PROT_R             (1 << 3)
 #define VT_EPT_QUAL_PROT_W             (1 << 4)
 #define VT_EPT_QUAL_PROT_X             (1 << 5)
+#define VT_EPT_QUAL_PROT_XS            (1 << 5)
+#define VT_EPT_QUAL_PROT_XU            (1 << 6)
 #define VT_EPT_QUAL_LA_VALID           (1 << 7)
 #define VT_EPT_QUAL_FINAL_ADDR         (1 << 8)
+#define VT_EPT_QUAL_GUEST_US           (1 << 9)
+#define VT_EPT_QUAL_GUEST_RW           (1 << 10)
+#define VT_EPT_QUAL_GUEST_NX           (1 << 11)
 #define VT_EPT_QUAL_NMIUNMASK          (1 << 12)
 
 
@@ -772,12 +743,120 @@ enum {
     MSR_VMX_EPT_VPID_INVVPID_ALL_CTX)
 
 #define VT_REQUIRED_EPT_SUPPORT                        \
-   (MSR_VMX_EPT_VPID_EPTE_X                          | \
-    MSR_VMX_EPT_VPID_GAW_48                          | \
+   (MSR_VMX_EPT_VPID_GAW_48                          | \
     MSR_VMX_EPT_VPID_ETMT_WB                         | \
     MSR_VMX_EPT_VPID_SP_2MB                          | \
-    MSR_VMX_EPT_VPID_INVEPT                          | \
-    MSR_VMX_EPT_VPID_INVEPT_EPT_CTX)
+    MSR_VMX_EPT_VPID_INVEPT)
+
+#define VT_TSQUAL_CALL  0
+#define VT_TSQUAL_IRET  1
+#define VT_TSQUAL_JMP   2
+#define VT_TSQUAL_GATE  3
+
+typedef union {
+   struct {
+      unsigned selVal:16;
+      unsigned rsvd0:14;
+      unsigned source:2;
+   } bits;
+   uint32 flat;
+} VTTSQualifier;
+
+#define VT_CRQUAL_WR    0
+#define VT_CRQUAL_RD    1
+#define VT_CRQUAL_CLTS  2
+#define VT_CRQUAL_LMSW  3
+
+/* Control register intercept qualifier */
+typedef union {
+   struct {
+      unsigned num:4;   // For MOV-CR
+      unsigned op:2;
+      unsigned mem:1;   // For LMSW
+      unsigned rsvd0:1;
+      unsigned gpr:4;   // For MOV-CR
+      unsigned rsvd1:4;
+      unsigned data:16; // For LMSW
+   } bits;
+   uint32 flat;
+} VTCRQualifier;
+
+#define VT_DRQUAL_WR    0
+#define VT_DRQUAL_RD    1
+
+/* Debug register intercept qualifier */
+typedef union {
+   struct {
+      unsigned num:3;
+      unsigned rsvd0:1;
+      unsigned op:1;
+      unsigned rsvd1:3;
+      unsigned gpr:4;
+      unsigned rsvd2:20;
+   } bits;
+   uint32 flat;
+} VTDRQualifier;
+
+#define VT_IOQUAL_SZ8     0
+#define VT_IOQUAL_SZ16    1
+#define VT_IOQUAL_SZ32    3
+
+/* I/O intercept qualifier */
+typedef union {
+   struct {
+      unsigned opSize:3; // 0 = 1-byte; 1 = 2-byte; 3 = 4-byte
+      unsigned in:1;
+      unsigned string:1;
+      unsigned rep:1;
+      unsigned imm:1;
+      unsigned rsvd0:9;
+      unsigned port:16;
+   } bits;
+   uint32 flat;
+} VTIOQualifier;
+
+#define VT_IINFO_SCALE1    0
+#define VT_IINFO_SCALE2    1
+#define VT_IINFO_SCALE4    2
+#define VT_IINFO_SCALE8    3
+
+#define VT_IINFO_SZ16      0
+#define VT_IINFO_SZ32      1
+#define VT_IINFO_SZ64      2
+
+#define VT_IINFO_SGDT      0
+#define VT_IINFO_SIDT      1
+#define VT_IINFO_LGDT      2
+#define VT_IINFO_LIDT      3
+
+#define VT_IINFO_SLDT      0
+#define VT_IINFO_STR       1
+#define VT_IINFO_LLDT      2
+#define VT_IINFO_LTR       3
+
+/* VM-Exit Instruction-Information */
+typedef union {
+   struct {
+      unsigned scale:2;    // Bits  1:0
+      unsigned rsvd0:1;    // Bit   2
+      unsigned reg1:4;     // Bits  6:3
+      unsigned aSize:3;    // Bits  9:7
+      unsigned modrmReg:1; // Bit  10
+      unsigned oSize:2;    // Bits 12:11
+      unsigned rsvd1:2;    // Bit  14:13
+      unsigned seg:3;      // Bits 17:15
+      unsigned indexReg:5; // Bits 22:18
+      unsigned baseReg:5;  // Bits 27:23
+      unsigned misc:4;     // Bits 31:28
+   } bits;
+   uint32 flat;
+} VTInstrInfo;
+
+typedef struct VTMSREntry {
+   uint32       index;
+   uint32       reserved;
+   uint64       data;
+} VTMSREntry;
 
 typedef uint64 VTConfig[NUM_VMX_MSRS];
 
@@ -941,6 +1020,25 @@ VT_RealModeSupportedFromFeatures(uint64 secondary)
 }
 
 
+/*
+ *----------------------------------------------------------------------
+ *
+ * VT_MBXSupportedFromFeatures --
+ *
+ *   Returns TRUE if the given VMX features provide support for
+ *   mode-based execute control for EPT.
+ *
+ *   Assumes that VT is supported.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE Bool
+VT_MBXSupportedFromFeatures(uint64 secondary)
+{
+   return (HIDWORD(secondary) & VT_VMCS_2ND_VMEXEC_CTL_EPT_MBX) != 0;
+}
+
+
 #if !defined(USERLEVEL) && !defined(MONITOR_APP) /* { */
 /*
  *----------------------------------------------------------------------

vmmon-only/include/x86vtinstr.h

@@ -34,6 +34,11 @@
 #include "x86_basic_defs.h"
 #include "community_source.h"
 
+#if defined __cplusplus
+extern "C" {
+#endif
+
+
 #define INVALID_VMCS_ADDR    ~0ULL
 
 /*
@@ -67,23 +72,23 @@ VMXStatus_Valid(VMXStatus status)
 #if defined __GNUC__
 
 static INLINE VMXStatus
-VMXON_2_STATUS(MA* vmcs)
+VMXON_2_STATUS(MA* vmxonRegion)
 {
    VMXStatus status;
    __asm__ __volatile__("vmxon %1; lahf; movzbl %%ah, %0"
                         : "=a"(status)
-                        : "m"(*vmcs)
+                        : "m"(*vmxonRegion)
                         : "cc", "memory");
    ASSERT(VMXStatus_Valid(status));
    return status;
 }
 
 static INLINE void
-VMXON_UNCHECKED(MA *vmcs)
+VMXON_UNCHECKED(MA *vmxonRegion)
 {
    __asm__ __volatile__("vmxon %0"
                         :
-                        : "m"(*vmcs)
+                        : "m"(*vmxonRegion)
                         : "cc", "memory");
 }
 
@@ -129,18 +134,6 @@ VMCLEAR_UNCHECKED(MA* vmcs)
                         : "cc", "memory");
 }
 
-static INLINE void
-VMCLEAR(MA* vmcs)
-{
-   if (vmx86_debug) {
-      VMXStatus status;
-      status = VMCLEAR_2_STATUS(vmcs);
-      ASSERT(status == VMX_Success);
-   } else {
-      VMCLEAR_UNCHECKED(vmcs);
-   }
-}
-
 static INLINE VMXStatus
 VMPTRLD_2_STATUS(MA *vmcs)
 {
@@ -208,20 +201,6 @@ VMREAD_UNCHECKED(size_t encoding)
    return retval;
 }
 
-static INLINE size_t
-VMREAD(size_t encoding)
-{
-   size_t retval;
-   if (vmx86_debug) {
-      VMXStatus status;
-      status = VMREAD_2_STATUS(encoding, &retval);
-      ASSERT(status == VMX_Success);
-   } else {
-      retval = VMREAD_UNCHECKED(encoding);
-   }
-   return retval;
-}
-
 static INLINE VMXStatus
 VMWRITE_2_STATUS(size_t encoding, size_t val)
 {
@@ -507,21 +486,21 @@ void          __vmx_vmptrst(unsigned __int64 *);
 
 
 static INLINE VMXStatus
-VMXON_2_STATUS(MA *vmcs)
+VMXON_2_STATUS(MA *vmxonRegion)
 {
    unsigned char mscStatus;
    static const VMXStatus MscToStatus[] =
       {VMX_Success, VMX_FailValid, VMX_FailInvalid};
 
-   mscStatus = __vmx_on((unsigned __int64 *)vmcs);
+   mscStatus = __vmx_on((unsigned __int64 *)vmxonRegion);
    ASSERT(mscStatus < ARRAYSIZE(MscToStatus));
    return MscToStatus[mscStatus];
 }
 
 static INLINE void
-VMXON_UNCHECKED(MA *vmcs)
+VMXON_UNCHECKED(MA *vmxonRegion)
 {
-   (void)__vmx_on((unsigned __int64 *)vmcs);
+   (void)__vmx_on((unsigned __int64 *)vmxonRegion);
 }
 
 static INLINE VMXStatus
@@ -537,6 +516,45 @@ VMXOFF_UNCHECKED(void)
    (void)__vmx_off();
 }
 
+static INLINE void
+VMCLEAR_UNCHECKED(MA *vmcs)
+{
+   (void)__vmx_vmclear((unsigned __int64 *)vmcs);
+}
+
+static INLINE size_t
+VMREAD_UNCHECKED(size_t encoding)
+{
+   size_t retval;
+   (void)__vmx_vmread(encoding, &retval);
+   return retval;
+}
+
+static INLINE VMXStatus
+VMCLEAR_2_STATUS(MA *vmcs)
+{
+   unsigned char mscStatus;
+   static const VMXStatus MscToStatus[] =
+      {VMX_Success, VMX_FailValid, VMX_FailInvalid};
+
+   mscStatus = __vmx_vmclear((unsigned __int64 *)vmcs);
+   ASSERT(mscStatus < ARRAYSIZE(MscToStatus));
+   return MscToStatus[mscStatus];
+}
+ 
+static INLINE VMXStatus
+VMREAD_2_STATUS(size_t encoding, size_t *retval)
+{
+   unsigned char mscStatus;
+   static const VMXStatus MscToStatus[] =
+      {VMX_Success, VMX_FailValid, VMX_FailInvalid};
+ 
+   mscStatus = __vmx_vmread(encoding, retval);
+   ASSERT(mscStatus < ARRAYSIZE(MscToStatus));
+   return MscToStatus[mscStatus];
+}
+
+
 static INLINE VMXStatus
 VMPTRLD_2_STATUS(MA *vmcs)
 {
@@ -573,14 +591,14 @@ VMPTRST_UNCHECKED(MA *vmcs)
 
 
 static INLINE void
-VMXON(MA *vmcs)
+VMXON(MA *vmxonRegion)
 {
    if (vmx86_debug) {
       VMXStatus status;
-      status = VMXON_2_STATUS(vmcs);
+      status = VMXON_2_STATUS(vmxonRegion);
       ASSERT(status == VMX_Success);
    } else {
-      VMXON_UNCHECKED(vmcs); 
+      VMXON_UNCHECKED(vmxonRegion); 
    }
 }
 
@@ -620,4 +638,36 @@ VMPTRST(MA *vmcs)
    }
 }
 
+
+static INLINE void
+VMCLEAR(MA* vmcs)
+{
+   if (vmx86_debug) {
+      VMXStatus status;
+      status = VMCLEAR_2_STATUS(vmcs);
+      ASSERT(status == VMX_Success);
+   } else {
+      VMCLEAR_UNCHECKED(vmcs);
+   }
+}
+
+static INLINE size_t
+VMREAD(size_t encoding)
+{
+   size_t retval;
+   if (vmx86_debug) {
+      VMXStatus status;
+      status = VMREAD_2_STATUS(encoding, &retval);
+      ASSERT(status == VMX_Success);
+   } else {
+      retval = VMREAD_UNCHECKED(encoding);
+   }
+   return retval;
+}
+
+
+#if defined __cplusplus
+}  // extern "C"
+#endif
+
 #endif /* _X86VTINSTR_H_ */

vmmon-only/linux/driver.h

@@ -26,10 +26,10 @@
 #include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
+#include <linux/semaphore.h>
 #include <linux/wait.h>
 
 #include "vmx86.h"
-#include "compat_semaphore.h"
 #include "driver_vmcore.h"
 
 
@@ -51,7 +51,7 @@ typedef struct VMMappedUserMem {
 struct VMDriver;
 
 /* 16 pages (64KB) looks as a good limit for one allocation */
-#define VMMON_MAX_LOWMEM_PAGES	16
+#define VMMON_MAX_LOWMEM_PAGES  16
 
 typedef struct VMLinux {
    struct VMLinux *next;
@@ -66,17 +66,6 @@ typedef struct VMLinux {
    struct semaphore lock4Gb;
    unsigned int size4Gb;
    struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
-
-   /*
-    * LinuxDriverPoll() support
-    */
-
-   wait_queue_head_t pollQueue;
-   volatile uint32 *pollTimeoutPtr;
-   VMMappedUserMem *pollTimeoutHandle;
-   VmTimeType pollTime;
-   struct VMLinux *pollForw;
-   struct VMLinux **pollBack;
 } VMLinux;
 
 
@@ -95,19 +84,8 @@ typedef struct VMXLinuxState {
    char buf[LINUXLOG_BUFFER_SIZE];
    VMLinux *head;
 
-   /*
-    * for LinuxDriverPoll()
-    */
-
-   struct timer_list pollTimer;
-   wait_queue_head_t pollQueue;
-
-   struct VMLinux *pollList;
-
    struct task_struct *fastClockThread;
-   struct file *fastClockFile;
    unsigned fastClockRate;
-   long fastClockPriority;
    uint64 swapSize;
 } VMXLinuxState;
 

vmmon-only/linux/hostif_priv.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2014 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -25,14 +25,6 @@
 #ifndef _HOSTIF_PRIV_H_
 #define _HOSTIF_PRIV_H_
 
-/* Functions for transferring data to/from userspace. */
-EXTERN int    HostIF_CopyFromUser(void *dst, const void *src, unsigned int len);
-EXTERN int    HostIF_CopyToUser(void *dst, const void *src, unsigned int len);
-
-/* Functions for dealing with the poll list lock. */
-EXTERN void   HostIF_PollListLock(int callerID);
-EXTERN void   HostIF_PollListUnlock(int callerID);
-
 /* Functions for mapping and unmapping userspace memory. */
 struct VMMappedUserMem;
 EXTERN void  *HostIF_MapUserMem(VA addr, size_t size,

vmmon-only/linux/vmhost.h

@@ -23,7 +23,7 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"
 
-#include "compat_semaphore.h"
+#include <linux/semaphore.h>
 
 
 #ifdef VMX86_DEBUG
@@ -73,7 +73,7 @@ typedef struct Mutex {
  */
 
 typedef struct VMHost {
-   /* 
+   /*
     * Used for shared modifications to VM's VMDriver data, mostly page locking.
     * It has higher rank than the global mutex.
     */
@@ -84,15 +84,15 @@ typedef struct VMHost {
    struct task_struct *vcpuSemaTask[MAX_INITBLOCK_CPUS];
    /*
     * Pages that were allocated/mapped by VMX and locked by the driver and
-    * don't have a particular VA. 
+    * don't have a particular VA.
     */
    struct PhysTracker *lockedPages;
    /*
-    * Locked pages that were allocated by the driver and don't have 
+    * Locked pages that were allocated by the driver and don't have
     * a particular VA. They are used as monitor anonymous pages or
     * as pages for "AWE" guest memory.
     */
-   struct PhysTracker *AWEPages; 
+   struct PhysTracker *AWEPages;
    /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */
    Bool               hostAPICIsMapped;
 } VMHost;

vmmon-only/linux/vmmonInt.h

@@ -1,109 +0,0 @@
-/*********************************************************
- * Copyright (C) 1998,2015 VMware, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation version 2 and no later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- *
- *********************************************************/
-
-#ifndef __VMMONINT_H__
-#define __VMMONINT_H__
-
-#define INCLUDE_ALLOW_VMMON
-#define INCLUDE_ALLOW_VMCORE
-#include "includeCheck.h"
-
-
-/*
- * Hide all kernel compatibility stuff in these macros and functions.
- */
-
-#ifdef VMW_HAVE_SMP_CALL_3ARG
-#define compat_smp_call_function(fn, info, wait) smp_call_function(fn, info, wait)
-#else
-#define compat_smp_call_function(fn, info, wait) smp_call_function(fn, info, 1, wait)
-#endif
-
-
-/*
- *----------------------------------------------------------------------
- *
- * compat_tsc_khz --
- *
- *    Compatibility wrapper for tsc_khz.
- *
- * Returns:
- *
- *    Returns 0 if symbol is not exported by the kernel, else
- *    value of tsc_khz.
- *
- *----------------------------------------------------------------------
- */
-
-static inline unsigned int
-compat_tsc_khz(void)
-{
-#if defined(VMW_HAVE_TSC_KHZ)
-   return tsc_khz;
-#else
-   return 0;
-#endif
-}
-
-
-/*
- *----------------------------------------------------------------------
- *
- * compat_smp_call_function_single --
- *
- *    Compatibility wrapper for calling smp_call_function_single.
- *    Versions prior to 2.6.20 did not export the symbol for both
- *    i386 and x86_64 kernels.
- *
- * Returns:
- *
- *    Returns -ENOSYS if the host kernel does not implement or export
- *    the function, else returns the error status of
- *    smp_call_function_single.
- *
- *----------------------------------------------------------------------
- */
-
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20)
-#   ifdef VMW_HAVE_SMP_CALL_3ARG
-#      define compat_smp_call_function_single(cpu, fn, info, wait) \
-              smp_call_function_single(cpu, fn, info, wait)
-#   else
-#      define compat_smp_call_function_single(cpu, fn, info, wait) \
-              smp_call_function_single(cpu, fn, info, 1, wait)
-#   endif // VMW_HAVE_SMP_CALL_3ARG
-#else // VERSION >= 2.6.20
-#      define compat_smp_call_function_single(cpu, fn, info, wait) (-ENOSYS)
-#endif
-
-
-/*
- * Although this is not really related to kernel-compatibility, I put this
- * helper macro here for now for a lack of better place --hpreg
- *
- * The exit(2) path does, in this order:
- * . set current->files to NULL
- * . close all fds, which potentially calls LinuxDriver_Close()
- *
- * fget() requires current->files != NULL, so we must explicitely check --hpreg
- */
-#define vmware_fget(_fd) (current->files ? fget(_fd) : NULL)
-
-extern void LinuxDriverWakeUp(Bool selective);
-
-#endif /* __VMMONINT_H__ */

vmmon-only/vmcore/moduleloop.c

@@ -41,6 +41,7 @@
 #include "memtrack.h"
 #include "driver_vmcore.h"
 #include "usercalldefs.h"
+#include "cpuid.h"
 
 /*
  *----------------------------------------------------------------------
@@ -74,7 +75,7 @@ Vmx86_RunVM(VMDriver *vm,   // IN:
    VMCrossPageData *crosspage = &vm->crosspage[vcpuid]->crosspageData;
    int              bailValue = 0;
 
-   ASSERT(crosspage);
+   ASSERT(crosspage && CPUID_HostSupportsHV());
 
    /*
     * Check if we were interrupted by signal.
@@ -172,15 +173,13 @@ skipTaskSwitch:;
       }
 
       case MODULECALL_SEMAFORCEWAKEUP: {
-         ASSERT_ON_COMPILE(sizeof(VCPUSet) <= sizeof(crosspage->args));
-         HostIF_SemaphoreForceWakeup(vm, (VCPUSet *) &crosspage->args[0]);
+         HostIF_SemaphoreForceWakeup(vm, &crosspage->vcpuSet);
          break;
       }
 
       case MODULECALL_IPI: {
          HostIFIPIMode mode;
-         ASSERT_ON_COMPILE(sizeof(VCPUSet) <= sizeof(crosspage->args));
-         mode = HostIF_IPI(vm, (VCPUSet *) &crosspage->args[0]);
+         mode = HostIF_IPI(vm, &crosspage->vcpuSet);
          retval = (mode != IPI_NONE);
          break;
       }
@@ -225,11 +224,9 @@ skipTaskSwitch:;
       }
 
       case MODULECALL_COSCHED: {
-         uint32 spinUS = (uint32)crosspage->args[2];
-         ASSERT_ON_COMPILE(sizeof(VCPUSet) + sizeof(uint32) <=
-                           sizeof(crosspage->args));
-         Vmx86_YieldToSet(vm, vcpuid, (VCPUSet *) &crosspage->args[0],
-                          spinUS, FALSE);
+         uint32 spinUS = (uint32)crosspage->args[0];
+         ASSERT_ON_COMPILE(sizeof(uint32) <= sizeof(crosspage->args));
+         Vmx86_YieldToSet(vm, vcpuid, &crosspage->vcpuSet, spinUS, FALSE);
          break;
       }
 
@@ -251,6 +248,18 @@ skipTaskSwitch:;
          retval = crosspage->retval;
       } break;
 
+      case MODULECALL_VMCLEAR_VMCS_ALL_CPUS: {
+         MA vmcs = (MA)crosspage->args[0];
+         Vmx86_FlushVMCSAllCPUs(vmcs);
+      } break;
+
+      case MODULECALL_GET_PAGE_ROOT: {
+         MPN mpn;
+         Vcpuid targetVcpuid = (Vcpuid)crosspage->args[0];
+         retval = Vmx86_GetPageRoot(vm, targetVcpuid, &mpn);
+         crosspage->args[0] = mpn;
+      } break;
+
       default:
          Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
       }

vmnet-only/Makefile

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 ##########################################################
-# Copyright (C) 1998-2015 VMware, Inc. All rights reserved.
+# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -84,8 +84,6 @@ endif
 
 ifneq ($(VM_KBUILD), no)
 
-VMCCVER := $(shell $(CC) -dumpversion)
-
 # If there is no version defined, we are in toplevel pass, not yet in kernel makefiles...
 ifeq ($(VERSION),)
 
@@ -100,14 +98,6 @@ auto-build: $(DRIVER_KO)
 $(DRIVER): $(DRIVER_KO)
 	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi
 
-# Pass gcc version down the chain, so we can detect if kernel attempts to use unapproved compiler
-VM_CCVER := $(VMCCVER)
-export VM_CCVER
-VM_CC := $(CC)
-export VM_CC
-
-MAKEOVERRIDES := $(filter-out CC=%,$(MAKEOVERRIDES))
-
 #
 # Define a setup target that gets built before the actual driver.
 # This target may not be used at all, but if it is then it will be defined

vmnet-only/Makefile.kernel

@@ -20,10 +20,7 @@
 INCLUDE := -I$(SRCROOT)
 
 EXTRA_CFLAGS := $(CC_OPTS) $(INCLUDE)
-EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/netdev_has_net.c,-DVMW_NETDEV_HAS_NET, )
-EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/netdev_has_dev_net.c,-DVMW_NETDEV_HAS_DEV_NET, )
-EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/nfhook_uses_skb.c,-DVMW_NFHOOK_USES_SKB, )
-EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/skblin.c,  -DVMW_SKB_LINEARIZE_2618, )
+EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/netif_trans_update.c,-DVMW_NETIF_TRANS_UPDATE, )
 
 obj-m += $(DRIVER).o
 

vmnet-only/Makefile.normal

@@ -46,10 +46,6 @@ INCLUDE := -I. -I$(HEADER_DIR)
 INCLUDE += $(shell $(CC) $(INCLUDE) -E geninclude.c \
 	   | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p')
 
-CC_OPTS += $(call vm_check_build, $(SRCROOT)/netdev_has_net.c,-DVMW_NETDEV_HAS_NET, )
-CC_OPTS += $(call vm_check_build, $(SRCROOT)/netdev_has_dev_net.c,-DVMW_NETDEV_HAS_DEV_NET, )
-CC_OPTS += $(call vm_check_build, $(SRCROOT)/skblin.c, -DVMW_SKB_LINEARIZE_2618, )
-
 CFLAGS := -O $(CC_WARNINGS) $(CC_OPTS) $(INCLUDE) $(GLOBAL_DEFS)
 
 OBJS := driver.o hub.o userif.o netif.o bridge.o procfs.o smac_compat.o \

vmnet-only/bridge.c

@@ -41,6 +41,7 @@
 #include <linux/file.h>
 #include <linux/ip.h>
 #include <linux/tcp.h>
+#include <linux/if_arp.h>
 #include <net/tcp.h>
 #include <net/ipv6.h>
 
@@ -77,7 +78,6 @@ struct VNetBridge {
    struct sock             *sk;             // socket associated with skb's
    struct packet_type       pt;             // used to add packet handler
    Bool                     enabledPromisc; // track if promisc enabled
-   Bool                     warnPromisc;    // tracks if warning has been logged
    Bool                     forceSmac;      // whether to use smac unconditionally
    struct sk_buff          *history[VNET_BRIDGE_HISTORY];  // avoid duplicate packets
    spinlock_t		    historyLock;    // protects 'history'
@@ -146,7 +146,6 @@ VNetBridgeStartPromisc(VNetBridge *bridge,      // IN:
    if (!bridge->enabledPromisc && !bridge->wirelessAdapter) {
       dev_set_promiscuity(dev, 1);
       bridge->enabledPromisc = TRUE;
-      bridge->warnPromisc = FALSE;
       LOG(0, (KERN_NOTICE "bridge-%s: enabled promiscuous mode\n",
 	      bridge->name));
    }
@@ -213,11 +212,9 @@ static INLINE_SINGLE_CALLER int
 VNetBridgeDevCompatible(VNetBridge *bridge,      // IN: Bridge
                         struct net_device *net)  // IN: Network device
 {
-#ifdef VMW_NETDEV_HAS_NET
-   if (compat_dev_net(net) != &init_net) {
+   if (dev_net(net) != &init_net) {
       return 0;
    }
-#endif
    return strcmp(net->name, bridge->name) == 0;
 }
 
@@ -636,7 +633,7 @@ VNetBridgeReceiveFromVNet(VNetJack        *this, // IN: jack
 	 unsigned long flags;
 	 int i;
 
-	 atomic_inc(&clone->users);
+	 clone = skb_get(clone);
 
 	 clone->dev = dev;
 	 clone->protocol = eth_type_trans(clone, dev);
@@ -811,22 +808,10 @@ VNetBridgeIsBridged(VNetJack *this) // IN: jack
 static Bool
 VNetBridgeIsDeviceWireless(struct net_device *dev) //IN: sock
 {
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
-#  if defined(CONFIG_WIRELESS_EXT)
+#if defined(CONFIG_WIRELESS_EXT)
    return dev->ieee80211_ptr != NULL || dev->wireless_handlers != NULL;
-#  else
-   return dev->ieee80211_ptr != NULL;
-#  endif
-#elif defined(CONFIG_WIRELESS_EXT)
-   return dev->wireless_handlers != NULL;
-#elif !defined(CONFIG_NET_RADIO)
-   return FALSE;
-#elif defined WIRELESS_EXT && WIRELESS_EXT > 19
-   return dev->wireless_handlers != NULL;
-#elif defined WIRELESS_EXT && WIRELESS_EXT > 12
-   return dev->wireless_handlers != NULL || dev->get_wireless_stats != NULL;
 #else
-   return dev->get_wireless_stats != NULL;
+   return dev->ieee80211_ptr != NULL;
 #endif
 }
 
@@ -911,7 +896,7 @@ VNetBridgeUp(VNetBridge *bridge, // IN: bridge struct
     */
 
    dev_lock_list();
-   bridge->dev = DEV_GET(bridge);
+   bridge->dev = __dev_get_by_name(&init_net, bridge->name);
    LOG(2, (KERN_DEBUG "bridge-%s: got dev %p\n",
 	   bridge->name, bridge->dev));
    if (bridge->dev == NULL) {
@@ -926,17 +911,10 @@ VNetBridgeUp(VNetBridge *bridge, // IN: bridge struct
       retval = -ENODEV;
       goto out;
    }
-
-   /*
-    * At a minimum, the header size should be the same as ours.
-    *
-    * XXX we should either do header translation or ensure this
-    * is an Ethernet.
-    */
-
-   if (bridge->dev->hard_header_len != ETH_HLEN) {
-      LOG(1, (KERN_DEBUG "bridge-%s: can't bridge with %s, bad header length %d\n",
-	      bridge->name, bridge->dev->name, bridge->dev->hard_header_len));
+   if (bridge->dev->type != ARPHRD_ETHER) {
+      LOG(1, (KERN_DEBUG "bridge-%s: can't bridge with %s (header length %d, "
+              "type %d).\n", bridge->name, bridge->dev->name,
+              bridge->dev->hard_header_len, bridge->dev->type));
       dev_unlock_list();
       retval = -EINVAL;
       goto out;
@@ -990,7 +968,6 @@ VNetBridgeUp(VNetBridge *bridge, // IN: bridge struct
 
    bridge->pt.af_packet_priv = bridge->sk;
    bridge->enabledPromisc = FALSE;
-   bridge->warnPromisc = FALSE;
    dev_add_pack(&bridge->pt);
    dev_unlock_list();
 
@@ -1406,154 +1383,6 @@ VNetBridgeComputeHeaderPos(struct sk_buff *skb) // IN: buffer to examine
 }
 
 
-/*
- * We deal with three types of kernels:
- * New kernels: skb_shinfo() has gso_size member, and there is
- *              skb_gso_segment() helper to split GSO skb into flat ones.
- * Older kernels: skb_shinfo() has tso_size member, and there is
- *                no helper.
- * Oldest kernels: without any segmentation offload support.
- */
-#if defined(NETIF_F_GSO) || LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
-#define VNetBridgeIsGSO(skb) skb_shinfo(skb)->gso_size
-#define VNetBridgeGSOSegment(skb) skb_gso_segment(skb, 0)
-#elif defined(NETIF_F_TSO)
-#define VNetBridgeIsGSO(skb) skb_shinfo(skb)->tso_size
-
-
-/*
- *----------------------------------------------------------------------
- *
- * VNetBridgeGSOSegment --
- *
- *	Split a large TCP/IPv4 sk_buff into multiple sk_buffs of
- *	size skb_shinfo(skb)->tso_size
- *	Called from VNetBridgeSendLargePacket().
- *
- * Results:
- *	List of skbs created.
- *
- * Side effects:
- *	The incoming packet is split into multiple packets.
- *
- *----------------------------------------------------------------------
- */
-
-static struct sk_buff *
-VNetBridgeGSOSegment(struct sk_buff *skb)        // IN: packet to split
-{
-   struct sk_buff *segs = NULL;
-   struct sk_buff **next = &segs;
-   int bytesPerPacket, bytesLeft;
-   int macHdrLen, ipHdrLen, tcpHdrLen, allHdrLen;
-   int curByteOffset;
-   uint16 ipID;
-   uint32 seqNo;
-
-   if (((struct ethhdr *)compat_skb_mac_header(skb))->h_proto != htons(ETH_P_IP)) {
-      return ERR_PTR(-EPFNOSUPPORT);
-   }
-
-   if (compat_skb_ip_header(skb)->protocol != IPPROTO_TCP) {
-      return ERR_PTR(-EPROTONOSUPPORT);
-   }
-
-   macHdrLen = compat_skb_network_header(skb) - compat_skb_mac_header(skb);
-   ipHdrLen = compat_skb_ip_header(skb)->ihl << 2;
-   tcpHdrLen = compat_skb_tcp_header(skb)->doff << 2;
-   allHdrLen = macHdrLen + ipHdrLen + tcpHdrLen;
-
-   ipID = ntohs(compat_skb_ip_header(skb)->id);
-   seqNo = ntohl(compat_skb_tcp_header(skb)->seq);
-
-   /* Host TCP stack populated this (MSS) for the host NIC driver */
-   bytesPerPacket = skb_shinfo(skb)->tso_size;
-
-   bytesLeft = skb->len - allHdrLen;
-   curByteOffset = allHdrLen;
-
-   while (bytesLeft) {
-      struct sk_buff *newSkb;
-      int payloadSize = (bytesLeft < bytesPerPacket) ? bytesLeft : bytesPerPacket;
-
-      newSkb = dev_alloc_skb(payloadSize + allHdrLen + NET_IP_ALIGN);
-      if (!newSkb) {
-         while (segs) {
-            newSkb = segs;
-            segs = segs->next;
-            newSkb->next = NULL;
-            dev_kfree_skb(newSkb);
-         }
-         return ERR_PTR(-ENOMEM);
-      }
-      skb_reserve(newSkb, NET_IP_ALIGN);
-      newSkb->dev = skb->dev;
-      newSkb->protocol = skb->protocol;
-      newSkb->pkt_type = skb->pkt_type;
-      newSkb->ip_summed = VM_TX_CHECKSUM_PARTIAL;
-
-      /*
-       * MAC+IP+TCP copy
-       * This implies that ALL fields in the IP and TCP headers are copied from
-       * the original skb. This is convenient: we'll only fix up fields that
-       * need to be changed below
-       */
-      memcpy(skb_put(newSkb, allHdrLen), skb->data, allHdrLen);
-
-      /* Fix up pointers to different layers */
-      compat_skb_reset_mac_header(newSkb);
-      compat_skb_set_network_header(newSkb, macHdrLen);
-      compat_skb_set_transport_header(newSkb, macHdrLen + ipHdrLen);
-
-      /* Payload copy */
-      skb_copy_bits(skb, curByteOffset, compat_skb_tail_pointer(newSkb), payloadSize);
-      skb_put(newSkb, payloadSize);
-
-      curByteOffset+=payloadSize;
-      bytesLeft -= payloadSize;
-
-      /* Fix up IP hdr */
-      compat_skb_ip_header(newSkb)->tot_len = htons(payloadSize + tcpHdrLen + ipHdrLen);
-      compat_skb_ip_header(newSkb)->id = htons(ipID);
-      compat_skb_ip_header(newSkb)->check = 0;
-      /* Recompute new IP checksum */
-      compat_skb_ip_header(newSkb)->check =
-              ip_fast_csum(compat_skb_network_header(newSkb),
-                           compat_skb_ip_header(newSkb)->ihl);
-
-      /* Fix up TCP hdr */
-      compat_skb_tcp_header(newSkb)->seq = htonl(seqNo);
-      /* Clear FIN/PSH if not last packet */
-      if (bytesLeft > 0) {
-         compat_skb_tcp_header(newSkb)->fin = 0;
-         compat_skb_tcp_header(newSkb)->psh = 0;
-      }
-      /* Recompute partial TCP checksum */
-      compat_skb_tcp_header(newSkb)->check =
-         ~csum_tcpudp_magic(compat_skb_ip_header(newSkb)->saddr,
-                            compat_skb_ip_header(newSkb)->daddr,
-                            payloadSize+tcpHdrLen, IPPROTO_TCP, 0);
-
-      /* Offset of field */
-      newSkb->csum = offsetof(struct tcphdr, check);
-
-      /* Join packet to the list of segments */
-      *next = newSkb;
-      next = &newSkb->next;
-
-      /* Bump up our counters */
-      ipID++;
-      seqNo += payloadSize;
-
-   }
-   return segs;
-}
-#else
-#define VNetBridgeIsGSO(skb) (0)
-#define VNetBridgeGSOSegment(skb) ERR_PTR(-ENOSYS)
-#endif
-
-
 /*
  *----------------------------------------------------------------------
  *
@@ -1582,7 +1411,7 @@ VNetBridgeSendLargePacket(struct sk_buff *skb,        // IN: packet to split
 {
    struct sk_buff *segs;
 
-   segs = VNetBridgeGSOSegment(skb);
+   segs = skb_gso_segment(skb, 0);
    dev_kfree_skb(skb);
    if (IS_ERR(segs)) {
       LOG(1, (KERN_DEBUG "bridge-%s: cannot segment packet: error %ld\n",
@@ -1713,7 +1542,7 @@ VNetBridgeReceiveFromDev(struct sk_buff *skb,         // IN: packet to receive
    /*
     * If this is a large packet, chop chop chop (if supported)...
     */
-   if (VNetBridgeIsGSO(skb)) {
+   if (skb_shinfo(skb)->gso_size) {
       VNetBridgeSendLargePacket(skb, bridge);
    } else {
       VNetSend(&bridge->port.jack, skb);

vmnet-only/community_source.h

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2009-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2009-2016 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the

vmnet-only/compat_netdevice.h

@@ -337,7 +337,7 @@ typedef netdev_features_t compat_netdev_features_t;
 typedef u32 compat_netdev_features_t;
 #endif
 
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) || defined(VMW_NETIF_TRANS_UPDATE)
 #define compat_netif_trans_update(d) netif_trans_update(d)
 #else
 #define compat_netif_trans_update(d) do { (d)->trans_start = jiffies; } while (0)