From e4df7e3461e7bf2fa3a4aae5cc2740c911ded9aa Mon Sep 17 00:00:00 2001 From: Michal Kubecek Date: Sat, 30 Sep 2017 21:05:11 +0200 Subject: [PATCH] import Workstation 14.0.0 module sources --- vmmon-only/Makefile | 12 +- vmmon-only/Makefile.kernel | 8 +- vmmon-only/Makefile.normal | 26 +- vmmon-only/README | 4 +- vmmon-only/autoconf/smpcall.c | 37 - .../tsc_khz.c => bootstrap/bootstrap.c} | 37 +- vmmon-only/bootstrap/monLoader.c | 997 +++++++++ vmmon-only/bootstrap/monLoaderVmmon.c | 411 ++++ vmmon-only/common/apic.c | 8 +- vmmon-only/common/comport.c | 16 +- vmmon-only/common/cpuid.c | 87 +- vmmon-only/common/cpuid.h | 8 +- vmmon-only/common/hashFunc.c | 6 +- vmmon-only/common/hostif.h | 26 +- vmmon-only/common/memtrack.c | 26 +- vmmon-only/common/memtrack.h | 4 +- vmmon-only/common/phystrack.c | 6 +- vmmon-only/common/task.c | 955 +++++++-- vmmon-only/common/task.h | 10 +- vmmon-only/common/vmx86.c | 747 ++++--- vmmon-only/common/vmx86.h | 37 +- vmmon-only/include/address_defs.h | 10 + vmmon-only/include/basic_initblock.h | 7 +- vmmon-only/include/bootstrap_vmm.h | 65 + vmmon-only/include/circList.h | 259 ++- vmmon-only/include/community_source.h | 2 +- vmmon-only/include/compat_cred.h | 51 - vmmon-only/include/compat_highmem.h | 32 - vmmon-only/include/compat_interrupt.h | 55 - vmmon-only/include/compat_ioport.h | 63 - vmmon-only/include/compat_pci.h | 72 - vmmon-only/include/compat_pgtable.h | 5 - vmmon-only/include/compat_semaphore.h | 49 - vmmon-only/include/cpuid_info.h | 11 +- vmmon-only/include/crossgdt.h | 25 +- vmmon-only/include/iocontrols.h | 82 +- vmmon-only/include/memDefaults.h | 11 +- vmmon-only/include/modulecall.h | 138 +- vmmon-only/include/modulecallstructs.h | 5 +- vmmon-only/include/monAddrLayout.h | 57 + vmmon-only/include/monLoader.h | 302 +++ vmmon-only/include/monLoaderLog.h | 66 + vmmon-only/include/mon_assert.h | 17 +- vmmon-only/include/monitorAction_exported.h | 16 +- vmmon-only/include/numa_defs.h | 11 +- vmmon-only/include/pagelist.h | 22 +- vmmon-only/include/pshare_ext.h | 13 +- vmmon-only/include/ptsc.h | 1 - vmmon-only/include/rateconv.h | 16 +- vmmon-only/include/vcpuid.h | 16 +- vmmon-only/include/vcpuset.h | 6 +- vmmon-only/include/vcpuset_types.h | 22 +- vmmon-only/include/versioned_atomic.h | 246 ++- vmmon-only/include/vm_asm_x86.h | 9 +- vmmon-only/include/vm_asm_x86_64.h | 3 +- vmmon-only/include/vm_assert.h | 19 +- vmmon-only/include/vm_atomic.h | 1709 ++++++++-------- vmmon-only/include/vm_basic_asm.h | 102 +- vmmon-only/include/vm_basic_asm_x86.h | 12 +- vmmon-only/include/vm_basic_asm_x86_64.h | 15 +- vmmon-only/include/vm_basic_asm_x86_common.h | 163 +- vmmon-only/include/vm_basic_defs.h | 72 +- vmmon-only/include/vm_basic_math.h | 37 +- vmmon-only/include/vm_basic_types.h | 76 +- vmmon-only/include/vm_pagetable.h | 128 ++ vmmon-only/include/vmm_constants.h | 3 - vmmon-only/include/vmmem_shared.h | 14 +- vmmon-only/include/vmware.h | 2 +- vmmon-only/include/vmware_pack_begin.h | 2 +- vmmon-only/include/vmware_pack_end.h | 2 +- vmmon-only/include/vmware_pack_init.h | 2 +- vmmon-only/include/x86_basic_defs.h | 31 +- vmmon-only/include/x86apic.h | 3 +- vmmon-only/include/x86cpuid.h | 1600 +++++++++------ vmmon-only/include/x86cpuid_asm.h | 33 +- vmmon-only/include/x86msr.h | 151 +- vmmon-only/include/x86paging_64.h | 87 + vmmon-only/include/x86paging_common.h | 113 + vmmon-only/include/x86perfctr.h | 65 +- vmmon-only/include/x86sel.h | 10 + vmmon-only/include/x86svm.h | 6 +- vmmon-only/include/x86types.h | 13 +- vmmon-only/include/x86vendor.h | 2 +- vmmon-only/include/x86vt-exit-reasons.h | 97 + vmmon-only/include/x86vt-vmcs-fields.h | 816 ++++++-- vmmon-only/include/x86vt.h | 292 ++- vmmon-only/include/x86vtinstr.h | 124 +- vmmon-only/linux/driver.c | 725 ++----- vmmon-only/linux/driver.h | 26 +- vmmon-only/linux/hostif.c | 935 +++------ vmmon-only/linux/hostif_priv.h | 10 +- vmmon-only/linux/vmhost.h | 10 +- vmmon-only/linux/vmmonInt.h | 109 - vmmon-only/vmcore/moduleloop.c | 29 +- vmnet-only/Makefile | 12 +- vmnet-only/Makefile.kernel | 4 - vmnet-only/Makefile.normal | 4 - vmnet-only/bridge.c | 195 +- vmnet-only/community_source.h | 2 +- vmnet-only/compat_netdevice.h | 4 +- vmnet-only/driver.c | 4 - vmnet-only/monitorAction_exported.h | 156 -- vmnet-only/net.h | 2 +- vmnet-only/netdev_has_dev_net.c | 37 - vmnet-only/netdev_has_net.c | 43 - vmnet-only/nfhook_uses_skb.c | 45 - vmnet-only/skblin.c | 41 - vmnet-only/smac.c | 17 +- vmnet-only/userif.c | 85 +- vmnet-only/vm_assert.h | 19 +- vmnet-only/vm_atomic.h | 1709 ++++++++-------- vmnet-only/vm_basic_asm.h | 102 +- vmnet-only/vm_basic_asm_x86.h | 12 +- vmnet-only/vm_basic_asm_x86_64.h | 15 +- vmnet-only/vm_basic_asm_x86_common.h | 163 +- vmnet-only/vm_basic_defs.h | 72 +- vmnet-only/vm_basic_types.h | 76 +- vmnet-only/vm_device_version.h | 71 +- vmnet-only/vm_oui.h | 8 + vmnet-only/vmnetInt.h | 39 +- vmnet-only/vmware_pack_begin.h | 43 - vmnet-only/vmware_pack_end.h | 44 - vmnet-only/vmware_pack_init.h | 65 - vmnet-only/vnet.h | 24 +- vmnet-only/vnetFilterInt.h | 0 vmnet-only/x86cpuid.h | 1815 ----------------- 126 files changed, 9534 insertions(+), 8272 deletions(-) delete mode 100644 vmmon-only/autoconf/smpcall.c rename vmmon-only/{autoconf/tsc_khz.c => bootstrap/bootstrap.c} (51%) create mode 100644 vmmon-only/bootstrap/monLoader.c create mode 100644 vmmon-only/bootstrap/monLoaderVmmon.c create mode 100644 vmmon-only/include/bootstrap_vmm.h delete mode 100644 vmmon-only/include/compat_cred.h delete mode 100644 vmmon-only/include/compat_highmem.h delete mode 100644 vmmon-only/include/compat_interrupt.h delete mode 100644 vmmon-only/include/compat_ioport.h delete mode 100644 vmmon-only/include/compat_pci.h delete mode 100644 vmmon-only/include/compat_semaphore.h create mode 100644 vmmon-only/include/monAddrLayout.h create mode 100644 vmmon-only/include/monLoader.h create mode 100644 vmmon-only/include/monLoaderLog.h create mode 100644 vmmon-only/include/vm_pagetable.h create mode 100644 vmmon-only/include/x86paging_64.h create mode 100644 vmmon-only/include/x86paging_common.h create mode 100644 vmmon-only/include/x86vt-exit-reasons.h delete mode 100644 vmmon-only/linux/vmmonInt.h delete mode 100644 vmnet-only/monitorAction_exported.h delete mode 100644 vmnet-only/netdev_has_dev_net.c delete mode 100644 vmnet-only/netdev_has_net.c delete mode 100644 vmnet-only/nfhook_uses_skb.c delete mode 100644 vmnet-only/skblin.c delete mode 100644 vmnet-only/vmware_pack_begin.h delete mode 100644 vmnet-only/vmware_pack_end.h delete mode 100644 vmnet-only/vmware_pack_init.h delete mode 100644 vmnet-only/vnetFilterInt.h delete mode 100644 vmnet-only/x86cpuid.h diff --git a/vmmon-only/Makefile b/vmmon-only/Makefile index de8162e..ccdd295 100644 --- a/vmmon-only/Makefile +++ b/vmmon-only/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f ########################################################## -# Copyright (C) 1998-2015 VMware, Inc. All rights reserved. +# Copyright (C) 1998-2016 VMware, Inc. All rights reserved. # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -84,8 +84,6 @@ endif ifneq ($(VM_KBUILD), no) -VMCCVER := $(shell $(CC) -dumpversion) - # If there is no version defined, we are in toplevel pass, not yet in kernel makefiles... ifeq ($(VERSION),) @@ -100,14 +98,6 @@ auto-build: $(DRIVER_KO) $(DRIVER): $(DRIVER_KO) if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi -# Pass gcc version down the chain, so we can detect if kernel attempts to use unapproved compiler -VM_CCVER := $(VMCCVER) -export VM_CCVER -VM_CC := $(CC) -export VM_CC - -MAKEOVERRIDES := $(filter-out CC=%,$(MAKEOVERRIDES)) - # # Define a setup target that gets built before the actual driver. # This target may not be used at all, but if it is then it will be defined diff --git a/vmmon-only/Makefile.kernel b/vmmon-only/Makefile.kernel index bf805e0..385068d 100644 --- a/vmmon-only/Makefile.kernel +++ b/vmmon-only/Makefile.kernel @@ -24,16 +24,14 @@ INCLUDE := -I$(SRCROOT)/include -I$(SRCROOT)/common -I$(SRCROOT)/linux \ EXTRA_CFLAGS := $(CC_OPTS) $(INCLUDE) -EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/autoconf/smpcall.c, -DVMW_HAVE_SMP_CALL_3ARG, ) -EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/autoconf/tsc_khz.c, -DVMW_HAVE_TSC_KHZ, ) - obj-m += $(DRIVER).o $(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, \ - $(wildcard $(SRCROOT)/linux/*.c $(SRCROOT)/common/*.c $(SRCROOT)/vmcore/*.c))) + $(wildcard $(SRCROOT)/linux/*.c $(SRCROOT)/common/*.c $(SRCROOT)/vmcore/*.c \ + $(SRCROOT)/bootstrap/*.c))) clean: rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \ Module.symvers Modules.symvers Module.markers modules.order \ - $(foreach dir,linux/ common/ vmcore/ \ + $(foreach dir,linux/ common/ vmcore/ bootstrap/ \ ./,$(addprefix $(dir),.*.cmd .*.o.flags *.o))) diff --git a/vmmon-only/Makefile.normal b/vmmon-only/Makefile.normal index 9e26a96..a38a6da 100644 --- a/vmmon-only/Makefile.normal +++ b/vmmon-only/Makefile.normal @@ -26,7 +26,7 @@ vm_check_build = $(shell if $(CC) $(CC_OPTS) $(INCLUDE) -Werror -S -o /dev/null DESTDIR := driver-$(VM_UNAME) #### -#### DRIVERNAME should be untouched unless you have a good reason to change +#### DRIVERNAME should be untouched unless you have a good reason to change #### it. The form below is how the scripts expect it. #### DRIVERNAME := $(DRIVER)-xxx-$(VM_UNAME) @@ -49,7 +49,7 @@ endif #### If desparate, I think that bringing in might #### suffice. #### -CC_WARNINGS := -Wall -Wstrict-prototypes +CC_WARNINGS := -Wall -Wstrict-prototypes # Don't use -pipe or egcs-2.91.66 (shipped with RedHat) will die CC_KFLAGS := -D__KERNEL__ -fno-strength-reduce -fno-omit-frame-pointer \ -fno-common -DKBUILD_MODNAME=$(DRIVER) @@ -63,7 +63,7 @@ else CC_KFLAGS += -DCPU=586 $(call check_gcc,-march=i586,-m486) endif -CC_OPTS := -O2 -DMODULE -DVMMON -DVMCORE $(GLOBAL_DEFS) $(CC_KFLAGS) $(CC_WARNINGS) +CC_OPTS := -O2 -DMODULE -DVMMON -DVMCORE $(GLOBAL_DEFS) $(CC_KFLAGS) $(CC_WARNINGS) INCLUDE := -I$(SRCROOT)/include -I$(SRCROOT)/common -I$(SRCROOT)/linux \ -I$(SRCROOT)/vmcore -I$(HEADER_DIR) @@ -71,16 +71,16 @@ INCLUDE := -I$(SRCROOT)/include -I$(SRCROOT)/common -I$(SRCROOT)/linux \ INCLUDE += $(shell $(CC) $(INCLUDE) -E $(SRCROOT)/autoconf/geninclude.c \ | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p') -CC_OPTS += $(call vm_check_build, $(SRCROOT)/autoconf/smpcall.c, -DVMW_HAVE_SMP_CALL_3ARG, ) -CC_OPTS += $(call vm_check_build, $(SRCROOT)/autoconf/tsc_khz.c, -DVMW_HAVE_TSC_KHZ, ) - C_TARGETS_LINUX := driver.o hostif.o driverLog.o C_TARGETS_COMMON := vmx86.o memtrack.o phystrack.o cpuid.o task.o hashFunc.o C_TARGETS_VMCORE := moduleloop.o +C_TARGETS_BOOTSTRAP := bootstrap.o C_TARGETS_LINUX_D := ${C_TARGETS_LINUX:.o=.d} C_TARGETS_COMMON_D := ${C_TARGETS_COMMON:.o=.d} C_TARGETS_VMCORE_D := ${C_TARGETS_VMCORE:.o=.d} -C_TARGETS := $(C_TARGETS_LINUX) $(C_TARGETS_COMMON) $(C_TARGETS_VMCORE) +C_TARGETS_BOOTSTRAP_D := ${C_TARGETS_BOOTSTRAP:.o=.d} +C_TARGETS := $(C_TARGETS_LINUX) $(C_TARGETS_COMMON) $(C_TARGETS_VMCORE) \ + $(C_TARGETS_BOOTSTRAP) #### #### Make Targets are beneath here. @@ -116,6 +116,10 @@ $(C_TARGETS_VMCORE): %.o: $(SRCROOT)/vmcore/%.c $(ECHO) "Compiling vmcore/$( $@ -deps: setup - $(MAKE) -C $(DESTDIR) -f ../Makefile SRCROOT=../$(SRCROOT) driver_deps +$(C_TARGETS_BOOTSTRAP_D): %.d: $(SRCROOT)/bootstrap/%.c + $(ECHO) "Dependencies for $( $@ + +deps: setup + $(MAKE) -C $(DESTDIR) -f ../Makefile SRCROOT=../$(SRCROOT) driver_deps driver_deps: ${C_TARGETS:.o=.d} diff --git a/vmmon-only/README b/vmmon-only/README index 6a4da4c..ef162c3 100644 --- a/vmmon-only/README +++ b/vmmon-only/README @@ -3,11 +3,11 @@ portion of the VMware Virtual Machine Monitor. In order to build, make certain the Makefile is correct, especially in whether or not your system is multi-processor and then just type - make + make from this directory. A copy of the module will be left in - driver-/vmmon.o + driver-/vmmon.o (e.g. driver-up-2.0.32/vmmon.o). diff --git a/vmmon-only/autoconf/smpcall.c b/vmmon-only/autoconf/smpcall.c deleted file mode 100644 index e27a873..0000000 --- a/vmmon-only/autoconf/smpcall.c +++ /dev/null @@ -1,37 +0,0 @@ -/********************************************************* - * Copyright (C) 2008 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * Detect whether smp_call_function has 4 or 3 arguments. - * Change happened between 2.6.26 and 2.6.27-rc1. - */ - -#include "compat_version.h" -#include "compat_autoconf.h" - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26) -# error This compile test intentionally fails. -#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27) -# include - -int -vmware_smp_call_function(void (*func)(void *info), void *info, int wait) -{ - return smp_call_function(func, info, wait); -} -#endif diff --git a/vmmon-only/autoconf/tsc_khz.c b/vmmon-only/bootstrap/bootstrap.c similarity index 51% rename from vmmon-only/autoconf/tsc_khz.c rename to vmmon-only/bootstrap/bootstrap.c index f2f262f..24ede07 100644 --- a/vmmon-only/autoconf/tsc_khz.c +++ b/vmmon-only/bootstrap/bootstrap.c @@ -17,16 +17,37 @@ *********************************************************/ /* - * Detect if tsc_khz is available. + * bootstrap.c -- + * + * Implements the early VMM bootstraping code that is executed + * by the host (vmmon/VMKernel) to create the VMM context. */ -#include "compat_version.h" -#include "compat_autoconf.h" +#include "vm_basic_types.h" +#include "vm_basic_defs.h" +#include "bootstrap_vmm.h" -#include - -int -vmw_tsc_khz(void) +/* + *--------------------------------------------------------------------- + * + * BSVMM_Validate -- + * + * Validates the VMM bootstrap blob. For now, we do it by checking + * the magic number. Returns the bootstrap parameter table if + * successful, NULL otherwise. + * + *--------------------------------------------------------------------- + */ +BSVMM_HostParams * +BSVMM_Validate(void *buf, uint32 nbytes) { - return tsc_khz; + BSVMM_HostParams *bsParams = buf; + + if (nbytes < sizeof *bsParams) { + return NULL; + } + if (bsParams->magic != BOOTSTRAP_MAGIC) { + return NULL; + } + return bsParams; } diff --git a/vmmon-only/bootstrap/monLoader.c b/vmmon-only/bootstrap/monLoader.c new file mode 100644 index 0000000..34797cf --- /dev/null +++ b/vmmon-only/bootstrap/monLoader.c @@ -0,0 +1,997 @@ +/********************************************************* + * Copyright (C) 2015-2017 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * monLoader.c -- + * + * Processes the monitor loader header. + * + * See monLoader.h for a full description. + */ + +#ifdef VMX86_VMX + +#include +#include +#include "vmx.h" +#include "vm_assert.h" + +#elif defined VMKERNEL + +#include "vmkernel.h" +#include "libc.h" +#include "vm_libc.h" + +#define UINT_MAX (~0U) + +#elif defined VMMON + +#include "hostKernel.h" + +#ifdef __linux__ +# include "driver-config.h" +# include +# include +#elif defined __APPLE__ +# include +# include +#else +# include +# include +#endif + +#include "vm_assert.h" + +#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */ +#error MonLoader cannot be built as part of this environment +#endif + +#include "vm_basic_types.h" +#include "monLoader.h" +#include "vcpuid.h" +#include "x86types.h" +#include "x86paging_64.h" +#include "monLoaderLog.h" + +#define CANONICAL_MASK MASK64(36) +/* The index of VPN v within an array of LxEs where index 0 maps VPN b. */ +#define L4EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 36)) +#define L3EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 27)) +#define L2EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 18)) +#define L1EArrayIdx(b,v) ((unsigned)((((v) - (b)) & CANONICAL_MASK) >> 9)) + +/* + * Import address space from VMX rather than building it. Once monitor loading + * is removed from VMX entirely, code to import will be deleted. + */ +#define IMPORT_AS_FROM_VMX (TRUE) + +/* The maximum (canonical-address) VPN */ +#define VPN_MAX MASK64(52) + +/* Sufficient MPN counts to accommodate the monitor's top 64MB. */ +#define L4MPNCOUNTMAX 1 +#define L3MPNCOUNTMAX 1 +#define L2MPNCOUNTMAX 1 +#define L1MPNCOUNTMAX 32 + +#define LINE_INVALID ((unsigned)-1) + +typedef struct MonPTMPNs { + MPN L4MPNs[L4MPNCOUNTMAX]; /* The page root. Definitely only one page. */ + MPN L3MPNs[L3MPNCOUNTMAX]; + MPN L2MPNs[L2MPNCOUNTMAX]; + MPN L1MPNs[L1MPNCOUNTMAX]; + unsigned L4MPNCount; + unsigned L3MPNCount; + unsigned L2MPNCount; + unsigned L1MPNCount; +} MonPTMPNs; + +typedef struct MonLoaderContext { + MonLoaderEnvContext *envCtx; /* Environment-specific context */ + struct { + MonPTMPNs ptMPNs; /* Mappings into the AS for the current VCPU. */ + VPN ASFirstVPN; /* first VPN in the address space */ + VPN ASLastVPN; /* last VPN in the address space (inclusive) */ + uint64 ASPTEFlags; /* PTE flags for L4->L1 connection. */ + Vcpuid currentVCPU; + Bool ASImported; /* Was this AS imported or created anew? */ + Bool hasAddrSpace; + } vcpu; +} MonLoaderContext; + + +static const Bool importFromVMX = IMPORT_AS_FROM_VMX; + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderTranslateMonVPNToL1E -- + * + * Translates a monitor VPN to a level 1 page table entry, for the current + * VCPU. Assumes preallocated and connected page tables, so only the L1 + * table is examined. + * + * Result: + * ML_OK and a PTE if successful, some other value on error. Fails if a + * page table page is not already allocated or the VPN is not in the + * context's address space's defined range. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderTranslateMonVPNToL1E(MonLoaderContext *ctx, // IN/OUT + VPN monVPN, // IN + PT_L1E *pte) // OUT +{ + Vcpuid vcpu = ctx->vcpu.currentVCPU; + /* Only modify the L1E as L4->L1 are guaranteed connected already. */ + unsigned L1Off = PT_LPN_2_L1OFF(monVPN); + unsigned L1Page = L1EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN); + + if (monVPN < ctx->vcpu.ASFirstVPN || monVPN > ctx->vcpu.ASLastVPN) { + return ML_ERROR_INVALID_VPN; /* Address is outside address space. */ + } + if (ctx->vcpu.ptMPNs.L1MPNs[L1Page] == INVALID_MPN) { + return ML_ERROR_MAP; /* Page table not sufficiently preallocated. */ + } + if (!MonLoaderCallout_GetPTE(ctx->envCtx, ctx->vcpu.ptMPNs.L1MPNs[L1Page], + L1Off, vcpu, pte)) { + return ML_ERROR_CALLOUT_GETPTE; + } + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderIsMapped -- + * + * Checks whether a VPN has an existing mapping on the current VCPU. + * Assumes preallocated and connected page tables. + * + * Result: + * ML_OK if successful, some other value on error. Fails if a page table + * page is not already allocated or the VPN is not in the context's + * address space's defined range. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderIsMapped(MonLoaderContext *ctx, // IN/OUT + VPN monVPN, // IN + Bool *mapped) // OUT +{ + PT_L1E pte; + MonLoaderError ret = MonLoaderTranslateMonVPNToL1E(ctx, monVPN, &pte); + + if (ret != ML_OK) { + return ret; + } + *mapped = PTE_PRESENT(pte); + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderAllocMPN -- + * + * Allocates a new MPN. + * + * Result: + * ML_OK and an MPN, on success. An error and no MPN on failure. + * + * Side effects: + * An MPN may be allocated. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderAllocMPN(MonLoaderContext *ctx, // IN + MPN *mpn) // OUT +{ + *mpn = MonLoaderCallout_AllocMPN(ctx->envCtx, ctx->vcpu.currentVCPU); + if (*mpn == INVALID_MPN) { + return ML_ERROR_ALLOC; + } + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderMapMPN -- + * + * Maps an MPN into the current VCPU's address space at a VPN. + * + * Result: + * ML_OK if successful, some other value on error. Fails if a page table + * page is not already allocated or the VPN is not in the context's + * address space's defined range. + * + * Side effects: + * An MPN is mapped on success. Nothing on failure. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderMapMPN(MonLoaderContext *ctx, // IN/OUT + MPN mpn, // IN + uint64 flags, // IN + VPN monVPN) // IN +{ + MonPTMPNs *ptMPNs = &ctx->vcpu.ptMPNs; + /* Only modify the L1E, as L4->L1 are guaranteed connected already. */ + unsigned L1Off = PT_LPN_2_L1OFF(monVPN); + unsigned L1Page = L1EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN); + + if (vmx86_debug) { + MonLoaderError ret; + Bool mapped; + if (monVPN < ctx->vcpu.ASFirstVPN || monVPN > ctx->vcpu.ASLastVPN) { + LOG(5, "%s: monVPN=0x%"FMTVPN"x, AS=0x%"FMTVPN"x-0x%"FMTVPN"x\n", + __FUNCTION__, monVPN, ctx->vcpu.ASFirstVPN, + ctx->vcpu.ASLastVPN); + return ML_ERROR_INVALID_VPN; /* Address is outside address space. */ + } + if (ptMPNs->L1MPNs[L1Page] == INVALID_MPN) { + return ML_ERROR_MAP; /* Page table not sufficiently preallocated. */ + } + ret = MonLoaderIsMapped(ctx, monVPN, &mapped); + if (ret != ML_OK) { + return ret; + } + if (mapped) { + return ML_ERROR_ALREADY_MAPPED; + } + } + if (!MonLoaderCallout_MapMPNInPTE(ctx->envCtx, ptMPNs->L1MPNs[L1Page], L1Off, + flags, mpn, ctx->vcpu.currentVCPU)) { + return ML_ERROR_CALLOUT_MAPINPTE; + } + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCreateAddressSpace -- + * + * Creates or (if importAS is set) verifies an address space. The VPN + * range specified by firstVPN and size is used to determine page counts + * at each page table level to map every page. Sufficient pages are then + * allocated or verified. + * + * Result: + * ML_OK if successful, some other value on error. + * + * Side effects: + * MPNs are allocated if not importing. Sets ctx->vcpu.hasAddrSpace. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderCreateAddressSpace(MonLoaderContext *ctx, // IN/OUT + VPN firstVPN, // IN + uint64 flags, // IN + uint64 monPages, // IN + Bool importAS) // IN +{ + Vcpuid vcpu = ctx->vcpu.currentVCPU; + VPN lastVPN = firstVPN + monPages - 1; + unsigned L3MPNsNeeded = L3EArrayIdx(0, lastVPN) - + L3EArrayIdx(0, firstVPN) + 1; + unsigned L2MPNsNeeded = L2EArrayIdx(0, lastVPN) - + L2EArrayIdx(0, firstVPN) + 1; + unsigned L1MPNsNeeded = L1EArrayIdx(0, lastVPN) - + L1EArrayIdx(0, firstVPN) + 1; + unsigned i; + MPN mpn; + MonPTMPNs *ptMPNs; + + if (vcpu == VCPUID_INVALID) { + return ML_ERROR_ARGS; + } + + /* Verify MonPTMPNs is large enough for this address space. */ + if (L3MPNsNeeded > L3MPNCOUNTMAX || L2MPNsNeeded > L2MPNCOUNTMAX || + L1MPNsNeeded > L1MPNCOUNTMAX) { + return ML_ERROR_ADDRSPACE_TOO_LARGE; + } + ctx->vcpu.ASFirstVPN = firstVPN; + ctx->vcpu.ASLastVPN = lastVPN; + ctx->vcpu.ASPTEFlags = flags; + ctx->vcpu.ASImported = importAS; + + ptMPNs = &ctx->vcpu.ptMPNs; + + if (importAS) { + mpn = MonLoaderCallout_GetPageRoot(ctx->envCtx, vcpu); + if (mpn == INVALID_MPN) { + return ML_ERROR_CALLOUT_PAGEROOT_GET; + } + LOG(5, "%s: vcpu %u page root=0x%"FMT64"x\n", __FUNCTION__, vcpu, mpn); + ptMPNs->L4MPNs[0] = mpn; + MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu); + } else { + NOT_IMPLEMENTED(); + } + ptMPNs->L4MPNCount = 1; + + for (i = 0; i < L3MPNsNeeded; i++) { + if (importAS) { + VPN monVPN = firstVPN + i * PT_PAGES_PER_L4E; + unsigned L4Off = PT_LPN_2_L4OFF(monVPN); + PT_L1E pte; + if (!MonLoaderCallout_GetPTE(ctx->envCtx, ptMPNs->L4MPNs[0], L4Off, + vcpu, &pte)) { + return ML_ERROR_CALLOUT_GETPTE; + } + LOG(5, "%s: monVPN=0x%"FMTVPN"x: L4E=0x%"FMT64"x\n", __FUNCTION__, + monVPN, pte); + mpn = LM_PTE_2_PFN(pte); + if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) { + return ML_ERROR_PAGE_TABLE_IMPORT; + } + MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu); + } else { + NOT_IMPLEMENTED(); + } + ptMPNs->L3MPNs[i] = mpn; + } + ptMPNs->L3MPNCount = L3MPNsNeeded; + + for (i = 0; i < L2MPNsNeeded; i++) { + if (importAS) { + VPN monVPN = firstVPN + i * PT_PAGES_PER_L3E; + unsigned L3Off = PT_LPN_2_L3OFF(monVPN); + unsigned L3Page = L3EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN); + PT_L1E pte; + if (!MonLoaderCallout_GetPTE(ctx->envCtx, ptMPNs->L3MPNs[L3Page], + L3Off, vcpu, &pte)) { + return ML_ERROR_CALLOUT_GETPTE; + } + LOG(5, "%s: monVPN=0x%"FMTVPN"x: L3E=0x%"FMT64"x\n", __FUNCTION__, + monVPN, pte); + mpn = LM_PTE_2_PFN(pte); + if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) { + return ML_ERROR_PAGE_TABLE_IMPORT; + } + MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu); + } else { + NOT_IMPLEMENTED(); + } + ptMPNs->L2MPNs[i] = mpn; + } + ptMPNs->L2MPNCount = L2MPNsNeeded; + + for (i = 0; i < L1MPNsNeeded; i++) { + if (importAS) { + VPN monVPN = firstVPN + i * PT_PAGES_PER_L2E; + unsigned L2Off = PT_LPN_2_L2OFF(monVPN); + unsigned L2Page = L2EArrayIdx(ctx->vcpu.ASFirstVPN, monVPN); + PT_L1E pte; + if (!MonLoaderCallout_GetPTE(ctx->envCtx, ptMPNs->L2MPNs[L2Page], + L2Off, vcpu, &pte)) { + return ML_ERROR_CALLOUT_GETPTE; + } + LOG(5, "%s: monVPN=0x%"FMTVPN"x: L2E=0x%"FMT64"x\n", __FUNCTION__, + monVPN, pte); + mpn = LM_PTE_2_PFN(pte); + if (mpn == INVALID_MPN || !ML_PERMS_MATCH(pte, flags)) { + return ML_ERROR_PAGE_TABLE_IMPORT; + } + MonLoaderCallout_ImportPage(ctx->envCtx, mpn, vcpu); + } else { + NOT_IMPLEMENTED(); + } + ptMPNs->L1MPNs[i] = mpn; + } + ptMPNs->L1MPNCount = L1MPNsNeeded; + + ctx->vcpu.hasAddrSpace = TRUE; + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderMapPageTables -- + * + * Maps in the page tables for the specified level, or verifies that the + * existing mappings match internal state and flags if the address space + * was imported. Assumes pre-allocated memory at all levels and + * pre-connected L4->L1 page tables. + * + * Result: + * ML_OK if successful, some other value on error. + * + * Side effects: + * Page tables are updated if mappings are created. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderMapPageTables(MonLoaderContext *ctx, // IN/OUT + unsigned level, // IN + uint64 flags, // IN + VPN monVPN, // IN + uint64 monPages) // IN +{ + Bool verify = ctx->vcpu.ASImported; + uint64 i; + unsigned count; + MPN *ptMPNs; + + if (vmx86_debug) { + if (!ctx->vcpu.hasAddrSpace) { + return ML_ERROR_NO_ADDRSPACE; + } + if (level < 1 || level > 4) { + return ML_ERROR_ARGS; + } + } + + switch (level) { + case 4: + count = ctx->vcpu.ptMPNs.L4MPNCount; + ptMPNs = ctx->vcpu.ptMPNs.L4MPNs; + break; + case 3: + count = ctx->vcpu.ptMPNs.L3MPNCount; + ptMPNs = ctx->vcpu.ptMPNs.L3MPNs; + break; + case 2: + count = ctx->vcpu.ptMPNs.L2MPNCount; + ptMPNs = ctx->vcpu.ptMPNs.L2MPNs; + break; + case 1: + count = ctx->vcpu.ptMPNs.L1MPNCount; + ptMPNs = ctx->vcpu.ptMPNs.L1MPNs; + break; + default: + return ML_ERROR_ARGS; + break; + + if (monPages != count) { + return ML_ERROR_PAGE_TABLE_MAP_SIZE; + } + + for (i = 0; i < monPages; i++) { + VPN vpn = monVPN + i; + if (verify) { + PT_L1E l1e; + if (MonLoaderTranslateMonVPNToL1E(ctx, vpn, &l1e) != ML_OK || + LM_PTE_2_PFN(l1e) != ptMPNs[i] || !ML_PERMS_MATCH(l1e, flags)) { + return ML_ERROR_PAGE_TABLE_VERIFY; + } + } else { + NOT_IMPLEMENTED(); + } + } + } + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderZero -- + * + * Allocates new pages, zeroes them and maps them on the current VCPU. + * + * Result: + * ML_OK if successful, some other value on error. The number of + * MPNs allocated in *allocs is incremented. + * + * Side effects: + * Allocations, zeroing, mapping and accounting are performed. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderZero(MonLoaderContext *ctx, // IN/OUT + uint64 flags, // IN + VPN monVPN, // IN + uint64 numPages, // IN + uint64 monPages, // IN + unsigned *allocs) // IN/OUT +{ + uint64 i; + + if (numPages == 0 || numPages > monPages) { + return ML_ERROR_SIZE; + } + for (i = 0; i < numPages; i++) { + MPN mpn; + MonLoaderError ret = MonLoaderAllocMPN(ctx, &mpn); + if (ret != ML_OK) { + return ret; + } + if (!MonLoaderCallout_FillPage(ctx->envCtx, 0, mpn, + ctx->vcpu.currentVCPU)) { + return ML_ERROR_CALLOUT_ZERO; + } + ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN + i); + if (ret != ML_OK) { + return ret; + } + *allocs += 1; + } + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCopyFromBlob -- + * + * Allocates new pages, copies blob contents and maps the pages. + * + * Result: + * ML_OK if successful, some other value on error. The number of + * MPNs allocated in *allocs is incremented. + * + * Side effects: + * Allocations, copying, mapping and accounting are performed. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderCopyFromBlob(MonLoaderContext *ctx, // IN/OUT + uint64 flags, // IN + VPN monVPN, // IN + uint64 monBytes, // IN + uint64 blobOffset, // IN + uint64 blobSize, // IN + unsigned *allocs) // IN/OUT +{ + uint64 bytesLeft = blobSize; + + if (blobSize > monBytes || blobSize == 0) { + return ML_ERROR_SIZE; /* Entry corrupt: size mismatch. */ + } + while (bytesLeft) { + MPN mpn; + uint64 toCopy = MIN(bytesLeft, PAGE_SIZE); + + MonLoaderError ret = MonLoaderAllocMPN(ctx, &mpn); + if (ret != ML_OK) { + return ret; + } + + ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN); + if (ret != ML_OK) { + return ret; + } + + if (!MonLoaderCallout_CopyFromBlob(ctx->envCtx, blobOffset, toCopy, mpn, + ctx->vcpu.currentVCPU)) { + return ML_ERROR_CALLOUT_COPY; + } + + monVPN++; + blobOffset += toCopy; + bytesLeft -= toCopy; + *allocs += 1; + } + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderShareWork -- + * + * Maps user or host pages in. monPages is a maximum, so partial sharing + * is considered successful and remaining monitor pages will not be mapped. + * + * Result: + * ML_OK if successful, some other value on error. + * + * Side effects: + * User pages may be mapped in. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderShareWork(MonLoaderContext *ctx, // IN/OUT + uint64 flags, // IN + VPN monVPN, // IN + uint64 monPages, // IN + uint64 subIndex, // IN + Bool user) // IN +{ + Vcpuid vcpu = ctx->vcpu.currentVCPU; + uint64 i; + + if (monPages == 0 || monPages > UINT_MAX) { + return ML_ERROR_SIZE; /* Entry corrupt: size mismatch. */ + } + for (i = 0; i < monPages; i++) { + MPN mpn; + MonLoaderError ret; + unsigned pgNum = (unsigned)i; + + if (user) { + mpn = MonLoaderCallout_GetSharedUserPage(ctx->envCtx, subIndex, pgNum, + vcpu); + } else { + mpn = MonLoaderCallout_GetSharedHostPage(ctx->envCtx, subIndex, pgNum, + vcpu); + } + if (mpn == INVALID_MPN) { + /* Partial sharing is allowed. Return success if any occurred. */ + return i != 0 ? ML_OK : ML_ERROR_SHARE; + } + ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN + i); + if (ret != ML_OK) { + return ret; + } + } + return ML_OK; +} + + +#ifndef VMMON +/* + *---------------------------------------------------------------------- + * + * MonLoaderSetEntrypoint -- + * + * Calls out to set the code and stack entrypoint. + * + * Result: + * ML_OK if successful, some other value on error. + * + * Side effects: + * The monitor entrypoint for the current VCPU may be set. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderSetEntrypoint(MonLoaderContext *ctx, // IN + uint16 codeSelector, // IN + VA64 code, // IN + uint16 stackSelector, // IN + VA64 stack) // IN +{ + if (VA_2_VPN(code) < ctx->vcpu.ASFirstVPN || + VA_2_VPN(code) > ctx->vcpu.ASLastVPN || + VA_2_VPN(stack) < ctx->vcpu.ASFirstVPN || + VA_2_VPN(stack) > ctx->vcpu.ASLastVPN) { + return ML_ERROR_INVALID_VPN; /* Address is outside address space. */ + } + if (MonLoaderCallout_SetEntrypoint(ctx->envCtx, codeSelector, code, + stackSelector, stack)) { + return ML_OK; + } else { + return ML_ERROR_CALLOUT_ENTRYPOINT; + } +} +#endif + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderShareFromUser -- + * + * Maps user pages in. monPages is a maximum, so partial sharing is + * considered successful and remaining monitor pages will not be mapped. + * + * Result: + * ML_OK if successful, some other value on error. + * + * Side effects: + * User pages may be mapped in. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderShareFromUser(MonLoaderContext *ctx, // IN/OUT + uint64 flags, // IN + VPN monVPN, // IN + uint64 monPages, // IN + uint64 subIndex) // IN +{ + return MonLoaderShareWork(ctx, flags, monVPN, monPages, subIndex, TRUE); +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderShareFromHost -- + * + * Maps host pages in. monPages is a maximum, so partial sharing is + * considered successful and remaining monitor pages will not be mapped. + * + * Result: + * ML_OK if successful, some other value on error. + * + * Side effects: + * Host pages may be mapped in. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderShareFromHost(MonLoaderContext *ctx, // IN/OUT + uint64 flags, // IN + VPN monVPN, // IN + uint64 monPages, // IN + uint64 subIndex) // IN +{ + return MonLoaderShareWork(ctx, flags, monVPN, monPages, subIndex, FALSE); +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderShareFromBlob -- + * + * Maps blob pages in. monPages is a maximum, so partial sharing is + * considered successful and remaining monitor pages will not be + * mapped. blobOffset must be page-aligned and blobSize must be a + * multiple of PAGE_SIZE. Writeable mappings are not allowed. + * + * Result: + * ML_OK if successful, some other value on error. + * + * Side effects: + * Blob pages may be mapped in. + * + *---------------------------------------------------------------------- + */ +static MonLoaderError +MonLoaderShareFromBlob(MonLoaderContext *ctx, // IN/OUT + uint64 flags, // IN + VPN monVPN, // IN + uint64 monBytes, // IN + uint64 blobOffset, // IN + uint64 blobSize) // IN +{ + uint64 endOff = blobOffset + blobSize; + + if ((blobOffset & PAGE_MASK) != 0 || (blobSize & PAGE_MASK) != 0 || + (flags & PTE_RW) != 0) { + return ML_ERROR_SHARE; + } + if (blobSize > monBytes || blobSize == 0) { + return ML_ERROR_SIZE; + } + while (blobOffset < endOff) { + MonLoaderError ret; + MPN mpn = MonLoaderCallout_GetBlobMpn(ctx->envCtx, blobOffset); + if (mpn == INVALID_MPN) { + return ML_ERROR_INVALID_VPN; + } + ret = MonLoaderMapMPN(ctx, mpn, flags, monVPN); + if (ret != ML_OK) { + return ret; + } + blobOffset += PAGE_SIZE; + monVPN++; + } + ASSERT(blobOffset == endOff); + return ML_OK; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoader_Process -- + * + * Builds and populates a monitor address space per the given a header and + * VCPU count. Calls out to support functions. + * + * Result: + * ML_OK if successful, some other value on error. If an error occurs, + * *line contains the line in the table at which the failure occurred and + * *vcpu contains the VCPU for which processing was running. + * + * Side effects: + * An address space and page table mappings are created or verified, for + * all VCPUs. Callouts are called. + * + *---------------------------------------------------------------------- + */ +MonLoaderError +MonLoader_Process(MonLoaderHeader *header, // IN/OUT + unsigned numVCPUs, // IN + void *args, // IN + unsigned *line, // OUT + Vcpuid *vcpu) // OUT +{ + unsigned i; + MonLoaderError ret; + MonLoaderContext ctx; + + if (numVCPUs > MAX_VCPUS) { + return ML_ERROR_ARGS; + } + + if (header->magic != LOADER_HEADER_MAGIC) { + return ML_ERROR_MAGIC; + } + + if (vmx86_debug && !line) { + return ML_ERROR_ARGS; + } + *line = LINE_INVALID; + + if (header->entries == 0 || header->count == 0) { + return ML_ERROR_TABLE_MISSING; + } + + if (!MonLoaderCallout_Init(args, &ctx.envCtx)) { + return ML_ERROR_CALLOUT_INIT; + } + ret = ML_OK; + for (*vcpu = 0; *vcpu < numVCPUs && ret == ML_OK; (*vcpu) += 1) { + /* Reset VCPU-specific part of context. */ + memset(&ctx.vcpu, 0, sizeof ctx.vcpu); + ctx.vcpu.currentVCPU = *vcpu; + ctx.vcpu.hasAddrSpace = FALSE; + for (i = 0; i < header->count && ret == ML_OK; i++) { + MonLoaderEntry *entry = &header->entries[i]; + MonLoaderContentType content = entry->content; + MonLoaderSourceType source = entry->source; + VPN monVPN = entry->monVPN; + uint64 blobOffset = entry->blobSrc.offset; + uint64 blobSize = entry->blobSrc.size; + uint64 flags = entry->flags; + uint64 monPages = entry->monPages; + uint64 monBytes = PAGES_2_BYTES(monPages); + uint64 subIndex = entry->subIndex; + uint64 bspOnly = entry->bspOnly; + uint64 isPriv = vmx86_server || entry->procVmmon; + *line = i; + + /* + * The entry is specific to the bootstrap, which only runs in VCPU 0. + */ + if (bspOnly && !IS_BOOT_VCPUID(*vcpu)) { + continue; + } + /* + * Entries marked as privileged (isPriv == TRUE) are processed + * by the privileged layer (VMKernel/vmmon). Others are + * processed by the VMX. One exception is the entry of type + * ML_CONTENT_ADDRSPACE: it is always processed by the VMX and, + * if marked as privileged, also by the privileged layer. + */ + if (isPriv != MonLoaderCallout_IsPrivileged(ctx.envCtx) && + (content != ML_CONTENT_ADDRSPACE || isPriv == 0)) { + continue; + } + if (monPages == 0 || monVPN + monPages - 1 > VPN_MAX) { + ret = ML_ERROR_SIZE; + continue; + } + + switch (content) { + case ML_CONTENT_ADDRSPACE: + if (ctx.vcpu.hasAddrSpace) { + ret = ML_ERROR_DUPLICATE; + continue; + } + /* Create or verify the address space and page table MPNs. */ + ret = MonLoaderCreateAddressSpace(&ctx, monVPN, flags, monPages, + importFromVMX); + break; + case ML_CONTENT_PAGETABLE_L4: + case ML_CONTENT_PAGETABLE_L3: + case ML_CONTENT_PAGETABLE_L2: + case ML_CONTENT_PAGETABLE_L1: + /* Create or verify page table mappings. */ + ret = MonLoaderMapPageTables(&ctx, CONTENT_TO_PTLEVEL(content), + flags, monVPN, monPages); + break; + case ML_CONTENT_ALLOCZERO: { + uint64 numPages; + if (source == ML_SOURCE_NONE) { + numPages = monPages; + } else if (source == ML_SOURCE_BLOB) { + /* + * Used for loading the uninitialized data section + * (.bss), which has a blob size but no blob image + * associated with it. + */ + numPages = CEILING(blobSize, PAGE_SIZE); + } else { + ret = ML_ERROR_SOURCE_INVALID; + break; + } + ret = MonLoaderZero(&ctx, flags, monVPN, numPages, monPages, + &entry->allocs); + break; + } + case ML_CONTENT_COPY: + switch (source) { + case ML_SOURCE_BLOB: + ret = MonLoaderCopyFromBlob(&ctx, flags, monVPN, monBytes, + blobOffset, blobSize, + &entry->allocs); + break; + default: + ret = ML_ERROR_SOURCE_INVALID; + break; + } + break; + case ML_CONTENT_SHARE: + switch (source) { + case ML_SOURCE_USER: + ret = MonLoaderShareFromUser(&ctx, flags, monVPN, + monPages, subIndex); + break; + case ML_SOURCE_HOST: + ret = MonLoaderShareFromHost(&ctx, flags, monVPN, + monPages, subIndex); + break; + case ML_SOURCE_BLOB: + ret = MonLoaderShareFromBlob(&ctx, flags, monVPN, + monBytes, blobOffset, + blobSize); + break; + default: + ret = ML_ERROR_SOURCE_INVALID; + break; + } + break; + case ML_CONTENT_INVALID: + default: + ret = ML_ERROR_CONTENT_INVALID; + break; + } + } + } +#ifndef VMMON + if (ret == ML_OK) { + if (ctx.vcpu.ASLastVPN == 0) { + ret = ML_ERROR_NO_ADDRSPACE; + } else { + ret = MonLoaderSetEntrypoint(&ctx, header->codeSelector, + header->codeEntrypoint, + header->stackSelector, + header->stackEntrypoint); + } + } +#endif + MonLoaderCallout_CleanUp(ctx.envCtx); + return ret; +} diff --git a/vmmon-only/bootstrap/monLoaderVmmon.c b/vmmon-only/bootstrap/monLoaderVmmon.c new file mode 100644 index 0000000..6879e61 --- /dev/null +++ b/vmmon-only/bootstrap/monLoaderVmmon.c @@ -0,0 +1,411 @@ +/********************************************************* + * Copyright (C) 2016-2017 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * monLoaderVmmon.c -- + * + * vmmon implementation of the MonLoader callouts. + * + */ + +#ifdef __linux__ +# include "driver-config.h" +# include +#else +# include +#endif + +#include "vm_assert.h" +#include "vcpuid.h" +#include "x86types.h" +#include "hostif.h" +#include "x86paging_common.h" +#include "x86paging_64.h" +#include "vmx86.h" +#include "monLoaderLog.h" + +typedef struct MonLoaderEnvContext { + VMDriver *vm; + VMSharedRegion *shRegions; +} MonLoaderEnvContext; + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCallout_Init -- + * + * Initializes platform-specific MonLoader context. + * + * Returns: + * TRUE if successful, FALSE if not. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +Bool +MonLoaderCallout_Init(void *args, MonLoaderEnvContext **ctx) +{ + MonLoaderEnvContext *c; + MonLoaderArgs *mlArgs = (MonLoaderArgs *)args; + + c = HostIF_AllocKernelMem(sizeof *c, FALSE); + if (c == NULL) { + Log("Failed to allocate memory for MonLoader context\n"); + return FALSE; + } + memset(c, 0, sizeof *c); + c->vm = mlArgs->vm; + c->shRegions = mlArgs->shRegions; + + *ctx = c; + return TRUE; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCallout_Cleanup -- + * + * Cleans up platform-specific MonLoader context. + * + * Returns: + * None. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +void +MonLoaderCallout_CleanUp(MonLoaderEnvContext *ctx) +{ + ASSERT(ctx != NULL); + HostIF_FreeKernelMem(ctx); +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCallout_GetPageRoot -- + * + * Gets the page root MPN for a VCPU. + * + * Returns: + * An MPN, or INVALID_MPN in case of error. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + + +MPN +MonLoaderCallout_GetPageRoot(MonLoaderEnvContext *ctx, // IN + Vcpuid vcpu) // IN +{ + VMDriver *vm = ctx->vm; + + if (vcpu >= vm->numVCPUs) { + return INVALID_MPN; + } + return vm->ptRootMpns[vcpu]; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCallout_GetPTE -- + * + * Gets the page table entry at index idx in a page table MPN. + * + * Returns: + * TRUE and sets *pte on success. FALSE otherwise. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +Bool +MonLoaderCallout_GetPTE(MonLoaderEnvContext *ctx, // IN + MPN ptMPN, // IN + unsigned idx, // IN + Vcpuid vcpu, // IN + PT_L1E *pte) // OUT +{ + VMDriver *vm = ctx->vm; + MA pteMA = MPN_2_MA(ptMPN) + idx * sizeof *pte; + + ASSERT(idx < (PAGE_SIZE / sizeof *pte)); + if (HostIF_ReadPhysical(vm, pteMA, PtrToVA64(pte), TRUE, sizeof *pte) != 0) { + Log("Failed to read PTE %d from MPN %"FMT64"x\n", idx, ptMPN); + return FALSE; + } + return TRUE; +} + + +Bool +MonLoaderCallout_ImportPage(MonLoaderEnvContext *ctx, // IN + MPN mpn, // IN + Vcpuid vcpu) // IN +{ + return TRUE; +} + + +MPN +MonLoaderCallout_AllocMPN(MonLoaderEnvContext *ctx, // IN + Vcpuid vcpu) // IN +{ + NOT_IMPLEMENTED(); +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCallout_MapMPNInPTE -- + * + * Maps an MPN into a page table at the specified index. + * + * Returns: + * TRUE on success, FALSE on failure. + * + * Side effects: + * A mapping is created. + * + *---------------------------------------------------------------------- + */ + +Bool +MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *ctx, // IN + MPN ptMPN, // IN + unsigned idx, // IN + uint64 flags, // IN + MPN mpn, // IN + Vcpuid vcpu) // IN +{ + VMDriver *vm = ctx->vm; + PT_L1E pte = LM_MAKE_PTE(mpn, 0, flags); + MA pteMA = MPN_2_MA(ptMPN) + idx * sizeof pte; + + ASSERT(idx < (PAGE_SIZE / sizeof pte)); + if (HostIF_WritePhysical(vm, pteMA, PtrToVA64(&pte), TRUE, sizeof pte) != + 0) { + Warning("Failed to map MPN %"FMT64"x\n", mpn); + return FALSE; + } + LOG(5, "MonLoader mapped MPN %"FMT64"x at PT index %u for VCPU %u\n", + mpn, idx, vcpu); + return TRUE; +} + + +Bool +MonLoaderCallout_FillPage(MonLoaderEnvContext *ctx, // IN + uint8 pattern, // IN + MPN mpn, // IN + Vcpuid vcpu) // IN +{ + NOT_IMPLEMENTED(); +} + + +Bool +MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *ctx, // IN + uint64 blobOffset, // IN + size_t copySize, // IN + MPN mpn, // IN + Vcpuid vcpu) // IN +{ + NOT_IMPLEMENTED(); +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderFindSharedRegion -- + * + * Searches the shared region table for the given index and returns + * the corresponding descriptor. + * + * Returns: + * A pointer to the shared region descriptor or NULL if not found. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static VMSharedRegion * +MonLoaderFindSharedRegion(MonLoaderEnvContext *ctx, + uint64 index) +{ + int i; + + for (i = 0; i < ML_SHARED_REGIONS_MAX; i++) { + if (ctx->shRegions[i].baseVpn == INVALID_VPN) { + /* Not found. */ + return NULL; + } else if (ctx->shRegions[i].index == index) { + return &ctx->shRegions[i]; + } + } + return NULL; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderGetSharedRegionMPN -- + * + * Gets shared page's MPN. Assumes vcpuid == 0 for the bootstrap. + * + * Returns: + * An MPN or INVALID_MPN. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static MPN +MonLoaderGetSharedRegionMPN(MonLoaderEnvContext *ctx, + uint64 index, + Vcpuid vcpuid, + unsigned pgOffset) +{ + VMSharedRegion *s; + VA addr; + MPN mpn; + + ASSERT(IS_BOOT_VCPUID(vcpuid)); + + if (index >= ML_SHARED_REGIONS_MAX) { + Log("Invalid shared region %"FMT64"x", index); + return INVALID_MPN; + } + s = MonLoaderFindSharedRegion(ctx, index); + if (s == NULL) { + return INVALID_MPN; + } + if (pgOffset >= s->numPages) { + return INVALID_MPN; + } + + addr = VPN_2_VA(s->baseVpn) + pgOffset * PAGE_SIZE; + HostIF_VMLock(ctx->vm, 38); + if (HostIF_LookupUserMPN(ctx->vm, addr, &mpn) != PAGE_LOOKUP_SUCCESS) { + HostIF_VMUnlock(ctx->vm, 38); + Log("Failed to lookup MPN for shared region VA %"FMTVA"x\n", addr); + return INVALID_MPN; + } + HostIF_VMUnlock(ctx->vm, 38); + return mpn; +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCallout_GetSharedUserPage -- + * + * Gets a user-provided shared page's MPN for a VCPU. + * + * Returns: + * An MPN or INVALID_MPN. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +MPN +MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *ctx, + uint64 subIndex, // IN + unsigned page, // IN + Vcpuid vcpu) // IN +{ + return MonLoaderGetSharedRegionMPN(ctx, subIndex, vcpu, page); +} + + +MPN +MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *ctx, // IN + uint64 subIndex, // IN + unsigned page, // IN + Vcpuid vcpu) // IN +{ + NOT_IMPLEMENTED(); +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoaderCallout_IsPrivileged -- + * + * Returns whether or not this is a privileged environment. + * + * Returns: + * TRUE if privileged, FALSE if not. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +Bool +MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *ctx) // IN +{ + return TRUE; +} + +MPN +MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *ctx, // IN + uint64 offset) // IN +{ + return INVALID_MPN; +} + +Bool +MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *ctx, // IN + uint16 codeSelector, // IN + VA64 code, // IN + uint16 stackSelector, // IN + VA64 stack) // IN +{ + NOT_IMPLEMENTED(); +} diff --git a/vmmon-only/common/apic.c b/vmmon-only/common/apic.c index a9eb1a4..753a729 100644 --- a/vmmon-only/common/apic.c +++ b/vmmon-only/common/apic.c @@ -165,7 +165,7 @@ APIC_ReadID(const APICDescriptor *desc) // IN * * APIC_Write -- * - * Writes the given value to the given APIC register using the proper + * Writes the given value to the given APIC register using the proper * interface. Does not check to see if the register number is valid. * * Side effects: @@ -195,7 +195,7 @@ APIC_Write(const APICDescriptor *desc, // IN * * APIC_ReadICR -- * - * Reads the APIC ICR using the proper interface. The semantics of the + * Reads the APIC ICR using the proper interface. The semantics of the * ICR are different in X2APIC mode so APIC_Read() should not be used. * * Side effects: @@ -207,7 +207,7 @@ APIC_Write(const APICDescriptor *desc, // IN *---------------------------------------------------------------------- */ -uint64 +uint64 APIC_ReadICR(const APICDescriptor *desc) // IN { if (desc->isX2) { @@ -225,7 +225,7 @@ APIC_ReadICR(const APICDescriptor *desc) // IN * * APIC_WriteICR -- * - * Writes the given value to the APIC ICR using the proper interface. + * Writes the given value to the APIC ICR using the proper interface. * The semantics of the ICR are different in X2APIC mode so APIC_Write() * should not be used. * diff --git a/vmmon-only/common/comport.c b/vmmon-only/common/comport.c index 4d59117..29291b2 100644 --- a/vmmon-only/common/comport.c +++ b/vmmon-only/common/comport.c @@ -19,20 +19,20 @@ /* * comport.c -- * - * Simple COM1 port routines for debugging code that can't do any real + * Simple COM1 port routines for debugging code that can't do any real * host IO, such as the worldswitch and related. * - * They all wait for the last character to complete transmission so if the - * system crashes immediately on return, the last character will be seen + * They all wait for the last character to complete transmission so if the + * system crashes immediately on return, the last character will be seen * by the remote end. * - * These routines do not have any external dependencies so can be called + * These routines do not have any external dependencies so can be called * by any code that has privilege to access IO ports. * - * Under Windows, they can be made to forward output to DbgPrint for - * printing via the debugger. Just have USE_DBGPRINT set to 1. If you - * let USE_DBGPRINT be 0 with Windows, make sure the comport hardware is - * powered-up by leaving Hyperterm running with the comport open, else + * Under Windows, they can be made to forward output to DbgPrint for + * printing via the debugger. Just have USE_DBGPRINT set to 1. If you + * let USE_DBGPRINT be 0 with Windows, make sure the comport hardware is + * powered-up by leaving Hyperterm running with the comport open, else * Windows will power the chip down. */ diff --git a/vmmon-only/common/cpuid.c b/vmmon-only/common/cpuid.c index 4984eb0..04a071e 100644 --- a/vmmon-only/common/cpuid.c +++ b/vmmon-only/common/cpuid.c @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998 VMware, Inc. All rights reserved. + * Copyright (C) 1998, 2016-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -31,6 +31,8 @@ #include "hostif.h" #include "cpuid.h" #include "x86cpuid_asm.h" +#include "x86svm.h" +#include "x86vt.h" uint32 cpuidFeatures; static CpuidVendor vendor = CPUID_NUM_VENDORS; @@ -111,68 +113,10 @@ CPUID_GetVendor(void) uint32 CPUID_GetVersion(void) { - ASSERT(vendor != CPUID_NUM_VENDORS); return version; } -/* - *----------------------------------------------------------------------------- - * - * CPUID_SyscallSupported -- - * - * Determine whether processor supports syscall opcode and MSRs. - * - * Results: - * FALSE if processor does not support syscall - * TRUE if processor supports syscall - * - * Side effects: - * It determines value only on first call, caching it for future. - * - *----------------------------------------------------------------------------- - */ - -Bool -CPUID_SyscallSupported(void) -{ - /* - * It is OK to use local static variables here as 'result' does not depend - * on any work done in CPUID_Init(). It purely depends on the CPU. - */ - static Bool initialized = FALSE; - static Bool result; - - if (UNLIKELY(!initialized)) { - result = CPUIDExtendedSupported() >= 0x80000001 - && (__GET_EDX_FROM_CPUID(0x80000001) & (1 << 11)); - initialized = TRUE; - } - - return result; -} - - -Bool -CPUID_LongModeSupported(void) -{ - /* - * It is OK to use local static variables here as 'result' does not depend - * on any work done in CPUID_Init(). It purely depends on the CPU. - */ - static Bool initialized = FALSE; - static Bool result; - - if (UNLIKELY(!initialized)) { - result = CPUIDExtendedSupported() >= 0x80000001 - && (__GET_EDX_FROM_CPUID(0x80000001) & (1 << 29)); - initialized = TRUE; - } - - return result; -} - - /* *----------------------------------------------------------------------------- * @@ -207,3 +151,28 @@ CPUID_AddressSizeSupported(void) return result; } + + +/* + *----------------------------------------------------------------------------- + * + * CPUID_HostSupportsHV -- + * + * Determine whether processor supports hardware virtualization. Two + * possibilities are valid: VMX on Intel or SVM on AMD. + * + * Results: + * True iff the processor supports the required features. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +Bool +CPUID_HostSupportsHV(void) +{ + return (vendor == CPUID_VENDOR_AMD && SVM_CapableCPU()) || + (vendor == CPUID_VENDOR_INTEL && VT_CapableCPU()); +} diff --git a/vmmon-only/common/cpuid.h b/vmmon-only/common/cpuid.h index 59bb806..c21b071 100644 --- a/vmmon-only/common/cpuid.h +++ b/vmmon-only/common/cpuid.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998,2016 VMware, Inc. All rights reserved. + * Copyright (C) 1998, 2016-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -36,11 +36,10 @@ extern uint32 cpuidFeatures; void CPUID_Init(void); CpuidVendor CPUID_GetVendor(void); uint32 CPUID_GetVersion(void); -Bool CPUID_SyscallSupported(void); -Bool CPUID_LongModeSupported(void); Bool CPUID_AddressSizeSupported(void); +Bool CPUID_HostSupportsHV(void); -static INLINE uint32 +static INLINE uint32 CPUID_GetFeatures(void) { return cpuidFeatures; @@ -53,4 +52,3 @@ CPUID_SSE2Supported(void) } #endif - diff --git a/vmmon-only/common/hashFunc.c b/vmmon-only/common/hashFunc.c index b851082..38f0516 100644 --- a/vmmon-only/common/hashFunc.c +++ b/vmmon-only/common/hashFunc.c @@ -20,7 +20,7 @@ /* * hashFunc.c -- * - * The core implementation lives in lib/shared/hashFunc.h because it + * The core implementation lives in lib/shared/hashFunc.h because it * is shared by the vmkernel and vmmon. */ @@ -31,10 +31,10 @@ */ // arbitrary constant -#define HASH_INIT_VALUE (42) +#define HASH_INIT_VALUE (42) // 64-bit hash for one 4K page -uint64 +uint64 HashFunc_HashPage(const void *addr) { return hash2((uint64 *)addr, PAGE_SIZE / sizeof (uint64), HASH_INIT_VALUE); diff --git a/vmmon-only/common/hostif.h b/vmmon-only/common/hostif.h index 2b94ce1..c11b865 100644 --- a/vmmon-only/common/hostif.h +++ b/vmmon-only/common/hostif.h @@ -17,8 +17,8 @@ *********************************************************/ /* - * hostif.h - Platform dependent interface for supporting - * the vmx86 device driver. + * hostif.h - Platform dependent interface for supporting + * the vmx86 device driver. */ @@ -36,7 +36,7 @@ #include "hostifGlobalLock.h" /* - * Host-specific definitions. + * Host-specific definitions. */ #if !__linux__ && !defined(WINNT_DDK) && !defined __APPLE__ #error "Only Linux or NT or Mac OS defined for now." @@ -69,11 +69,12 @@ typedef enum { EXTERN int HostIF_Init(VMDriver *vm); EXTERN int HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn); EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr); -EXTERN void HostIF_InitFP(VMDriver *vm); EXTERN void *HostIF_AllocPage(void); EXTERN void HostIF_FreePage(void *ptr); +EXTERN VPN HostIF_MapPage(MPN mpn); +EXTERN void HostIF_UnmapPage(VPN vpn); EXTERN int HostIF_LockPage(VMDriver *vm, VA64 uAddr, Bool allowMultipleMPNsPerVA, MPN *mpn); EXTERN int HostIF_UnlockPage(VMDriver *vm, VA64 uAddr); @@ -86,9 +87,8 @@ EXTERN unsigned int HostIF_EstimateLockedPageLimit(const VMDriver *vm, unsigned int lockedPages); EXTERN void HostIF_Wait(unsigned int timeoutMs); EXTERN void HostIF_WaitForFreePages(unsigned int timeoutMs); -EXTERN void *HostIF_AllocCrossGDT(uint32 numPages, MPN maxValidFirst, - MPN *crossGDTMPNs); -EXTERN void HostIF_FreeCrossGDT(uint32 numPages, void *crossGDT); +EXTERN void *HostIF_AllocKernelPages(unsigned numPages, MPN *mpns); +EXTERN void HostIF_FreeKernelPages(unsigned numPages, void *ptr); EXTERN void HostIF_VMLock(VMDriver *vm, int callerID); EXTERN void HostIF_VMUnlock(VMDriver *vm, int callerID); #ifdef VMX86_DEBUG @@ -119,12 +119,11 @@ EXTERN int HostIF_AllocLockedPages(VMDriver *vm, VA64 addr, EXTERN int HostIF_FreeLockedPages(VMDriver *vm, VA64 addr, unsigned int numPages, Bool kernelMPNBuffer); EXTERN MPN HostIF_GetNextAnonPage(VMDriver *vm, MPN mpn); -EXTERN int HostIF_GetLockedPageList(VMDriver *vm, VA64 uAddr, - unsigned int numPages); -EXTERN int HostIF_ReadPage(VMDriver *vm, MPN mpn, VA64 addr, Bool kernelBuffer); -EXTERN int HostIF_WritePage(VMDriver *vm, MPN mpn, VA64 addr, - Bool kernelBuffer); +EXTERN int HostIF_ReadPhysical(VMDriver *vm, MA ma, VA64 addr, + Bool kernelBuffer, size_t len); +EXTERN int HostIF_WritePhysical(VMDriver *vm, MA ma, VA64 addr, + Bool kernelBuffer, size_t len); EXTERN int HostIF_WriteMachinePage(MPN mpn, VA64 addr); #if defined __APPLE__ // There is no need for a fast clock lock on Mac OS. @@ -141,6 +140,9 @@ EXTERN void HostIF_FreeMachinePage(MPN mpn); EXTERN int HostIF_SafeRDMSR(uint32 msr, uint64 *val); +EXTERN int HostIF_CopyFromUser(void *dst, VA64 src, size_t len); +EXTERN int HostIF_CopyToUser(VA64 dst, const void *src, size_t len); + #if defined __APPLE__ EXTERN void HostIF_PageUnitTest(void); #endif diff --git a/vmmon-only/common/memtrack.c b/vmmon-only/common/memtrack.c index eec9844..b8dd351 100644 --- a/vmmon-only/common/memtrack.c +++ b/vmmon-only/common/memtrack.c @@ -151,7 +151,7 @@ typedef struct MemTrack { unsigned numPages; /* Number of pages tracked. */ MemTrackDir1 dir1; /* First level directory. */ MemTrackHT vpnHashTable; /* VPN to entry hashtable. */ - MemTrackHT mpnHashTable; /* MPN to entry hashtable. */ + MemTrackHT *mpnHashTable; /* MPN to entry hashtable. */ } MemTrack; /* @@ -276,6 +276,7 @@ MemTrackCleanup(MemTrack *mt) // IN unsigned idx; unsigned p1; MemTrackDir1 *dir1; + Bool freeBackMap = mt != NULL && mt->mpnHashTable != NULL; if (mt == NULL) { return; @@ -304,10 +305,15 @@ MemTrackCleanup(MemTrack *mt) // IN if (mt->vpnHashTable.pages[idx] != NULL) { HostIF_FreePage(mt->vpnHashTable.pages[idx]); } - if (mt->mpnHashTable.pages[idx] != NULL) { - HostIF_FreePage(mt->mpnHashTable.pages[idx]); + if (freeBackMap) { + if (mt->mpnHashTable->pages[idx] != NULL) { + HostIF_FreePage(mt->mpnHashTable->pages[idx]); + } } } + if (freeBackMap) { + HostIF_FreeKernelMem(mt->mpnHashTable); + } HostIF_FreeKernelMem(mt); } @@ -359,14 +365,19 @@ MemTrack_Init(VMDriver *vm) // IN: mt->vpnHashTable.pages[idx] = htPage; } + mt->mpnHashTable = HostIF_AllocKernelMem(sizeof *mt->mpnHashTable, FALSE); + if (mt->mpnHashTable == NULL) { + Warning("MemTrack failed to allocate MPN hash table.\n"); + goto error; + } + memset(mt->mpnHashTable, 0, sizeof *mt->mpnHashTable); for (idx = 0; idx < MEMTRACK_HT_PAGES; idx++) { MemTrackHTPage *htPage = MemTrackAllocPage(); - if (htPage == NULL) { Warning("MemTrack failed to allocate MPN hash table (%d).\n", idx); goto error; } - mt->mpnHashTable.pages[idx] = htPage; + mt->mpnHashTable->pages[idx] = htPage; } return mt; @@ -406,6 +417,7 @@ MemTrack_Add(MemTrack *mt, // IN MemTrackDir3 *dir3; MEMTRACK_IDX2DIR(idx, p1, p2, p3); + ASSERT(HostIF_VMLockIsHeld(mt->vm)); if (p1 >= MEMTRACK_DIR1_ENTRIES || @@ -429,7 +441,7 @@ MemTrack_Add(MemTrack *mt, // IN ent->mpn = mpn; MemTrackHTInsert(&mt->vpnHashTable, ent, &ent->vpnChain, ent->vpn); - MemTrackHTInsert(&mt->mpnHashTable, ent, &ent->mpnChain, ent->mpn); + MemTrackHTInsert(mt->mpnHashTable, ent, &ent->mpnChain, ent->mpn); mt->numPages++; @@ -492,7 +504,7 @@ MemTrack_LookupMPN(MemTrack *mt, // IN { MemTrackEntry *next; ASSERT(HostIF_VMLockIsHeld(mt->vm)); - next = *MemTrackHTLookup(&mt->mpnHashTable, mpn); + next = *MemTrackHTLookup(mt->mpnHashTable, mpn); while (next != NULL) { if (next->mpn == mpn) { diff --git a/vmmon-only/common/memtrack.h b/vmmon-only/common/memtrack.h index 6d104ad..1f9cb78 100644 --- a/vmmon-only/common/memtrack.h +++ b/vmmon-only/common/memtrack.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. + * Copyright (C) 1998,2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -20,7 +20,7 @@ * memtrack.h -- * * Utility module for tracking pinned memory, which allows later - * lookup by VPN. + * lookup by VPN and optionally by MPN. */ diff --git a/vmmon-only/common/phystrack.c b/vmmon-only/common/phystrack.c index 5972e24..cd95fdb 100644 --- a/vmmon-only/common/phystrack.c +++ b/vmmon-only/common/phystrack.c @@ -226,7 +226,7 @@ PhysTrack_Alloc(VMDriver *vm) } return tracker; -} +} /* @@ -238,7 +238,7 @@ PhysTrack_Alloc(VMDriver *vm) * * Results: * reallocates all structures, including 'tracker' - * + * * Side effects: * tracker deallocated * @@ -246,7 +246,7 @@ PhysTrack_Alloc(VMDriver *vm) */ void -PhysTrack_Free(PhysTracker *tracker) +PhysTrack_Free(PhysTracker *tracker) { unsigned int p1; diff --git a/vmmon-only/common/task.c b/vmmon-only/common/task.c index 228996e..abb6630 100644 --- a/vmmon-only/common/task.c +++ b/vmmon-only/common/task.c @@ -62,6 +62,9 @@ #include "x86vtinstr.h" #include "apic.h" #include "x86perfctr.h" +#include "x86paging_common.h" +#include "x86paging_64.h" +#include "memtrack.h" #ifdef LINUX_GDT_IS_RO # include @@ -85,12 +88,32 @@ static CrossGDT *crossGDT = NULL; static MPN crossGDTMPNs[CROSSGDT_NUMPAGES]; -static DTR crossGDTDescHKLA; static Selector kernelStackSegment = 0; static uint32 dummyLVT; static Atomic_uint64 hvRootPage[MAX_PCPUS]; static Atomic_Ptr tmpGDT[MAX_PCPUS]; static Bool pebsAvailable = FALSE; +static Bool ptAvailable = FALSE; + +#define L1_FLAGS (PTE_RW | PTE_A | PTE_D | PTE_P) +#define L2_FLAGS (PTE_RW | PTE_A | PTE_D) +#define L3_FLAGS (PTE_RW | PTE_A | PTE_D) +#define L4_FLAGS (PTE_RW | PTE_A | PTE_D) + +#define CANONICAL_MASK MASK64(36) + +/* + * Index of the PTE in the given PT level when translating the given + * LPN. This index is global, i.e. it considers the full page table + * hierarchy, not just the containing page table. + */ +#define PTE_GLOBAL_INDEX(_lpn, _lvl) (((_lpn) & CANONICAL_MASK) >> ((_lvl) - 1) * 9) + +/* + * Index of the PTE in the page table at the given PT level when + * translating the given LPN. + */ +#define PTE_INDEX(_lpn, _lvl) (PTE_GLOBAL_INDEX(_lpn, _lvl) & PT_OFF_MASK) /* @@ -148,7 +171,7 @@ TaskAllocHVRootPage(Atomic_uint64 *slot) // IN/OUT * the same 'slot', only the first one to pass this finish line will win. */ if (HostIF_WriteMachinePage(mpn, PtrToVA64(content)) != 0 || - !Atomic_CMPXCHG64(slot, &invalidMPN, &mpn)) { + !Atomic_CMPXCHG64(slot, invalidMPN, mpn)) { /* * Either we couldn't set up the page or this thread lost the race. * We must free its HV root page. @@ -507,10 +530,8 @@ Task_Terminate(void) TaskFreeHVRootPages(); if (crossGDT != NULL) { - HostIF_FreeCrossGDT(CROSSGDT_NUMPAGES, crossGDT); + HostIF_FreeKernelPages(ARRAYSIZE(crossGDTMPNs), crossGDT); crossGDT = NULL; - crossGDTDescHKLA.limit = 0; - crossGDTDescHKLA.offset = 0; } if (USE_TEMPORARY_GDT) { @@ -600,6 +621,11 @@ gotnzss:; */ pebsAvailable = PerfCtr_PEBSAvailable(); + /* + * Check if PT is supported. For simplicity we assume there will not + * be mixed CPU models. + */ + ptAvailable = PerfCtr_PTAvailable(); return TRUE; } @@ -731,12 +757,248 @@ TaskRestoreHostGDTTRLDT(Descriptor *tempGDTBase, /* *----------------------------------------------------------------------------- * - * Task_AllocCrossGDT -- + * TaskSwitchPTPAllocPage -- * - * Make sure the crossGDT is allocated and initialized. + * Allocates and zeroes a page to be used as part of a page table patch. + * Adds a corresponding entry to the PTP memtracker. * * Results: - * TRUE iff crossGDT was already initialized or successfully initialized. + * On success, a pointer to the page's starting address. NULL on + * failure. + * + *----------------------------------------------------------------------------- + */ + +static void * +TaskSwitchPTPAllocPage(VMDriver *vm) +{ + MPN mpn; + MemTrackEntry *entry; + uint8 *ptr; + + ptr = HostIF_AllocKernelPages(1, &mpn); + if (ptr == NULL) { + return NULL; + } + memset(ptr, 0, PAGE_SIZE); + entry = MemTrack_Add(vm->ptpTracker, VA_2_VPN(PtrToVA64(ptr)), mpn); + + if (entry == NULL) { + HostIF_FreeKernelPages(1, ptr); + return NULL; + } + + return ptr; +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskSwitchPTPLookupVPN -- + * + * Looks up the MPN for the given PTP page VPN by querying the PTP + * memtracker. + * + * Results: + * The VPN's corresponding MPN. + * + *----------------------------------------------------------------------------- + */ + +static MPN +TaskSwitchPTPLookupVPN(VMDriver *vm, VPN vpn) +{ + MemTrackEntry *entry = MemTrack_LookupVPN(vm->ptpTracker, vpn); + + ASSERT(entry != NULL && entry->mpn != 0); + return entry->mpn; +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskSwitchPTPPageFree -- + * + * Frees the PTP page matching the VPN stored by the given + * MemTrackEntry. This function is used as a callback by + * MemTrack_Cleanup(). + * + * Results: + * None. + * + *----------------------------------------------------------------------------- + */ + +static void +TaskSwitchPTPPageFree(void *unused, MemTrackEntry *entry) +{ + ASSERT(entry->mpn != 0); + HostIF_FreeKernelPages(1, VA64ToPtr(VPN_2_VA(entry->vpn))); +} + + +/* + *----------------------------------------------------------------------------- + * + * Task_SwitchPTPPageCleanup -- + * + * Frees all PTP pages allocated for the given VM and frees the + * PTP MemTracker. + * + * Results: + * None. + * + *----------------------------------------------------------------------------- + */ + +void +Task_SwitchPTPPageCleanup(VMDriver *vm) +{ + MemTrack_Cleanup(vm->ptpTracker, TaskSwitchPTPPageFree, NULL); + vm->ptpTracker = NULL; +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskSetCrossGDTEntry -- + * + * Fill in a crossGDT entry from the given descriptor. If the entry has + * already been initialized, it is compared to the given descriptor. Any + * discrepancy is logged and an error is returned. + * + * Results: + * TRUE on success, FALSE otherwise. + * + * Side effects: + * crossGDT entry is initialized, if not already. + * + *----------------------------------------------------------------------------- + */ + +static Bool +TaskSetCrossGDTEntry(uint16 index, Descriptor d) +{ + Bool res = FALSE; + + ASSERT(HostIF_GlobalLockIsHeld()); + ASSERT(Desc_Present(&d)); + + if (index >= ARRAYSIZE(crossGDT->gdtes)) { + Warning("%s: index %u too big\n", __FUNCTION__, index); + } else if (!CROSSGDT_TESTINDEXMASK(index)) { + Warning("%s: index %u not in CROSSGDT_PAGEMASK %x\n", __FUNCTION__, + index, CROSSGDT_PAGEMASK); + } else if (!Desc_Present(crossGDT->gdtes + index)) { + crossGDT->gdtes[index] = d; + res = TRUE; + } else if (Desc_EqualIgnoreAccessed(crossGDT->gdtes + index, &d)) { + res = TRUE; + } else { + Warning("%s: entry 0x%X mismatch\n", __FUNCTION__, index); + Warning("%s: crossGDT %16.16" FMT64 "X\n", __FUNCTION__, + *(uint64 *)(crossGDT->gdtes + index)); + Warning("%s: template %16.16" FMT64 "X\n", __FUNCTION__, *(uint64 *)&d); + } + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskSetCrossGDTVMM -- + * + * Initialize/compare the VMM portion of the crossGDT. + * + * Results: + * TRUE if success, FALSE otherwise. + * + * Side effects: + * VMM portion of the crossGDT is initialized, if not already. + * + *----------------------------------------------------------------------------- + */ + +static Bool +TaskSetCrossGDTVMM(BSVMM_GDTInit *gdt) +{ + uint8 i; + + for (i = 0; i < ARRAYSIZE(gdt->entries); i++) { + Descriptor d; + if (gdt->entries[i].present == 1) { + Desc_SetDescriptor(&d, gdt->entries[i].base, + gdt->entries[i].limit, + gdt->entries[i].type, + gdt->entries[i].S, + gdt->entries[i].DPL, + gdt->entries[i].present, + gdt->entries[i].DB, + gdt->entries[i].gran); + d.longmode = gdt->entries[i].longmode; + + if (!TaskSetCrossGDTEntry(gdt->entries[i].index, d)) { + return FALSE; + } + } + } + return TRUE; +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskSetCrossGDTHost -- + * + * Initializes the host portion of the crossGDT by copying it directly from + * the host kernel's GDT. We assume that all the host segments we + * will ever need are below CROSSGDT_HOSTLIMIT. + * + * Results: + * TRUE on success, FALSE otherwise. + * + * Side effects: + * The host portion of the crossGDT is initialized. + * + *----------------------------------------------------------------------------- + */ + +static void +TaskSetCrossGDTHost(void) +{ + DTR64 hostGDT; + /* + * All copied host segment descriptors will come from the first page of + * the host kernel GDT. + */ + ASSERT(HostIF_GlobalLockIsHeld()); + ASSERT_ON_COMPILE(CROSSGDT_HOSTLIMIT <= PAGE_SIZE / sizeof(Descriptor)); + ASSERT_ON_COMPILE(CROSSGDT_PAGEMASK & 1); + + TaskSaveGDT64(&hostGDT); + if (hostGDT.limit > CROSSGDT_HOSTLIMIT * sizeof(Descriptor) - 1) { + hostGDT.limit = CROSSGDT_HOSTLIMIT * sizeof(Descriptor) - 1; + } + TaskCopyGDT64(&hostGDT, crossGDT->gdtes); +} + + +/* + *----------------------------------------------------------------------------- + * + * Task_CreateCrossGDT -- + * + * Allocate and initialize the crossGDT, if not already done. A single + * crossGDT is shared by all VMs managed by this driver. + * + * Results: + * TRUE if success, FALSE otherwise. * * Side effects: * crossGDT static vars set up if not already. @@ -745,99 +1007,483 @@ TaskRestoreHostGDTTRLDT(Descriptor *tempGDTBase, */ Bool -Task_AllocCrossGDT(InitBlock *initBlock) // OUT: crossGDT values filled in +Task_CreateCrossGDT(BSVMM_GDTInit *gdt) { - DTR64 hostGDT64; - - /* - * Make sure only one of these runs at a time on the whole system, because - * there is only one crossGDT for the whole system. - */ - HostIF_GlobalLock(2); - /* - * Maybe the crossGDT has already been set up. - */ - if (crossGDT == NULL) { - MPN maxValidFirst = - 0xFFC00 /* 32-bit MONITOR_LINEAR_START */ - CROSSGDT_NUMPAGES; - /* - * The host entries must fit on pages of the crossGDT that are mapped. - * Since we know they are below CROSSGDT_LOWSEG, we can just check that - * CROSSGDT_LOWSEG and below are mapped. - * - * Because the CROSSGDT_LOWSEG segments must reside on the first page of - * the crossGDT (as they must remain valid with paging off), all we need - * do is check that bit 0 of CROSSGDT_PAGEMASK is set (indicating that - * page 0 of the crossGDT will be mapped). - */ - - ASSERT_ON_COMPILE(CROSSGDT_LOWSEG < PAGE_SIZE); - ASSERT_ON_COMPILE(CROSSGDT_PAGEMASK & 1); - - /* - * Allocate the crossGDT. + * The crossGDT has not yet been created. This must be the first + * VM that this driver powers on. */ ASSERT_ON_COMPILE(sizeof *crossGDT == CROSSGDT_NUMPAGES * PAGE_SIZE); - crossGDT = HostIF_AllocCrossGDT(CROSSGDT_NUMPAGES, maxValidFirst, - crossGDTMPNs); + crossGDT = HostIF_AllocKernelPages(ARRAYSIZE(crossGDTMPNs), + crossGDTMPNs); if (crossGDT == NULL) { HostIF_GlobalUnlock(2); Warning("%s: unable to allocate crossGDT\n", __FUNCTION__); return FALSE; } - - /* - * Check that the crossGDT meets the address requirements documented in - * bora/doc/worldswitch-pages.txt. - */ - - if (crossGDTMPNs[0] > maxValidFirst) { - HostIF_FreeCrossGDT(CROSSGDT_NUMPAGES, crossGDT); - crossGDT = NULL; - HostIF_GlobalUnlock(2); - Warning("%s: crossGDT MPN %"FMT64"X gt %"FMT64"X\n", __FUNCTION__, - crossGDTMPNs[0], maxValidFirst); - - return FALSE; - } - - /* - * Fill the crossGDT with a copy of our host GDT. VMX will have to fill - * in monitor segments via Task_InitCrossGDT. - * - * We are assuming that all the host segments we will ever need are below - * CROSSGDT_LOWSEG. If this assumption ever breaks, the host segments - * would have to be unconditionally transitioned to the CROSSGDT - * intermediate segments before switching to the monitor. The only time - * the GDT has been found to be bigger than CROSSGDT_LOWSEG is when they - * are running KVM or Xen, and we never see the large segment numbers. - */ - memset(crossGDT, 0, sizeof *crossGDT); - TaskSaveGDT64(&hostGDT64); - if (hostGDT64.limit > CROSSGDT_LOWSEG * 8 - 1) { - hostGDT64.limit = CROSSGDT_LOWSEG * 8 - 1; - } - TaskCopyGDT64(&hostGDT64, crossGDT->gdtes); - - /* - * Set up descriptor for the crossGDT using host kernel LA as a base. - */ - - crossGDTDescHKLA.limit = sizeof *crossGDT - 1; - crossGDTDescHKLA.offset = HOST_KERNEL_VA_2_LA((VA)crossGDT); + TaskSetCrossGDTHost(); + } + if (!TaskSetCrossGDTVMM(gdt)) { + HostIF_GlobalUnlock(2); + return FALSE; } HostIF_GlobalUnlock(2); - initBlock->crossGDTHKLA = crossGDTDescHKLA.offset; - ASSERT_ON_COMPILE(sizeof initBlock->crossGDTMPNs == sizeof crossGDTMPNs); - memcpy(initBlock->crossGDTMPNs, crossGDTMPNs, sizeof crossGDTMPNs); + return TRUE; +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskFixupHostSwitchIDTE -- + * + * Fixes up a 64-bit host IDT entry in the crosspage, given the + * host kernel CS and the crosspage's code section address on the + * host. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static void +TaskFixupHostSwitchIDTE(Gate64 *entry, Selector cs, VMCrossPageCode *cpCode) +{ + VA64 handlerVA; + DEBUG_ONLY(const VA64 codeBase = offsetof(VMCrossPageCode, codeBlock);) + + ASSERT_ON_COMPILE(sizeof *entry == 16); + + ASSERT(entry->segment == 0 && entry->ist == 0 && + entry->reserved0 == 0 && entry->type == INTER_GATE && + entry->DPL == 0 && entry->present == 0 && + entry->offset_16_31 == 0 && entry->offset_32_63 == 0 && + entry->reserved1 == 0); + + /* + * entry->offset_0_15 is initialized in crosspage.S with an offset relative + * to the crosspage's code section. Here, we transform it into an absolute + * address. + */ + ASSERT(entry->offset_0_15 >= codeBase && + entry->offset_0_15 < codeBase + sizeof cpCode->codeBlock); + + handlerVA = PtrToVA64(cpCode) + entry->offset_0_15; + + entry->offset_0_15 = LOWORD(handlerVA); + entry->segment = cs; + entry->present = 1; + entry->offset_16_31 = HIWORD(handlerVA); + entry->offset_32_63 = HIDWORD(handlerVA); +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskFixupHostSwitchIDT -- + * + * Fixes up the contents of the host-context switch IDT and IDTR, + * which were partially initialized at compile-time. At this point, + * we have the necessary runtime information needed to fully + * initialize them: the crosspage's address in the host and the + * host kernel's CS. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static void +TaskFixupHostSwitchIDT(VMCrossPage *crosspage) +{ + VMCrossPageData *cpData = &crosspage->crosspageData; + VMCrossPageCode *cpCode = &crosspage->crosspageCode; + const Selector cs = cpData->hostInitial64CS; + Gate64 *idt = (Gate64 *)&cpData->switchHostIDT; + + ASSERT(cpData->switchHostIDTR.limit == sizeof cpData->switchHostIDT - 1); + ASSERT(cpData->switchHostIDTR.offset == 0); + cpData->switchHostIDTR.offset = PtrToVA64(&cpData->switchHostIDT); + + TaskFixupHostSwitchIDTE(&idt[EXC_DB ], cs, cpCode); + TaskFixupHostSwitchIDTE(&idt[EXC_NMI], cs, cpCode); + TaskFixupHostSwitchIDTE(&idt[EXC_UD ], cs, cpCode); + TaskFixupHostSwitchIDTE(&idt[EXC_MC ], cs, cpCode); +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskFixupPatchPT -- + * TaskFixupPatchPTE -- + * + * These functions recursively walk the specified page table patch + * subtree and convert all entries containing a 'VA | flags' into + * their actual PTE values at that level. + * + * When 'PTE_P' is not present in an entry, the value is actually + * a VA. All other flag values, besides 'PTE_P' may be set in + * the entry; the flag values are preserved during conversion to + * a PTE (and 'PTE_P' is included). + * + * Level 1 entries are always entered as true PTE values. Since + * they always have PTE_P set, they never need fixing up. + * + * Results: + * None. + * + *----------------------------------------------------------------------------- + */ + +static void +TaskFixupPatchPT(VMDriver *vm, + VM_PDPTE *ptPage, + unsigned ptLevel); + +static void +TaskFixupPatchPTE(VMDriver *vm, + VM_PDPTE *pte, + unsigned ptLevel) +{ + const VM_PDPTE entry = *pte; + const VM_PDPTE paEntry = VM_PAGE_BASE(entry); + const uint32 flags = entry & (PAGE_SIZE - 1); + + ASSERT_ON_COMPILE(sizeof(VM_PDPTE) == sizeof(uint64)); + ASSERT_ON_COMPILE(sizeof(PT_L4E) == sizeof(VM_PDPTE)); + ASSERT_ON_COMPILE(sizeof(PT_L3E) == sizeof(VM_PDPTE)); + ASSERT_ON_COMPILE(sizeof(PT_L2E) == sizeof(VM_PDPTE)); + ASSERT_ON_COMPILE(sizeof(PT_L1E) == sizeof(VM_PDPTE)); + ASSERT(PTP_LEVEL_L1 <= ptLevel && ptLevel <= PTP_LEVEL_L4); + + if (ptLevel == 1) { + ASSERT(PTE_PRESENT(flags)); /* Already in PTE form. */ + } else if (paEntry != 0) { + MPN mpn; + VM_PDPTE *nextPt = (VM_PDPTE *)(uintptr_t)paEntry; + + ASSERT(!PTE_PRESENT(flags)); /* Not in PTE form. */ + mpn = TaskSwitchPTPLookupVPN(vm, VA_2_VPN(PtrToVA64(nextPt))); + + *pte = LM_MAKE_PTE(mpn, 0, flags | PTE_P); + + TaskFixupPatchPT(vm, nextPt, ptLevel - 1); + } +} + +static void +TaskFixupPatchPT(VMDriver *vm, + VM_PDPTE *ptPage, + unsigned ptLevel) +{ + unsigned i; + + ASSERT(PTP_LEVEL_L1 <= ptLevel && ptLevel <= PTP_LEVEL_L4); + + for (i = 0; i < PAGE_SIZE / sizeof(VM_PDPTE); i++) { + VM_PDPTE *pte = &ptPage[i]; + if (*pte != 0) { + TaskFixupPatchPTE(vm, pte, ptLevel); + } + } +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskFixupPTPatches -- + * + * Fixes up the PTEs for the entire page table hierarchy rooted at + * each PTP. It translates PTEs from the 'VA | flags' intermediate + * format to the actual PTE format. + * + * Results: + * None. + * + *----------------------------------------------------------------------------- + */ + +static void +TaskFixupPTPatches(VMDriver *vm, VMCrossPageData *cpData) +{ + unsigned i; + + for (i = 0; i < MAX_SWITCH_PT_PATCHES; i++) { + VMMPageTablePatch *patch = &cpData->vmmPTP[i]; + if (patch->level == PTP_EMPTY) { + break; + } + TaskFixupPatchPTE(vm, &patch->pte, patch->level); + } +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskSavePTPatch -- + * + * This function saves a page table patch entry. The patch + * entries are used during worldswitch to ensure that the cross + * page and portions of the host GDT are mapped at all times (for + * NMI-safe worldswitches). + * + * The code searches for an empty slot rather than using a static + * local index (incremented after use) because it is run once on + * each VCPU. Each VCPU has a distinct 'crosspage', and thus a + * distinct patch table. + * + * Results: + * Saved patch entry. + * + *----------------------------------------------------------------------------- + */ + +static VMMPageTablePatch * +TaskSavePTPatch(VMCrossPageData *crosspage, + LPN64 lpn, + unsigned level, + VM_PDPTE pte) +{ + unsigned i; + const unsigned pteIdx = PTE_INDEX(lpn, level); + const uint64 pteGlobalIdx = PTE_GLOBAL_INDEX(lpn, level); + + ASSERT(PTP_LEVEL_L1 <= level && level <= PTP_LEVEL_L4); + + for (i = 0; i < MAX_SWITCH_PT_PATCHES; ++i) { + ASSERT(crosspage->vmmPTP[i].level != level || /* No duplicates. */ + crosspage->vmmPTP[i].pteGlobalIdx != pteGlobalIdx || + crosspage->vmmPTP[i].pteIdx != pteIdx); + + if (crosspage->vmmPTP[i].level == PTP_EMPTY) { + crosspage->vmmPTP[i].level = level; + crosspage->vmmPTP[i].pteIdx = pteIdx; + crosspage->vmmPTP[i].pte = pte; + crosspage->vmmPTP[i].pteGlobalIdx = pteGlobalIdx; + + return &crosspage->vmmPTP[i]; + } + } + Panic("Internal error: PTP table is full"); +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskSearchPTPatch -- + * + * Looks for an entry in the PT patch table. + * + * Results: + * Pointer to the PTP entry, or NULL if not found. + * + *----------------------------------------------------------------------------- + */ + +static VMMPageTablePatch * +TaskSearchPTPatch(VMCrossPageData *cpData, + LPN64 lpn, + unsigned level) +{ + unsigned i; + + ASSERT(PTP_LEVEL_L1 <= level && level <= PTP_LEVEL_L4); + + for (i = 0; i < MAX_SWITCH_PT_PATCHES; i++) { + VMMPageTablePatch *patch = &cpData->vmmPTP[i]; + if (patch->level == level && + patch->pteGlobalIdx == PTE_GLOBAL_INDEX(lpn, level)) { + + return patch; + } + } + return NULL; +} + +/* + *----------------------------------------------------------------------------- + * + * TaskCreatePTPatch -- + * + * Creates a page table patch for BackToHost to apply to the + * monitor's page table before switching to the host. These patches + * map the crosspage and crossGDT at their host addresses so there + * is common ground during the switch. + * + * A patch is essentially a PTE-rooted page table subtree. The + * location of the root PTE is determined by calculating the lowest + * page table that is shared with the monitor in the translation + * path for the LPN associated with the patch. We disallow patches + * inside the monitor address space. + * + * At creation time, the patch's PTEs are initialized to a 'VA | flags' + * intermediary representation, where VA is the virtual address of + * a page, and 'flags' are the PTE flags. This allows easy + * traversal of the structure during patch creation. PTEs are fixed + * up later on after all patches have been created. + * + * Results: + * TRUE if the patch was created successfully, FALSE otherwise. + * + * Side effects: + * Patch is created and saved to the crosspage's PTP table. + * + *----------------------------------------------------------------------------- + */ + +static Bool +TaskCreatePTPatch(VMDriver *vm, + VMCrossPageData *cpData, + LPN64 monStartLPN, + LPN64 monEndLPN, + LPN64 lpn, + MPN mpn, + uint16 *numPages) +{ + const PT_L1E l1pte = LM_MAKE_PTE(mpn, 0, L1_FLAGS); + const int flags[4] = {L1_FLAGS, L2_FLAGS, L3_FLAGS, L4_FLAGS}; + VM_PDPTE *pte; + unsigned level; + VMMPageTablePatch *patch; + + ASSERT_ON_COMPILE(sizeof(PT_L4E) == sizeof(VM_PDPTE)); + ASSERT_ON_COMPILE(sizeof(PT_L3E) == sizeof(VM_PDPTE)); + ASSERT_ON_COMPILE(sizeof(PT_L2E) == sizeof(VM_PDPTE)); + ASSERT_ON_COMPILE(sizeof(PT_L1E) == sizeof(VM_PDPTE)); + + /* + * Find the level of the first non-VMM PTE in the LPN's translation + * path. This will be the level in which the patch's root PTE will be + * placed. + */ + for (level = 4; level > 0; level--) { + uint64 lpnIdx = PTE_GLOBAL_INDEX(lpn, level); + if (lpnIdx < PTE_GLOBAL_INDEX(monStartLPN, level) || + lpnIdx > PTE_GLOBAL_INDEX(monEndLPN, level)) { + break; + } + } + + /* + * Ensure that we actually found something. If not (i.e. the l4e, l3e, + * l2e and l1e are all VMM entries), the LPN overlaps with the VMM's + * address space, so we return an error. + */ + if (level == 0) { + Warning("%s: cannot create page table patch because LPN 0x%" FMT64 "x " + "overlaps with the VMM address space.\n", __FUNCTION__, lpn); + return FALSE; + } + + patch = TaskSearchPTPatch(cpData, lpn, level); + if (patch == NULL) { + patch = TaskSavePTPatch(cpData, lpn, level, 0); /* PTE filled later */ + } + + /* + * Traverse the patch-rooted page table subtree until we reach L1. + * Allocate new page tables as needed. + */ + pte = &patch->pte; + while (level > 1) { + VM_PDPTE *page; + ASSERT(!PTE_PRESENT(*pte & (PAGE_SIZE - 1))); /* Pre-fixup entry. */ + page = (VM_PDPTE *)VM_PAGE_BASE((uintptr_t)*pte); + if (page == NULL) { + page = TaskSwitchPTPAllocPage(vm); + if (page == NULL) { + Warning("%s: failure allocating memory for page table patch.\n", + __FUNCTION__); + return FALSE; + } + *numPages += 1; + *pte = (VM_PDPTE)((uintptr_t)page | flags[level - 1]); /* VA | flags */ + } + level -= 1; + pte = (VM_PDPTE *)&page[PTE_INDEX(lpn, level)]; + } + + ASSERT(level == 1 && *pte == 0); + ASSERT_ON_COMPILE(PTE_PRESENT(L1_FLAGS)); + *pte = l1pte; + + return TRUE; +} + + +/* + *----------------------------------------------------------------------------- + * + * TaskCreatePTPatches + * + * Creates VMM mappings for the crosspage and crossGDT that match + * those from the vmmon address space, so that they are mapped at + * a common linear address on both address spaces. Such mappings + * are created in the form of patches that are applied to/removed + * from the VMM's page tables. + * + * Results: + * TRUE if all patches were created successfully, FALSE otherwise. + * + *----------------------------------------------------------------------------- + */ + +static Bool +TaskCreatePTPatches(VMDriver *vm, + VMCrossPage *crosspage, + LPN64 monStartLPN, + LPN64 monEndLPN, + uint16 *numPages) +{ + VMCrossPageData * const cpData = &crosspage->crosspageData; + LPN64 xgCPN = LA_2_LPN(cpData->vmm64CrossGDTLA); + LPN64 xpCPN = LA_2_LPN(cpData->vmm64CrossPageLA); + MPN xpMPN = MA_2_MPN(cpData->crosspageMA); + int i; + + /* + * Set up patches that the BackToHost code will use to map the + * crosspage and crossGDT at their crossover addresses (HKLA for + * the host). + */ + if (!TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN, xpCPN, xpMPN, + numPages)) { + return FALSE; + } + + for (i = 0; CROSSGDT_PAGEMASK >> i; i++) { + if ((CROSSGDT_PAGEMASK >> i) & 1) { + /* + * Set up patches for just first and last pages of cross-gdt + * due to the selected value of CROSSGDT_PAGEMASK. + */ + if (!TaskCreatePTPatch(vm, cpData, monStartLPN, monEndLPN, xgCPN + i, + crossGDTMPNs[i], numPages)) { + return FALSE; + } + } + } + TaskFixupPTPatches(vm, cpData); return TRUE; } @@ -866,12 +1512,13 @@ Task_InitCrosspage(VMDriver *vm, // IN // VM { Vcpuid vcpuid; + uint16 numPTPPages = 0; if (crossGDT == NULL) { return 1; } - initParams->crossGDTHKLA = crossGDTDescHKLA.offset; + initParams->crossGDTHKLA = HOST_KERNEL_VA_2_LA((VA)crossGDT); ASSERT_ON_COMPILE(sizeof initParams->crossGDTMPNs == sizeof crossGDTMPNs); memcpy(initParams->crossGDTMPNs, crossGDTMPNs, sizeof crossGDTMPNs); @@ -936,7 +1583,20 @@ Task_InitCrosspage(VMDriver *vm, // IN } p->crosspageData.crosspageMA = (uint32)MPN_2_MA(crossPageMPN); p->crosspageData.hostCrossPageLA = (LA64)(uintptr_t)p; + p->crosspageData.vmm64CrossPageLA = p->crosspageData.hostCrossPageLA; + p->crosspageData.vmm64CrossGDTLA = HOST_KERNEL_VA_2_LA((VA)crossGDT); + HostIF_VMLock(vm, 39); + if (!TaskCreatePTPatches(vm, p, + initParams->monStartLPN, + initParams->monEndLPN, + &numPTPPages)) { + HostIF_VMUnlock(vm, 39); + Warning("%s: Could not create page table patches for VCPU %d\n", + __FUNCTION__, vcpuid); + return 1; + } + HostIF_VMUnlock(vm, 39); /* * Pass our kernel code segment numbers back to MonitorPlatformInit. * They have to be in the GDT so they will be valid when the crossGDT is @@ -948,85 +1608,29 @@ Task_InitCrosspage(VMDriver *vm, // IN SELECTOR_TABLE(p->crosspageData.hostInitial64CS) == 0); p->crosspageData.moduleCallInterrupted = FALSE; + VersionedAtomic_BeginWrite(&p->crosspageData.pseudoTSCConv.vers); p->crosspageData.pseudoTSCConv.p.mult = 1; p->crosspageData.pseudoTSCConv.p.shift = 0; p->crosspageData.pseudoTSCConv.p.add = 0; - p->crosspageData.pseudoTSCConv.changed = TRUE; + VersionedAtomic_EndWrite(&p->crosspageData.pseudoTSCConv.vers); p->crosspageData.worldSwitchPTSC = Vmx86_GetPseudoTSC(); p->crosspageData.timerIntrTS = MAX_ABSOLUTE_TS; p->crosspageData.hstTimerExpiry = MAX_ABSOLUTE_TS; p->crosspageData.monTimerExpiry = MAX_ABSOLUTE_TS; vm->crosspage[vcpuid] = p; + + TaskFixupHostSwitchIDT(p); } + /* + * Report back to the VMX the number of pages allocated for this VM's + * page table patches. + */ + initParams->numPTPPages = numPTPPages; return 0; } -/* - *----------------------------------------------------------------------------- - * - * Task_InitCrossGDT -- - * - * Fill in a crossGDT entry from the given template. - * - * Results: - * 0 on success - * != 0 on failure - * - * Side effects: - * CrossGDT entry filled from template. If crossGDT has already been - * initialized, the entry is compared to the given template. Any - * discrepancy is logged and an error is returned. This is necessary - * because this same GDT is shared among all VMs on this host, so really, - * the first call initializes it and the others just do compares. - * - *----------------------------------------------------------------------------- - */ - -int -Task_InitCrossGDT(InitCrossGDT *initCrossGDT) // IN -{ - Descriptor v; - int rc; - uint32 i; - - rc = 1; - i = initCrossGDT->index; - v = initCrossGDT->value; - - HostIF_GlobalLock(3); - if (i >= sizeof crossGDT->gdtes / sizeof crossGDT->gdtes[0]) { - HostIF_GlobalUnlock(3); - Warning("%s: index %u too big\n", __FUNCTION__, i); - } else if (!( (1 << (i * sizeof crossGDT->gdtes[0] / PAGE_SIZE)) - & CROSSGDT_PAGEMASK)) { - HostIF_GlobalUnlock(3); - Warning("%s: index %u not in CROSSGDT_PAGEMASK %x\n", __FUNCTION__, - i, CROSSGDT_PAGEMASK); - } else if (!Desc_Present(&v)) { - HostIF_GlobalUnlock(3); - Warning("%s: entry %u not present\n", __FUNCTION__, i); - } else if (!Desc_Present(crossGDT->gdtes + i)) { - crossGDT->gdtes[i] = v; - HostIF_GlobalUnlock(3); - rc = 0; - } else if (Desc_EqualIgnoreAccessed(crossGDT->gdtes + i, &v)) { - HostIF_GlobalUnlock(3); - rc = 0; - } else { - HostIF_GlobalUnlock(3); - Warning("%s: entry 0x%X mismatch\n", __FUNCTION__, i); - Warning("%s: crossGDT %16.16llX\n", __FUNCTION__, - (long long unsigned)*(uint64 *)(crossGDT->gdtes + i)); - Warning("%s: template %16.16llX\n", __FUNCTION__, - (long long unsigned)*(uint64 *)&v); - } - - return rc; -} - - /* *----------------------------------------------------------------------------- * @@ -1172,14 +1776,14 @@ TaskDisableTF(void) static INLINE Bool TaskGotException(const VMCrossPage *crosspage, unsigned exc) { - return crosspage->crosspageCode.faultHandler.wsException[exc]; + return crosspage->crosspageData.wsException[exc]; } static INLINE void TaskSetException(VMCrossPage *crosspage, unsigned exc, Bool v) { - crosspage->crosspageCode.faultHandler.wsException[exc] = v; + crosspage->crosspageData.wsException[exc] = v; } @@ -1375,7 +1979,7 @@ TaskUpdateLatestPTSC(VMDriver *vm, VMCrossPageData *crosspage) if (ptsc <= latest) { break; } - } while (!Atomic_CMPXCHG64(&vm->ptscLatest, &latest, &ptsc)); + } while (!Atomic_CMPXCHG64(&vm->ptscLatest, latest, ptsc)); /* After updating the latest PTSC, decrement the reference count. */ Atomic_Dec32((Atomic_uint32 *)&vm->ptscOffsetInfo.inVMMCnt); } @@ -1519,10 +2123,11 @@ TaskUpdatePTSCParameters(VMDriver *vm, } new.inVMMCnt++; } while (!Atomic_CMPXCHG64((Atomic_uint64 *)&vm->ptscOffsetInfo, - (uint64 *)&old, (uint64 *)&new)); + *(uint64 *)&old, *(uint64 *)&new)); /* Use the designated global offset as this thread's offset. */ + VersionedAtomic_BeginWrite(&crosspage->pseudoTSCConv.vers); crosspage->pseudoTSCConv.p.add = vm->ptscOffsets[new.vcpuid]; - crosspage->pseudoTSCConv.changed = TRUE; + VersionedAtomic_EndWrite(&crosspage->pseudoTSCConv.vers); /* * Need to derive the worldSwitchPTSC value from TSC since the * PTSC, when calculated from TSC, may drift from the reference @@ -1530,8 +2135,9 @@ TaskUpdatePTSCParameters(VMDriver *vm, */ ptsc = tsc + crosspage->pseudoTSCConv.p.add; } else { + VersionedAtomic_BeginWrite(&crosspage->pseudoTSCConv.vers); crosspage->pseudoTSCConv.p.add = ptsc - tsc; - crosspage->pseudoTSCConv.changed = TRUE; + VersionedAtomic_EndWrite(&crosspage->pseudoTSCConv.vers); } /* Cache PTSC value for BackToHost. */ crosspage->worldSwitchPTSC = ptsc; @@ -1581,8 +2187,8 @@ TaskUpdatePTSCParameters(VMDriver *vm, static INLINE_SINGLE_CALLER void TaskSwitchToMonitor(VMCrossPage *crosspage) { - const uint8 *codePtr = ((uint8 *)&crosspage->crosspageCode.worldswitch + - crosspage->crosspageCode.worldswitch.hostToVmm); + const uint8 *codePtr = ((uint8 *)&crosspage->crosspageCode + + crosspage->crosspageCode.offsets.hostToVmm); #if defined(__GNUC__) /* @@ -1751,6 +2357,7 @@ Task_Switch(VMDriver *vm, // IN uint64 fs64 = 0; uint64 gs64 = 0; uint64 kgs64 = 0; + uint64 ptMSR = 0; uint64 pebsMSR = 0; DTR64 hostGDT64, hostIDT64; Selector cs, ds, es, fs, gs, ss; @@ -1783,9 +2390,7 @@ Task_Switch(VMDriver *vm, // IN * world switch. We must be careful not to overwrite the * crosspages arguments when doing this though, see bug 820257. */ - if (hvRootMPN == INVALID_MPN && - (crosspage->crosspageData.activateVMX || - crosspage->crosspageData.activateSVM)) { + if (hvRootMPN == INVALID_MPN) { crosspage->crosspageData.userCallType = MODULECALL_USERCALL_NONE; crosspage->crosspageData.moduleCallType = MODULECALL_ALLOC_VMX_PAGE; crosspage->crosspageData.pcpuNum = pCPU; @@ -1816,6 +2421,17 @@ Task_Switch(VMDriver *vm, // IN } } + /* + * The same as PEBS disable PT(Processor Trace) if it is supported and + * enabled. + */ + if (ptAvailable) { + ptMSR = __GET_MSR(MSR_RTIT_CTL); + if ((ptMSR & MSR_RTIT_CTL_TRACE_EN) != 0) { + __SET_MSR(MSR_RTIT_CTL, ptMSR & ~MSR_RTIT_CTL_TRACE_EN); + } + } + /* * Save the host's standard IDT and set up an IDT that only * has space for all the hardware exceptions (though only a @@ -1826,7 +2442,7 @@ Task_Switch(VMDriver *vm, // IN TaskLoadIDT64(&crosspage->crosspageData.switchHostIDTR); TaskTestCrossPageExceptionHandlers(crosspage); - if (crosspage->crosspageData.activateVMX) { + if (CPUID_GetVendor() == CPUID_VENDOR_INTEL) { /* * Ensure that VMX is enabled and locked in the feature control MSR, * so that we can set CR4.VMXE to activate VMX. @@ -1860,10 +2476,9 @@ Task_Switch(VMDriver *vm, // IN crosspage->crosspageData.wsCR0 |= (cr0reg & CR0_RESERVED); /* - * CR4.VMXE must be enabled to support VMX in the monitor, and it - * can't be cleared if it is set on the host. + * CR4.VMXE must be enabled to support VMX in the monitor. */ - if (crosspage->crosspageData.activateVMX || (cr4reg & CR4_VMXE) != 0) { + if (CPUID_GetVendor() == CPUID_VENDOR_INTEL) { crosspage->crosspageData.wsCR4 |= CR4_VMXE; } @@ -1895,7 +2510,7 @@ Task_Switch(VMDriver *vm, // IN TaskSaveGDT64(&hostGDT64); - if (crosspage->crosspageData.activateVMX) { + if (CPUID_GetVendor() == CPUID_VENDOR_INTEL) { MA vmxonRegion = MPN_2_MA(hvRootMPN); VMXStatus status = VMXON_2_STATUS(&vmxonRegion); if (status == VMX_Success) { @@ -1905,7 +2520,7 @@ Task_Switch(VMDriver *vm, // IN } } - if (crosspage->crosspageData.activateSVM) { + if (CPUID_GetVendor() == CPUID_VENDOR_AMD) { efer = __GET_MSR(MSR_EFER); if ((efer & MSR_EFER_SVME) == 0) { __SET_MSR(MSR_EFER, efer | MSR_EFER_SVME); @@ -1993,12 +2608,12 @@ Task_Switch(VMDriver *vm, // IN } else if (foreignVMCS != ~0ULL) { VMPTRLD_UNCHECKED(&foreignVMCS); } - + if (WS_INTR_STRESS) { TaskEnableTF(); } - if (crosspage->crosspageData.activateSVM) { + if (CPUID_GetVendor() == CPUID_VENDOR_AMD) { __SET_MSR(MSR_VM_HSAVE_PA, foreignHSAVE); if ((efer & MSR_EFER_SVME) == 0) { __SET_MSR(MSR_EFER, efer); @@ -2079,6 +2694,10 @@ Task_Switch(VMDriver *vm, // IN __SET_MSR(IA32_MSR_PEBS_ENABLE, pebsMSR); } + if ((ptMSR & MSR_RTIT_CTL_TRACE_EN) != 0) { + __SET_MSR(MSR_RTIT_CTL, ptMSR); + } + TaskUpdateLatestPTSC(vm, &crosspage->crosspageData); vm->currentHostCpu[vcpuid] = INVALID_PCPU; diff --git a/vmmon-only/common/task.h b/vmmon-only/common/task.h index b2fcfd9..0ea0b01 100644 --- a/vmmon-only/common/task.h +++ b/vmmon-only/common/task.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2013 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2013,2015 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -16,26 +16,24 @@ * *********************************************************/ - - #ifndef TASK_H #define TASK_H #define INCLUDE_ALLOW_VMMON #define INCLUDE_ALLOW_VMCORE #include "includeCheck.h" +#include "bootstrap_vmm.h" struct InitBlock; -struct InitCrossGDT; -extern Bool Task_AllocCrossGDT(struct InitBlock *initBlock); +extern Bool Task_CreateCrossGDT(BSVMM_GDTInit *gdt); extern int Task_InitCrosspage(VMDriver *vm, struct InitBlock *params); -extern int Task_InitCrossGDT(struct InitCrossGDT *initCrossGDT); extern void Task_Switch(VMDriver *vm, Vcpuid vcpuid); extern Bool Task_Initialize(void); extern void Task_Terminate(void); extern MPN Task_GetHVRootPageForPCPU(uint32 pCPU); extern Descriptor *Task_GetTmpGDT(uint32 pCPU); +extern void Task_SwitchPTPPageCleanup(VMDriver *vm); #endif diff --git a/vmmon-only/common/vmx86.c b/vmmon-only/common/vmx86.c index 156e94a..f6b0d6f 100644 --- a/vmmon-only/common/vmx86.c +++ b/vmmon-only/common/vmx86.c @@ -61,6 +61,9 @@ #include #endif #include "x86perfctr.h" +#include "x86vtinstr.h" +#include "bootstrap_vmm.h" +#include "monLoader.h" PseudoTSC pseudoTSC; @@ -105,6 +108,12 @@ static int vmIDsUnused; /* Max rate requested for fast clock by any virtual machine. */ static unsigned globalFastClockRate; +typedef struct { + Atomic_uint32 index; + MSRQuery *query; +} Vmx86GetMSRData; + + /* *---------------------------------------------------------------------- * @@ -198,7 +207,7 @@ Vmx86LockedPageLimit(const VMDriver* vm) // IN: static INLINE Bool Vmx86HasFreePages(VMDriver *vm, - unsigned int numPages, + unsigned int numPages, Bool checkVM) { /* @@ -218,11 +227,11 @@ Vmx86HasFreePages(VMDriver *vm, ASSERT(HostIF_VMLockIsHeld(vm)); if (vm->memInfo.admitted) { - if (vm->memInfo.maxAllocation <= vm->memInfo.locked) { - return FALSE; - } else if (vm->memInfo.maxAllocation - vm->memInfo.locked < numPages) { - return FALSE; - } + if (vm->memInfo.maxAllocation <= vm->memInfo.locked) { + return FALSE; + } else if (vm->memInfo.maxAllocation - vm->memInfo.locked < numPages) { + return FALSE; + } } } else { /* @@ -232,9 +241,9 @@ Vmx86HasFreePages(VMDriver *vm, unsigned limit = Vmx86LockedPageLimit(vm); if (limit <= numLockedPages) { - return FALSE; + return FALSE; } else if (limit - numLockedPages < numPages) { - return FALSE; + return FALSE; } } @@ -415,7 +424,7 @@ Vmx86AllocVMID(void) * * Side effects: * Add VM to linked list. - * Increment count of VMs. + * Increment count of VMs. * *---------------------------------------------------------------- */ @@ -458,7 +467,7 @@ Vmx86RegisterVMOnList(VMDriver *vm) // IN * * Side effects: * Remove VM from linked list. - * Decrement count of VMs. + * Decrement count of VMs. * *---------------------------------------------------------------- */ @@ -520,6 +529,10 @@ Vmx86FreeAllVMResources(VMDriver *vm) Vmx86_SetHostClockRate(vm, 0); + if (vm->ptpTracker) { + Task_SwitchPTPPageCleanup(vm); + } + HostIF_FreeAllResources(vm); HostIF_FreeKernelMem(vm); @@ -539,9 +552,9 @@ Vmx86FreeAllVMResources(VMDriver *vm) * * If ignoreLimits is TRUE then additional pages may be reserved even * if limits are violated. The request to ignore limits may come in - * cases of anonymous page allocations. Swapping is not always possible + * cases of anonymous page allocations. Swapping is not always possible * at those points but a swap target will have been posted so that the - * vmm will release memory shortly allowing the excessive reservation + * vmm will release memory shortly allowing the excessive reservation * to be reduced. * * Results: @@ -555,7 +568,7 @@ Vmx86FreeAllVMResources(VMDriver *vm) static Bool Vmx86ReserveFreePages(VMDriver *vm, - unsigned int numPages, + unsigned int numPages, Bool ignoreLimits) { Bool retval = FALSE; @@ -572,26 +585,26 @@ Vmx86ReserveFreePages(VMDriver *vm, if (!retval) { HostIF_VMUnlock(vm, 0); HostIF_GlobalUnlock(17); - break; + break; } else { - // Wait to satisfy the global limit. - retval = Vmx86HasFreePages(vm, numPages, FALSE); - if (retval) { - numLockedPages += numPages; - vm->memInfo.locked += numPages; + // Wait to satisfy the global limit. + retval = Vmx86HasFreePages(vm, numPages, FALSE); + if (retval) { + numLockedPages += numPages; + vm->memInfo.locked += numPages; HostIF_VMUnlock(vm, 0); - HostIF_GlobalUnlock(17); - break; - } else { + HostIF_GlobalUnlock(17); + break; + } else { /* * There are not enough pages -- drop the locks and wait for * the host and/or other VMs to produce free pages. - */ + */ HostIF_VMUnlock(vm, 0); - HostIF_GlobalUnlock(17); - HostIF_WaitForFreePages(10); - } + HostIF_GlobalUnlock(17); + HostIF_WaitForFreePages(10); + } } } @@ -628,7 +641,7 @@ Vmx86ReserveFreePages(VMDriver *vm, static void Vmx86UnreserveFreePages(VMDriver *vm, - unsigned int numPages) + unsigned int numPages) { ASSERT(vm); @@ -663,10 +676,17 @@ Vmx86UnreserveFreePages(VMDriver *vm, */ VMDriver * -Vmx86_CreateVM(void) +Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize) { VMDriver *vm; Vcpuid v; + void *bsBuf = NULL; + BSVMM_HostParams *bsParams; + + /* Disallow VM creation if HV is not available, as the VM cannot be run. */ + if (!CPUID_HostSupportsHV()) { + return NULL; + } vm = HostIF_AllocKernelMem(sizeof *vm, TRUE); if (vm == NULL) { @@ -676,14 +696,36 @@ Vmx86_CreateVM(void) vm->userID = 0; vm->memInfo.admitted = FALSE; - vm->fastSuspResFlag = 0; for (v = 0; v < MAX_INITBLOCK_CPUS; v++) { vm->currentHostCpu[v] = INVALID_PCPU; } - + for (v = 0; v < MAX_VCPUS; v++) { + vm->ptRootMpns[v] = INVALID_MPN; + } if (HostIF_Init(vm)) { goto cleanup; } + bsBuf = HostIF_AllocKernelMem(bsBlobSize, FALSE); + if (bsBuf == NULL) { + goto cleanup; + } + if (HostIF_CopyFromUser(bsBuf, bsBlob, bsBlobSize) != 0) { + goto cleanup; + } + bsParams = BSVMM_Validate(bsBuf, bsBlobSize); + if (bsParams == NULL) { + Warning("Could not validate the VMM bootstrap blob"); + goto cleanup; + } + memcpy(&vm->bsParams, bsParams, sizeof *bsParams); + + if (!Task_CreateCrossGDT(&bsParams->gdtInit)) { + goto cleanup; + } + vm->ptpTracker = MemTrack_Init(vm); + if (vm->ptpTracker == NULL) { + goto cleanup; + } HostIF_GlobalLock(0); @@ -702,9 +744,13 @@ Vmx86_CreateVM(void) HostIF_GlobalUnlock(0); + HostIF_FreeKernelMem(bsBuf); return vm; cleanup: + if (bsBuf != NULL) { + HostIF_FreeKernelMem(bsBuf); + } /* * The VM is not on a list, "vmCount" has not been incremented, * "vm->cowID" is INVALID_VMID, and either the VM's mutex hasn't @@ -720,6 +766,47 @@ cleanup: } +/* + *---------------------------------------------------------------------- + * + * Vmx86SetPageTableRoots -- + * + * Translates the user VA corresponding to the root page tables + * for all VCPUs into MPNs and stores them in VMDriver. + * + * Results: + * TRUE if successful, FALSE otherwise. + * + *---------------------------------------------------------------------- + */ + +static Bool +Vmx86SetPageTableRoots(VMDriver *vm, UserVA64 *ptRootVAs, uint16 numVCPUs) +{ + uint16 vcpu; + + ASSERT_ON_COMPILE(MAX_VCPUS == MAX_INITBLOCK_CPUS); + + if (numVCPUs > vm->numVCPUs) { + return FALSE; + } + for (vcpu = 0; vcpu < numVCPUs; vcpu++) { + if ((ptRootVAs[vcpu] & (PAGE_SIZE - 1)) != 0) { + Warning("Error: page table VA %"FMT64"x is not page-aligned\n", + ptRootVAs[vcpu]); + return FALSE; + } + ASSERT(vm->ptRootMpns[vcpu] == INVALID_MPN); + if (HostIF_LookupUserMPN(vm, ptRootVAs[vcpu], &vm->ptRootMpns[vcpu]) != + PAGE_LOOKUP_SUCCESS) { + Warning("Failure looking up page table root MPN for VCPU %d\n", vcpu); + return FALSE; + } + } + return TRUE; +} + + /* *---------------------------------------------------------------------- * @@ -749,6 +836,87 @@ Vmx86_LookupUserMPN(VMDriver *vm, // IN: VMDriver } +/* + *---------------------------------------------------------------------- + * + * Vmx86_ProcessBootstrap -- + * + * Copies the VMM bootstrap blob header and processes it by invoking + * MonLoader. + * + * Results: + * TRUE if successful, FALSE otherwise. + * + *---------------------------------------------------------------------- + */ + +Bool +Vmx86_ProcessBootstrap(VMDriver *vm, + VA64 bsBlobAddr, + uint32 numBytes, + uint32 headerOffset, + uint16 numVCPUs, + UserVA64 *ptRootVAs, + VMSharedRegion *shRegions) +{ + void *blobHdr = NULL; + unsigned errLine; + Vcpuid errVcpu; + MonLoaderError ret; + MonLoaderHeader *header; + Bool res = FALSE; + MonLoaderArgs args; + + /* + * Only the beginning of the blob may be passed in: the header at + * offset 0, which must fit on one page. + */ + if (numBytes > PAGE_SIZE || headerOffset != 0 || + MonLoader_GetFixedHeaderSize() > numBytes) { + Warning("Invalid arguments for processing bootstrap. " + "Header offset: %d, Fixed header size: %zd bytes, " + "Blob size: %d bytes\n", + headerOffset, MonLoader_GetFixedHeaderSize(), numBytes); + goto cleanup; + } + blobHdr = HostIF_AllocKernelMem(numBytes, FALSE); + if (blobHdr == NULL) { + Warning("Failure allocating kernel buffer for bootstrap blob\n"); + goto cleanup; + } + if (HostIF_CopyFromUser(blobHdr, bsBlobAddr, numBytes) != 0) { + Warning("Failure copying bootstrap blob from userspace\n"); + goto cleanup; + } + ASSERT(headerOffset == 0); + header = (MonLoaderHeader *)blobHdr; + if (MonLoader_GetFullHeaderSize(header) > numBytes) { + Warning("Invalid arguments for processing bootstrap. " + "Full header size: %zd bytes, Blob size: %d bytes\n", + MonLoader_GetFullHeaderSize(header), numBytes); + goto cleanup; + } + if (!Vmx86SetPageTableRoots(vm, ptRootVAs, numVCPUs)) { + goto cleanup; + } + args.vm = vm; + args.shRegions = shRegions; + ret = MonLoader_Process(header, numVCPUs, &args, &errLine, &errVcpu); + if (ret != ML_OK) { + Warning("Error processing bootstrap: error %d at line %u, vcpu %u\n", + ret, errLine, errVcpu); + goto cleanup; + } + res = TRUE; + +cleanup: + if (blobHdr != NULL) { + HostIF_FreeKernelMem(blobHdr); + } + return res; +} + + /* *---------------------------------------------------------------------- * @@ -790,7 +958,7 @@ Vmx86_ReleaseVM(VMDriver *vm) // IN: * None. * * Side effects: - * Bumps fdCount. + * Bumps fdCount. * *---------------------------------------------------------------------- */ @@ -819,7 +987,7 @@ Vmx86_Open(void) * * Side effects: * Decrements fdCount - * May de-initialize ptsc. + * May de-initialize ptsc. * *---------------------------------------------------------------------- */ @@ -890,8 +1058,6 @@ Vmx86_InitVM(VMDriver *vm, // IN } vm->numVCPUs = initParams->numVCPUs; - HostIF_InitFP(vm); - /* * Initialize the driver's part of the cross-over page used to * talk to the monitor @@ -946,7 +1112,7 @@ Vmx86_InitVM(VMDriver *vm, // IN */ void -Vmx86_ReadTSCAndUptime(VmTimeStart *st) // OUT: return value +Vmx86_ReadTSCAndUptime(VmTimeStart *st) // OUT: return value { uintptr_t flags; @@ -994,7 +1160,7 @@ Vmx86_ComputekHz(uint64 cycles, uint64 uptime) cycles >>= 1; uptime >>= 1; } - + if (uptime == 0) { return 0; } @@ -1065,7 +1231,7 @@ Vmx86GetBusyKHzEstimate(void) */ static INLINE_SINGLE_CALLER uint32 -Vmx86GetkHzEstimate(VmTimeStart *st) // IN: start time +Vmx86GetkHzEstimate(VmTimeStart *st) // IN: start time { uint64 cDiff, tDiff; uintptr_t flags; @@ -1102,7 +1268,7 @@ Vmx86GetkHzEstimate(VmTimeStart *st) // IN: start time */ uint32 -Vmx86_GetkHzEstimate(VmTimeStart *st) // IN: start time +Vmx86_GetkHzEstimate(VmTimeStart *st) // IN: start time { static uint32 kHz; @@ -1328,7 +1494,7 @@ Vmx86MinAllocationFunc(unsigned nonpaged, * memory. * * Results: - * The minAllocation for this vm. + * The minAllocation for this vm. * * * Side effects: @@ -1357,7 +1523,7 @@ Vmx86MinAllocation(VMDriver *vm, * percentage of guest memory must fit within host RAM. * * Results: - * Number of pages that must fit within host ram for a given overcommit + * Number of pages that must fit within host ram for a given overcommit * level. * * @@ -1610,7 +1776,7 @@ Vmx86_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver * allocated MPNs. MPNs of the allocated pages are copied to the * caller's buffer at 'addr'. * - * Negative system specific error code on error (NTSTATUS on Windows, + * Negative system specific error code on error (NTSTATUS on Windows, * etc.) * * Side effects: @@ -1620,11 +1786,11 @@ Vmx86_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver */ int -Vmx86_AllocLockedPages(VMDriver *vm, // IN: VMDriver - VA64 addr, // OUT: VA of an array for +Vmx86_AllocLockedPages(VMDriver *vm, // IN: VMDriver + VA64 addr, // OUT: VA of an array for // allocated MPNs. - unsigned numPages, // IN: number of pages to allocate - Bool kernelMPNBuffer, // IN: is the MPN buffer in kernel + unsigned numPages, // IN: number of pages to allocate + Bool kernelMPNBuffer, // IN: is the MPN buffer in kernel // or user address space? Bool ignoreLimits) // IN: should limits be ignored? { @@ -1659,8 +1825,8 @@ Vmx86_AllocLockedPages(VMDriver *vm, // IN: VMDriver * by Vmx86_AllocLockedPages(). * * Results: - * 0 on success, - * non-0 system specific error code on error (NTSTATUS on Windows, etc.) + * 0 on success, + * non-0 system specific error code on error (NTSTATUS on Windows, etc.) * * Side effects: * Number of global and per-VM locked pages is decreased. @@ -1669,10 +1835,10 @@ Vmx86_AllocLockedPages(VMDriver *vm, // IN: VMDriver */ int -Vmx86_FreeLockedPages(VMDriver *vm, // IN: VM instance pointer - VA64 addr, // IN: user or kernel array of MPNs to free - unsigned numPages, // IN: number of pages to free - Bool kernelMPNBuffer) // IN: is the MPN buffer in kernel or user address space? +Vmx86_FreeLockedPages(VMDriver *vm, // IN: VM instance pointer + VA64 addr, // IN: user or kernel array of MPNs to free + unsigned numPages, // IN: number of pages to free + Bool kernelMPNBuffer) // IN: is the MPN buffer in kernel or user address space? { int ret; @@ -1693,11 +1859,11 @@ Vmx86_FreeLockedPages(VMDriver *vm, // IN: VM instance pointer * * Vmx86_GetNextAnonPage -- * - * Queries the driver to retrieve the list of anonymous pages. + * Queries the driver to retrieve the list of anonymous pages. * A supplied value of INVALID_MPN will start the query from * the head of the list. Callers supply the previously received * mpn to retrieve the next in the chain. Note: There is no - * guarantee of coherency. + * guarantee of coherency. * * Results: * A valid mpn or INVALID_MPN if the list has been exhausted. @@ -1722,55 +1888,22 @@ Vmx86_GetNextAnonPage(VMDriver *vm, // IN: VM instance pointer } -/* - *---------------------------------------------------------------------- - * - * Vmx86_GetLockedPageList -- - * - * puts MPNs of pages that were allocated by HostIF_AllocLockedPages() - * into user mode buffer. - * - * Results: - * non-negative number of the MPNs in the buffer on success. - * negative error code on error. - * - * Side effects: - * none - * - *---------------------------------------------------------------------- - */ - -int -Vmx86_GetLockedPageList(VMDriver *vm, // IN: VM instance pointer - VA64 uAddr, // OUT: user mode buffer for MPNs - unsigned int numPages) // IN: size of the buffer in MPNs -{ - int ret; - - HostIF_VMLock(vm, 9); - ret = HostIF_GetLockedPageList(vm, uAddr, numPages); - HostIF_VMUnlock(vm, 9); - - return ret; -} - - /* *---------------------------------------------------------------------- * * Vmx86_GetMemInfo -- * - * Return the info about all VMs. + * Return the info about all VMs. * * Results: * TRUE if all info was successfully copied. * * Side effects: - * VMGetMemInfoArgs is filled in. If the supplied curVM is null + * VMGetMemInfoArgs is filled in. If the supplied curVM is null * then only the baseline information will be returned. Calling - * with a null curVM may return results for maxLockedPages - * that differ from those when the vm is passed if huge pages - * are in use. + * with a null curVM may return results for maxLockedPages + * that differ from those when the vm is passed if huge pages + * are in use. * *---------------------------------------------------------------------- */ @@ -1846,7 +1979,7 @@ Vmx86_GetMemInfo(VMDriver *curVM, * * Vmx86SetMemoryUsage -- * - * Updates the paged, nonpaged, and anonymous memory reserved memory + * Updates the paged, nonpaged, and anonymous memory reserved memory * values for the vm. * * Results: @@ -1968,8 +2101,8 @@ Vmx86_Admit(VMDriver *curVM, // IN while (allocatedPages < nonpaged) { pages = Vmx86_AllocLockedPages(curVM, PtrToVA64(mpns + allocatedPages), - MIN(ALLOCATE_CHUNK_SIZE, nonpaged - allocatedPages), - TRUE, + MIN(ALLOCATE_CHUNK_SIZE, nonpaged - allocatedPages), + TRUE, FALSE); if (pages <= 0) { break; @@ -2078,34 +2211,7 @@ Vmx86_UpdateMemInfo(VMDriver *curVM, /* *---------------------------------------------------------------------- * - * Vmx86_PAEEnabled -- - * - * Is PAE enabled? - * - * Results: - * TRUE if PAE enabled. - * - * Side effects: - * None. - * - *---------------------------------------------------------------------- - */ - -Bool -Vmx86_PAEEnabled(void) -{ - uintptr_t cr4; - - GET_CR4(cr4); - - return (cr4 & CR4_PAE) != 0; -} - - -/* - *---------------------------------------------------------------------- - * - * Vmx86_VMXEnabled -- + * Vmx86VMXEnabled -- * * Test the VMXE bit as an easy proxy for whether VMX operation * is enabled. @@ -2119,8 +2225,8 @@ Vmx86_PAEEnabled(void) *---------------------------------------------------------------------- */ -Bool -Vmx86_VMXEnabled(void) +static Bool +Vmx86VMXEnabled(void) { if (VT_CapableCPU()) { uintptr_t cr4; @@ -2134,154 +2240,6 @@ Vmx86_VMXEnabled(void) } -/* - *----------------------------------------------------------------------------- - * - * Vmx86LookupVMByUserIDLocked -- - * - * Lookup a VM by userID. The caller must hold the global lock. - * - * Returns: - * On success: Pointer to the driver's VM instance. - * On failure: NULL (not found). - * - * Side effects: - * None - * - *----------------------------------------------------------------------------- - */ - -static VMDriver * -Vmx86LookupVMByUserIDLocked(int userID) // IN -{ - VMDriver *vm; - - ASSERT(HostIF_GlobalLockIsHeld()); - - for (vm = vmDriverList; vm != NULL; vm = vm->nextDriver) { - if (vm->userID == userID) { - return vm; - } - } - - return NULL; -} - - -/* - *----------------------------------------------------------------------------- - * - * Vmx86_LookupVMByUserID -- - * - * Lookup a VM by userID. - * - * Returns: - * On success: Pointer to the driver's VM instance. - * On failure: NULL (not found). - * - * Side effects: - * None - * - *----------------------------------------------------------------------------- - */ - -VMDriver * -Vmx86_LookupVMByUserID(int userID) // IN -{ - VMDriver *vm; - - HostIF_GlobalLock(10); - vm = Vmx86LookupVMByUserIDLocked(userID); - HostIF_GlobalUnlock(10); - - return vm; -} - - -/* - *---------------------------------------------------------------------- - * - * Vmx86_FastSuspResSetOtherFlag -- - * - * Sets the value of other VM's fastSuspResFlag. - * - * Returns: - * TRUE if VM was found and flag was set successfully. - * FALSE if VM was not found. - * - * Side effects: - * The value we set the flag to is this VM's userID. - * - *---------------------------------------------------------------------- - */ - -Bool -Vmx86_FastSuspResSetOtherFlag(VMDriver *vm, // IN - int otherVmUserId) // IN -{ - VMDriver *otherVM; - - HostIF_GlobalLock(35); - otherVM = Vmx86LookupVMByUserIDLocked(otherVmUserId); - if (otherVM != NULL) { - ASSERT(otherVM->fastSuspResFlag == 0); - otherVM->fastSuspResFlag = vm->userID; - } else { - Warning("otherVmUserId (%d) is invalid", otherVmUserId); - } - HostIF_GlobalUnlock(35); - - return otherVM != NULL; -} - - -/* - *---------------------------------------------------------------------- - * - * Vmx86_FastSuspResGetMyFlag -- - * - * Gets the value of fastSuspResFlag. If blockWait is true, this - * function will not return until the flag is non-zero, or until - * timeout. - * - * Returns: - * The value of the flag which, if non-zero, should be the userID of - * the vm that set it. - * - * Side effects: - * The flag is reset to zero once read. - * - *---------------------------------------------------------------------- - */ - -int -Vmx86_FastSuspResGetMyFlag(VMDriver *vm, // IN - Bool blockWait) // IN -{ - int retval = 0; - int ntries = 1; - const int waitInterval = 10; /* Wait 10ms at a time. */ - const int maxWaitTime = 100000; /* Wait maximum of 100 seconds. */ - - if (blockWait) { - ntries = maxWaitTime / waitInterval; - } - - while (ntries--) { - HostIF_GlobalLock(6); - retval = vm->fastSuspResFlag; - vm->fastSuspResFlag = 0; - HostIF_GlobalUnlock(6); - if (retval || !ntries) { - break; - } - HostIF_Wait(waitInterval); - } - - return retval; -} - - /* *----------------------------------------------------------------------------- * @@ -2301,7 +2259,7 @@ Vmx86_FastSuspResGetMyFlag(VMDriver *vm, // IN static void Vmx86EnableHVOnCPU(void) { - if (SVM_CapableCPU()) { + if (CPUID_GetVendor() == CPUID_VENDOR_AMD) { uint64 vmCR = __GET_MSR(MSR_VM_CR); if (!SVM_LockedFromFeatures(vmCR)) { CPUIDRegs regs; @@ -2311,12 +2269,13 @@ Vmx86EnableHVOnCPU(void) MSR_VM_CR_SVM_LOCK); } } - } - if (VT_CapableCPU()) { + } else if (CPUID_GetVendor() == CPUID_VENDOR_INTEL) { uint64 featCtl = __GET_MSR(MSR_FEATCTL); if (!VT_LockedFromFeatures(featCtl)) { __SET_MSR(MSR_FEATCTL, featCtl | MSR_FEATCTL_LOCK | MSR_FEATCTL_VMXE); } + } else { + NOT_REACHED(); } } @@ -2334,7 +2293,7 @@ Vmx86EnableHVOnCPU(void) static INLINE uint64 Vmx86RefClockInCycles(uint64 uptime) { - return Mul64x3264(uptime, + return Mul64x3264(uptime, pseudoTSC.refClockToPTSC.ratio.mult, pseudoTSC.refClockToPTSC.ratio.shift); } @@ -2353,7 +2312,7 @@ Vmx86RefClockInCycles(uint64 uptime) static INLINE uint64 Vmx86RefClockToPTSC(uint64 uptime) { - return Vmx86RefClockInCycles(uptime) + + return Vmx86RefClockInCycles(uptime) + Atomic_Read64(&pseudoTSC.refClockToPTSC.add); } @@ -2411,7 +2370,7 @@ Vmx86_InitPseudoTSC(PTSCInitParams *params) // IN/OUT pseudoTSC.initialized = TRUE; } - /* + /* * Allow the calling vmx to respect ptsc.noTSC=TRUE config option * even if another vmx is already running (pseudoTSC was already * initialized). Useful for testing. @@ -2533,12 +2492,6 @@ Vmx86_CheckPseudoTSC(uint64 *lastTSC, // IN/OUT: last/current value of the TSC } -typedef struct { - Atomic_uint32 index; - MSRQuery *query; -} Vmx86GetMSRData; - - /* *----------------------------------------------------------------------------- * @@ -2546,7 +2499,7 @@ typedef struct { * * Collect MSR value on the current logical CPU. * - * Function must not block (it is invoked from interrupt context). + * Function must not block (it is invoked from interrupt context). * Only VT MSRs are supported on VT-capable processors. * * Results: @@ -2653,6 +2606,143 @@ Vmx86_GetAllMSRs(MSRQuery *query) // IN/OUT } +/* + *----------------------------------------------------------------------------- + * + * Vmx86CheckVMXStatus -- + * + * Checks the status of the given operation and issues a warning if it was + * not successful. If it is a valid failure, the error code will be read + * and logged. + * + * Results: + * None. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +static void +Vmx86CheckVMXStatus(const char *operation, // IN: Operation string + VMXStatus status) // IN: Status to check +{ + if (status != VMX_Success) { + Warning("%s failed with status %s.\n", operation, + status == VMX_FailValid ? "VMX_FailValid" : + status == VMX_FailInvalid ? "VMX_FailInvalid" : "UNKNOWN"); +/* + * We use a broken in-house version of binutils (2.16.1-vt) with gcc 4.3 which + * doesn't handle VMREAD/VMWRITE operands properly. + */ +#ifdef __GNUC__ +#if __GNUC__== 4 && __GNUC_MINOR__ > 3 + if (status == VMX_FailValid) { + uint64 errorCode; + VMREAD_2_STATUS(VT_VMCS_VMINSTR_ERR, &errorCode); + Log("VM-instruction error: Error %"FMT64"d\n", errorCode); + } +#endif +#endif + } +} + + +/* + *----------------------------------------------------------------------------- + * + * Vmx86FlushVMCSPage -- + * + * VMCLEAR the given VMCS page on the current logical CPU. We first enable + * HV if necessary, and execute a VMXON using the given VMXON region MPN. + * If HV was already enabled, it will remain enabled. If we enabled HV or + * executed a VMXON in non-root operation, we will restore the state of + * each respectively after the VMCLEAR. + * + * Results: + * None. + * + * Side effects: + * The hardware VMCS cache will be flushed. + * + *----------------------------------------------------------------------------- + */ + +static void +Vmx86FlushVMCSPage(void *clientData) // IN: The MA of the VMCS to VMCLEAR +{ + MA vmxonRegion; + Bool hvWasEnabled; + MA vmcs = (MA)clientData; + Bool vmxWasInRootOperation = FALSE; + VMXStatus vmxonStatus, vmclearStatus, vmxoffStatus; + + ASSERT(vmcs); + + /* Enable HV if it isn't already enabled. */ + hvWasEnabled = Vmx86VMXEnabled(); + if (!hvWasEnabled) { + uintptr_t cr4reg; + ASSERT(VT_CapableCPU()); + Vmx86EnableHVOnCPU(); + GET_CR4(cr4reg); + SET_CR4(cr4reg | CR4_VMXE); + } + + /* VMXON using this CPUs's VMXON region. */ + vmxonRegion = MPN_2_MA(Task_GetHVRootPageForPCPU(HostIF_GetCurrentPCPU())); + vmxonStatus = VMXON_2_STATUS(&vmxonRegion); + if (vmxonStatus != VMX_Success) { + /* VMXON failed, we must already be in VMX root operation. */ + vmxWasInRootOperation = TRUE; + } + + /* VMCLEAR the given VMCS page. */ + vmclearStatus = VMCLEAR_2_STATUS(&vmcs); + Vmx86CheckVMXStatus("VMCLEAR", vmclearStatus); + + /* VMXOFF if we were initially in VMX non-root operation. */ + if (!vmxWasInRootOperation) { + vmxoffStatus = VMXOFF_2_STATUS(); + Vmx86CheckVMXStatus("VMXOFF", vmxoffStatus); + } + + /* Disable HV if it was initially disabled. */ + if (!hvWasEnabled) { + uintptr_t cr4reg; + GET_CR4(cr4reg); + SET_CR4(cr4reg & ~CR4_VMXE); + } +} + + +/* + *----------------------------------------------------------------------------- + * + * Vmx86_FlushVMCSAllCPUs -- + * + * Enable HV (if necessary) and VMCLEAR a VMCS page on all logical CPUs. + * This will prevent stale data from surfacing out of the VMCS cache when + * executing VMREADs. + * + * Results: + * None. + * + * Side effects: + * HV will be enabled and hardware VMCS caches will be flushed across all + * CPUs. + * + *----------------------------------------------------------------------------- + */ + +void +Vmx86_FlushVMCSAllCPUs(MA vmcs) // IN: The MA of the VMCS to VMCLEAR +{ + HostIF_CallOnEachCPU(Vmx86FlushVMCSPage, (void *)vmcs); +} + + /* *---------------------------------------------------------------------- * @@ -2662,7 +2752,7 @@ Vmx86_GetAllMSRs(MSRQuery *query) // IN/OUT * * usecs is the total spin time in monitor. Very low numbers * indicate we detected there was a vCPU thread that was not - * in the monitor, so we didn't spin. In that case, simply + * in the monitor, so we didn't spin. In that case, simply * nudge the threads we want and return. * * Results: @@ -2838,7 +2928,7 @@ Vmx86PerfCtrInUse(Bool isGen, unsigned pmcNum, unsigned ctrlMSR, * Vmx86GetUnavailPerfCtrsOnCPU -- * * Determine which performance counters are already in use by the - * host on the current PCPU. + * host on the current PCPU. * * Results: * A bitset representing unavailable performance counter. @@ -2879,10 +2969,11 @@ Vmx86GetUnavailPerfCtrsOnCPU(void *data) selBase = PERFCTR_CORE_PERFEVTSEL0_ADDR; ctrBase = PERFCTR_CORE_PERFCTR0_ADDR; } else if (CPUID_GetVendor() == CPUID_VENDOR_AMD) { - if (CPUID_FAMILY_IS_BULLDOZER(__GET_EAX_FROM_CPUID(1))) { + if(CPUID_ISSET(0x80000001, ECX, PERFCORE, + __GET_ECX_FROM_CPUID(0x80000001))) { numGen = 6; - selBase = PERFCTR_BD_BASE_ADDR + PERFCTR_BD_EVENTSEL; - ctrBase = PERFCTR_BD_BASE_ADDR + PERFCTR_BD_CTR; + selBase = PERFCTR_AMD_EXT_BASE_ADDR + PERFCTR_AMD_EXT_EVENTSEL; + ctrBase = PERFCTR_AMD_EXT_BASE_ADDR + PERFCTR_AMD_EXT_CTR; stride = 2; } else { numGen = 4; @@ -2947,3 +3038,79 @@ Vmx86_GetUnavailablePerfCtrs(void) return Atomic_Read64(&unavailCtrs); } + +/* + *---------------------------------------------------------------------- + * + * Vmx86_GetPageRoot -- + * + * Get the page root MPN for the specified VCPU. + * + * Results: + * TRUE and an MPN on success, FALSE on failure. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +Bool +Vmx86_GetPageRoot(VMDriver *vm, // IN: + Vcpuid vcpuid, // IN: + MPN *mpn) // OUT: +{ + if (vcpuid >= vm->numVCPUs) { + return FALSE; + } + *mpn = vm->ptRootMpns[vcpuid]; + return TRUE; +} + + +/* + *---------------------------------------------------------------------- + * + * Vmx86_MapPage -- + * + * Maps the specified MPN into the host kernel address space. + * returns the VPN of the mapping. + * + * Results: + * The VPN in the kernel address space of the new mapping, or 0 if + * the mapping failed. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +VPN +Vmx86_MapPage(MPN mpn) // IN: +{ + return HostIF_MapPage(mpn); +} + + +/* + *---------------------------------------------------------------------- + * + * Vmx86_UnmapPage -- + * + * Unmaps the specified VPN from the host kernel address space. + * + * Results: + * None. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +void +Vmx86_UnmapPage(VPN vpn) // IN: +{ + HostIF_UnmapPage(vpn); +} diff --git a/vmmon-only/common/vmx86.h b/vmmon-only/common/vmx86.h index 9e227ca..7f0f33f 100644 --- a/vmmon-only/common/vmx86.h +++ b/vmmon-only/common/vmx86.h @@ -38,6 +38,7 @@ #include "rateconv.h" #include "vmmem_shared.h" #include "apic.h" +#include "bootstrap_vmm.h" typedef struct PseudoTSCOffsetInfo { uint32 inVMMCnt; /* Number of vcpus executing in the VMM. */ @@ -63,8 +64,12 @@ typedef struct VMDriver { Vcpuid numVCPUs; /* Number of vcpus in VM. */ struct VMHost *vmhost; /* Host-specific fields. */ + BSVMM_HostParams bsParams; + MPN ptRootMpns[MAX_VCPUS]; + /* Pointers to the crossover pages shared with the monitor. */ struct VMCrossPage *crosspage[MAX_INITBLOCK_CPUS]; + struct MemTrack *ptpTracker; volatile uint32 currentHostCpu[MAX_INITBLOCK_CPUS]; VCPUSet crosscallWaitSet[MAX_INITBLOCK_CPUS]; APICDescriptor hostAPIC; @@ -74,13 +79,17 @@ typedef struct VMDriver { struct PerfCounter *perfCounter; VMMemMgmtInfo memInfo; unsigned fastClockRate;/* Protected by FastClockLock. */ - int fastSuspResFlag; volatile PseudoTSCOffsetInfo ptscOffsetInfo; /* Volatile per PR 699101#29. */ Atomic_uint64 ptscLatest; int64 ptscOffsets[MAX_INITBLOCK_CPUS]; } VMDriver; +typedef struct MonLoaderArgs { + VMDriver *vm; + VMSharedRegion *shRegions; +} MonLoaderArgs; + typedef struct VmTimeStart { uint64 count; uint64 time; @@ -105,7 +114,14 @@ extern PseudoTSC pseudoTSC; #define MAX_LOCKED_PAGES (-1) -extern VMDriver *Vmx86_CreateVM(void); +extern VMDriver *Vmx86_CreateVM(VA64 bsBlob, uint32 bsBlobSize); +extern Bool Vmx86_ProcessBootstrap(VMDriver *vm, + VA64 bsBlobAddr, + uint32 numBytes, + uint32 headerOffset, + uint16 numVCPUs, + UserVA64 *ptRootVAs, + VMSharedRegion *shRegions); extern int Vmx86_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn); extern int Vmx86_ReleaseVM(VMDriver *vm); extern int Vmx86_InitVM(VMDriver *vm, InitBlock *initParams); @@ -126,13 +142,11 @@ extern int Vmx86_UnlockPageByMPN(VMDriver *vm, MPN mpn, VA64 uAddr); extern MPN Vmx86_GetRecycledPage(VMDriver *vm); extern int Vmx86_ReleaseAnonPage(VMDriver *vm, MPN mpn); extern int Vmx86_AllocLockedPages(VMDriver *vm, VA64 addr, - unsigned numPages, Bool kernelMPNBuffer, + unsigned numPages, Bool kernelMPNBuffer, Bool ignoreLimits); extern int Vmx86_FreeLockedPages(VMDriver *vm, VA64 addr, - unsigned numPages, Bool kernelMPNBuffer); + unsigned numPages, Bool kernelMPNBuffer); extern MPN Vmx86_GetNextAnonPage(VMDriver *vm, MPN mpn); -extern int Vmx86_GetLockedPageList(VMDriver *vm, VA64 uAddr, - unsigned int numPages); extern int32 Vmx86_GetNumVMs(void); extern int32 Vmx86_GetTotalMemUsage(void); @@ -147,15 +161,12 @@ extern Bool Vmx86_Readmit(VMDriver *curVM, OvhdMem_Deltas *delta); extern void Vmx86_UpdateMemInfo(VMDriver *curVM, const VMMemMgmtInfoPatch *patch); extern void Vmx86_Add2MonPageTable(VMDriver *vm, VPN vpn, MPN mpn, - Bool readOnly); -extern Bool Vmx86_PAEEnabled(void); -extern Bool Vmx86_VMXEnabled(void); + Bool readOnly); extern Bool Vmx86_GetAllMSRs(MSRQuery *query); +extern void Vmx86_FlushVMCSAllCPUs(MA vmcs); extern void Vmx86_MonTimerIPI(void); extern void Vmx86_InitIDList(void); -extern VMDriver *Vmx86_LookupVMByUserID(int userID); -extern Bool Vmx86_FastSuspResSetOtherFlag(VMDriver *vm, int otherVmUserId); -extern int Vmx86_FastSuspResGetMyFlag(VMDriver *vm, Bool blockWait); +extern Bool Vmx86_GetPageRoot(VMDriver *vm, Vcpuid vcpuid, MPN *mpn); extern void Vmx86_Open(void); extern void Vmx86_Close(void); @@ -199,4 +210,4 @@ extern uint64 Vmx86_GetPseudoTSC(void); extern uint64 Vmx86_GetUnavailablePerfCtrs(void); -#endif +#endif diff --git a/vmmon-only/include/address_defs.h b/vmmon-only/include/address_defs.h index f7ba064..94259e8 100644 --- a/vmmon-only/include/address_defs.h +++ b/vmmon-only/include/address_defs.h @@ -36,6 +36,11 @@ #include "vm_basic_defs.h" // For PAGE_SHIFT +#if defined __cplusplus +extern "C" { +#endif + + /* * Virtual, physical, machine address and page conversion macros */ @@ -92,4 +97,9 @@ IsGoodMPNOrMemref(MPN mpn) return IsGoodMPN(mpn) || mpn == MEMREF_MPN; } + +#if defined __cplusplus +} // extern "C" #endif + +#endif // _ADDRESS_DEFS_H_ diff --git a/vmmon-only/include/basic_initblock.h b/vmmon-only/include/basic_initblock.h index 438e9b6..2932655 100644 --- a/vmmon-only/include/basic_initblock.h +++ b/vmmon-only/include/basic_initblock.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2006 VMware, Inc. All rights reserved. + * Copyright (C) 2006, 2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -40,7 +40,7 @@ #include "vcpuid.h" -#define MAX_INITBLOCK_CPUS 128 +#define MAX_INITBLOCK_CPUS MAX_VCPUS typedef @@ -49,9 +49,12 @@ struct InitBlock { uint32 magicNumber; /* Magic number (INIT_BLOCK_MAGIC) */ Vcpuid numVCPUs; VA64 crosspage[MAX_INITBLOCK_CPUS]; + LPN64 monStartLPN; + LPN64 monEndLPN; uint32 vmInitFailurePeriod; LA64 crossGDTHKLA; MPN crossGDTMPNs[5]; // CROSSGDT_NUMPAGES + uint16 numPTPPages; } #include "vmware_pack_end.h" InitBlock; diff --git a/vmmon-only/include/bootstrap_vmm.h b/vmmon-only/include/bootstrap_vmm.h new file mode 100644 index 0000000..1e48122 --- /dev/null +++ b/vmmon-only/include/bootstrap_vmm.h @@ -0,0 +1,65 @@ +/********************************************************* + * Copyright (C) 2015,2017 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * bootstrap_vmm.h -- + * + * Public VMM bootstrap declarations. + */ + +#ifndef _BOOTSTRAP_VMM_H +#define _BOOTSTRAP_VMM_H + +#define INCLUDE_ALLOW_VMCORE +#define INCLUDE_ALLOW_VMMON +#define INCLUDE_ALLOW_VMKERNEL +#define INCLUDE_ALLOW_USERLEVEL + +#include "includeCheck.h" +#include "vm_basic_types.h" +#include "monAddrLayout.h" + +#define BOOTSTRAP_MAGIC 42 +#define BOOTSTRAP_MAX_GDT_DESCS 2 + +typedef struct BSVMM_GDTInitEntry { + uint16 index; + LA32 base; + VA32 limit; + uint32 type; + uint32 S; + uint32 DPL; + uint32 present; + uint32 longmode; + uint32 DB; + uint32 gran; +} BSVMM_GDTInitEntry; + +typedef struct BSVMM_GDTInit { + BSVMM_GDTInitEntry entries[BOOTSTRAP_MAX_GDT_DESCS]; +} BSVMM_GDTInit; + +typedef struct BSVMM_HostParams { + uint32 magic; + VMM64_AddrLayout addrLayout; + BSVMM_GDTInit gdtInit; +} BSVMM_HostParams; + +BSVMM_HostParams* BSVMM_Validate(void *buf, uint32 nbytes); + +#endif diff --git a/vmmon-only/include/circList.h b/vmmon-only/include/circList.h index ac4f48f..850b8e6 100644 --- a/vmmon-only/include/circList.h +++ b/vmmon-only/include/circList.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -32,75 +32,126 @@ #define INCLUDE_ALLOW_MODULE #define INCLUDE_ALLOW_VMKERNEL #include "includeCheck.h" + #include "vmware.h" +#if defined(__cplusplus) +extern "C" { +#endif + typedef struct ListItem { struct ListItem *prev; struct ListItem *next; } ListItem; -/* A list with no elements is a null pointer. */ -#define LIST_ITEM_DEF(name) \ - ListItem * name = NULL - -#define LIST_EMPTY(l) ((l) == NULL) - -/* initialize list item */ -#define INIT_LIST_ITEM(p) \ - do { \ - (p)->prev = (p)->next = (p); \ - } while (0) - -/* check if initialized */ -#define IS_LIST_ITEM_INITIALIZED(li) \ - (((li) == (li)->prev) && ((li) == (li)->next)) - -/* return first element in the list */ -#define LIST_FIRST(l) (l) -#define LIST_FIRST_CHK(l) (l) - -/* return last element in the list */ -#define LIST_LAST(l) ((l)->prev) -#define LIST_LAST_CHK(l) (LIST_EMPTY(l) ? NULL : LIST_LAST(l)) /* - * LIST_CONTAINER - get the struct for this entry (like list_entry) + *---------------------------------------------------------------------- + * + * CircList_IsEmpty -- + * + * A NULL list is an empty list. + * + * Result: + * TRUE if list is empty, FALSE otherwise. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static INLINE Bool +CircList_IsEmpty(const ListItem *item) // IN +{ + return item == NULL; +} + + +/* + *---------------------------------------------------------------------- + * + * CircList_InitItem -- + * + * Initialize item as a single-element circular list. + * + * Result: + * None. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static INLINE void +CircList_InitItem(ListItem *item) // OUT +{ + item->prev = item->next = item; +} + + +/* + *---------------------------------------------------------------------- + * + * CircList_First -- + * + * Return first item in the list. + * + * Result: + * First item. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static INLINE ListItem * +CircList_First(ListItem *item) // IN +{ + return item; +} + + +/* + *---------------------------------------------------------------------- + * + * CircList_Last -- + * + * Return last item in the list. + * + * Result: + * Last item. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static INLINE ListItem * +CircList_Last(ListItem *item) +{ + return item->prev; +} + + +/* + * CIRC_LIST_CONTAINER - get the struct for this entry (like list_entry) * @ptr: the &struct ListItem pointer. * @type: the type of the struct this is embedded in. * @member: the name of the list struct within the struct. */ -#define LIST_CONTAINER(ptr, type, member) \ +#define CIRC_LIST_CONTAINER(ptr, type, member) \ VMW_CONTAINER_OF(ptr, type, member) - -/* - * delete item from the list +/* + * Historical name, left here to reduce churn. + * TODO: remove, all LIST_CONTAINER uses should be + * VMW_CONTAINER_OF and stop depending on circList.h + * to provide the definition. */ -#define LIST_DEL DelListItem - -/* - * link two lists together - */ -#define LIST_SPLICE SpliceLists - -/* - * Split a list into two lists - */ -#define LIST_SPLIT SplitLists - -/* - * Add item to front of stack. List pointer points to new head. - */ -#define LIST_PUSH PushListItem - -/* - * Add item at back of queue. List pointer only changes if list was empty. - */ -#define LIST_QUEUE QueueListItem - -/* - * Get the list size. - */ -#define LIST_SIZE GetListSize +#define LIST_CONTAINER(ptr, type, member) VMW_CONTAINER_OF(ptr, type, member) /* * LIST_SCAN_FROM scans the list from "from" up until "until". @@ -109,38 +160,32 @@ typedef struct ListItem { * "until" is the element where search should stop. * member is the field to use for the search - either "next" or "prev". */ -#define LIST_SCAN_FROM(p, from, until, member) \ +#define CIRC_LIST_SCAN_FROM(p, from, until, member) \ for (p = (from); (p) != NULL; \ (p) = (((p)->member == (until)) ? NULL : (p)->member)) /* scan the entire list (non-destructively) */ -#define LIST_SCAN(p, l) \ - LIST_SCAN_FROM(p, LIST_FIRST(l), LIST_FIRST(l), next) +#define CIRC_LIST_SCAN(p, l) \ + CIRC_LIST_SCAN_FROM(p, CircList_First(l), CircList_First(l), next) -/* scan a list backward from last element to first (non-destructively) */ -#define LIST_SCAN_BACK(p, l) \ - LIST_SCAN_FROM(p, LIST_LAST_CHK(l), LIST_LAST(l), prev) - /* scan the entire list where loop element may be destroyed */ -#define LIST_SCAN_SAFE(p, pn, l) \ - if (!LIST_EMPTY(l)) \ - for (p = (l), (pn) = NextListItem(p, l); (p) != NULL; \ - (p) = (pn), (pn) = NextListItem(p, l)) +#define CIRC_LIST_SCAN_SAFE(p, pn, l) \ + if (!CircList_IsEmpty(l)) \ + for (p = (l), (pn) = CircList_Next(p, l); (p) != NULL; \ + (p) = (pn), (pn) = CircList_Next(p, l)) /* scan the entire list backwards where loop element may be destroyed */ -#define LIST_SCAN_BACK_SAFE(p, pn, l) \ - if (!LIST_EMPTY(l)) \ - for (p = LIST_LAST(l), (pn) = PrevListItem(p, l); (p) != NULL; \ - (p) = (pn), (pn) = PrevListItem(p, l)) +#define CIRC_LIST_SCAN_BACK_SAFE(p, pn, l) \ + if (!CircList_IsEmpty(l)) \ + for (p = CircList_Last(l), (pn) = CircList_Prev(p, l); (p) != NULL; \ + (p) = (pn), (pn) = CircList_Prev(p, l)) -/* function definitions */ - /* *---------------------------------------------------------------------- * - * NextListItem -- + * CircList_Next -- * * Returns the next member of a doubly linked list, or NULL if last. * Assumes: p is member of the list headed by head. @@ -156,8 +201,8 @@ typedef struct ListItem { */ static INLINE ListItem * -NextListItem(ListItem *p, // IN - ListItem *head) // IN +CircList_Next(ListItem *p, // IN + ListItem *head) // IN { if (head == NULL || p == NULL) { return NULL; @@ -171,7 +216,7 @@ NextListItem(ListItem *p, // IN /* *---------------------------------------------------------------------- * - * PrevListItem -- + * CircList_Prev -- * * Returns the prev member of a doubly linked list, or NULL if first. * Assumes: p is member of the list headed by head. @@ -187,8 +232,8 @@ NextListItem(ListItem *p, // IN */ static INLINE ListItem * -PrevListItem(ListItem *p, // IN - ListItem *head) // IN +CircList_Prev(ListItem *p, // IN + ListItem *head) // IN { if (head == NULL || p == NULL) { return NULL; @@ -201,7 +246,7 @@ PrevListItem(ListItem *p, // IN /* *---------------------------------------------------------------------- * - * DelListItem -- + * CircList_DeleteItem -- * * Deletes a member of a doubly linked list, possibly modifies the * list header itself. @@ -217,8 +262,8 @@ PrevListItem(ListItem *p, // IN */ static INLINE void -DelListItem(ListItem *p, // IN - ListItem **headp) // IN/OUT +CircList_DeleteItem(ListItem *p, // IN + ListItem **headp) // IN/OUT { ListItem *next; @@ -241,7 +286,7 @@ DelListItem(ListItem *p, // IN /* *---------------------------------------------------------------------- * - * QueueListItem -- + * CircList_Queue -- * * Adds a new member to the back of a doubly linked list (queue) * Assumes neither p nor headp is null and p is not a member of *headp. @@ -256,14 +301,14 @@ DelListItem(ListItem *p, // IN */ static INLINE void -QueueListItem(ListItem *p, // IN - ListItem **headp) // IN/OUT +CircList_Queue(ListItem *p, // IN + ListItem **headp) // IN/OUT { ListItem *head; head = *headp; - if (LIST_EMPTY(head)) { - INIT_LIST_ITEM(p); + if (CircList_IsEmpty(head)) { + CircList_InitItem(p); *headp = p; } else { p->prev = head->prev; @@ -277,7 +322,7 @@ QueueListItem(ListItem *p, // IN /* *---------------------------------------------------------------------- * - * PushListItem -- + * CircList_Push -- * * Adds a new member to the front of a doubly linked list (stack) * Assumes neither p nor headp is null and p is not a member of *headp. @@ -292,10 +337,10 @@ QueueListItem(ListItem *p, // IN */ static INLINE void -PushListItem(ListItem *p, // IN - ListItem **headp) // IN/OUT +CircList_Push(ListItem *p, // IN + ListItem **headp) // IN/OUT { - QueueListItem(p, headp); + CircList_Queue(p, headp); *headp = p; } @@ -303,7 +348,7 @@ PushListItem(ListItem *p, // IN /* *---------------------------------------------------------------------- * - * SpliceLists -- + * CircList_Splice -- * * Make a single list {l1 l2} from {l1} and {l2} and return it. * It is okay for one or both lists to be NULL. @@ -320,16 +365,16 @@ PushListItem(ListItem *p, // IN */ static INLINE ListItem * -SpliceLists(ListItem *l1, // IN - ListItem *l2) // IN +CircList_Splice(ListItem *l1, // IN + ListItem *l2) // IN { ListItem *l1Last, *l2Last; - if (LIST_EMPTY(l1)) { + if (CircList_IsEmpty(l1)) { return l2; } - if (LIST_EMPTY(l2)) { + if (CircList_IsEmpty(l2)) { return l1; } @@ -349,10 +394,11 @@ SpliceLists(ListItem *l1, // IN } +#if 0 /* Presently unused, enable if a use is found */ /* *---------------------------------------------------------------------- * - * SplitLists -- + * CircList_Split -- * * Make a list l = {l1 l2} into two separate lists {l1} and {l2}, where: * l = { ... x -> p -> ... } split into: @@ -372,14 +418,14 @@ SpliceLists(ListItem *l1, // IN */ static INLINE void -SplitLists(ListItem *p, // IN - ListItem *l, // IN - ListItem **l1p, // OUT - ListItem **l2p) // OUT +CircList_Split(ListItem *p, // IN + ListItem *l, // IN + ListItem **l1p, // OUT + ListItem **l2p) // OUT { ListItem *last; - if (p == LIST_FIRST(l)) { /* first element */ + if (p == CircList_First(l)) { /* first element */ *l1p = NULL; *l2p = l; return; @@ -395,12 +441,13 @@ SplitLists(ListItem *p, // IN p->prev = last; last->next = p; } +#endif /* *---------------------------------------------------------------------- * - * GetListSize -- + * CircList_Size -- * * Return the number of items in the list. * @@ -414,15 +461,19 @@ SplitLists(ListItem *p, // IN */ static INLINE int -GetListSize(ListItem *head) // IN +CircList_Size(ListItem *head) // IN { ListItem *li; int ret = 0; - LIST_SCAN(li, head) { + CIRC_LIST_SCAN(li, head) { ret++; } return ret; } +#if defined(__cplusplus) +} // extern "C" +#endif + #endif /* _CIRCLIST_H_ */ diff --git a/vmmon-only/include/community_source.h b/vmmon-only/include/community_source.h index 2d32d8f..754416e 100644 --- a/vmmon-only/include/community_source.h +++ b/vmmon-only/include/community_source.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2009-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2009-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmmon-only/include/compat_cred.h b/vmmon-only/include/compat_cred.h deleted file mode 100644 index 4c6d940..0000000 --- a/vmmon-only/include/compat_cred.h +++ /dev/null @@ -1,51 +0,0 @@ -/********************************************************* - * Copyright (C) 2002 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef __COMPAT_CRED_H__ -# define __COMPAT_CRED_H__ - - -/* - * Include linux/cred.h via linux/sched.h - it is not nice, but - * as cpp does not have #ifexist... - */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -#include -#else -#include -#endif - -#if !defined(current_fsuid) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29) -#define current_uid() (current->uid) -#define current_euid() (current->euid) -#define current_fsuid() (current->fsuid) -#define current_gid() (current->gid) -#define current_egid() (current->egid) -#define current_fsgid() (current->fsgid) -#endif - -#if !defined(cap_set_full) -/* cap_set_full was removed in kernel version 3.0-rc4. */ -#define cap_set_full(_c) do { (_c) = CAP_FULL_SET; } while (0) -#endif - -#if !defined(GLOBAL_ROOT_UID) -#define GLOBAL_ROOT_UID (0) -#endif - -#endif /* __COMPAT_CRED_H__ */ diff --git a/vmmon-only/include/compat_highmem.h b/vmmon-only/include/compat_highmem.h deleted file mode 100644 index 263380d..0000000 --- a/vmmon-only/include/compat_highmem.h +++ /dev/null @@ -1,32 +0,0 @@ -/********************************************************* - * Copyright (C) 2012 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef __COMPAT_HIGHMEM_H__ -# define __COMPAT_HIGHMEM_H__ - -#include - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) -# define compat_kmap_atomic(_page) kmap_atomic(_page) -# define compat_kunmap_atomic(_page) kunmap_atomic(_page) -#else -# define compat_kmap_atomic(_page) kmap_atomic((_page), KM_USER0) -# define compat_kunmap_atomic(_page) kunmap_atomic((_page), KM_USER0) -#endif - -#endif /* __COMPAT_HIGHMEM_H__ */ diff --git a/vmmon-only/include/compat_interrupt.h b/vmmon-only/include/compat_interrupt.h deleted file mode 100644 index 1d72a4b..0000000 --- a/vmmon-only/include/compat_interrupt.h +++ /dev/null @@ -1,55 +0,0 @@ -/********************************************************* - * Copyright (C) 2003 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef __COMPAT_INTERRUPT_H__ -# define __COMPAT_INTERRUPT_H__ - - -#include - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 69) -/* - * We cannot just define irqreturn_t, as some 2.4.x kernels have - * typedef void irqreturn_t; for "increasing" backward compatibility. - */ -typedef void compat_irqreturn_t; -#define COMPAT_IRQ_NONE -#define COMPAT_IRQ_HANDLED -#define COMPAT_IRQ_RETVAL(x) -#else -typedef irqreturn_t compat_irqreturn_t; -#define COMPAT_IRQ_NONE IRQ_NONE -#define COMPAT_IRQ_HANDLED IRQ_HANDLED -#define COMPAT_IRQ_RETVAL(x) IRQ_RETVAL(x) -#endif - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) -#define COMPAT_IRQF_DISABLED SA_INTERRUPT -#define COMPAT_IRQF_SHARED SA_SHIRQ -#else -#define COMPAT_IRQF_DISABLED IRQF_DISABLED -#define COMPAT_IRQF_SHARED IRQF_SHARED -#endif - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) -#define COMPAT_IRQ_HANDLER_ARGS(irq, devp) (int irq, void *devp, struct pt_regs *regs) -#else -#define COMPAT_IRQ_HANDLER_ARGS(irq, devp) (int irq, void *devp) -#endif - -#endif /* __COMPAT_INTERRUPT_H__ */ diff --git a/vmmon-only/include/compat_ioport.h b/vmmon-only/include/compat_ioport.h deleted file mode 100644 index bacdb50..0000000 --- a/vmmon-only/include/compat_ioport.h +++ /dev/null @@ -1,63 +0,0 @@ -/********************************************************* - * Copyright (C) 2003 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef __COMPAT_IOPORT_H__ -# define __COMPAT_IOPORT_H__ - - -#include - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0) -static inline void * -compat_request_region(unsigned long start, unsigned long len, const char *name) -{ - if (check_region(start, len)) { - return NULL; - } - request_region(start, len, name); - return (void*)1; -} -#else -#define compat_request_region(start, len, name) request_region(start, len, name) -#endif - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 7) -/* mmap io support starts from 2.3.7, fail the call for kernel prior to that */ -static inline void * -compat_request_mem_region(unsigned long start, unsigned long len, const char *name) -{ - return NULL; -} - -static inline void -compat_release_mem_region(unsigned long start, unsigned long len) -{ - return; -} -#else -#define compat_request_mem_region(start, len, name) request_mem_region(start, len, name) -#define compat_release_mem_region(start, len) release_mem_region(start, len) -#endif - -/* these two macro defs are needed by compat_pci_request_region */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 15) -# define IORESOURCE_IO 0x00000100 -# define IORESOURCE_MEM 0x00000200 -#endif - -#endif /* __COMPAT_IOPORT_H__ */ diff --git a/vmmon-only/include/compat_pci.h b/vmmon-only/include/compat_pci.h deleted file mode 100644 index d1f897a..0000000 --- a/vmmon-only/include/compat_pci.h +++ /dev/null @@ -1,72 +0,0 @@ -/********************************************************* - * Copyright (C) 1999 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * compat_pci.h: PCI compatibility wrappers. - */ - -#ifndef __COMPAT_PCI_H__ -#define __COMPAT_PCI_H__ - -#include "compat_ioport.h" -#include - -#ifndef DMA_BIT_MASK -# define DMA_BIT_MASK(n) DMA_##n##BIT_MASK -#endif - -/* - * Power Management related compat wrappers. - */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 10) -# define compat_pci_save_state(pdev) pci_save_state((pdev), NULL) -# define compat_pci_restore_state(pdev) pci_restore_state((pdev), NULL) -#else -# define compat_pci_save_state(pdev) pci_save_state((pdev)) -# define compat_pci_restore_state(pdev) pci_restore_state((pdev)) -#endif - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11) -# define pm_message_t u32 -# define compat_pci_choose_state(pdev, state) (state) -# define PCI_D0 0 -# define PCI_D3hot 3 -#else -# define compat_pci_choose_state(pdev, state) pci_choose_state((pdev), (state)) -#endif - -/* 2.6.14 changed the PCI shutdown callback */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14) -# define COMPAT_PCI_SHUTDOWN(func) .driver = { .shutdown = (func), } -# define COMPAT_PCI_DECLARE_SHUTDOWN(func, var) (func)(struct device *(var)) -# define COMPAT_PCI_TO_DEV(dev) (to_pci_dev(dev)) -#else -# define COMPAT_PCI_SHUTDOWN(func) .shutdown = (func) -# define COMPAT_PCI_DECLARE_SHUTDOWN(func, var) (func)(struct pci_dev *(var)) -# define COMPAT_PCI_TO_DEV(dev) (dev) -#endif - -/* 2.6.26 introduced the device_set_wakeup_enable() function */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26) -# define compat_device_set_wakeup_enable(dev, val) do {} while(0) -#else -# define compat_device_set_wakeup_enable(dev, val) \ - device_set_wakeup_enable(dev, val) -#endif - -#endif /* __COMPAT_PCI_H__ */ diff --git a/vmmon-only/include/compat_pgtable.h b/vmmon-only/include/compat_pgtable.h index 4722d4e..aca2f5d 100644 --- a/vmmon-only/include/compat_pgtable.h +++ b/vmmon-only/include/compat_pgtable.h @@ -48,11 +48,6 @@ typedef p4d_t compat_p4d_t; # define COMPAT_P4D_MASK 0 /* Not used */ typedef pgd_t compat_p4d_t; #endif -/* p[gu]d_large did not exist before 2.6.25 */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25) -# define pud_large(pud) 0 -# define pgd_large(pgd) 0 -#endif /* pud_pfn did not exist before 3.8. */ #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) # define pud_pfn(pud) INVALID_MPN diff --git a/vmmon-only/include/compat_semaphore.h b/vmmon-only/include/compat_semaphore.h deleted file mode 100644 index f5527b9..0000000 --- a/vmmon-only/include/compat_semaphore.h +++ /dev/null @@ -1,49 +0,0 @@ -/********************************************************* - * Copyright (C) 2002 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef __COMPAT_SEMAPHORE_H__ -# define __COMPAT_SEMAPHORE_H__ - - -/* <= 2.6.25 have asm only, 2.6.26 has both, and 2.6.27-rc2+ has linux only. */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27) -# include -#else -# include -#endif - - -/* -* The init_MUTEX_LOCKED() API appeared in 2.2.18, and is also in -* 2.2.17-21mdk --hpreg -*/ - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 18) - #ifndef init_MUTEX_LOCKED - #define init_MUTEX_LOCKED(_sem) *(_sem) = MUTEX_LOCKED - #endif - #ifndef DECLARE_MUTEX - #define DECLARE_MUTEX(name) struct semaphore name = MUTEX - #endif - #ifndef DECLARE_MUTEX_LOCKED - #define DECLARE_MUTEX_LOCKED(name) struct semaphore name = MUTEX_LOCKED - #endif -#endif - - -#endif /* __COMPAT_SEMAPHORE_H__ */ diff --git a/vmmon-only/include/cpuid_info.h b/vmmon-only/include/cpuid_info.h index 1d3ba30..bd1cdbd 100644 --- a/vmmon-only/include/cpuid_info.h +++ b/vmmon-only/include/cpuid_info.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -29,6 +29,10 @@ #include "vm_basic_asm.h" #include "x86cpuid_asm.h" +#if defined __cplusplus +extern "C" { +#endif + typedef struct CPUID0 { int numEntries; @@ -92,4 +96,9 @@ CPUIDSummary_RegsFromCpuid0(CPUID0* id0In, return id0Out; } + +#if defined __cplusplus +} // extern "C" #endif + +#endif // _CPUID_INFO_H diff --git a/vmmon-only/include/crossgdt.h b/vmmon-only/include/crossgdt.h index ad823ea..b3374a4 100644 --- a/vmmon-only/include/crossgdt.h +++ b/vmmon-only/include/crossgdt.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2006-2014 VMware, Inc. All rights reserved. + * Copyright (C) 2006-2015 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -42,7 +42,7 @@ typedef struct CrossGDT { Descriptor gdtes[0x5000 / sizeof (Descriptor)]; // 0x5000 > GDT_LIMIT } CrossGDT; -#define CROSSGDT_NUMPAGES ((sizeof (CrossGDT) + PAGE_SIZE - 1) / PAGE_SIZE) +#define CROSSGDT_NUMPAGES CEILING(sizeof (CrossGDT), PAGE_SIZE) /* * Out of the 5 pages, only the first and last are really used. @@ -52,28 +52,17 @@ typedef struct CrossGDT { * mapping and invalidation code which pages are mapped and unmapped. */ #define CROSSGDT_PAGEMASK 0x11 +#define CROSSGDT_GETINDEXMASK(i) (1 << ((i) * sizeof (Descriptor) / PAGE_SIZE)) +#define CROSSGDT_TESTINDEXMASK(i) (CROSSGDT_GETINDEXMASK(i) & CROSSGDT_PAGEMASK) /* - * These segments are placed in the first crossGDT page. We assume - * they do not overlap any host segments (checked by - * Task_InitCrossGDT). The only real requirement is that they (and - * the host CS/SS) be physically contiguous with the start of the - * crossGDT so they will remain valid when paging is turned off. + * All necessary host segments must be below CROSSGDT_HOSTLIMIT. In Nov 2006, + * host GDT limits for the various guest OSes were: * - * As of this writing (Nov 2006), host GDT limits: * Linux 64 bit: 80 (yes 80, not 7F) - * 32 bit: FF * MacOS 64 bit: 8F - * 32 bit: 8F * Windows 64 bit: 6F - * 32 bit: 3FF */ -#define CROSSGDT_LOWSEG (0x0FD0 / 8) // all host segs must be below this -#define CROSSGDT_64BITCS (0x0FD0 / 8) // 64-bit code segment -#define CROSSGDT_64BITSS (0x0FD8 / 8) // 64-bit data segment -#define CROSSGDT_FLAT32CS (0x0FE0 / 8) // 32-bit flat code seg -#define CROSSGDT_FLAT32SS (0x0FE8 / 8) // 32-bit flat data seg -#define CROSSGDT_MON32CS (0x0FF0 / 8) // 32-bit FFC00000 base code seg -#define CROSSGDT_MON32SS (0x0FF8 / 8) // 32-bit FFC00000 base data seg +#define CROSSGDT_HOSTLIMIT (PAGE_SIZE / sizeof (Descriptor)) /* 1st page */ #endif diff --git a/vmmon-only/include/iocontrols.h b/vmmon-only/include/iocontrols.h index 6bce93e..f528291 100644 --- a/vmmon-only/include/iocontrols.h +++ b/vmmon-only/include/iocontrols.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -43,6 +43,17 @@ #include "overheadmem_types.h" #include "pageLock_defs.h" #include "numa_defs.h" +#include "bootstrap_vmm.h" + +#if defined __cplusplus +extern "C" { +#endif + + +/* + * Maximum number of shared regions that can be passed to MonLoader. + */ +#define ML_SHARED_REGIONS_MAX 3 /* *----------------------------------------------------------------------------- @@ -133,7 +144,7 @@ PtrToVA64(void const *ptr) // IN * */ -#define VMMON_VERSION (308 << 16 | 0) +#define VMMON_VERSION (329 << 16 | 0) #define VMMON_VERSION_MAJOR(v) ((uint32) (v) >> 16) #define VMMON_VERSION_MINOR(v) ((uint16) (v)) @@ -197,11 +208,10 @@ enum IOCTLCmd { #endif IOCTLCMD(VERSION) = IOCTLCMD(FIRST), IOCTLCMD(CREATE_VM), + IOCTLCMD(PROCESS_BOOTSTRAP), IOCTLCMD(RELEASE_VM), IOCTLCMD(GET_NUM_VMS), - IOCTLCMD(ALLOC_CROSSGDT), IOCTLCMD(INIT_VM), - IOCTLCMD(INIT_CROSSGDT), IOCTLCMD(RUN_VM), IOCTLCMD(LOOK_UP_MPN), IOCTLCMD(LOCK_PAGE), @@ -212,7 +222,6 @@ enum IOCTLCmd { IOCTLCMD(ADMIT), IOCTLCMD(UPDATE_MEM_INFO), IOCTLCMD(READMIT), - IOCTLCMD(PAE_ENABLED), IOCTLCMD(GET_TOTAL_MEM_USAGE), IOCTLCMD(GET_KHZ_ESTIMATE), IOCTLCMD(SET_HOST_CLOCK_RATE), @@ -224,7 +233,6 @@ enum IOCTLCmd { IOCTLCMD(ALLOC_LOCKED_PAGES), IOCTLCMD(FREE_LOCKED_PAGES), IOCTLCMD(GET_NEXT_ANON_PAGE), - IOCTLCMD(GET_LOCKED_PAGES_LIST), IOCTLCMD(GET_ALL_MSRS), @@ -251,7 +259,6 @@ enum IOCTLCmd { #if defined __linux__ || defined __APPLE__ IOCTLCMD(GET_ALL_CPUID), - IOCTLCMD(GET_KERNEL_CLOCK_RATE), #endif #if defined _WIN32 || defined __APPLE__ @@ -260,9 +267,8 @@ enum IOCTLCmd { #if defined _WIN32 IOCTLCMD(FREE_CONTIG_PAGES), - IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // Windows 2000 only - IOCTLCMD(BLUE_SCREEN), // USE_BLUE_SCREEN only - IOCTLCMD(CHANGE_HARD_LIMIT), + IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows + IOCTLCMD(CHANGE_HARD_LIMIT), // used by vmauthd on Windows IOCTLCMD(GET_KERNEL_PROC_ADDRESS), IOCTLCMD(READ_VA64), IOCTLCMD(SET_MEMORY_PARAMS), @@ -277,14 +283,7 @@ enum IOCTLCmd { IOCTLCMD(BLUEPILL), #endif - IOCTLCMD(SET_POLL_TIMEOUT_PTR), - - IOCTLCMD(FAST_SUSP_RES_SET_OTHER_FLAG), - IOCTLCMD(FAST_SUSP_RES_GET_MY_FLAG), - #if defined __linux__ - IOCTLCMD(SET_HOST_CLOCK_PRIORITY), - IOCTLCMD(VMX_ENABLED), IOCTLCMD(SET_HOST_SWAP_SIZE), #endif @@ -318,10 +317,9 @@ enum IOCTLCmd { #define IOCTL_VMX86_VERSION VMIOCTL_BUFFERED(VERSION) #define IOCTL_VMX86_CREATE_VM VMIOCTL_BUFFERED(CREATE_VM) #define IOCTL_VMX86_RELEASE_VM VMIOCTL_BUFFERED(RELEASE_VM) +#define IOCTL_VMX86_PROCESS_BOOTSTRAP VMIOCTL_BUFFERED(PROCESS_BOOTSTRAP) #define IOCTL_VMX86_GET_NUM_VMS VMIOCTL_BUFFERED(GET_NUM_VMS) -#define IOCTL_VMX86_ALLOC_CROSSGDT VMIOCTL_BUFFERED(ALLOC_CROSSGDT) #define IOCTL_VMX86_INIT_VM VMIOCTL_BUFFERED(INIT_VM) -#define IOCTL_VMX86_INIT_CROSSGDT VMIOCTL_BUFFERED(INIT_CROSSGDT) #define IOCTL_VMX86_RUN_VM VMIOCTL_NEITHER(RUN_VM) #define IOCTL_VMX86_SEND_IPI VMIOCTL_NEITHER(SEND_IPI) #define IOCTL_VMX86_GET_IPI_VECTORS VMIOCTL_BUFFERED(GET_IPI_VECTORS) @@ -334,8 +332,6 @@ enum IOCTLCmd { #define IOCTL_VMX86_ADMIT VMIOCTL_BUFFERED(ADMIT) #define IOCTL_VMX86_READMIT VMIOCTL_BUFFERED(READMIT) #define IOCTL_VMX86_UPDATE_MEM_INFO VMIOCTL_BUFFERED(UPDATE_MEM_INFO) -#define IOCTL_VMX86_PAE_ENABLED VMIOCTL_BUFFERED(PAE_ENABLED) -#define IOCTL_VMX86_BEEP VMIOCTL_BUFFERED(BEEP) #define IOCTL_VMX86_HARD_LIMIT_MONITOR_STATUS VMIOCTL_BUFFERED(HARD_LIMIT_MONITOR_STATUS) #define IOCTL_VMX86_CHANGE_HARD_LIMIT VMIOCTL_BUFFERED(CHANGE_HARD_LIMIT) #define IOCTL_VMX86_ALLOC_CONTIG_PAGES VMIOCTL_BUFFERED(ALLOC_CONTIG_PAGES) @@ -352,7 +348,6 @@ enum IOCTLCmd { #define IOCTL_VMX86_ALLOC_LOCKED_PAGES VMIOCTL_BUFFERED(ALLOC_LOCKED_PAGES) #define IOCTL_VMX86_FREE_LOCKED_PAGES VMIOCTL_BUFFERED(FREE_LOCKED_PAGES) #define IOCTL_VMX86_GET_NEXT_ANON_PAGE VMIOCTL_BUFFERED(GET_NEXT_ANON_PAGE) -#define IOCTL_VMX86_GET_LOCKED_PAGES_LIST VMIOCTL_BUFFERED(GET_LOCKED_PAGES_LIST) #define IOCTL_VMX86_GET_KERNEL_PROC_ADDRESS VMIOCTL_BUFFERED(GET_KERNEL_PROC_ADDRESS) #define IOCTL_VMX86_READ_VA64 VMIOCTL_BUFFERED(READ_VA64) @@ -361,16 +356,11 @@ enum IOCTLCmd { #define IOCTL_VMX86_REMEMBER_KHZ_ESTIMATE VMIOCTL_BUFFERED(REMEMBER_KHZ_ESTIMATE) #define IOCTL_VMX86_GET_ALL_MSRS VMIOCTL_BUFFERED(GET_ALL_MSRS) -#define IOCTL_VMX86_COUNT_PRESENT_PAGES VMIOCTL_BUFFERED(COUNT_PRESENT_PAGES) - -#define IOCTL_VMX86_FAST_SUSP_RES_SET_OTHER_FLAG VMIOCTL_BUFFERED(FAST_SUSP_RES_SET_OTHER_FLAG) -#define IOCTL_VMX86_FAST_SUSP_RES_GET_MY_FLAG VMIOCTL_BUFFERED(FAST_SUSP_RES_GET_MY_FLAG) #define IOCTL_VMX86_GET_REFERENCE_CLOCK_HZ VMIOCTL_BUFFERED(GET_REFERENCE_CLOCK_HZ) #define IOCTL_VMX86_INIT_PSEUDO_TSC VMIOCTL_BUFFERED(INIT_PSEUDO_TSC) #define IOCTL_VMX86_CHECK_PSEUDO_TSC VMIOCTL_BUFFERED(CHECK_PSEUDO_TSC) #define IOCTL_VMX86_GET_PSEUDO_TSC VMIOCTL_NEITHER(GET_PSEUDO_TSC) -#define IOCTL_VMX86_SET_HOST_CLOCK_PRIORITY VMIOCTL_BUFFERED(SET_HOST_CLOCK_PRIORITY) #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS) #define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST) #define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST) @@ -563,13 +553,34 @@ typedef struct IPIVectors { #endif /* - * This struct is passed to IOCTL_VMX86_INIT_CROSSGDT to fill in a crossGDT - * entry. + * Arguments and return value for VM creation. */ -typedef struct InitCrossGDT { - uint32 index; // index in crossGDT to update (offset / 8) - Descriptor value; // value to set the crossGDT entry to -} InitCrossGDT; +typedef struct VMCreateBlock { + VA64 bsBlob; // IN: User VA of the VMM bootstrap blob. + uint32 bsBlobSize; // IN: Size of VMM bootstrap blob. + uint16 vmid; // OUT: VM ID for the created VM. +} VMCreateBlock; + +/* + * Information about a shared region. + */ +typedef struct VMSharedRegion { + uint32 index; + VPN baseVpn; + uint32 numPages; +} VMSharedRegion; + +/* + * Arguments for VMM bootstrap processing. + */ +typedef struct VMProcessBootstrapBlock { + VA64 bsBlobAddr; // IN: User VA of the VMM bootstrap blob. + uint32 numBytes; // IN: Size of VMM bootstrap blob. + uint32 headerOffset; // IN: Offset of header in blob. + uint16 numVCPUs; // IN: Number of VCPUs. + VA64 ptRootVAs[MAX_VCPUS]; // IN: User VA of PT roots. + VMSharedRegion shRegions[ML_SHARED_REGIONS_MAX]; // IN: Shared regions. +} VMProcessBootstrapBlock; #if defined __linux__ @@ -620,4 +631,9 @@ typedef struct VMAllocContiguousMem { /* Clean up helper macros */ #undef IOCTLCMD + +#if defined __cplusplus +} // extern "C" +#endif + #endif // ifndef _IOCONTROLS_H_ diff --git a/vmmon-only/include/memDefaults.h b/vmmon-only/include/memDefaults.h index 6861170..3c98645 100644 --- a/vmmon-only/include/memDefaults.h +++ b/vmmon-only/include/memDefaults.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2012 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -24,12 +24,15 @@ #define INCLUDE_ALLOW_VMMON #define INCLUDE_ALLOW_VMCORE #define INCLUDE_ALLOW_MODULE - #include "includeCheck.h" #include "vm_basic_math.h" #include "vm_basic_defs.h" +#if defined(__cplusplus) +extern "C" { +#endif + #define MEMDEFAULTS_MIN_HOST_PAGES MBYTES_2_PAGES(128) @@ -151,4 +154,8 @@ MemDefaults_CalcMinReservedMBs(uint32 hostMem) // IN: void MemDefaults_GetReservedMemory(uint32 *host, uint32 *min, uint32 *max, uint32 *recommended); +#if defined(__cplusplus) +} // extern "C" +#endif + #endif diff --git a/vmmon-only/include/modulecall.h b/vmmon-only/include/modulecall.h index 3f6f444..9487cd5 100644 --- a/vmmon-only/include/modulecall.h +++ b/vmmon-only/include/modulecall.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -56,7 +56,9 @@ MC(COSCHED) \ MC(ALLOC_VMX_PAGE) \ MC(ALLOC_TMP_GDT) \ - MC(PIN_MPN) + MC(PIN_MPN) \ + MC(VMCLEAR_VMCS_ALL_CPUS) \ + MC(GET_PAGE_ROOT) /* *---------------------------------------------------------------------- @@ -124,55 +126,14 @@ typedef enum UCCostStamp { } UCCostStamp; #endif // VMX86_SERVER -/* - * Header for the wsBody64.S worldswitch code file. - */ -typedef struct WSModule { - uint32 vmmonVersion; // VMMON_VERSION when assembled as part of monitor - uint16 moduleSize; // size of whole wsBody64 module - uint16 hostToVmm; // offset from beginning of header to hostToVmm - uint16 vmm64ToHost; - uint16 _pad[3]; - - uint8 code[1024]; // big enough for MAX('.wsBody64', 'wsBody64Log') -} WSModule; - typedef #include "vmware_pack_begin.h" -struct SwitchNMIOffsets { - uint16 db; // offset to start of #DB handler - uint16 nmi; // offset to start of #NMI handler - uint16 df; // offset to start of #DF handler - uint16 ud; // offset to start of #UD handler - uint16 gp; // offset to start of #GP handler - uint16 pf; // offset to start of #PF handler - uint16 mce; // offset to start of #MCE handler +struct CodeOffsets { + uint16 hostToVmm; + uint16 vmmToHost; } #include "vmware_pack_end.h" -SwitchNMIOffsets; - -/* - * This is a header for the switchNMI.S module. It contains code for - * exceptions occurring during worldswitch. The code gets copied to - * the crosspage by initialization. - */ -typedef -#include "vmware_pack_begin.h" -struct SwitchNMI { // see switchNMI.S - uint16 switchNMISize; - SwitchNMIOffsets host; // offsets to handlers - volatile Bool wsException[NUM_EXCEPTIONS]; // EXC_DE ... EXC_XF - // TRUE -> fault occurred in - // worldswitch - uint64 wsUD2; // IP of ud2 instruction - // 0ULL == unset - // other == worldswitch IP - uint8 codeBlock[768]; // Enough for - // max('.switchNMI', - // '.switchNMILog'). -} -#include "vmware_pack_end.h" -SwitchNMI; +CodeOffsets; #define SHADOW_DR(cpData, n) (cpData)->shadowDR[n].ureg64 @@ -211,7 +172,7 @@ SwitchNMI; /*---------------------------------------------------------------------- * - * VMM64PageTablePatch + * VMMPageTablePatch * * Describes an entry in the monitor page table which needs to be * patched during the back-to-host worldswitch. @@ -224,28 +185,16 @@ SwitchNMI; * This is the level in the page table to which the patch must * be applied: L4, L3, L2, L1. This information is used to * determine the base of the region of memory which must be - * patched. The level value corresponds to the following - * regions in monitor memory: + * patched. The value zero is reserved to indicate an empty spot + * in the array of patches. * - * MMU_ROOT_64 - * MMU_L3_64 - * MMU_L2_64 - * MON_PAGE_TABLE_64 + * o index * - * The value zero is reserved to indicate an empty spot in the - * array of patches. + * The index of the PTE at the given page table. * - * o level offset + * o ptIdx * - * The monitor memory regions corresponding to the page table - * levels may be more than one page in length, so a 'page - * offset' is required to know the starting address of the page - * table page which must be patched in 'level'. - * - * o page index - * - * The 'index' value specifies the element in the page which - * should be patched. + * The index of the page table at the given level. * * o pte * @@ -256,19 +205,19 @@ SwitchNMI; */ typedef #include "vmware_pack_begin.h" -struct VMM64PageTablePatch { +struct VMMPageTablePatch { #define PTP_EMPTY (0U) /* Unused array entry. (must be 0) */ -#define PTP_LEVEL_L1 (1U) /* leaf level */ +#define PTP_LEVEL_L1 (1U) #define PTP_LEVEL_L2 (2U) #define PTP_LEVEL_L3 (3U) #define PTP_LEVEL_L4 (4U) /* root level */ - uint16 level; /* [0, 4] (maximal size: 3 bits) */ - uint16 page; /* Index of 'page' in 'level'. */ - uint32 index; /* Index of 'pte' in 'page'. */ - VM_PDPTE pte; /* PTE. */ + uint32 level; /* [0, 4] (maximal size: 3 bits) */ + uint32 pteIdx; /* Index of the PTE in the page table. */ + uint64 pteGlobalIdx; /* Global index of the PTE in 'level'. */ + VM_PDPTE pte; /* PTE. */ } #include "vmware_pack_end.h" -VMM64PageTablePatch; +VMMPageTablePatch; #define MODULECALL_NUM_ARGS 4 @@ -345,16 +294,17 @@ struct VMCrossPageData { uint64 mon64RIP; Task64 monTask64; /* vmm64's task */ - VMM64PageTablePatch vmm64PTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */ - LA64 vmm64CrossPageLA; - LA64 vmm64CrossGDTLA; // where crossGDT mapped by PT patch - // 64-bit host: host kernel linear - // address + VMMPageTablePatch vmmPTP[MAX_SWITCH_PT_PATCHES]; /* page table patch */ + LA64 vmm64CrossPageLA; + LA64 vmm64CrossGDTLA; // where crossGDT mapped by PT patch + // 64-bit host: host kernel linear + // address /* * The monitor may requests up to two actions when returning to the * host. The moduleCallType field and args encode a request for - * some action in the driver. The userCallType field (together + * some action in the driver. The vcpuSet field is an additional + * argument used in some calls. The userCallType field (together * with the RPC block) encodes a user call request. The two * requests are independent. The user call is executed first, with * the exception of MODULECALL_INTR which has a special effect. @@ -362,6 +312,9 @@ struct VMCrossPageData { ModuleCallType moduleCallType; uint32 retval; uint64 args[MODULECALL_NUM_ARGS]; +#if !defined(VMX86_SERVER) + VCPUSet vcpuSet; +#endif int userCallType; uint32 pcpuNum; /* Used as extra module call arg within vmmon. */ @@ -397,8 +350,6 @@ struct VMCrossPageData { VmAbsoluteTS hstTimerExpiry; // PTSC of host timer interrupt VmAbsoluteTS monTimerExpiry; // PTSC of next MonTimer callback - Bool activateVMX; // TRUE -> activate Intel VMX extensions - Bool activateSVM; // TRUE -> activate AMD SVM extensions Bool retryWorldSwitch; // TRUE -> return to host on host->vmm switch /* * TRUE if moduleCall was interrupted by signal. Only @@ -406,12 +357,11 @@ struct VMCrossPageData { * restart RunVM call, nobody else should look at it. */ Bool moduleCallInterrupted; - uint8 _pad3[4]; + uint8 _pad3[6]; - DTR64 switchHostIDTR; // baseLA = switchHostIDT's host knl LA + DTR64 switchHostIDTR; // baseLA = switchHostIDT's host kernel LA uint16 _pad4[3]; - DTR64 switchMon64IDTR; // has baseLA = switchMon64IDT's monitor LA - // contains 64-bit DB,NMI,MCE entries + DTR64 switchMonIDTR; // baseLA = switchMonIDT's monitor LA uint16 _pad5[3]; /* @@ -419,12 +369,16 @@ struct VMCrossPageData { * IDT has only enough space for the hardware exceptions; they are * sized to accommodate 64-bit descriptors. */ - uint8 switchHostIDT [sizeof(Gate64) * NUM_EXCEPTIONS]; // hostCS:hostVA - uint8 switchMon64IDT[sizeof(Gate64) * NUM_EXCEPTIONS]; // 64-bit monCS:monVA + uint8 switchHostIDT[sizeof(Gate64) * NUM_EXCEPTIONS]; // hostCS:hostVA + uint8 switchMonIDT[sizeof(Gate64) * NUM_EXCEPTIONS]; // monCS:monVA + + volatile Bool wsException[NUM_EXCEPTIONS]; // Tracks faults in worldswitch. + uint64 wsUD2; // IP of ud2 instr or 0 if unset. } #include "vmware_pack_end.h" VMCrossPageData; + /* *---------------------------------------------------------------------- * @@ -437,8 +391,10 @@ VMCrossPageData; typedef #include "vmware_pack_begin.h" struct VMCrossPageCode { - WSModule worldswitch; - SwitchNMI faultHandler; + uint16 size; // Size of the code module. + uint32 vmmonVersion; // VMMON_VERSION + CodeOffsets offsets; // Offsets to handlers. + uint8 codeBlock[512]; // Code for worldswitch and fault handling. } #include "vmware_pack_end.h" VMCrossPageCode; @@ -465,16 +421,16 @@ struct VMCrossPage { uint32 version; /* 4 bytes. Must be at offset zero. */ uint32 crosspage_size; /* 4 bytes. Must be at offset 4. */ VMCrossPageData crosspageData; + VMCrossPageCode crosspageCode; uint8 _pad[PAGE_SIZE - (sizeof(uint32) /* version */ + sizeof(uint32) /* crosspage_size */ + sizeof(VMCrossPageData) + sizeof(VMCrossPageCode))]; - VMCrossPageCode crosspageCode; } #include "vmware_pack_end.h" VMCrossPage; -#define CROSSPAGE_VERSION_BASE 0xbf1 /* increment by 1 */ +#define CROSSPAGE_VERSION_BASE 0xbfb /* increment by 1 */ #define CROSSPAGE_VERSION ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS) #if !defined(VMX86_SERVER) && defined(VMM) diff --git a/vmmon-only/include/modulecallstructs.h b/vmmon-only/include/modulecallstructs.h index 7943a2b..1537b27 100644 --- a/vmmon-only/include/modulecallstructs.h +++ b/vmmon-only/include/modulecallstructs.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2006,2009-2011,2013-2014 VMware, Inc. All rights reserved. + * Copyright (C) 2006,2009-2011,2013-2015 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -66,7 +66,8 @@ SWMSR(MSR_CSTAR) \ SWMSR(MSR_SFMASK) \ SWMSR(MSR_TSC_AUX) \ - SWMSR(MSR_BD_TSC_RATIO) + SWMSR(MSR_BD_TSC_RATIO) \ + SWMSR(MSR_BNDCFGS) /* * Data structures for dealing with the context-switched MSRs that need diff --git a/vmmon-only/include/monAddrLayout.h b/vmmon-only/include/monAddrLayout.h new file mode 100644 index 0000000..f47800d --- /dev/null +++ b/vmmon-only/include/monAddrLayout.h @@ -0,0 +1,57 @@ +/********************************************************* + * Copyright (C) 2005,2007,2009,2013,2015 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * monAddrLayout.h -- + * + * Address layout of the monitor + */ + +#ifndef _MON_ADDR_LAYOUT_H +#define _MON_ADDR_LAYOUT_H + +#define INCLUDE_ALLOW_VMCORE +#define INCLUDE_ALLOW_VMMON +#define INCLUDE_ALLOW_VMKERNEL +#define INCLUDE_ALLOW_USERLEVEL +#include "includeCheck.h" + +#pragma pack(push, 1) +typedef struct VMM64_AddrLayout { + /* + *All address are VPNs and all lengths are numPages + */ + uint64 monBase; // MONITOR_BASE_VPN + uint64 mmuRootStart; // MMU_ROOT_START + uint32 mmuRootLen; // MMU_ROOT_LEN + uint64 mmuL3Start; // MMU_L3_START + uint32 mmuL3Len; // MMU_L3_LEN + uint64 mmuL2Start; // MMU_L2_START + uint32 mmuL2Len; // MMU_L2_LEN + uint64 monPageTableStart; // MON_PAGE_TABLE_START + uint32 monPageTableLen; // MON_PAGE_TABLE_LEN +#ifndef VMX86_SERVER + uint64 monCpStart; // CROSS_PAGE_START +#endif +} VMM64_AddrLayout; +#pragma pack(pop) + +#define VMM_SCRATCHAS_FIRST_L4OFF 1 +#define VMM_SCRATCHAS_LAST_L4OFF 127 + +#endif diff --git a/vmmon-only/include/monLoader.h b/vmmon-only/include/monLoader.h new file mode 100644 index 0000000..bd69830 --- /dev/null +++ b/vmmon-only/include/monLoader.h @@ -0,0 +1,302 @@ +/********************************************************* + * Copyright (C) 2015-2017 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * monLoader.h -- + * + * Describes the monitor loader, its header and support code for + * processing this header. + * + * + * Overview + * ======== + * + * Before the monitor or its in-context bootstrap code can run, a monitor + * address space must be created and partially populated. This address space + * contains code and data statically built, empty but allocated space, shared + * and run-time initialized content. The monitor loader header regularizes + * encoding of address space information, allowing a common representation and + * common code to be re-used for different contexts (vmmon for hosted, vmkernel + * for ESX, and the VMX while deprivileging of bootstrap is ongoing). + * + * The Header + * ========== + * + * The monitor loader header contains a sequence of entries describing content. + * Each entry has a start and end VPN, an optional content source, permission + * flags to be applied when mapping, an optional subIndex specifying a shared + * region (for shared content types) and a flag specifying whether to process + * this entry for all VCPUs or just the bootstrap processor. + * + * Content types + * ============= + * + * ADDRSPACE: Must be first. Describes the address space in which all other + * entries reside. Page tables from L4 to L1 will be allocated, in order, + * eagerly. These page tables will be wired with the permissions of this entry + * from L4 to L1. This allows easy access to the L1E for any VPN in the space. + * + * ML_CONTENT_PAGETABLE_Lx: Maps the page tables for a given level. It must be + * exactly the size of the tables preallocated by ADDRSPACE for the level. + * + * ML_CONTENT_ALLOCZERO: Allocates new MPNs, zeroes the pages and maps them. + * + * ML_CONTENT_COPY: Allocates new MPNs, maps them and copies from the specified + * source. + * + * ML_CONTENT_SHARE: Memory provided by user or host kernel and mapped into the + * monitor address space. The subIndex field specifies which region, as + * multiple regions may be shared for each source. + * + * Processing + * ========== + * + * The monitor loader can be built in different contexts. Each context must + * provide implementations of callout functions. To fully build the context, + * callouts for memory allocation, access and reporting of resources will be + * added, as will accounting functionality. + * + * Callouts + * ======== + * + * Memory-accessing callouts take a Vcpuid as memory is tracked per-VCPU. + * + * MonLoaderCallout_AllocMPN(MPN): Allocate a new MPN. + * MonLoaderCallout_CleanUp(): Release temporary MonLoader callout resources. + * MonLoaderCallout_CopyFromBlob(blobOffset, size, MPN): Copy blob contents. + * MonLoaderCallout_FillPage(pattern, mpn): fill a page with a pattern. + * MonLoaderCallout_GetPTE(MPN, index, *pte): Get indexth PTE in a PT MPN. + * MonLoaderCallout_GetPageRoot(vcpu): Get a VCPU's page root. + * MonLoaderCallout_ImportPage(MPN): Import an MPN for tracking and future use. + * MonLoaderCallout_Init(): Initialize MonLoader callouts. + * MonLoaderCallout_GetSharedHostPage(subIdx, page): Get shared host page MPN. + * MonLoaderCallout_GetSharedUserPage(subIdx, page): Get shared user page MPN. + * MonLoaderCallout_SetEntrypoint(): Sets the entry code/stack for execution. + * + * Building vs Importing + * ===================== + * + * While legacy VMX is still responsible for building part of the monitor + * context, the monitor loader must cooperate. Rather than building the + * context itself, the monitor loader verifies its environment matches its + * header's expectations. This includes verifying sufficient allocations, + * permissions of page table wiring and page table self-mapping. + * + * Errors + * ====== + * + * To avoid ASSERTs across contexts, errors are returned and clean-up is done + * carefully if processing fails at any stage. Some errors are debug-only and + * essentially verify invariants for debug builds. The line in the table at + * which the error was encountered is returned for logging and debugging. + * + * Compatibility + * ============= + * + * bootstrap-offsets.pl relies on MonLoaderContentType, MonLoaderSourceType and + * the structure of MonLoaderEntry to determine and set blob offsets for + * ML_CONTENT_COPY + ML_SOURCE_BLOB. If these are changed, the script should + * be updated accordingly. + */ + +#ifndef _MON_LOADER +#define _MON_LOADER + +#include "vm_basic_types.h" +#include "x86paging_common.h" +#include "x86types.h" /* PT_L1E */ +#include "vcpuid.h" /* Vcpuid */ + +#define ML_NAME_MAX 16 + +/* ML perms are simple and abbreviated. */ + +#define ML_PERM_RWX (PTE_P | PTE_RW) +#define ML_PERM_RW (PTE_P | PTE_RW | PTE_NX) +#define ML_PERM_RO (PTE_P | PTE_NX) +#define ML_PERM_RX PTE_P + +#define ML_PERM_MASK (PTE_P| PTE_RW | PTE_NX | PTE_US) + +#define ML_PERMS_MATCH(x,p) (((x) & ML_PERM_MASK) == ((p) & ML_PERM_MASK)) + +#define LOADER_HEADER_MAGIC 0x8675309E98675309 + +typedef enum { + ML_CONTENT_INVALID = 0, + ML_CONTENT_ADDRSPACE, /* The static address space. */ + ML_CONTENT_ALLOCZERO, /* Allocate, zero-fill and map. */ + ML_CONTENT_COPY, /* Copy data from external source. */ + ML_CONTENT_PAGETABLE_L4, /* Mappings for level 4 page tables in AS. */ + ML_CONTENT_PAGETABLE_L3, /* Mappings for level 3 page tables in AS. */ + ML_CONTENT_PAGETABLE_L2, /* Mappings for level 2 page tables in AS. */ + ML_CONTENT_PAGETABLE_L1, /* Mappings for level 1 page tables in AS. */ + ML_CONTENT_SHARE, /* Share data from external source. */ +} MonLoaderContentType; + +#define CONTENT_TO_PTLEVEL(x) (x == ML_CONTENT_PAGETABLE_L4 ? 4 : \ + x == ML_CONTENT_PAGETABLE_L3 ? 3 : \ + x == ML_CONTENT_PAGETABLE_L2 ? 2 : \ + x == ML_CONTENT_PAGETABLE_L1 ? 1 : \ + 0) + +/* Sources of content, for pages not zeroed, unmapped or otherwise special. */ +typedef enum MonLoaderSourceType { + ML_SOURCE_INVALID = 0x0, + ML_SOURCE_NONE, + ML_SOURCE_BLOB, + ML_SOURCE_USER, + ML_SOURCE_HOST, +} MonLoaderSourceType; + +/* + * NOTE: When modifying MonLoaderEntry, corresponding changes must be + * made to vmcore/make/misc/bootstrap-offsets.pl. + */ +typedef struct { + MonLoaderContentType content; /* Content type. */ + MonLoaderSourceType source; /* Source, if any, for content. */ + VPN monVPN; /* Destination in monitor AS. */ + uint64 monPages; /* Size in pages. */ + uint64 flags; /* PTE permissions */ + unsigned allocs; /* MPNs allocated (all VCPUs). */ + /* Set for source ML_SOURCE_BLOB: */ + struct blobSrc { + uint64 offset; /* offset within the blob, in bytes. */ + uint64 size; /* size of content, in bytes. */ + } blobSrc; + uint64 bspOnly; /* Process only on BSP. */ + uint64 subIndex; /* Region ID for ML_CONTENT_COPY. */ + uint64 procVmmon; /* Processed by the vmmon on hosted. */ +} MonLoaderEntry; + + +/* Packed for easy consumption by bootstrap-offsets.pl. */ +#pragma pack(push, 1) +typedef struct { + uint64 magic; + uint32 entrySize; + uint32 count; + /* cs:rip */ + uint16 codeSelector; + VA64 codeEntrypoint; + /* ss:rip */ + uint16 stackSelector; + VA64 stackEntrypoint; + MonLoaderEntry entries[]; +} MonLoaderHeader; +#pragma pack(pop) + +/* Environment context structure, defined by the environment. */ +struct MonLoaderEnvContext; +typedef struct MonLoaderEnvContext MonLoaderEnvContext; + +/* Callout prototypes */ +MPN MonLoaderCallout_AllocMPN(MonLoaderEnvContext *, Vcpuid); +void MonLoaderCallout_CleanUp(MonLoaderEnvContext *); +Bool MonLoaderCallout_CopyFromBlob(MonLoaderEnvContext *, uint64, size_t, MPN, + Vcpuid); +Bool MonLoaderCallout_FillPage(MonLoaderEnvContext *, uint8, MPN, Vcpuid); +MPN MonLoaderCallout_GetPageRoot(MonLoaderEnvContext *, Vcpuid); +Bool MonLoaderCallout_GetPTE(MonLoaderEnvContext *, MPN, unsigned, Vcpuid, + PT_L1E *); +Bool MonLoaderCallout_ImportPage(MonLoaderEnvContext *, MPN, Vcpuid); +Bool MonLoaderCallout_Init(void *, MonLoaderEnvContext **); +Bool MonLoaderCallout_MapMPNInPTE(MonLoaderEnvContext *, MPN, unsigned, uint64, + MPN, Vcpuid); +MPN MonLoaderCallout_GetSharedUserPage(MonLoaderEnvContext *, uint64, unsigned, + Vcpuid); +MPN MonLoaderCallout_GetSharedHostPage(MonLoaderEnvContext *, uint64, unsigned, + Vcpuid); +Bool MonLoaderCallout_IsPrivileged(MonLoaderEnvContext *); +Bool MonLoaderCallout_SetEntrypoint(MonLoaderEnvContext *, uint16, VA64, uint16, + VA64); +MPN MonLoaderCallout_GetBlobMpn(MonLoaderEnvContext *, uint64); + +typedef enum MonLoaderError { + ML_OK = 0, + ML_ERROR_ADDRSPACE_TOO_LARGE, + ML_ERROR_ALLOC, + ML_ERROR_ALREADY_MAPPED, + ML_ERROR_ARGS, + ML_ERROR_CALLOUT_INIT, + ML_ERROR_CALLOUT_COPY, + ML_ERROR_CALLOUT_ENTRYPOINT, + ML_ERROR_CALLOUT_GETPTE, + ML_ERROR_CALLOUT_MAPINPTE, + ML_ERROR_CALLOUT_PAGEROOT_GET, + ML_ERROR_CALLOUT_ZERO, + ML_ERROR_CONTENT_INVALID, + ML_ERROR_CONTEXT_INIT, + ML_ERROR_DUPLICATE, + ML_ERROR_INVALID_VPN, + ML_ERROR_MAGIC, + ML_ERROR_MAP, + ML_ERROR_NO_ADDRSPACE, + ML_ERROR_PAGE_TABLE_IMPORT, + ML_ERROR_PAGE_TABLE_MAP_SIZE, + ML_ERROR_PAGE_TABLE_VERIFY, + ML_ERROR_SHARE, + ML_ERROR_SIZE, + ML_ERROR_SOURCE_INVALID, + ML_ERROR_TABLE_MISSING, +} MonLoaderError; + + +/* A subindex above shared area subindices for sharing of MonLoaderHeader. */ +#define MONLOADER_HEADER_IDX (NUM_SHARED_AREAS + 1) + + +MonLoaderError MonLoader_Process(MonLoaderHeader *header, unsigned numVCPUs, + void *args, unsigned *line, Vcpuid *vcpu); + + +/* + *---------------------------------------------------------------------- + * + * MonLoader_GetFixedHeaderSize -- + * + * Returns the size of the fixed portion of MonLoaderHeader. + * + *---------------------------------------------------------------------- + */ +static INLINE size_t +MonLoader_GetFixedHeaderSize(void) +{ + return sizeof(MonLoaderHeader); +} + + +/* + *---------------------------------------------------------------------- + * + * MonLoader_GetFullHeaderSize -- + * + * Returns the size of the full MonLoaderHeader, including the + * variable size portion. + * + *---------------------------------------------------------------------- + */ +static INLINE size_t +MonLoader_GetFullHeaderSize(MonLoaderHeader *header) +{ + return MonLoader_GetFixedHeaderSize() + + header->count * sizeof(MonLoaderEntry); +} + +#endif /* !_MON_LOADER */ diff --git a/vmmon-only/include/monLoaderLog.h b/vmmon-only/include/monLoaderLog.h new file mode 100644 index 0000000..3fa0da6 --- /dev/null +++ b/vmmon-only/include/monLoaderLog.h @@ -0,0 +1,66 @@ +/********************************************************* + * Copyright (C) 2016 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * monLoaderLog.h -- + * + * Logging definitions for MonLoader. + */ + +#ifndef _MONLOADER_LOG_H_ +#define _MONLOADER_LOG_H_ + +#ifdef VMX86_VMX + +#define LOGLEVEL_MODULE main +#include "loglevel_vmcore.h" +/* + * Redefine LOG() to match the vmkernel calling convention. + */ +#undef LOG_BYNAME +#undef LOG + +#define LOG_BYNAME(_mod, _min, _fmt, ...) \ + (DOLOG_BYNAME(_mod, _min) ? Log(_fmt "\n", ## __VA_ARGS__) : (void) 0) + +#define LOG(_min, _fmt, ...) \ + LOG_BYNAME(LOGLEVEL_MODULE, _min, _fmt, ## __VA_ARGS__) + +#elif defined VMKERNEL + +#define LOGLEVEL_MODULE MonLoader +#include "log.h" + +#elif defined VMMON + +#include "vm_assert.h" +#define ML_LOGLEVEL_VMMON 0 /* MonLoader/vmmon loglevel. */ + +#undef LOG +#define LOG(_min, _fmt, ...) \ +do { \ + if (vmx86_log && ML_LOGLEVEL_VMMON >= (_min)) { \ + Log(_fmt "\n", ## __VA_ARGS__); \ + } \ +} while (0) + +#else /* !defined VMX86_VMX && !defined VMKERNEL && !defined VMMON */ +#error MonLoader cannot be built as part of this environment +#endif + +#endif /* _MONLOADER_LOG_H_ */ diff --git a/vmmon-only/include/mon_assert.h b/vmmon-only/include/mon_assert.h index b86832c..3d047f8 100644 --- a/vmmon-only/include/mon_assert.h +++ b/vmmon-only/include/mon_assert.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2007-2014 VMware, Inc. All rights reserved. + * Copyright (C) 2007-2015 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -169,8 +169,13 @@ extern const char __vmm_pathnames_start; * seen to hoist code into the failing arm of the assertion, where it * can then tell that, because of the assertion failure, the code ends * up accessing an array out of bounds. + * + * Assertion failures for the monitor's bootstrap are reduced to panics + * logging the current %rip. As such, no .assert_info section is created + * and VMX merely processes the panic. */ +#ifndef VMM_BOOTSTRAP #define ASSERT_RECORDINFO(assembly, assertType, bugNr) \ __asm__ __volatile__(".pushsection .assert_info;" \ ".quad 0f;" \ @@ -182,6 +187,16 @@ extern const char __vmm_pathnames_start; [file] "i" (__VMM__FILE__), \ [type] "i" (assertType), \ [bug] "i" (bugNr)) +#else + +extern uint64 bsAssertRIP; +#define ASSERT_RECORDINFO(assembly, assertType, bugNr) \ + __asm__ __volatile__("lea 0(%%rip), %0\n\t" \ + : "=r"(bsAssertRIP)); \ + Panic("Bootstrap: %s failure at rip=0x%lx", \ + assertType == AssertType_AssertVerify ? "VERIFY" : "ASSERT", \ + bsAssertRIP); +#endif /* VMM_BOOTSTRAP */ #define _ASSERT_PANIC(name) \ ({COMPILER_MEM_BARRIER(); \ diff --git a/vmmon-only/include/monitorAction_exported.h b/vmmon-only/include/monitorAction_exported.h index 28b57c2..745b1d9 100644 --- a/vmmon-only/include/monitorAction_exported.h +++ b/vmmon-only/include/monitorAction_exported.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2010-2013 VMware, Inc. All rights reserved. + * Copyright (C) 2010-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -22,8 +22,6 @@ #define INCLUDE_ALLOW_VMX #define INCLUDE_ALLOW_VMCORE #define INCLUDE_ALLOW_USERLEVEL -#define INCLUDE_ALLOW_MODULE -#define INCLUDE_ALLOW_VMMON #define INCLUDE_ALLOW_VMKERNEL #define INCLUDE_ALLOW_DISTRIBUTE #include "includeCheck.h" @@ -32,6 +30,11 @@ #include "vm_atomic.h" #include "vm_basic_types.h" +#if defined __cplusplus +extern "C" { +#endif + + /* * Please bump the version number if your change will break the * compatability to the drivers. @@ -106,7 +109,7 @@ MonitorActionSet_AtomicInclude(MonitorActionSet *set, const uint32 actionID) do { oldWord = Atomic_Read64(atomicSet); newWord = oldWord | mask; - } while (!Atomic_CMPXCHG64(atomicSet, &oldWord, &newWord)); + } while (!Atomic_CMPXCHG64(atomicSet, oldWord, newWord)); return (oldWord & mask) == 0; } @@ -153,4 +156,9 @@ typedef enum { VMM_C1_STATE_PAUSE } vmmC1StateType; + +#if defined __cplusplus +} // extern "C" +#endif + #endif // _MONITORACTION_EXPORTED_H_ diff --git a/vmmon-only/include/numa_defs.h b/vmmon-only/include/numa_defs.h index f14a8b5..fd2a0fc 100644 --- a/vmmon-only/include/numa_defs.h +++ b/vmmon-only/include/numa_defs.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2006 VMware, Inc. All rights reserved. + * Copyright (C) 2006,2009-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -39,15 +39,16 @@ /* Machine NUMA nodes */ typedef uint32 NUMA_Node; -typedef uint32 NUMA_NodeMask; +typedef uint32 NUMA_PxmID; +typedef uint64 NUMA_NodeMask; typedef uint8 NUMA_MemRangeID; /* * Constants */ -#define NUMA_MAX_NODES 32 +#define NUMA_MAX_NODES 64 #define NUMA_MAX_CPUS_PER_NODE (vmx86_server ? MAX_PCPUS : 32) -#define NUMA_MAX_MEM_RANGES 64 +#define NUMA_MAX_MEM_RANGES (vm_arm_64 ? 128 : 64) #define INVALID_NUMANODE ((NUMA_Node)-1) #define NUMA_NODE_MASK_ANY ((NUMA_NodeMask)-1) #define NUMA_NODE_MASK_NONE ((NUMA_NodeMask)0) @@ -59,7 +60,7 @@ typedef uint8 NUMA_MemRangeID; typedef struct { MPN startMPN; MPN endMPN; - NUMA_Node id; + NUMA_PxmID id; Bool isReliable; Bool isVolatile; } NUMA_MemRange; diff --git a/vmmon-only/include/pagelist.h b/vmmon-only/include/pagelist.h index 46ed142..8628c35 100644 --- a/vmmon-only/include/pagelist.h +++ b/vmmon-only/include/pagelist.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2010-2014 VMware, Inc. All rights reserved. + * Copyright (C) 2010-2014,2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -36,6 +36,11 @@ #include "vm_assert.h" #include "vmcore_types.h" +#if defined __cplusplus +extern "C" { +#endif + + /* * Sets of pages are passed between the monitor and the platform to be * shared, invalidated, remapped, or swapped. @@ -53,13 +58,11 @@ typedef struct PageListEntry { #define PAGELIST_MAX (PAGE_SIZE / sizeof(PageListEntry)) -static INLINE PageListEntry -PageList_CreateEntry(BPN bpn) +static INLINE void +PageList_SetEntry(PageListEntry *ple, BPN bpn) { - PageListEntry ple; - CompressedBPN_Write(&ple.cbpn, bpn); - ple.voided = FALSE; - return ple; + CompressedBPN_Write(&ple->cbpn, bpn); + ple->voided = FALSE; } static INLINE BPN @@ -98,4 +101,9 @@ PageList_IsBPNDup(const PageListEntry *pageList, unsigned i, BPN bpn) return FALSE; } + +#if defined __cplusplus +} // extern "C" #endif + +#endif // _PAGELIST_H diff --git a/vmmon-only/include/pshare_ext.h b/vmmon-only/include/pshare_ext.h index ab42c3d..2fab2ab 100644 --- a/vmmon-only/include/pshare_ext.h +++ b/vmmon-only/include/pshare_ext.h @@ -44,6 +44,7 @@ #define PSHARE_P2M_BUFFER_MPNS_MAX (16) #define PSHARE_P2M_BUFFER_MPNS_DEFAULT (4) #define PSHARE_P2M_BUFFER_SLOTS_PER_MPN (PAGE_SIZE / sizeof(PShare_P2MUpdate)) +#define PSHARE_P2M_BUFFER_SLOTS_MIN (2) #define PSHARE_POISON_MARKER (CONST64U(0xAAAAAAAAAAAAAAAA)) @@ -57,8 +58,16 @@ MY_ASSERTS(PSHARE_EXT, * types */ +typedef union { + MPN mpn; + uint64 vpmemRef; +} PShare_P2MUpdateReference; + +#define PSHARE_SET_INVALID_P2MUPDATE_REFERENCE(ref) ((ref).vpmemRef = CONST64U(-1)) +#define PSHARE_IS_INVALID_P2MUPDATE_REFERENCE(ref) ((ref).vpmemRef == CONST64U(-1)) + typedef struct PShare_P2MUpdate { - BPN bpn; - MPN mpn; + BPN bpn; + PShare_P2MUpdateReference reference; } PShare_P2MUpdate; #endif diff --git a/vmmon-only/include/ptsc.h b/vmmon-only/include/ptsc.h index dfa0328..584cf19 100644 --- a/vmmon-only/include/ptsc.h +++ b/vmmon-only/include/ptsc.h @@ -170,7 +170,6 @@ int64 PTSC_CyclesToUS(VmRelativeTS ts); /* * ESX with userworld VMX */ -#include "x86types.h" #include "user_layout.h" #ifdef VM_ARM_64 diff --git a/vmmon-only/include/rateconv.h b/vmmon-only/include/rateconv.h index 14f0195..bc76f84 100644 --- a/vmmon-only/include/rateconv.h +++ b/vmmon-only/include/rateconv.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -31,7 +31,6 @@ #define _VM_RATECONV_H_ #define INCLUDE_ALLOW_USERLEVEL - #define INCLUDE_ALLOW_MODULE #define INCLUDE_ALLOW_VMKERNEL #define INCLUDE_ALLOW_VMK_MODULE @@ -44,6 +43,12 @@ #include "vm_basic_asm.h" #include "vm_assert.h" #include "vm_atomic.h" +#include "versioned_atomic.h" + +#if defined __cplusplus +extern "C" { +#endif + /* RateConv_Params is part of vmx<->vmmon interface (INIT_PSEUDO_TSC ioctl) */ typedef struct RateConv_Params { @@ -54,8 +59,7 @@ typedef struct RateConv_Params { typedef struct RateConv_ParamsVolatile { RateConv_Params p; - Bool changed; - uint8 pad[7]; + VersionedAtomic vers; } RateConv_ParamsVolatile; typedef struct RateConv_Ratio { @@ -115,4 +119,8 @@ RateConv_Signed(const RateConv_Params *conv, int64 x) } +#if defined __cplusplus +} // extern "C" +#endif + #endif // _VM_RATECONV_H_ diff --git a/vmmon-only/include/vcpuid.h b/vmmon-only/include/vcpuid.h index c12dcef..a1bde88 100644 --- a/vmmon-only/include/vcpuid.h +++ b/vmmon-only/include/vcpuid.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2014 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2014, 2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -38,15 +38,19 @@ #include "vm_basic_types.h" +#if defined __cplusplus +extern "C" { +#endif -typedef uint32 Vcpuid; // VCPU number -#define VCPUID_INVALID (~0U) +typedef uint32 Vcpuid; // VCPU number + +#define VCPUID_INVALID (~0U) #define BOOT_VCPU_ID 0 #define IS_BOOT_VCPUID(vcpuid) ((vcpuid) == BOOT_VCPU_ID) -#define MAX_VCPUS 128 +#define MAX_VCPUS 192 #define MAX_CORES_PER_SOCKET 64 @@ -62,4 +66,8 @@ extern const VCPUSet curVcpuidSet; #endif /* VMM */ +#if defined __cplusplus +} // extern "C" +#endif + #endif // ifndef _VCPUID_H_ diff --git a/vmmon-only/include/vcpuset.h b/vmmon-only/include/vcpuset.h index 4bbe3a0..838726e 100644 --- a/vmmon-only/include/vcpuset.h +++ b/vmmon-only/include/vcpuset.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2002-2014 VMware, Inc. All rights reserved. + * Copyright (C) 2002-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -417,13 +417,13 @@ VCPUSet_PackCareful(unsigned numVCPUs, const VCPUSet *vcs, void *ptr) * * VCPUSet_UnpackCareful -- * - * Unpack a VCPUSet from the bytes at "src". + * Unpack a VCPUSet from the bytes at "ptr". * *---------------------------------------------------------------------- */ static INLINE void -VCPUSet_UnpackCareful(unsigned numVCPUs, VCPUSet *vcs, void *ptr) +VCPUSet_UnpackCareful(unsigned numVCPUs, VCPUSet *vcs, const void *ptr) { memcpy(vcs->subset, ptr, (numVCPUs + 7) / 8); } diff --git a/vmmon-only/include/vcpuset_types.h b/vmmon-only/include/vcpuset_types.h index bbb1990..3df1f8c 100644 --- a/vmmon-only/include/vcpuset_types.h +++ b/vmmon-only/include/vcpuset_types.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2002-2013 VMware, Inc. All rights reserved. + * Copyright (C) 2002-2013, 2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -19,7 +19,7 @@ /* * vcpuset_types.h -- * - * ADT for a set of VCPUs. Implemented as an array of bitmasks. + * ADT for a set of VCPUs. Implemented as an array of bitmasks. * */ @@ -35,14 +35,14 @@ #define INCLUDE_ALLOW_VMCORE #include "includeCheck.h" -#include "vm_basic_asm.h" -#include "vm_atomic.h" #include "vcpuid.h" +#include "vm_assert.h" #define VCS_SUBSET_WIDTH 64 #define VCS_SUBSET_SHIFT 6 #define VCS_SUBSET_MASK ((CONST64U(1) << VCS_SUBSET_SHIFT) - 1) -#define VCS_SUBSET_COUNT 2 +#define VCS_SUBSET_COUNT 4 + #define VCS_VCPUID_SUBSET_IDX(v) ((v) >> VCS_SUBSET_SHIFT) #define VCS_VCPUID_SUBSET_BIT(v) (CONST64U(1) << ((v) & VCS_SUBSET_MASK)) @@ -55,4 +55,16 @@ typedef struct VCPUSet { uint64 subset[VCS_SUBSET_COUNT]; } VCPUSet; +MY_ASSERTS(VCPUSET_ASSERTS, + /* + * Catch changes in VCPUSet which need to be reflected in + * bora/public/iocontrolsMacosTable.h. + */ + ASSERT_ON_COMPILE(VCS_SUBSET_COUNT == 4); + /* + * There is code that depends on sizeof(VCPUSet) being a power of + * 2 in at least vcpuHotPlug.c and possible other places. + */ + ASSERT_ON_COMPILE((sizeof(VCPUSet) & (sizeof(VCPUSet) - 1)) == 0); +) #endif diff --git a/vmmon-only/include/versioned_atomic.h b/vmmon-only/include/versioned_atomic.h index 7eb190d..650b1e6 100644 --- a/vmmon-only/include/versioned_atomic.h +++ b/vmmon-only/include/versioned_atomic.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998 VMware, Inc. All rights reserved. + * Copyright (C) 1998,2015-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -17,28 +17,62 @@ *********************************************************/ /* - * Versioned atomic synchronization: + * versioned_atomic.h -- * - * These synchronization macros allow single-writer/many-reader - * access to data, based on Leslie Lamport's paper "Concurrent - * Reading and Writing", Communications of the ACM, November 1977. + * This file implements versioned atomic synchronization, which allows + * concurrent accesses to shared data by a single writer and multiple + * readers. The algorithm is lock-free (it does not use a mutual exclusion + * mechanism), and the writer is wait-free (it never waits for readers). * - * many-writer/many-reader can be implemented on top of versioned - * atomics by using an additional spin lock to synchronize - * writers. This is preferable for cases where readers are expected to - * greatly outnumber writers. + * Many-writer/many-reader can be implemented on top of versioned atomics + * by adding an external mutual exclusion mechanism to serialize all + * writer threads into a single logical writer thread. This is preferable + * for cases where readers are expected to greatly outnumber writers. * - * Multiple concurrent writers to the version variables are not - * allowed. Even if writers are working on lock-free or disjoint - * data, the version counters are not interlocked for read-modify-write. + * Implementation + * -------------- + * This implementation is based on Leslie Lamport's paper "Concurrent + * Reading and Writing", Communications of the ACM, November 1977 + * (http://url.eng.vmware.com/udcc ). * - * Recursive use of versioned atomics in writers is currently not supported. + * Lamport's algorithm was designed for systems in which the reader and + * the writers run on separate computers and where the version variables + * (v0 and v1 in this implementation) are stored on a shared hard disk. So + * it did not assume that the version variables could be accessed + * atomically. Instead, it assumed that the version variables were made of + * multiple basic units of data, called "digits", and that each digit + * could be accessed atomically. The upper-left corner of page 4 of the + * paper precisely and concisely describes the algorithm, and in + * particular the order in which digits of the version variables must be + * accessed (left to right, right to left, or no particular order). + * + * This implementation is designed for systems in which the reader and the + * writers run on the same computer and where the version variables are + * stored in shared memory. So it assumes that the version variables can + * be accessed atomically. As a result, this implementation uses a + * simplified version of Lamport's algorithm, where: + * o Each left to right or right to left access in Lamport's algorithm is + * replaced with a single atomic access in this implementation. + * o Each access in no particular order in Lamport's algorithm is replaced + * with a single non-atomic access in this implementation. + * Note: The writer is the only thread which writes to version variables, + * so it is allowed to read them non-atomically. + * + * Multiple concurrent writers to the version variables are not allowed. + * Even if writers are working on lock-free or disjoint data, the version + * variables are not interlocked for read-modify-write. See PR514764. + * + * Recursive use of versioned atomics in writers is currently not + * supported. See PR514764. */ #ifndef _VERSIONED_ATOMIC_H #define _VERSIONED_ATOMIC_H - + +#define INCLUDE_ALLOW_MODULE #define INCLUDE_ALLOW_USERLEVEL +#define INCLUDE_ALLOW_VMK_MODULE +#define INCLUDE_ALLOW_DISTRIBUTE #define INCLUDE_ALLOW_VMKERNEL #define INCLUDE_ALLOW_VMCORE #define INCLUDE_ALLOW_VMMON @@ -46,6 +80,12 @@ #include "vm_basic_asm.h" #include "vm_assert.h" +#include "vm_atomic.h" + +#if defined __cplusplus +extern "C" { +#endif + /* * Users with attribute(packed) structs must ensure any @@ -56,115 +96,193 @@ */ typedef struct { - volatile uint32 v0; - volatile uint32 v1; + uint32 v0; + uint32 v1; } ALIGNED(4) VersionedAtomic; + /* *----------------------------------------------------------------------------- * * VersionedAtomic_BeginWrite -- - * Called by a writer to indicate that the data protected by - * a given atomic version is about to change. Effectively locks out - * all readers until EndWrite is called. - * + * + * Called by the writer to indicate that it is about to write to the + * shared data protected by 'versions'. Effectively locks out all readers + * until VersionedAtomic_EndWrite() is called. + * * Results: - * . + * None * * Side effects: - * + * None * *----------------------------------------------------------------------------- */ + static INLINE void -VersionedAtomic_BeginWrite(VersionedAtomic *versions) +VersionedAtomic_BeginWrite(VersionedAtomic *versions) // IN { - /* - * As long as the variable is on natural boundary it is guaranteed - * to be atomic, so we don't need to write from left to right as in - * Lamport's original algorithm. - */ - ASSERT(((size_t)(&versions->v0) & (sizeof(versions->v0) - 1)) == 0); - /* If recursive use is needed demand for support in PR514764 */ + /* Catch improper initialization or unsupported recursive use. */ ASSERT(versions->v1 == versions->v0); - versions->v0++; - COMPILER_MEM_BARRIER(); + + /* + * Do not use Atomic_Inc32() below: it is more expensive than + * Atomic_Write32() and unnecessary: only the write needs to be atomic, not + * the whole read + inc + write. + */ +#if !defined VMM + /* + * Atomic_Write32() below requires the address to be naturally aligned, but + * currently only checks that condition when VMM is defined (tracked by + * PR1055836). So until that PR is fixed, we must check that condition here + * when VMM is not defined. + */ + ASSERT(((uintptr_t)&versions->v0 % sizeof versions->v0) == 0); +#endif + Atomic_Write32((Atomic_uint32 *)&versions->v0, versions->v0 + 1); + + /* + * The write to 'versions->v0' must be observed by all other OS CPUs before + * the write(s) to the shared data. + * + * On arm64, this barrier is Full System, which is probably too strong: + * Inner Shareable (== all OS CPUs) should work just fine. + */ + SMP_W_BARRIER_W(); } + /* *----------------------------------------------------------------------------- * * VersionedAtomic_EndWrite -- - * Called by a writer after it is done updating shared data. Lets - * pending and new readers proceed on shared data. - * + * + * Called by the writer to indicate that it has finished writing to the + * shared data protected by 'versions'. Lets pending and new readers + * proceed. + * * Results: - * . + * None * * Side effects: - * + * None * *----------------------------------------------------------------------------- */ + static INLINE void -VersionedAtomic_EndWrite(VersionedAtomic *versions) +VersionedAtomic_EndWrite(VersionedAtomic *versions) // IN { - ASSERT(((size_t)(&versions->v1) & (sizeof(versions->v1) - 1)) == 0); + /* + * The write(s) to the shared data must be observed by all other OS CPUs + * before the write to 'versions->v1'. + * + * On arm64, this barrier is Full System, which is probably too strong: + * Inner Shareable (== all OS CPUs) should work just fine. + */ + SMP_W_BARRIER_W(); + ASSERT(versions->v1 + 1 == versions->v0); - COMPILER_MEM_BARRIER(); - versions->v1 = versions->v0; + +#if !defined VMM + /* + * Atomic_Write32() below requires the address to be naturally aligned, but + * currently only checks that condition when VMM is defined (tracked by + * PR1055836). So until that PR is fixed, we must check that condition here + * when VMM is not defined. + */ + ASSERT(((uintptr_t)&versions->v1 % sizeof versions->v1) == 0); +#endif + Atomic_Write32((Atomic_uint32 *)&versions->v1, versions->v0); } + /* *----------------------------------------------------------------------------- * * VersionedAtomic_BeginTryRead -- - * Called by a reader before it tried to read shared data. - * + * + * Called by a reader before it starts reading the shared data protected + * by 'versions'. + * * Results: - * Returns a version number to the reader. This version number - * is required to confirm validity of the read operation when reader - * calls EndTryRead. + * A version number for the caller to pass to the matching call to + * VersionedAtomic_EndTryRead(), if any. * * Side effects: - * + * None * *----------------------------------------------------------------------------- */ -static INLINE uint32 -VersionedAtomic_BeginTryRead(const VersionedAtomic *versions) -{ - uint32 readVersion; - readVersion = versions->v1; - COMPILER_MEM_BARRIER(); +static INLINE uint32 +VersionedAtomic_BeginTryRead(VersionedAtomic const *versions) // IN +{ + uint32 readVersion = Atomic_Read32((Atomic_uint32 *)&versions->v1); + + /* + * The read from 'versions->v1' must happen on this CPU before the read(s) + * from the shared data. + * + * On arm64, this barrier is Full System, which is probably too strong: + * Inner Shareable (== all OS CPUs) should work just fine, and maybe + * Non-shareable (== current CPU) would work too. + */ + SMP_R_BARRIER_R(); return readVersion; } + /* *----------------------------------------------------------------------------- * * VersionedAtomic_EndTryRead -- - * Called by a reader after it finishes reading shared data, to confirm - * validity of the data that was just read (IOW, to make sure that a - * writer did not intervene while the read was in progress). - * + * + * Called by a reader after it has finished reading the shared data + * protected by 'versions', to confirm that the data is consistent, i.e. + * that the writer has not modified the data while the reader was reading + * it. + * * Results: - * TRUE if the data read between BeginTryRead() and this call is - * valid. FALSE otherwise. + * TRUE if the shared data read between VersionedAtomic_BeginTryRead() and + * this call is consistent. + * FALSE otherwise. * * Side effects: - * + * None * *----------------------------------------------------------------------------- */ + static INLINE Bool -VersionedAtomic_EndTryRead(const VersionedAtomic *versions, - uint32 readVersion) +VersionedAtomic_EndTryRead(VersionedAtomic const *versions, // IN + uint32 readVersion) // IN { - COMPILER_MEM_BARRIER(); - return LIKELY(versions->v0 == readVersion); + /* + * The read(s) from the shared data must happen on this CPU before the read + * from 'versions->v0'. + * + * On arm64, this barrier is Full System, which is probably too strong: + * Inner Shareable (== all OS CPUs) should work just fine, and maybe + * Non-shareable (== current CPU) would work too. + */ + SMP_R_BARRIER_R(); + + /* + * There is a theoretical ABA issue here: if the writer updates the shared + * data exactly 2^32 times while a reader reads it, the read will + * incorrectly be considered consistent. In practice, this issue is + * unlikely, so we ignore it. But should we need it, we could make the issue + * even less likely by using 64-bit version variables. + */ + return LIKELY( Atomic_Read32((Atomic_uint32 *)&versions->v0) + == readVersion); } + +#if defined __cplusplus +} // extern "C" +#endif + #endif //_VERSIONED_ATOMIC_H diff --git a/vmmon-only/include/vm_asm_x86.h b/vmmon-only/include/vm_asm_x86.h index 2b37976..364a420 100644 --- a/vmmon-only/include/vm_asm_x86.h +++ b/vmmon-only/include/vm_asm_x86.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2014 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2014,2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -435,16 +435,13 @@ CLTS(void) __asm__ __volatile__("fs; invlpg %0": :"m" (*(char *) (_addr)):"memory"); \ } while (0) -#if ! defined(VMKBOOT) #define RESTORE_FLAGS _Set_flags #define ENABLE_INTERRUPTS() __asm__ __volatile__ ("sti": : :"memory") #define CLEAR_INTERRUPTS() __asm__ __volatile__ ("cli": : :"memory") -#endif - #define RAISE_INTERRUPT(_x) __asm__ __volatile__("int %0" :: "g" (_x)) #define RETURN_FROM_INT() __asm__ __volatile__("iret" :: ) -#if ! defined(VMKERNEL) && ! defined(VMKBOOT) +#if ! defined(VMKERNEL) #define NO_INTERRUPTS_BEGIN() do { \ uintptr_t _flags; \ SAVE_FLAGS(_flags); \ @@ -958,10 +955,12 @@ static INLINE uint64 GET_XCR0(void) static INLINE void SET_XCR0_IF_NEEDED(uint64 newVal, uint64 oldVal) { +#ifndef VMM_BOOTSTRAP ASSERT(oldVal == GET_XCR0()); if (newVal != oldVal) { SET_XCR0(newVal); } +#endif } #endif diff --git a/vmmon-only/include/vm_asm_x86_64.h b/vmmon-only/include/vm_asm_x86_64.h index 902cd81..db2131c 100644 --- a/vmmon-only/include/vm_asm_x86_64.h +++ b/vmmon-only/include/vm_asm_x86_64.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2014 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -32,6 +32,7 @@ #define INCLUDE_ALLOW_VMMON #define INCLUDE_ALLOW_VMK_MODULE #define INCLUDE_ALLOW_VMKERNEL +#define INCLUDE_ALLOW_USERLEVEL #define INCLUDE_ALLOW_DISTRIBUTE #include "includeCheck.h" diff --git a/vmmon-only/include/vm_assert.h b/vmmon-only/include/vm_assert.h index b368e1d..8cdbc93 100644 --- a/vmmon-only/include/vm_assert.h +++ b/vmmon-only/include/vm_assert.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -99,24 +99,13 @@ void Warning(const char *fmt, ...) PRINTF_DECL(1, 2); #if defined VMKPANIC void Panic_SaveRegs(void); -#ifdef VMX86_DEBUG -void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2); -#else NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2); -#endif - -NORETURN void Panic_NoSaveNoReturn(const char *fmt, ...) PRINTF_DECL(1, 2); #define Panic(fmt...) do { \ Panic_SaveRegs(); \ Panic_NoSave(fmt); \ } while(0) -#define Panic_NoReturn(fmt...) do { \ - Panic_SaveRegs(); \ - Panic_NoSaveNoReturn(fmt); \ -} while(0) - #else NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2); #endif @@ -176,8 +165,6 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3); #define ASSERT_NOT_IMPLEMENTED(cond) \ ASSERT_IFNOT(cond, NOT_IMPLEMENTED()) -#define ASSERT_NOT_IMPLEMENTED_BUG(bug, cond) \ - ASSERT_IFNOT(cond, NOT_IMPLEMENTED_BUG(bug)) #if defined VMKPANIC || defined VMM #define NOT_IMPLEMENTED() _ASSERT_PANIC_NORETURN(AssertNotImplemented) @@ -202,10 +189,8 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3); ASSERT_IFNOT(cond, _ASSERT_PANIC(AssertMemAlloc)) #ifdef VMX86_DEVEL -#define ASSERT_DEVEL(cond) ASSERT(cond) #define NOT_TESTED() Warning(_AssertNotTestedFmt "\n", __FILE__, __LINE__) #else -#define ASSERT_DEVEL(cond) ((void)0) #define NOT_TESTED() Log(_AssertNotTestedFmt "\n", __FILE__, __LINE__) #endif @@ -305,7 +290,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3); #define ASSERT_ON_COMPILE(e) \ do { \ _Static_assert(e, #e); \ - } while (0); + } while (0) #endif /* diff --git a/vmmon-only/include/vm_atomic.h b/vmmon-only/include/vm_atomic.h index 07f32ea..38d45fd 100644 --- a/vmmon-only/include/vm_atomic.h +++ b/vmmon-only/include/vm_atomic.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -32,7 +32,6 @@ #define _ATOMIC_H_ #define INCLUDE_ALLOW_USERLEVEL - #define INCLUDE_ALLOW_MODULE #define INCLUDE_ALLOW_VMMON #define INCLUDE_ALLOW_VMKDRIVERS @@ -45,6 +44,10 @@ #include "vm_basic_types.h" #include "vm_assert.h" +#if defined __cplusplus +extern "C" { +#endif + /* * In the Atomic_* definitions below, memory ordering and atomicity are somewhat * conflated in an inconsistent manner. First, we have Atomic_{Read,Write}, @@ -84,153 +87,15 @@ */ #ifdef VM_ARM_64 -#include "vm_basic_asm_arm64.h" - -#define ARM_64_ATOMIC_16_OPV_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint16 _sample; \ - __asm__ __volatile__ ( \ - "1: ldxrh %w0, [%2] \n\t" \ - #op " %w0, %w0, %w3 \n\t" \ - " stxrh %w1, %w0, [%2] \n\t" \ - " cbnz %w1, 1b \n\t" \ - : "=&r" (_sample), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ -}) - -#define ARM_64_ATOMIC_16_OPO_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint16 _newval; \ - uint16 _oldval; \ - __asm__ __volatile__ ( \ - "1: ldxrh %w0, [%3] \n\t" \ - #op " %w1, %w0, %w4 \n\t" \ - " stxrh %w2, %w1, [%3] \n\t" \ - " cbnz %w2, 1b \n\t" \ - : "=&r" (_oldval), \ - "=&r" (_newval), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ - _oldval; \ -}) - -#define ARM_64_ATOMIC_32_OPV_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint32 _sample; \ - __asm__ __volatile__ ( \ - "1: ldxr %w0, [%2] \n\t" \ - #op " %w0, %w0, %w3 \n\t" \ - " stxr %w1, %w0, [%2] \n\t" \ - " cbnz %w1, 1b \n\t" \ - : "=&r" (_sample), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ -}) - -#define ARM_64_ATOMIC_32_OPO_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint32 _newval; \ - uint32 _oldval; \ - __asm__ __volatile__ ( \ - "1: ldxr %w0, [%3] \n\t" \ - #op " %w1, %w0, %w4 \n\t" \ - " stxr %w2, %w1, [%3] \n\t" \ - " cbnz %w2, 1b \n\t" \ - : "=&r" (_oldval), \ - "=&r" (_newval), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ - _oldval; \ -}) - -#define ARM_64_ATOMIC_64_OPV_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint64 _sample; \ - __asm__ __volatile__ ( \ - "1: ldxr %x0, [%2] \n\t" \ - #op " %x0, %x0, %x3 \n\t" \ - " stxr %w1, %x0, [%2] \n\t" \ - " cbnz %w1, 1b \n\t" \ - : "=&r" (_sample), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ -}) - -#define ARM_64_ATOMIC_64_OPO_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint64 _newval; \ - uint64 _oldval; \ - __asm__ __volatile__ ( \ - "1: ldxr %x0, [%3] \n\t" \ - #op " %x1, %x0, %x4 \n\t" \ - " stxr %w2, %x1, [%3] \n\t" \ - " cbnz %w2, 1b \n\t" \ - : "=&r" (_oldval), \ - "=&r" (_newval), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ - _oldval; \ -}) - -#define ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(atm,modval,op) ({ \ - DMB(); \ - ARM_64_ATOMIC_16_OPV_PRIVATE(atm,modval,op); \ - DMB(); \ -}) - -#define ARM_64_ATOMIC_16_OPO_PRIVATE_FENCED(atm,modval,op) ({ \ - uint16 _oldval_fenced; \ - DMB(); \ - _oldval_fenced = ARM_64_ATOMIC_16_OPO_PRIVATE(atm,modval,op); \ - DMB(); \ - _oldval_fenced; \ -}) - -#define ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(atm,modval,op) ({ \ - DMB(); \ - ARM_64_ATOMIC_32_OPV_PRIVATE(atm,modval,op); \ - DMB(); \ -}) - -#define ARM_64_ATOMIC_32_OPO_PRIVATE_FENCED(atm,modval,op) ({ \ - uint32 _oldval_fenced; \ - DMB(); \ - _oldval_fenced = ARM_64_ATOMIC_32_OPO_PRIVATE(atm,modval,op); \ - DMB(); \ - _oldval_fenced; \ -}) - -#define ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(atm,modval,op) ({ \ - DMB(); \ - ARM_64_ATOMIC_64_OPV_PRIVATE(atm,modval,op); \ - DMB(); \ -}) - -#define ARM_64_ATOMIC_64_OPO_PRIVATE_FENCED(atm,modval,op) ({ \ - uint64 _oldval_fenced; \ - DMB(); \ - _oldval_fenced = ARM_64_ATOMIC_64_OPO_PRIVATE(atm,modval,op); \ - DMB(); \ - _oldval_fenced; \ -}) - -#endif /* VM_ARM_64 */ +# include "vm_atomic_arm64_begin.h" +#endif -/* Basic atomic types: 16, 32 and 64 bits */ +/* Basic atomic types: 8, 16, 32, 64 and 128 bits */ +typedef struct Atomic_uint8 { + volatile uint8 value; +} Atomic_uint8 ALIGNED(1); + typedef struct Atomic_uint16 { volatile uint16 value; } Atomic_uint16 ALIGNED(2); @@ -239,10 +104,17 @@ typedef struct Atomic_uint32 { volatile uint32 value; } Atomic_uint32 ALIGNED(4); -typedef struct Atomic_uint64 { +typedef struct Atomic_uint64 { volatile uint64 value; } Atomic_uint64 ALIGNED(8); +#if defined __GNUC__ && defined VM_64BIT && \ + (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64) +typedef struct Atomic_uint128 { + volatile uint128 value; +} Atomic_uint128 ALIGNED(16); +#endif + /* * Prototypes for msft atomics. These are defined & inlined by the * compiler so no function definition is needed. The prototypes are @@ -258,7 +130,7 @@ typedef struct Atomic_uint64 { * Windows typedefs. This avoids having to include windows header files * to get to the windows types. */ -#if defined(_MSC_VER) && _MSC_VER >= 1310 && !defined(BORA_NO_WIN32_INTRINS) +#if defined _MSC_VER && _MSC_VER >= 1310 && !defined BORA_NO_WIN32_INTRINS #ifdef __cplusplus extern "C" { #endif @@ -267,10 +139,11 @@ long _InterlockedCompareExchange(long volatile*, long, long); long _InterlockedExchangeAdd(long volatile*, long); long _InterlockedDecrement(long volatile*); long _InterlockedIncrement(long volatile*); -void _ReadWriteBarrier(void); +__int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); #pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange) #pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement) -#pragma intrinsic(_InterlockedIncrement, _ReadWriteBarrier) +#pragma intrinsic(_InterlockedIncrement) +#pragma intrinsic(_InterlockedCompareExchange64) # if _MSC_VER >= 1600 char _InterlockedExchange8(char volatile *, char); @@ -278,7 +151,7 @@ char _InterlockedCompareExchange8(char volatile *, char, char); #pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8) #endif -#if defined(VM_X86_64) +#if defined VM_X86_64 long _InterlockedAnd(long volatile*, long); __int64 _InterlockedAnd64(__int64 volatile*, __int64); long _InterlockedOr(long volatile*, long); @@ -289,14 +162,12 @@ __int64 _InterlockedExchangeAdd64(__int64 volatile*, __int64); __int64 _InterlockedIncrement64(__int64 volatile*); __int64 _InterlockedDecrement64(__int64 volatile*); __int64 _InterlockedExchange64(__int64 volatile*, __int64); -__int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); -#if !defined(_WIN64) +#if !defined _WIN64 #pragma intrinsic(_InterlockedAnd, _InterlockedAnd64) #pragma intrinsic(_InterlockedOr, _InterlockedOr64) #pragma intrinsic(_InterlockedXor, _InterlockedXor64) #pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64) #pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64) -#pragma intrinsic(_InterlockedCompareExchange64) #endif /* !_WIN64 */ #endif /* __x86_64__ */ @@ -305,7 +176,7 @@ __int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); #endif #endif /* _MSC_VER */ -#if defined(__arm__) +#if defined __arm__ /* * LDREX without STREX or CLREX may cause problems in environments where the * context switch may not clear the reference monitor - according ARM manual @@ -313,8 +184,8 @@ __int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); * may not like Linux kernel's non-preemptive context switch path. So use of * ARM routines in kernel code may not be safe. */ -# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ - defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__) +# if defined __ARM_ARCH_7__ || defined __ARM_ARCH_7A__ || \ + defined __ARM_ARCH_7R__|| defined __ARM_ARCH_7M__ # define VM_ARM_V7 # ifdef __KERNEL__ # warning LDREX/STREX may not be safe in linux kernel, since it \ @@ -335,10 +206,12 @@ __int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); /* Convert a volatile uint32 to Atomic_uint32. */ static INLINE Atomic_uint32 * -Atomic_VolatileToAtomic(volatile uint32 *var) // IN: +Atomic_VolatileToAtomic32(volatile uint32 *var) // IN: { return (Atomic_uint32 *)var; } +#define Atomic_VolatileToAtomic Atomic_VolatileToAtomic32 + /* Convert a volatile uint64 to Atomic_uint64. */ static INLINE Atomic_uint64 * @@ -347,135 +220,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var) // IN: return (Atomic_uint64 *)var; } -/* - *----------------------------------------------------------------------------- - * - * Atomic_Init, Atomic_SetFence, AtomicUseFence -- - * - * Determine whether an lfence intruction is executed after - * every locked instruction. - * - * Certain AMD processors have a bug (see bug 107024) that - * requires an lfence after every locked instruction. - * - * The global variable AtomicUseFence controls whether lfence - * is used (see AtomicEpilogue). - * - * Atomic_SetFence sets AtomicUseFence to the given value. - * - * Atomic_Init computes and sets AtomicUseFence for x86. - * It does not take into account the number of processors. - * - * The rationale for all this complexity is that Atomic_Init - * is the easy-to-use interface. It can be called a number - * of times cheaply, and does not depend on other libraries. - * However, because the number of CPUs is difficult to compute, - * it does without it and always assumes there are more than one. - * - * For programs that care or have special requirements, - * Atomic_SetFence can be called directly, in addition to Atomic_Init. - * It overrides the effect of Atomic_Init, and can be called - * before, after, or between calls to Atomic_Init. - * - *----------------------------------------------------------------------------- - */ - -// The freebsd assembler doesn't know the lfence instruction -#if defined(__GNUC__) && \ - __GNUC__ >= 3 && \ - (defined(__VMKERNEL__) || !defined(__FreeBSD__)) && \ - (!defined(MODULE) || defined(__VMKERNEL_MODULE__)) && \ - !defined(__APPLE__) && \ - (defined(__i386__) || defined(__x86_64__)) /* PR136775 */ -#define ATOMIC_USE_FENCE -#endif - -/* - * Starting with vSphere 2014, we no longer support ESX on AMD Rev F. - * Thus, we can eliminate all dynamic checks for whether to enable - * the Errata 147 work-around when compiling many of our binaries. - * However, we use an opt-in approach here rather than assuming all - * parts of our builds are safe. For example, the "fdm" binary from - * a new build may time travel back to hosts running older versions - * of ESX on Rev F, so "fdm" continues to require the ability to - * dynamically enable the errata work-around. With vSphere 2017, - * this will no longer be required as the oldest version of ESX that - * VC 2017 will support is ESX 2014 (which won't run on Rev F). - * - * Modules may explicitly define MAY_NEED_AMD_REVF_WORKAROUND as 0 prior to - * inclusion of vm_atomic.h when they are safe on AMD Rev F with the elided - * lfence. - */ -#if !defined(MAY_NEED_AMD_REVF_WORKAROUND) -#if (!defined(VMX86_SERVER) || \ - (!defined(VMX86_VMX) && !defined(VMKERNEL) && \ - !defined(VMM) && !defined(VMCORE))) -#define MAY_NEED_AMD_REVF_WORKAROUND 1 -#else -#define MAY_NEED_AMD_REVF_WORKAROUND 0 -#endif -#endif - -#if MAY_NEED_AMD_REVF_WORKAROUND -#if defined(VMATOMIC_IMPORT_DLLDATA) -VMX86_EXTERN_DATA Bool AtomicUseFence; -#else -EXTERN Bool AtomicUseFence; -#endif -EXTERN Bool atomicFenceInitialized; -#else /* MAY_NEED_AMD_REVF_WORKAROUND */ -#define AtomicUseFence FALSE -#define atomicFenceInitialized TRUE -#endif /* MAY_NEED_AMD_REVF_WORKAROUND */ - - -void AtomicInitFence(void); - -static INLINE void -Atomic_Init(void) -{ -#ifdef ATOMIC_USE_FENCE - if (MAY_NEED_AMD_REVF_WORKAROUND && !atomicFenceInitialized) { - AtomicInitFence(); - } -#endif -} - -static INLINE void -Atomic_SetFence(Bool fenceAfterLock) // IN: -{ - (void)fenceAfterLock; /* Work around unused parameter. */ -#if MAY_NEED_AMD_REVF_WORKAROUND - AtomicUseFence = fenceAfterLock; - atomicFenceInitialized = TRUE; -#endif -} - - -/* Conditionally execute fence after interlocked instruction. */ -static INLINE void -AtomicEpilogue(void) -{ -#if MAY_NEED_AMD_REVF_WORKAROUND && defined(ATOMIC_USE_FENCE) -#ifdef VMM - /* The monitor conditionally patches out the lfence when not needed.*/ - /* Construct a MonitorPatchTextEntry in the .patchtext section. */ - asm volatile ("1:\n\t" - "lfence\n\t" - "2:\n\t" - ".pushsection .patchtext\n\t" - ".quad 1b\n\t" - ".quad 2b\n\t" - ".quad 0\n\t" - ".popsection\n\t" ::: "memory"); -#else - if (UNLIKELY(AtomicUseFence)) { - asm volatile ("lfence" ::: "memory"); - } -#endif -#endif -} - /* * All the assembly code is tricky and written conservatively. @@ -512,26 +256,86 @@ AtomicEpilogue(void) * - walken */ -#if defined(_MSC_VER) && _MSC_VER < 1600 && defined(__x86_64__) -Bool VMWInterlockedExchangeBool(Bool volatile *ptr, - Bool val); +#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__ +uint8 VMWInterlockedExchange8(uint8 volatile *ptr, + uint8 val); -Bool VMWInterlockedCompareExchangeBool(Bool volatile *ptr, - Bool newVal, - Bool oldVal); +uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr, + uint8 newVal, + uint8 oldVal); #endif -typedef struct Atomic_Bool { - volatile Bool value; -} Atomic_Bool; +#if defined __GNUC__ && defined VM_ARM_32 +/* Force the link step to fail for unimplemented functions. */ +extern int AtomicUndefined(void const *); +#endif -/* The ARM team can come along and add the code real soon now */ -extern Bool AtomicUndefinedOnARM(void); /* *----------------------------------------------------------------------------- * - * Atomic_ReadBool -- + * Atomic_ReadIfEqualWrite128 -- + * + * Compare and exchange a 16 byte tuple. + * + * Results: + * old value + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ +#if defined __GNUC__ && defined VM_64BIT && \ + (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64) +static INLINE uint128 +Atomic_ReadIfEqualWrite128(Atomic_uint128 *ptr, // IN/OUT + uint128 oldVal, // IN + uint128 newVal) // IN +{ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 + return __sync_val_compare_and_swap(&ptr->value, oldVal, newVal); +#elif defined VM_ARM_64 + union { + uint128 raw; + struct { + uint64 lo; + uint64 hi; + }; + } res, _old = { oldVal }, _new = { newVal }; + uint32 failed; + + SMP_RW_BARRIER_RW(); + __asm__ __volatile__( + "1: ldxp %x0, %x1, %3 \n\t" + " cmp %x0, %x4 \n\t" + " ccmp %x1, %x5, #0, eq \n\t" + " b.ne 2f \n\t" + " stxp %w2, %x6, %x7, %3 \n\t" + " cbnz %w2, 1b \n\t" + "2: \n\t" + : "=&r" (res.lo), + "=&r" (res.hi), + "=&r" (failed), + "+Q" (ptr->value) + : "r" (_old.lo), + "r" (_old.hi), + "r" (_new.lo), + "r" (_new.hi) + : "cc" + ); + SMP_RW_BARRIER_RW(); + + return res.raw; +#endif +} +#endif + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Read8 -- * * Read the value of the specified object atomically. * @@ -544,23 +348,25 @@ extern Bool AtomicUndefinedOnARM(void); *----------------------------------------------------------------------------- */ -static INLINE Bool -Atomic_ReadBool(Atomic_Bool const *var) // IN: +static INLINE uint8 +Atomic_Read8(Atomic_uint8 const *var) // IN: { - Bool val; + uint8 val; -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - val = AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#if defined __GNUC__ && defined VM_ARM_32 + val = AtomicUndefined(var); +#elif defined VM_ARM_64 + val = _VMATOM_X(R, 8, &var->value); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) __asm__ __volatile__( "movb %1, %0" : "=q" (val) : "m" (var->value) ); -#elif defined(_MSC_VER) +#elif defined _MSC_VER val = var->value; #else -#error No compiler defined for Atomic_ReadBool +#error No compiler defined for Atomic_Read8 #endif return val; @@ -570,7 +376,7 @@ Atomic_ReadBool(Atomic_Bool const *var) // IN: /* *----------------------------------------------------------------------------- * - * Atomic_ReadWriteBool -- + * Atomic_ReadWrite8 -- * * Read followed by write. * @@ -583,36 +389,37 @@ Atomic_ReadBool(Atomic_Bool const *var) // IN: *----------------------------------------------------------------------------- */ -static INLINE Bool -Atomic_ReadWriteBool(Atomic_Bool *var, // IN/OUT: - Bool val) // IN: +static INLINE uint8 +Atomic_ReadWrite8(Atomic_uint8 *var, // IN/OUT: + uint8 val) // IN: { -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - return AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#if defined __GNUC__ && defined VM_ARM_32 + return AtomicUndefined(var + val); +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 8, TRUE, &var->value, val); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) __asm__ __volatile__( "xchgb %0, %1" : "=q" (val), "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; -#elif defined(_MSC_VER) && _MSC_VER >= 1600 - return _InterlockedExchange8(&var->value, val); -#elif defined(_MSC_VER) && defined(__i386__) +#elif defined _MSC_VER && _MSC_VER >= 1600 + return _InterlockedExchange8((volatile char *)&var->value, val); +#elif defined _MSC_VER && defined __i386__ #pragma warning(push) #pragma warning(disable : 4035) // disable no-return warning { __asm movzx eax, val __asm mov ebx, var - __asm xchg [ebx]Atomic_Bool.value, al + __asm xchg [ebx]Atomic_uint8.value, al } #pragma warning(pop) -#elif defined(_MSC_VER) && defined(__x86_64__) - return VMWInterlockedExchangeBool(&var->value, val); +#elif defined _MSC_VER && defined __x86_64__ + return VMWInterlockedExchange8(&var->value, val); #else -#error No compiler defined for Atomic_ReadBool +#error No compiler defined for Atomic_ReadWrite8 #endif } @@ -620,7 +427,7 @@ Atomic_ReadWriteBool(Atomic_Bool *var, // IN/OUT: /* *----------------------------------------------------------------------------- * - * Atomic_WriteBool -- + * Atomic_Write8 -- * * Write the specified value to the specified object atomically. * @@ -634,21 +441,23 @@ Atomic_ReadWriteBool(Atomic_Bool *var, // IN/OUT: */ static INLINE void -Atomic_WriteBool(Atomic_Bool *var, // IN/OUT: - Bool val) // IN: +Atomic_Write8(Atomic_uint8 *var, // IN/OUT: + uint8 val) // IN: { -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#if defined __GNUC__ && defined VM_ARM_32 + AtomicUndefined(var + val); +#elif defined VM_ARM_64 + _VMATOM_X(W, 8, &var->value, val); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) __asm__ __volatile__( "movb %1, %0" : "=m" (var->value) : "qn" (val) ); -#elif defined(_MSC_VER) +#elif defined _MSC_VER var->value = val; #else -#error No compiler defined for Atomic_WriteBool +#error No compiler defined for Atomic_Write8 #endif } @@ -656,7 +465,7 @@ Atomic_WriteBool(Atomic_Bool *var, // IN/OUT: /* *----------------------------------------------------------------------------- * - * Atomic_ReadIfEqualWriteBool -- + * Atomic_ReadIfEqualWrite8 -- * * Compare exchange: Read variable, if equal to oldVal, write newVal. * @@ -669,15 +478,17 @@ Atomic_WriteBool(Atomic_Bool *var, // IN/OUT: *----------------------------------------------------------------------------- */ -static INLINE Bool -Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: - Bool oldVal, // IN: - Bool newVal) // IN: +static INLINE uint8 +Atomic_ReadIfEqualWrite8(Atomic_uint8 *var, // IN/OUT: + uint8 oldVal, // IN: + uint8 newVal) // IN: { -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - return AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - Bool val; +#if defined __GNUC__ && defined VM_ARM_32 + return AtomicUndefined(var + oldVal + newVal); +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 8, TRUE, &var->value, oldVal, newVal); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) + uint8 val; __asm__ __volatile__( "lock; cmpxchgb %2, %1" @@ -687,26 +498,27 @@ Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: "0" (oldVal) : "cc" ); - AtomicEpilogue(); + return val; -#elif defined(_MSC_VER) && _MSC_VER >= 1600 - return _InterlockedCompareExchange8(&var->value, newVal, oldVal); -#elif defined(_MSC_VER) && defined(__i386__) +#elif defined _MSC_VER && _MSC_VER >= 1600 + return _InterlockedCompareExchange8((volatile char *)&var->value, + newVal, oldVal); +#elif defined _MSC_VER && defined __i386__ #pragma warning(push) #pragma warning(disable : 4035) // disable no-return warning { __asm mov al, oldVal __asm mov ebx, var __asm mov cl, newVal - __asm lock cmpxchg [ebx]Atomic_Bool.value, cl + __asm lock cmpxchg [ebx]Atomic_uint8.value, cl __asm movzx eax, al // eax is the return value, this is documented to work - edward } #pragma warning(pop) -#elif defined(_MSC_VER) && defined(__x86_64__) - return VMWInterlockedCompareExchangeBool(&var->value, newVal, oldVal); +#elif defined _MSC_VER && defined __x86_64__ + return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal); #else -#error No compiler defined for Atomic_ReadIfEqualWriteBool +#error No compiler defined for Atomic_ReadIfEqualWrite8 #endif } @@ -714,7 +526,375 @@ Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: /* *----------------------------------------------------------------------------- * - * Atomic_Read -- + * Atomic_ReadAnd8 -- + * + * Atomic read (returned), bitwise AND with a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadAnd8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, and, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res & val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_And8 -- + * + * Atomic read, bitwise AND with a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_And8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, and, val); +#else + (void)Atomic_ReadAnd8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadOr8 -- + * + * Atomic read (returned), bitwise OR with a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadOr8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, orr, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res | val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Or8 -- + * + * Atomic read, bitwise OR with a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Or8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, orr, val); +#else + (void)Atomic_ReadOr8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadXor8 -- + * + * Atomic read (returned), bitwise XOR with a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadXor8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, eor, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res ^ val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Xor8 -- + * + * Atomic read, bitwise XOR with a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Xor8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, eor, val); +#else + (void)Atomic_ReadXor8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadAdd8 -- + * + * Atomic read (returned), add a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadAdd8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, add, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res + val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Add8 -- + * + * Atomic read, add a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Add8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, add, val); +#else + (void)Atomic_ReadAdd8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Sub8 -- + * + * Atomic read, subtract a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Sub8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, sub, val); +#else + Atomic_Add8(var, -val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Inc8 -- + * + * Atomic read, increment, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Inc8(Atomic_uint8 *var) // IN/OUT +{ + Atomic_Add8(var, 1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Dec8 -- + * + * Atomic read, decrement, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Dec8(Atomic_uint8 *var) // IN/OUT +{ + Atomic_Sub8(var, 1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadInc8 -- + * + * Atomic read (returned), increment, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadInc8(Atomic_uint8 *var) // IN/OUT +{ + return Atomic_ReadAdd8(var, 1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadDec8 -- + * + * Atomic read (returned), decrement, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadDec8(Atomic_uint8 *var) // IN/OUT +{ + return Atomic_ReadAdd8(var, (uint8)-1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Read32 -- * * Read * @@ -728,33 +908,29 @@ Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: */ static INLINE uint32 -Atomic_Read(Atomic_uint32 const *var) // IN +Atomic_Read32(Atomic_uint32 const *var) // IN { uint32 value; -#if defined(VMM) +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT(((uintptr_t)var % 4) == 0); #endif -#if defined(__GNUC__) +#if defined __GNUC__ /* * Use inline assembler to force using a single load instruction to * ensure that the compiler doesn't split a transfer operation into multiple * instructions. */ -#if defined(VM_ARM_32) +#if defined VM_ARM_32 __asm__ __volatile__( "ldr %0, [%1]" : "=r" (value) : "r" (&var->value) ); -#elif defined(VM_ARM_64) - __asm__ __volatile__ ( - "ldr %w0, [%1]" - : "=r" (value) - : "r" (&var->value) - ); +#elif defined VM_ARM_64 + value = _VMATOM_X(R, 32, &var->value); #else __asm__ __volatile__( "mov %1, %0" @@ -762,7 +938,7 @@ Atomic_Read(Atomic_uint32 const *var) // IN : "m" (var->value) ); #endif -#elif defined(_MSC_VER) +#elif defined _MSC_VER /* * Microsoft docs guarantee simple reads and writes to properly * aligned 32-bit variables use only a single instruction. @@ -776,13 +952,13 @@ Atomic_Read(Atomic_uint32 const *var) // IN return value; } -#define Atomic_Read32 Atomic_Read +#define Atomic_Read Atomic_Read32 /* *----------------------------------------------------------------------------- * - * Atomic_ReadWrite -- + * Atomic_ReadWrite32 -- * * Read followed by write * @@ -796,10 +972,10 @@ Atomic_Read(Atomic_uint32 const *var) // IN */ static INLINE uint32 -Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_ReadWrite32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 retVal; register volatile uint32 res; @@ -819,22 +995,8 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint32 retVal; - register uint32 failed; - - __asm__ __volatile__( - " dmb sy \n\t" - "1: ldxr %w0, [%2] \n\t" - " stxr %w1, %w3, [%2] \n\t" - " cbnz %w1, 1b \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (val) - : "memory" - ); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 32, TRUE, &var->value, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -843,7 +1005,6 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; #endif /* VM_X86_ANY */ #elif defined _MSC_VER @@ -864,13 +1025,13 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_ReadWrite #endif // __GNUC__ } -#define Atomic_ReadWrite32 Atomic_ReadWrite +#define Atomic_ReadWrite Atomic_ReadWrite32 /* *----------------------------------------------------------------------------- * - * Atomic_Write -- + * Atomic_Write32 -- * * Write * @@ -884,16 +1045,21 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Write(Atomic_uint32 *var, // OUT - uint32 val) // IN +Atomic_Write32(Atomic_uint32 *var, // OUT + uint32 val) // IN { -#if defined(VMM) +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT(((uintptr_t)var % 4) == 0); #endif -#if defined(__GNUC__) -#if defined(VM_ARM_ANY) +#if defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(W, 32, &var->value, val); +#elif defined VM_ARM_32 /* + * Best left this way due to the intricacies of exclusive load/store + * operations on legacy (32-bit) ARM. + * * A3.4.1 ARM DDI 0406C: * * When a processor writes using any instruction other than a @@ -915,7 +1081,7 @@ Atomic_Write(Atomic_uint32 *var, // OUT * - it is not necessary for every LDREX to have a subsequent STREX. */ - Atomic_ReadWrite(var, val); + Atomic_ReadWrite32(var, val); #else /* * Use inline assembler to force using a single store instruction to @@ -929,7 +1095,7 @@ Atomic_Write(Atomic_uint32 *var, // OUT : "r" (val) ); #endif -#elif defined(_MSC_VER) +#elif defined _MSC_VER /* * Microsoft docs guarantee simple reads and writes to properly * aligned 32-bit variables use only a single instruction. @@ -941,13 +1107,13 @@ Atomic_Write(Atomic_uint32 *var, // OUT #error No compiler defined for Atomic_Write #endif } -#define Atomic_Write32 Atomic_Write +#define Atomic_Write Atomic_Write32 /* *----------------------------------------------------------------------------- * - * Atomic_ReadIfEqualWrite -- + * Atomic_ReadIfEqualWrite32 -- * * Compare exchange: Read variable, if equal to oldVal, write newVal * @@ -961,11 +1127,11 @@ Atomic_Write(Atomic_uint32 *var, // OUT */ static INLINE uint32 -Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT - uint32 oldVal, // IN - uint32 newVal) // IN +Atomic_ReadIfEqualWrite32(Atomic_uint32 *var, // IN/OUT + uint32 oldVal, // IN + uint32 newVal) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register uint32 retVal; register uint32 res; @@ -987,24 +1153,8 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint32 retVal; - register uint32 failed; - - __asm__ __volatile__ ( - " dmb sy \n\t" - "1: ldxr %w0, [%2] \n\t" - " cmp %w0, %w3 \n\t" - " b.ne 2f \n\t" - " stxr %w1, %w4, [%2] \n\t" - " cbnz %w1, 1b \n\t" - "2: clrex \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (oldVal), "r" (newVal) - : "cc", "memory"); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 32, TRUE, &var->value, oldVal, newVal); #else /* VM_X86_ANY */ uint32 val; @@ -1017,7 +1167,6 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT "0" (oldVal) : "cc" ); - AtomicEpilogue(); return val; #endif /* VM_X86_ANY */ #elif defined _MSC_VER @@ -1041,10 +1190,10 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_ReadIfEqualWrite #endif } -#define Atomic_ReadIfEqualWrite32 Atomic_ReadIfEqualWrite +#define Atomic_ReadIfEqualWrite Atomic_ReadIfEqualWrite32 -#if defined(VM_64BIT) || defined(VM_ARM_V7) +#if defined VM_64BIT || defined VM_ARM_V7 /* *----------------------------------------------------------------------------- * @@ -1066,7 +1215,7 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT uint64 oldVal, // IN uint64 newVal) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register uint64 retVal; register uint32 res; @@ -1087,7 +1236,7 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT * Furthermore, using a 32-bits register to store a * 64-bits value of an variable looks risky. */ -#if defined(__APPLE__) && __clang__ == 1 && __clang_major__ >= 5 +#if defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5 #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wasm-operand-widths" #endif @@ -1103,30 +1252,14 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal) : "cc" ); -#if defined(__APPLE__) && __clang__ == 1 && __clang_major__ >= 5 +#if defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5 #pragma clang diagnostic pop -#endif // defined(__APPLE__) && __clang__ == 1 && __clang_major__ >= 5 +#endif // defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5 dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint64 retVal; - register uint32 failed; - - __asm__ __volatile__ ( - " dmb sy \n\t" - "1: ldxr %x0, [%2] \n\t" - " cmp %x0, %x3 \n\t" - " b.ne 2f \n\t" - " stxr %w1, %x4, [%2] \n\t" - " cbnz %w1, 1b \n\t" - "2: clrex \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (oldVal), "r" (newVal) - : "cc", "memory"); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 64, TRUE, &var->value, oldVal, newVal); #else /* VM_X86_64 */ uint64 val; @@ -1139,7 +1272,6 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT "0" (oldVal) : "cc" ); - AtomicEpilogue(); return val; #endif //VM_ARM_V7 #elif defined _MSC_VER @@ -1156,7 +1288,7 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT /* *----------------------------------------------------------------------------- * - * Atomic_And -- + * Atomic_And32 -- * * Atomic read, bitwise AND with a value, write. * @@ -1170,10 +1302,10 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT */ static INLINE void -Atomic_And(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_And32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1192,8 +1324,8 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, and); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, and, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1202,10 +1334,9 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER -#if defined(__x86_64__) +#if defined __x86_64__ || defined VM_ARM_32 _InterlockedAnd((long *)&var->value, (long)val); #else __asm mov eax, val @@ -1216,13 +1347,13 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_And #endif } -#define Atomic_And32 Atomic_And +#define Atomic_And Atomic_And32 /* *----------------------------------------------------------------------------- * - * Atomic_Or -- + * Atomic_Or32 -- * * Atomic read, bitwise OR with a value, write. * @@ -1236,10 +1367,10 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Or(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Or32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1258,8 +1389,8 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, orr); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, orr, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1268,10 +1399,9 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER -#if defined(__x86_64__) +#if defined __x86_64__ || defined VM_ARM_32 _InterlockedOr((long *)&var->value, (long)val); #else __asm mov eax, val @@ -1282,13 +1412,13 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Or #endif } -#define Atomic_Or32 Atomic_Or +#define Atomic_Or Atomic_Or32 /* *----------------------------------------------------------------------------- * - * Atomic_Xor -- + * Atomic_Xor32 -- * * Atomic read, bitwise XOR with a value, write. * @@ -1302,10 +1432,10 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Xor(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Xor32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1324,8 +1454,8 @@ Atomic_Xor(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, eor); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, eor, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1334,10 +1464,9 @@ Atomic_Xor(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER -#if defined(__x86_64__) +#if defined __x86_64__ || defined VM_ARM_32 _InterlockedXor((long *)&var->value, (long)val); #else __asm mov eax, val @@ -1348,10 +1477,10 @@ Atomic_Xor(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Xor #endif } -#define Atomic_Xor32 Atomic_Xor +#define Atomic_Xor Atomic_Xor32 -#if defined(VM_64BIT) +#if defined VM_64BIT /* *----------------------------------------------------------------------------- * @@ -1372,9 +1501,9 @@ static INLINE void Atomic_Xor64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__GNUC__) -#if defined(VM_ARM_64) - ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, eor); +#if defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, eor, val); #else /* VM_X86_64 */ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( @@ -1383,7 +1512,6 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #endif #elif defined _MSC_VER _InterlockedXor64((__int64 *)&var->value, (__int64)val); @@ -1397,7 +1525,7 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT /* *----------------------------------------------------------------------------- * - * Atomic_Add -- + * Atomic_Add32 -- * * Atomic read, add a value, write. * @@ -1411,10 +1539,10 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT */ static INLINE void -Atomic_Add(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Add32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1433,8 +1561,8 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, add, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1443,7 +1571,6 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1457,13 +1584,13 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Add #endif } -#define Atomic_Add32 Atomic_Add +#define Atomic_Add Atomic_Add32 /* *----------------------------------------------------------------------------- * - * Atomic_Sub -- + * Atomic_Sub32 -- * * Atomic read, subtract a value, write. * @@ -1477,10 +1604,10 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Sub(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Sub32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1499,8 +1626,8 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, sub); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, sub, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1509,7 +1636,6 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1523,13 +1649,13 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Sub #endif } -#define Atomic_Sub32 Atomic_Sub +#define Atomic_Sub Atomic_Sub32 /* *----------------------------------------------------------------------------- * - * Atomic_Inc -- + * Atomic_Inc32 -- * * Atomic read, increment, write. * @@ -1543,11 +1669,11 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Inc(Atomic_uint32 *var) // IN/OUT +Atomic_Inc32(Atomic_uint32 *var) // IN/OUT { #ifdef __GNUC__ -#if defined(VM_ARM_ANY) - Atomic_Add(var, 1); +#if defined VM_ARM_ANY + Atomic_Add32(var, 1); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1556,7 +1682,6 @@ Atomic_Inc(Atomic_uint32 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1569,13 +1694,13 @@ Atomic_Inc(Atomic_uint32 *var) // IN/OUT #error No compiler defined for Atomic_Inc #endif } -#define Atomic_Inc32 Atomic_Inc +#define Atomic_Inc Atomic_Inc32 /* *----------------------------------------------------------------------------- * - * Atomic_Dec -- + * Atomic_Dec32 -- * * Atomic read, decrement, write. * @@ -1589,11 +1714,11 @@ Atomic_Inc(Atomic_uint32 *var) // IN/OUT */ static INLINE void -Atomic_Dec(Atomic_uint32 *var) // IN/OUT +Atomic_Dec32(Atomic_uint32 *var) // IN/OUT { #ifdef __GNUC__ -#if defined(VM_ARM_ANY) - Atomic_Sub(var, 1); +#if defined VM_ARM_ANY + Atomic_Sub32(var, 1); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1602,7 +1727,6 @@ Atomic_Dec(Atomic_uint32 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1615,7 +1739,7 @@ Atomic_Dec(Atomic_uint32 *var) // IN/OUT #error No compiler defined for Atomic_Dec #endif } -#define Atomic_Dec32 Atomic_Dec +#define Atomic_Dec Atomic_Dec32 /* @@ -1646,9 +1770,13 @@ Atomic_ReadOr32(Atomic_uint32 *var, // IN/OUT { uint32 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 32, TRUE, &var->value, orr, val); +#else do { - res = Atomic_Read(var); - } while (res != Atomic_ReadIfEqualWrite(var, res, res | val)); + res = Atomic_Read32(var); + } while (res != Atomic_ReadIfEqualWrite32(var, res, res | val)); +#endif return res; } @@ -1676,15 +1804,19 @@ Atomic_ReadAnd32(Atomic_uint32 *var, // IN/OUT { uint32 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 32, TRUE, &var->value, and, val); +#else do { - res = Atomic_Read(var); - } while (res != Atomic_ReadIfEqualWrite(var, res, res & val)); + res = Atomic_Read32(var); + } while (res != Atomic_ReadIfEqualWrite32(var, res, res & val)); +#endif return res; } -#if defined(VM_64BIT) +#if defined VM_64BIT /* *----------------------------------------------------------------------------- * @@ -1707,9 +1839,13 @@ Atomic_ReadOr64(Atomic_uint64 *var, // IN/OUT { uint64 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 64, TRUE, &var->value, orr, val); +#else do { res = var->value; } while (res != Atomic_ReadIfEqualWrite64(var, res, res | val)); +#endif return res; } @@ -1737,13 +1873,17 @@ Atomic_ReadAnd64(Atomic_uint64 *var, // IN/OUT { uint64 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 64, TRUE, &var->value, and, val); +#else do { res = var->value; } while (res != Atomic_ReadIfEqualWrite64(var, res, res & val)); +#endif return res; } -#endif /* defined(VM_64BIT) */ +#endif /* defined VM_64BIT */ /* @@ -1770,7 +1910,7 @@ static INLINE uint32 Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 retVal; @@ -1792,8 +1932,8 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - return ARM_64_ATOMIC_32_OPO_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + return _VMATOM_X(ROP, 32, TRUE, &var->value, add, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1803,7 +1943,6 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT : "0" (val) : "cc" ); - AtomicEpilogue(); return val; #endif /* VM_X86_ANY */ #elif defined _MSC_VER @@ -1871,51 +2010,6 @@ Atomic_ReadDec32(Atomic_uint32 *var) // IN/OUT } -#if defined VMKERNEL || defined VMM -#if !defined(VM_ARM_64) -/* - *----------------------------------------------------------------------------- - * - * CMPXCHG1B -- - * - * Compare and exchange a single byte. - * - * Results: - * The value read from ptr. - * - * Side effects: - * None - * - *----------------------------------------------------------------------------- - */ -static INLINE uint8 -CMPXCHG1B(volatile uint8 *ptr, // IN/OUT - uint8 oldVal, // IN - uint8 newVal) // IN -{ - uint8 val; - __asm__ __volatile__("lock; cmpxchgb %b2, %1" - : "=a" (val), - "+m" (*ptr) - : "r" (newVal), - "0" (oldVal) - : "cc"); - return val; -} -#endif /* !defined(VM_ARM_64) */ -#endif - - -/* - * Usage of this helper struct is strictly reserved to the following - * function. --hpreg - */ -typedef struct { - uint32 lowValue; - uint32 highValue; -} S_uint64; - - /* *----------------------------------------------------------------------------- * @@ -1923,12 +2017,6 @@ typedef struct { * * Compare exchange: Read variable, if equal to oldVal, write newVal * - * XXX: Ensure that if this function is to be inlined by gcc, it is - * compiled with -fno-strict-aliasing. Otherwise it will break. - * Unfortunately we know that gcc 2.95.3 (used to build the FreeBSD 3.2 - * Tools) does not honor -fno-strict-aliasing. As a workaround, we avoid - * inlining the function entirely for versions of gcc under 3.0. - * * Results: * TRUE if equal, FALSE if not equal * @@ -1938,23 +2026,19 @@ typedef struct { *----------------------------------------------------------------------------- */ -#if defined(__GNUC__) && __GNUC__ < 3 -static Bool -#else static INLINE Bool -#endif Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT - uint64 const *oldVal, // IN - uint64 const *newVal) // IN + uint64 oldVal, // IN + uint64 newVal) // IN { -#if defined(__GNUC__) -#if defined(VM_ARM_ANY) - return (Atomic_ReadIfEqualWrite64(var, *oldVal, *newVal) == *oldVal); +#if defined __GNUC__ +#if defined VM_ARM_ANY + return Atomic_ReadIfEqualWrite64(var, oldVal, newVal) == oldVal; #else /* VM_X86_ANY */ Bool equal; /* Checked against the Intel manual and GCC --walken */ -#if defined(__x86_64__) +#if defined __x86_64__ uint64 dummy; __asm__ __volatile__( "lock; cmpxchgq %3, %0" "\n\t" @@ -1962,11 +2046,16 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT : "+m" (*var), "=qm" (equal), "=a" (dummy) - : "r" (*newVal), - "2" (*oldVal) + : "r" (newVal), + "2" (oldVal) : "cc" ); #else /* 32-bit version for non-ARM */ + typedef struct { + uint32 lowValue; + uint32 highValue; + } S_uint64; + int dummy1, dummy2; # if defined __PIC__ /* @@ -1994,28 +2083,6 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT * where it temporarily modifies %ebx as small as possible, and should * prefer specific register assignments. */ -# if __GNUC__ < 3 // Part of #188541 - for RHL 6.2 etc. - __asm__ __volatile__( - "xchg %%ebx, %6" "\n\t" - "mov 4(%%ebx), %%ecx" "\n\t" - "mov (%%ebx), %%ebx" "\n\t" - "lock; cmpxchg8b (%3)" "\n\t" - "xchg %%ebx, %6" "\n\t" - "sete %0" - : "=a" (equal), - "=d" (dummy2), - "=D" (dummy1) - : /* - * See the "Rules for __asm__ statements in __PIC__ code" above: %3 - * must use a register class which does not contain %ebx. - */ - "S" (var), - "0" (((S_uint64 const *)oldVal)->lowValue), - "1" (((S_uint64 const *)oldVal)->highValue), - "D" (newVal) - : "ecx", "cc", "memory" - ); -# else __asm__ __volatile__( "xchgl %%ebx, %6" "\n\t" "lock; cmpxchg8b (%3)" "\n\t" @@ -2033,13 +2100,12 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT * stack, %ebp reserved for frame, %ebx reserved for PIC). */ "S" (var), - "1" (((S_uint64 const *)oldVal)->lowValue), - "2" (((S_uint64 const *)oldVal)->highValue), - "D" (((S_uint64 const *)newVal)->lowValue), - "c" (((S_uint64 const *)newVal)->highValue) + "1" (((S_uint64 *)&oldVal)->lowValue), + "2" (((S_uint64 *)&oldVal)->highValue), + "D" (((S_uint64 *)&newVal)->lowValue), + "c" (((S_uint64 *)&newVal)->highValue) : "cc", "memory" ); -# endif # else __asm__ __volatile__( "lock; cmpxchg8b %0" "\n\t" @@ -2048,40 +2114,20 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT "=qm" (equal), "=a" (dummy1), "=d" (dummy2) - : "2" (((S_uint64 const *)oldVal)->lowValue), - "3" (((S_uint64 const *)oldVal)->highValue), - "b" (((S_uint64 const *)newVal)->lowValue), - "c" (((S_uint64 const *)newVal)->highValue) + : "2" (((S_uint64 *)&oldVal)->lowValue), + "3" (((S_uint64 *)&oldVal)->highValue), + "b" (((S_uint64 *)&newVal)->lowValue), + "c" (((S_uint64 *)&newVal)->highValue) : "cc" ); # endif #endif - AtomicEpilogue(); return equal; #endif //VM_ARM_V7 #elif defined _MSC_VER -#if defined(__x86_64__) - return (__int64)*oldVal == _InterlockedCompareExchange64((__int64 *)&var->value, - (__int64)*newVal, - (__int64)*oldVal); -#else -#pragma warning(push) -#pragma warning(disable : 4035) // disable no-return warning - { - __asm mov esi, var - __asm mov edx, oldVal - __asm mov ecx, newVal - __asm mov eax, [edx]S_uint64.lowValue - __asm mov edx, [edx]S_uint64.highValue - __asm mov ebx, [ecx]S_uint64.lowValue - __asm mov ecx, [ecx]S_uint64.highValue - __asm lock cmpxchg8b [esi] - __asm sete al - __asm movzx eax, al - // eax is the return value, this is documented to work - edward - } -#pragma warning(pop) -#endif + return (__int64)oldVal == _InterlockedCompareExchange64((__int64 *)&var->value, + (__int64)newVal, + (__int64)oldVal); #else #error No compiler defined for Atomic_CMPXCHG64 #endif // !GNUC @@ -2109,9 +2155,9 @@ Atomic_CMPXCHG32(Atomic_uint32 *var, // IN/OUT uint32 oldVal, // IN uint32 newVal) // IN { -#if defined(__GNUC__) -#if defined(VM_ARM_ANY) - return (Atomic_ReadIfEqualWrite(var, oldVal, newVal) == oldVal); +#if defined __GNUC__ +#if defined VM_ARM_ANY + return Atomic_ReadIfEqualWrite32(var, oldVal, newVal) == oldVal; #else /* VM_X86_ANY */ Bool equal; uint32 dummy; @@ -2126,12 +2172,11 @@ Atomic_CMPXCHG32(Atomic_uint32 *var, // IN/OUT "2" (oldVal) : "cc" ); - AtomicEpilogue(); return equal; #endif /* VM_X86_ANY */ -#else // defined(__GNUC__) - return (Atomic_ReadIfEqualWrite(var, oldVal, newVal) == oldVal); -#endif // !defined(__GNUC__) +#else // defined __GNUC__ + return Atomic_ReadIfEqualWrite32(var, oldVal, newVal) == oldVal; +#endif // !defined __GNUC__ } @@ -2154,12 +2199,15 @@ Atomic_CMPXCHG32(Atomic_uint32 *var, // IN/OUT static INLINE uint64 Atomic_Read64(Atomic_uint64 const *var) // IN { -#if defined(__GNUC__) && defined(__x86_64__) +#if defined __GNUC__ uint64 value; +#endif -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 8 == 0); #endif + +#if defined __GNUC__ && defined __x86_64__ /* * Use asm to ensure we emit a single load. */ @@ -2168,9 +2216,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN : "=r" (value) : "m" (var->value) ); - return value; -#elif defined(__GNUC__) && defined(__i386__) - uint64 value; +#elif defined __GNUC__ && defined __i386__ /* * Since cmpxchg8b will replace the contents of EDX:EAX with the * value in memory if there is no match, we need only execute the @@ -2188,9 +2234,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN : "m" (*var) : "cc" ); - AtomicEpilogue(); - return value; -#elif defined (_MSC_VER) && defined(__x86_64__) +#elif defined _MSC_VER && defined __x86_64__ /* * Microsoft docs guarantee "Simple reads and writes to properly * aligned 64-bit variables are atomic on 64-bit Windows." @@ -2199,7 +2243,9 @@ Atomic_Read64(Atomic_uint64 const *var) // IN * XXX Verify that value is properly aligned. Bug 61315. */ return var->value; -#elif defined (_MSC_VER) && defined(__i386__) +#elif defined _MSC_VER && defined VM_ARM_32 + return _InterlockedAdd64((__int64 *)&var->value, 0); +#elif defined _MSC_VER && defined __i386__ # pragma warning(push) # pragma warning(disable : 4035) // disable no-return warning { @@ -2210,26 +2256,19 @@ Atomic_Read64(Atomic_uint64 const *var) // IN // edx:eax is the return value; this is documented to work. --mann } # pragma warning(pop) -#elif defined(__GNUC__) && defined (VM_ARM_V7) - uint64 value; - +#elif defined __GNUC__ && defined VM_ARM_V7 __asm__ __volatile__( "ldrexd %[value], %H[value], [%[var]] \n\t" : [value] "=&r" (value) : [var] "r" (&var->value) ); +#elif defined VM_ARM_64 + value = _VMATOM_X(R, 64, &var->value); +#endif +#if defined __GNUC__ return value; -#elif defined(__GNUC__) && defined(VM_ARM_64) - uint64 value; - - __asm__ __volatile__( - "ldr %0, [%1]" - : "=r" (value) - : "r" (&var->value)); - - return value; -#endif /* defined(__GNUC__) && defined(VM_ARM_64) */ +#endif } @@ -2241,7 +2280,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN * Atomically read a 64 bit integer, possibly misaligned. * This function can be *very* expensive, costing over 50 kcycles * on Nehalem. - * + * * Note that "var" needs to be writable, even though it will not * be modified. * @@ -2254,7 +2293,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN *---------------------------------------------------------------------- */ -#if defined(VM_64BIT) +#if defined VM_64BIT static INLINE uint64 Atomic_ReadUnaligned64(Atomic_uint64 const *var) // IN: { @@ -2283,11 +2322,11 @@ static INLINE uint64 Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(VM_ARM_64) - return ARM_64_ATOMIC_64_OPO_PRIVATE_FENCED(var, val, add); -#elif defined(__x86_64__) +#if defined VM_ARM_64 + return _VMATOM_X(ROP, 64, TRUE, &var->value, add, val); +#elif defined __x86_64__ -#if defined(__GNUC__) +#if defined __GNUC__ __asm__ __volatile__( "lock; xaddq %0, %1" : "=r" (val), @@ -2295,7 +2334,6 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT : "0" (val) : "cc" ); - AtomicEpilogue(); return val; #elif defined _MSC_VER return _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)val); @@ -2310,7 +2348,7 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal + val; - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); return oldVal; #endif @@ -2337,7 +2375,11 @@ static INLINE uint64 Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { +#if defined VM_ARM_64 + return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val); +#else return Atomic_ReadAdd64(var, -val); +#endif } @@ -2407,12 +2449,12 @@ static INLINE void Atomic_Add64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if !defined(VM_64BIT) +#if !defined VM_64BIT Atomic_ReadAdd64(var, val); /* Return value is unused. */ -#elif defined(__GNUC__) -#if defined(VM_ARM_64) - ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, add); -#else /* defined(VM_X86_64) */ +#elif defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, add, val); +#else /* defined VM_X86_64 */ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; addq %1, %0" @@ -2420,7 +2462,6 @@ Atomic_Add64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #endif #elif defined _MSC_VER _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)val); @@ -2450,11 +2491,11 @@ static INLINE void Atomic_Sub64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if !defined(__x86_64__) +#if !defined VM_64BIT Atomic_ReadSub64(var, val); /* Return value is unused. */ -#elif defined(__GNUC__) -#if defined(VM_ARM_64) - ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, sub); +#elif defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, sub, val); #else /* VM_X86_64 */ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( @@ -2463,7 +2504,6 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #endif #elif defined _MSC_VER _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)-val); @@ -2492,9 +2532,11 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT static INLINE void Atomic_Inc64(Atomic_uint64 *var) // IN/OUT { -#if !defined(__x86_64__) +#if defined VM_ARM_64 + Atomic_Add64(var, 1); +#elif !defined __x86_64__ Atomic_ReadInc64(var); /* Return value is unused. */ -#elif defined(__GNUC__) +#elif defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; incq %0" @@ -2502,7 +2544,6 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedIncrement64((__int64 *)&var->value); #else @@ -2530,9 +2571,11 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT static INLINE void Atomic_Dec64(Atomic_uint64 *var) // IN/OUT { -#if !defined(__x86_64__) +#if defined VM_ARM_64 + Atomic_Sub64(var, 1); +#elif !defined __x86_64__ Atomic_ReadDec64(var); /* Return value is unused. */ -#elif defined(__GNUC__) +#elif defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; decq %0" @@ -2540,7 +2583,6 @@ Atomic_Dec64(Atomic_uint64 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedDecrement64((__int64 *)&var->value); #else @@ -2569,8 +2611,8 @@ static INLINE uint64 Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "xchgq %0, %1" @@ -2578,19 +2620,20 @@ Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; #elif defined _MSC_VER return _InterlockedExchange64((__int64 *)&var->value, (__int64)val); #else #error No compiler defined for Atomic_ReadWrite64 #endif +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 64, TRUE, &var->value, val); #else uint64 oldVal; do { oldVal = var->value; - } while (!Atomic_CMPXCHG64(var, &oldVal, &val)); + } while (!Atomic_CMPXCHG64(var, oldVal, val)); return oldVal; #endif @@ -2617,12 +2660,12 @@ static INLINE void Atomic_Write64(Atomic_uint64 *var, // OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) - -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 8 == 0); #endif + +#if defined __x86_64__ +#if defined __GNUC__ /* * There is no move instruction for 64-bit immediate to memory, so unless * the immediate value fits in 32-bit (i.e. can be sign-extended), GCC @@ -2637,7 +2680,7 @@ Atomic_Write64(Atomic_uint64 *var, // OUT ); #elif defined _MSC_VER /* - * Microsoft docs guarantee "Simple reads and writes to properly aligned + * Microsoft docs guarantee "Simple reads and writes to properly aligned * 64-bit variables are atomic on 64-bit Windows." * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx * @@ -2648,7 +2691,9 @@ Atomic_Write64(Atomic_uint64 *var, // OUT #else #error No compiler defined for Atomic_Write64 #endif -#else /* defined(__x86_64__) */ +#elif defined VM_ARM_64 + _VMATOM_X(W, 64, &var->value, val); +#else (void)Atomic_ReadWrite64(var, val); #endif } @@ -2674,8 +2719,8 @@ static INLINE void Atomic_Or64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; orq %1, %0" @@ -2683,19 +2728,20 @@ Atomic_Or64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedOr64((__int64 *)&var->value, (__int64)val); #else #error No compiler defined for Atomic_Or64 #endif +#elif defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, orr, val); #else // __x86_64__ uint64 oldVal; uint64 newVal; do { oldVal = var->value; newVal = oldVal | val; - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2720,8 +2766,8 @@ static INLINE void Atomic_And64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; andq %1, %0" @@ -2729,19 +2775,20 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedAnd64((__int64 *)&var->value, (__int64)val); #else #error No compiler defined for Atomic_And64 #endif +#elif defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, and, val); #else // __x86_64__ uint64 oldVal; uint64 newVal; do { oldVal = var->value; newVal = oldVal & val; - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2766,8 +2813,8 @@ static INLINE void Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT uint64 bit) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ ASSERT(bit <= 63); __asm__ __volatile__( "lock; bts %1, %0" @@ -2775,7 +2822,6 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT : "ri" (bit) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER uint64 oldVal; uint64 newVal; @@ -2783,7 +2829,7 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal | (CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #else #error No compiler defined for Atomic_SetBit64 #endif @@ -2794,7 +2840,7 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal | (CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2819,8 +2865,8 @@ static INLINE void Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT uint64 bit) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ ASSERT(bit <= 63); __asm__ __volatile__( "lock; btr %1, %0" @@ -2828,7 +2874,6 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT : "ri" (bit) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER uint64 oldVal; uint64 newVal; @@ -2836,7 +2881,7 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal & ~(CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #else #error No compiler defined for Atomic_ClearBit64 #endif @@ -2847,7 +2892,7 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal & ~(CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2872,8 +2917,8 @@ static INLINE Bool Atomic_TestBit64(Atomic_uint64 *var, // IN uint64 bit) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ Bool out = FALSE; ASSERT(bit <= 63); __asm__ __volatile__( @@ -2897,7 +2942,7 @@ Atomic_TestBit64(Atomic_uint64 *var, // IN } -#if defined(__GNUC__) +#if defined __GNUC__ /* *----------------------------------------------------------------------------- * @@ -2919,19 +2964,19 @@ Atomic_Read16(Atomic_uint16 const *var) // IN { uint16 value; -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 2 == 0); #endif -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "movw %1, %0" : "=r" (value) : "m" (var->value) ); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 NOT_TESTED(); __asm__ __volatile__( @@ -2939,13 +2984,8 @@ Atomic_Read16(Atomic_uint16 const *var) // IN : "=r" (value) : "r" (&var->value) ); -#elif defined(VM_ARM_64) - __asm__ __volatile__ ( - "ldrh %w0, [%1]" - : "=r" (value) - : "r" (&var->value) - ); - +#elif defined VM_ARM_64 + value = _VMATOM_X(R, 16, &var->value); #else #error No 16-bits atomics. #endif @@ -2975,17 +3015,16 @@ static INLINE uint16 Atomic_ReadWrite16(Atomic_uint16 *var, // IN/OUT: uint16 val) // IN: { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "xchgw %0, %1" : "=r" (val), "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 retVal; register volatile uint16 res; @@ -3006,22 +3045,8 @@ Atomic_ReadWrite16(Atomic_uint16 *var, // IN/OUT: dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint16 retVal; - register uint16 failed; - - __asm__ __volatile__( - " dmb sy \n\t" - "1: ldxrh %w0, [%2] \n\t" - " stxrh %w1, %w3, [%2] \n\t" - " cbnz %w1, 1b \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (val) - : "memory" - ); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 16, TRUE, &var->value, val); #else #error No 16-bits atomics. #endif @@ -3049,19 +3074,24 @@ static INLINE void Atomic_Write16(Atomic_uint16 *var, // OUT: uint16 val) // IN: { -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 2 == 0); #endif -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) - +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "movw %1, %0" : "=m" (var->value) : "r" (val) ); -#elif defined(VM_ARM_ANY) +#elif defined VM_ARM_64 + _VMATOM_X(W, 16, &var->value, val); +#elif defined VM_ARM_32 + /* + * Best left this way due to the intricacies of exclusive load/store + * operations on legacy (32-bit) ARM. + */ Atomic_ReadWrite16(var, val); #else #error No 16-bits atomics. @@ -3091,8 +3121,8 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var, // IN/OUT uint16 oldVal, // IN uint16 newVal) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ uint16 val; __asm__ __volatile__( @@ -3103,9 +3133,8 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var, // IN/OUT "0" (oldVal) : "cc" ); - AtomicEpilogue(); return val; -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register uint16 retVal; register uint16 res; @@ -3128,24 +3157,8 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint16 retVal; - register uint16 failed; - - __asm__ __volatile__ ( - " dmb sy \n\t" - "1: ldxrh %w0, [%2] \n\t" - " cmp %w0, %w3 \n\t" - " b.ne 2f \n\t" - " stxrh %w1, %w4, [%2] \n\t" - " cbnz %w1, 1b \n\t" - "2: clrex \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (oldVal), "r" (newVal) - : "cc", "memory"); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 16, TRUE, &var->value, oldVal, newVal); #else #error No 16-bits atomics. #endif @@ -3173,16 +3186,15 @@ static INLINE void Atomic_And16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; andw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3202,8 +3214,8 @@ Atomic_And16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, and); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, and, val); #else #error No 16-bits atomics. #endif @@ -3231,16 +3243,15 @@ static INLINE void Atomic_Or16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; orw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3260,14 +3271,15 @@ Atomic_Or16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, orr); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, orr, val); #else #error No 16-bits atomics. #endif #endif } + /* *----------------------------------------------------------------------------- * @@ -3288,16 +3300,15 @@ static INLINE void Atomic_Xor16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; xorw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3317,8 +3328,8 @@ Atomic_Xor16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, eor); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, eor, val); #else #error No 16-bits atomics. #endif @@ -3346,16 +3357,15 @@ static INLINE void Atomic_Add16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; addw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3375,8 +3385,8 @@ Atomic_Add16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, add, val); #else #error No 16-bits atomics. #endif @@ -3404,16 +3414,15 @@ static INLINE void Atomic_Sub16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; subw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3433,8 +3442,8 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, sub); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, sub, val); #else #error No 16-bits atomics. #endif @@ -3461,16 +3470,15 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT static INLINE void Atomic_Inc16(Atomic_uint16 *var) // IN/OUT { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; incw %0" : "+m" (var->value) : : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_ANY) +#elif defined VM_ARM_ANY Atomic_Add16(var, 1); #else #error No 16-bits atomics. @@ -3498,16 +3506,15 @@ Atomic_Inc16(Atomic_uint16 *var) // IN/OUT static INLINE void Atomic_Dec16(Atomic_uint16 *var) // IN/OUT { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; decw %0" : "+m" (var->value) : : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_ANY) +#elif defined VM_ARM_ANY Atomic_Sub16(var, 1); #else #error No 16-bits atomics. @@ -3538,9 +3545,13 @@ Atomic_ReadOr16(Atomic_uint16 *var, // IN/OUT { uint16 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 16, TRUE, &var->value, orr, val); +#else do { res = var->value; } while (res != Atomic_ReadIfEqualWrite16(var, res, res | val)); +#endif return res; } @@ -3566,8 +3577,8 @@ static INLINE uint16 Atomic_ReadAdd16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN: { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; xaddw %0, %1" : "=r" (val), @@ -3575,9 +3586,8 @@ Atomic_ReadAdd16(Atomic_uint16 *var, // IN/OUT : "0" (val) : "cc" ); - AtomicEpilogue(); return val; -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 retVal; register volatile uint16 tmp; @@ -3600,8 +3610,8 @@ Atomic_ReadAdd16(Atomic_uint16 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - return ARM_64_ATOMIC_16_OPO_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + return _VMATOM_X(ROP, 16, TRUE, &var->value, add, val); #else #error No 16-bits atomics. #endif @@ -3631,8 +3641,8 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT return Atomic_ReadAdd16(var, 1); } -/* +/* *---------------------------------------------------------------------- * * Atomic_ReadDec16 -- @@ -3840,57 +3850,38 @@ Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT * Atomic_ReadAddInt -- * Atomic_ReadIncInt -- * Atomic_ReadDecInt -- + * + * Atomic_Bool + * Atomic_ReadBool -- + * Atomic_WriteBool -- + * Atomic_ReadWriteBool -- + * Atomic_ReadIfEqualWriteBool -- + * Atomic_AndBool -- + * Atomic_OrBool -- + * Atomic_XorBool -- + * Atomic_AddBool -- + * Atomic_SubBool -- + * Atomic_IncBool -- + * Atomic_DecBool -- + * Atomic_ReadOrBool -- + * Atomic_ReadAddBool -- + * Atomic_ReadIncBool -- + * Atomic_ReadDecBool -- */ -#if defined(VM_64BIT) +#if defined VM_64BIT MAKE_ATOMIC_TYPE(Ptr, 64, void const *, void *, uintptr_t) #else MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t) #endif MAKE_ATOMIC_TYPE(Int, 32, int, int, int) +MAKE_ATOMIC_TYPE(Bool, 8, Bool, Bool, Bool) - -/* Prevent the compiler from re-ordering memory references. */ -#ifdef __GNUC__ -#define ATOMIC_COMPILER_BARRIER() __asm__ __volatile__ ("": : :"memory") -#elif defined(_MSC_VER) -#define ATOMIC_COMPILER_BARRIER() _ReadWriteBarrier() -#else -#error No compiler defined for ATOMIC_COMPILER_BARRIER +#ifdef VM_ARM_64 +# include "vm_atomic_arm64_end.h" #endif - -/* - *----------------------------------------------------------------------------- - * - * Atomic_MFence -- - * - * Implements mfence in terms of a lock xor. The reason for implementing - * our own mfence is that not all of our supported cpus have an assembly - * mfence (P3, Athlon). We put it here to avoid duplicating code which is - * also why it is prefixed with "Atomic_". Moreover, this implementation - * performs slightly better than 'mfence' on modern CPUs (See PR 817059). - * - * Results: - * None. - * - * Side effects: - * Cause loads and stores prior to this to be globally - * visible. - * - *----------------------------------------------------------------------------- - */ - -static INLINE void -Atomic_MFence(void) -{ - Atomic_uint32 fence; - ATOMIC_COMPILER_BARRIER(); - Atomic_Xor(&fence, 0x1); - ATOMIC_COMPILER_BARRIER(); -} - -#ifdef ATOMIC_COMPILER_BARRIER -#undef ATOMIC_COMPILER_BARRIER -#endif /* ATOMIC_COMPILER_BARRIER */ +#if defined __cplusplus +} // extern "C" +#endif #endif // ifndef _ATOMIC_H_ diff --git a/vmmon-only/include/vm_basic_asm.h b/vmmon-only/include/vm_basic_asm.h index 0e4adae..11c3aba 100644 --- a/vmmon-only/include/vm_basic_asm.h +++ b/vmmon-only/include/vm_basic_asm.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -43,6 +43,10 @@ #elif defined VM_X86_32 #include "vm_basic_asm_x86_common.h" #include "vm_basic_asm_x86.h" +#elif defined VM_ARM_32 +#include "vm_basic_asm_arm32.h" +#define MUL64_NO_ASM 1 +#include "mul64.h" #elif defined VM_ARM_64 #include "arm64_basic_defs.h" #include "vm_basic_asm_arm64.h" @@ -51,6 +55,11 @@ #include "mul64.h" #endif +#if defined __cplusplus +extern "C" { +#endif + + /* * Locate most and least significant bit set functions. Use our own name * space to avoid namespace collisions. The new names follow a pattern, @@ -623,6 +632,11 @@ uint16set(void *dst, uint16 val, size_t count) { #ifdef VM_X86_64 __stosw((uint16*)dst, val, count); +#elif defined(VM_ARM_32) + size_t i; + for (i = 0; i < count; i++) { + ((uint16 *)dst)[i] = val; + } #else __asm { pushf; mov ax, val; @@ -641,6 +655,11 @@ uint32set(void *dst, uint32 val, size_t count) { #ifdef VM_X86_64 __stosd((unsigned long*)dst, (unsigned long)val, count); +#elif defined(VM_ARM_32) + size_t i; + for (i = 0; i < count; i++) { + ((uint32 *)dst)[i] = val; + } #else __asm { pushf; mov eax, val; @@ -673,7 +692,7 @@ static INLINE uint16 Bswap16(uint16 v) { #if defined(VM_ARM_64) - __asm__("rev16 %0, %0" : "+r"(v)); + __asm__("rev16 %w0, %w0" : "+r"(v)); return v; #else return ((v >> 8) & 0x00ff) | ((v << 8) & 0xff00); @@ -702,12 +721,12 @@ Bswap32(uint32 v) // IN : "0" (v) ); return v; -#elif defined(VM_ARM_32) && !defined(__ANDROID__) +#elif defined(VM_ARM_32) && !defined(__ANDROID__) && !defined(_MSC_VER) __asm__("rev %0, %0" : "+r"(v)); return v; #elif defined(VM_ARM_64) - __asm__("rev32 %0, %0" : "+r"(v)); - return v; + __asm__("rev32 %x0, %x0" : "+r"(v)); + return v; #else return (v >> 24) | ((v >> 8) & 0xFF00) @@ -740,63 +759,6 @@ Bswap64(uint64 v) // IN } -/* - * COMPILER_MEM_BARRIER prevents the compiler from re-ordering memory - * references accross the barrier. NOTE: It does not generate any - * instruction, so the CPU is free to do whatever it wants to... - */ -#ifdef __GNUC__ -#define COMPILER_MEM_BARRIER() __asm__ __volatile__ ("": : :"memory") -#define COMPILER_READ_BARRIER() COMPILER_MEM_BARRIER() -#define COMPILER_WRITE_BARRIER() COMPILER_MEM_BARRIER() -#elif defined(_MSC_VER) -#define COMPILER_MEM_BARRIER() _ReadWriteBarrier() -#define COMPILER_READ_BARRIER() _ReadBarrier() -#define COMPILER_WRITE_BARRIER() _WriteBarrier() -#endif - - -/* - *---------------------------------------------------------------------- - * - * COMPILER_FORCED_LOAD_AND_MEM_BARRIER -- - * - * This macro prevents the compiler from re-ordering memory references - * across the barrier. In addition it emits a forced load from the given - * memory reference. The memory reference has to be either 1, 2, 4 or 8 - * bytes wide. - * The forced load of a memory reference can be used exploit details of a - * given CPUs memory model. For example x86 CPUs won't reorder stores to - * a memory location x with loads from a memory location x. - * NOTE: It does not generate any fencing instruction, so the CPU is free - * to reorder instructions according to its memory model. - * - * Results: - * None - * - * Side Effects: - * None. - * - *---------------------------------------------------------------------- - */ - -#ifdef VM_X86_64 -#ifdef __GNUC__ - -#define COMPILER_FORCED_LOAD_AND_MEM_BARRIER(_memory_reference) \ - do { \ - typeof(_memory_reference) _dummy; \ - \ - asm volatile("mov %1, %0\n\t" \ - : "=r" (_dummy) /* Let compiler choose reg for _dummy */ \ - : "m" (_memory_reference) \ - : "memory"); \ - } while(0) - -#endif /* __GNUC__ */ -#endif /* VM_X86_64 */ - - /* * PAUSE is a P4 instruction that improves spinlock power+performance; * on non-P4 IA32 systems, the encoding is interpreted as a REPZ-NOP. @@ -804,7 +766,7 @@ Bswap64(uint64 v) // IN */ static INLINE void PAUSE(void) -#ifdef __GNUC__ +#if defined(__GNUC__) || defined(VM_ARM_32) { #ifdef VM_ARM_ANY /* @@ -878,6 +840,15 @@ RDTSC(void) { return __rdtsc(); } +#elif defined(VM_ARM_32) +{ + /* + * We need to do more inverstagetion here to find + * a microsoft equivalent of that code + */ + NOT_IMPLEMENTED(); + return 0; +} #else #pragma warning( disable : 4035) { @@ -1254,5 +1225,10 @@ RoundUpPow2_32(uint32 value) #endif } + +#if defined __cplusplus +} // extern "C" +#endif + #endif // _VM_BASIC_ASM_H_ diff --git a/vmmon-only/include/vm_basic_asm_x86.h b/vmmon-only/include/vm_basic_asm_x86.h index 2c4232c..b9c684f 100644 --- a/vmmon-only/include/vm_basic_asm_x86.h +++ b/vmmon-only/include/vm_basic_asm_x86.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -35,6 +35,11 @@ #define INCLUDE_ALLOW_VMCORE #include "includeCheck.h" +#if defined __cplusplus +extern "C" { +#endif + + #ifdef VM_X86_64 /* * The gcc inline asm uses the "A" constraint which differs in 32 & 64 @@ -545,4 +550,9 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift) #pragma warning(default: 4035) #endif + +#if defined __cplusplus +} // extern "C" #endif + +#endif // _VM_BASIC_ASM_X86_H_ diff --git a/vmmon-only/include/vm_basic_asm_x86_64.h b/vmmon-only/include/vm_basic_asm_x86_64.h index bef0056..4f9fd4f 100644 --- a/vmmon-only/include/vm_basic_asm_x86_64.h +++ b/vmmon-only/include/vm_basic_asm_x86_64.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -59,24 +59,23 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift); #if defined(__GNUC__) /* - * GET_CURRENT_RIP + * GET_CURRENT_PC * - * Returns the current instruction pointer. In the example below: + * Returns the current program counter (i.e. instruction pointer i.e. rip + * register on x86_64). In the example below: * * foo.c - * L123: Foo(GET_CURRENT_RIP()) + * L123: Foo(GET_CURRENT_PC()) * - * the return value from GET_CURRENT_RIP will point a debugger to L123. + * the return value from GET_CURRENT_PC will point a debugger to L123. */ -#define GET_CURRENT_RIP() ({ \ +#define GET_CURRENT_PC() ({ \ void *__rip; \ asm("lea 0(%%rip), %0;\n\t" \ : "=r" (__rip)); \ __rip; \ }) -#define GET_CURRENT_PC() GET_CURRENT_RIP() - /* * GET_CURRENT_LOCATION * diff --git a/vmmon-only/include/vm_basic_asm_x86_common.h b/vmmon-only/include/vm_basic_asm_x86_common.h index 6bbbc3e..06ae1b5 100644 --- a/vmmon-only/include/vm_basic_asm_x86_common.h +++ b/vmmon-only/include/vm_basic_asm_x86_common.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2013 VMware, Inc. All rights reserved. + * Copyright (C) 2013-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -89,6 +89,9 @@ void _mm_mfence(void); void _mm_lfence(void); #pragma intrinsic(_mm_mfence, _mm_lfence) +long _InterlockedXor(long volatile *, long); +#pragma intrinsic(_InterlockedXor) + unsigned int __getcallerseflags(void); #pragma intrinsic(__getcallerseflags) @@ -294,20 +297,150 @@ RDTSC_BARRIER(void) /* - * Compiler/CPU barriers. These take the form of __MEM_BARRIER, where is either LD (load), ST (store) or - * LDST (any). On x86 we only need to care specifically about store-load - * reordering on normal memory types and mfence, otherwise only a compiler - * barrier is needed. + * Memory Barriers + * =============== + * + * Terminology + * ----------- + * + * A compiler memory barrier prevents the compiler from re-ordering memory + * accesses accross the barrier. It is not a CPU instruction, it is a compiler + * directive (i.e. it does not emit any code). + * + * A CPU memory barrier prevents the CPU from re-ordering memory accesses + * accross the barrier. It is a CPU instruction. + * + * A memory barrier is the union of a compiler memory barrier and a CPU memory + * barrier. A compiler memory barrier is a useless construct by itself. It is + * only useful when combined with a CPU memory barrier, to implement a memory + * barrier. + * + * Semantics + * --------- + * + * At the time COMPILER_*_BARRIER were created (and references to them were + * added to the code), the code was only targetting x86. The intent of the code + * was really to use a memory barrier, but because x86 uses a strongly ordered + * memory model, the CPU would not re-order memory accesses, and the code could + * get away with using just a compiler memory barrier. So COMPILER_*_BARRIER + * were born and were implemented as compiler memory barriers _on x86_. But + * make no mistake, _the semantics that the code expects from + * COMPILER_*_BARRIER are that of a memory barrier_! + * + * DO NOT USE! + * ----------- + * + * On at least one non-x86 architecture, COMPILER_*_BARRIER are + * 1) Misnomers + * 2) Not fine-grained enough to provide the best performance. + * For the above two reasons, usage of COMPILER_*_BARRIER is now deprecated. + * _Do not add new references to COMPILER_*_BARRIER._ Instead, precisely + * document the intent of your code by using + * __BARRIER_. + * Existing references to COMPILER_*_BARRIER are being slowly but surely + * converted, and when no references are left, COMPILER_*_BARRIER will be + * retired. + * + * Thanks for pasting this whole comment into every architecture header. */ -#define LD_LD_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define LD_ST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define LD_LDST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define ST_LD_MEM_BARRIER() asm volatile ("mfence" ::: "memory") -#define ST_ST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define ST_LDST_MEM_BARRIER() ST_LD_MEM_BARRIER() -#define LDST_LD_MEM_BARRIER() ST_LD_MEM_BARRIER() -#define LDST_ST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define LDST_LDST_MEM_BARRIER() ST_LD_MEM_BARRIER() + +#if defined __GNUC__ +# define COMPILER_READ_BARRIER() COMPILER_MEM_BARRIER() +# define COMPILER_WRITE_BARRIER() COMPILER_MEM_BARRIER() +# define COMPILER_MEM_BARRIER() __asm__ __volatile__("" ::: "memory") +#elif defined _MSC_VER +# define COMPILER_READ_BARRIER() _ReadBarrier() +# define COMPILER_WRITE_BARRIER() _WriteBarrier() +# define COMPILER_MEM_BARRIER() _ReadWriteBarrier() +#endif + + +/* + * Memory barriers. These take the form of + * + * __BARRIER_ + * + * where: + * is either SMP, DMA, or MMIO. + * <*_access type> is either R(load), W(store) or RW(any). + * + * Above every use of these memory barriers in the code, there _must_ be a + * comment to justify the use, i.e. a comment which: + * + * 1) Precisely identifies which memory accesses must not be re-ordered across + * the memory barrier. + * 2) Explains why it is important that the memory accesses not be re-ordered. + * + * Thanks for pasting this whole comment into every architecture header. + * + * On x86, we only need to care specifically about store-load reordering on + * normal memory types. In other cases, only a compiler barrier is needed. The + * ST_LD barrier is implemented with a locked xor operation (instead of the + * mfence instruction) for performance reasons. See PR 1674199 for more + * details. + * + * On x64, special instructions are only provided for load-load (lfence) and + * store-store (sfence) ordering, and they don't apply to normal memory. + */ + + +static INLINE void +SMP_W_BARRIER_R(void) +{ + volatile long temp; + + COMPILER_MEM_BARRIER(); +#if defined __GNUC__ + __asm__ __volatile__ ( + "lock xorl $1, %0\n" + : "+m" (temp) + : /* no additional inputs */ + : "cc"); +#elif defined _MSC_VER + _InterlockedXor(&temp, 1); +#else +#error SMP_W_BARRIER_R not defined for this compiler +#endif + COMPILER_MEM_BARRIER(); +} + +#define SMP_R_BARRIER_R() COMPILER_READ_BARRIER() +#define SMP_R_BARRIER_W() COMPILER_MEM_BARRIER() +#define SMP_R_BARRIER_RW() COMPILER_MEM_BARRIER() +#define SMP_W_BARRIER_W() COMPILER_WRITE_BARRIER() +#define SMP_W_BARRIER_RW() SMP_W_BARRIER_R() +#define SMP_RW_BARRIER_R() SMP_W_BARRIER_R() +#define SMP_RW_BARRIER_W() COMPILER_MEM_BARRIER() +#define SMP_RW_BARRIER_RW() SMP_W_BARRIER_R() + +/* + * Like the above, only for use with observers other than CPUs, + * i.e. DMA masters. + */ + +#define DMA_R_BARRIER_R() SMP_R_BARRIER_R() +#define DMA_R_BARRIER_W() SMP_R_BARRIER_W() +#define DMA_R_BARRIER_RW() SMP_R_BARRIER_RW() +#define DMA_W_BARRIER_R() SMP_W_BARRIER_R() +#define DMA_W_BARRIER_W() SMP_W_BARRIER_W() +#define DMA_W_BARRIER_RW() SMP_W_BARRIER_RW() +#define DMA_RW_BARRIER_R() SMP_RW_BARRIER_R() +#define DMA_RW_BARRIER_W() SMP_RW_BARRIER_W() +#define DMA_RW_BARRIER_RW() SMP_RW_BARRIER_RW() + +/* + * And finally a set for use with MMIO accesses. + */ + +#define MMIO_R_BARRIER_R() SMP_R_BARRIER_R() +#define MMIO_R_BARRIER_W() SMP_R_BARRIER_W() +#define MMIO_R_BARRIER_RW() SMP_R_BARRIER_RW() +#define MMIO_W_BARRIER_R() SMP_W_BARRIER_R() +#define MMIO_W_BARRIER_W() SMP_W_BARRIER_W() +#define MMIO_W_BARRIER_RW() SMP_W_BARRIER_RW() +#define MMIO_RW_BARRIER_R() SMP_RW_BARRIER_R() +#define MMIO_RW_BARRIER_W() SMP_RW_BARRIER_W() +#define MMIO_RW_BARRIER_RW() SMP_RW_BARRIER_RW() + #endif // _VM_BASIC_ASM_X86_COMMON_H_ diff --git a/vmmon-only/include/vm_basic_defs.h b/vmmon-only/include/vm_basic_defs.h index efe492a..4702238 100644 --- a/vmmon-only/include/vm_basic_defs.h +++ b/vmmon-only/include/vm_basic_defs.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -251,12 +251,16 @@ Max(int a, int b) #define PAGES_2_BYTES(_npages) (((uint64)(_npages)) << PAGE_SHIFT) #endif +#ifndef MBYTES_SHIFT +#define MBYTES_SHIFT 20 +#endif + #ifndef MBYTES_2_PAGES -#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (20 - PAGE_SHIFT)) +#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT)) #endif #ifndef PAGES_2_MBYTES -#define PAGES_2_MBYTES(_npages) ((_npages) >> (20 - PAGE_SHIFT)) +#define PAGES_2_MBYTES(_npages) ((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT)) #endif #ifndef GBYTES_2_PAGES @@ -268,11 +272,11 @@ Max(int a, int b) #endif #ifndef BYTES_2_MBYTES -#define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> 20) +#define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT) #endif #ifndef MBYTES_2_BYTES -#define MBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << 20) +#define MBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << MBYTES_SHIFT) #endif #ifndef BYTES_2_GBYTES @@ -299,6 +303,26 @@ Max(int a, int b) #define VM_PAE_LARGE_2_SMALL_PAGES (BYTES_2_PAGES(VM_PAE_LARGE_PAGE_SIZE)) #endif +#ifndef VM_PAE_LARGE_2_BYTES +#define VM_PAE_LARGE_2_BYTES(_2mbytes) ((_2mbytes) << VM_PAE_LARGE_PAGE_SHIFT) +#endif + +#ifndef VM_1GB_PAGE_SHIFT +#define VM_1GB_PAGE_SHIFT 30 +#endif + +#ifndef VM_1GB_PAGE_SIZE +#define VM_1GB_PAGE_SIZE (1 << VM_1GB_PAGE_SHIFT) +#endif + +#ifndef VM_1GB_2_PAGES +#define VM_1GB_2_PAGES (BYTES_2_PAGES(VM_1GB_PAGE_SIZE)) +#endif + +#ifndef VM_1GB_2_PDIRS +#define VM_1GB_2_PDIRS (VM_1GB_PAGE_SIZE / VM_PAE_LARGE_PAGE_SIZE) +#endif + /* * Word operations */ @@ -394,9 +418,7 @@ void *_ReturnAddress(void); * guarantee. Bummer. --Jeremy. */ -#if defined(N_PLAT_NLM) -/* We do not have YIELD() as we do not need it yet... */ -#elif defined(_WIN32) +#if defined(_WIN32) # define YIELD() Sleep(0) #elif defined(VMKERNEL) /* We don't have a YIELD macro in the vmkernel */ @@ -412,7 +434,11 @@ void *_ReturnAddress(void); #ifdef _WIN32 // { +/* Conflict with definition of Visual Studio 2015 */ +#if (_MSC_VER < 1900) #define snprintf _snprintf +#endif + #define strtok_r strtok_s #if (_MSC_VER < 1500) @@ -652,14 +678,24 @@ typedef int pid_t; #define APPLE_ONLY(x) x #else #define vmx86_apple 0 -#define APPLE_ONLY(x) +#define APPLE_ONLY(x) +#endif + +#if defined(__APPLE__) && defined(VMW_APPLE_SANDBOX) +#define vmw_apple_sandbox 1 +#else +#define vmw_apple_sandbox 0 #endif #ifdef VMM #define VMM_ONLY(x) x -#define USER_ONLY(x) #else #define VMM_ONLY(x) +#endif + +#if defined(VMM) || defined(VMKERNEL) +#define USER_ONLY(x) +#else #define USER_ONLY(x) x #endif @@ -781,6 +817,22 @@ typedef int pid_t; #define SIZE_80BIT 10 #define SIZE_128BIT 16 #define SIZE_256BIT 32 +#define SIZE_512BIT 64 +/* + * Allocate a variable of type _type, aligned to _align bytes, returning a + * pointer to the variable in _var. Potentially _align - 1 bytes may be + * wasted. On x86, GCC 6.3.0 behaves sub-optimally when variables are declared + * on the stack using the aligned attribute, so this pattern is preferred. + * See PRs 1795155, 1819963. + */ +#define WITH_PTR_TO_ALIGNED_VAR(_type, _align, _var) \ + do { \ + uint8 _buf_##_var[sizeof(_type) + (_align) - 1]; \ + _type *_var = (_type *) ((uintptr_t)(_buf_##_var + (_align) - 1) & \ + ~((uintptr_t) ((_align) - 1))); + +#define END_PTR_TO_ALIGNED_VAR \ + } while (0) #endif // ifndef _VM_BASIC_DEFS_H_ diff --git a/vmmon-only/include/vm_basic_math.h b/vmmon-only/include/vm_basic_math.h index cef94cf..b6f5486 100644 --- a/vmmon-only/include/vm_basic_math.h +++ b/vmmon-only/include/vm_basic_math.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2008-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2008-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -38,6 +38,10 @@ #include "vm_basic_types.h" // For INLINE. #include "vm_basic_asm.h" // For Div64... +#if defined __cplusplus +extern "C" { +#endif + static INLINE uint32 RatioOf(uint32 numer1, uint32 numer2, uint32 denom) @@ -62,13 +66,36 @@ ExponentialAvg(uint32 avg, uint32 value, uint32 gainNumer, uint32 gainDenom) return (term1 + term2) / gainDenom; } + +/* + *----------------------------------------------------------------------------- + * + * IsZeroOrPowerOfTwo -- + * IsZeroOrPowerOfTwo64 -- + * + * Results: + * TRUE iff the value is 0 or a power of two. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + static INLINE Bool -IsPowerOfTwo(uint32 x) +IsZeroOrPowerOfTwo64(uint64 x) { - /* Does not check for zero. Callers depend on this. */ return !(x & (x - 1)); } + +static INLINE Bool +IsZeroOrPowerOfTwo(uint32 x) // IN +{ + return !(x & (x - 1)); +} + + static INLINE uint32 GetPowerOfTwo(uint32 x) { @@ -163,4 +190,8 @@ RotateRight64(uint64 value, uint8 shift) #endif // if !defined(_WIN32) && !defined(_WIN64) +#if defined __cplusplus +} // extern "C" +#endif + #endif // ifndef _VM_BASIC_MATH_H_ diff --git a/vmmon-only/include/vm_basic_types.h b/vmmon-only/include/vm_basic_types.h index 17b11b9..01aa0c9 100644 --- a/vmmon-only/include/vm_basic_types.h +++ b/vmmon-only/include/vm_basic_types.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -40,7 +40,8 @@ /* STRICT ANSI means the Xserver build and X defines Bool differently. */ #if !defined(_XTYPEDEF_BOOL) && \ - (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || defined(__MINGW32__)) + (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || \ + defined(__MINGW32__) || defined(__APPLE__)) #define _XTYPEDEF_BOOL typedef char Bool; #endif @@ -53,8 +54,7 @@ typedef char Bool; #define TRUE 1 #endif -#define IsBool(x) (((x) & ~1) == 0) -#define IsBool2(x, y) ((((x) | (y)) & ~1) == 0) +#define IS_BOOL(x) (((x) & ~1) == 0) /* * Macros __i386__ and __ia64 are intrinsically defined by GCC @@ -312,6 +312,19 @@ typedef uint32 uintptr_t; #endif +#if defined(__GNUC__) && defined(__SIZEOF_INT128__) + +typedef unsigned __int128 uint128; +typedef __int128 int128; + +#define MIN_INT128 ((int128)1 << 127) +#define MAX_INT128 (~MIN_INT128) +#define MIN_UINT128 ((uint128)0) +#define MAX_UINT128 (~MIN_UINT128) + +#endif + + /* * Time * XXX These should be cleaned up. -- edward @@ -353,14 +366,7 @@ typedef int64 VmTimeVirtualClock; /* Virtual Clock kept in CPU cycles */ #define FMTH "" #elif __GNUC__ #define FMTH "" - #if defined(N_PLAT_NLM) || defined(sun) || \ - (defined(__FreeBSD__) && (__FreeBSD__ + 0) && ((__FreeBSD__ + 0) < 5)) - /* - * Why (__FreeBSD__ + 0)? See bug 141008. - * Yes, we really need to test both (__FreeBSD__ + 0) and - * ((__FreeBSD__ + 0) < 5). No, we can't remove "+ 0" from - * ((__FreeBSD__ + 0) < 5). - */ + #if defined(sun) #if defined(VM_X86_64) || defined(VM_ARM_64) #define FMTSZ "l" #define FMTPD "l" @@ -368,7 +374,8 @@ typedef int64 VmTimeVirtualClock; /* Virtual Clock kept in CPU cycles */ #define FMTSZ "" #define FMTPD "" #endif - #elif defined(__linux__) \ + #elif defined(__linux__) || \ + (defined(__FreeBSD__) && (__FreeBSD__ + 0))\ || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) \ || (defined(_POSIX_VERSION) && _POSIX_VERSION >= 200112L) \ || (defined(_POSIX2_VERSION) && _POSIX2_VERSION >= 200112L) @@ -543,6 +550,11 @@ typedef uint16 UReg16; typedef uint32 UReg32; typedef uint64 UReg64; +#if defined(__GNUC__) && defined(__SIZEOF_INT128__) +typedef int128 Reg128; +typedef uint128 UReg128; +#endif + #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) || \ defined (VMKERNEL) || defined (VMKBOOT) typedef Reg64 Reg; @@ -673,10 +685,17 @@ typedef void * UserVA; #endif #define CONST const - #ifndef INLINE # ifdef _MSC_VER -# define INLINE __inline + /* + * On UWP(Universal Windows Platform), + * Only X86 32bit support '__inline' + */ +# if defined(VM_WIN_UWP) && !defined(VM_X86_32) +# define INLINE +# else +# define INLINE __inline +# endif # else # define INLINE inline # endif @@ -887,6 +906,8 @@ typedef void * UserVA; #ifndef UNUSED_PARAM # if defined(__GNUC__) # define UNUSED_PARAM(_parm) _parm __attribute__((__unused__)) +# elif defined _MSC_VER +# define UNUSED_PARAM(_parm) __pragma(warning(suppress:4100)) _parm # else # define UNUSED_PARAM(_parm) _parm # endif @@ -921,6 +942,21 @@ typedef void * UserVA; #define ALIGNED(n) #endif + +/* + * Encapsulate the syntactic differences between gcc and msvc alignment control. + * BOUNDARY must match in the prefix and suffix. + */ + +#ifdef _WIN32 +#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY)) +#define ALIGN_SUFFIX(BOUNDRY) +#else +#define ALIGN_PREFIX(BOUNDRY) +#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY))) +#endif + + /* * Once upon a time, this was used to silence compiler warnings that * get generated when the compiler thinks that a function returns @@ -1102,16 +1138,6 @@ typedef void * UserVA; #endif -/* - * Define MXSemaHandle here so both vmmon and vmx see this definition. - */ - -#ifdef _WIN32 -typedef uintptr_t MXSemaHandle; -#else -typedef int MXSemaHandle; -#endif - /* * Define type for poll device handles. */ diff --git a/vmmon-only/include/vm_pagetable.h b/vmmon-only/include/vm_pagetable.h new file mode 100644 index 0000000..1aaac73 --- /dev/null +++ b/vmmon-only/include/vm_pagetable.h @@ -0,0 +1,128 @@ +/********************************************************* + * Copyright (C) 2014,2016 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * vm_pagetable.h -- + * + * Virtual memory page table definitions + */ + +#ifndef _VM_PAGETABLE_H_ +#define _VM_PAGETABLE_H_ + +#define INCLUDE_ALLOW_MODULE +#define INCLUDE_ALLOW_VMK_MODULE +#define INCLUDE_ALLOW_VMKERNEL +#define INCLUDE_ALLOW_DISTRIBUTE +#define INCLUDE_ALLOW_VMCORE +#define INCLUDE_ALLOW_VMMON +#define INCLUDE_ALLOW_USERLEVEL +#include "includeCheck.h" + +#include "vm_basic_types.h" +#include "vm_basic_defs.h" + +/* + * These definitions suit both the x64 and arm64 architectures. In particular + * - At all levels, a page table is a 4KB page containing 512 8-byte entries. + * - Each entry maps 39, 30, 21 and 12 bits at the respective page table + * levels (with standard page sizes). + */ + +typedef uint64 PT_L4E; +typedef uint64 PT_L3E; +typedef uint64 PT_L2E; +typedef uint64 PT_L1E; + +typedef enum { + PT_LEVEL_1 = 1, + PT_LEVEL_2, + PT_LEVEL_3, + PT_LEVEL_4, + PT_MAX_LEVELS = PT_LEVEL_4 +} PT_Level; + +#define PT_PTE_SIZE 8 +#define PT_LEVEL_SHIFT 9 + +#define PT_PTE_PFN_SHIFT PAGE_SHIFT +#define PT_ENTRIES_PER_PT (1 << PT_LEVEL_SHIFT) +#define PT_OFF_MASK (PT_ENTRIES_PER_PT - 1) + +/* + * log2 of the + * address space size (in 4KB pages) covered by a level-_l page table entry. + */ +#define PT_LE_PG_SHIFT(_l) (PT_LEVEL_SHIFT * ((_l) - PT_LEVEL_1)) + +/* Address space size (in 4KB pages) covered by a level-_l page table entry. */ +#define PT_PAGES_PER_LE(_l) ((uint64)1 << PT_LE_PG_SHIFT(_l)) +#define PT_PAGES_PER_L4E PT_PAGES_PER_LE(PT_LEVEL_4) +#define PT_PAGES_PER_L3E PT_PAGES_PER_LE(PT_LEVEL_3) +#define PT_PAGES_PER_L2E PT_PAGES_PER_LE(PT_LEVEL_2) +#define PT_PAGES_PER_L1E PT_PAGES_PER_LE(PT_LEVEL_1) + +/* + * log2 of the + * address space size (in bytes) covered by a level-_l page table entry. + */ +#define PT_LE_SHIFT(_l) (PAGE_SHIFT + PT_LE_PG_SHIFT(_l)) +#define PT_L4E_SHIFT PT_LE_SHIFT(PT_LEVEL_4) +#define PT_L3E_SHIFT PT_LE_SHIFT(PT_LEVEL_3) +#define PT_L2E_SHIFT PT_LE_SHIFT(PT_LEVEL_2) +#define PT_L1E_SHIFT PT_LE_SHIFT(PT_LEVEL_1) + +/* Address space size (in bytes) covered by a level-_l page table entry. */ +#define PT_LE_SIZE(_l) ((uint64)1 << PT_LE_SHIFT(_l)) +#define PT_L4E_SIZE PT_LE_SIZE(PT_LEVEL_4) +#define PT_L3E_SIZE PT_LE_SIZE(PT_LEVEL_3) +#define PT_L2E_SIZE PT_LE_SIZE(PT_LEVEL_2) +#define PT_L1E_SIZE PT_LE_SIZE(PT_LEVEL_1) + +/* + * For a given LA, + * determine the offset (== index) of the level-_l page table entry. + */ +#define PT_LA_2_LOFF(_a, _l) (((_a) >> PT_LE_SHIFT(_l)) & PT_OFF_MASK) +#define PT_LA_2_L4OFF(_a) PT_LA_2_LOFF(_a, PT_LEVEL_4) +#define PT_LA_2_L3OFF(_a) PT_LA_2_LOFF(_a, PT_LEVEL_3) +#define PT_LA_2_L2OFF(_a) PT_LA_2_LOFF(_a, PT_LEVEL_2) +#define PT_LA_2_L1OFF(_a) PT_LA_2_LOFF(_a, PT_LEVEL_1) + +/* + * For a given LPN, + * determine the offset (== index) of the level-_l page table entry. + */ +#define PT_LPN_2_LOFF(_a, _l) (((_a) >> PT_LE_PG_SHIFT(_l)) & PT_OFF_MASK) +#define PT_LPN_2_L4OFF(_a) PT_LPN_2_LOFF(_a, PT_LEVEL_4) +#define PT_LPN_2_L3OFF(_a) PT_LPN_2_LOFF(_a, PT_LEVEL_3) +#define PT_LPN_2_L2OFF(_a) PT_LPN_2_LOFF(_a, PT_LEVEL_2) +#define PT_LPN_2_L1OFF(_a) PT_LPN_2_LOFF(_a, PT_LEVEL_1) + +/* Size (in bytes) of a 1GB page. */ +#define PT_1G_PAGE_SIZE PT_L3E_SIZE +#define PT_1G_PAGE_MASK (PT_L3E_SIZE - 1) + +/* Size (in bytes) of a 2MB page. */ +#define PT_2M_PAGE_SIZE PT_L2E_SIZE +#define PT_2M_PAGE_MASK (PT_L2E_SIZE - 1) + +#define PT_LE_NXT_PG(_a, _l) ((((_a) >> PT_LE_PG_SHIFT(_l)) + 1) \ + << PT_LE_PG_SHIFT(_l)) + +#endif diff --git a/vmmon-only/include/vmm_constants.h b/vmmon-only/include/vmm_constants.h index 54ad672..b3e820d 100644 --- a/vmmon-only/include/vmm_constants.h +++ b/vmmon-only/include/vmm_constants.h @@ -42,7 +42,4 @@ #define VMM_PANIC_MSG_SIZE 256 -#define VMM_CPL0_STACK 1 -#define VMM_CPL1_STACK 2 - #endif diff --git a/vmmon-only/include/vmmem_shared.h b/vmmon-only/include/vmmem_shared.h index 4f9fefc..291d170 100644 --- a/vmmon-only/include/vmmem_shared.h +++ b/vmmon-only/include/vmmem_shared.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2000-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2000-2015,2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -40,7 +40,7 @@ #define VMMEM_FLAG_BIT(x) (1 << (x)) #define VMMEM_ANON_LOW_MEM VMMEM_FLAG_BIT(0) -#define VMMEM_ANON_LARGE_PAGE VMMEM_FLAG_BIT(1) +#define VMMEM_ANON_CONTIG VMMEM_FLAG_BIT(1) #define VMMEM_ANON_CAN_FAIL VMMEM_FLAG_BIT(2) #define VMMEM_ANON_USE_PREALLOC VMMEM_FLAG_BIT(3) #define VMMEM_ANON_IOABLE_PAGE VMMEM_FLAG_BIT(4) @@ -64,10 +64,14 @@ #define VMMEM_PLATFORM_COW VMMEM_FLAG_BIT(2) #define VMMEM_PLATFORM_EXPOSED_TO_VMM VMMEM_FLAG_BIT(3) #define VMMEM_PLATFORM_P2M_UPDATE_PENDING VMMEM_FLAG_BIT(4) -#define VMMEM_PLATFORM_TRY_COW_SUCCESS VMMEM_FLAG_BIT(5) -#define VMMEM_PLATFORM_DIRTY VMMEM_FLAG_BIT(6) -#define VMMEM_PLATFORM_BACKED_LARGE VMMEM_FLAG_BIT(7) +#define VMMEM_PLATFORM_DIRTY VMMEM_FLAG_BIT(5) +#define VMMEM_PLATFORM_IS_2M_PAGE VMMEM_FLAG_BIT(6) +#define VMMEM_PLATFORM_IS_1G_PAGE VMMEM_FLAG_BIT(7) #define VMMEM_PLATFORM_LARGE_RETRY VMMEM_FLAG_BIT(8) +#define VMMEM_PLATFORM_TRY_COW_SUCCESS VMMEM_FLAG_BIT(9) + +#define VMMEM_PLATFORM_BACKED_LARGE (VMMEM_PLATFORM_IS_2M_PAGE | \ + VMMEM_PLATFORM_IS_1G_PAGE) #define MAX_PLATFORM_PAGE_INFO_PAGES 240 diff --git a/vmmon-only/include/vmware.h b/vmmon-only/include/vmware.h index fa6881f..4e2fa8b 100644 --- a/vmmon-only/include/vmware.h +++ b/vmmon-only/include/vmware.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmmon-only/include/vmware_pack_begin.h b/vmmon-only/include/vmware_pack_begin.h index d0eb4bd..fe0aec8 100644 --- a/vmmon-only/include/vmware_pack_begin.h +++ b/vmmon-only/include/vmware_pack_begin.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2002-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2002-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmmon-only/include/vmware_pack_end.h b/vmmon-only/include/vmware_pack_end.h index f02e215..7557559 100644 --- a/vmmon-only/include/vmware_pack_end.h +++ b/vmmon-only/include/vmware_pack_end.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2002-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2002-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmmon-only/include/vmware_pack_init.h b/vmmon-only/include/vmware_pack_init.h index c401d66..4aa426e 100644 --- a/vmmon-only/include/vmware_pack_init.h +++ b/vmmon-only/include/vmware_pack_init.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2002-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2002-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmmon-only/include/x86_basic_defs.h b/vmmon-only/include/x86_basic_defs.h index 7a5ba99..abfb0b8 100644 --- a/vmmon-only/include/x86_basic_defs.h +++ b/vmmon-only/include/x86_basic_defs.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2006-2013 VMware, Inc. All rights reserved. + * Copyright (C) 2006-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -54,7 +54,20 @@ #define CR0_NW 0x20000000 #define CR0_CD 0x40000000 #define CR0_PG 0x80000000 + +#define CR0_CACHE_CONTROL (CR0_CD | CR0_NW) + #define CR0_RESERVED CONST64U(0xffffffff1ffaffc0) +/* + * Note: The "Not Reserved" bits in CR0 are: + * PG, CD, NW, AM, WP, NE, ET, TS, EM, MP, PE + * | | | | | | + * | | +---------------+---+---+---> CR0_MUTABLE + * | | + * +---+--> CR0_CACHE_CONTROL + * + * (CR0_MUTABLE is defined in vmkernel/private/x86/cpu.h) + */ #define CR3_PWT 0x00000008 #define CR3_PCD 0x00000010 @@ -83,8 +96,8 @@ #define CR4_OSXSAVE 0x00040000 #define CR4_SMEP 0x00100000 #define CR4_SMAP 0x00200000 -/* Removing a bit from CR4_RESERVED causes Task_Switch to leave the bit set. */ -#define CR4_RESERVED CONST64U(0xffffffffffc89800) +#define CR4_PKE 0x00400000 +#define CR4_RESERVED CONST64U(0xffffffffff889800) #define CR8_RESERVED CONST64U(0xfffffffffffffff0) /* @@ -186,8 +199,9 @@ #define EXC_MF 16 #define EXC_AC 17 #define EXC_MC 18 -#define EXC_XF 19 /* SIMD exception. */ -#define EXC_SX 30 /* Security exception (SVM only). */ +#define EXC_XF 19 // SIMD exception. +#define EXC_VE 20 // Virtualization exception - VT only. +#define EXC_SX 30 // Security exception (SVM only). /* * eflag/rflag definitions. @@ -238,5 +252,12 @@ typedef enum x86_FLAGS { EFLAGS__4 = 0x7fffffff /* ensure 4 byte encoding */ } x86_FLAGS; +/* + * MPX bound configuration registers + */ +#define BNDCFG_EN 0x00000001 +#define BNDCFG_BNDPRSV 0x00000002 +#define BNDCFG_RSVD 0x00000ffc +#define BNDCFG_BDBASE CONST64U(0xfffffffffffff000) #endif // ifndef _VM_BASIC_DEFS_H_ diff --git a/vmmon-only/include/x86apic.h b/vmmon-only/include/x86apic.h index 325b6d0..791ca8d 100644 --- a/vmmon-only/include/x86apic.h +++ b/vmmon-only/include/x86apic.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998 VMware, Inc. All rights reserved. + * Copyright (C) 1998,2015 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -45,6 +45,7 @@ #define X2APIC_VERSION_0x15 0x15 #define APIC_MAXLVT_0x4 0x4 #define XAPIC_MAXLVT_0x5 0x5 +#define XAPIC_MAXLVT_0x6 0x6 // Intel only: Nehalem (1A) and onward #define APIC_VERSION_MASK 0xff #define APIC_MAX_LVT_MASK 0xff #define APIC_MAX_LVT_SHIFT 16 diff --git a/vmmon-only/include/x86cpuid.h b/vmmon-only/include/x86cpuid.h index 2a237ac..251e526 100644 --- a/vmmon-only/include/x86cpuid.h +++ b/vmmon-only/include/x86cpuid.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -37,6 +37,11 @@ #include "x86vendor.h" #include "vm_assert.h" +#if defined __cplusplus +extern "C" { +#endif + + /* * The linux kernel's ptrace.h stupidly defines the bare * EAX/EBX/ECX/EDX, which wrecks havoc with our preprocessor tricks. @@ -59,7 +64,10 @@ typedef union CPUIDRegsUnion { * Results of calling cpuid(eax, ecx) on all host logical CPU. */ #ifdef _MSC_VER +// TODO: Move this under the push #pragma warning (disable :4200) // non-std extension: zero-sized array in struct +#pragma warning (push) +#pragma warning (disable :4100) // unreferenced parameters #endif typedef @@ -91,56 +99,73 @@ CPUIDQuery; * CPUID levels the monitor caches. * * The first parameter defines whether the level has its default masks - * generated from the values in this file. Any level which is marked - * as FALSE here *must* have all monitor support types set to NA. A - * static assert in lib/cpuidcompat/cpuidcompat.c will check this. + * generated from the values in this file. Any level which is marked as FALSE + * here *must* have all monitor support types set to NA. A static assert in + * lib/cpuidcompat/cpuidcompat.c will check this. * - * The fourth parameter is a "sub leaf count", where 0 means that ecx - * is ignored, otherwise is the count of sub-leaves cached/supported. + * The second parameter is the "short name" of the level. It's mainly used for + * token concatenation in various macros. + * + * The third parameter is the actual numeric value of that level (the EAX input + * value). + * + * The fourth parameter is a "subleaf count", where 0 means that ecx is + * ignored, otherwise is the count of sub-leaves. + * + * The fifth parameter is the first hardware version that is *aware* of the + * CPUID level (0 = existed since dawn of time), even though we may not expose + * this level or parts of it to guest. */ #define CPUID_CACHED_LEVELS \ - CPUIDLEVEL(TRUE, 0, 0, 0) \ - CPUIDLEVEL(TRUE, 1, 1, 0) \ - CPUIDLEVEL(FALSE, 2, 2, 0) \ - CPUIDLEVEL(FALSE, 4, 4, 7) \ - CPUIDLEVEL(FALSE, 5, 5, 0) \ - CPUIDLEVEL(FALSE, 6, 6, 0) \ - CPUIDLEVEL(TRUE, 7, 7, 1) \ - CPUIDLEVEL(FALSE, A, 0xA, 0) \ - CPUIDLEVEL(FALSE, B, 0xB, 2) \ - CPUIDLEVEL(TRUE, D, 0xD, 4) \ - CPUIDLEVEL(FALSE, 12, 0x12, 4) \ - CPUIDLEVEL(FALSE, 400, 0x40000000, 0) \ - CPUIDLEVEL(FALSE, 401, 0x40000001, 0) \ - CPUIDLEVEL(FALSE, 402, 0x40000002, 0) \ - CPUIDLEVEL(FALSE, 403, 0x40000003, 0) \ - CPUIDLEVEL(FALSE, 404, 0x40000004, 0) \ - CPUIDLEVEL(FALSE, 405, 0x40000005, 0) \ - CPUIDLEVEL(FALSE, 406, 0x40000006, 0) \ - CPUIDLEVEL(FALSE, 410, 0x40000010, 0) \ - CPUIDLEVEL(FALSE, 80, 0x80000000, 0) \ - CPUIDLEVEL(TRUE, 81, 0x80000001, 0) \ - CPUIDLEVEL(FALSE, 82, 0x80000002, 0) \ - CPUIDLEVEL(FALSE, 83, 0x80000003, 0) \ - CPUIDLEVEL(FALSE, 84, 0x80000004, 0) \ - CPUIDLEVEL(FALSE, 85, 0x80000005, 0) \ - CPUIDLEVEL(FALSE, 86, 0x80000006, 0) \ - CPUIDLEVEL(FALSE, 87, 0x80000007, 0) \ - CPUIDLEVEL(FALSE, 88, 0x80000008, 0) \ - CPUIDLEVEL(TRUE, 8A, 0x8000000A, 0) \ - CPUIDLEVEL(FALSE, 819, 0x80000019, 0) \ - CPUIDLEVEL(FALSE, 81A, 0x8000001A, 0) \ - CPUIDLEVEL(FALSE, 81B, 0x8000001B, 0) \ - CPUIDLEVEL(FALSE, 81C, 0x8000001C, 0) \ - CPUIDLEVEL(FALSE, 81D, 0x8000001D, 5) \ - CPUIDLEVEL(FALSE, 81E, 0x8000001E, 0) + CPUIDLEVEL(TRUE, 0, 0, 0, 0) \ + CPUIDLEVEL(TRUE, 1, 1, 0, 0) \ + CPUIDLEVEL(FALSE, 2, 2, 0, 0) \ + CPUIDLEVEL(FALSE, 4, 4, 7, 0) \ + CPUIDLEVEL(FALSE, 5, 5, 0, 0) \ + CPUIDLEVEL(TRUE, 6, 6, 0, 0) \ + CPUIDLEVEL(TRUE, 7, 7, 1, 0) \ + CPUIDLEVEL(FALSE, A, 0xA, 0, 0) \ + CPUIDLEVEL(FALSE, B, 0xB, 2, 0) \ + CPUIDLEVEL(TRUE, D, 0xD, 10, 0) \ + CPUIDLEVEL(TRUE, F, 0xF, 2, 13) \ + CPUIDLEVEL(TRUE, 10, 0x10, 2, 13) \ + CPUIDLEVEL(TRUE, 12, 0x12, 4, 13) \ + CPUIDLEVEL(TRUE, 14, 0x14, 2, 13) \ + CPUIDLEVEL(TRUE, 15, 0x15, 0, 13) \ + CPUIDLEVEL(TRUE, 16, 0x16, 0, 13) \ + CPUIDLEVEL(TRUE, 17, 0x17, 4, 14) \ + CPUIDLEVEL(FALSE, 400, 0x40000000, 0, 0) \ + CPUIDLEVEL(FALSE, 401, 0x40000001, 0, 0) \ + CPUIDLEVEL(FALSE, 402, 0x40000002, 0, 0) \ + CPUIDLEVEL(FALSE, 403, 0x40000003, 0, 0) \ + CPUIDLEVEL(FALSE, 404, 0x40000004, 0, 0) \ + CPUIDLEVEL(FALSE, 405, 0x40000005, 0, 0) \ + CPUIDLEVEL(FALSE, 406, 0x40000006, 0, 0) \ + CPUIDLEVEL(FALSE, 410, 0x40000010, 0, 0) \ + CPUIDLEVEL(FALSE, 80, 0x80000000, 0, 0) \ + CPUIDLEVEL(TRUE, 81, 0x80000001, 0, 0) \ + CPUIDLEVEL(FALSE, 82, 0x80000002, 0, 0) \ + CPUIDLEVEL(FALSE, 83, 0x80000003, 0, 0) \ + CPUIDLEVEL(FALSE, 84, 0x80000004, 0, 0) \ + CPUIDLEVEL(FALSE, 85, 0x80000005, 0, 0) \ + CPUIDLEVEL(FALSE, 86, 0x80000006, 0, 0) \ + CPUIDLEVEL(FALSE, 87, 0x80000007, 0, 0) \ + CPUIDLEVEL(TRUE, 88, 0x80000008, 0, 0) \ + CPUIDLEVEL(TRUE, 8A, 0x8000000A, 0, 0) \ + CPUIDLEVEL(FALSE, 819, 0x80000019, 0, 0) \ + CPUIDLEVEL(FALSE, 81A, 0x8000001A, 0, 0) \ + CPUIDLEVEL(FALSE, 81B, 0x8000001B, 0, 0) \ + CPUIDLEVEL(FALSE, 81C, 0x8000001C, 0, 0) \ + CPUIDLEVEL(FALSE, 81D, 0x8000001D, 5, 0) \ + CPUIDLEVEL(FALSE, 81E, 0x8000001E, 0, 0) \ + CPUIDLEVEL(TRUE, 81F, 0x8000001F, 0, 14) #define CPUID_ALL_LEVELS CPUID_CACHED_LEVELS /* Define cached CPUID levels in the form: CPUID_LEVEL_ */ typedef enum { -#define CPUIDLEVEL(t, s, v, c) CPUID_LEVEL_##s, +#define CPUIDLEVEL(t, s, v, c, h) CPUID_LEVEL_##s, CPUID_CACHED_LEVELS #undef CPUIDLEVEL CPUID_NUM_CACHED_LEVELS @@ -148,7 +173,7 @@ typedef enum { /* Enum to translate between shorthand name and actual CPUID level value. */ enum { -#define CPUIDLEVEL(t, s, v, c) CPUID_LEVEL_VAL_##s = v, +#define CPUIDLEVEL(t, s, v, c, h) CPUID_LEVEL_VAL_##s = v, CPUID_ALL_LEVELS #undef CPUIDLEVEL }; @@ -159,6 +184,8 @@ enum { #define CPUID_PROCESSOR_TOPOLOGY 4 #define CPUID_MWAIT_FEATURES 5 #define CPUID_XSAVE_FEATURES 0xd +#define CPUID_SGX_FEATURES 0x12 +#define CPUID_PT_FEATURES 0x14 #define CPUID_HYPERVISOR_LEVEL_0 0x40000000 #define CPUID_SVM_FEATURES 0x8000000a @@ -196,28 +223,41 @@ typedef enum { #define CPUID_VIA_VENDOR_STRING_FIXED "CentaurHauls" /* - * FIELD can be defined to process the CPUID information provided - * in the following CPUID_FIELD_DATA macro. The first parameter is - * the CPUID level of the feature (must be defined in - * CPUID_ALL_LEVELS, above. The second parameter is the CPUID result - * register in which the field is returned (defined in CPUID_REGS). - * The third field is the vendor(s) this feature applies to. "COMMON" - * means all vendors apply. UNKNOWN may not be used here. The fourth - * and fifth parameters are the bit position of the field and the - * width, respectively. The sixth is the text name of the field. + * FIELD can be defined to process the CPUID information provided in the + * following CPUID_FIELD_DATA macro. * - * The seventh parameters specifies the monitor support - * characteristics for this field. The value must be a valid - * CpuidFieldSupported value (omitting CPUID_FIELD_SUPPORT_ for - * convenience). The meaning of those values are described below. + * The first parameter is the CPUID level of the feature (must be defined in + * CPUID_ALL_LEVELS, above). * - * The eighth parameter describes whether the feature is capable of - * being used by usermode code (TRUE), or just CPL0 kernel code - * (FALSE). + * The second parameter is the CPUID sub-level (subleaf) of the feature. Please + * make sure here the number is consistent with the "subleaf count" in + * CPUIDLEVEL macro. I.e., if a feature is being added to a _new_ subleaf, + * update the subleaf count above as well. * - * FLAG is defined identically to FIELD, but its accessors are more - * appropriate for 1-bit flags, and compile-time asserts enforce that - * the size is 1 bit wide. + * The third parameter is the result register. + * + * The fourth and fifth parameters are the bit position of the field and the + * width, respectively. + * + * The sixth is the name of the field. + * + * The seventh parameter specifies the monitor support characteristics for + * this field. The value must be a valid CpuidFieldSupported value (omitting + * CPUID_FIELD_SUPPORT_ for convenience). The meaning of those values are + * described below. + * + * The eighth parameter specifies the first virtual hardware version that + * implements the field (if 7th field is YES or ANY), or 0 (if 7th field is + * NO or NA). The field's hardware version must match the version in + * defaultMasks (cpuidcompat.c) if defined there, and must be less than or + * equal to the version of the cpuid leaf it's in. + * + * The eighth parameter describes whether the feature is capable of being used + * by usermode code (TRUE), or just CPL0 kernel code (FALSE). + * + * FLAG is defined identically to FIELD, but its accessors are more appropriate + * for 1-bit flags, and compile-time asserts enforce that the size is 1 bit + * wide. */ @@ -273,580 +313,775 @@ typedef enum { CPUID_NUM_FIELD_SUPPORTEDS } CpuidFieldSupported; -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_0 \ -FIELD( 0, 0, EAX, 0, 32, NUMLEVELS, ANY, FALSE) \ -FIELD( 0, 0, EBX, 0, 32, VENDOR1, YES, TRUE) \ -FIELD( 0, 0, ECX, 0, 32, VENDOR3, YES, TRUE) \ -FIELD( 0, 0, EDX, 0, 32, VENDOR2, YES, TRUE) +FIELD( 0, 0, EAX, 0, 32, NUMLEVELS, ANY, 4, FALSE) \ +FIELD( 0, 0, EBX, 0, 32, VENDOR1, YES, 4, TRUE) \ +FIELD( 0, 0, ECX, 0, 32, VENDOR3, YES, 4, TRUE) \ +FIELD( 0, 0, EDX, 0, 32, VENDOR2, YES, 4, TRUE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_1 \ -FIELD( 1, 0, EAX, 0, 4, STEPPING, ANY, FALSE) \ -FIELD( 1, 0, EAX, 4, 4, MODEL, ANY, FALSE) \ -FIELD( 1, 0, EAX, 8, 4, FAMILY, YES, FALSE) \ -FIELD( 1, 0, EAX, 12, 2, TYPE, ANY, FALSE) \ -FIELD( 1, 0, EAX, 16, 4, EXTENDED_MODEL, ANY, FALSE) \ -FIELD( 1, 0, EAX, 20, 8, EXTENDED_FAMILY, YES, FALSE) \ -FIELD( 1, 0, EBX, 0, 8, BRAND_ID, ANY, FALSE) \ -FIELD( 1, 0, EBX, 8, 8, CLFL_SIZE, ANY, FALSE) \ -FIELD( 1, 0, EBX, 16, 8, LCPU_COUNT, ANY, FALSE) \ -FIELD( 1, 0, EBX, 24, 8, APICID, ANY, FALSE) \ -FLAG( 1, 0, ECX, 0, 1, SSE3, YES, TRUE) \ -FLAG( 1, 0, ECX, 1, 1, PCLMULQDQ, YES, TRUE) \ -FLAG( 1, 0, ECX, 2, 1, DTES64, NO, FALSE) \ -FLAG( 1, 0, ECX, 3, 1, MWAIT, YES, FALSE) \ -FLAG( 1, 0, ECX, 4, 1, DSCPL, NO, FALSE) \ -FLAG( 1, 0, ECX, 5, 1, VMX, YES, FALSE) \ -FLAG( 1, 0, ECX, 6, 1, SMX, NO, FALSE) \ -FLAG( 1, 0, ECX, 7, 1, EIST, NO, FALSE) \ -FLAG( 1, 0, ECX, 8, 1, TM2, NO, FALSE) \ -FLAG( 1, 0, ECX, 9, 1, SSSE3, YES, TRUE) \ -FLAG( 1, 0, ECX, 10, 1, CNXTID, NO, FALSE) \ -FLAG( 1, 0, ECX, 11, 1, SDBG, NO, FALSE) \ -FLAG( 1, 0, ECX, 12, 1, FMA, YES, TRUE) \ -FLAG( 1, 0, ECX, 13, 1, CMPXCHG16B, YES, TRUE) \ -FLAG( 1, 0, ECX, 14, 1, xTPR, NO, FALSE) \ -FLAG( 1, 0, ECX, 15, 1, PDCM, NO, FALSE) \ -FLAG( 1, 0, ECX, 17, 1, PCID, YES, FALSE) \ -FLAG( 1, 0, ECX, 18, 1, DCA, NO, FALSE) \ -FLAG( 1, 0, ECX, 19, 1, SSE41, YES, TRUE) \ -FLAG( 1, 0, ECX, 20, 1, SSE42, YES, TRUE) \ -FLAG( 1, 0, ECX, 21, 1, x2APIC, ANY, FALSE) \ -FLAG( 1, 0, ECX, 22, 1, MOVBE, YES, TRUE) \ -FLAG( 1, 0, ECX, 23, 1, POPCNT, YES, TRUE) \ -FLAG( 1, 0, ECX, 24, 1, TSC_DEADLINE, ANY, FALSE) \ -FLAG( 1, 0, ECX, 25, 1, AES, YES, TRUE) \ -FLAG( 1, 0, ECX, 26, 1, XSAVE, YES, FALSE) \ -FLAG( 1, 0, ECX, 27, 1, OSXSAVE, ANY, FALSE) \ -FLAG( 1, 0, ECX, 28, 1, AVX, YES, FALSE) \ -FLAG( 1, 0, ECX, 29, 1, F16C, YES, TRUE) \ -FLAG( 1, 0, ECX, 30, 1, RDRAND, YES, TRUE) \ -FLAG( 1, 0, ECX, 31, 1, HYPERVISOR, ANY, TRUE) \ -FLAG( 1, 0, EDX, 0, 1, FPU, YES, TRUE) \ -FLAG( 1, 0, EDX, 1, 1, VME, YES, FALSE) \ -FLAG( 1, 0, EDX, 2, 1, DE, YES, FALSE) \ -FLAG( 1, 0, EDX, 3, 1, PSE, YES, FALSE) \ -FLAG( 1, 0, EDX, 4, 1, TSC, YES, TRUE) \ -FLAG( 1, 0, EDX, 5, 1, MSR, YES, FALSE) \ -FLAG( 1, 0, EDX, 6, 1, PAE, YES, FALSE) \ -FLAG( 1, 0, EDX, 7, 1, MCE, YES, FALSE) \ -FLAG( 1, 0, EDX, 8, 1, CX8, YES, TRUE) \ -FLAG( 1, 0, EDX, 9, 1, APIC, ANY, FALSE) \ -FLAG( 1, 0, EDX, 11, 1, SEP, YES, TRUE) \ -FLAG( 1, 0, EDX, 12, 1, MTRR, YES, FALSE) \ -FLAG( 1, 0, EDX, 13, 1, PGE, YES, FALSE) \ -FLAG( 1, 0, EDX, 14, 1, MCA, YES, FALSE) \ -FLAG( 1, 0, EDX, 15, 1, CMOV, YES, TRUE) \ -FLAG( 1, 0, EDX, 16, 1, PAT, YES, FALSE) \ -FLAG( 1, 0, EDX, 17, 1, PSE36, YES, FALSE) \ -FLAG( 1, 0, EDX, 18, 1, PSN, YES, FALSE) \ -FLAG( 1, 0, EDX, 19, 1, CLFSH, YES, TRUE) \ -FLAG( 1, 0, EDX, 21, 1, DS, YES, FALSE) \ -FLAG( 1, 0, EDX, 22, 1, ACPI, ANY, FALSE) \ -FLAG( 1, 0, EDX, 23, 1, MMX, YES, TRUE) \ -FLAG( 1, 0, EDX, 24, 1, FXSR, YES, TRUE) \ -FLAG( 1, 0, EDX, 25, 1, SSE, YES, TRUE) \ -FLAG( 1, 0, EDX, 26, 1, SSE2, YES, TRUE) \ -FLAG( 1, 0, EDX, 27, 1, SS, YES, FALSE) \ -FLAG( 1, 0, EDX, 28, 1, HTT, ANY, FALSE) \ -FLAG( 1, 0, EDX, 29, 1, TM, NO, FALSE) \ -FLAG( 1, 0, EDX, 30, 1, IA64, NO, FALSE) \ -FLAG( 1, 0, EDX, 31, 1, PBE, NO, FALSE) +FIELD( 1, 0, EAX, 0, 4, STEPPING, ANY, 4, FALSE) \ +FIELD( 1, 0, EAX, 4, 4, MODEL, ANY, 4, FALSE) \ +FIELD( 1, 0, EAX, 8, 4, FAMILY, YES, 4, FALSE) \ +FIELD( 1, 0, EAX, 12, 2, TYPE, ANY, 4, FALSE) \ +FIELD( 1, 0, EAX, 16, 4, EXTENDED_MODEL, ANY, 4, FALSE) \ +FIELD( 1, 0, EAX, 20, 8, EXTENDED_FAMILY, YES, 4, FALSE) \ +FIELD( 1, 0, EBX, 0, 8, BRAND_ID, ANY, 4, FALSE) \ +FIELD( 1, 0, EBX, 8, 8, CLFL_SIZE, ANY, 4, FALSE) \ +FIELD( 1, 0, EBX, 16, 8, LCPU_COUNT, ANY, 4, FALSE) \ +FIELD( 1, 0, EBX, 24, 8, APICID, ANY, 4, FALSE) \ +FLAG( 1, 0, ECX, 0, 1, SSE3, YES, 4, TRUE) \ +FLAG( 1, 0, ECX, 1, 1, PCLMULQDQ, YES, 7, TRUE) \ +FLAG( 1, 0, ECX, 2, 1, DTES64, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 3, 1, MWAIT, YES, 4, FALSE) \ +FLAG( 1, 0, ECX, 4, 1, DSCPL, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 5, 1, VMX, YES, 4, FALSE) \ +FLAG( 1, 0, ECX, 6, 1, SMX, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 7, 1, EIST, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 8, 1, TM2, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 9, 1, SSSE3, YES, 4, TRUE) \ +FLAG( 1, 0, ECX, 10, 1, CNXTID, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 11, 1, SDBG, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 12, 1, FMA, YES, 8, TRUE) \ +FLAG( 1, 0, ECX, 13, 1, CMPXCHG16B, YES, 4, TRUE) \ +FLAG( 1, 0, ECX, 14, 1, xTPR, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 15, 1, PDCM, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 17, 1, PCID, YES, 8, FALSE) \ +FLAG( 1, 0, ECX, 18, 1, DCA, NO, 0, FALSE) \ +FLAG( 1, 0, ECX, 19, 1, SSE41, YES, 4, TRUE) \ +FLAG( 1, 0, ECX, 20, 1, SSE42, YES, 4, TRUE) \ +FLAG( 1, 0, ECX, 21, 1, x2APIC, ANY, 9, FALSE) \ +FLAG( 1, 0, ECX, 22, 1, MOVBE, YES, 7, TRUE) \ +FLAG( 1, 0, ECX, 23, 1, POPCNT, YES, 4, TRUE) \ +FLAG( 1, 0, ECX, 24, 1, TSC_DEADLINE, ANY, 11, FALSE) \ +FLAG( 1, 0, ECX, 25, 1, AES, YES, 7, TRUE) \ +FLAG( 1, 0, ECX, 26, 1, XSAVE, YES, 8, FALSE) \ +FLAG( 1, 0, ECX, 27, 1, OSXSAVE, ANY, 8, FALSE) \ +FLAG( 1, 0, ECX, 28, 1, AVX, YES, 8, FALSE) \ +FLAG( 1, 0, ECX, 29, 1, F16C, YES, 9, TRUE) \ +FLAG( 1, 0, ECX, 30, 1, RDRAND, YES, 9, TRUE) \ +FLAG( 1, 0, ECX, 31, 1, HYPERVISOR, ANY, 4, TRUE) \ +FLAG( 1, 0, EDX, 0, 1, FPU, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 1, 1, VME, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 2, 1, DE, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 3, 1, PSE, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 4, 1, TSC, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 5, 1, MSR, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 6, 1, PAE, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 7, 1, MCE, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 8, 1, CX8, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 9, 1, APIC, ANY, 4, FALSE) \ +FLAG( 1, 0, EDX, 11, 1, SEP, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 12, 1, MTRR, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 13, 1, PGE, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 14, 1, MCA, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 15, 1, CMOV, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 16, 1, PAT, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 17, 1, PSE36, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 18, 1, PSN, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 19, 1, CLFSH, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 21, 1, DS, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 22, 1, ACPI, ANY, 4, FALSE) \ +FLAG( 1, 0, EDX, 23, 1, MMX, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 24, 1, FXSR, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 25, 1, SSE, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 26, 1, SSE2, YES, 4, TRUE) \ +FLAG( 1, 0, EDX, 27, 1, SS, YES, 4, FALSE) \ +FLAG( 1, 0, EDX, 28, 1, HTT, ANY, 7, FALSE) \ +FLAG( 1, 0, EDX, 29, 1, TM, NO, 0, FALSE) \ +FLAG( 1, 0, EDX, 30, 1, IA64, NO, 0, FALSE) \ +FLAG( 1, 0, EDX, 31, 1, PBE, NO, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_2 \ -FIELD( 2, 0, EAX, 0, 8, LEAF2_COUNT, NA, FALSE) \ -FIELD( 2, 0, EAX, 8, 8, LEAF2_CACHE1, NA, FALSE) \ -FIELD( 2, 0, EAX, 16, 8, LEAF2_CACHE2, NA, FALSE) \ -FIELD( 2, 0, EAX, 24, 8, LEAF2_CACHE3, NA, FALSE) \ -FIELD( 2, 0, EBX, 0, 8, LEAF2_CACHE4, NA, FALSE) \ -FIELD( 2, 0, EBX, 8, 8, LEAF2_CACHE5, NA, FALSE) \ -FIELD( 2, 0, EBX, 16, 8, LEAF2_CACHE6, NA, FALSE) \ -FIELD( 2, 0, EBX, 24, 8, LEAF2_CACHE7, NA, FALSE) \ -FIELD( 2, 0, ECX, 0, 8, LEAF2_CACHE8, NA, FALSE) \ -FIELD( 2, 0, ECX, 8, 8, LEAF2_CACHE9, NA, FALSE) \ -FIELD( 2, 0, ECX, 16, 8, LEAF2_CACHE10, NA, FALSE) \ -FIELD( 2, 0, ECX, 24, 8, LEAF2_CACHE11, NA, FALSE) \ -FIELD( 2, 0, EDX, 0, 8, LEAF2_CACHE12, NA, FALSE) \ -FIELD( 2, 0, EDX, 8, 8, LEAF2_CACHE13, NA, FALSE) \ -FIELD( 2, 0, EDX, 16, 8, LEAF2_CACHE14, NA, FALSE) \ -FIELD( 2, 0, EDX, 24, 8, LEAF2_CACHE15, NA, FALSE) \ +FIELD( 2, 0, EAX, 0, 8, LEAF2_COUNT, NA, 0, FALSE) \ +FIELD( 2, 0, EAX, 8, 8, LEAF2_CACHE1, NA, 0, FALSE) \ +FIELD( 2, 0, EAX, 16, 8, LEAF2_CACHE2, NA, 0, FALSE) \ +FIELD( 2, 0, EAX, 24, 8, LEAF2_CACHE3, NA, 0, FALSE) \ +FIELD( 2, 0, EBX, 0, 8, LEAF2_CACHE4, NA, 0, FALSE) \ +FIELD( 2, 0, EBX, 8, 8, LEAF2_CACHE5, NA, 0, FALSE) \ +FIELD( 2, 0, EBX, 16, 8, LEAF2_CACHE6, NA, 0, FALSE) \ +FIELD( 2, 0, EBX, 24, 8, LEAF2_CACHE7, NA, 0, FALSE) \ +FIELD( 2, 0, ECX, 0, 8, LEAF2_CACHE8, NA, 0, FALSE) \ +FIELD( 2, 0, ECX, 8, 8, LEAF2_CACHE9, NA, 0, FALSE) \ +FIELD( 2, 0, ECX, 16, 8, LEAF2_CACHE10, NA, 0, FALSE) \ +FIELD( 2, 0, ECX, 24, 8, LEAF2_CACHE11, NA, 0, FALSE) \ +FIELD( 2, 0, EDX, 0, 8, LEAF2_CACHE12, NA, 0, FALSE) \ +FIELD( 2, 0, EDX, 8, 8, LEAF2_CACHE13, NA, 0, FALSE) \ +FIELD( 2, 0, EDX, 16, 8, LEAF2_CACHE14, NA, 0, FALSE) \ +FIELD( 2, 0, EDX, 24, 8, LEAF2_CACHE15, NA, 0, FALSE) \ -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_4 \ -FIELD( 4, 0, EAX, 0, 5, LEAF4_CACHE_TYPE, NA, FALSE) \ -FIELD( 4, 0, EAX, 5, 3, LEAF4_CACHE_LEVEL, NA, FALSE) \ -FLAG( 4, 0, EAX, 8, 1, LEAF4_CACHE_SELF_INIT, NA, FALSE) \ -FLAG( 4, 0, EAX, 9, 1, LEAF4_CACHE_FULLY_ASSOC, NA, FALSE) \ -FIELD( 4, 0, EAX, 14, 12, LEAF4_CACHE_NUMHT_SHARING, NA, FALSE) \ -FIELD( 4, 0, EAX, 26, 6, LEAF4_CORE_COUNT, NA, FALSE) \ -FIELD( 4, 0, EBX, 0, 12, LEAF4_CACHE_LINE, NA, FALSE) \ -FIELD( 4, 0, EBX, 12, 10, LEAF4_CACHE_PART, NA, FALSE) \ -FIELD( 4, 0, EBX, 22, 10, LEAF4_CACHE_WAYS, NA, FALSE) \ -FIELD( 4, 0, ECX, 0, 32, LEAF4_CACHE_SETS, NA, FALSE) \ -FLAG( 4, 0, EDX, 0, 1, LEAF4_CACHE_WBINVD_NOT_GUARANTEED, NA, FALSE) \ -FLAG( 4, 0, EDX, 1, 1, LEAF4_CACHE_IS_INCLUSIVE, NA, FALSE) \ -FLAG( 4, 0, EDX, 2, 1, LEAF4_CACHE_COMPLEX_INDEXING, NA, FALSE) +FIELD( 4, 0, EAX, 0, 5, LEAF4_CACHE_TYPE, NA, 0, FALSE) \ +FIELD( 4, 0, EAX, 5, 3, LEAF4_CACHE_LEVEL, NA, 0, FALSE) \ +FLAG( 4, 0, EAX, 8, 1, LEAF4_CACHE_SELF_INIT, NA, 0, FALSE) \ +FLAG( 4, 0, EAX, 9, 1, LEAF4_CACHE_FULLY_ASSOC, NA, 0, FALSE) \ +FIELD( 4, 0, EAX, 14, 12, LEAF4_CACHE_NUMHT_SHARING, NA, 0, FALSE) \ +FIELD( 4, 0, EAX, 26, 6, LEAF4_CORE_COUNT, NA, 0, FALSE) \ +FIELD( 4, 0, EBX, 0, 12, LEAF4_CACHE_LINE, NA, 0, FALSE) \ +FIELD( 4, 0, EBX, 12, 10, LEAF4_CACHE_PART, NA, 0, FALSE) \ +FIELD( 4, 0, EBX, 22, 10, LEAF4_CACHE_WAYS, NA, 0, FALSE) \ +FIELD( 4, 0, ECX, 0, 32, LEAF4_CACHE_SETS, NA, 0, FALSE) \ +FLAG( 4, 0, EDX, 0, 1, LEAF4_CACHE_WBINVD_NOT_GUARANTEED, NA, 0, FALSE) \ +FLAG( 4, 0, EDX, 1, 1, LEAF4_CACHE_IS_INCLUSIVE, NA, 0, FALSE) \ +FLAG( 4, 0, EDX, 2, 1, LEAF4_CACHE_COMPLEX_INDEXING, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_5 \ -FIELD( 5, 0, EAX, 0, 16, MWAIT_MIN_SIZE, NA, FALSE) \ -FIELD( 5, 0, EBX, 0, 16, MWAIT_MAX_SIZE, NA, FALSE) \ -FLAG( 5, 0, ECX, 0, 1, MWAIT_EXTENSIONS, NA, FALSE) \ -FLAG( 5, 0, ECX, 1, 1, MWAIT_INTR_BREAK, NA, FALSE) \ -FIELD( 5, 0, EDX, 0, 4, MWAIT_C0_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 4, 4, MWAIT_C1_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 8, 4, MWAIT_C2_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 12, 4, MWAIT_C3_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 16, 4, MWAIT_C4_SUBSTATE, NA, FALSE) +FIELD( 5, 0, EAX, 0, 16, MWAIT_MIN_SIZE, NA, 0, FALSE) \ +FIELD( 5, 0, EBX, 0, 16, MWAIT_MAX_SIZE, NA, 0, FALSE) \ +FLAG( 5, 0, ECX, 0, 1, MWAIT_EXTENSIONS, NA, 0, FALSE) \ +FLAG( 5, 0, ECX, 1, 1, MWAIT_INTR_BREAK, NA, 0, FALSE) \ +FIELD( 5, 0, EDX, 0, 4, MWAIT_C0_SUBSTATE, NA, 0, FALSE) \ +FIELD( 5, 0, EDX, 4, 4, MWAIT_C1_SUBSTATE, NA, 0, FALSE) \ +FIELD( 5, 0, EDX, 8, 4, MWAIT_C2_SUBSTATE, NA, 0, FALSE) \ +FIELD( 5, 0, EDX, 12, 4, MWAIT_C3_SUBSTATE, NA, 0, FALSE) \ +FIELD( 5, 0, EDX, 16, 4, MWAIT_C4_SUBSTATE, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_6 \ -FLAG( 6, 0, EAX, 0, 1, THERMAL_SENSOR, NA, FALSE) \ -FLAG( 6, 0, EAX, 1, 1, TURBO_MODE, NA, FALSE) \ -FLAG( 6, 0, EAX, 2, 1, APIC_INVARIANT, NA, FALSE) \ -FLAG( 6, 0, EAX, 4, 1, PLN, NA, FALSE) \ -FLAG( 6, 0, EAX, 5, 1, ECMD, NA, FALSE) \ -FLAG( 6, 0, EAX, 6, 1, PTM, NA, FALSE) \ -FLAG( 6, 0, EAX, 7, 1, HWP, NA, FALSE) \ -FLAG( 6, 0, EAX, 8, 1, HWP_NOTIFICATION, NA, FALSE) \ -FLAG( 6, 0, EAX, 9, 1, HWP_ACTIVITY_WINDOW, NA, FALSE) \ -FLAG( 6, 0, EAX, 10, 1, HWP_ENERGY_PERFORMANCE_PREFERENCE, NA, FALSE) \ -FLAG( 6, 0, EAX, 11, 1, HWP_PACKAGE_LEVEL_REQUEST, NA, FALSE) \ -FLAG( 6, 0, EAX, 13, 1, HDC, NA, FALSE) \ -FIELD( 6, 0, EBX, 0, 4, NUM_INTR_THRESHOLDS, NA, FALSE) \ -FLAG( 6, 0, ECX, 0, 1, HW_COORD_FEEDBACK, NA, FALSE) \ -FLAG( 6, 0, ECX, 3, 1, ENERGY_PERF_BIAS, NA, FALSE) +FLAG( 6, 0, EAX, 0, 1, THERMAL_SENSOR, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 1, 1, TURBO_MODE, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 2, 1, APIC_INVARIANT, ANY, 13, FALSE) \ +FLAG( 6, 0, EAX, 4, 1, PLN, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 5, 1, ECMD, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 6, 1, PTM, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 7, 1, HWP, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 8, 1, HWP_NOTIFICATION, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 9, 1, HWP_ACTIVITY_WINDOW, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 10, 1, HWP_ENERGY_PERFORMANCE_PREFERENCE, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 11, 1, HWP_PACKAGE_LEVEL_REQUEST, NO, 0, FALSE) \ +FLAG( 6, 0, EAX, 13, 1, HDC, NO, 0, FALSE) \ +FIELD( 6, 0, EBX, 0, 4, NUM_INTR_THRESHOLDS, NO, 0, FALSE) \ +FLAG( 6, 0, ECX, 0, 1, HW_COORD_FEEDBACK, NO, 0, FALSE) \ +FLAG( 6, 0, ECX, 1, 1, ACNT2, ANY, 13, FALSE) \ +FLAG( 6, 0, ECX, 3, 1, ENERGY_PERF_BIAS, NO, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_7 \ -FLAG( 7, 0, EBX, 0, 1, FSGSBASE, YES, FALSE) \ -FLAG( 7, 0, EBX, 1, 1, TSC_ADJUST, ANY, FALSE) \ -FLAG( 7, 0, EBX, 3, 1, BMI1, YES, TRUE) \ -FLAG( 7, 0, EBX, 2, 1, SGX, NO, FALSE) \ -FLAG( 7, 0, EBX, 4, 1, HLE, YES, TRUE) \ -FLAG( 7, 0, EBX, 5, 1, AVX2, YES, TRUE) \ -FLAG( 7, 0, EBX, 7, 1, SMEP, YES, FALSE) \ -FLAG( 7, 0, EBX, 8, 1, BMI2, YES, TRUE) \ -FLAG( 7, 0, EBX, 9, 1, ENFSTRG, YES, FALSE) \ -FLAG( 7, 0, EBX, 10, 1, INVPCID, YES, FALSE) \ -FLAG( 7, 0, EBX, 11, 1, RTM, YES, TRUE) \ -FLAG( 7, 0, EBX, 12, 1, PQM, NO, FALSE) \ -FLAG( 7, 0, EBX, 13, 1, FP_SEGMENT_ZERO, ANY, TRUE) \ -FLAG( 7, 0, EBX, 15, 1, PQE, NO, FALSE) \ -FLAG( 7, 0, EBX, 18, 1, RDSEED, YES, TRUE) \ -FLAG( 7, 0, EBX, 19, 1, ADX, YES, TRUE) \ -FLAG( 7, 0, EBX, 20, 1, SMAP, YES, FALSE) \ -FLAG( 7, 0, EBX, 25, 1, PT, NO, FALSE) \ -FLAG( 7, 0, ECX, 0, 1, PREFETCHWT1, NO, TRUE) +FLAG( 7, 0, EBX, 0, 1, FSGSBASE, YES, 9, FALSE) \ +FLAG( 7, 0, EBX, 1, 1, TSC_ADJUST, ANY, 11, FALSE) \ +FLAG( 7, 0, EBX, 2, 1, SGX, NO, 0, FALSE) \ +FLAG( 7, 0, EBX, 3, 1, BMI1, YES, 9, TRUE) \ +FLAG( 7, 0, EBX, 4, 1, HLE, YES, 11, TRUE) \ +FLAG( 7, 0, EBX, 5, 1, AVX2, YES, 11, TRUE) \ +FLAG( 7, 0, EBX, 6, 1, FDP_EXCPTN_ONLY, ANY, 13, TRUE) \ +FLAG( 7, 0, EBX, 7, 1, SMEP, YES, 9, FALSE) \ +FLAG( 7, 0, EBX, 8, 1, BMI2, YES, 11, TRUE) \ +FLAG( 7, 0, EBX, 9, 1, ENFSTRG, YES, 9, FALSE) \ +FLAG( 7, 0, EBX, 10, 1, INVPCID, YES, 11, FALSE) \ +FLAG( 7, 0, EBX, 11, 1, RTM, YES, 11, TRUE) \ +FLAG( 7, 0, EBX, 12, 1, PQM, NO, 0, FALSE) \ +FLAG( 7, 0, EBX, 13, 1, FP_SEGMENT_ZERO, ANY, 11, TRUE) \ +FLAG( 7, 0, EBX, 14, 1, MPX, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 15, 1, PQE, NO, 0, FALSE) \ +FLAG( 7, 0, EBX, 16, 1, AVX512F, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 17, 1, AVX512DQ, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 18, 1, RDSEED, YES, 11, TRUE) \ +FLAG( 7, 0, EBX, 19, 1, ADX, YES, 11, TRUE) \ +FLAG( 7, 0, EBX, 20, 1, SMAP, YES, 11, FALSE) \ +FLAG( 7, 0, EBX, 21, 1, AVX512IFMA, YES, 15, TRUE) \ +FLAG( 7, 0, EBX, 23, 1, CLFLUSHOPT, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 24, 1, CLWB, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 25, 1, PT, NO, 0, FALSE) \ +FLAG( 7, 0, EBX, 26, 1, AVX512PF, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 27, 1, AVX512ER, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 28, 1, AVX512CD, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 29, 1, SHA, YES, 14, TRUE) \ +FLAG( 7, 0, EBX, 30, 1, AVX512BW, YES, 13, TRUE) \ +FLAG( 7, 0, EBX, 31, 1, AVX512VL, YES, 13, TRUE) \ +FLAG( 7, 0, ECX, 0, 1, PREFETCHWT1, YES, 13, TRUE) \ +FLAG( 7, 0, ECX, 1, 1, AVX512VBMI, YES, 15, TRUE) \ +FLAG( 7, 0, ECX, 3, 1, PKU, YES, 13, TRUE) \ +FLAG( 7, 0, ECX, 4, 1, OSPKE, ANY, 13, TRUE) \ +FLAG( 7, 0, ECX, 30, 1, SGX_LC, NO, 0, FALSE) \ -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_A \ -FIELD( A, 0, EAX, 0, 8, PMC_VERSION, NA, FALSE) \ -FIELD( A, 0, EAX, 8, 8, PMC_NUM_GEN, NA, FALSE) \ -FIELD( A, 0, EAX, 16, 8, PMC_WIDTH_GEN, NA, FALSE) \ -FIELD( A, 0, EAX, 24, 8, PMC_EBX_LENGTH, NA, FALSE) \ -FLAG( A, 0, EBX, 0, 1, PMC_CORE_CYCLES, NA, FALSE) \ -FLAG( A, 0, EBX, 1, 1, PMC_INSTR_RETIRED, NA, FALSE) \ -FLAG( A, 0, EBX, 2, 1, PMC_REF_CYCLES, NA, FALSE) \ -FLAG( A, 0, EBX, 3, 1, PMC_LAST_LVL_CREF, NA, FALSE) \ -FLAG( A, 0, EBX, 4, 1, PMC_LAST_LVL_CMISS, NA, FALSE) \ -FLAG( A, 0, EBX, 5, 1, PMC_BR_INST_RETIRED, NA, FALSE) \ -FLAG( A, 0, EBX, 6, 1, PMC_BR_MISS_RETIRED, NA, FALSE) \ -FIELD( A, 0, EDX, 0, 5, PMC_NUM_FIXED, NA, FALSE) \ -FIELD( A, 0, EDX, 5, 8, PMC_WIDTH_FIXED, NA, FALSE) +FIELD( A, 0, EAX, 0, 8, PMC_VERSION, NA, 0, FALSE) \ +FIELD( A, 0, EAX, 8, 8, PMC_NUM_GEN, NA, 0, FALSE) \ +FIELD( A, 0, EAX, 16, 8, PMC_WIDTH_GEN, NA, 0, FALSE) \ +FIELD( A, 0, EAX, 24, 8, PMC_EBX_LENGTH, NA, 0, FALSE) \ +FLAG( A, 0, EBX, 0, 1, PMC_CORE_CYCLES, NA, 0, FALSE) \ +FLAG( A, 0, EBX, 1, 1, PMC_INSTR_RETIRED, NA, 0, FALSE) \ +FLAG( A, 0, EBX, 2, 1, PMC_REF_CYCLES, NA, 0, FALSE) \ +FLAG( A, 0, EBX, 3, 1, PMC_LAST_LVL_CREF, NA, 0, FALSE) \ +FLAG( A, 0, EBX, 4, 1, PMC_LAST_LVL_CMISS, NA, 0, FALSE) \ +FLAG( A, 0, EBX, 5, 1, PMC_BR_INST_RETIRED, NA, 0, FALSE) \ +FLAG( A, 0, EBX, 6, 1, PMC_BR_MISS_RETIRED, NA, 0, FALSE) \ +FIELD( A, 0, EDX, 0, 5, PMC_NUM_FIXED, NA, 0, FALSE) \ +FIELD( A, 0, EDX, 5, 8, PMC_WIDTH_FIXED, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_B \ -FIELD( B, 0, EAX, 0, 5, TOPOLOGY_MASK_WIDTH, NA, FALSE) \ -FIELD( B, 0, EBX, 0, 16, TOPOLOGY_CPUS_SHARING_LEVEL, NA, FALSE) \ -FIELD( B, 0, ECX, 0, 8, TOPOLOGY_LEVEL_NUMBER, NA, FALSE) \ -FIELD( B, 0, ECX, 8, 8, TOPOLOGY_LEVEL_TYPE, NA, FALSE) \ -FIELD( B, 0, EDX, 0, 32, TOPOLOGY_X2APIC_ID, NA, FALSE) +FIELD( B, 0, EAX, 0, 5, TOPOLOGY_MASK_WIDTH, NA, 0, FALSE) \ +FIELD( B, 0, EBX, 0, 16, TOPOLOGY_CPUS_SHARING_LEVEL, NA, 0, FALSE) \ +FIELD( B, 0, ECX, 0, 8, TOPOLOGY_LEVEL_NUMBER, NA, 0, FALSE) \ +FIELD( B, 0, ECX, 8, 8, TOPOLOGY_LEVEL_TYPE, NA, 0, FALSE) \ +FIELD( B, 0, EDX, 0, 32, TOPOLOGY_X2APIC_ID, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_D \ -FLAG( D, 0, EAX, 0, 1, XCR0_MASTER_LEGACY_FP, YES, FALSE) \ -FLAG( D, 0, EAX, 1, 1, XCR0_MASTER_SSE, YES, FALSE) \ -FLAG( D, 0, EAX, 2, 1, XCR0_MASTER_YMM_H, YES, FALSE) \ -FIELD( D, 0, EAX, 3, 29, XCR0_MASTER_LOWER, NO, FALSE) \ -FIELD( D, 0, EBX, 0, 32, XSAVE_ENABLED_SIZE, ANY, FALSE) \ -FIELD( D, 0, ECX, 0, 32, XSAVE_MAX_SIZE, YES, FALSE) \ -FIELD( D, 0, EDX, 0, 29, XCR0_MASTER_UPPER, NO, FALSE) \ -FLAG( D, 0, EDX, 30, 1, XCR0_MASTER_LWP, NO, FALSE) \ -FLAG( D, 0, EDX, 31, 1, XCR0_MASTER_EXTENDED_XSAVE, NO, FALSE) \ -FLAG( D, 1, EAX, 0, 1, XSAVEOPT, YES, FALSE) \ -FLAG( D, 1, EAX, 1, 1, XSAVEC, NO, FALSE) \ -FLAG( D, 1, EAX, 2, 1, XGETBV_ECX1, NO, FALSE) \ -FLAG( D, 1, EAX, 3, 1, XSAVES, NO, FALSE) \ -FIELD( D, 1, EBX, 0, 32, XSAVE_XSS_SIZE, NO, FALSE) \ -FIELD( D, 1, ECX, 0, 32, XSS_LOWER, NO, FALSE) \ -FIELD( D, 1, EDX, 0, 32, XSS_UPPER, NO, FALSE) \ -FIELD( D, 2, EAX, 0, 32, XSAVE_YMM_SIZE, YES, FALSE) \ -FIELD( D, 2, EBX, 0, 32, XSAVE_YMM_OFFSET, YES, FALSE) \ -FIELD( D, 2, ECX, 0, 32, XSAVE_YMM_RSVD1, YES, FALSE) \ -FIELD( D, 2, EDX, 0, 32, XSAVE_YMM_RSVD2, YES, FALSE) \ -FIELD( D, 62, EAX, 0, 32, XSAVE_LWP_SIZE, NO, FALSE) \ -FIELD( D, 62, EBX, 0, 32, XSAVE_LWP_OFFSET, NO, FALSE) \ -FIELD( D, 62, ECX, 0, 32, XSAVE_LWP_RSVD1, NO, FALSE) \ -FIELD( D, 62, EDX, 0, 32, XSAVE_LWP_RSVD2, NO, FALSE) +FLAG( D, 0, EAX, 0, 1, XCR0_MASTER_LEGACY_FP, YES, 8, FALSE) \ +FLAG( D, 0, EAX, 1, 1, XCR0_MASTER_SSE, YES, 8, FALSE) \ +FLAG( D, 0, EAX, 2, 1, XCR0_MASTER_YMM_H, YES, 8, FALSE) \ +FLAG( D, 0, EAX, 3, 1, XCR0_MASTER_BNDREGS, YES, 13, FALSE) \ +FLAG( D, 0, EAX, 4, 1, XCR0_MASTER_BNDCSR, YES, 13, FALSE) \ +FLAG( D, 0, EAX, 5, 1, XCR0_MASTER_OPMASK, YES, 13, FALSE) \ +FLAG( D, 0, EAX, 6, 1, XCR0_MASTER_ZMM_H, YES, 13, FALSE) \ +FLAG( D, 0, EAX, 7, 1, XCR0_MASTER_HI16_ZMM, YES, 13, FALSE) \ +FLAG( D, 0, EAX, 8, 1, XCR0_MASTER_XSS, NO, 0, FALSE) \ +FLAG( D, 0, EAX, 9, 1, XCR0_MASTER_PKRU, YES, 13, FALSE) \ +FIELD( D, 0, EAX, 10,22, XCR0_MASTER_LOWER, NO, 0, FALSE) \ +FIELD( D, 0, EBX, 0, 32, XSAVE_ENABLED_SIZE, ANY, 8, FALSE) \ +FIELD( D, 0, ECX, 0, 32, XSAVE_MAX_SIZE, YES, 8, FALSE) \ +FIELD( D, 0, EDX, 0, 29, XCR0_MASTER_UPPER, NO, 0, FALSE) \ +FLAG( D, 0, EDX, 30, 1, XCR0_MASTER_LWP, NO, 0, FALSE) \ +FLAG( D, 0, EDX, 31, 1, XCR0_MASTER_EXTENDED_XSAVE, NO, 0, FALSE) \ +FLAG( D, 1, EAX, 0, 1, XSAVEOPT, YES, 11, FALSE) \ +FLAG( D, 1, EAX, 1, 1, XSAVEC, YES, 13, FALSE) \ +FLAG( D, 1, EAX, 2, 1, XGETBV_ECX1, NO, 0, FALSE) \ +FLAG( D, 1, EAX, 3, 1, XSAVES, YES, 13, FALSE) \ +FIELD( D, 1, EBX, 0, 32, XSAVES_ENABLED_SIZE, ANY, 13, FALSE) \ +FIELD( D, 1, ECX, 0, 7, XSS_XCR0_USED0, NO, 0, FALSE) \ +FLAG( D, 1, ECX, 8, 1, XSS_PT, NO, 0, FALSE) \ +FIELD( D, 1, ECX, 9, 1, XSS_XCR0_USED1, NO, 0, FALSE) \ +FIELD( D, 1, ECX, 10,22, XSS_RSVD0, NO, 0, FALSE) \ +FIELD( D, 1, EDX, 0, 32, XSS_RSVD1, NO, 0, FALSE) \ +FIELD( D, 2, EAX, 0, 32, XSAVE_YMM_SIZE, YES, 9, FALSE) \ +FIELD( D, 2, EBX, 0, 32, XSAVE_YMM_OFFSET, YES, 9, FALSE) \ +FLAG( D, 2, ECX, 0, 1, XSAVE_YMM_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 2, ECX, 1, 1, XSAVE_YMM_ALIGN, YES, 13, FALSE) \ +FIELD( D, 2, ECX, 2, 30, XSAVE_YMM_RSVD1, NO, 0, FALSE) \ +FIELD( D, 2, EDX, 0, 32, XSAVE_YMM_RSVD2, NO, 0, FALSE) \ +FIELD( D, 3, EAX, 0, 32, XSAVE_BNDREGS_SIZE, YES, 13, FALSE) \ +FIELD( D, 3, EBX, 0, 32, XSAVE_BNDREGS_OFFSET, YES, 13, FALSE) \ +FLAG( D, 3, ECX, 0, 1, XSAVE_BNDREGS_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 3, ECX, 1, 1, XSAVE_BNDREGS_ALIGN, YES, 13, FALSE) \ +FIELD( D, 3, ECX, 2, 30, XSAVE_BNDREGS_RSVD1, NO, 0, FALSE) \ +FIELD( D, 3, EDX, 0, 32, XSAVE_BNDREGS_RSVD2, NO, 0, FALSE) \ +FIELD( D, 4, EAX, 0, 32, XSAVE_BNDCSR_SIZE, YES, 13, FALSE) \ +FIELD( D, 4, EBX, 0, 32, XSAVE_BNDCSR_OFFSET, YES, 13, FALSE) \ +FLAG( D, 4, ECX, 0, 1, XSAVE_BNDCSR_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 4, ECX, 1, 1, XSAVE_BNDCSR_ALIGN, YES, 13, FALSE) \ +FIELD( D, 4, ECX, 2, 30, XSAVE_BNDCSR_RSVD1, NO, 0, FALSE) \ +FIELD( D, 4, EDX, 0, 32, XSAVE_BNDCSR_RSVD2, NO, 0, FALSE) \ +FIELD( D, 5, EAX, 0, 32, XSAVE_OPMASK_SIZE, YES, 13, FALSE) \ +FIELD( D, 5, EBX, 0, 32, XSAVE_OPMASK_OFFSET, YES, 13, FALSE) \ +FLAG( D, 5, ECX, 0, 1, XSAVE_OPMASK_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 5, ECX, 1, 1, XSAVE_OPMASK_ALIGN, YES, 13, FALSE) \ +FIELD( D, 5, ECX, 2, 30, XSAVE_OPMASK_RSVD1, NO, 0, FALSE) \ +FIELD( D, 5, EDX, 0, 32, XSAVE_OPMASK_RSVD2, NO, 0, FALSE) \ +FIELD( D, 6, EAX, 0, 32, XSAVE_ZMM_H_SIZE, YES, 13, FALSE) \ +FIELD( D, 6, EBX, 0, 32, XSAVE_ZMM_H_OFFSET, YES, 13, FALSE) \ +FLAG( D, 6, ECX, 0, 1, XSAVE_ZMM_H_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 6, ECX, 1, 1, XSAVE_ZMM_H_ALIGN, YES, 13, FALSE) \ +FIELD( D, 6, ECX, 2, 30, XSAVE_ZMM_H_RSVD1, NO, 0, FALSE) \ +FIELD( D, 6, EDX, 0, 32, XSAVE_ZMM_H_RSVD2, NO, 0, FALSE) \ +FIELD( D, 7, EAX, 0, 32, XSAVE_HI16_ZMM_SIZE, YES, 13, FALSE) \ +FIELD( D, 7, EBX, 0, 32, XSAVE_HI16_ZMM_OFFSET, YES, 13, FALSE) \ +FLAG( D, 7, ECX, 0, 1, XSAVE_HI16_ZMM_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 7, ECX, 1, 1, XSAVE_HI16_ZMM_ALIGN, YES, 13, FALSE) \ +FIELD( D, 7, ECX, 2, 30, XSAVE_HI16_ZMM_RSVD1, NO, 0, FALSE) \ +FIELD( D, 7, EDX, 0, 32, XSAVE_HI16_ZMM_RSVD2, NO, 0, FALSE) \ +FIELD( D, 8, EAX, 0, 32, XSAVE_PT_STATE_SIZE, NO, 0, FALSE) \ +FIELD( D, 8, EBX, 0, 32, XSAVE_PT_STATE_OFFSET, NO, 0, FALSE) \ +FLAG( D, 8, ECX, 0, 1, XSAVE_PT_STATE_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 8, ECX, 1, 1, XSAVE_PT_STATE_ALIGN, NO, 0, FALSE) \ +FIELD( D, 8, ECX, 2, 30, XSAVE_PT_STATE_RSVD1, NO, 0, FALSE) \ +FIELD( D, 8, EDX, 0, 32, XSAVE_PT_STATE_RSVD2, NO, 0, FALSE) \ +FIELD( D, 9, EAX, 0, 32, XSAVE_PKRU_SIZE, YES, 13, FALSE) \ +FIELD( D, 9, EBX, 0, 32, XSAVE_PKRU_OFFSET, YES, 13, FALSE) \ +FLAG( D, 9, ECX, 0, 1, XSAVE_PKRU_SUP_BY_XSS, NO, 0, FALSE) \ +FLAG( D, 9, ECX, 1, 1, XSAVE_PKRU_ALIGN, YES, 13, FALSE) \ +FIELD( D, 9, ECX, 2, 30, XSAVE_PKRU_RSVD1, NO, 0, FALSE) \ +FIELD( D, 9, EDX, 0, 32, XSAVE_PKRU_RSVD2, NO, 0, FALSE) \ +FIELD( D, 62, EAX, 0, 32, XSAVE_LWP_SIZE, NO, 0, FALSE) \ +FIELD( D, 62, EBX, 0, 32, XSAVE_LWP_OFFSET, NO, 0, FALSE) \ +FIELD( D, 62, ECX, 0, 32, XSAVE_LWP_RSVD1, NO, 0, FALSE) \ +FIELD( D, 62, EDX, 0, 32, XSAVE_LWP_RSVD2, NO, 0, FALSE) +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_F \ +FIELD( F, 0, EBX, 0, 32, PQM_MAX_RMID, NO, 0, FALSE) \ +FLAG( F, 0, EDX, 1, 1, PQM_CMT_SUPPORT, NO, 0, FALSE) \ +FIELD( F, 1, EBX, 0, 32, PQM_CMT_CONV, NO, 0, FALSE) \ +FIELD( F, 1, ECX, 0, 32, PQM_CMT_NUM_RMID, NO, 0, FALSE) \ +FLAG( F, 1, EDX, 0, 1, PQM_CMT_OCCUPANCY, NO, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_10 \ +FLAG( 10, 0, EBX, 1, 1, L3_QOS_ENFORCEMENT, NO, 0, FALSE) \ +FIELD( 10, 1, EAX, 0, 4, RESID_CAPACITY_MASK_LENGTH, NO, 0, FALSE) \ +FIELD( 10, 1, EBX, 0, 32, ISOLATION_UNIT_MAP, NO, 0, FALSE) \ +FLAG( 10, 1, ECX, 1, 1, INFREQUENT_COS_UPDATE, NO, 0, FALSE) \ +FLAG( 10, 1, ECX, 2, 1, CODE_AND_DATA_PRIORITIZATION, NO, 0, FALSE) \ +FIELD( 10, 1, EDX, 0, 16, MAX_COS_NUMBER, NO, 0, FALSE) + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_12 \ -FLAG( 12, 0, EAX, 0, 1, SGX1, NA, FALSE) \ -FLAG( 12, 0, EAX, 1, 1, SGX2, NA, FALSE) \ -FLAG( 12, 0, EBX, 31, 1, SGX_MISCSELECT, NA, FALSE) \ -FIELD( 12, 0, EDX, 0, 8, MAX_ENCLAVE_SIZE_NOT64, NA, FALSE) \ -FIELD( 12, 0, EDX, 8, 8, MAX_ENCLAVE_SIZE_64, NA, FALSE) \ -FIELD( 12, 1, EAX, 0, 32, SECS_ATTRIBUTES0, NA, FALSE) \ -FIELD( 12, 1, EBX, 0, 32, SECS_ATTRIBUTES1, NA, FALSE) \ -FIELD( 12, 1, ECX, 0, 32, SECS_ATTRIBUTES2, NA, FALSE) \ -FIELD( 12, 1, EDX, 0, 32, SECS_ATTRIBUTES3, NA, FALSE) \ -FIELD( 12, 2, EAX, 0, 4, EPC00_VALID, NA, FALSE) \ -FIELD( 12, 2, EAX, 12, 20, EPC00_BASE_LOW, NA, FALSE) \ -FIELD( 12, 2, EBX, 0, 20, EPC00_BASE_HIGH, NA, FALSE) \ -FIELD( 12, 2, ECX, 0, 4, EPC00_PROTECTED, NA, FALSE) \ -FIELD( 12, 2, ECX, 12, 20, EPC00_SIZE_LOW, NA, FALSE) \ -FIELD( 12, 2, EDX, 0, 20, EPC00_SIZE_HIGH, NA, FALSE) \ -FIELD( 12, 3, EAX, 0, 4, EPC01_VALID, NA, FALSE) \ -FIELD( 12, 3, EAX, 12, 20, EPC01_BASE_LOW, NA, FALSE) \ -FIELD( 12, 3, EBX, 0, 20, EPC01_BASE_HIGH, NA, FALSE) \ -FIELD( 12, 3, ECX, 0, 4, EPC01_PROTECTED, NA, FALSE) \ -FIELD( 12, 3, ECX, 12, 20, EPC01_SIZE_LOW, NA, FALSE) \ -FIELD( 12, 3, EDX, 0, 20, EPC01_SIZE_HIGH, NA, FALSE) +FLAG( 12, 0, EAX, 0, 1, SGX1, NO, 0, FALSE) \ +FLAG( 12, 0, EAX, 1, 1, SGX2, NO, 0, FALSE) \ +FLAG( 12, 0, EBX, 0, 1, SGX_MISCSELECT_EXINFO, NO, 0, FALSE) \ +FIELD( 12, 0, EBX, 1, 31, SGX_MISCSELECT_RSVD, NO, 0, FALSE) \ +FIELD( 12, 0, EDX, 0, 8, MAX_ENCLAVE_SIZE_NOT64, NO, 0, FALSE) \ +FIELD( 12, 0, EDX, 8, 8, MAX_ENCLAVE_SIZE_64, NO, 0, FALSE) \ +FIELD( 12, 1, EAX, 0, 32, SECS_ATTRIBUTES0, NO, 0, FALSE) \ +FIELD( 12, 1, EBX, 0, 32, SECS_ATTRIBUTES1, NO, 0, FALSE) \ +FIELD( 12, 1, ECX, 0, 32, SECS_ATTRIBUTES2, NO, 0, FALSE) \ +FIELD( 12, 1, EDX, 0, 32, SECS_ATTRIBUTES3, NO, 0, FALSE) \ +FIELD( 12, 2, EAX, 0, 15, EPC00_VALID, NO, 0, FALSE) \ +FIELD( 12, 2, EAX, 12, 20, EPC00_BASE_LOW, NO, 0, FALSE) \ +FIELD( 12, 2, EBX, 0, 20, EPC00_BASE_HIGH, NO, 0, FALSE) \ +FIELD( 12, 2, ECX, 0, 15, EPC00_PROTECTED, NO, 0, FALSE) \ +FIELD( 12, 2, ECX, 12, 20, EPC00_SIZE_LOW, NO, 0, FALSE) \ +FIELD( 12, 2, EDX, 0, 20, EPC00_SIZE_HIGH, NO, 0, FALSE) \ +FIELD( 12, 3, EAX, 0, 4, EPC01_VALID, NO, 0, FALSE) \ +FIELD( 12, 3, EAX, 12, 20, EPC01_BASE_LOW, NO, 0, FALSE) \ +FIELD( 12, 3, EBX, 0, 20, EPC01_BASE_HIGH, NO, 0, FALSE) \ +FIELD( 12, 3, ECX, 0, 4, EPC01_PROTECTED, NO, 0, FALSE) \ +FIELD( 12, 3, ECX, 12, 20, EPC01_SIZE_LOW, NO, 0, FALSE) \ +FIELD( 12, 3, EDX, 0, 20, EPC01_SIZE_HIGH, NO, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_14 \ +FIELD( 14, 0, EAX, 0, 32, MAX_SUB_LEAF, NO, 0, FALSE) \ +FLAG( 14, 0, EBX, 0, 1, CR3FTR_AND_MATCHMSR_AVAILABLE, NO, 0, FALSE) \ +FLAG( 14, 0, EBX, 1, 1, PSB_AND_CYCLE_ACCURATE_MODE, NO, 0, FALSE) \ +FLAG( 14, 0, EBX, 2, 1, IP_TRACESTOP_FTR_PTMSR_PERSIST, NO, 0, FALSE) \ +FLAG( 14, 0, EBX, 3, 1, MTC_PKT_GENERATION_SUPPORTED, NO, 0, FALSE) \ +FLAG( 14, 0, ECX, 0, 1, TOPA_OUTPUT_SUPPORTED, NO, 0, FALSE) \ +FLAG( 14, 0, ECX, 1, 1, TOPA_ALLOW_MULTIPLE_ENTRIES, NO, 0, FALSE) \ +FLAG( 14, 0, ECX, 2, 1, SINGLE_RANGE_OUTPUT_SCHEME, NO, 0, FALSE) \ +FLAG( 14, 0, ECX, 3, 1, TRACE_TRANSPORT_SUBSYSTEM, NO, 0, FALSE) \ +FLAG( 14, 0, ECX, 31, 1, LIP_PRESENT_FOR_IP_PAYLOADS, NO, 0, FALSE) \ +FIELD( 14, 1, EAX, 0, 2, NUM_ADDR_RANGE_FOR_FILTERING, NO, 0, FALSE) \ +FIELD( 14, 1, EAX, 16, 16, SUPPORTED_MTC_ENCODINGS, NO, 0, FALSE) \ +FIELD( 14, 1, EBX, 0, 16, SUPPORTED_CYCLE_THRESHOLD_ENCODINGS, NO, 0, FALSE) \ +FIELD( 14, 1, EBX, 16, 16, SUPPORTED_PSB_FREQ_ENCODINGS, NO, 0, FALSE) \ + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_15 \ +FIELD( 15, 0, EAX, 0, 32, DENOM_TSC_TO_CORE_CRYSTAL_CLK, NO, 0, FALSE) \ +FIELD( 15, 0, EBX, 0, 32, NUMER_TSC_TO_CORE_CRYSTAL_CLK, NO, 0, FALSE) \ + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_16 \ +FIELD( 16, 0, EAX, 0, 16, PROC_BASE_FREQ, NO, 0, FALSE) \ +FIELD( 16, 0, EBX, 0, 16, PROC_MIN_FREQ, NO, 0, FALSE) \ +FIELD( 16, 0, ECX, 0, 16, BUS_FREQ, NO, 0, FALSE) \ + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_17 \ +FIELD( 17, 0, EAX, 0, 31, MAX_SOCID_INDEX, NO, 0, FALSE) \ +FIELD( 17, 0, EBX, 0, 16, SOC_VENDOR_ID, NO, 0, FALSE) \ +FIELD( 17, 0, EBX, 16, 1, SOC_INDUSTRY_STD, NO, 0, FALSE) \ +FIELD( 17, 0, ECX, 0, 31, SOC_PROJECT_ID, NO, 0, FALSE) \ +FIELD( 17, 0, EDX, 0, 31, SOC_STEPPING_ID, NO, 0, FALSE) \ +FIELD( 17, 1, EAX, 0, 32, SOC_VENDOR_BRAND_STRING_1_0, NO, 0, FALSE) \ +FIELD( 17, 1, EBX, 0, 32, SOC_VENDOR_BRAND_STRING_1_1, NO, 0, FALSE) \ +FIELD( 17, 1, ECX, 0, 32, SOC_VENDOR_BRAND_STRING_1_2, NO, 0, FALSE) \ +FIELD( 17, 1, EDX, 0, 32, SOC_VENDOR_BRAND_STRING_1_3, NO, 0, FALSE) \ +FIELD( 17, 2, EAX, 0, 32, SOC_VENDOR_BRAND_STRING_2_0, NO, 0, FALSE) \ +FIELD( 17, 2, EBX, 0, 32, SOC_VENDOR_BRAND_STRING_2_1, NO, 0, FALSE) \ +FIELD( 17, 2, ECX, 0, 32, SOC_VENDOR_BRAND_STRING_2_2, NO, 0, FALSE) \ +FIELD( 17, 2, EDX, 0, 32, SOC_VENDOR_BRAND_STRING_2_3, NO, 0, FALSE) \ +FIELD( 17, 3, EAX, 0, 32, SOC_VENDOR_BRAND_STRING_3_0, NO, 0, FALSE) \ +FIELD( 17, 3, EBX, 0, 32, SOC_VENDOR_BRAND_STRING_3_1, NO, 0, FALSE) \ +FIELD( 17, 3, ECX, 0, 32, SOC_VENDOR_BRAND_STRING_3_2, NO, 0, FALSE) \ +FIELD( 17, 3, EDX, 0, 32, SOC_VENDOR_BRAND_STRING_3_3, NO, 0, FALSE) \ + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_400 \ -FIELD(400, 0, EAX, 0, 32, MAX_HYP_LEVEL, NA, FALSE) \ -FIELD(400, 0, EBX, 0, 32, HYPERVISOR_VENDOR0, NA, FALSE) \ -FIELD(400, 0, ECX, 0, 32, HYPERVISOR_VENDOR1, NA, FALSE) \ -FIELD(400, 0, EDX, 0, 32, HYPERVISOR_VENDOR2, NA, FALSE) +FIELD(400, 0, EAX, 0, 32, MAX_HYP_LEVEL, NA, 0, FALSE) \ +FIELD(400, 0, EBX, 0, 32, HYPERVISOR_VENDOR0, NA, 0, FALSE) \ +FIELD(400, 0, ECX, 0, 32, HYPERVISOR_VENDOR1, NA, 0, FALSE) \ +FIELD(400, 0, EDX, 0, 32, HYPERVISOR_VENDOR2, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_401 \ -FIELD(401, 0, EAX, 0, 32, HV_INTERFACE_SIGNATURE, NA, FALSE) +FIELD(401, 0, EAX, 0, 32, HV_INTERFACE_SIGNATURE, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_402 \ -FIELD(402, 0, EAX, 0, 32, BUILD_NUMBER, NA, FALSE) \ -FIELD(402, 0, EBX, 0, 16, MINOR_VERSION, NA, FALSE) \ -FIELD(402, 0, EBX, 16, 16, MAJOR_VERSION, NA, FALSE) \ -FIELD(402, 0, ECX, 0, 32, SERVICE_PACK, NA, FALSE) \ -FIELD(402, 0, EDX, 0, 24, SERVICE_NUMBER, NA, FALSE) \ -FIELD(402, 0, EDX, 24, 8, SERVICE_BRANCH, NA, FALSE) +FIELD(402, 0, EAX, 0, 32, BUILD_NUMBER, NA, 0, FALSE) \ +FIELD(402, 0, EBX, 0, 16, MINOR_VERSION, NA, 0, FALSE) \ +FIELD(402, 0, EBX, 16, 16, MAJOR_VERSION, NA, 0, FALSE) \ +FIELD(402, 0, ECX, 0, 32, SERVICE_PACK, NA, 0, FALSE) \ +FIELD(402, 0, EDX, 0, 24, SERVICE_NUMBER, NA, 0, FALSE) \ +FIELD(402, 0, EDX, 24, 8, SERVICE_BRANCH, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_403 \ -FLAG( 403, 0, EAX, 0, 1, VP_RUNTIME_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 1, 1, REF_COUNTER_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 2, 1, BASIC_SYNIC_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 3, 1, SYNTH_TIMER_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 4, 1, APIC_ACCESS_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 5, 1, HYPERCALL_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 6, 1, VP_INDEX_MSR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 7, 1, VIRT_RESET_MSR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 8, 1, STATS_PAGES_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 9, 1, REF_TSC_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 10, 1, GUEST_IDLE_MSR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 11, 1, FREQUENCY_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 12, 1, SYNTH_DEBUG_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EBX, 0, 1, CREATE_PARTITIONS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 1, 1, ACCESS_PARTITION_ID_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 2, 1, ACCESS_MEMORY_POOL_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 3, 1, ADJUST_MESSAGE_BUFFERS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 4, 1, POST_MESSAGES_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 5, 1, SIGNAL_EVENTS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 6, 1, CREATE_PORT_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 7, 1, CONNECT_PORT_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 8, 1, ACCESS_STATS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 11, 1, DEBUGGING_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 12, 1, CPU_MANAGEMENT_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 13, 1, CONFIGURE_PROFILER_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 14, 1, ENABLE_EXPANDED_STACKWALKING_FLAG, NA, FALSE) \ -FIELD(403, 0, ECX, 0, 4, MAX_POWER_STATE, NA, FALSE) \ -FLAG( 403, 0, ECX, 4, 1, HPET_NEEDED_FOR_C3, NA, FALSE) \ -FLAG( 403, 0, EDX, 0, 1, MWAIT_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 1, 1, GUEST_DEBUGGING_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 2, 1, PERFORMANCE_MONITOR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 3, 1, CPU_DYN_PARTITIONING_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 4, 1, XMM_REGISTERS_FOR_HYPERCALL_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 5, 1, GUEST_IDLE_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 6, 1, HYPERVISOR_SLEEP_STATE_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 7, 1, NUMA_DISTANCE_QUERY_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 8, 1, TIMER_FREQUENCY_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 9, 1, SYNTH_MACHINE_CHECK_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 10, 1, GUEST_CRASH_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 11, 1, DEBUG_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 12, 1, NPIEP1_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 13, 1, DISABLE_HYPERVISOR_AVAIL, NA, FALSE) +FLAG( 403, 0, EAX, 0, 1, VP_RUNTIME_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 1, 1, REF_COUNTER_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 2, 1, BASIC_SYNIC_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 3, 1, SYNTH_TIMER_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 4, 1, APIC_ACCESS_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 5, 1, HYPERCALL_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 6, 1, VP_INDEX_MSR_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 7, 1, VIRT_RESET_MSR_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 8, 1, STATS_PAGES_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 9, 1, REF_TSC_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 10, 1, GUEST_IDLE_MSR_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 11, 1, FREQUENCY_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EAX, 12, 1, SYNTH_DEBUG_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 0, 1, CREATE_PARTITIONS_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 1, 1, ACCESS_PARTITION_ID_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 2, 1, ACCESS_MEMORY_POOL_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 3, 1, ADJUST_MESSAGE_BUFFERS_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 4, 1, POST_MESSAGES_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 5, 1, SIGNAL_EVENTS_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 6, 1, CREATE_PORT_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 7, 1, CONNECT_PORT_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 8, 1, ACCESS_STATS_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 11, 1, DEBUGGING_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 12, 1, CPU_MANAGEMENT_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 13, 1, CONFIGURE_PROFILER_FLAG, NA, 0, FALSE) \ +FLAG( 403, 0, EBX, 14, 1, ENABLE_EXPANDED_STACKWALKING_FLAG, NA, 0, FALSE) \ +FIELD(403, 0, ECX, 0, 4, MAX_POWER_STATE, NA, 0, FALSE) \ +FLAG( 403, 0, ECX, 4, 1, HPET_NEEDED_FOR_C3, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 0, 1, MWAIT_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 1, 1, GUEST_DEBUGGING_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 2, 1, PERFORMANCE_MONITOR_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 3, 1, CPU_DYN_PARTITIONING_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 4, 1, XMM_REGISTERS_FOR_HYPERCALL_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 5, 1, GUEST_IDLE_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 6, 1, HYPERVISOR_SLEEP_STATE_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 7, 1, NUMA_DISTANCE_QUERY_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 8, 1, TIMER_FREQUENCY_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 9, 1, SYNTH_MACHINE_CHECK_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 10, 1, GUEST_CRASH_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 11, 1, DEBUG_MSRS_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 12, 1, NPIEP1_AVAIL, NA, 0, FALSE) \ +FLAG( 403, 0, EDX, 13, 1, DISABLE_HYPERVISOR_AVAIL, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_404 \ -FLAG( 404, 0, EAX, 0, 1, USE_HYPERCALL_TO_SWITCH_ADDR_SPACE, NA, FALSE) \ -FLAG( 404, 0, EAX, 1, 1, USE_HYPERCALL_TO_FLUSH_TLB, NA, FALSE) \ -FLAG( 404, 0, EAX, 2, 1, USE_HYPERCALL_FOR_TLB_SHOOTDOWN, NA, FALSE) \ -FLAG( 404, 0, EAX, 3, 1, USE_MSRS_FOR_EOI_ICR_TPR, NA, FALSE) \ -FLAG( 404, 0, EAX, 4, 1, USE_MSR_FOR_RESET, NA, FALSE) \ -FLAG( 404, 0, EAX, 5, 1, USE_RELAXED_TIMING, NA, FALSE) \ -FLAG( 404, 0, EAX, 6, 1, USE_DMA_REMAPPING, NA, FALSE) \ -FLAG( 404, 0, EAX, 7, 1, USE_INTERRUPT_REMAPPING, NA, FALSE) \ -FLAG( 404, 0, EAX, 8, 1, USE_X2APIC, NA, FALSE) \ -FLAG( 404, 0, EAX, 9, 1, DEPRECATE_AUTOEOI, NA, FALSE) \ -FIELD(404, 0, EBX, 0, 32, SPINLOCK_RETRIES, NA, FALSE) +FLAG( 404, 0, EAX, 0, 1, USE_HYPERCALL_TO_SWITCH_ADDR_SPACE,NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 1, 1, USE_HYPERCALL_TO_FLUSH_TLB, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 2, 1, USE_HYPERCALL_FOR_TLB_SHOOTDOWN, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 3, 1, USE_MSRS_FOR_EOI_ICR_TPR, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 4, 1, USE_MSR_FOR_RESET, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 5, 1, USE_RELAXED_TIMING, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 6, 1, USE_DMA_REMAPPING, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 7, 1, USE_INTERRUPT_REMAPPING, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 8, 1, USE_X2APIC, NA, 0, FALSE) \ +FLAG( 404, 0, EAX, 9, 1, DEPRECATE_AUTOEOI, NA, 0, FALSE) \ +FIELD(404, 0, EBX, 0, 32, SPINLOCK_RETRIES, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_405 \ -FIELD(405, 0, EAX, 0, 32, MAX_VCPU, NA, FALSE) \ -FIELD(405, 0, EBX, 0, 32, MAX_LCPU, NA, FALSE) \ -FIELD(405, 0, ECX, 0, 32, MAX_REMAPPABLE_VECTORS, NA, FALSE) +FIELD(405, 0, EAX, 0, 32, MAX_VCPU, NA, 0, FALSE) \ +FIELD(405, 0, EBX, 0, 32, MAX_LCPU, NA, 0, FALSE) \ +FIELD(405, 0, ECX, 0, 32, MAX_REMAPPABLE_VECTORS, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_406 \ -FLAG( 406, 0, EAX, 0, 1, APIC_OVERLAY_ASSIST, NA, FALSE) \ -FLAG( 406, 0, EAX, 1, 1, MSR_BITMAPS, NA, FALSE) \ -FLAG( 406, 0, EAX, 2, 1, ARCH_PMCS, NA, FALSE) \ -FLAG( 406, 0, EAX, 3, 1, SLAT, NA, FALSE) \ -FLAG( 406, 0, EAX, 4, 1, DMA_REMAPPING, NA, FALSE) \ -FLAG( 406, 0, EAX, 5, 1, INTERRUPT_REMAPPING, NA, FALSE) \ -FLAG( 406, 0, EAX, 6, 1, MEMORY_PATROL_SCRUBBER, NA, FALSE) +FLAG( 406, 0, EAX, 0, 1, APIC_OVERLAY_ASSIST, NA, 0, FALSE) \ +FLAG( 406, 0, EAX, 1, 1, MSR_BITMAPS, NA, 0, FALSE) \ +FLAG( 406, 0, EAX, 2, 1, ARCH_PMCS, NA, 0, FALSE) \ +FLAG( 406, 0, EAX, 3, 1, SLAT, NA, 0, FALSE) \ +FLAG( 406, 0, EAX, 4, 1, DMA_REMAPPING, NA, 0, FALSE) \ +FLAG( 406, 0, EAX, 5, 1, INTERRUPT_REMAPPING, NA, 0, FALSE) \ +FLAG( 406, 0, EAX, 6, 1, MEMORY_PATROL_SCRUBBER, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_410 \ -FIELD(410, 0, EAX, 0, 32, TSC_HZ, NA, FALSE) \ -FIELD(410, 0, EBX, 0, 32, ACPIBUS_HZ, NA, FALSE) +FIELD(410, 0, EAX, 0, 32, TSC_HZ, NA, 0, FALSE) \ +FIELD(410, 0, EBX, 0, 32, APICBUS_HZ, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_80 \ -FIELD( 80, 0, EAX, 0, 32, NUM_EXT_LEVELS, NA, FALSE) \ -FIELD( 80, 0, EBX, 0, 32, LEAF80_VENDOR1, NA, FALSE) \ -FIELD( 80, 0, ECX, 0, 32, LEAF80_VENDOR3, NA, FALSE) \ -FIELD( 80, 0, EDX, 0, 32, LEAF80_VENDOR2, NA, FALSE) +FIELD( 80, 0, EAX, 0, 32, NUM_EXT_LEVELS, NA, 0, FALSE) \ +FIELD( 80, 0, EBX, 0, 32, LEAF80_VENDOR1, NA, 0, FALSE) \ +FIELD( 80, 0, ECX, 0, 32, LEAF80_VENDOR3, NA, 0, FALSE) \ +FIELD( 80, 0, EDX, 0, 32, LEAF80_VENDOR2, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_81 \ -FIELD( 81, 0, EAX, 0, 32, UNKNOWN81EAX, ANY, FALSE) \ -FIELD( 81, 0, EAX, 0, 4, LEAF81_STEPPING, ANY, FALSE) \ -FIELD( 81, 0, EAX, 4, 4, LEAF81_MODEL, ANY, FALSE) \ -FIELD( 81, 0, EAX, 8, 4, LEAF81_FAMILY, ANY, FALSE) \ -FIELD( 81, 0, EAX, 12, 2, LEAF81_TYPE, ANY, FALSE) \ -FIELD( 81, 0, EAX, 16, 4, LEAF81_EXTENDED_MODEL, ANY, FALSE) \ -FIELD( 81, 0, EAX, 20, 8, LEAF81_EXTENDED_FAMILY, ANY, FALSE) \ -FIELD( 81, 0, EBX, 0, 32, UNKNOWN81EBX, ANY, FALSE) \ -FIELD( 81, 0, EBX, 0, 16, LEAF81_BRAND_ID, ANY, FALSE) \ -FIELD( 81, 0, EBX, 16, 16, UNDEF, ANY, FALSE) \ -FLAG( 81, 0, ECX, 0, 1, LAHF64, YES, TRUE) \ -FLAG( 81, 0, ECX, 1, 1, CMPLEGACY, ANY, FALSE) \ -FLAG( 81, 0, ECX, 2, 1, SVM, YES, FALSE) \ -FLAG( 81, 0, ECX, 3, 1, EXTAPICSPC, YES, FALSE) \ -FLAG( 81, 0, ECX, 4, 1, CR8AVAIL, YES, FALSE) \ -FLAG( 81, 0, ECX, 5, 1, ABM, YES, TRUE) \ -FLAG( 81, 0, ECX, 6, 1, SSE4A, YES, TRUE) \ -FLAG( 81, 0, ECX, 7, 1, MISALIGNED_SSE, YES, TRUE) \ -FLAG( 81, 0, ECX, 8, 1, 3DNPREFETCH, YES, TRUE) \ -FLAG( 81, 0, ECX, 9, 1, OSVW, ANY, FALSE) \ -FLAG( 81, 0, ECX, 10, 1, IBS, NO, FALSE) \ -FLAG( 81, 0, ECX, 11, 1, XOP, YES, TRUE) \ -FLAG( 81, 0, ECX, 12, 1, SKINIT, NO, FALSE) \ -FLAG( 81, 0, ECX, 13, 1, WATCHDOG, NO, FALSE) \ -FLAG( 81, 0, ECX, 15, 1, LWP, NO, FALSE) \ -FLAG( 81, 0, ECX, 16, 1, FMA4, YES, TRUE) \ -FLAG( 81, 0, ECX, 17, 1, TCE, NO, FALSE) \ -FLAG( 81, 0, ECX, 19, 1, NODEID_MSR, NO, FALSE) \ -FLAG( 81, 0, ECX, 21, 1, TBM, YES, TRUE) \ -FLAG( 81, 0, ECX, 22, 1, TOPOLOGY, NO, FALSE) \ -FLAG( 81, 0, ECX, 23, 1, PERFCORE, ANY, TRUE) \ -FLAG( 81, 0, EDX, 0, 1, LEAF81_FPU, YES, TRUE) \ -FLAG( 81, 0, EDX, 1, 1, LEAF81_VME, YES, FALSE) \ -FLAG( 81, 0, EDX, 2, 1, LEAF81_DE, YES, FALSE) \ -FLAG( 81, 0, EDX, 3, 1, LEAF81_PSE, YES, FALSE) \ -FLAG( 81, 0, EDX, 4, 1, LEAF81_TSC, YES, TRUE) \ -FLAG( 81, 0, EDX, 5, 1, LEAF81_MSR, YES, FALSE) \ -FLAG( 81, 0, EDX, 6, 1, LEAF81_PAE, YES, FALSE) \ -FLAG( 81, 0, EDX, 7, 1, LEAF81_MCE, YES, FALSE) \ -FLAG( 81, 0, EDX, 8, 1, LEAF81_CX8, YES, TRUE) \ -FLAG( 81, 0, EDX, 9, 1, LEAF81_APIC, ANY, FALSE) \ -FLAG( 81, 0, EDX, 11, 1, SYSC, ANY, TRUE) \ -FLAG( 81, 0, EDX, 12, 1, LEAF81_MTRR, YES, FALSE) \ -FLAG( 81, 0, EDX, 13, 1, LEAF81_PGE, YES, FALSE) \ -FLAG( 81, 0, EDX, 14, 1, LEAF81_MCA, YES, FALSE) \ -FLAG( 81, 0, EDX, 15, 1, LEAF81_CMOV, YES, TRUE) \ -FLAG( 81, 0, EDX, 16, 1, LEAF81_PAT, YES, FALSE) \ -FLAG( 81, 0, EDX, 17, 1, LEAF81_PSE36, YES, FALSE) \ -FLAG( 81, 0, EDX, 20, 1, NX, YES, FALSE) \ -FLAG( 81, 0, EDX, 22, 1, MMXEXT, YES, TRUE) \ -FLAG( 81, 0, EDX, 23, 1, LEAF81_MMX, YES, TRUE) \ -FLAG( 81, 0, EDX, 24, 1, LEAF81_FXSR, YES, TRUE) \ -FLAG( 81, 0, EDX, 25, 1, FFXSR, YES, FALSE) \ -FLAG( 81, 0, EDX, 26, 1, PDPE1GB, YES, FALSE) \ -FLAG( 81, 0, EDX, 27, 1, RDTSCP, YES, TRUE) \ -FLAG( 81, 0, EDX, 29, 1, LM, YES, FALSE) \ -FLAG( 81, 0, EDX, 30, 1, 3DNOWPLUS, YES, TRUE) \ -FLAG( 81, 0, EDX, 31, 1, 3DNOW, YES, TRUE) +FIELD( 81, 0, EAX, 0, 32, UNKNOWN81EAX, ANY, 4, FALSE) \ +FIELD( 81, 0, EAX, 0, 4, LEAF81_STEPPING, ANY, 4, FALSE) \ +FIELD( 81, 0, EAX, 4, 4, LEAF81_MODEL, ANY, 4, FALSE) \ +FIELD( 81, 0, EAX, 8, 4, LEAF81_FAMILY, ANY, 4, FALSE) \ +FIELD( 81, 0, EAX, 12, 2, LEAF81_TYPE, ANY, 4, FALSE) \ +FIELD( 81, 0, EAX, 16, 4, LEAF81_EXTENDED_MODEL, ANY, 4, FALSE) \ +FIELD( 81, 0, EAX, 20, 8, LEAF81_EXTENDED_FAMILY, ANY, 4, FALSE) \ +FIELD( 81, 0, EBX, 0, 32, UNKNOWN81EBX, ANY, 4, FALSE) \ +FIELD( 81, 0, EBX, 0, 16, LEAF81_BRAND_ID, ANY, 4, FALSE) \ +FIELD( 81, 0, EBX, 16, 16, UNDEF, ANY, 4, FALSE) \ +FLAG( 81, 0, ECX, 0, 1, LAHF64, YES, 4, TRUE) \ +FLAG( 81, 0, ECX, 1, 1, CMPLEGACY, ANY, 9, FALSE) \ +FLAG( 81, 0, ECX, 2, 1, SVM, YES, 7, FALSE) \ +FLAG( 81, 0, ECX, 3, 1, EXTAPICSPC, YES, 4, FALSE) \ +FLAG( 81, 0, ECX, 4, 1, CR8AVAIL, YES, 4, FALSE) \ +FLAG( 81, 0, ECX, 5, 1, ABM, YES, 7, TRUE) \ +FLAG( 81, 0, ECX, 6, 1, SSE4A, YES, 4, TRUE) \ +FLAG( 81, 0, ECX, 7, 1, MISALIGNED_SSE, YES, 4, TRUE) \ +FLAG( 81, 0, ECX, 8, 1, 3DNPREFETCH, YES, 4, TRUE) \ +FLAG( 81, 0, ECX, 9, 1, OSVW, ANY, 8, FALSE) \ +FLAG( 81, 0, ECX, 10, 1, IBS, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 11, 1, XOP, YES, 8, TRUE) \ +FLAG( 81, 0, ECX, 12, 1, SKINIT, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 13, 1, WATCHDOG, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 15, 1, LWP, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 16, 1, FMA4, YES, 8, TRUE) \ +FLAG( 81, 0, ECX, 17, 1, TCE, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 19, 1, NODEID_MSR, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 21, 1, TBM, YES, 9, TRUE) \ +FLAG( 81, 0, ECX, 22, 1, TOPOLOGY, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 23, 1, PERFCORE, ANY, 4, TRUE) \ +FLAG( 81, 0, ECX, 24, 1, PERFNB, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 26, 1, DATABK, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 27, 1, PERFTSC, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 28, 1, PERFL3, NO, 0, FALSE) \ +FLAG( 81, 0, ECX, 29, 1, MWAITX, NO, 0, FALSE) \ +FLAG( 81, 0, EDX, 0, 1, LEAF81_FPU, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 1, 1, LEAF81_VME, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 2, 1, LEAF81_DE, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 3, 1, LEAF81_PSE, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 4, 1, LEAF81_TSC, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 5, 1, LEAF81_MSR, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 6, 1, LEAF81_PAE, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 7, 1, LEAF81_MCE, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 8, 1, LEAF81_CX8, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 9, 1, LEAF81_APIC, ANY, 4, FALSE) \ +FLAG( 81, 0, EDX, 11, 1, SYSC, ANY, 4, TRUE) \ +FLAG( 81, 0, EDX, 12, 1, LEAF81_MTRR, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 13, 1, LEAF81_PGE, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 14, 1, LEAF81_MCA, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 15, 1, LEAF81_CMOV, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 16, 1, LEAF81_PAT, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 17, 1, LEAF81_PSE36, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 20, 1, NX, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 22, 1, MMXEXT, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 23, 1, LEAF81_MMX, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 24, 1, LEAF81_FXSR, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 25, 1, FFXSR, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 26, 1, PDPE1GB, YES, 7, FALSE) \ +FLAG( 81, 0, EDX, 27, 1, RDTSCP, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 29, 1, LM, YES, 4, FALSE) \ +FLAG( 81, 0, EDX, 30, 1, 3DNOWPLUS, YES, 4, TRUE) \ +FLAG( 81, 0, EDX, 31, 1, 3DNOW, YES, 4, TRUE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_82 \ -FIELD( 82, 0, EAX, 0, 32, LEAF82_BRAND_STRING_EAX, NA, FALSE) \ -FIELD( 82, 0, EBX, 0, 32, LEAF82_BRAND_STRING_EBX, NA, FALSE) \ -FIELD( 82, 0, ECX, 0, 32, LEAF82_BRAND_STRING_ECX, NA, FALSE) \ -FIELD( 82, 0, EDX, 0, 32, LEAF82_BRAND_STRING_EDX, NA, FALSE) +FIELD( 82, 0, EAX, 0, 32, LEAF82_BRAND_STRING_EAX, NA, 0, FALSE) \ +FIELD( 82, 0, EBX, 0, 32, LEAF82_BRAND_STRING_EBX, NA, 0, FALSE) \ +FIELD( 82, 0, ECX, 0, 32, LEAF82_BRAND_STRING_ECX, NA, 0, FALSE) \ +FIELD( 82, 0, EDX, 0, 32, LEAF82_BRAND_STRING_EDX, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_83 \ -FIELD( 83, 0, EAX, 0, 32, LEAF83_BRAND_STRING_EAX, NA, FALSE) \ -FIELD( 83, 0, EBX, 0, 32, LEAF83_BRAND_STRING_EBX, NA, FALSE) \ -FIELD( 83, 0, ECX, 0, 32, LEAF83_BRAND_STRING_ECX, NA, FALSE) \ -FIELD( 83, 0, EDX, 0, 32, LEAF83_BRAND_STRING_EDX, NA, FALSE) +FIELD( 83, 0, EAX, 0, 32, LEAF83_BRAND_STRING_EAX, NA, 0, FALSE) \ +FIELD( 83, 0, EBX, 0, 32, LEAF83_BRAND_STRING_EBX, NA, 0, FALSE) \ +FIELD( 83, 0, ECX, 0, 32, LEAF83_BRAND_STRING_ECX, NA, 0, FALSE) \ +FIELD( 83, 0, EDX, 0, 32, LEAF83_BRAND_STRING_EDX, NA, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ #define CPUID_FIELD_DATA_LEVEL_84 \ -FIELD( 84, 0, EAX, 0, 32, LEAF84_BRAND_STRING_EAX, NA, FALSE) \ -FIELD( 84, 0, EBX, 0, 32, LEAF84_BRAND_STRING_EBX, NA, FALSE) \ -FIELD( 84, 0, ECX, 0, 32, LEAF84_BRAND_STRING_ECX, NA, FALSE) \ -FIELD( 84, 0, EDX, 0, 32, LEAF84_BRAND_STRING_EDX, NA, FALSE) +FIELD( 84, 0, EAX, 0, 32, LEAF84_BRAND_STRING_EAX, NA, 0, FALSE) \ +FIELD( 84, 0, EBX, 0, 32, LEAF84_BRAND_STRING_EBX, NA, 0, FALSE) \ +FIELD( 84, 0, ECX, 0, 32, LEAF84_BRAND_STRING_ECX, NA, 0, FALSE) \ +FIELD( 84, 0, EDX, 0, 32, LEAF84_BRAND_STRING_EDX, NA, 0, FALSE) + +/* LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_85 \ +FIELD( 85, 0, EAX, 0, 8, ITLB_ENTRIES_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, EAX, 8, 8, ITLB_ASSOC_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, EAX, 16, 8, DTLB_ENTRIES_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, EAX, 24, 8, DTLB_ASSOC_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, EBX, 0, 8, ITLB_ENTRIES_4K_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, EBX, 8, 8, ITLB_ASSOC_4K_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, EBX, 16, 8, DTLB_ENTRIES_4K_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, EBX, 24, 8, DTLB_ASSOC_4K_PGS, NA, 0, FALSE) \ +FIELD( 85, 0, ECX, 0, 8, L1_DCACHE_LINE_SIZE, NA, 0, FALSE) \ +FIELD( 85, 0, ECX, 8, 8, L1_DCACHE_LINES_PER_TAG, NA, 0, FALSE) \ +FIELD( 85, 0, ECX, 16, 8, L1_DCACHE_ASSOC, NA, 0, FALSE) \ +FIELD( 85, 0, ECX, 24, 8, L1_DCACHE_SIZE, NA, 0, FALSE) \ +FIELD( 85, 0, EDX, 0, 8, L1_ICACHE_LINE_SIZE, NA, 0, FALSE) \ +FIELD( 85, 0, EDX, 8, 8, L1_ICACHE_LINES_PER_TAG, NA, 0, FALSE) \ +FIELD( 85, 0, EDX, 16, 8, L1_ICACHE_ASSOC, NA, 0, FALSE) \ +FIELD( 85, 0, EDX, 24, 8, L1_ICACHE_SIZE, NA, 0, FALSE) + +/* LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_86 \ +FIELD( 86, 0, EAX, 0, 12, L2_ITLB_ENTRIES_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, EAX, 12, 4, L2_ITLB_ASSOC_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, EAX, 16, 12, L2_DTLB_ENTRIES_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, EAX, 28, 4, L2_DTLB_ASSOC_2M4M_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, EBX, 0, 12, L2_ITLB_ENTRIES_4K_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, EBX, 12, 4, L2_ITLB_ASSOC_4K_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, EBX, 16, 12, L2_DTLB_ENTRIES_4K_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, EBX, 28, 4, L2_DTLB_ASSOC_4K_PGS, NA, 0, FALSE) \ +FIELD( 86, 0, ECX, 0, 8, L2CACHE_LINE, NA, 0, FALSE) \ +FIELD( 86, 0, ECX, 8, 4, L2CACHE_LINE_PER_TAG, NA, 0, FALSE) \ +FIELD( 86, 0, ECX, 12, 4, L2CACHE_WAYS, NA, 0, FALSE) \ +FIELD( 86, 0, ECX, 16, 16, L2CACHE_SIZE, NA, 0, FALSE) \ +FIELD( 86, 0, EDX, 0, 8, L3CACHE_LINE, NA, 0, FALSE) \ +FIELD( 86, 0, EDX, 8, 4, L3CACHE_LINE_PER_TAG, NA, 0, FALSE) \ +FIELD( 86, 0, EDX, 12, 4, L3CACHE_WAYS, NA, 0, FALSE) \ +FIELD( 86, 0, EDX, 18, 14, L3CACHE_SIZE, NA, 0, FALSE) + +/* LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_87 \ +FLAG( 87, 0, EDX, 0, 1, TS, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 1, 1, FID, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 2, 1, VID, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 3, 1, TTP, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 4, 1, LEAF87_TM, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 5, 1, STC, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 6, 1, 100MHZSTEPS, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 7, 1, HWPSTATE, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 8, 1, TSC_INVARIANT, NA, 0, FALSE) \ +FLAG( 87, 0, EDX, 9, 1, CORE_PERF_BOOST, NA, 0, FALSE) + +/* LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_88 \ +FIELD( 88, 0, EAX, 0, 8, PHYS_BITS, YES, 4, FALSE) \ +FIELD( 88, 0, EAX, 8, 8, VIRT_BITS, YES, 4, FALSE) \ +FIELD( 88, 0, EAX, 16, 8, GUEST_PHYS_ADDR_SZ, YES, 8, FALSE) \ +FLAG( 88, 0, EBX, 0, 1, CLZERO, YES, 14, TRUE) \ +FLAG( 88, 0, EBX, 1, 1, IRPERF, NO, 0, FALSE) \ +FLAG( 88, 0, EBX, 2, 1, XSAVE_ERR_PTR, NO, 0, FALSE) \ +FIELD( 88, 0, ECX, 0, 8, LEAF88_CORE_COUNT, YES, 4, FALSE) \ +FIELD( 88, 0, ECX, 12, 4, APICID_COREID_SIZE, YES, 7, FALSE) \ +FIELD( 88, 0, ECX, 16, 2, PERFTSC_SIZE, NO, 0, FALSE) #define CPUID_8A_EDX_11 \ -FLAG( 8A, 0, EDX, 11, 1, SVMEDX_RSVD1, NO, FALSE) -#define CPUID_8A_EDX_14_31 \ -FIELD( 8A, 0, EDX, 14, 18, SVMEDX_RSVD2, NO, FALSE) +FLAG( 8A, 0, EDX, 11, 1, SVMEDX_RSVD1, NO, 0, FALSE) +#define CPUID_8A_EDX_14 \ +FLAG( 8A, 0, EDX, 14, 1, SVMEDX_RSVD2, NO, 0, FALSE) -/* LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_8x \ -FIELD( 85, 0, EAX, 0, 8, ITLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EAX, 8, 8, ITLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EAX, 16, 8, DTLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EAX, 24, 8, DTLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 0, 8, ITLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 8, 8, ITLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 16, 8, DTLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 24, 8, DTLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, ECX, 0, 8, L1_DCACHE_LINE_SIZE, NA, FALSE) \ -FIELD( 85, 0, ECX, 8, 8, L1_DCACHE_LINES_PER_TAG, NA, FALSE) \ -FIELD( 85, 0, ECX, 16, 8, L1_DCACHE_ASSOC, NA, FALSE) \ -FIELD( 85, 0, ECX, 24, 8, L1_DCACHE_SIZE, NA, FALSE) \ -FIELD( 85, 0, EDX, 0, 8, L1_ICACHE_LINE_SIZE, NA, FALSE) \ -FIELD( 85, 0, EDX, 8, 8, L1_ICACHE_LINES_PER_TAG, NA, FALSE) \ -FIELD( 85, 0, EDX, 16, 8, L1_ICACHE_ASSOC, NA, FALSE) \ -FIELD( 85, 0, EDX, 24, 8, L1_ICACHE_SIZE, NA, FALSE) \ -FIELD( 86, 0, EAX, 0, 12, L2_ITLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EAX, 12, 4, L2_ITLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EAX, 16, 12, L2_DTLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EAX, 28, 4, L2_DTLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 0, 12, L2_ITLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 12, 4, L2_ITLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 16, 12, L2_DTLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 28, 4, L2_DTLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, ECX, 0, 8, L2CACHE_LINE, NA, FALSE) \ -FIELD( 86, 0, ECX, 8, 4, L2CACHE_LINE_PER_TAG, NA, FALSE) \ -FIELD( 86, 0, ECX, 12, 4, L2CACHE_WAYS, NA, FALSE) \ -FIELD( 86, 0, ECX, 16, 16, L2CACHE_SIZE, NA, FALSE) \ -FIELD( 86, 0, EDX, 0, 8, L3CACHE_LINE, NA, FALSE) \ -FIELD( 86, 0, EDX, 8, 4, L3CACHE_LINE_PER_TAG, NA, FALSE) \ -FIELD( 86, 0, EDX, 12, 4, L3CACHE_WAYS, NA, FALSE) \ -FIELD( 86, 0, EDX, 18, 14, L3CACHE_SIZE, NA, FALSE) \ -FLAG( 87, 0, EDX, 0, 1, TS, NA, FALSE) \ -FLAG( 87, 0, EDX, 1, 1, FID, NA, FALSE) \ -FLAG( 87, 0, EDX, 2, 1, VID, NA, FALSE) \ -FLAG( 87, 0, EDX, 3, 1, TTP, NA, FALSE) \ -FLAG( 87, 0, EDX, 4, 1, LEAF87_TM, NA, FALSE) \ -FLAG( 87, 0, EDX, 5, 1, STC, NA, FALSE) \ -FLAG( 87, 0, EDX, 6, 1, 100MHZSTEPS, NA, FALSE) \ -FLAG( 87, 0, EDX, 7, 1, HWPSTATE, NA, FALSE) \ -FLAG( 87, 0, EDX, 8, 1, TSC_INVARIANT, NA, FALSE) \ -FLAG( 87, 0, EDX, 9, 1, CORE_PERF_BOOST, NA, FALSE) \ -FIELD( 88, 0, EAX, 0, 8, PHYS_BITS, NA, FALSE) \ -FIELD( 88, 0, EAX, 8, 8, VIRT_BITS, NA, FALSE) \ -FIELD( 88, 0, EAX, 16, 8, GUEST_PHYS_ADDR_SZ, NA, FALSE) \ -FIELD( 88, 0, ECX, 0, 8, LEAF88_CORE_COUNT, NA, FALSE) \ -FIELD( 88, 0, ECX, 12, 4, APICID_COREID_SIZE, NA, FALSE) \ -FIELD( 8A, 0, EAX, 0, 8, SVM_REVISION, YES, FALSE) \ -FLAG( 8A, 0, EAX, 8, 1, SVM_HYPERVISOR, NO, FALSE) \ -FIELD( 8A, 0, EAX, 9, 23, SVMEAX_RSVD, NO, FALSE) \ -FIELD( 8A, 0, EBX, 0, 32, SVM_NUM_ASIDS, YES, FALSE) \ -FIELD( 8A, 0, ECX, 0, 32, SVMECX_RSVD, NO, FALSE) \ -FLAG( 8A, 0, EDX, 0, 1, SVM_NPT, YES, FALSE) \ -FLAG( 8A, 0, EDX, 1, 1, SVM_LBR, NO, FALSE) \ -FLAG( 8A, 0, EDX, 2, 1, SVM_LOCK, ANY, FALSE) \ -FLAG( 8A, 0, EDX, 3, 1, SVM_NRIP, YES, FALSE) \ -FLAG( 8A, 0, EDX, 4, 1, SVM_TSC_RATE_MSR, NO, FALSE) \ -FLAG( 8A, 0, EDX, 5, 1, SVM_VMCB_CLEAN, YES, FALSE) \ -FLAG( 8A, 0, EDX, 6, 1, SVM_FLUSH_BY_ASID, YES, FALSE) \ -FLAG( 8A, 0, EDX, 7, 1, SVM_DECODE_ASSISTS, YES, FALSE) \ -FIELD( 8A, 0, EDX, 8, 2, SVMEDX_RSVD0, NO, FALSE) \ -FLAG( 8A, 0, EDX, 10, 1, SVM_PAUSE_FILTER, NO, FALSE) \ +/* LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_8A \ +FIELD( 8A, 0, EAX, 0, 8, SVM_REVISION, YES, 4, FALSE) \ +FLAG( 8A, 0, EAX, 8, 1, SVM_HYPERVISOR, NO, 0, FALSE) \ +FIELD( 8A, 0, EAX, 9, 23, SVMEAX_RSVD, NO, 0, FALSE) \ +FIELD( 8A, 0, EBX, 0, 32, SVM_NUM_ASIDS, YES, 7, FALSE) \ +FIELD( 8A, 0, ECX, 0, 32, SVMECX_RSVD, NO, 0, FALSE) \ +FLAG( 8A, 0, EDX, 0, 1, SVM_NPT, YES, 7, FALSE) \ +FLAG( 8A, 0, EDX, 1, 1, SVM_LBR, NO, 0, FALSE) \ +FLAG( 8A, 0, EDX, 2, 1, SVM_LOCK, ANY, 7, FALSE) \ +FLAG( 8A, 0, EDX, 3, 1, SVM_NRIP, YES, 7, FALSE) \ +FLAG( 8A, 0, EDX, 4, 1, SVM_TSC_RATE_MSR, NO, 0, FALSE) \ +FLAG( 8A, 0, EDX, 5, 1, SVM_VMCB_CLEAN, YES, 7, FALSE) \ +FLAG( 8A, 0, EDX, 6, 1, SVM_FLUSH_BY_ASID, YES, 7, FALSE) \ +FLAG( 8A, 0, EDX, 7, 1, SVM_DECODE_ASSISTS, YES, 7, FALSE) \ +FIELD( 8A, 0, EDX, 8, 2, SVMEDX_RSVD0, NO, 0, FALSE) \ +FLAG( 8A, 0, EDX, 10, 1, SVM_PAUSE_FILTER, NO, 0, FALSE) \ CPUID_8A_EDX_11 \ -FLAG( 8A, 0, EDX, 12, 1, SVM_PAUSE_THRESHOLD, NO, FALSE) \ -FLAG( 8A, 0, EDX, 13, 1, SVM_AVIC, NO, FALSE) \ -CPUID_8A_EDX_14_31 +FLAG( 8A, 0, EDX, 12, 1, SVM_PAUSE_THRESHOLD, NO, 0, FALSE) \ +FLAG( 8A, 0, EDX, 13, 1, SVM_AVIC, NO, 0, FALSE) \ +CPUID_8A_EDX_14 \ +FLAG( 8A, 0, EDX, 15, 1, SVM_VIRT_VMSAVE_VMLOAD, NO, 0, FALSE) \ +FLAG( 8A, 0, EDX, 16, 1, SVM_VGIF, NO, 0, FALSE) \ +FIELD( 8A, 0, EDX, 17, 15, SVMEDX_RSVD, NO, 0, FALSE) -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_81x \ -FIELD(819, 0, EAX, 0, 12, L1_ITLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EAX, 12, 4, L1_ITLB_ASSOC_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EAX, 16, 12, L1_DTLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EAX, 28, 4, L1_DTLB_ASSOC_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 0, 12, L2_ITLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 12, 4, L2_ITLB_ASSOC_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 16, 12, L2_DTLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 28, 4, L2_DTLB_ASSOC_1G_PGS, NA, FALSE) \ -FLAG( 81A, 0, EAX, 0, 1, FP128, NA, FALSE) \ -FLAG( 81A, 0, EAX, 1, 1, MOVU, NA, FALSE) \ -FLAG( 81B, 0, EAX, 0, 1, IBS_FFV, NA, FALSE) \ -FLAG( 81B, 0, EAX, 1, 1, IBS_FETCHSAM, NA, FALSE) \ -FLAG( 81B, 0, EAX, 2, 1, IBS_OPSAM, NA, FALSE) \ -FLAG( 81B, 0, EAX, 3, 1, RW_OPCOUNT, NA, FALSE) \ -FLAG( 81B, 0, EAX, 4, 1, OPCOUNT, NA, FALSE) \ -FLAG( 81B, 0, EAX, 5, 1, BRANCH_TARGET_ADDR, NA, FALSE) \ -FLAG( 81B, 0, EAX, 6, 1, OPCOUNT_EXT, NA, FALSE) \ -FLAG( 81B, 0, EAX, 7, 1, RIP_INVALID_CHECK, NA, FALSE) \ -FLAG( 81C, 0, EAX, 0, 1, LWP_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 1, 1, LWP_VAL_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 2, 1, LWP_IRE_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 3, 1, LWP_BRE_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 4, 1, LWP_DME_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 5, 1, LWP_CNH_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 6, 1, LWP_RNH_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 31, 1, LWP_INT_AVAIL, NA, FALSE) \ -FIELD(81C, 0, EBX, 0, 8, LWP_CB_SIZE, NA, FALSE) \ -FIELD(81C, 0, EBX, 8, 8, LWP_EVENT_SIZE, NA, FALSE) \ -FIELD(81C, 0, EBX, 16, 8, LWP_MAX_EVENTS, NA, FALSE) \ -FIELD(81C, 0, EBX, 24, 8, LWP_EVENT_OFFSET, NA, FALSE) \ -FIELD(81C, 0, ECX, 0, 4, LWP_LATENCY_MAX, NA, FALSE) \ -FLAG( 81C, 0, ECX, 5, 1, LWP_DATA_ADDR_VALID, NA, FALSE) \ -FIELD(81C, 0, ECX, 6, 3, LWP_LATENCY_ROUND, NA, FALSE) \ -FIELD(81C, 0, ECX, 9, 7, LWP_VERSION, NA, FALSE) \ -FIELD(81C, 0, ECX, 16, 8, LWP_MIN_BUF_SIZE, NA, FALSE) \ -FLAG( 81C, 0, ECX, 28, 1, LWP_BRANCH_PRED, NA, FALSE) \ -FLAG( 81C, 0, ECX, 29, 1, LWP_IP_FILTERING, NA, FALSE) \ -FLAG( 81C, 0, ECX, 30, 1, LWP_CACHE_LEVEL, NA, FALSE) \ -FLAG( 81C, 0, ECX, 31, 1, LWP_CACHE_LATENCY, NA, FALSE) \ -FLAG( 81C, 0, EDX, 0, 1, LWP_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 1, 1, LWP_VAL_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 2, 1, LWP_IRE_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 3, 1, LWP_BRE_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 4, 1, LWP_DME_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 5, 1, LWP_CNH_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 6, 1, LWP_RNH_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 31, 1, LWP_INT_SUPPORTED, NA, FALSE) \ -FIELD(81D, 0, EAX, 0, 5, LEAF81D_CACHE_TYPE, NA, FALSE) \ -FIELD(81D, 0, EAX, 5, 3, LEAF81D_CACHE_LEVEL, NA, FALSE) \ -FLAG( 81D, 0, EAX, 8, 1, LEAF81D_CACHE_SELF_INIT, NA, FALSE) \ -FLAG( 81D, 0, EAX, 9, 1, LEAF81D_CACHE_FULLY_ASSOC, NA, FALSE) \ -FIELD(81D, 0, EAX, 14, 12, LEAF81D_NUM_SHARING_CACHE, NA, FALSE) \ -FIELD(81D, 0, EBX, 0, 12, LEAF81D_CACHE_LINE_SIZE, NA, FALSE) \ -FIELD(81D, 0, EBX, 12, 10, LEAF81D_CACHE_PHYS_PARTITIONS, NA, FALSE) \ -FIELD(81D, 0, EBX, 22, 10, LEAF81D_CACHE_WAYS, NA, FALSE) \ -FIELD(81D, 0, ECX, 0, 32, LEAF81D_CACHE_NUM_SETS, NA, FALSE) \ -FLAG( 81D, 0, EDX, 0, 1, LEAF81D_CACHE_WBINVD, NA, FALSE) \ -FLAG( 81D, 0, EDX, 1, 1, LEAF81D_CACHE_INCLUSIVE, NA, FALSE) \ -FIELD(81E, 0, EAX, 0, 32, EXTENDED_APICID, NA, FALSE) \ -FIELD(81E, 0, EBX, 0, 8, COMPUTE_UNIT_ID, NA, FALSE) \ -FIELD(81E, 0, EBX, 8, 2, CORES_PER_COMPUTE_UNIT, NA, FALSE) \ -FIELD(81E, 0, ECX, 0, 8, NODEID_VAL, NA, FALSE) \ -FIELD(81E, 0, ECX, 8, 3, NODES_PER_PKG, NA, FALSE) +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_819 \ +FIELD(819, 0, EAX, 0, 12, L1_ITLB_ENTRIES_1G_PGS, NA, 0, FALSE) \ +FIELD(819, 0, EAX, 12, 4, L1_ITLB_ASSOC_1G_PGS, NA, 0, FALSE) \ +FIELD(819, 0, EAX, 16, 12, L1_DTLB_ENTRIES_1G_PGS, NA, 0, FALSE) \ +FIELD(819, 0, EAX, 28, 4, L1_DTLB_ASSOC_1G_PGS, NA, 0, FALSE) \ +FIELD(819, 0, EBX, 0, 12, L2_ITLB_ENTRIES_1G_PGS, NA, 0, FALSE) \ +FIELD(819, 0, EBX, 12, 4, L2_ITLB_ASSOC_1G_PGS, NA, 0, FALSE) \ +FIELD(819, 0, EBX, 16, 12, L2_DTLB_ENTRIES_1G_PGS, NA, 0, FALSE) \ +FIELD(819, 0, EBX, 28, 4, L2_DTLB_ASSOC_1G_PGS, NA, 0, FALSE) + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_81A \ +FLAG( 81A, 0, EAX, 0, 1, FP128, NA, 0, FALSE) \ +FLAG( 81A, 0, EAX, 1, 1, MOVU, NA, 0, FALSE) \ +FLAG( 81A, 0, EAX, 2, 1, FP256, NA, 0, FALSE) + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_81B \ +FLAG( 81B, 0, EAX, 0, 1, IBS_FFV, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 1, 1, IBS_FETCHSAM, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 2, 1, IBS_OPSAM, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 3, 1, RW_OPCOUNT, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 4, 1, OPCOUNT, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 5, 1, BRANCH_TARGET_ADDR, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 6, 1, OPCOUNT_EXT, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 7, 1, RIP_INVALID_CHECK, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 8, 1, OP_BRN_FUSE, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 9, 1, IBS_FETCH_CTL_EXTD, NA, 0, FALSE) \ +FLAG( 81B, 0, EAX, 10, 1, IBS_OP_DATA4, NA, 0, FALSE) + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_81C \ +FLAG( 81C, 0, EAX, 0, 1, LWP_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 1, 1, LWP_VAL_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 2, 1, LWP_IRE_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 3, 1, LWP_BRE_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 4, 1, LWP_DME_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 5, 1, LWP_CNH_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 6, 1, LWP_RNH_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 29, 1, LWP_CONT_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 30, 1, LWP_PTSC_AVAIL, NA, 0, FALSE) \ +FLAG( 81C, 0, EAX, 31, 1, LWP_INT_AVAIL, NA, 0, FALSE) \ +FIELD(81C, 0, EBX, 0, 8, LWP_CB_SIZE, NA, 0, FALSE) \ +FIELD(81C, 0, EBX, 8, 8, LWP_EVENT_SIZE, NA, 0, FALSE) \ +FIELD(81C, 0, EBX, 16, 8, LWP_MAX_EVENTS, NA, 0, FALSE) \ +FIELD(81C, 0, EBX, 24, 8, LWP_EVENT_OFFSET, NA, 0, FALSE) \ +FIELD(81C, 0, ECX, 0, 4, LWP_LATENCY_MAX, NA, 0, FALSE) \ +FLAG( 81C, 0, ECX, 5, 1, LWP_DATA_ADDR_VALID, NA, 0, FALSE) \ +FIELD(81C, 0, ECX, 6, 3, LWP_LATENCY_ROUND, NA, 0, FALSE) \ +FIELD(81C, 0, ECX, 9, 7, LWP_VERSION, NA, 0, FALSE) \ +FIELD(81C, 0, ECX, 16, 8, LWP_MIN_BUF_SIZE, NA, 0, FALSE) \ +FLAG( 81C, 0, ECX, 28, 1, LWP_BRANCH_PRED, NA, 0, FALSE) \ +FLAG( 81C, 0, ECX, 29, 1, LWP_IP_FILTERING, NA, 0, FALSE) \ +FLAG( 81C, 0, ECX, 30, 1, LWP_CACHE_LEVEL, NA, 0, FALSE) \ +FLAG( 81C, 0, ECX, 31, 1, LWP_CACHE_LATENCY, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 0, 1, LWP_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 1, 1, LWP_VAL_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 2, 1, LWP_IRE_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 3, 1, LWP_BRE_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 4, 1, LWP_DME_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 5, 1, LWP_CNH_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 6, 1, LWP_RNH_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 29, 1, LWP_CONT_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 30, 1, LWP_PTSC_SUPPORTED, NA, 0, FALSE) \ +FLAG( 81C, 0, EDX, 31, 1, LWP_INT_SUPPORTED, NA, 0, FALSE) + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_81D \ +FIELD(81D, 0, EAX, 0, 5, LEAF81D_CACHE_TYPE, NA, 0, FALSE) \ +FIELD(81D, 0, EAX, 5, 3, LEAF81D_CACHE_LEVEL, NA, 0, FALSE) \ +FLAG( 81D, 0, EAX, 8, 1, LEAF81D_CACHE_SELF_INIT, NA, 0, FALSE) \ +FLAG( 81D, 0, EAX, 9, 1, LEAF81D_CACHE_FULLY_ASSOC, NA, 0, FALSE) \ +FIELD(81D, 0, EAX, 14, 12, LEAF81D_NUM_SHARING_CACHE, NA, 0, FALSE) \ +FIELD(81D, 0, EBX, 0, 12, LEAF81D_CACHE_LINE_SIZE, NA, 0, FALSE) \ +FIELD(81D, 0, EBX, 12, 10, LEAF81D_CACHE_PHYS_PARTITIONS, NA, 0, FALSE) \ +FIELD(81D, 0, EBX, 22, 10, LEAF81D_CACHE_WAYS, NA, 0, FALSE) \ +FIELD(81D, 0, ECX, 0, 32, LEAF81D_CACHE_NUM_SETS, NA, 0, FALSE) \ +FLAG( 81D, 0, EDX, 0, 1, LEAF81D_CACHE_WBINVD, NA, 0, FALSE) \ +FLAG( 81D, 0, EDX, 1, 1, LEAF81D_CACHE_INCLUSIVE, NA, 0, FALSE) + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_81E \ +FIELD(81E, 0, EAX, 0, 32, EXTENDED_APICID, NA, 0, FALSE) \ +FIELD(81E, 0, EBX, 0, 8, COMPUTE_UNIT_ID, NA, 0, FALSE) \ +FIELD(81E, 0, EBX, 8, 2, CORES_PER_COMPUTE_UNIT, NA, 0, FALSE) \ +FIELD(81E, 0, ECX, 0, 8, NODEID_VAL, NA, 0, FALSE) \ +FIELD(81E, 0, ECX, 8, 3, NODES_PER_PKG, NA, 0, FALSE) + +/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, HWV, CPL3 */ +#define CPUID_FIELD_DATA_LEVEL_81F \ +FLAG( 81F, 0, EAX, 0, 1, SME, NO, 0, FALSE) \ +FLAG( 81F, 0, EAX, 1, 1, SEV, NO, 0, FALSE) \ +FLAG( 81F, 0, EAX, 2, 1, PAGE_FLUSH_MSR, NO, 0, FALSE) \ +FLAG( 81F, 0, EAX, 3, 1, SEV_ES, NO, 0, FALSE) \ +FIELD(81F, 0, EBX, 0, 5, SME_PAGE_TABLE_BIT_NUM, NO, 0, FALSE) \ +FIELD(81F, 0, EBX, 6, 6, SME_PHYS_ADDR_SPACE_REDUCTION, NO, 0, FALSE) \ +FIELD(81F, 0, ECX, 0, 32, NUM_ENCRYPTED_GUESTS, NO, 0, FALSE) \ +FIELD(81F, 0, EDX, 0, 32, SEV_MIN_ASID, NO, 0, FALSE) #define INTEL_CPUID_FIELD_DATA @@ -863,7 +1098,13 @@ FIELD(81E, 0, ECX, 8, 3, NODES_PER_PKG, NA, FALSE) CPUID_FIELD_DATA_LEVEL_A \ CPUID_FIELD_DATA_LEVEL_B \ CPUID_FIELD_DATA_LEVEL_D \ + CPUID_FIELD_DATA_LEVEL_F \ + CPUID_FIELD_DATA_LEVEL_10 \ CPUID_FIELD_DATA_LEVEL_12 \ + CPUID_FIELD_DATA_LEVEL_14 \ + CPUID_FIELD_DATA_LEVEL_15 \ + CPUID_FIELD_DATA_LEVEL_16 \ + CPUID_FIELD_DATA_LEVEL_17 \ CPUID_FIELD_DATA_LEVEL_400 \ CPUID_FIELD_DATA_LEVEL_401 \ CPUID_FIELD_DATA_LEVEL_402 \ @@ -877,8 +1118,18 @@ FIELD(81E, 0, ECX, 8, 3, NODES_PER_PKG, NA, FALSE) CPUID_FIELD_DATA_LEVEL_82 \ CPUID_FIELD_DATA_LEVEL_83 \ CPUID_FIELD_DATA_LEVEL_84 \ - CPUID_FIELD_DATA_LEVEL_8x \ - CPUID_FIELD_DATA_LEVEL_81x \ + CPUID_FIELD_DATA_LEVEL_85 \ + CPUID_FIELD_DATA_LEVEL_86 \ + CPUID_FIELD_DATA_LEVEL_87 \ + CPUID_FIELD_DATA_LEVEL_88 \ + CPUID_FIELD_DATA_LEVEL_8A \ + CPUID_FIELD_DATA_LEVEL_819 \ + CPUID_FIELD_DATA_LEVEL_81A \ + CPUID_FIELD_DATA_LEVEL_81B \ + CPUID_FIELD_DATA_LEVEL_81C \ + CPUID_FIELD_DATA_LEVEL_81D \ + CPUID_FIELD_DATA_LEVEL_81E \ + CPUID_FIELD_DATA_LEVEL_81F \ INTEL_CPUID_FIELD_DATA \ AMD_CPUID_FIELD_DATA @@ -891,20 +1142,19 @@ FIELD(81E, 0, ECX, 8, 3, NODES_PER_PKG, NA, FALSE) * * e.g. - CPUID_VIRT_BITS_MASK = 0xff00 * - CPUID_VIRT_BITS_SHIFT = 8 - * - * Note: The MASK definitions must use some gymnastics to get - * around a warning when shifting left by 32. */ -#define VMW_BIT_MASK(shift) (((1 << (shift - 1)) << 1) - 1) +#define VMW_BIT_MASK(shift) (0xffffffffu >> (32 - shift)) -#define FIELD(lvl, ecxIn, reg, bitpos, size, name, s, c3) \ + +#define FIELD(lvl, ecxIn, reg, bitpos, size, name, s, hwv, c3) \ CPUID_##name##_SHIFT = bitpos, \ CPUID_##name##_MASK = VMW_BIT_MASK(size) << bitpos, \ CPUID_INTERNAL_SHIFT_##name = bitpos, \ CPUID_INTERNAL_MASK_##name = VMW_BIT_MASK(size) << bitpos, \ CPUID_INTERNAL_REG_##name = CPUID_REG_##reg, \ CPUID_INTERNAL_EAXIN_##name = CPUID_LEVEL_VAL_##lvl, \ - CPUID_INTERNAL_ECXIN_##name = ecxIn, + CPUID_INTERNAL_ECXIN_##name = ecxIn, \ + CPUID_INTERNAL_HWV_##name = hwv, #define FLAG FIELD @@ -1114,8 +1364,9 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck, #define CPUID_FAMILY_K8MOBILE 0x11 #define CPUID_FAMILY_LLANO 0x12 #define CPUID_FAMILY_BOBCAT 0x14 -#define CPUID_FAMILY_BULLDOZER 0x15 // Bulldozer Piledriver Steamroller +#define CPUID_FAMILY_BULLDOZER 0x15 // BD PD SR EX #define CPUID_FAMILY_KYOTO 0x16 // Note: Jaguar microarch +#define CPUID_FAMILY_ZEN 0x17 /* Effective VIA CPU Families */ #define CPUID_FAMILY_C7 6 @@ -1157,12 +1408,21 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck, #define CPUID_MODEL_ATOM_4A 0x4a // Future Silvermont #define CPUID_MODEL_ATOM_4C 0x4c // Airmont #define CPUID_MODEL_ATOM_4D 0x4d // Avoton -#define CPUID_MODEL_SKYLAKE_4E 0x4e // Skylake-Y +#define CPUID_MODEL_SKYLAKE_4E 0x4e // Skylake-Y / Kabylake U/Y ES #define CPUID_MODEL_BROADWELL_4F 0x4f // Broadwell EP/EN/EX #define CPUID_MODEL_BROADWELL_56 0x56 // Broadwell DE +#define CPUID_MODEL_KNL_57 0x57 // Knights Landing #define CPUID_MODEL_ATOM_5A 0x5a // Future Silvermont #define CPUID_MODEL_ATOM_5D 0x5d // Future Silvermont -#define CPUID_MODEL_SKYLAKE_5E 0x5e // Skylake-S +#define CPUID_MODEL_SKYLAKE_5E 0x5e // Skylake-S / Kabylake S/H ES +#define CPUID_MODEL_ATOM_5F 0x5f // Denverton +#define CPUID_MODEL_KABYLAKE_8E 0x8e // Kabylake U/Y QS +#define CPUID_MODEL_KABYLAKE_9E 0x9e // Kabylake S/H QS + +/* Intel stepping information */ +#define CPUID_STEPPING_KABYLAKE_ES 0x8 // Kabylake S/H/U/Y ES +#define CPUID_STEPPING_COFFEELAKE_A 0xA // Coffeelake U/S/H +#define CPUID_STEPPING_COFFEELAKE_B 0xB // Coffeelake S/H #define CPUID_MODEL_PIII_07 7 #define CPUID_MODEL_PIII_08 8 @@ -1181,6 +1441,12 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck, #define CPUID_MODEL_PILEDRIVER_02 0x02 // family == CPUID_FAMILY_BULLDOZER #define CPUID_MODEL_OPTERON_REVF_41 0x41 // family == CPUID_FAMILY_K8 #define CPUID_MODEL_KYOTO_00 0x00 // family == CPUID_FAMILY_KYOTO +#define CPUID_MODEL_STEAMROLLER_3F 0x3F // Max Steamroller model defined in BKDG +#define CPUID_MODEL_STEAMROLLER_30 0x30 // family == CPUID_FAMILY_BULLDOZER +#define CPUID_MODEL_EXCAVATOR_60 0x60 // family == CPUID_FAMILY_BULLDOZER +#define CPUID_MODEL_EXCAVATOR_6F 0x6F // Max Excavator model defined in BKDG +#define CPUID_MODEL_ZEN_00 0x00 // family == CPUID_FAMILY_ZEN +#define CPUID_MODEL_ZEN_1F 0x1F // Max Zen model defined in BKDG /* VIA model information */ #define CPUID_MODEL_NANO 15 // Isaiah @@ -1247,6 +1513,12 @@ CPUID_EFFECTIVE_MODEL(uint32 v) /* %eax from CPUID with %eax=1. */ return m + (em << 4); } +static INLINE uint32 +CPUID_EFFECTIVE_STEPPING(uint32 v) /* %eax from CPUID with %eax=1. */ +{ + return CPUID_GET(1, EAX, STEPPING, v); +} + /* * Notice that CPUID families for Intel and AMD overlap. The following macros * should only be used AFTER the manufacturer has been established (through @@ -1371,19 +1643,48 @@ CPUID_MODEL_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1. { /* Assumes the CPU manufacturer is Intel. */ return CPUID_FAMILY_IS_P6(v) && - (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_5E || - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_4E); + ((CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_5E && + CPUID_EFFECTIVE_STEPPING(v) != CPUID_STEPPING_KABYLAKE_ES) || + (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_4E && + CPUID_EFFECTIVE_STEPPING(v) != CPUID_STEPPING_KABYLAKE_ES)); } +static INLINE Bool +CPUID_MODEL_IS_COFFEELAKE(uint32 v) // IN: %eax from CPUID with %eax=1. +{ + /* Assumes the CPU manufacturer is Intel. */ + return CPUID_FAMILY_IS_P6(v) && + ((CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_9E && + (CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_COFFEELAKE_A || + CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_COFFEELAKE_B)) || + (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_8E && + CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_COFFEELAKE_A)); +} + +static INLINE Bool +CPUID_MODEL_IS_KABYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1. +{ + /* Assumes the CPU manufacturer is Intel. */ + return CPUID_FAMILY_IS_P6(v) && + !CPUID_MODEL_IS_COFFEELAKE(v) && + (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_9E || + CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_8E || + (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_5E && + CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_KABYLAKE_ES) || + (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_4E && + CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_KABYLAKE_ES)); +} static INLINE Bool CPUID_UARCH_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1. { /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && CPUID_MODEL_IS_SKYLAKE(v); + return CPUID_FAMILY_IS_P6(v) && + (CPUID_MODEL_IS_COFFEELAKE(v) || + CPUID_MODEL_IS_KABYLAKE(v) || + CPUID_MODEL_IS_SKYLAKE(v)); } - static INLINE Bool CPUID_UARCH_IS_HASWELL(uint32 v) // IN: %eax from CPUID with %eax=1. { @@ -1409,6 +1710,30 @@ CPUID_MODEL_IS_AVOTON(uint32 v) // IN: %eax from CPUID with %eax=1. CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_4D; } +static INLINE Bool +CPUID_MODEL_IS_BAYTRAIL(uint32 v) // IN: %eax from CPUID with %eax=1. +{ + /* Assumes the CPU manufacturer is Intel. */ + return CPUID_FAMILY_IS_P6(v) && + CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_37; +} + +static INLINE Bool +CPUID_UARCH_IS_SILVERMONT(uint32 v) // IN: %eax from CPUID with %eax=1. +{ + /* Assumes the CPU manufacturer is Intel. */ + return CPUID_FAMILY_IS_P6(v) && + (CPUID_MODEL_IS_AVOTON(v) || CPUID_MODEL_IS_BAYTRAIL(v)); +} + +static INLINE Bool +CPUID_MODEL_IS_DENVERTON(uint32 v) // IN: %eax from CPUID with %eax=1. +{ + /* Assumes the CPU manufacturer is Intel. */ + return CPUID_FAMILY_IS_P6(v) && + CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_5F; +} + static INLINE Bool CPUID_MODEL_IS_WESTMERE(uint32 v) // IN: %eax from CPUID with %eax=1. { @@ -1446,6 +1771,15 @@ CPUID_MODEL_IS_IVYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1. } +static INLINE Bool +CPUID_MODEL_IS_KNIGHTS_LANDING(uint32 v) // IN: %eax from CPUID with %eax=1. +{ + /* Assumes the CPU manufacturer is Intel. */ + return CPUID_FAMILY_IS_P6(v) && + CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KNL_57; +} + + static INLINE Bool CPUID_FAMILY_IS_K7(uint32 eax) { @@ -1458,13 +1792,18 @@ CPUID_FAMILY_IS_K8(uint32 eax) return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K8; } +/* + *---------------------------------------------------------------------- + * + * CPUID_FAMILY_IS_K8EXT -- + * + * Return TRUE for family K8 with effective model >= 0x10. + * + *---------------------------------------------------------------------- + */ static INLINE Bool CPUID_FAMILY_IS_K8EXT(uint32 eax) { - /* - * We check for this pattern often enough that it's - * worth a separate function, for syntactic sugar. - */ return CPUID_FAMILY_IS_K8(eax) && CPUID_GET(1, EAX, EXTENDED_MODEL, eax) != 0; } @@ -1518,6 +1857,12 @@ CPUID_FAMILY_IS_KYOTO(uint32 eax) return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_KYOTO; } +static INLINE Bool +CPUID_FAMILY_IS_ZEN(uint32 eax) +{ + return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN; +} + /* * AMD Barcelona (of either Opteron or Phenom kind). */ @@ -1583,6 +1928,24 @@ CPUID_MODEL_IS_PILEDRIVER(uint32 eax) } +static INLINE Bool +CPUID_MODEL_IS_STEAMROLLER(uint32 eax) +{ + /* Steamroller is model 0x30 of family 0x15 (so far). */ + return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER && + (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_STEAMROLLER_30 && + CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_STEAMROLLER_3F); +} + + +static INLINE Bool +CPUID_MODEL_IS_EXCAVATOR(uint32 eax) +{ + /* Excavator is model 0x60 of family 0x15 (so far). */ + return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER && + (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_EXCAVATOR_60 && + CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_EXCAVATOR_6F); +} static INLINE Bool @@ -1593,6 +1956,14 @@ CPUID_MODEL_IS_KYOTO(uint32 eax) CPUID_EFFECTIVE_MODEL(eax) == CPUID_MODEL_KYOTO_00; } + +static INLINE Bool +CPUID_MODEL_IS_ZEN(uint32 eax) +{ + return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN && + CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN_1F; +} + #define CPUID_TYPE_PRIMARY 0 #define CPUID_TYPE_OVERDRIVE 1 #define CPUID_TYPE_SECONDARY 2 @@ -1708,8 +2079,20 @@ CPUID_IsHypervisorLevel(uint32 level) static INLINE Bool CPUID_LevelUsesEcx(uint32 level) { - return level == 4 || level == 7 || level == 0xb || level == 0xd || - level == 0x8000001d; + switch (level) + { + +#define CPUIDLEVEL(t, s, v, c, h) \ + case v: \ + return c != 0; + + CPUID_ALL_LEVELS + +#undef CPUIDLEVEL + + default: + return FALSE; + } } /* @@ -1777,7 +2160,18 @@ CPUID_SupportsMsrPlatformInfo(CpuidVendor vendor, uint32 version) CPUID_UARCH_IS_SANDYBRIDGE(version) || CPUID_UARCH_IS_HASWELL(version) || CPUID_UARCH_IS_SKYLAKE(version) || - CPUID_MODEL_IS_AVOTON(version)); + CPUID_MODEL_IS_KNIGHTS_LANDING(version) || + CPUID_MODEL_IS_DENVERTON(version) || + CPUID_UARCH_IS_SILVERMONT(version)); } +#ifdef _MSC_VER +#pragma warning (pop) #endif + + +#if defined __cplusplus +} // extern "C" +#endif + +#endif // _X86CPUID_H_ diff --git a/vmmon-only/include/x86cpuid_asm.h b/vmmon-only/include/x86cpuid_asm.h index 5a96ea6..e596acc 100644 --- a/vmmon-only/include/x86cpuid_asm.h +++ b/vmmon-only/include/x86cpuid_asm.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -63,6 +63,14 @@ extern "C" { */ void __cpuid(int regs[4], int eax); #pragma intrinsic(__cpuid) + +/* + * __cpuidex has been supported since VS2008 + */ +#if _MSC_VER >= 1500 +void __cpuidex(int regs[4], int eax, int ecx); +#pragma intrinsic(__cpuidex) +#endif /* _MSC_VER >= 1500 */ #endif /* VM_X86_64 */ #ifdef __cplusplus @@ -247,6 +255,20 @@ __GET_CPUID(int input, CPUIDRegs *regs) #ifdef VM_X86_64 +#if _MSC_VER >= 1500 + +/* + * __cpuidex has been supported since VS2008 + */ + +static INLINE void +__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs) +{ + __cpuidex((int *)regs, inputEax, inputEcx); +} + +#else // _MSC_VER >= 1500 + /* * No inline assembly in Win64. Implemented in bora/lib/misc in * cpuidMasm64.asm. @@ -254,6 +276,7 @@ __GET_CPUID(int input, CPUIDRegs *regs) extern void __GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs); +#endif // _MSC_VER >= 1500 #else // VM_X86_64 @@ -441,5 +464,13 @@ __GET_EAX_FROM_CPUID4(int inputEcx) _dx = regs.edx; \ } +#define GET_CPUID2(_ax, _bx, _cx, _dx) {\ + CPUIDRegs regs; \ + __GET_CPUID2(_ax, _cx, ®s); \ + _ax = regs.eax; \ + _bx = regs.ebx; \ + _cx = regs.ecx; \ + _dx = regs.edx; \ +} #endif diff --git a/vmmon-only/include/x86msr.h b/vmmon-only/include/x86msr.h index e1675bd..3a6bfe8 100644 --- a/vmmon-only/include/x86msr.h +++ b/vmmon-only/include/x86msr.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -38,6 +38,11 @@ #include "vm_basic_types.h" #include "community_source.h" +#if defined __cplusplus +extern "C" { +#endif + + /* * Results of calling rdmsr(msrNum) on all logical processors. */ @@ -92,6 +97,7 @@ MSRQuery; #define MSR_MCG_CAP 0x00000179 #define MSR_MCG_STATUS 0x0000017a #define MSR_MCG_CTL 0x0000017b +#define MSR_MCG_EXT_CTL 0x000004d0 #define MSR_EVNTSEL0 0x00000186 #define MSR_EVNTSEL1 0x00000187 #define MSR_FLEX_RATIO 0x00000194 // Intel Nehalem Family @@ -240,6 +246,133 @@ typedef enum { #define MSR_X2APIC_DIVIDER 0x0000083e #define MSR_X2APIC_SELFIPI 0x0000083f +#define MSR_BNDCFGS 0x00000d90 // Sup. mode bounds configuration + +#define MSR_XSS 0x00000da0 // Extended Supervisor State Mask + +/* RTIT MSRs */ +#define MSR_RTIT_CTL 0x00000570 +#define MSR_RTIT_STATUS 0x00000571 +#define MSR_RTIT_OUTPUT_BASE 0x00000560 +#define MSR_RTIT_OUTPUT_MASK_PTRS 0x00000561 +#define MSR_RTIT_CR3_MATCH 0x00000572 +#define MSR_RTIT_ADDR0_A 0x00000580 +#define MSR_RTIT_ADDR0_B 0x00000581 +#define MSR_RTIT_ADDR1_A 0x00000582 +#define MSR_RTIT_ADDR1_B 0x00000583 +#define MSR_RTIT_ADDR2_A 0x00000584 +#define MSR_RTIT_ADDR2_B 0x00000585 +#define MSR_RTIT_ADDR3_A 0x00000586 +#define MSR_RTIT_ADDR3_B 0x00000587 + +#define MSR_RTIT_NUM_OF_ADDR_REG 8 + +/* RTIT control MSR bits */ +#define MSR_RTIT_CTL_TRACE_EN (1LL<<0) // Enable tracing +#define MSR_RTIT_CTL_CYC_EN (1LL<<1) // Enable CYC Packet +#define MSR_RTIT_CTL_OS (1LL<<2) // CPL0 filter +#define MSR_RTIT_CTL_USER (1LL<<3) // CPL > 0 filter +#define MSR_RTIT_CTL_FABRIC_EN (1LL<<6) // Trace output direction +#define MSR_RTIT_CTL_CR3_FILTER_EN (1LL<<7) // Enable CR3 filter +#define MSR_RTIT_CTL_TOPA (1LL<<8) // Enable ToPA output scheme +#define MSR_RTIT_CTL_MTC_EN (1LL<<9) // Enable MTC packet +#define MSR_RTIT_CTL_TSC_EN (1LL<<10) // Enable TSC packet +#define MSR_RTIT_CTL_DIS_RETC (1LL<<11) // Disable RET compression +#define MSR_RTIT_CTL_BRANCH_EN (1LL<<13) // Disable COFI-based packets +#define MSR_RTIT_CTL_RSVD CONST64U(0xffff0000f0841030) // Reserved bits + +#define MSR_RTIT_CTL_MTCFREQ_MASK CONST64U(0xf) +#define MSR_RTIT_CTL_MTCFREQ_SHIFT 14 +#define MSR_RTIT_CTL_MTCFREQ(_msr) \ + (((_msr) >> MSR_RTIT_CTL_MTCFREQ_SHIFT) & MSR_RTIT_CTL_MTCFREQ_MASK) + +#define MSR_RTIT_CTL_CYCTHRESH_MASK CONST64U(0xf) +#define MSR_RTIT_CTL_CYCTHRESH_SHIFT 19 +#define MSR_RTIT_CTL_CYCTHRESH(_msr) \ + (((_msr) >> MSR_RTIT_CTL_CYCTHRESH_SHIFT) & MSR_RTIT_CTL_CYCTHRESH_MASK) + +#define MSR_RTIT_CTL_PSBFREQ_MASK CONST64U(0xf) +#define MSR_RTIT_CTL_PSBFREQ_SHIFT 24 +#define MSR_RTIT_CTL_PSBFREQ(_msr) \ + (((_msr) >> MSR_RTIT_CTL_PSBFREQ_SHIFT) & MSR_RTIT_CTL_PSBFREQ_MASK) + +#define MSR_RTIT_CTL_ADDR0_CFG_MASK CONST64U(0xf) +#define MSR_RTIT_CTL_ADDR0_CFG_SHIFT 32 +#define MSR_RTIT_CTL_ADDR0_CFG(_msr) \ + (((_msr) >> MSR_RTIT_CTL_ADDR0_CFG_SHIFT) & MSR_RTIT_CTL_ADDR0_CFG_MASK) + +#define MSR_RTIT_CTL_ADDR1_CFG_MASK CONST64U(0xf) +#define MSR_RTIT_CTL_ADDR1_CFG_SHIFT 36 +#define MSR_RTIT_CTL_ADDR1_CFG(_msr) \ + (((_msr) >> MSR_RTIT_CTL_ADDR1_CFG_SHIFT) & MSR_RTIT_CTL_ADDR1_CFG_MASK) + +#define MSR_RTIT_CTL_ADDR2_CFG_MASK CONST64U(0xf) +#define MSR_RTIT_CTL_ADDR2_CFG_SHIFT 40 +#define MSR_RTIT_CTL_ADDR2_CFG(_msr) \ + (((_msr) >> MSR_RTIT_CTL_ADDR2_CFG_SHIFT) & MSR_RTIT_CTL_ADDR2_CFG_MASK) + +#define MSR_RTIT_CTL_ADDR3_CFG_MASK CONST64U(0xf) +#define MSR_RTIT_CTL_ADDR3_CFG_SHIFT 44 +#define MSR_RTIT_CTL_ADDR3_CFG(_msr) \ + (((_msr) >> MSR_RTIT_CTL_ADDR3_CFG_SHIFT) & MSR_RTIT_CTL_ADDR3_CFG_MASK) + +/* RTIT status MSR bits */ +#define MSR_RTIT_STATUS_FILTER_EN (1LL<<0) +#define MSR_RTIT_STATUS_CONTEXT_EN (1LL<<1) +#define MSR_RTIT_STATUS_TRIGGER_EN (1LL<<2) +#define MSR_RTIT_STATUS_ERROR (1LL<<4) +#define MSR_RTIT_STATUS_STOPPED (1LL<<5) +#define MSR_RTIT_STATUS_RSVD CONST64U(0xfffe0000ffffffc8) + +#define MSR_RTIT_STATUS_PKT_BYTES_CNT_MASK CONST64U(0x1ffff) +#define MSR_RTIT_STATUS_PKT_BYTES_CNT_SHIFT 32 +#define MSR_RTIT_STATUS_PKT_BYTES_CNT(_msr) \ + (((_msr) >> MSR_RTIT_STATUS_PKT_BYTES_CNT_SHIFT) & \ + MSR_RTIT_STATUS_PKT_BYTES_CNT_MASK) + +/* RTIT CR3 MSR bits */ +#define MSR_RTIT_CR3_MATCH_MASK CONST64U(~0x1f) + +/* RTIT output base MSR bits */ +#define MSR_RTIT_OUTPUT_BASE_MASK CONST64U(~0x7f) +#define MSR_RTIT_OUTPUT_BASE_RSVD CONST64U(0x7f) + +/* RTIT output mask ptrs MSR bits */ +#define MSR_RTIT_OUTPUT_MASK_LOWERMASK CONST64U(0x7f) + +/* + * Get the mask value for the single contiguous output region + * when MSR_RTIT_CTL.ToPA is masked + */ +#define MSR_RTIT_OUTPUT_MASK_MASK CONST64U(0xffffffff) +#define MSR_RTIT_OUTPUT_MASK(_msr) \ + ((_msr) & MSR_RTIT_OUTPUT_MASK_MASK) + +/* Get the offset pointer of the ToPA table when MSR_RTIT_CTL.ToPA is set */ +#define MSR_RTIT_OUTPUT_TABLE_OFFSET_MASK CONST64U(0x1ffffff) +#define MSR_RTIT_OUTPUT_TABLE_OFFSET_SHIFT 7 +#define MSR_RTIT_OUTPUT_TABLE_OFFSET(_msr) \ + ((((_msr) >> MSR_RTIT_OUTPUT_TABLE_OFFSET_SHIFT) & \ + MSR_RTIT_OUTPUT_TABLE_OFFSET_MASK) << 3) +#define MSR_RTIT_OUTPUT_TABLE_ENTRY(_msr) \ + (((_msr) >> MSR_RTIT_OUTPUT_TABLE_OFFSET_SHIFT) & \ + MSR_RTIT_OUTPUT_TABLE_OFFSET_MASK) + +#define MSR_RTIT_OUTPUT_OFFSET_MASK CONST64U(0xffffffff) +#define MSR_RTIT_OUTPUT_OFFSET_SHIFT 32 +#define MSR_RTIT_OUTPUT_OFFSET(_msr) \ + (((_msr) >> MSR_RTIT_OUTPUT_OFFSET_SHIFT) & \ + MSR_RTIT_OUTPUT_OFFSET_MASK) + +#define MSR_RTIT_CR3_MATCH_RSVD CONST64U(0x1f) + +/* SGX MSRs */ +#define MSR_SGX_SVN_STATUS 0x00000500 + +/* SGX SVN status MSR fields */ +#define MSR_SGX_SVN_STATUS_LOCK 0x1 +#define MSR_SGX_SVN_STATUS_SINIT_SVN CONST64U(0xff0000) +#define MSR_SGX_SVN_STATUS_RSVD CONST64U(0xffffffffff00fffe) /* MSR_CR_PAT power-on value */ #define MSR_CR_PAT_DEFAULT 0x0007040600070406ULL @@ -277,7 +410,10 @@ typedef enum { #define MSR_FEATCTL_LOCK 0x00000001 #define MSR_FEATCTL_SMXE 0x00000002 #define MSR_FEATCTL_VMXE 0x00000004 +#define MSR_FEATCTL_SENTERP 0x00007F00 +#define MSR_FEATCTL_SENTERE 0x00008000 #define MSR_FEATCTL_SGXE 0x00040000 +#define MSR_FEATCTL_LMCE 0x00100000 /* MSR_EFER bits. */ #define MSR_EFER_SCE 0x0000000000000001ULL /* Sys call ext'ns: r/w */ @@ -423,6 +559,7 @@ typedef enum { #define MSR_HYPERV_HYPERCALL_EN 1ULL #define MSR_HYPERV_REFERENCE_TSC_EN 1ULL +#define MSR_HYPERV_VP_ASSIST_EN 1ULL #define MSR_HYPERV_GUESTOSID_VENDOR_SHIFT 48 #define MSR_HYPERV_GUESTOSID_VENDOR_MASK 0xfULL @@ -481,6 +618,14 @@ typedef enum { +/* + * Platform Quality of Service + */ + +#define MSR_INTEL_QOS_EVTSEL 0xc8d +#define MSR_INTEL_QOS_CTR 0xc8e +#define MSR_INTEL_QOS_ASSOC 0xc8f + static INLINE uint32 X86MSR_SysCallEIP(uint64 star) { @@ -502,4 +647,8 @@ X86MSR_SysRetCS(uint64 star) } +#if defined __cplusplus +} +#endif + #endif /* _X86MSR_H_ */ diff --git a/vmmon-only/include/x86paging_64.h b/vmmon-only/include/x86paging_64.h new file mode 100644 index 0000000..a8e0946 --- /dev/null +++ b/vmmon-only/include/x86paging_64.h @@ -0,0 +1,87 @@ +/********************************************************* + * Copyright (C) 1998-2014,2016 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * x86paging_64.h -- + * + * Contains definitions for the x86 page table layout specific to + * long mode. + */ + +#ifndef _X86PAGING_64_H_ +#define _X86PAGING_64_H_ + +#define INCLUDE_ALLOW_USERLEVEL + +#define INCLUDE_ALLOW_MODULE +#define INCLUDE_ALLOW_VMMON +#define INCLUDE_ALLOW_VMK_MODULE +#define INCLUDE_ALLOW_VMKERNEL +#define INCLUDE_ALLOW_DISTRIBUTE +#define INCLUDE_ALLOW_VMCORE +#include "includeCheck.h" + +#include "vm_basic_types.h" +#include "x86types.h" +#include "vm_pagetable.h" + +#define LM_PTE_PFN_MASK CONST64U(0xffffffffff000) +#define LM_PTE_2_PFN(_pte) (((_pte) & LM_PTE_PFN_MASK) >> PT_PTE_PFN_SHIFT) + +#define LM_PDE_PFN_MASK 0xfffffffe00000LL +#define LM_PDPTE_PFN_MASK 0xfffffc0000000LL + +#define LM_AVAIL_SHIFT 9 + +#define LM_AVAIL_MASK (CONST64(0x7) << LM_AVAIL_SHIFT) +#define LM_FLAGS_MASK CONST64(0x80000000000001ff) +#define LM_CR3_FLAGS_MASK CONST64(0x18) +#define LM_L4_L3_RSVD_MASK CONST64(0x180) +#define LM_L3_1G_RSVD_MASK CONST64(0x3fffe000) + +#define LM_MAKE_CR3(_mpfn, _flags) \ + (((uint64)(_mpfn) << PT_PTE_PFN_SHIFT) | \ + ((_flags) & LM_CR3_FLAGS_MASK)) + +#define LM_MAKE_PTE(_mpfn, _avail, _flags) \ + (((uint64)(_mpfn) << PT_PTE_PFN_SHIFT) | \ + (((_avail) << LM_AVAIL_SHIFT) & LM_AVAIL_MASK) | \ + ((uint64)(_flags) & LM_FLAGS_MASK)) + +#define LM_MAKE_PDE(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags) +#define LM_MAKE_L4E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags) +#define LM_MAKE_L3E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags) +#define LM_MAKE_L2E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags) +#define LM_MAKE_L1E(_pfn, _avail, _flags) LM_MAKE_PTE(_pfn, _avail, _flags) + + +/* + * x86-64 architecture requires implementations supporting less than + * full 64-bit VAs to ensure that all virtual addresses are in canonical + * form. An address is in canonical form if the address bits from the + * most significant implemented bit up to bit 63 are all ones or all + * zeros. If this is not the case, the processor generates #GP/#SS. Our + * VCPU implements 48 bits of virtual address space. + */ + +#define VA64_IMPL_BITS 48 +#define VA64_CANONICAL_MASK ~((CONST64U(1) << (VA64_IMPL_BITS - 1)) - 1) +#define VA64_CANONICAL_HOLE_START (CONST64U(1) << (VA64_IMPL_BITS - 1)) +#define VA64_CANONICAL_HOLE_LEN VA64_CANONICAL_MASK - VA64_CANONICAL_HOLE_START + +#endif /* _X86PAGING_64_H_ */ diff --git a/vmmon-only/include/x86paging_common.h b/vmmon-only/include/x86paging_common.h new file mode 100644 index 0000000..a93fd41 --- /dev/null +++ b/vmmon-only/include/x86paging_common.h @@ -0,0 +1,113 @@ +/********************************************************* + * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * x86paging_common.h -- + * + * Contains definitions for the x86 page table layout that are common + * to legacy and long modes. + */ + +#ifndef _X86PAGING_COMMON_H_ +#define _X86PAGING_COMMON_H_ + +#define INCLUDE_ALLOW_USERLEVEL + +#define INCLUDE_ALLOW_MODULE +#define INCLUDE_ALLOW_VMMON +#define INCLUDE_ALLOW_VMK_MODULE +#define INCLUDE_ALLOW_VMKERNEL +#define INCLUDE_ALLOW_DISTRIBUTE +#define INCLUDE_ALLOW_VMCORE +#include "includeCheck.h" + +#include "vm_basic_types.h" + + +/* + * page tables + */ + +#define PTE_P 0x00000001 +#define PTE_RW 0x00000002 +#define PTE_US 0x00000004 +#define PTE_PWT 0x00000008 +#define PTE_PCD 0x00000010 // careful: MMU_PTE_EX has same value. +#define PTE_A 0x00000020 +#define PTE_D 0x00000040 +#define PTE_PS 0x00000080 +#define PTE_PAT 0x00000080 +#define PTE_G 0x00000100 +#define PTE_LARGE_PAT 0x00001000 +#define PTE_RSVD CONST64(0x0008000000000000) +#define PTE_NX CONST64(0x8000000000000000) +#define PTE_NX_SHIFT 63 + +#define PTE_PK_MASK CONST64(0x7800000000000000) +#define PTE_PK_SHIFT 59 + +#define PTE_LARGE_PAT_2_PS(_pte) (((_pte) & PTE_LARGE_PAT) >> 5) + +#define PTE_PROT_FLAGS (PTE_P|PTE_RW|PTE_US) +#define PTE_FLAGS (PTE_PROT_FLAGS|PTE_G) + +#define PTE_KERNEL (PTE_P|PTE_RW) +#define PTE_KERNEL_NX (PTE_P|PTE_RW|PTE_NX) + +#define PTE_PRESENT(_pte) (((_pte) & PTE_P) != 0) +#define PTE_WRITEABLE(_pte) (((_pte) & PTE_RW) != 0) +#define PTE_ACCESS(_pte) (((_pte) & PTE_A) != 0) +#define PTE_DIRTY(_pte) (((_pte) & PTE_D) != 0) +#define PTE_USER(_pte) (((_pte) & PTE_US) != 0) +#define PTE_GLOBAL(_pte) (((_pte) & PTE_G) != 0) +#define PTE_LARGEPAGE(_pte) (((_pte) & PTE_PS) != 0) +#define PTE_UNCACHED(_pte) (((_pte) & PTE_PCD) != 0) +#define PTE_NOEXECUTE(_pte) (((_pte) & PTE_NX) != 0) +#define PTE_PK(_pte) (((_pte) & PTE_PK_MASK) >> PTE_PK_SHIFT) + +#define PK_AD 1 /* Access disable bit */ +#define PK_WD 2 /* Write disable bit */ +#define PKR_WIDTH 2 +#define PKR_MASK 3 +#define PK_RIGHTS(_pkru, _key) (((_pkru) >> ((_key) * PKR_WIDTH)) & PKR_MASK); + +/* Error code flags */ +#define PF_P 0x0001 +#define PF_RW 0x0002 +#define PF_US 0x0004 +#define PF_RSVD 0x0008 +#define PF_ID 0x0010 +#define PF_PK 0x0020 +#define PF_SGX 0x8000 + +/* + * Operand definitions for the INVPCID instruction. See SDM Vol. 2A. + */ +#define INVPCID_EXTENT_ADDR 0 +#define INVPCID_EXTENT_PCID_CTX 1 +#define INVPCID_EXTENT_ALL_CTX 2 +#define INVPCID_EXTENT_ALL_CTX_LOCAL 3 +#define INVPCID_EXTENT_MAX 3 +#define INVPCID_MAX_PCID 0xfff + +typedef struct { + uint64 pcid; /* Bits 63:12 reserved. */ + uint64 la; /* Used for single address invalidation. */ +} InvpcidArg; + +#endif /* _X86PAGING_COMMON_H_ */ diff --git a/vmmon-only/include/x86perfctr.h b/vmmon-only/include/x86perfctr.h index 606c898..5ab6c8c 100644 --- a/vmmon-only/include/x86perfctr.h +++ b/vmmon-only/include/x86perfctr.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2012,2014-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2012,2014-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -39,7 +39,7 @@ #define PERFCTR_PENTIUM4_NUM_COUNTERS 18 #define PERFCTR_PENTIUM4_NUM_COUNTERS_WITH_L3 26 #define PERFCTR_AMD_NUM_COUNTERS 4 -#define PERFCTR_BD_NUM_COUNTERS 6 +#define PERFCTR_AMD_EXT_NUM_COUNTERS 6 #define PERFCTR_P6_NUM_COUNTERS 2 #define PERFCTR_NEHALEM_NUM_GEN_COUNTERS 4 #define PERFCTR_NEHALEM_NUM_FIXED_COUNTERS 3 @@ -109,10 +109,10 @@ /* AMD Performance Counter MSR Definitions */ #define PERFCTR_AMD_PERFEVTSEL0_ADDR 0xC0010000 #define PERFCTR_AMD_PERFCTR0_ADDR 0xC0010004 -#define PERFCTR_BD_BASE_ADDR 0xC0010200 -#define PERFCTR_BD_EVENTSEL 0 -#define PERFCTR_BD_CTR 1 -#define PERFCTR_BD_MSR_STRIDE 2 +#define PERFCTR_AMD_EXT_BASE_ADDR 0xC0010200 +#define PERFCTR_AMD_EXT_EVENTSEL 0 +#define PERFCTR_AMD_EXT_CTR 1 +#define PERFCTR_AMD_EXT_MSR_STRIDE 2 /* AMD Clocks */ #define PERFCTR_AMD_CPU_CLK_UNHALTED 0x76 @@ -144,10 +144,6 @@ #define PERFCTR_AMD_L2_MISS 0x7e #define PERFCTR_AMD_L2_FILL_WRITEBACK 0x7f -/* AMD L3 Cache */ -#define PERFCTR_AMD_REQUESTS_TO_L3 (0x4e0 | 0xf000) -#define PERFCTR_AMD_L3_MISS (0x4e1 | 0xf000) - /* AMD Instruction Cache Events */ #define PERFCTR_AMD_INSTR_FETCHES 0x80 @@ -395,7 +391,7 @@ * Use CPUID 0xa to get perf capabilities. Some (7) events are * architectural; most are version-specific. * - * V1: Yonah, V2: Merom, V2+: Penryn + * V1: Yonah, V2: Merom, V2+: Penryn, V3: Nehalem, V4: Skylake * * V1 is similar to P6, with some additions: * - Global control MSR @@ -404,6 +400,14 @@ * - Ability to freeze a counter before PMI delivery. * - Freeze during SMI (Penryn+ only) * - VMCS global enable (Penryn+ only) + * V3 introduces: + * - (nothing we virtualize) + * V4 introduces: + * - Global unavailable (in-use) MSR + * - Freeze on PMI bit + * - ASCI bit + * - Global status set MSR + * - Global status reset MSR (previously called Ovf ctrl) * * ---------------------------------------------------------------------- */ @@ -443,6 +447,7 @@ #define PERFCTR_CORE_PMI_UNAVAILABLE_IN_USE (CONST64U(1) << 63) // XXX serebrin/dhecht: 1-10-11: Make ANYTHREAD depend on number of fixed PMCs +#define PERFCTR_CORE_GLOBAL_STATUS_TOPA_PMI (1ULL << 55) #define PERFCTR_CORE_GLOBAL_STATUS_CTR_FRZ (1ULL << 59) #define PERFCTR_CORE_GLOBAL_STATUS_ASCI (1ULL << 60) #define PERFCTR_CORE_GLOBAL_STATUS_OVFBUFFER (1ULL << 62) @@ -632,7 +637,7 @@ * escrAddr: MSR # of the Perf Event Selector (0xc0010000 + index). * escrVal: Value placed in PerfEvtSel MSR; what to measure. * - * On AMD BD: + * On AMD with PerfCtrExtCore support: * index: Which perf ctr, 0 to 5. RDPMC argument * addr: MSR of raw perf ctr (0xc0010201 + 2 * index). * aliased PMCs 0 - 3 (0xc0010004 + index). @@ -731,13 +736,17 @@ PerfCtr_PgcToOvfValidBits(uint64 pgcValBits) static INLINE uint64 PerfCtr_PgcToStsRstValidBits(uint64 pgcValBits) { - return pgcValBits | (1ULL << 55) | MASKRANGE64(63, 58); + return pgcValBits | + PERFCTR_CORE_GLOBAL_STATUS_TOPA_PMI | + MASKRANGE64(63, 58); } static INLINE uint64 PerfCtr_PgcToGssValidBits(uint64 pgcValBits) { - return pgcValBits | (1ULL << 55) | MASKRANGE64(62, 58); + return pgcValBits | + PERFCTR_CORE_GLOBAL_STATUS_TOPA_PMI | + MASKRANGE64(62, 58); } /* @@ -815,6 +824,34 @@ PerfCtr_PEBSAvailable(void) return FALSE; } +/* + *---------------------------------------------------------------------- + * + * PerfCtr_PTAvailable -- + * + * Checks if this CPU is capable of PT(Intel processor trace). + * + * Results: + * TRUE if PT is supported, FALSE otherwise. + * + * Side effects: + * None + * + *---------------------------------------------------------------------- + */ + +static INLINE Bool +PerfCtr_PTAvailable(void) +{ + CPUIDRegs regs; + __GET_CPUID(0, ®s); + if (CPUID_IsVendorIntel(®s)) { + __GET_CPUID2(7, 0, ®s); + return (regs.ecx & CPUID_INTERNAL_MASK_PT) != 0; + } + return FALSE; +} + /* The following are taken from the Intel Architecture Manual, * Book 3, Table 14-4. */ diff --git a/vmmon-only/include/x86sel.h b/vmmon-only/include/x86sel.h index 934bfde..8399368 100644 --- a/vmmon-only/include/x86sel.h +++ b/vmmon-only/include/x86sel.h @@ -35,6 +35,11 @@ #define INCLUDE_ALLOW_VMCORE #include "includeCheck.h" +#if defined __cplusplus +extern "C" { +#endif + + #define SELECTOR_GDT 0 #define SELECTOR_LDT 1 #define SELECTOR_RPL_SHIFT 0 @@ -64,4 +69,9 @@ MAKE_SELECTOR(unsigned index, unsigned ti, unsigned rpl) return MAKE_SELECTOR_UNCHECKED(index, ti, rpl); } + +#if defined __cplusplus +} // extern "C" +#endif + #endif /* !defined _X86SEL_H_ */ diff --git a/vmmon-only/include/x86svm.h b/vmmon-only/include/x86svm.h index 584a4af..dc5d60a 100644 --- a/vmmon-only/include/x86svm.h +++ b/vmmon-only/include/x86svm.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2005-2014 VMware, Inc. All rights reserved. + * Copyright (C) 2005-2014,2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -47,10 +47,6 @@ #endif #include "x86cpuid_asm.h" -#ifdef VMM -#include "cpuidMonitor.h" -#endif - #define SVM_VMCB_IO_BITMAP_SIZE (3 * PAGE_SIZE) #define SVM_VMCB_MSR_BITMAP_SIZE (2 * PAGE_SIZE) diff --git a/vmmon-only/include/x86types.h b/vmmon-only/include/x86types.h index cf7966e..c070e4e 100644 --- a/vmmon-only/include/x86types.h +++ b/vmmon-only/include/x86types.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -39,16 +39,7 @@ #include "vm_basic_defs.h" #include "address_defs.h" #include "x86segdescrs.h" - -/* - * Types used for PL4 page table in x86_64 and arm64 - */ - -typedef uint64 VM_L4E; -typedef uint64 VM_L3E; -typedef uint64 VM_L2E; -typedef uint64 VM_L1E; - +#include "vm_pagetable.h" /* * Page table diff --git a/vmmon-only/include/x86vendor.h b/vmmon-only/include/x86vendor.h index 0c9ccdd..62248ec 100644 --- a/vmmon-only/include/x86vendor.h +++ b/vmmon-only/include/x86vendor.h @@ -1,6 +1,6 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmmon-only/include/x86vt-exit-reasons.h b/vmmon-only/include/x86vt-exit-reasons.h new file mode 100644 index 0000000..07d6cf6 --- /dev/null +++ b/vmmon-only/include/x86vt-exit-reasons.h @@ -0,0 +1,97 @@ +/********************************************************* + * Copyright (C) 2016 VMware, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2 and no later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *********************************************************/ + +/* + * x86vt-exit-reasons.h -- + * + * VT exit reasons. + * + */ + +#include "community_source.h" + +#define VT_EXITREASON_VMENTRYFAIL (1U << 31) + +VT_EXIT(EXC_OR_NMI, 0) +VT_EXIT(EXTINT, 1) +VT_EXIT(TRIPLEFAULT, 2) +VT_EXIT(INIT, 3) +VT_EXIT(SIPI, 4) +VT_EXIT(IOSMI, 5) +VT_EXIT(OTHERSMI, 6) +VT_EXIT(VINTR_WINDOW, 7) +VT_EXIT(VNMI_WINDOW, 8) +VT_EXIT(TS, 9) +VT_EXIT(CPUID, 10) +VT_EXIT(GETSEC, 11) +VT_EXIT(HLT, 12) +VT_EXIT(INVD, 13) +VT_EXIT(INVLPG, 14) +VT_EXIT(RDPMC, 15) +VT_EXIT(RDTSC, 16) +VT_EXIT(RSM, 17) +VT_EXIT(VMCALL, 18) +VT_EXIT(VMCLEAR, 19) +VT_EXIT(VMLAUNCH, 20) +VT_EXIT(VMPTRLD, 21) +VT_EXIT(VMPTRST, 22) +VT_EXIT(VMREAD, 23) +VT_EXIT(VMRESUME, 24) +VT_EXIT(VMWRITE, 25) +VT_EXIT(VMXOFF, 26) +VT_EXIT(VMXON, 27) +VT_EXIT(CR, 28) +VT_EXIT(DR, 29) +VT_EXIT(IO, 30) +VT_EXIT(RDMSR, 31) +VT_EXIT(WRMSR, 32) +VT_EXIT(VMENTRYFAIL_GUEST, (33 | VT_EXITREASON_VMENTRYFAIL)) +VT_EXIT(VMENTRYFAIL_MSR, (34 | VT_EXITREASON_VMENTRYFAIL)) +VT_EXIT(VMEXIT35, 35) +VT_EXIT(MWAIT, 36) +VT_EXIT(MTF, 37) +VT_EXIT(VMEXIT38, 38) +VT_EXIT(MONITOR, 39) +VT_EXIT(PAUSE, 40) +VT_EXIT(VMENTRYFAIL_MC, (41 | VT_EXITREASON_VMENTRYFAIL)) +VT_EXIT(VMEXIT42, 42) +VT_EXIT(TPR, 43) +VT_EXIT(APIC, 44) +VT_EXIT(EOI, 45) +VT_EXIT(GDTR_IDTR, 46) +VT_EXIT(LDTR_TR, 47) +VT_EXIT(EPT_VIOLATION, 48) +VT_EXIT(EPT_MISCONFIG, 49) +VT_EXIT(INVEPT, 50) +VT_EXIT(RDTSCP, 51) +VT_EXIT(TIMER, 52) +VT_EXIT(INVVPID, 53) +VT_EXIT(WBINVD, 54) +VT_EXIT(XSETBV, 55) +VT_EXIT(APIC_WRITE, 56) +VT_EXIT(RDRAND, 57) +VT_EXIT(INVPCID, 58) +VT_EXIT(VMFUNC, 59) +VT_EXIT(ENCLS, 60) +VT_EXIT(RDSEED, 61) +VT_EXIT(PML_LOGFULL, 62) +VT_EXIT(XSAVES, 63) +VT_EXIT(XRSTORS, 64) + +/* Bump this up if you add an exit reason. */ +#define VT_NUM_EXIT_REASONS 65 diff --git a/vmmon-only/include/x86vt-vmcs-fields.h b/vmmon-only/include/x86vt-vmcs-fields.h index 321b533..74b9b4c 100644 --- a/vmmon-only/include/x86vt-vmcs-fields.h +++ b/vmmon-only/include/x86vt-vmcs-fields.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2015 VMware, Inc. All rights reserved. + * Copyright (C) 2015-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -19,195 +19,697 @@ /* * x86vt-vmcs-fields.h -- * - * VMCS encodings; SDM volume 3 Appendix B. - * These are the values passed to VMWRITE and VMREAD. + * VMCS fields (SDM volume 3 Appendix B). * - * VMCS_FIELD(name, encoding) */ #include "community_source.h" /* - * VMCS_FIELD(name, encoding, vVT support) + * There are 16 groups of VMCS fields: 4 sizes crossed with 4 types. + * The 4 sizes are 16-bit, 64-bit, 32-bit, and "natural", in that order, + * per the enumeration of the size in the field encoding. + * The 4 types are control, read-only data, guest-state, and host-state, + * in that order, per the enumeration of the type in the field encoding. */ +/* + * Use of this table: + * A typical consumer will define a VMCS_FIELD macro to extract and + * collate the desired information for each VMCS field, and then it will + * include this file to invoke that macro with the property list for each + * VMCS field. Some properties are represented mnemonically (e.g. _S16 for + * "16-bit size," so that the extracted property can be defined by another + * macro. For example, some consumers may choose to interpret the size + * property "_S16" as "uint16", while others may wish to interpret it as "s16." + * For convenience, each of the 16 groups is preceded by a VMCS_GROUP_START + * macro and followed by a VMCS_GROUP_END macro. Similarly, each set of four + * groups with the same size is preceded by a VMCS_SET_START macro and followed + * by a VMCS_SET_END macro. This file provides empty definitions of + * VMCS_GROUP_START, VMCS_GROUP_END, VMCS_SET_START, and VMCS_SET_END if the + * consumer has not defined them. + */ + +#ifndef VMCS_GROUP_START +#define VMCS_GROUP_START(size, type) +#define LOCAL_GROUP_START +#endif + +#ifndef VMCS_GROUP_END +#define VMCS_GROUP_END(size, type) +#define LOCAL_GROUP_END +#endif + +#ifndef VMCS_SET_START +#define VMCS_SET_START(size) +#define LOCAL_SET_START +#endif + +#ifndef VMCS_SET_END +#define VMCS_SET_END(size) +#define LOCAL_SET_END +#endif + +#ifndef VMCS_UNUSED +#define VMCS_UNUSED(val, size, type, index) +#define LOCAL_UNUSED +#endif + +/* + * VMCS_FIELD(_name, _val, _size, _type, _index, _cache, _vvt, _access) + * _type is one of: + * _TC -- control + * _TG -- guest-state + * _TH -- host-state + * _TD -- read-only data + * _size is one of: + * _S16 -- 16-bits + * _S32 -- 32-bits + * _S64 -- 64-bits + * _SN -- natural width + * _cache is one of: + * _C -- cached + * _NC -- not cached + * _vvt is one of: + * _V -- virtualized + * _NV -- not virtualized + * _access is one of: + * _NA -- no access + * _RW -- read/write access + * _RO -- read-only access + * + */ + + +VMCS_SET_START(16) /* 16-bit control fields. */ -VMCS_FIELD(VPID, 0x0000, TRUE) -VMCS_FIELD(PI_NOTIFY, 0x0002, FALSE) -VMCS_FIELD(EPTP_INDEX, 0x0004, FALSE) +VMCS_GROUP_START(16, CTL) +VMCS_FIELD(VPID, 0x0000, _S16, _TC, 0, _C, _V, _NA) +VMCS_FIELD(PI_NOTIFY, 0x0002, _S16, _TC, 1, _NC, _NV, _NA) +VMCS_FIELD(EPTP_INDEX, 0x0004, _S16, _TC, 2, _NC, _V, _NA) +VMCS_UNUSED( 0x0006, _S16, _TC, 3) +VMCS_UNUSED( 0x0008, _S16, _TC, 4) +VMCS_UNUSED( 0x000A, _S16, _TC, 5) +VMCS_UNUSED( 0x000C, _S16, _TC, 6) +VMCS_UNUSED( 0x000E, _S16, _TC, 7) +VMCS_UNUSED( 0x0010, _S16, _TC, 8) +VMCS_UNUSED( 0x0012, _S16, _TC, 9) +VMCS_UNUSED( 0x0014, _S16, _TC, 10) +VMCS_UNUSED( 0x0016, _S16, _TC, 11) +VMCS_UNUSED( 0x0018, _S16, _TC, 12) +VMCS_UNUSED( 0x001A, _S16, _TC, 13) +VMCS_UNUSED( 0x001C, _S16, _TC, 14) +VMCS_UNUSED( 0x001E, _S16, _TC, 15) +VMCS_UNUSED( 0x0020, _S16, _TC, 16) +VMCS_UNUSED( 0x0022, _S16, _TC, 17) +VMCS_UNUSED( 0x0024, _S16, _TC, 18) +VMCS_UNUSED( 0x0026, _S16, _TC, 19) +VMCS_UNUSED( 0x0028, _S16, _TC, 20) +VMCS_UNUSED( 0x002A, _S16, _TC, 21) +VMCS_UNUSED( 0x002C, _S16, _TC, 22) +VMCS_UNUSED( 0x002E, _S16, _TC, 23) +VMCS_UNUSED( 0x0030, _S16, _TC, 24) +VMCS_UNUSED( 0x0032, _S16, _TC, 25) +VMCS_UNUSED( 0x0034, _S16, _TC, 26) +VMCS_UNUSED( 0x0036, _S16, _TC, 27) +VMCS_UNUSED( 0x0038, _S16, _TC, 28) +VMCS_UNUSED( 0x003A, _S16, _TC, 29) +VMCS_UNUSED( 0x003C, _S16, _TC, 30) +VMCS_UNUSED( 0x003E, _S16, _TC, 31) +VMCS_GROUP_END(16, CTL) + +/* 16-bit read-only data fields. */ +VMCS_GROUP_START(16, DATA) +VMCS_UNUSED( 0x0400, _S16, _TD, 0) +VMCS_UNUSED( 0x0402, _S16, _TD, 1) +VMCS_UNUSED( 0x0404, _S16, _TD, 2) +VMCS_UNUSED( 0x0406, _S16, _TD, 3) +VMCS_UNUSED( 0x0408, _S16, _TD, 4) +VMCS_UNUSED( 0x040A, _S16, _TD, 5) +VMCS_UNUSED( 0x040C, _S16, _TD, 6) +VMCS_UNUSED( 0x040E, _S16, _TD, 7) +VMCS_UNUSED( 0x0410, _S16, _TD, 8) +VMCS_UNUSED( 0x0412, _S16, _TD, 9) +VMCS_UNUSED( 0x0414, _S16, _TD, 10) +VMCS_UNUSED( 0x0416, _S16, _TD, 11) +VMCS_UNUSED( 0x0418, _S16, _TD, 12) +VMCS_UNUSED( 0x041A, _S16, _TD, 13) +VMCS_UNUSED( 0x041C, _S16, _TD, 14) +VMCS_UNUSED( 0x041E, _S16, _TD, 15) +VMCS_UNUSED( 0x0420, _S16, _TD, 16) +VMCS_UNUSED( 0x0422, _S16, _TD, 17) +VMCS_UNUSED( 0x0424, _S16, _TD, 18) +VMCS_UNUSED( 0x0426, _S16, _TD, 19) +VMCS_UNUSED( 0x0428, _S16, _TD, 20) +VMCS_UNUSED( 0x042A, _S16, _TD, 21) +VMCS_UNUSED( 0x042C, _S16, _TD, 22) +VMCS_UNUSED( 0x042E, _S16, _TD, 23) +VMCS_UNUSED( 0x0430, _S16, _TD, 24) +VMCS_UNUSED( 0x0432, _S16, _TD, 25) +VMCS_UNUSED( 0x0434, _S16, _TD, 26) +VMCS_UNUSED( 0x0436, _S16, _TD, 27) +VMCS_UNUSED( 0x0438, _S16, _TD, 28) +VMCS_UNUSED( 0x043A, _S16, _TD, 29) +VMCS_UNUSED( 0x043C, _S16, _TD, 30) +VMCS_UNUSED( 0x043E, _S16, _TD, 31) +VMCS_GROUP_END(16, DATA) /* 16-bit guest state. */ -VMCS_FIELD(ES, 0x0800, TRUE) -VMCS_FIELD(CS, 0x0802, TRUE) -VMCS_FIELD(SS, 0x0804, TRUE) -VMCS_FIELD(DS, 0x0806, TRUE) -VMCS_FIELD(FS, 0x0808, TRUE) -VMCS_FIELD(GS, 0x080A, TRUE) -VMCS_FIELD(LDTR, 0x080C, TRUE) -VMCS_FIELD(TR, 0x080E, TRUE) -VMCS_FIELD(INTR_STATUS, 0x0810, FALSE) +VMCS_GROUP_START(16, GUEST) +VMCS_FIELD(ES, 0x0800, _S16, _TG, 0, _NC, _V, _RW) +VMCS_FIELD(CS, 0x0802, _S16, _TG, 1, _NC, _V, _RW) +VMCS_FIELD(SS, 0x0804, _S16, _TG, 2, _NC, _V, _RW) +VMCS_FIELD(DS, 0x0806, _S16, _TG, 3, _NC, _V, _RW) +VMCS_FIELD(FS, 0x0808, _S16, _TG, 4, _NC, _V, _RW) +VMCS_FIELD(GS, 0x080A, _S16, _TG, 5, _NC, _V, _RW) +VMCS_FIELD(LDTR, 0x080C, _S16, _TG, 6, _NC, _V, _RW) +VMCS_FIELD(TR, 0x080E, _S16, _TG, 7, _NC, _V, _RW) +VMCS_FIELD(INTR_STATUS, 0x0810, _S16, _TG, 8, _NC, _NV, _NA) +VMCS_FIELD(PML_INDEX, 0x0812, _S16, _TG, 9, _NC, _NV, _NA) +VMCS_UNUSED( 0x0814, _S16, _TG, 10) +VMCS_UNUSED( 0x0816, _S16, _TG, 11) +VMCS_UNUSED( 0x0818, _S16, _TG, 12) +VMCS_UNUSED( 0x081A, _S16, _TG, 13) +VMCS_UNUSED( 0x081C, _S16, _TG, 14) +VMCS_UNUSED( 0x081E, _S16, _TG, 15) +VMCS_UNUSED( 0x0820, _S16, _TG, 16) +VMCS_UNUSED( 0x0822, _S16, _TG, 17) +VMCS_UNUSED( 0x0824, _S16, _TG, 18) +VMCS_UNUSED( 0x0826, _S16, _TG, 19) +VMCS_UNUSED( 0x0828, _S16, _TG, 20) +VMCS_UNUSED( 0x082A, _S16, _TG, 21) +VMCS_UNUSED( 0x082C, _S16, _TG, 22) +VMCS_UNUSED( 0x082E, _S16, _TG, 23) +VMCS_UNUSED( 0x0830, _S16, _TG, 24) +VMCS_UNUSED( 0x0832, _S16, _TG, 25) +VMCS_UNUSED( 0x0834, _S16, _TG, 26) +VMCS_UNUSED( 0x0836, _S16, _TG, 27) +VMCS_UNUSED( 0x0838, _S16, _TG, 28) +VMCS_UNUSED( 0x083A, _S16, _TG, 29) +VMCS_UNUSED( 0x083C, _S16, _TG, 30) +VMCS_UNUSED( 0x083E, _S16, _TG, 31) +VMCS_GROUP_END(16, GUEST) /* 16-bit host state. */ -VMCS_FIELD(HOST_ES, 0x0C00, TRUE) -VMCS_FIELD(HOST_CS, 0x0C02, TRUE) -VMCS_FIELD(HOST_SS, 0x0C04, TRUE) -VMCS_FIELD(HOST_DS, 0x0C06, TRUE) -VMCS_FIELD(HOST_FS, 0x0C08, TRUE) -VMCS_FIELD(HOST_GS, 0x0C0A, TRUE) -VMCS_FIELD(HOST_TR, 0x0C0C, TRUE) +VMCS_GROUP_START(16, HOST) +VMCS_FIELD(HOST_ES, 0x0C00, _S16, _TH, 0, _NC, _V, _NA) +VMCS_FIELD(HOST_CS, 0x0C02, _S16, _TH, 1, _NC, _V, _NA) +VMCS_FIELD(HOST_SS, 0x0C04, _S16, _TH, 2, _NC, _V, _NA) +VMCS_FIELD(HOST_DS, 0x0C06, _S16, _TH, 3, _NC, _V, _NA) +VMCS_FIELD(HOST_FS, 0x0C08, _S16, _TH, 4, _NC, _V, _NA) +VMCS_FIELD(HOST_GS, 0x0C0A, _S16, _TH, 5, _NC, _V, _NA) +VMCS_FIELD(HOST_TR, 0x0C0C, _S16, _TH, 6, _NC, _V, _NA) +VMCS_UNUSED( 0x0C0E, _S16, _TH, 7) +VMCS_UNUSED( 0x0C10, _S16, _TH, 8) +VMCS_UNUSED( 0x0C12, _S16, _TH, 9) +VMCS_UNUSED( 0x0C14, _S16, _TH, 10) +VMCS_UNUSED( 0x0C16, _S16, _TH, 11) +VMCS_UNUSED( 0x0C18, _S16, _TH, 12) +VMCS_UNUSED( 0x0C1A, _S16, _TH, 13) +VMCS_UNUSED( 0x0C1C, _S16, _TH, 14) +VMCS_UNUSED( 0x0C1E, _S16, _TH, 15) +VMCS_UNUSED( 0x0C20, _S16, _TH, 16) +VMCS_UNUSED( 0x0C22, _S16, _TH, 17) +VMCS_UNUSED( 0x0C24, _S16, _TH, 18) +VMCS_UNUSED( 0x0C26, _S16, _TH, 19) +VMCS_UNUSED( 0x0C28, _S16, _TH, 20) +VMCS_UNUSED( 0x0C2A, _S16, _TH, 21) +VMCS_UNUSED( 0x0C2C, _S16, _TH, 22) +VMCS_UNUSED( 0x0C2E, _S16, _TH, 23) +VMCS_UNUSED( 0x0C30, _S16, _TH, 24) +VMCS_UNUSED( 0x0C32, _S16, _TH, 25) +VMCS_UNUSED( 0x0C34, _S16, _TH, 26) +VMCS_UNUSED( 0x0C36, _S16, _TH, 27) +VMCS_UNUSED( 0x0C38, _S16, _TH, 28) +VMCS_UNUSED( 0x0C3A, _S16, _TH, 29) +VMCS_UNUSED( 0x0C3C, _S16, _TH, 30) +VMCS_UNUSED( 0x0C3E, _S16, _TH, 31) +VMCS_GROUP_END(16, HOST) +VMCS_SET_END(16) + +VMCS_SET_START(64) /* 64-bit control fields. */ -VMCS_FIELD(IOBITMAPA, 0x2000, TRUE) -VMCS_FIELD(IOBITMAPB, 0x2002, TRUE) -VMCS_FIELD(MSRBITMAP, 0x2004, TRUE) -VMCS_FIELD(VMEXIT_MSR_STORE_ADDR, 0x2006, TRUE) -VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR, 0x2008, TRUE) -VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR, 0x200A, TRUE) -VMCS_FIELD(EXECUTIVE_VMCS_PTR, 0x200C, TRUE) -VMCS_FIELD(TSC_OFF, 0x2010, TRUE) -VMCS_FIELD(VIRT_APIC_ADDR, 0x2012, TRUE) -VMCS_FIELD(APIC_ACCESS_ADDR, 0x2014, FALSE) -VMCS_FIELD(PI_DESC_ADDR, 0x2016, FALSE) -VMCS_FIELD(VMFUNC_CTLS, 0x2018, FALSE) -VMCS_FIELD(EPTP, 0x201A, TRUE) -VMCS_FIELD(EOI_EXIT0, 0x201C, FALSE) -VMCS_FIELD(EOI_EXIT1, 0x201E, FALSE) -VMCS_FIELD(EOI_EXIT2, 0x2020, FALSE) -VMCS_FIELD(EOI_EXIT3, 0x2022, FALSE) -VMCS_FIELD(EPTP_LIST_ADDR, 0x2024, FALSE) -VMCS_FIELD(VMREAD_BITMAP, 0x2026, FALSE) -VMCS_FIELD(VMWRITE_BITMAP, 0x2028, FALSE) -VMCS_FIELD(VE_INFO_ADDR, 0x202A, FALSE) -VMCS_FIELD(XSS_EXITING_BITMAP, 0x202C, FALSE) -VMCS_FIELD(ENCLS_EXITING_BITMAP, 0x202E, FALSE) +VMCS_GROUP_START(64, CTL) +VMCS_FIELD(IOBITMAPA, 0x2000, _S64, _TC, 0, _NC, _V, _NA) +VMCS_FIELD(IOBITMAPB, 0x2002, _S64, _TC, 1, _NC, _V, _NA) +VMCS_FIELD(MSRBITMAP, 0x2004, _S64, _TC, 2, _NC, _V, _NA) +VMCS_FIELD(VMEXIT_MSR_STORE_ADDR, 0x2006, _S64, _TC, 3, _NC, _V, _NA) +VMCS_FIELD(VMEXIT_MSR_LOAD_ADDR, 0x2008, _S64, _TC, 4, _NC, _V, _NA) +VMCS_FIELD(VMENTRY_MSR_LOAD_ADDR, 0x200A, _S64, _TC, 5, _NC, _V, _NA) +VMCS_FIELD(EXECUTIVE_VMCS_PTR, 0x200C, _S64, _TC, 6, _NC, _V, _NA) +VMCS_FIELD(PML_ADDR, 0x200E, _S64, _TC, 7, _NC, _NV, _NA) +VMCS_FIELD(TSC_OFF, 0x2010, _S64, _TC, 8, _C, _V, _NA) +VMCS_FIELD(VIRT_APIC_ADDR, 0x2012, _S64, _TC, 9, _NC, _V, _NA) +VMCS_FIELD(APIC_ACCESS_ADDR, 0x2014, _S64, _TC, 10, _NC, _V, _NA) +VMCS_FIELD(PI_DESC_ADDR, 0x2016, _S64, _TC, 11, _NC, _NV, _NA) +VMCS_FIELD(VMFUNC_CTLS, 0x2018, _S64, _TC, 12, _NC, _V, _NA) +VMCS_FIELD(EPTP, 0x201A, _S64, _TC, 13, _NC, _V, _NA) +VMCS_FIELD(EOI_EXIT0, 0x201C, _S64, _TC, 14, _NC, _NV, _NA) +VMCS_FIELD(EOI_EXIT1, 0x201E, _S64, _TC, 15, _NC, _NV, _NA) +VMCS_FIELD(EOI_EXIT2, 0x2020, _S64, _TC, 16, _NC, _NV, _NA) +VMCS_FIELD(EOI_EXIT3, 0x2022, _S64, _TC, 17, _NC, _NV, _NA) +VMCS_FIELD(EPTP_LIST_ADDR, 0x2024, _S64, _TC, 18, _NC, _V, _NA) +VMCS_FIELD(VMREAD_BITMAP, 0x2026, _S64, _TC, 19, _NC, _NV, _NA) +VMCS_FIELD(VMWRITE_BITMAP, 0x2028, _S64, _TC, 20, _NC, _NV, _NA) +VMCS_FIELD(VE_INFO_ADDR, 0x202A, _S64, _TC, 21, _NC, _V, _NA) +VMCS_FIELD(XSS_EXITING_BITMAP, 0x202C, _S64, _TC, 22, _NC, _V, _NA) +VMCS_FIELD(ENCLS_EXITING_BITMAP, 0x202E, _S64, _TC, 23, _NC, _NV, _NA) +VMCS_UNUSED( 0x2030, _S64, _TC, 24) +VMCS_FIELD(TSC_MULTIPLIER, 0x2032, _S64, _TC, 25, _NC, _NV, _NA) +VMCS_UNUSED( 0x2034, _S64, _TC, 26) +VMCS_UNUSED( 0x2036, _S64, _TC, 27) +VMCS_UNUSED( 0x2038, _S64, _TC, 28) +VMCS_UNUSED( 0x203A, _S64, _TC, 29) +VMCS_UNUSED( 0x203C, _S64, _TC, 30) +VMCS_UNUSED( 0x203E, _S64, _TC, 31) +VMCS_GROUP_END(64, CTL) /* 64-bit read-only data field. */ -VMCS_FIELD(PHYSADDR, 0x2400, TRUE) +VMCS_GROUP_START(64, DATA) +VMCS_FIELD(PHYSADDR, 0x2400, _S64, _TD, 0, _NC, _V, _RO) +VMCS_UNUSED( 0x2402, _S64, _TD, 1) +VMCS_UNUSED( 0x2404, _S64, _TD, 2) +VMCS_UNUSED( 0x2406, _S64, _TD, 3) +VMCS_UNUSED( 0x2408, _S64, _TD, 4) +VMCS_UNUSED( 0x240A, _S64, _TD, 5) +VMCS_UNUSED( 0x240C, _S64, _TD, 6) +VMCS_UNUSED( 0x240E, _S64, _TD, 7) +VMCS_UNUSED( 0x2410, _S64, _TD, 8) +VMCS_UNUSED( 0x2412, _S64, _TD, 9) +VMCS_UNUSED( 0x2414, _S64, _TD, 10) +VMCS_UNUSED( 0x2416, _S64, _TD, 11) +VMCS_UNUSED( 0x2418, _S64, _TD, 12) +VMCS_UNUSED( 0x241A, _S64, _TD, 13) +VMCS_UNUSED( 0x241C, _S64, _TD, 14) +VMCS_UNUSED( 0x241E, _S64, _TD, 15) +VMCS_UNUSED( 0x2420, _S64, _TD, 16) +VMCS_UNUSED( 0x2422, _S64, _TD, 17) +VMCS_UNUSED( 0x2424, _S64, _TD, 18) +VMCS_UNUSED( 0x2426, _S64, _TD, 19) +VMCS_UNUSED( 0x2428, _S64, _TD, 20) +VMCS_UNUSED( 0x242A, _S64, _TD, 21) +VMCS_UNUSED( 0x242C, _S64, _TD, 22) +VMCS_UNUSED( 0x242E, _S64, _TD, 23) +VMCS_UNUSED( 0x2430, _S64, _TD, 24) +VMCS_UNUSED( 0x2432, _S64, _TD, 25) +VMCS_UNUSED( 0x2434, _S64, _TD, 26) +VMCS_UNUSED( 0x2436, _S64, _TD, 27) +VMCS_UNUSED( 0x2438, _S64, _TD, 28) +VMCS_UNUSED( 0x243A, _S64, _TD, 29) +VMCS_UNUSED( 0x243C, _S64, _TD, 30) +VMCS_UNUSED( 0x243E, _S64, _TD, 31) +VMCS_GROUP_END(64, DATA) /* 64-bit guest state. */ -VMCS_FIELD(LINK_PTR, 0x2800, TRUE) -VMCS_FIELD(DEBUGCTL, 0x2802, TRUE) -VMCS_FIELD(PAT, 0x2804, FALSE) -VMCS_FIELD(EFER, 0x2806, TRUE) -VMCS_FIELD(PGC, 0x2808, TRUE) -VMCS_FIELD(PDPTE0, 0x280A, TRUE) -VMCS_FIELD(PDPTE1, 0x280C, TRUE) -VMCS_FIELD(PDPTE2, 0x280E, TRUE) -VMCS_FIELD(PDPTE3, 0x2810, TRUE) +VMCS_GROUP_START(64, GUEST) +VMCS_FIELD(LINK_PTR, 0x2800, _S64, _TG, 0, _NC, _V, _NA) +VMCS_FIELD(DEBUGCTL, 0x2802, _S64, _TG, 1, _NC, _V, _NA) +VMCS_FIELD(PAT, 0x2804, _S64, _TG, 2, _NC, _V, _NA) +VMCS_FIELD(EFER, 0x2806, _S64, _TG, 3, _NC, _V, _NA) +VMCS_FIELD(PGC, 0x2808, _S64, _TG, 4, _NC, _V, _NA) +VMCS_FIELD(PDPTE0, 0x280A, _S64, _TG, 5, _NC, _V, _RW) +VMCS_FIELD(PDPTE1, 0x280C, _S64, _TG, 6, _NC, _V, _RW) +VMCS_FIELD(PDPTE2, 0x280E, _S64, _TG, 7, _NC, _V, _RW) +VMCS_FIELD(PDPTE3, 0x2810, _S64, _TG, 8, _NC, _V, _RW) +VMCS_FIELD(BNDCFGS, 0x2812, _S64, _TG, 9, _NC, _V, _NA) +VMCS_UNUSED( 0x2814, _S64, _TG, 10) +VMCS_UNUSED( 0x2816, _S64, _TG, 11) +VMCS_UNUSED( 0x2818, _S64, _TG, 12) +VMCS_UNUSED( 0x281A, _S64, _TG, 13) +VMCS_UNUSED( 0x281C, _S64, _TG, 14) +VMCS_UNUSED( 0x281E, _S64, _TG, 15) +VMCS_UNUSED( 0x2820, _S64, _TG, 16) +VMCS_UNUSED( 0x2822, _S64, _TG, 17) +VMCS_UNUSED( 0x2824, _S64, _TG, 18) +VMCS_UNUSED( 0x2826, _S64, _TG, 19) +VMCS_UNUSED( 0x2828, _S64, _TG, 20) +VMCS_UNUSED( 0x282A, _S64, _TG, 21) +VMCS_UNUSED( 0x282C, _S64, _TG, 22) +VMCS_UNUSED( 0x282E, _S64, _TG, 23) +VMCS_UNUSED( 0x2830, _S64, _TG, 24) +VMCS_UNUSED( 0x2832, _S64, _TG, 25) +VMCS_UNUSED( 0x2834, _S64, _TG, 26) +VMCS_UNUSED( 0x2836, _S64, _TG, 27) +VMCS_UNUSED( 0x2838, _S64, _TG, 28) +VMCS_UNUSED( 0x283A, _S64, _TG, 29) +VMCS_UNUSED( 0x283C, _S64, _TG, 30) +VMCS_UNUSED( 0x283E, _S64, _TG, 31) +VMCS_GROUP_END(64, GUEST) /* 64-bit host state. */ -VMCS_FIELD(HOST_PAT, 0x2C00, FALSE) -VMCS_FIELD(HOST_EFER, 0x2C02, TRUE) -VMCS_FIELD(HOST_PGC, 0x2C04, TRUE) +VMCS_GROUP_START(64, HOST) +VMCS_FIELD(HOST_PAT, 0x2C00, _S64, _TH, 0, _NC, _V, _NA) +VMCS_FIELD(HOST_EFER, 0x2C02, _S64, _TH, 1, _NC, _V, _NA) +VMCS_FIELD(HOST_PGC, 0x2C04, _S64, _TH, 2, _NC, _V, _NA) +VMCS_UNUSED( 0x2C06, _S64, _TH, 3) +VMCS_UNUSED( 0x2C08, _S64, _TH, 4) +VMCS_UNUSED( 0x2C0A, _S64, _TH, 5) +VMCS_UNUSED( 0x2C0C, _S64, _TH, 6) +VMCS_UNUSED( 0x2C0E, _S64, _TH, 7) +VMCS_UNUSED( 0x2C10, _S64, _TH, 8) +VMCS_UNUSED( 0x2C12, _S64, _TH, 9) +VMCS_UNUSED( 0x2C14, _S64, _TH, 10) +VMCS_UNUSED( 0x2C16, _S64, _TH, 11) +VMCS_UNUSED( 0x2C18, _S64, _TH, 12) +VMCS_UNUSED( 0x2C1A, _S64, _TH, 13) +VMCS_UNUSED( 0x2C1C, _S64, _TH, 14) +VMCS_UNUSED( 0x2C1E, _S64, _TH, 15) +VMCS_UNUSED( 0x2C20, _S64, _TH, 16) +VMCS_UNUSED( 0x2C22, _S64, _TH, 17) +VMCS_UNUSED( 0x2C24, _S64, _TH, 18) +VMCS_UNUSED( 0x2C26, _S64, _TH, 19) +VMCS_UNUSED( 0x2C28, _S64, _TH, 20) +VMCS_UNUSED( 0x2C2A, _S64, _TH, 21) +VMCS_UNUSED( 0x2C2C, _S64, _TH, 22) +VMCS_UNUSED( 0x2C2E, _S64, _TH, 23) +VMCS_UNUSED( 0x2C30, _S64, _TH, 24) +VMCS_UNUSED( 0x2C32, _S64, _TH, 25) +VMCS_UNUSED( 0x2C34, _S64, _TH, 26) +VMCS_UNUSED( 0x2C36, _S64, _TH, 27) +VMCS_UNUSED( 0x2C38, _S64, _TH, 28) +VMCS_UNUSED( 0x2C3A, _S64, _TH, 29) +VMCS_UNUSED( 0x2C3C, _S64, _TH, 30) +VMCS_UNUSED( 0x2C3E, _S64, _TH, 31) +VMCS_GROUP_END(64, HOST) +VMCS_SET_END(64) + +VMCS_SET_START(32) /* 32-bit control fields. */ -VMCS_FIELD(PIN_VMEXEC_CTL, 0x4000, TRUE) -VMCS_FIELD(CPU_VMEXEC_CTL, 0x4002, TRUE) -VMCS_FIELD(XCP_BITMAP, 0x4004, TRUE) -VMCS_FIELD(PF_ERR_MASK, 0x4006, TRUE) -VMCS_FIELD(PF_ERR_MATCH, 0x4008, TRUE) -VMCS_FIELD(CR3_TARG_COUNT, 0x400A, TRUE) -VMCS_FIELD(VMEXIT_CTL, 0x400C, TRUE) -VMCS_FIELD(VMEXIT_MSR_STORE_COUNT, 0x400E, TRUE) -VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT, 0x4010, TRUE) -VMCS_FIELD(VMENTRY_CTL, 0x4012, TRUE) -VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT, 0x4014, TRUE) -VMCS_FIELD(VMENTRY_INTR_INFO, 0x4016, TRUE) -VMCS_FIELD(VMENTRY_XCP_ERR, 0x4018, TRUE) -VMCS_FIELD(VMENTRY_INSTR_LEN, 0x401A, TRUE) -VMCS_FIELD(TPR_THRESHOLD, 0x401C, TRUE) -VMCS_FIELD(2ND_VMEXEC_CTL, 0x401E, TRUE) -VMCS_FIELD(PAUSE_LOOP_GAP, 0x4020, FALSE) -VMCS_FIELD(PAUSE_LOOP_WINDOW, 0x4022, FALSE) +VMCS_GROUP_START(32, CTL) +VMCS_FIELD(PIN_VMEXEC_CTL, 0x4000, _S32, _TC, 0, _C, _V, _NA) +VMCS_FIELD(CPU_VMEXEC_CTL, 0x4002, _S32, _TC, 1, _C, _V, _NA) +VMCS_FIELD(XCP_BITMAP, 0x4004, _S32, _TC, 2, _NC, _V, _NA) +VMCS_FIELD(PF_ERR_MASK, 0x4006, _S32, _TC, 3, _NC, _V, _NA) +VMCS_FIELD(PF_ERR_MATCH, 0x4008, _S32, _TC, 4, _NC, _V, _NA) +VMCS_FIELD(CR3_TARG_COUNT, 0x400A, _S32, _TC, 5, _NC, _V, _NA) +VMCS_FIELD(VMEXIT_CTL, 0x400C, _S32, _TC, 6, _NC, _V, _NA) +VMCS_FIELD(VMEXIT_MSR_STORE_COUNT, 0x400E, _S32, _TC, 7, _NC, _V, _NA) +VMCS_FIELD(VMEXIT_MSR_LOAD_COUNT, 0x4010, _S32, _TC, 8, _NC, _V, _NA) +VMCS_FIELD(VMENTRY_CTL, 0x4012, _S32, _TC, 9, _NC, _V, _NA) +VMCS_FIELD(VMENTRY_MSR_LOAD_COUNT, 0x4014, _S32, _TC, 10, _NC, _V, _NA) +VMCS_FIELD(VMENTRY_INTR_INFO, 0x4016, _S32, _TC, 11, _NC, _V, _NA) +VMCS_FIELD(VMENTRY_XCP_ERR, 0x4018, _S32, _TC, 12, _NC, _V, _NA) +VMCS_FIELD(VMENTRY_INSTR_LEN, 0x401A, _S32, _TC, 13, _NC, _V, _NA) +VMCS_FIELD(TPR_THRESHOLD, 0x401C, _S32, _TC, 14, _C, _V, _NA) +VMCS_FIELD(2ND_VMEXEC_CTL, 0x401E, _S32, _TC, 15, _C, _V, _NA) +VMCS_FIELD(PAUSE_LOOP_GAP, 0x4020, _S32, _TC, 16, _NC, _V, _NA) +VMCS_FIELD(PAUSE_LOOP_WINDOW, 0x4022, _S32, _TC, 17, _NC, _V, _NA) +VMCS_UNUSED( 0x4024, _S32, _TC, 18) +VMCS_UNUSED( 0x4026, _S32, _TC, 19) +VMCS_UNUSED( 0x4028, _S32, _TC, 20) +VMCS_UNUSED( 0x402A, _S32, _TC, 21) +VMCS_UNUSED( 0x402C, _S32, _TC, 22) +VMCS_UNUSED( 0x402E, _S32, _TC, 23) +VMCS_UNUSED( 0x4030, _S32, _TC, 24) +VMCS_UNUSED( 0x4032, _S32, _TC, 25) +VMCS_UNUSED( 0x4034, _S32, _TC, 26) +VMCS_UNUSED( 0x4036, _S32, _TC, 27) +VMCS_UNUSED( 0x4038, _S32, _TC, 28) +VMCS_UNUSED( 0x403A, _S32, _TC, 29) +VMCS_UNUSED( 0x403C, _S32, _TC, 30) +VMCS_UNUSED( 0x403E, _S32, _TC, 31) +VMCS_GROUP_END(32, CTL) /* 32-bit read-only data fields. */ -VMCS_FIELD(VMINSTR_ERR, 0x4400, TRUE) -VMCS_FIELD(EXIT_REASON, 0x4402, TRUE) -VMCS_FIELD(EXIT_INTR_INFO, 0x4404, TRUE) -VMCS_FIELD(EXIT_INTR_ERR, 0x4406, TRUE) -VMCS_FIELD(IDTVEC_INFO, 0x4408, TRUE) -VMCS_FIELD(IDTVEC_ERR, 0x440A, TRUE) -VMCS_FIELD(INSTRLEN, 0x440C, TRUE) -VMCS_FIELD(INSTR_INFO, 0x440E, TRUE) +VMCS_GROUP_START(32, DATA) +VMCS_FIELD(VMINSTR_ERR, 0x4400, _S32, _TD, 0, _NC, _V, _NA) +VMCS_FIELD(EXIT_REASON, 0x4402, _S32, _TD, 1, _NC, _V, _RO) +VMCS_FIELD(EXIT_INTR_INFO, 0x4404, _S32, _TD, 2, _C, _V, _RO) +VMCS_FIELD(EXIT_INTR_ERR, 0x4406, _S32, _TD, 3, _NC, _V, _RO) +VMCS_FIELD(IDTVEC_INFO, 0x4408, _S32, _TD, 4, _NC, _V, _RO) +VMCS_FIELD(IDTVEC_ERR, 0x440A, _S32, _TD, 5, _NC, _V, _RO) +VMCS_FIELD(INSTRLEN, 0x440C, _S32, _TD, 6, _NC, _V, _RO) +VMCS_FIELD(INSTR_INFO, 0x440E, _S32, _TD, 7, _NC, _V, _RO) +VMCS_UNUSED( 0x4410, _S32, _TD, 8) +VMCS_UNUSED( 0x4412, _S32, _TD, 9) +VMCS_UNUSED( 0x4414, _S32, _TD, 10) +VMCS_UNUSED( 0x4416, _S32, _TD, 11) +VMCS_UNUSED( 0x4418, _S32, _TD, 12) +VMCS_UNUSED( 0x441A, _S32, _TD, 13) +VMCS_UNUSED( 0x441C, _S32, _TD, 14) +VMCS_UNUSED( 0x441E, _S32, _TD, 15) +VMCS_UNUSED( 0x4420, _S32, _TD, 16) +VMCS_UNUSED( 0x4422, _S32, _TD, 17) +VMCS_UNUSED( 0x4424, _S32, _TD, 18) +VMCS_UNUSED( 0x4426, _S32, _TD, 19) +VMCS_UNUSED( 0x4428, _S32, _TD, 20) +VMCS_UNUSED( 0x442A, _S32, _TD, 21) +VMCS_UNUSED( 0x442C, _S32, _TD, 22) +VMCS_UNUSED( 0x442E, _S32, _TD, 23) +VMCS_UNUSED( 0x4430, _S32, _TD, 24) +VMCS_UNUSED( 0x4432, _S32, _TD, 25) +VMCS_UNUSED( 0x4434, _S32, _TD, 26) +VMCS_UNUSED( 0x4436, _S32, _TD, 27) +VMCS_UNUSED( 0x4438, _S32, _TD, 28) +VMCS_UNUSED( 0x443A, _S32, _TD, 29) +VMCS_UNUSED( 0x443C, _S32, _TD, 30) +VMCS_UNUSED( 0x443E, _S32, _TD, 31) +VMCS_GROUP_END(32, DATA) /* 32-bit guest state. */ -VMCS_FIELD(ES_LIMIT, 0x4800, TRUE) -VMCS_FIELD(CS_LIMIT, 0x4802, TRUE) -VMCS_FIELD(SS_LIMIT, 0x4804, TRUE) -VMCS_FIELD(DS_LIMIT, 0x4806, TRUE) -VMCS_FIELD(FS_LIMIT, 0x4808, TRUE) -VMCS_FIELD(GS_LIMIT, 0x480A, TRUE) -VMCS_FIELD(LDTR_LIMIT, 0x480C, TRUE) -VMCS_FIELD(TR_LIMIT, 0x480E, TRUE) -VMCS_FIELD(GDTR_LIMIT, 0x4810, TRUE) -VMCS_FIELD(IDTR_LIMIT, 0x4812, TRUE) -VMCS_FIELD(ES_AR, 0x4814, TRUE) -VMCS_FIELD(CS_AR, 0x4816, TRUE) -VMCS_FIELD(SS_AR, 0x4818, TRUE) -VMCS_FIELD(DS_AR, 0x481A, TRUE) -VMCS_FIELD(FS_AR, 0x481C, TRUE) -VMCS_FIELD(GS_AR, 0x481E, TRUE) -VMCS_FIELD(LDTR_AR, 0x4820, TRUE) -VMCS_FIELD(TR_AR, 0x4822, TRUE) -VMCS_FIELD(HOLDOFF, 0x4824, TRUE) -VMCS_FIELD(ACTSTATE, 0x4826, TRUE) -VMCS_FIELD(SMBASE, 0x4828, TRUE) -VMCS_FIELD(SYSENTER_CS, 0x482A, TRUE) -VMCS_FIELD(TIMER, 0x482E, FALSE) +VMCS_GROUP_START(32, GUEST) +VMCS_FIELD(ES_LIMIT, 0x4800, _S32, _TG, 0, _NC, _V, _RW) +VMCS_FIELD(CS_LIMIT, 0x4802, _S32, _TG, 1, _NC, _V, _RW) +VMCS_FIELD(SS_LIMIT, 0x4804, _S32, _TG, 2, _NC, _V, _RW) +VMCS_FIELD(DS_LIMIT, 0x4806, _S32, _TG, 3, _NC, _V, _RW) +VMCS_FIELD(FS_LIMIT, 0x4808, _S32, _TG, 4, _NC, _V, _RW) +VMCS_FIELD(GS_LIMIT, 0x480A, _S32, _TG, 5, _NC, _V, _RW) +VMCS_FIELD(LDTR_LIMIT, 0x480C, _S32, _TG, 6, _NC, _V, _RW) +VMCS_FIELD(TR_LIMIT, 0x480E, _S32, _TG, 7, _NC, _V, _RW) +VMCS_FIELD(GDTR_LIMIT, 0x4810, _S32, _TG, 8, _NC, _V, _RW) +VMCS_FIELD(IDTR_LIMIT, 0x4812, _S32, _TG, 9, _NC, _V, _RW) +VMCS_FIELD(ES_AR, 0x4814, _S32, _TG, 10, _NC, _V, _RW) +VMCS_FIELD(CS_AR, 0x4816, _S32, _TG, 11, _NC, _V, _RW) +VMCS_FIELD(SS_AR, 0x4818, _S32, _TG, 12, _NC, _V, _RW) +VMCS_FIELD(DS_AR, 0x481A, _S32, _TG, 13, _NC, _V, _RW) +VMCS_FIELD(FS_AR, 0x481C, _S32, _TG, 14, _NC, _V, _RW) +VMCS_FIELD(GS_AR, 0x481E, _S32, _TG, 15, _NC, _V, _RW) +VMCS_FIELD(LDTR_AR, 0x4820, _S32, _TG, 16, _NC, _V, _RW) +VMCS_FIELD(TR_AR, 0x4822, _S32, _TG, 17, _NC, _V, _RW) +VMCS_FIELD(HOLDOFF, 0x4824, _S32, _TG, 18, _C, _V, _RW) +VMCS_FIELD(ACTSTATE, 0x4826, _S32, _TG, 19, _NC, _V, _NA) +VMCS_FIELD(SMBASE, 0x4828, _S32, _TG, 20, _NC, _V, _NA) +VMCS_FIELD(SYSENTER_CS, 0x482A, _S32, _TG, 21, _NC, _V, _RW) +VMCS_UNUSED( 0x482C, _S32, _TG, 22) +VMCS_FIELD(TIMER, 0x482E, _S32, _TG, 23, _NC, _NV, _NA) +VMCS_UNUSED( 0x4830, _S32, _TG, 24) +VMCS_UNUSED( 0x4832, _S32, _TG, 25) +VMCS_UNUSED( 0x4834, _S32, _TG, 26) +VMCS_UNUSED( 0x4836, _S32, _TG, 27) +VMCS_UNUSED( 0x4838, _S32, _TG, 28) +VMCS_UNUSED( 0x483A, _S32, _TG, 29) +VMCS_UNUSED( 0x483C, _S32, _TG, 30) +VMCS_UNUSED( 0x483E, _S32, _TG, 31) + +VMCS_GROUP_END(32, GUEST) /* 32-bit host state. */ -VMCS_FIELD(HOST_SYSENTER_CS, 0x4C00, TRUE) +VMCS_GROUP_START(32, HOST) +VMCS_FIELD(HOST_SYSENTER_CS, 0x4C00, _S32, _TH, 0, _NC, _V, _NA) +VMCS_UNUSED( 0x4C02, _S32, _TH, 1) +VMCS_UNUSED( 0x4C04, _S32, _TH, 2) +VMCS_UNUSED( 0x4C06, _S32, _TH, 3) +VMCS_UNUSED( 0x4C08, _S32, _TH, 4) +VMCS_UNUSED( 0x4C0A, _S32, _TH, 5) +VMCS_UNUSED( 0x4C0C, _S32, _TH, 6) +VMCS_UNUSED( 0x4C0E, _S32, _TH, 7) +VMCS_UNUSED( 0x4C10, _S32, _TH, 8) +VMCS_UNUSED( 0x4C12, _S32, _TH, 9) +VMCS_UNUSED( 0x4C14, _S32, _TH, 10) +VMCS_UNUSED( 0x4C16, _S32, _TH, 11) +VMCS_UNUSED( 0x4C18, _S32, _TH, 12) +VMCS_UNUSED( 0x4C1A, _S32, _TH, 13) +VMCS_UNUSED( 0x4C1C, _S32, _TH, 14) +VMCS_UNUSED( 0x4C1E, _S32, _TH, 15) +VMCS_UNUSED( 0x4C20, _S32, _TH, 16) +VMCS_UNUSED( 0x4C22, _S32, _TH, 17) +VMCS_UNUSED( 0x4C24, _S32, _TH, 18) +VMCS_UNUSED( 0x4C26, _S32, _TH, 19) +VMCS_UNUSED( 0x4C28, _S32, _TH, 20) +VMCS_UNUSED( 0x4C2A, _S32, _TH, 21) +VMCS_UNUSED( 0x4C2C, _S32, _TH, 22) +VMCS_UNUSED( 0x4C2E, _S32, _TH, 23) +VMCS_UNUSED( 0x4C30, _S32, _TH, 24) +VMCS_UNUSED( 0x4C32, _S32, _TH, 25) +VMCS_UNUSED( 0x4C34, _S32, _TH, 26) +VMCS_UNUSED( 0x4C36, _S32, _TH, 27) +VMCS_UNUSED( 0x4C38, _S32, _TH, 28) +VMCS_UNUSED( 0x4C3A, _S32, _TH, 29) +VMCS_UNUSED( 0x4C3C, _S32, _TH, 30) +VMCS_UNUSED( 0x4C3E, _S32, _TH, 31) +VMCS_GROUP_END(32, HOST) +VMCS_SET_END(32) + +VMCS_SET_START(NAT) /* natural-width control fields. */ -VMCS_FIELD(CR0_GHMASK, 0x6000, TRUE) -VMCS_FIELD(CR4_GHMASK, 0x6002, TRUE) -VMCS_FIELD(CR0_SHADOW, 0x6004, TRUE) -VMCS_FIELD(CR4_SHADOW, 0x6006, TRUE) -VMCS_FIELD(CR3_TARGVAL0, 0x6008, TRUE) -VMCS_FIELD(CR3_TARGVAL1, 0x600A, TRUE) -VMCS_FIELD(CR3_TARGVAL2, 0x600C, TRUE) -VMCS_FIELD(CR3_TARGVAL3, 0x600E, TRUE) +VMCS_GROUP_START(NAT, CTL) +VMCS_FIELD(CR0_GHMASK, 0x6000, _SN, _TC, 0, _C, _V, _NA) +VMCS_FIELD(CR4_GHMASK, 0x6002, _SN, _TC, 1, _C, _V, _NA) +VMCS_FIELD(CR0_SHADOW, 0x6004, _SN, _TC, 2, _NC, _V, _NA) +VMCS_FIELD(CR4_SHADOW, 0x6006, _SN, _TC, 3, _NC, _V, _NA) +VMCS_FIELD(CR3_TARGVAL0, 0x6008, _SN, _TC, 4, _NC, _V, _NA) +VMCS_FIELD(CR3_TARGVAL1, 0x600A, _SN, _TC, 5, _NC, _V, _NA) +VMCS_FIELD(CR3_TARGVAL2, 0x600C, _SN, _TC, 6, _NC, _V, _NA) +VMCS_FIELD(CR3_TARGVAL3, 0x600E, _SN, _TC, 7, _NC, _V, _NA) +VMCS_UNUSED( 0x6010, _SN, _TC, 8) +VMCS_UNUSED( 0x6012, _SN, _TC, 9) +VMCS_UNUSED( 0x6014, _SN, _TC, 10) +VMCS_UNUSED( 0x6016, _SN, _TC, 11) +VMCS_UNUSED( 0x6018, _SN, _TC, 12) +VMCS_UNUSED( 0x601A, _SN, _TC, 13) +VMCS_UNUSED( 0x601C, _SN, _TC, 14) +VMCS_UNUSED( 0x601E, _SN, _TC, 15) +VMCS_UNUSED( 0x6020, _SN, _TC, 16) +VMCS_UNUSED( 0x6022, _SN, _TC, 17) +VMCS_UNUSED( 0x6024, _SN, _TC, 18) +VMCS_UNUSED( 0x6026, _SN, _TC, 19) +VMCS_UNUSED( 0x6028, _SN, _TC, 20) +VMCS_UNUSED( 0x602A, _SN, _TC, 21) +VMCS_UNUSED( 0x602C, _SN, _TC, 22) +VMCS_UNUSED( 0x602E, _SN, _TC, 23) +VMCS_UNUSED( 0x6030, _SN, _TC, 24) +VMCS_UNUSED( 0x6032, _SN, _TC, 25) +VMCS_UNUSED( 0x6034, _SN, _TC, 26) +VMCS_UNUSED( 0x6036, _SN, _TC, 27) +VMCS_UNUSED( 0x6038, _SN, _TC, 28) +VMCS_UNUSED( 0x603A, _SN, _TC, 29) +VMCS_UNUSED( 0x603C, _SN, _TC, 30) +VMCS_UNUSED( 0x603E, _SN, _TC, 31) +VMCS_GROUP_END(NAT, CTL) -/* natural-width read-only data fields. */ -VMCS_FIELD(EXIT_QUAL, 0x6400, TRUE) -VMCS_FIELD(IO_ECX, 0x6402, TRUE) -VMCS_FIELD(IO_ESI, 0x6404, TRUE) -VMCS_FIELD(IO_EDI, 0x6406, TRUE) -VMCS_FIELD(IO_EIP, 0x6408, TRUE) -VMCS_FIELD(LINEAR_ADDR, 0x640A, TRUE) +VMCS_GROUP_START(NAT, DATA) +VMCS_FIELD(EXIT_QUAL, 0x6400, _SN, _TD, 0, _NC, _V, _RO) +VMCS_FIELD(IO_RCX, 0x6402, _SN, _TD, 1, _NC, _V, _RO) +VMCS_FIELD(IO_RSI, 0x6404, _SN, _TD, 2, _NC, _V, _RO) +VMCS_FIELD(IO_RDI, 0x6406, _SN, _TD, 3, _NC, _V, _RO) +VMCS_FIELD(IO_RIP, 0x6408, _SN, _TD, 4, _NC, _V, _RO) +VMCS_FIELD(LINEAR_ADDR, 0x640A, _SN, _TD, 5, _NC, _V, _RO) +VMCS_UNUSED( 0x640C, _SN, _TD, 6) +VMCS_UNUSED( 0x640E, _SN, _TD, 7) +VMCS_UNUSED( 0x6410, _SN, _TD, 8) +VMCS_UNUSED( 0x6412, _SN, _TD, 9) +VMCS_UNUSED( 0x6414, _SN, _TD, 10) +VMCS_UNUSED( 0x6416, _SN, _TD, 11) +VMCS_UNUSED( 0x6418, _SN, _TD, 12) +VMCS_UNUSED( 0x641A, _SN, _TD, 13) +VMCS_UNUSED( 0x641C, _SN, _TD, 14) +VMCS_UNUSED( 0x641E, _SN, _TD, 15) +VMCS_UNUSED( 0x6420, _SN, _TD, 16) +VMCS_UNUSED( 0x6422, _SN, _TD, 17) +VMCS_UNUSED( 0x6424, _SN, _TD, 18) +VMCS_UNUSED( 0x6426, _SN, _TD, 19) +VMCS_UNUSED( 0x6428, _SN, _TD, 20) +VMCS_UNUSED( 0x642A, _SN, _TD, 21) +VMCS_UNUSED( 0x642C, _SN, _TD, 22) +VMCS_UNUSED( 0x642E, _SN, _TD, 23) +VMCS_UNUSED( 0x6430, _SN, _TD, 24) +VMCS_UNUSED( 0x6432, _SN, _TD, 25) +VMCS_UNUSED( 0x6434, _SN, _TD, 26) +VMCS_UNUSED( 0x6436, _SN, _TD, 27) +VMCS_UNUSED( 0x6438, _SN, _TD, 28) +VMCS_UNUSED( 0x643A, _SN, _TD, 29) +VMCS_UNUSED( 0x643C, _SN, _TD, 30) +VMCS_UNUSED( 0x643E, _SN, _TD, 31) +VMCS_GROUP_END(NAT, DATA) /* natural-width guest state. */ -VMCS_FIELD(CR0, 0x6800, TRUE) -VMCS_FIELD(CR3, 0x6802, TRUE) -VMCS_FIELD(CR4, 0x6804, TRUE) -VMCS_FIELD(ES_BASE, 0x6806, TRUE) -VMCS_FIELD(CS_BASE, 0x6808, TRUE) -VMCS_FIELD(SS_BASE, 0x680A, TRUE) -VMCS_FIELD(DS_BASE, 0x680C, TRUE) -VMCS_FIELD(FS_BASE, 0x680E, TRUE) -VMCS_FIELD(GS_BASE, 0x6810, TRUE) -VMCS_FIELD(LDTR_BASE, 0x6812, TRUE) -VMCS_FIELD(TR_BASE, 0x6814, TRUE) -VMCS_FIELD(GDTR_BASE, 0x6816, TRUE) -VMCS_FIELD(IDTR_BASE, 0x6818, TRUE) -VMCS_FIELD(DR7, 0x681A, TRUE) -VMCS_FIELD(ESP, 0x681C, TRUE) -VMCS_FIELD(EIP, 0x681E, TRUE) -VMCS_FIELD(EFLAGS, 0x6820, TRUE) -VMCS_FIELD(PENDDBG, 0x6822, TRUE) -VMCS_FIELD(SYSENTER_ESP, 0x6824, TRUE) -VMCS_FIELD(SYSENTER_EIP, 0x6826, TRUE) +VMCS_GROUP_START(NAT, GUEST) +VMCS_FIELD(CR0, 0x6800, _SN, _TG, 0, _NC, _V, _NA) +VMCS_FIELD(CR3, 0x6802, _SN, _TG, 1, _NC, _V, _RW) +VMCS_FIELD(CR4, 0x6804, _SN, _TG, 2, _NC, _V, _RW) +VMCS_FIELD(ES_BASE, 0x6806, _SN, _TG, 3, _NC, _V, _RW) +VMCS_FIELD(CS_BASE, 0x6808, _SN, _TG, 4, _NC, _V, _RW) +VMCS_FIELD(SS_BASE, 0x680A, _SN, _TG, 5, _NC, _V, _RW) +VMCS_FIELD(DS_BASE, 0x680C, _SN, _TG, 6, _NC, _V, _RW) +VMCS_FIELD(FS_BASE, 0x680E, _SN, _TG, 7, _NC, _V, _RW) +VMCS_FIELD(GS_BASE, 0x6810, _SN, _TG, 8, _NC, _V, _RW) +VMCS_FIELD(LDTR_BASE, 0x6812, _SN, _TG, 9, _NC, _V, _RW) +VMCS_FIELD(TR_BASE, 0x6814, _SN, _TG, 10, _NC, _V, _RW) +VMCS_FIELD(GDTR_BASE, 0x6816, _SN, _TG, 11, _NC, _V, _RW) +VMCS_FIELD(IDTR_BASE, 0x6818, _SN, _TG, 12, _NC, _V, _RW) +VMCS_FIELD(DR7, 0x681A, _SN, _TG, 13, _NC, _V, _NA) +VMCS_FIELD(RSP, 0x681C, _SN, _TG, 14, _NC, _V, _RW) +VMCS_FIELD(RIP, 0x681E, _SN, _TG, 15, _NC, _V, _RW) +VMCS_FIELD(RFLAGS, 0x6820, _SN, _TG, 16, _NC, _V, _RW) +VMCS_FIELD(PENDDBG, 0x6822, _SN, _TG, 17, _NC, _V, _RW) +VMCS_FIELD(SYSENTER_ESP, 0x6824, _SN, _TG, 18, _NC, _V, _RW) +VMCS_FIELD(SYSENTER_EIP, 0x6826, _SN, _TG, 19, _NC, _V, _RW) +VMCS_UNUSED( 0x6828, _SN, _TG, 20) +VMCS_UNUSED( 0x682A, _SN, _TG, 21) +VMCS_UNUSED( 0x682C, _SN, _TG, 22) +VMCS_UNUSED( 0x682E, _SN, _TG, 23) +VMCS_UNUSED( 0x6830, _SN, _TG, 24) +VMCS_UNUSED( 0x6832, _SN, _TG, 25) +VMCS_UNUSED( 0x6834, _SN, _TG, 26) +VMCS_UNUSED( 0x6836, _SN, _TG, 27) +VMCS_UNUSED( 0x6838, _SN, _TG, 28) +VMCS_UNUSED( 0x683A, _SN, _TG, 29) +VMCS_UNUSED( 0x683C, _SN, _TG, 30) +VMCS_UNUSED( 0x683E, _SN, _TG, 31) +VMCS_GROUP_END(NAT, GUEST) /* natural-width host state. */ -VMCS_FIELD(HOST_CR0, 0x6C00, TRUE) -VMCS_FIELD(HOST_CR3, 0x6C02, TRUE) -VMCS_FIELD(HOST_CR4, 0x6C04, TRUE) -VMCS_FIELD(HOST_FSBASE, 0x6C06, TRUE) -VMCS_FIELD(HOST_GSBASE, 0x6C08, TRUE) -VMCS_FIELD(HOST_TRBASE, 0x6C0A, TRUE) -VMCS_FIELD(HOST_GDTRBASE, 0x6C0C, TRUE) -VMCS_FIELD(HOST_IDTRBASE, 0x6C0E, TRUE) -VMCS_FIELD(HOST_SYSENTER_ESP, 0x6C10, TRUE) -VMCS_FIELD(HOST_SYSENTER_EIP, 0x6C12, TRUE) -VMCS_FIELD(HOST_ESP, 0x6C14, TRUE) -VMCS_FIELD(HOST_EIP, 0x6C16, TRUE) +VMCS_GROUP_START(NAT, HOST) +VMCS_FIELD(HOST_CR0, 0x6C00, _SN, _TH, 0, _NC, _V, _NA) +VMCS_FIELD(HOST_CR3, 0x6C02, _SN, _TH, 1, _NC, _V, _NA) +VMCS_FIELD(HOST_CR4, 0x6C04, _SN, _TH, 2, _NC, _V, _NA) +VMCS_FIELD(HOST_FSBASE, 0x6C06, _SN, _TH, 3, _NC, _V, _NA) +VMCS_FIELD(HOST_GSBASE, 0x6C08, _SN, _TH, 4, _NC, _V, _NA) +VMCS_FIELD(HOST_TRBASE, 0x6C0A, _SN, _TH, 5, _NC, _V, _NA) +VMCS_FIELD(HOST_GDTRBASE, 0x6C0C, _SN, _TH, 6, _NC, _V, _NA) +VMCS_FIELD(HOST_IDTRBASE, 0x6C0E, _SN, _TH, 7, _NC, _V, _NA) +VMCS_FIELD(HOST_SYSENTER_ESP, 0x6C10, _SN, _TH, 8, _NC, _V, _NA) +VMCS_FIELD(HOST_SYSENTER_EIP, 0x6C12, _SN, _TH, 9, _NC, _V, _NA) +VMCS_FIELD(HOST_RSP, 0x6C14, _SN, _TH, 10, _NC, _V, _NA) +VMCS_FIELD(HOST_RIP, 0x6C16, _SN, _TH, 11, _NC, _V, _NA) +VMCS_UNUSED( 0x6C18, _SN, _TH, 12) +VMCS_UNUSED( 0x6C1A, _SN, _TH, 13) +VMCS_UNUSED( 0x6C1C, _SN, _TH, 14) +VMCS_UNUSED( 0x6C1E, _SN, _TH, 15) +VMCS_UNUSED( 0x6C20, _SN, _TH, 16) +VMCS_UNUSED( 0x6C22, _SN, _TH, 17) +VMCS_UNUSED( 0x6C24, _SN, _TH, 18) +VMCS_UNUSED( 0x6C26, _SN, _TH, 19) +VMCS_UNUSED( 0x6C28, _SN, _TH, 20) +VMCS_UNUSED( 0x6C2A, _SN, _TH, 21) +VMCS_UNUSED( 0x6C2C, _SN, _TH, 22) +VMCS_UNUSED( 0x6C2E, _SN, _TH, 23) +VMCS_UNUSED( 0x6C30, _SN, _TH, 24) +VMCS_UNUSED( 0x6C32, _SN, _TH, 25) +VMCS_UNUSED( 0x6C34, _SN, _TH, 26) +VMCS_UNUSED( 0x6C36, _SN, _TH, 27) +VMCS_UNUSED( 0x6C38, _SN, _TH, 28) +VMCS_UNUSED( 0x6C3A, _SN, _TH, 29) +VMCS_UNUSED( 0x6C3C, _SN, _TH, 30) +VMCS_UNUSED( 0x6C3E, _SN, _TH, 31) +VMCS_GROUP_END(NAT, HOST) +VMCS_SET_END(NAT) + + +#ifdef LOCAL_GROUP_START +#undef VMCS_GROUP_START +#undef LOCAL_GROUP_START +#endif + +#ifdef LOCAL_GROUP_END +#undef VMCS_GROUP_END +#undef LOCAL_GROUP_END +#endif + +#ifdef LOCAL_SET_START +#undef VMCS_SET_START +#undef LOCAL_SET_START +#endif + +#ifdef LOCAL_SET_END +#undef VMCS_SET_END +#undef LOCAL_SET_END +#endif + +#ifdef LOCAL_UNUSED +#undef VMCS_UNUSED +#undef LOCAL_UNUSED +#endif diff --git a/vmmon-only/include/x86vt.h b/vmmon-only/include/x86vt.h index 993a8a9..2ca47e0 100644 --- a/vmmon-only/include/x86vt.h +++ b/vmmon-only/include/x86vt.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2004-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2004-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -90,6 +90,8 @@ (CONST64U(1) << MSR_VMX_MISC_ACTSTATE_SHUTDOWN_SHIFT) #define MSR_VMX_MISC_ACTSTATE_SIPI \ (CONST64U(1) << MSR_VMX_MISC_ACTSTATE_SIPI_SHIFT) +#define MSR_VMX_MISC_PROCESSOR_TRACE_IN_VMX \ + (CONST64U(1) << MSR_VMX_MISC_PROCESSOR_TRACE_IN_VMX_SHIFT) #define MSR_VMX_MISC_RDMSR_SMBASE_IN_SMM \ (CONST64U(1) << MSR_VMX_MISC_RDMSR_SMBASE_IN_SMM_SHIFT) #define MSR_VMX_MISC_ALLOW_ALL_VMWRITES \ @@ -128,6 +130,8 @@ (CONST64U(1) << MSR_VMX_EPT_VPID_INVVPID_ALL_CTX_SHIFT) #define MSR_VMX_EPT_VPID_INVVPID_VPID_CTX_LOCAL \ (CONST64U(1) << MSR_VMX_EPT_VPID_INVVPID_VPID_CTX_LOCAL_SHIFT) +#define MSR_VMX_EPT_VPID_ADV_EXIT_INFO \ + (CONST64U(1) << MSR_VMX_EPT_VPID_ADV_EXIT_INFO_SHIFT) #define VT_VMCS_STANDARD_TAG 0x00000000 #define VT_VMCS_SHADOW_TAG 0x80000000 @@ -154,8 +158,14 @@ #define VT_ENCODING_NUM_SIZES 4 #define VT_ENCODING_RSVD 0xffff9000 +/* + * The highest index of any currently defined field is 25, for + * TSC_MULTIPLIER. + */ +#define VT_ENCODING_MAX_INDEX 25 + enum { -#define VMCS_FIELD(name, encoding, vvt) VT_VMCS_##name = encoding, +#define VMCS_FIELD(_name, _val, ...) VT_VMCS_##_name = _val, #include "x86vt-vmcs-fields.h" #undef VMCS_FIELD }; @@ -278,8 +288,12 @@ enum { VMX_CPU2(VMCS_SHADOW, 14) \ VMX_CPU2(ENCL, 15) \ VMX_CPU2(RDSEED, 16) \ + VMX_CPU2(PML, 17) \ VMX_CPU2(EPT_VIOL_VE, 18) \ - VMX_CPU2(XSAVES, 20) + VMX_CPU2(PT_SUPPRESS_NR_BIT, 19) \ + VMX_CPU2(XSAVES, 20) \ + VMX_CPU2(EPT_MBX, 22) \ + VMX_CPU2(TSC_SCALING, 25) #define VMX_PROCBASED_CTLS2_CAP \ VMX_PROCBASED_CTLS2_CAP_NDA \ @@ -292,15 +306,17 @@ enum { VMXCTL(_EXIT_CTLS, _field, _pos) #define VMX_EXIT_CTLS_CAP_NDA #define VMX_EXIT_CTLS_CAP_PUB \ - VMX_EXIT(SAVE_DEBUGCTL, 2) \ - VMX_EXIT(LONGMODE, 9) \ - VMX_EXIT(LOAD_PGC, 12) \ - VMX_EXIT(INTRACK, 15) \ - VMX_EXIT(SAVE_PAT, 18) \ - VMX_EXIT(LOAD_PAT, 19) \ - VMX_EXIT(SAVE_EFER, 20) \ - VMX_EXIT(LOAD_EFER, 21) \ - VMX_EXIT(SAVE_TIMER, 22) + VMX_EXIT(SAVE_DEBUGCTL, 2) \ + VMX_EXIT(LONGMODE, 9) \ + VMX_EXIT(LOAD_PGC, 12) \ + VMX_EXIT(INTRACK, 15) \ + VMX_EXIT(SAVE_PAT, 18) \ + VMX_EXIT(LOAD_PAT, 19) \ + VMX_EXIT(SAVE_EFER, 20) \ + VMX_EXIT(LOAD_EFER, 21) \ + VMX_EXIT(SAVE_TIMER, 22) \ + VMX_EXIT(CLEAR_BNDCFGS, 23) \ + VMX_EXIT(PT_SUPPRESS_VMX_PKT, 24) #define VMX_EXIT_CTLS_CAP \ VMX_EXIT_CTLS_CAP_NDA \ @@ -313,13 +329,15 @@ enum { VMXCTL(_ENTRY_CTLS, _field, _pos) #define VMX_ENTRY_CTLS_CAP_NDA #define VMX_ENTRY_CTLS_CAP_PUB \ - VMX_ENTRY(LOAD_DEBUGCTL, 2) \ - VMX_ENTRY(LONGMODE, 9) \ - VMX_ENTRY(ENTRY_TO_SMM, 10) \ - VMX_ENTRY(SMM_TEARDOWN, 11) \ - VMX_ENTRY(LOAD_PGC, 13) \ - VMX_ENTRY(LOAD_PAT, 14) \ - VMX_ENTRY(LOAD_EFER, 15) + VMX_ENTRY(LOAD_DEBUGCTL, 2) \ + VMX_ENTRY(LONGMODE, 9) \ + VMX_ENTRY(ENTRY_TO_SMM, 10) \ + VMX_ENTRY(SMM_TEARDOWN, 11) \ + VMX_ENTRY(LOAD_PGC, 13) \ + VMX_ENTRY(LOAD_PAT, 14) \ + VMX_ENTRY(LOAD_EFER, 15) \ + VMX_ENTRY(LOAD_BNDCFGS, 16) \ + VMX_ENTRY(PT_SUPPRESS_VMX_PKT, 17) #define VMX_ENTRY_CTLS_CAP \ VMX_ENTRY_CTLS_CAP_NDA \ @@ -332,17 +350,18 @@ enum { VMXCAP(_MISC, _field, _pos, _len) #define VMX_MISC_CAP_NDA #define VMX_MISC_CAP_PUB \ - VMX_MISC(TMR_RATIO, 0, 5) \ - VMX_MISC(VMEXIT_SAVES_LMA, 5, 1) \ - VMX_MISC(ACTSTATE_HLT, 6, 1) \ - VMX_MISC(ACTSTATE_SHUTDOWN, 7, 1) \ - VMX_MISC(ACTSTATE_SIPI, 8, 1) \ - VMX_MISC(RDMSR_SMBASE_IN_SMM, 15, 1) \ - VMX_MISC(CR3_TARGETS, 16, 9) \ - VMX_MISC(MAX_MSRS, 25, 3) \ - VMX_MISC(ALLOW_ALL_VMWRITES, 29, 1) \ - VMX_MISC(ZERO_VMENTRY_INSTLEN, 30, 1) \ - VMX_MISC(MSEG_ID, 32, 32) \ + VMX_MISC(TMR_RATIO, 0, 5) \ + VMX_MISC(VMEXIT_SAVES_LMA, 5, 1) \ + VMX_MISC(ACTSTATE_HLT, 6, 1) \ + VMX_MISC(ACTSTATE_SHUTDOWN, 7, 1) \ + VMX_MISC(ACTSTATE_SIPI, 8, 1) \ + VMX_MISC(PROCESSOR_TRACE_IN_VMX, 14, 1) \ + VMX_MISC(RDMSR_SMBASE_IN_SMM, 15, 1) \ + VMX_MISC(CR3_TARGETS, 16, 9) \ + VMX_MISC(MAX_MSRS, 25, 3) \ + VMX_MISC(ALLOW_ALL_VMWRITES, 29, 1) \ + VMX_MISC(ZERO_VMENTRY_INSTLEN, 30, 1) \ + VMX_MISC(MSEG_ID, 32, 32) \ #define VMX_MISC_CAP \ VMX_MISC_CAP_NDA \ @@ -427,6 +446,7 @@ enum { VMX_EPT(SP_1GB, 17, 1) \ VMX_EPT(INVEPT, 20, 1) \ VMX_EPT(ACCESS_DIRTY, 21, 1) \ + VMX_EPT(ADV_EXIT_INFO, 22, 1) \ VMX_EPT(INVEPT_EPT_CTX, 25, 1) \ VMX_EPT(INVEPT_GLOBAL, 26, 1) \ VMX_EPT(INVVPID, 32, 1) \ @@ -532,68 +552,14 @@ enum { /* Exception error must-be-zero bits for VMEntry */ #define VT_XCP_ERR_MBZ 0xffff8000 + /* Exit reasons. */ -#define VT_EXITREASON_EXC_OR_NMI 0 -#define VT_EXITREASON_EXTINT 1 -#define VT_EXITREASON_TRIPLEFAULT 2 -#define VT_EXITREASON_INIT 3 -#define VT_EXITREASON_SIPI 4 -#define VT_EXITREASON_IOSMI 5 -#define VT_EXITREASON_OTHERSMI 6 -#define VT_EXITREASON_VINTR_WINDOW 7 -#define VT_EXITREASON_VNMI_WINDOW 8 -#define VT_EXITREASON_TS 9 -#define VT_EXITREASON_CPUID 10 -#define VT_EXITREASON_GETSEC 11 -#define VT_EXITREASON_HLT 12 -#define VT_EXITREASON_INVD 13 -#define VT_EXITREASON_INVLPG 14 -#define VT_EXITREASON_RDPMC 15 -#define VT_EXITREASON_RDTSC 16 -#define VT_EXITREASON_RSM 17 -#define VT_EXITREASON_VMCALL 18 -#define VT_EXITREASON_VMCLEAR 19 -#define VT_EXITREASON_VMLAUNCH 20 -#define VT_EXITREASON_VMPTRLD 21 -#define VT_EXITREASON_VMPTRST 22 -#define VT_EXITREASON_VMREAD 23 -#define VT_EXITREASON_VMRESUME 24 -#define VT_EXITREASON_VMWRITE 25 -#define VT_EXITREASON_VMXOFF 26 -#define VT_EXITREASON_VMXON 27 -#define VT_EXITREASON_CR 28 -#define VT_EXITREASON_DR 29 -#define VT_EXITREASON_IO 30 -#define VT_EXITREASON_RDMSR 31 -#define VT_EXITREASON_WRMSR 32 -#define VT_EXITREASON_VMENTRYFAIL_GUEST (33 | VT_EXITREASON_VMENTRYFAIL) -#define VT_EXITREASON_VMENTRYFAIL_MSR (34 | VT_EXITREASON_VMENTRYFAIL) -#define VT_EXITREASON_MWAIT 36 -#define VT_EXITREASON_MTF 37 -#define VT_EXITREASON_MONITOR 39 -#define VT_EXITREASON_PAUSE 40 -#define VT_EXITREASON_VMENTRYFAIL_MC (41 | VT_EXITREASON_VMENTRYFAIL) -#define VT_EXITREASON_TPR 43 -#define VT_EXITREASON_APIC 44 -#define VT_EXITREASON_EOI 45 -#define VT_EXITREASON_GDTR_IDTR 46 -#define VT_EXITREASON_LDTR_TR 47 -#define VT_EXITREASON_EPT_VIOLATION 48 -#define VT_EXITREASON_EPT_MISCONFIG 49 -#define VT_EXITREASON_INVEPT 50 -#define VT_EXITREASON_RDTSCP 51 -#define VT_EXITREASON_TIMER 52 -#define VT_EXITREASON_INVVPID 53 -#define VT_EXITREASON_WBINVD 54 -#define VT_EXITREASON_XSETBV 55 -#define VT_EXITREASON_APIC_WRITE 56 -#define VT_EXITREASON_RDRAND 57 -#define VT_EXITREASON_INVPCID 58 -#define VT_EXITREASON_VMFUNC 59 -#define VT_EXITREASON_ENCLS 60 -#define VT_EXITREASON_RDSEED 61 -#define VT_EXITREASON_XSAVES 63 -#define VT_EXITREASON_XRSTORS 64 +enum { +#define VT_EXIT(_name, _val) VT_EXITREASON_##_name = _val, +#include "x86vt-exit-reasons.h" +#undef VT_EXIT +}; + /* * VT synthesized exit reasons: @@ -618,7 +584,6 @@ enum { (VT_EXITREASON_SYNTH_EXC_BASE + gatenum) /* 0-31 */ #define VT_EXITREASON_INSIDE_ENCLAVE (1U << 27) -#define VT_EXITREASON_VMENTRYFAIL (1U << 31) /* Instruction error codes. */ #define VT_ERROR_VMCALL_VMX_ROOT 1 @@ -690,12 +655,18 @@ enum { #define VT_EPT_QUAL_ACCESS_W (1 << 1) #define VT_EPT_QUAL_ACCESS_X (1 << 2) #define VT_EPT_QUAL_PROT_SHIFT 3 -#define VT_EPT_QUAL_PROT_MASK (0x7 << VT_EPT_QUAL_PROT_SHIFT) +#define VT_EPT_QUAL_PROT_MASK(_mbx) (((_mbx) ? 0xf : 0x7) << \ + VT_EPT_QUAL_PROT_SHIFT) #define VT_EPT_QUAL_PROT_R (1 << 3) #define VT_EPT_QUAL_PROT_W (1 << 4) #define VT_EPT_QUAL_PROT_X (1 << 5) +#define VT_EPT_QUAL_PROT_XS (1 << 5) +#define VT_EPT_QUAL_PROT_XU (1 << 6) #define VT_EPT_QUAL_LA_VALID (1 << 7) #define VT_EPT_QUAL_FINAL_ADDR (1 << 8) +#define VT_EPT_QUAL_GUEST_US (1 << 9) +#define VT_EPT_QUAL_GUEST_RW (1 << 10) +#define VT_EPT_QUAL_GUEST_NX (1 << 11) #define VT_EPT_QUAL_NMIUNMASK (1 << 12) @@ -772,12 +743,120 @@ enum { MSR_VMX_EPT_VPID_INVVPID_ALL_CTX) #define VT_REQUIRED_EPT_SUPPORT \ - (MSR_VMX_EPT_VPID_EPTE_X | \ - MSR_VMX_EPT_VPID_GAW_48 | \ + (MSR_VMX_EPT_VPID_GAW_48 | \ MSR_VMX_EPT_VPID_ETMT_WB | \ MSR_VMX_EPT_VPID_SP_2MB | \ - MSR_VMX_EPT_VPID_INVEPT | \ - MSR_VMX_EPT_VPID_INVEPT_EPT_CTX) + MSR_VMX_EPT_VPID_INVEPT) + +#define VT_TSQUAL_CALL 0 +#define VT_TSQUAL_IRET 1 +#define VT_TSQUAL_JMP 2 +#define VT_TSQUAL_GATE 3 + +typedef union { + struct { + unsigned selVal:16; + unsigned rsvd0:14; + unsigned source:2; + } bits; + uint32 flat; +} VTTSQualifier; + +#define VT_CRQUAL_WR 0 +#define VT_CRQUAL_RD 1 +#define VT_CRQUAL_CLTS 2 +#define VT_CRQUAL_LMSW 3 + +/* Control register intercept qualifier */ +typedef union { + struct { + unsigned num:4; // For MOV-CR + unsigned op:2; + unsigned mem:1; // For LMSW + unsigned rsvd0:1; + unsigned gpr:4; // For MOV-CR + unsigned rsvd1:4; + unsigned data:16; // For LMSW + } bits; + uint32 flat; +} VTCRQualifier; + +#define VT_DRQUAL_WR 0 +#define VT_DRQUAL_RD 1 + +/* Debug register intercept qualifier */ +typedef union { + struct { + unsigned num:3; + unsigned rsvd0:1; + unsigned op:1; + unsigned rsvd1:3; + unsigned gpr:4; + unsigned rsvd2:20; + } bits; + uint32 flat; +} VTDRQualifier; + +#define VT_IOQUAL_SZ8 0 +#define VT_IOQUAL_SZ16 1 +#define VT_IOQUAL_SZ32 3 + +/* I/O intercept qualifier */ +typedef union { + struct { + unsigned opSize:3; // 0 = 1-byte; 1 = 2-byte; 3 = 4-byte + unsigned in:1; + unsigned string:1; + unsigned rep:1; + unsigned imm:1; + unsigned rsvd0:9; + unsigned port:16; + } bits; + uint32 flat; +} VTIOQualifier; + +#define VT_IINFO_SCALE1 0 +#define VT_IINFO_SCALE2 1 +#define VT_IINFO_SCALE4 2 +#define VT_IINFO_SCALE8 3 + +#define VT_IINFO_SZ16 0 +#define VT_IINFO_SZ32 1 +#define VT_IINFO_SZ64 2 + +#define VT_IINFO_SGDT 0 +#define VT_IINFO_SIDT 1 +#define VT_IINFO_LGDT 2 +#define VT_IINFO_LIDT 3 + +#define VT_IINFO_SLDT 0 +#define VT_IINFO_STR 1 +#define VT_IINFO_LLDT 2 +#define VT_IINFO_LTR 3 + +/* VM-Exit Instruction-Information */ +typedef union { + struct { + unsigned scale:2; // Bits 1:0 + unsigned rsvd0:1; // Bit 2 + unsigned reg1:4; // Bits 6:3 + unsigned aSize:3; // Bits 9:7 + unsigned modrmReg:1; // Bit 10 + unsigned oSize:2; // Bits 12:11 + unsigned rsvd1:2; // Bit 14:13 + unsigned seg:3; // Bits 17:15 + unsigned indexReg:5; // Bits 22:18 + unsigned baseReg:5; // Bits 27:23 + unsigned misc:4; // Bits 31:28 + } bits; + uint32 flat; +} VTInstrInfo; + +typedef struct VTMSREntry { + uint32 index; + uint32 reserved; + uint64 data; +} VTMSREntry; typedef uint64 VTConfig[NUM_VMX_MSRS]; @@ -941,6 +1020,25 @@ VT_RealModeSupportedFromFeatures(uint64 secondary) } +/* + *---------------------------------------------------------------------- + * + * VT_MBXSupportedFromFeatures -- + * + * Returns TRUE if the given VMX features provide support for + * mode-based execute control for EPT. + * + * Assumes that VT is supported. + * + *---------------------------------------------------------------------- + */ +static INLINE Bool +VT_MBXSupportedFromFeatures(uint64 secondary) +{ + return (HIDWORD(secondary) & VT_VMCS_2ND_VMEXEC_CTL_EPT_MBX) != 0; +} + + #if !defined(USERLEVEL) && !defined(MONITOR_APP) /* { */ /* *---------------------------------------------------------------------- diff --git a/vmmon-only/include/x86vtinstr.h b/vmmon-only/include/x86vtinstr.h index 844f92f..8363092 100644 --- a/vmmon-only/include/x86vtinstr.h +++ b/vmmon-only/include/x86vtinstr.h @@ -34,6 +34,11 @@ #include "x86_basic_defs.h" #include "community_source.h" +#if defined __cplusplus +extern "C" { +#endif + + #define INVALID_VMCS_ADDR ~0ULL /* @@ -67,23 +72,23 @@ VMXStatus_Valid(VMXStatus status) #if defined __GNUC__ static INLINE VMXStatus -VMXON_2_STATUS(MA* vmcs) +VMXON_2_STATUS(MA* vmxonRegion) { VMXStatus status; __asm__ __volatile__("vmxon %1; lahf; movzbl %%ah, %0" : "=a"(status) - : "m"(*vmcs) + : "m"(*vmxonRegion) : "cc", "memory"); ASSERT(VMXStatus_Valid(status)); return status; } static INLINE void -VMXON_UNCHECKED(MA *vmcs) +VMXON_UNCHECKED(MA *vmxonRegion) { __asm__ __volatile__("vmxon %0" : - : "m"(*vmcs) + : "m"(*vmxonRegion) : "cc", "memory"); } @@ -129,18 +134,6 @@ VMCLEAR_UNCHECKED(MA* vmcs) : "cc", "memory"); } -static INLINE void -VMCLEAR(MA* vmcs) -{ - if (vmx86_debug) { - VMXStatus status; - status = VMCLEAR_2_STATUS(vmcs); - ASSERT(status == VMX_Success); - } else { - VMCLEAR_UNCHECKED(vmcs); - } -} - static INLINE VMXStatus VMPTRLD_2_STATUS(MA *vmcs) { @@ -208,20 +201,6 @@ VMREAD_UNCHECKED(size_t encoding) return retval; } -static INLINE size_t -VMREAD(size_t encoding) -{ - size_t retval; - if (vmx86_debug) { - VMXStatus status; - status = VMREAD_2_STATUS(encoding, &retval); - ASSERT(status == VMX_Success); - } else { - retval = VMREAD_UNCHECKED(encoding); - } - return retval; -} - static INLINE VMXStatus VMWRITE_2_STATUS(size_t encoding, size_t val) { @@ -507,21 +486,21 @@ void __vmx_vmptrst(unsigned __int64 *); static INLINE VMXStatus -VMXON_2_STATUS(MA *vmcs) +VMXON_2_STATUS(MA *vmxonRegion) { unsigned char mscStatus; static const VMXStatus MscToStatus[] = {VMX_Success, VMX_FailValid, VMX_FailInvalid}; - mscStatus = __vmx_on((unsigned __int64 *)vmcs); + mscStatus = __vmx_on((unsigned __int64 *)vmxonRegion); ASSERT(mscStatus < ARRAYSIZE(MscToStatus)); return MscToStatus[mscStatus]; } static INLINE void -VMXON_UNCHECKED(MA *vmcs) +VMXON_UNCHECKED(MA *vmxonRegion) { - (void)__vmx_on((unsigned __int64 *)vmcs); + (void)__vmx_on((unsigned __int64 *)vmxonRegion); } static INLINE VMXStatus @@ -537,6 +516,45 @@ VMXOFF_UNCHECKED(void) (void)__vmx_off(); } +static INLINE void +VMCLEAR_UNCHECKED(MA *vmcs) +{ + (void)__vmx_vmclear((unsigned __int64 *)vmcs); +} + +static INLINE size_t +VMREAD_UNCHECKED(size_t encoding) +{ + size_t retval; + (void)__vmx_vmread(encoding, &retval); + return retval; +} + +static INLINE VMXStatus +VMCLEAR_2_STATUS(MA *vmcs) +{ + unsigned char mscStatus; + static const VMXStatus MscToStatus[] = + {VMX_Success, VMX_FailValid, VMX_FailInvalid}; + + mscStatus = __vmx_vmclear((unsigned __int64 *)vmcs); + ASSERT(mscStatus < ARRAYSIZE(MscToStatus)); + return MscToStatus[mscStatus]; +} + +static INLINE VMXStatus +VMREAD_2_STATUS(size_t encoding, size_t *retval) +{ + unsigned char mscStatus; + static const VMXStatus MscToStatus[] = + {VMX_Success, VMX_FailValid, VMX_FailInvalid}; + + mscStatus = __vmx_vmread(encoding, retval); + ASSERT(mscStatus < ARRAYSIZE(MscToStatus)); + return MscToStatus[mscStatus]; +} + + static INLINE VMXStatus VMPTRLD_2_STATUS(MA *vmcs) { @@ -573,14 +591,14 @@ VMPTRST_UNCHECKED(MA *vmcs) static INLINE void -VMXON(MA *vmcs) +VMXON(MA *vmxonRegion) { if (vmx86_debug) { VMXStatus status; - status = VMXON_2_STATUS(vmcs); + status = VMXON_2_STATUS(vmxonRegion); ASSERT(status == VMX_Success); } else { - VMXON_UNCHECKED(vmcs); + VMXON_UNCHECKED(vmxonRegion); } } @@ -620,4 +638,36 @@ VMPTRST(MA *vmcs) } } + +static INLINE void +VMCLEAR(MA* vmcs) +{ + if (vmx86_debug) { + VMXStatus status; + status = VMCLEAR_2_STATUS(vmcs); + ASSERT(status == VMX_Success); + } else { + VMCLEAR_UNCHECKED(vmcs); + } +} + +static INLINE size_t +VMREAD(size_t encoding) +{ + size_t retval; + if (vmx86_debug) { + VMXStatus status; + status = VMREAD_2_STATUS(encoding, &retval); + ASSERT(status == VMX_Success); + } else { + retval = VMREAD_UNCHECKED(encoding); + } + return retval; +} + + +#if defined __cplusplus +} // extern "C" +#endif + #endif /* _X86VTINSTR_H_ */ diff --git a/vmmon-only/linux/driver.c b/vmmon-only/linux/driver.c index 3595237..3080525 100644 --- a/vmmon-only/linux/driver.c +++ b/vmmon-only/linux/driver.c @@ -23,7 +23,7 @@ #include #include -#include +#include #include #include #include @@ -33,12 +33,11 @@ #include "compat_version.h" #include "compat_module.h" -#include "compat_page.h" #include "usercalldefs.h" -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16) -#error Linux before 2.6.16 is not supported +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) +#error Linux kernels before 2.6.32 are not supported #endif #include @@ -66,16 +65,10 @@ #include "hostif_priv.h" #include "vmhost.h" -#include "vmmonInt.h" - static void LinuxDriverQueue(VMLinux *vmLinux); static void LinuxDriverDequeue(VMLinux *vmLinux); static Bool LinuxDriverCheckPadding(void); -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24) -#define VMW_NOPAGE_2624 -#endif - #define VMMON_UNKNOWN_SWAP_SIZE -1ULL struct VMXLinuxState linuxState; @@ -103,27 +96,17 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd, unsigned long ioarg); static int LinuxDriver_Close(struct inode *inode, struct file *filp); -static unsigned int LinuxDriverPoll(struct file *file, poll_table *wait); #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) static int LinuxDriverFault(struct vm_fault *fault); -#elif defined(VMW_NOPAGE_2624) -static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault); #else -static struct page *LinuxDriverNoPage(struct vm_area_struct *vma, - unsigned long address, - int *type); +static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault); #endif static int LinuxDriverMmap(struct file *filp, struct vm_area_struct *vma); -static void LinuxDriverPollTimeout(unsigned long clientData); static unsigned int LinuxDriverEstimateTSCkHz(void); static struct vm_operations_struct vmuser_mops = { -#ifdef VMW_NOPAGE_2624 .fault = LinuxDriverFault -#else - .nopage = LinuxDriverNoPage -#endif }; static struct file_operations vmuser_fops; @@ -135,7 +118,41 @@ static VmTimeStart tsckHzStartTime; /* *---------------------------------------------------------------------- * - * LinuxDriverEstimateTSCkHzWork -- + * LinuxDriverReadTSCAndUptimeSmpCB -- + * LinuxDriverReadTSCAndUptime -- + * + * Read TSC and uptime on CPU 0. Reading on CPU 0 is best + * effort, and the remote smp function call may fail for two + * reasons: either the function is not supportd by the kernel, + * or the cpu went offline. In this unlikely event, we perform + * the read on the local cpu. + * + *---------------------------------------------------------------------- + */ + +static void +LinuxDriverReadTSCAndUptimeSmpCB(void *arg) +{ + VmTimeStart *time = (VmTimeStart *) arg; + Vmx86_ReadTSCAndUptime(time); + /* Ensure the above write is visible to the remote caller. */ + SMP_RW_BARRIER_RW(); +} + +static void +LinuxDriverReadTSCAndUptime(VmTimeStart *time) +{ + if (smp_call_function_single(0, LinuxDriverReadTSCAndUptimeSmpCB, + (void *)time, 1) != 0) { + LinuxDriverReadTSCAndUptimeSmpCB(time); + } +} + + +/* + *---------------------------------------------------------------------- + * + * LinuxDriverEstimateTSCkHz -- * * Estimates TSC frequency in terms of cycles and system uptime * elapsed since module init. At module init, the starting cycle @@ -149,19 +166,30 @@ static VmTimeStart tsckHzStartTime; * query races with the timer, the first thread to write to 'tsckHz' * wins. * + * Results: + * + * Returns the estimated TSC khz, cached in tscKhz. If tsckHz is + * 0, the reads uptime on CPU 0 and estimates tsc khz, followed + * by caching it in tsckHz. + * *---------------------------------------------------------------------- */ -static void -LinuxDriverEstimateTSCkHzWork(void *data) +static uint32 +LinuxDriverEstimateTSCkHz(void) { + uint32 khz; VmTimeStart curTime; uint64 cycles; uint64 uptime; - unsigned int khz; - ASSERT(tsckHzStartTime.count != 0 && tsckHzStartTime.time != 0); - Vmx86_ReadTSCAndUptime(&curTime); + khz = Atomic_Read(&tsckHz); + if (khz != 0) { + return khz; + } + + ASSERT(tsckHzStartTime.count != 0); + LinuxDriverReadTSCAndUptime(&curTime); cycles = curTime.count - tsckHzStartTime.count; uptime = curTime.time - tsckHzStartTime.time; khz = Vmx86_ComputekHz(cycles, uptime); @@ -173,45 +201,6 @@ LinuxDriverEstimateTSCkHzWork(void *data) } else if (Atomic_ReadIfEqualWrite(&tsckHz, 0, cpu_khz) == 0) { Log("Failed to compute TSC frequency, using cpu_khz: %u\n", cpu_khz); } -} - - -/* - *---------------------------------------------------------------------- - * - * LinuxDriverEstimateTSCkHz -- - * - * Returns the estimated TSC khz, cached in tscKhz. If tsckHz is - * 0, the routine kicks off estimation work on CPU 0. - * - * Results: - * - * Returns the estimated TSC khz value. - * - *---------------------------------------------------------------------- - */ - -static unsigned int -LinuxDriverEstimateTSCkHz(void) -{ - int err; - uint32 khz; - - khz = Atomic_Read(&tsckHz); - if (khz != 0) { - return khz; - } - err = compat_smp_call_function_single(0, LinuxDriverEstimateTSCkHzWork, - NULL, 1); - /* - * The smp function call may fail for two reasons, either - * the function is not supportd by the kernel, or the cpu - * went offline. In this unlikely event, we just perform - * the work wherever we can. - */ - if (err != 0) { - LinuxDriverEstimateTSCkHzWork(NULL); - } return Atomic_Read(&tsckHz); } @@ -255,23 +244,20 @@ LinuxDriverEstimateTSCkHzDeferred(unsigned long data) static void LinuxDriverInitTSCkHz(void) { - unsigned int khz; - - khz = compat_tsc_khz(); - if (khz != 0) { - Atomic_Write(&tsckHz, khz); - Log("Using tsc_khz as TSC frequency: %u\n", khz); + if (tsc_khz != 0) { /* Exported kernel value */ + Atomic_Write(&tsckHz, tsc_khz); + Log("Using tsc_khz as TSC frequency: %u\n", tsc_khz); return; } - Vmx86_ReadTSCAndUptime(&tsckHzStartTime); + LinuxDriverReadTSCAndUptime(&tsckHzStartTime); tscTimer.function = LinuxDriverEstimateTSCkHzDeferred; tscTimer.expires = jiffies + 4 * HZ; tscTimer.data = 0; add_timer(&tscTimer); } - + /* *---------------------------------------------------------------------- * @@ -300,23 +286,13 @@ init_module(void) } CPUID_Init(); + if (!Task_Initialize()) { return -ENOEXEC; } - /* - * Initialize LinuxDriverPoll state - */ - - init_waitqueue_head(&linuxState.pollQueue); - init_timer(&linuxState.pollTimer); - linuxState.pollTimer.data = 0; - linuxState.pollTimer.function = LinuxDriverPollTimeout; - linuxState.fastClockThread = NULL; - linuxState.fastClockFile = NULL; linuxState.fastClockRate = 0; - linuxState.fastClockPriority = -20; linuxState.swapSize = VMMON_UNKNOWN_SWAP_SIZE; /* @@ -327,7 +303,6 @@ init_module(void) memset(&vmuser_fops, 0, sizeof vmuser_fops); vmuser_fops.owner = THIS_MODULE; - vmuser_fops.poll = LinuxDriverPoll; vmuser_fops.unlocked_ioctl = LinuxDriver_Ioctl; vmuser_fops.compat_ioctl = LinuxDriver_Ioctl; vmuser_fops.open = LinuxDriver_Open; @@ -394,7 +369,6 @@ cleanup_module(void) Log("Module %s: unloaded\n", linuxState.deviceName); - del_timer_sync(&linuxState.pollTimer); del_timer_sync(&tscTimer); Task_Terminate(); @@ -435,7 +409,6 @@ LinuxDriver_Open(struct inode *inode, // IN memset(vmLinux, 0, sizeof *vmLinux); sema_init(&vmLinux->lock4Gb, 1); - init_waitqueue_head(&vmLinux->pollQueue); filp->private_data = vmLinux; LinuxDriverQueue(vmLinux); @@ -591,20 +564,6 @@ LinuxDriver_Close(struct inode *inode, // IN LinuxDriverDestructor4Gb(vmLinux); - /* - * Clean up poll state. - */ - - HostIF_PollListLock(0); - if (vmLinux->pollBack != NULL) { - if ((*vmLinux->pollBack = vmLinux->pollForw) != NULL) { - vmLinux->pollForw->pollBack = vmLinux->pollBack; - } - } - HostIF_PollListUnlock(0); - // XXX call wake_up()? - HostIF_UnmapUserMem(vmLinux->pollTimeoutHandle); - kfree(vmLinux); filp->private_data = NULL; @@ -612,262 +571,10 @@ LinuxDriver_Close(struct inode *inode, // IN } -#define POLLQUEUE_MAX_TASK 1000 -static DEFINE_SPINLOCK(pollQueueLock); -static void *pollQueue[POLLQUEUE_MAX_TASK]; -static unsigned int pollQueueCount = 0; - - /* *----------------------------------------------------------------------------- * - * LinuxDriverQueuePoll -- - * - * Remember that current process waits for next timer event. - * - * Results: - * None. - * - * Side effects: - * None. - * - *----------------------------------------------------------------------------- - */ - -static INLINE_SINGLE_CALLER void -LinuxDriverQueuePoll(void) -{ - unsigned long flags; - - spin_lock_irqsave(&pollQueueLock, flags); - - /* - * Under normal circumstances every process should be listed - * only once in this array. If it becomes problem that process - * can be in the array twice, walk array! Maybe you can keep - * it sorted by 'current' value then, making IsPollQueued - * a bit faster... - */ - - if (pollQueueCount < POLLQUEUE_MAX_TASK) { - pollQueue[pollQueueCount++] = current; - } - spin_unlock_irqrestore(&pollQueueLock, flags); -} - - -/* - *----------------------------------------------------------------------------- - * - * LinuxDriverIsPollQueued -- - * - * Determine whether timer event occurred since we queued for it using - * LinuxDriverQueuePoll. - * - * Results: - * 0 Event already occurred. - * 1 Event did not occur yet. - * - * Side effects: - * None. - * - *----------------------------------------------------------------------------- - */ - -static INLINE_SINGLE_CALLER int -LinuxDriverIsPollQueued(void) -{ - unsigned long flags; - unsigned int i; - int retval = 0; - - spin_lock_irqsave(&pollQueueLock, flags); - for (i = 0; i < pollQueueCount; i++) { - if (current == pollQueue[i]) { - retval = 1; - break; - } - } - spin_unlock_irqrestore(&pollQueueLock, flags); - - return retval; -} - - -/* - *----------------------------------------------------------------------------- - * - * LinuxDriverFlushPollQueue -- - * - * Signal to queue that timer event occurred. - * - * Results: - * None. - * - * Side effects: - * None. - * - *----------------------------------------------------------------------------- - */ - -static INLINE_SINGLE_CALLER void -LinuxDriverFlushPollQueue(void) -{ - unsigned long flags; - - spin_lock_irqsave(&pollQueueLock, flags); - pollQueueCount = 0; - spin_unlock_irqrestore(&pollQueueLock, flags); -} - - -/* - *----------------------------------------------------------------------------- - * - * LinuxDriverWakeUp -- - * - * Wake up processes waiting on timer event. - * - * Results: - * None. - * - * Side effects: - * None. - * - *----------------------------------------------------------------------------- - */ - -void -LinuxDriverWakeUp(Bool selective) // IN: -{ - if (selective && linuxState.pollList != NULL) { - struct timeval tv; - VmTimeType now; - VMLinux *p; - VMLinux *next; - - HostIF_PollListLock(1); - do_gettimeofday(&tv); - now = tv.tv_sec * 1000000ULL + tv.tv_usec; - - for (p = linuxState.pollList; p != NULL; p = next) { - next = p->pollForw; - - if (p->pollTime <= now) { - if ((*p->pollBack = next) != NULL) { - next->pollBack = p->pollBack; - } - p->pollForw = NULL; - p->pollBack = NULL; - wake_up(&p->pollQueue); - } - } - HostIF_PollListUnlock(1); - } - - LinuxDriverFlushPollQueue(); - wake_up(&linuxState.pollQueue); -} - - -/* - *---------------------------------------------------------------------- - * - * LinuxDriverPoll -- - * - * This is used to wake up the VMX when a user call arrives, or - * to wake up select() or poll() at the next clock tick. - * - *---------------------------------------------------------------------- - */ - -static unsigned int -LinuxDriverPoll(struct file *filp, // IN: - poll_table *wait) // IN: -{ - VMLinux *vmLinux = (VMLinux *) filp->private_data; - unsigned int mask = 0; - - /* - * Set up or check the timeout for fast wakeup. - * - * Thanks to Petr for this simple and correct implementation: - * - * There are four cases of wait == NULL: - * another file descriptor is ready in the same poll() - * just slept and woke up - * nonblocking poll() - * did not sleep due to memory allocation on 2.4.21-9.EL - * In first three cases, it's okay to return POLLIN. - * Unfortunately, for 4th variant we have to do some - * bookkeeping to not return POLLIN when timer did not expire - * yet. - * - * We may schedule a timer unnecessarily if an existing - * timer fires between poll_wait() and timer_pending(). - * - * -- edward - */ - - if (wait == NULL) { - if (vmLinux->pollBack == NULL && !LinuxDriverIsPollQueued()) { - mask = POLLIN; - } - } else { - if (linuxState.fastClockThread && vmLinux->pollTimeoutPtr != NULL) { - struct timeval tv; - - do_gettimeofday(&tv); - poll_wait(filp, &vmLinux->pollQueue, wait); - vmLinux->pollTime = *vmLinux->pollTimeoutPtr + - tv.tv_sec * 1000000ULL + tv.tv_usec; - if (vmLinux->pollBack == NULL) { - HostIF_PollListLock(2); - if (vmLinux->pollBack == NULL) { - if ((vmLinux->pollForw = linuxState.pollList) != NULL) { - vmLinux->pollForw->pollBack = &vmLinux->pollForw; - } - linuxState.pollList = vmLinux; - vmLinux->pollBack = &linuxState.pollList; - } - HostIF_PollListUnlock(2); - } - } else { - LinuxDriverQueuePoll(); - poll_wait(filp, &linuxState.pollQueue, wait); - - if (!timer_pending(&linuxState.pollTimer)) { - mod_timer(&linuxState.pollTimer, jiffies + 1); - } - } - } - - return mask; -} - - -/* - *---------------------------------------------------------------------- - * - * LinuxDriverPollTimeout -- - * - * Wake up a process waiting in poll/select. This is called from - * the timer, and hence processed in the bottom half - * - *---------------------------------------------------------------------- - */ - -static void -LinuxDriverPollTimeout(unsigned long clientData) // IN: -{ - LinuxDriverWakeUp(FALSE); -} - - -/* - *----------------------------------------------------------------------------- - * - * LinuxDriverNoPage/LinuxDriverFault -- + * LinuxDriverFault -- * * Callback for returning allocated page for memory mapping * @@ -883,16 +590,12 @@ LinuxDriverPollTimeout(unsigned long clientData) // IN: *----------------------------------------------------------------------------- */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) static int +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) LinuxDriverFault(struct vm_fault *fault) //IN/OUT -#elif defined(VMW_NOPAGE_2624) -static int LinuxDriverFault(struct vm_area_struct *vma, //IN - struct vm_fault *fault) //IN/OUT #else -static struct page *LinuxDriverNoPage(struct vm_area_struct *vma, //IN - unsigned long address, //IN - int *type) //OUT: Fault type +LinuxDriverFault(struct vm_area_struct *vma, //IN + struct vm_fault *fault) //IN/OUT #endif { #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) @@ -902,28 +605,15 @@ static struct page *LinuxDriverNoPage(struct vm_area_struct *vma, //IN unsigned long pg; struct page* page; -#ifdef VMW_NOPAGE_2624 pg = fault->pgoff; -#else - pg = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; -#endif pg = VMMON_MAP_OFFSET(pg); if (pg >= vmLinux->size4Gb) { -#ifdef VMW_NOPAGE_2624 return VM_FAULT_SIGBUS; -#else - return 0; -#endif } page = vmLinux->pages4Gb[pg]; get_page(page); -#ifdef VMW_NOPAGE_2624 fault->page = page; return 0; -#else - *type = VM_FAULT_MINOR; - return page; -#endif } @@ -1198,7 +888,7 @@ LinuxDriverSyncCallOnEachCPU(SyncFunc func, // IN: * on this CPU too. */ - compat_smp_call_function(LinuxDriverSyncCallHook, &args, 0); + smp_call_function(LinuxDriverSyncCallHook, &args, 0); /* * smp_call_function doesn't return until all cpus have been @@ -1262,10 +952,10 @@ LinuxDriverReadTSC(void *data, // OUT: TSC values /* Any looping means another CPU changed min/max. */ do { old = Atomic_Read64(&tscDelta->min); - } while (old > tsc && !Atomic_CMPXCHG64(&tscDelta->min, &old, &tsc)); + } while (old > tsc && !Atomic_CMPXCHG64(&tscDelta->min, old, tsc)); do { old = Atomic_Read64(&tscDelta->max); - } while (old < tsc && !Atomic_CMPXCHG64(&tscDelta->max, &old, &tsc)); + } while (old < tsc && !Atomic_CMPXCHG64(&tscDelta->max, old, tsc)); return TRUE; } @@ -1356,24 +1046,18 @@ LinuxDriver_Ioctl(struct file *filp, // IN: switch (iocmd) { case IOCTL_VMX86_VERSION: case IOCTL_VMX86_CREATE_VM: - case IOCTL_VMX86_INIT_CROSSGDT: case IOCTL_VMX86_SET_UID: case IOCTL_VMX86_GET_NUM_VMS: case IOCTL_VMX86_GET_TOTAL_MEM_USAGE: case IOCTL_VMX86_SET_HARD_LIMIT: - case IOCTL_VMX86_PAE_ENABLED: - case IOCTL_VMX86_VMX_ENABLED: case IOCTL_VMX86_GET_IPI_VECTORS: case IOCTL_VMX86_GET_KHZ_ESTIMATE: case IOCTL_VMX86_GET_ALL_CPUID: case IOCTL_VMX86_GET_ALL_MSRS: - case IOCTL_VMX86_SET_POLL_TIMEOUT_PTR: - case IOCTL_VMX86_GET_KERNEL_CLOCK_RATE: case IOCTL_VMX86_GET_REFERENCE_CLOCK_HZ: case IOCTL_VMX86_INIT_PSEUDO_TSC: case IOCTL_VMX86_CHECK_PSEUDO_TSC: case IOCTL_VMX86_GET_PSEUDO_TSC: - case IOCTL_VMX86_SET_HOST_CLOCK_PRIORITY: case IOCTL_VMX86_SYNC_GET_TSCS: case IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS: break; @@ -1394,65 +1078,79 @@ LinuxDriver_Ioctl(struct file *filp, // IN: retval = VMMON_VERSION; break; - case IOCTL_VMX86_CREATE_VM: + case IOCTL_VMX86_CREATE_VM: { + VMCreateBlock args; + if (vm != NULL) { retval = -EINVAL; break; } - - vm = Vmx86_CreateVM(); + retval = HostIF_CopyFromUser(&args, (VA64)ioarg, sizeof args); + if (retval != 0) { + break; + } + vm = Vmx86_CreateVM(args.bsBlob, args.bsBlobSize); if (vm == NULL) { retval = -ENOMEM; } else { vmLinux->vm = vm; - retval = vm->userID; + args.vmid = vm->userID; + retval = HostIF_CopyToUser((VA64)ioarg, &args, sizeof args); } break; + } + + case IOCTL_VMX86_PROCESS_BOOTSTRAP: { + VMProcessBootstrapBlock *args; + Bool res; + + args = HostIF_AllocKernelMem(sizeof *args, TRUE); + if (args == NULL) { + retval = -ENOMEM; + break; + } + retval = HostIF_CopyFromUser(args, (VA64)ioarg, sizeof *args); + if (retval != 0) { + HostIF_FreeKernelMem(args); + break; + } + res = Vmx86_ProcessBootstrap(vm, + args->bsBlobAddr, + args->numBytes, + args->headerOffset, + args->numVCPUs, + args->ptRootVAs, + args->shRegions); + if (!res) { + retval = -ENOMEM; + } + HostIF_FreeKernelMem(args); + break; + } case IOCTL_VMX86_RELEASE_VM: vmLinux->vm = NULL; Vmx86_ReleaseVM(vm); break; - case IOCTL_VMX86_ALLOC_CROSSGDT: { - InitBlock initBlock; - - if (Task_AllocCrossGDT(&initBlock)) { - retval = HostIF_CopyToUser((char *)ioarg, &initBlock, - sizeof initBlock); - } else { - retval = -EINVAL; - } - break; - } - case IOCTL_VMX86_INIT_VM: { - InitBlock initParams; + InitBlock *initParams; - retval = HostIF_CopyFromUser(&initParams, (char *)ioarg, - sizeof initParams); - if (retval != 0) { + initParams = HostIF_AllocKernelMem(sizeof *initParams, TRUE); + if (initParams == NULL) { + retval = -ENOMEM; break; } - if (Vmx86_InitVM(vm, &initParams)) { - retval = -EINVAL; - break; - } - retval = HostIF_CopyToUser((char *)ioarg, &initParams, - sizeof initParams); - break; - } - - case IOCTL_VMX86_INIT_CROSSGDT: { - InitCrossGDT initCrossGDT; - - retval = HostIF_CopyFromUser(&initCrossGDT, (char *)ioarg, - sizeof initCrossGDT); - - if ((retval == 0) && Task_InitCrossGDT(&initCrossGDT)) { - retval = -EIO; + retval = HostIF_CopyFromUser(initParams, ioarg, sizeof *initParams); + if (retval == 0) { + if (Vmx86_InitVM(vm, initParams) == 0) { + retval = HostIF_CopyToUser(ioarg, initParams, sizeof *initParams); + } else { + retval = -EINVAL; + } } + HostIF_FreeKernelMem(initParams); break; } @@ -1477,31 +1175,31 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_LOCK_PAGE: { VMLockPage args; - retval = HostIF_CopyFromUser(&args, (void *)ioarg, sizeof args); + retval = HostIF_CopyFromUser(&args, ioarg, sizeof args); if (retval) { break; } args.ret.status = Vmx86_LockPage(vm, args.uAddr, FALSE, &args.ret.mpn); - retval = HostIF_CopyToUser((void *)ioarg, &args, sizeof args); + retval = HostIF_CopyToUser(ioarg, &args, sizeof args); break; } case IOCTL_VMX86_LOCK_PAGE_NEW: { VMLockPage args; - retval = HostIF_CopyFromUser(&args, (void *)ioarg, sizeof args); + retval = HostIF_CopyFromUser(&args, ioarg, sizeof args); if (retval) { break; } args.ret.status = Vmx86_LockPage(vm, args.uAddr, TRUE, &args.ret.mpn); - retval = HostIF_CopyToUser((void *)ioarg, &args, sizeof args); + retval = HostIF_CopyToUser(ioarg, &args, sizeof args); break; } case IOCTL_VMX86_UNLOCK_PAGE: { VA64 uAddr; - retval = HostIF_CopyFromUser(&uAddr, (void *)ioarg, sizeof uAddr); + retval = HostIF_CopyFromUser(&uAddr, ioarg, sizeof uAddr); if (retval) { break; } @@ -1512,7 +1210,7 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_UNLOCK_PAGE_BY_MPN: { VMMUnlockPageByMPN args; - retval = HostIF_CopyFromUser(&args, (void *)ioarg, sizeof args); + retval = HostIF_CopyFromUser(&args, ioarg, sizeof args); if (retval) { break; } @@ -1523,12 +1221,12 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_LOOK_UP_MPN: { VMLockPage args; - retval = HostIF_CopyFromUser(&args, (void *)ioarg, sizeof args); + retval = HostIF_CopyFromUser(&args, ioarg, sizeof args); if (retval) { break; } args.ret.status = Vmx86_LookupUserMPN(vm, args.uAddr, &args.ret.mpn); - retval = HostIF_CopyToUser((void *)ioarg, &args, sizeof args); + retval = HostIF_CopyToUser(ioarg, &args, sizeof args); break; } @@ -1543,7 +1241,7 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_SET_HARD_LIMIT: { int32 limit; - retval = HostIF_CopyFromUser(&limit, (void *)ioarg, sizeof limit); + retval = HostIF_CopyFromUser(&limit, ioarg, sizeof limit); if (retval != 0) { break; } @@ -1556,19 +1254,19 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_ADMIT: { VMMemInfoArgs args; - retval = HostIF_CopyFromUser(&args, (void *)ioarg, sizeof args); + retval = HostIF_CopyFromUser(&args, ioarg, sizeof args); if (retval != 0) { break; } Vmx86_Admit(vm, &args); - retval = HostIF_CopyToUser((void *)ioarg, &args, sizeof args); + retval = HostIF_CopyToUser(ioarg, &args, sizeof args); break; } case IOCTL_VMX86_READMIT: { OvhdMem_Deltas delta; - retval = HostIF_CopyFromUser(&delta, (void *)ioarg, sizeof delta); + retval = HostIF_CopyFromUser(&delta, ioarg, sizeof delta); if (retval != 0) { break; } @@ -1582,7 +1280,7 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_UPDATE_MEM_INFO: { VMMemMgmtInfoPatch patch; - retval = HostIF_CopyFromUser(&patch, (void *)ioarg, sizeof patch); + retval = HostIF_CopyFromUser(&patch, ioarg, sizeof patch); if (retval == 0) { Vmx86_UpdateMemInfo(vm, &patch); } @@ -1591,17 +1289,15 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_GET_MEM_INFO: { VA64 uAddr; - VMMemInfoArgs *userVA; VMMemInfoArgs in; VMMemInfoArgs *out; - retval = HostIF_CopyFromUser(&uAddr, (void *)ioarg, sizeof uAddr); + retval = HostIF_CopyFromUser(&uAddr, ioarg, sizeof uAddr); if (retval) { break; } - userVA = VA64ToPtr(uAddr); - retval = HostIF_CopyFromUser(&in, userVA, sizeof in); + retval = HostIF_CopyFromUser(&in, uAddr, sizeof in); if (retval) { break; } @@ -1623,26 +1319,18 @@ LinuxDriver_Ioctl(struct file *filp, // IN: break; } - retval = HostIF_CopyToUser(userVA, out, + retval = HostIF_CopyToUser(uAddr, out, VM_GET_MEM_INFO_SIZE(out->numVMs)); HostIF_FreeKernelMem(out); break; } - case IOCTL_VMX86_PAE_ENABLED: - retval = Vmx86_PAEEnabled(); - break; - - case IOCTL_VMX86_VMX_ENABLED: - retval = Vmx86_VMXEnabled(); - break; - case IOCTL_VMX86_APIC_INIT: { VMAPICInfo info; Bool setVMPtr; Bool probe; - retval = HostIF_CopyFromUser(&info, (VMAPICInfo *)ioarg, sizeof info); + retval = HostIF_CopyFromUser(&info, ioarg, sizeof info); if (retval != 0) { break; } @@ -1667,8 +1355,7 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_SEND_IPI: { VCPUSet ipiTargets; - retval = HostIF_CopyFromUser(&ipiTargets, (VCPUSet *) ioarg, - sizeof ipiTargets); + retval = HostIF_CopyFromUser(&ipiTargets, ioarg, sizeof ipiTargets); if (retval == 0) { HostIF_IPI(vm, &ipiTargets); @@ -1689,8 +1376,7 @@ LinuxDriver_Ioctl(struct file *filp, // IN: ipiVectors.monitorIPIVector = monitorIPIVector; ipiVectors.hvIPIVector = hvIPIVector; - retval = HostIF_CopyToUser((void *)ioarg, &ipiVectors, - sizeof ipiVectors); + retval = HostIF_CopyToUser(ioarg, &ipiVectors, sizeof ipiVectors); break; } @@ -1700,17 +1386,15 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_GET_ALL_CPUID: { VA64 uAddr; - CPUIDQuery *userVA; CPUIDQuery in; CPUIDQuery *out; - retval = HostIF_CopyFromUser(&uAddr, (void *)ioarg, sizeof uAddr); + retval = HostIF_CopyFromUser(&uAddr, ioarg, sizeof uAddr); if (retval) { break; } - userVA = VA64ToPtr(uAddr); - retval = HostIF_CopyFromUser(&in, userVA, sizeof in); + retval = HostIF_CopyFromUser(&in, uAddr, sizeof in); if (retval) { break; } @@ -1740,26 +1424,25 @@ LinuxDriver_Ioctl(struct file *filp, // IN: break; } - retval = HostIF_CopyToUser((int8 *)userVA + sizeof *userVA, - &out->logicalCPUs[0], - out->numLogicalCPUs * sizeof out->logicalCPUs[0]); + retval = HostIF_CopyToUser(uAddr + sizeof in, + &out->logicalCPUs[0], + out->numLogicalCPUs * + sizeof out->logicalCPUs[0]); HostIF_FreeKernelMem(out); break; } case IOCTL_VMX86_GET_ALL_MSRS: { VA64 uAddr; - MSRQuery *userVA; MSRQuery in; MSRQuery *out; - retval = HostIF_CopyFromUser(&uAddr, (void *)ioarg, sizeof uAddr); + retval = HostIF_CopyFromUser(&uAddr, ioarg, sizeof uAddr); if (retval) { break; } - userVA = VA64ToPtr(uAddr); - retval = HostIF_CopyFromUser(&in, userVA, sizeof in); + retval = HostIF_CopyFromUser(&in, uAddr, sizeof in); if (retval) { break; } @@ -1789,9 +1472,10 @@ LinuxDriver_Ioctl(struct file *filp, // IN: break; } - retval = HostIF_CopyToUser((int8 *)userVA + sizeof *userVA, - &out->logicalCPUs[0], - out->numLogicalCPUs * sizeof out->logicalCPUs[0]); + retval = HostIF_CopyToUser(uAddr + sizeof in, + &out->logicalCPUs[0], + out->numLogicalCPUs * + sizeof out->logicalCPUs[0]); HostIF_FreeKernelMem(out); break; } @@ -1800,7 +1484,7 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_FREE_LOCKED_PAGES: { VMMPNList req; - retval = HostIF_CopyFromUser(&req, (void *)ioarg, sizeof req); + retval = HostIF_CopyFromUser(&req, ioarg, sizeof req); if (retval) { break; } @@ -1818,141 +1502,98 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_GET_NEXT_ANON_PAGE: { VMMPNNext req; - retval = HostIF_CopyFromUser(&req, (void *)ioarg, sizeof req); + retval = HostIF_CopyFromUser(&req, ioarg, sizeof req); if (retval) { req.outMPN = INVALID_MPN; } else { req.outMPN = Vmx86_GetNextAnonPage(vm, req.inMPN); } - retval = HostIF_CopyToUser((void *)ioarg, &req, sizeof req); + retval = HostIF_CopyToUser(ioarg, &req, sizeof req); break; } - case IOCTL_VMX86_GET_LOCKED_PAGES_LIST: { - VMMPNList req; - - retval = HostIF_CopyFromUser(&req, (void *)ioarg, sizeof req); - if (retval) { - break; - } - retval = Vmx86_GetLockedPageList(vm, req.mpnList, req.mpnCount); - break; - } - case IOCTL_VMX86_READ_PAGE: { + void *tempPage; VMMReadWritePage req; - retval = HostIF_CopyFromUser(&req, (void *)ioarg, sizeof req); + retval = HostIF_CopyFromUser(&req, ioarg, sizeof req); if (retval) { break; } - retval = HostIF_ReadPage(vm, req.mpn, req.uAddr, FALSE); + + tempPage = HostIF_AllocPage(); + if (tempPage == NULL) { + retval = -ENOMEM; + break; + } + + retval = HostIF_ReadPhysical(vm, MPN_2_MA(req.mpn), + PtrToVA64(tempPage), TRUE, PAGE_SIZE); + if (retval == 0) { + retval = HostIF_CopyToUser(req.uAddr, tempPage, PAGE_SIZE); + } + + HostIF_FreePage(tempPage); break; } case IOCTL_VMX86_WRITE_PAGE: { VMMReadWritePage req; - retval = HostIF_CopyFromUser(&req, (void *)ioarg, sizeof req); + retval = HostIF_CopyFromUser(&req, ioarg, sizeof req); if (retval) { break; } - retval = HostIF_WritePage(vm, req.mpn, req.uAddr, FALSE); + retval = HostIF_WritePhysical(vm, MPN_2_MA(req.mpn), req.uAddr, FALSE, + PAGE_SIZE); break; } - case IOCTL_VMX86_SET_POLL_TIMEOUT_PTR: { - vmLinux->pollTimeoutPtr = NULL; - HostIF_UnmapUserMem(vmLinux->pollTimeoutHandle); - if (ioarg != 0) { - vmLinux->pollTimeoutPtr = HostIF_MapUserMem((VA)ioarg, - sizeof *vmLinux->pollTimeoutPtr, - &vmLinux->pollTimeoutHandle); - - if (vmLinux->pollTimeoutPtr == NULL) { - retval = -EINVAL; - break; - } - } - break; - } - - case IOCTL_VMX86_GET_KERNEL_CLOCK_RATE: - retval = HZ; - break; - - case IOCTL_VMX86_FAST_SUSP_RES_SET_OTHER_FLAG: - retval = Vmx86_FastSuspResSetOtherFlag(vm, ioarg); - break; - - case IOCTL_VMX86_FAST_SUSP_RES_GET_MY_FLAG: - retval = Vmx86_FastSuspResGetMyFlag(vm, ioarg); - break; - case IOCTL_VMX86_GET_REFERENCE_CLOCK_HZ: { uint64 refClockHz = HostIF_UptimeFrequency(); - retval = HostIF_CopyToUser((void *)ioarg, &refClockHz, - sizeof refClockHz); + retval = HostIF_CopyToUser(ioarg, &refClockHz, sizeof refClockHz); break; } case IOCTL_VMX86_INIT_PSEUDO_TSC: { PTSCInitParams params; - retval = HostIF_CopyFromUser(¶ms, (void *)ioarg, sizeof params); + retval = HostIF_CopyFromUser(¶ms, ioarg, sizeof params); if (retval != 0) { break; } Vmx86_InitPseudoTSC(¶ms); - retval = HostIF_CopyToUser((void *)ioarg, ¶ms, sizeof params); + retval = HostIF_CopyToUser(ioarg, ¶ms, sizeof params); break; } case IOCTL_VMX86_CHECK_PSEUDO_TSC: { PTSCCheckParams params; - retval = HostIF_CopyFromUser(¶ms, (void *)ioarg, sizeof params); + retval = HostIF_CopyFromUser(¶ms, ioarg, sizeof params); if (retval != 0) { break; } params.usingRefClock = Vmx86_CheckPseudoTSC(¶ms.lastTSC, ¶ms.lastRC); - retval = HostIF_CopyToUser((void *)ioarg, ¶ms, sizeof params); + retval = HostIF_CopyToUser(ioarg, ¶ms, sizeof params); break; } case IOCTL_VMX86_GET_PSEUDO_TSC: { uint64 ptsc = Vmx86_GetPseudoTSC(); - retval = HostIF_CopyToUser((void *)ioarg, &ptsc, sizeof ptsc); + retval = HostIF_CopyToUser(ioarg, &ptsc, sizeof ptsc); break; } - case IOCTL_VMX86_SET_HOST_CLOCK_PRIORITY: - /* - * This affects the global fast clock priority, and it only - * takes effect when the fast clock rate transitions from zero - * to a non-zero value. - * - * This is used to allow VMs to optionally work around - * bug 218750 by disabling our default priority boost. If any - * VM chooses to apply this workaround, the effect is permanent - * until vmmon is reloaded! - */ - - HostIF_FastClockLock(3); - linuxState.fastClockPriority = MAX(-20, MIN(19, (int)ioarg)); - HostIF_FastClockUnlock(3); - retval = 0; - break; - case IOCTL_VMX86_SYNC_GET_TSCS: { uint64 delta; if (LinuxDriverSyncReadTSCs(&delta)) { - retval = HostIF_CopyToUser((void *)ioarg, &delta, sizeof delta); + retval = HostIF_CopyToUser(ioarg, &delta, sizeof delta); } else { retval = -EBUSY; } @@ -1961,10 +1602,10 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_SET_HOST_SWAP_SIZE: { uint64 swapSize; - retval = HostIF_CopyFromUser(&swapSize, (void *)ioarg, sizeof swapSize); + retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize); if (retval != 0) { Warning("Could not copy swap size from user, status %d\n", retval); - break; + break; } linuxState.swapSize = swapSize; break; @@ -1972,11 +1613,11 @@ LinuxDriver_Ioctl(struct file *filp, // IN: case IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS: { uint64 ctrs = Vmx86_GetUnavailablePerfCtrs(); - retval = HostIF_CopyToUser((void *)ioarg, &ctrs, sizeof ctrs); + retval = HostIF_CopyToUser(ioarg, &ctrs, sizeof ctrs); break; } - default: + default: Warning("Unknown ioctl %d\n", iocmd); retval = -EINVAL; } diff --git a/vmmon-only/linux/driver.h b/vmmon-only/linux/driver.h index 698afc9..650acc0 100644 --- a/vmmon-only/linux/driver.h +++ b/vmmon-only/linux/driver.h @@ -26,10 +26,10 @@ #include #include #include +#include #include #include "vmx86.h" -#include "compat_semaphore.h" #include "driver_vmcore.h" @@ -51,7 +51,7 @@ typedef struct VMMappedUserMem { struct VMDriver; /* 16 pages (64KB) looks as a good limit for one allocation */ -#define VMMON_MAX_LOWMEM_PAGES 16 +#define VMMON_MAX_LOWMEM_PAGES 16 typedef struct VMLinux { struct VMLinux *next; @@ -66,17 +66,6 @@ typedef struct VMLinux { struct semaphore lock4Gb; unsigned int size4Gb; struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES]; - - /* - * LinuxDriverPoll() support - */ - - wait_queue_head_t pollQueue; - volatile uint32 *pollTimeoutPtr; - VMMappedUserMem *pollTimeoutHandle; - VmTimeType pollTime; - struct VMLinux *pollForw; - struct VMLinux **pollBack; } VMLinux; @@ -95,19 +84,8 @@ typedef struct VMXLinuxState { char buf[LINUXLOG_BUFFER_SIZE]; VMLinux *head; - /* - * for LinuxDriverPoll() - */ - - struct timer_list pollTimer; - wait_queue_head_t pollQueue; - - struct VMLinux *pollList; - struct task_struct *fastClockThread; - struct file *fastClockFile; unsigned fastClockRate; - long fastClockPriority; uint64 swapSize; } VMXLinuxState; diff --git a/vmmon-only/linux/hostif.c b/vmmon-only/linux/hostif.c index 80b5787..78199da 100644 --- a/vmmon-only/linux/hostif.c +++ b/vmmon-only/linux/hostif.c @@ -29,7 +29,6 @@ #include "driver-config.h" /* Must come before vmware.h --hpreg */ -#include "compat_page.h" #include #include #include @@ -40,43 +39,21 @@ #include #include +#include #include #include -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25) -# include -#endif -#if defined(_ASM_EXTABLE) -# define VMW_ASM_EXTABLE(from, to) _ASM_EXTABLE(from, to) -#else - /* Compat version copied from asm.h of 2.6.25 kernel */ -# define VMW_ASM_FORM(x) " " #x " " -# define VMW_ASM_EX_SEC " .section __ex_table,\"a\"\n" -# ifdef CONFIG_X86_32 -# define VMW_ASM_SEL(a,b) VMW_ASM_FORM(a) -# else -# define VMW_ASM_SEL(a,b) VMW_ASM_FORM(b) -# endif -# define VMW_ASM_PTR VMW_ASM_SEL(.long, .quad) -# define VMW_ASM_ALIGN VMW_ASM_SEL(.balign 4, .balign 8) -# define VMW_ASM_EXTABLE(from,to) \ - VMW_ASM_EX_SEC \ - VMW_ASM_ALIGN "\n" \ - VMW_ASM_PTR #from "," #to "\n" \ - " .previous\n" -#endif - +#include #include +#include #include -#include #include #include #include +#include #include -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) #include // For linux/sched/signal.h without version check -#endif #include "vmware.h" #include "x86apic.h" @@ -96,28 +73,24 @@ #include "vcpuid.h" #include "pgtbl.h" -#include "vmmonInt.h" #include "versioned_atomic.h" -/* - * Determine if we can use high resolution timers. - */ - -#ifdef CONFIG_HIGH_RES_TIMERS -# include -# define VMMON_USE_HIGH_RES_TIMERS -# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28) -# define VMMON_USE_SCHEDULE_HRTIMEOUT -# else -# define VMMON_USE_COMPAT_SCHEDULE_HRTIMEOUT -static void HostIFWakeupClockThread(unsigned long data); -static DECLARE_TASKLET(timerTasklet, HostIFWakeupClockThread, 0); -# endif -# define close_rtc(filp, files) do {} while(0) -#else -# define close_rtc(filp, files) filp_close(filp, files) +#if !defined(CONFIG_HIGH_RES_TIMERS) +#error CONFIG_HIGH_RES_TIMERS required for acceptable performance #endif +/* + * Although this is not really related to kernel-compatibility, I put this + * helper macro here for now for a lack of better place --hpreg + * + * The exit(2) path does, in this order: + * . set current->files to NULL + * . close all fds, which potentially calls LinuxDriver_Close() + * + * fget() requires current->files != NULL, so we must explicitely check --hpreg + */ +#define vmware_fget(_fd) (current->files ? fget(_fd) : NULL) + #define UPTIME_FREQ CONST64(1000000) /* @@ -346,9 +319,6 @@ static Mutex globalMutex; */ static Mutex fastClockMutex; -/* This mutex protects linuxState.pollList. */ -static Mutex pollListMutex; - /* *---------------------------------------------------------------------- @@ -397,13 +367,8 @@ HostIF_WaitForThreads(VMDriver *vm, // UNUSED: Vcpuid currVcpu) // UNUSED: { -#ifdef VMMON_USE_SCHEDULE_HRTIMEOUT ktime_t timeout = ktime_set(0, CROSSCALL_SLEEP_US * 1000); schedule_hrtimeout(&timeout, HRTIMER_MODE_REL); -#else - /* Fallback to ms timer resolution is fine for older kernels. */ - schedule_timeout(msecs_to_jiffies(CROSSCALL_SLEEP_US / 1000) + 1); -#endif } @@ -438,7 +403,7 @@ HostIF_CancelWaitForThreads(VMDriver *vm, // IN: * HostIF_WakeUpYielders -- * * Wakeup vCPUs that are waiting for the current vCPU. - * + * * Results: * The requested vCPUs are nudged if they are sleeping due to * Vmx86_YieldToSet. @@ -506,7 +471,6 @@ HostIF_InitGlobalLock(void) { MutexInit(&globalMutex, "global"); MutexInit(&fastClockMutex, "fastClock"); - MutexInit(&pollListMutex, "pollList"); } @@ -521,7 +485,7 @@ HostIF_InitGlobalLock(void) * None * * Side effects: - * Should be a very low contention lock. + * Should be a very low contention lock. * The current thread is rescheduled if the lock is busy. * *----------------------------------------------------------------------------- @@ -564,7 +528,7 @@ HostIF_GlobalUnlock(int callerID) // IN * HostIF_GlobalLockIsHeld -- * * Determine if the global lock is held by the current thread. - * + * * Results: * TRUE if yes * FALSE if no @@ -594,7 +558,7 @@ HostIF_GlobalLockIsHeld(void) * None * * Side effects: - * Should be a very low contention lock. + * Should be a very low contention lock. * The current thread is rescheduled if the lock is busy. * *----------------------------------------------------------------------------- @@ -630,52 +594,6 @@ HostIF_FastClockUnlock(int callerID) // IN } -/* - *----------------------------------------------------------------------------- - * - * HostIF_PollListLock -- - * - * Grabs the linuxState.pollList lock. - * - * Results: - * None - * - * Side effects: - * The current thread is rescheduled if the lock is busy. - * - *----------------------------------------------------------------------------- - */ - -void -HostIF_PollListLock(int callerID) // IN -{ - MutexLock(&pollListMutex, callerID); -} - - -/* - *----------------------------------------------------------------------------- - * - * HostIF_PollListUnlock -- - * - * Releases the linuxState.pollList lock. - * - * Results: - * None - * - * Side effects: - * None - * - *----------------------------------------------------------------------------- - */ - -void -HostIF_PollListUnlock(int callerID) // IN -{ - MutexUnlock(&pollListMutex, callerID); -} - - /* *---------------------------------------------------------------------- * @@ -729,7 +647,7 @@ static int HostIFHostMemInit(VMDriver *vm) // IN: { VMHost *vmh = vm->vmhost; - + vmh->lockedPages = PhysTrack_Alloc(vm); if (!vmh->lockedPages) { return -1; @@ -824,8 +742,8 @@ HostIF_AllocMachinePage(void) * * HostIF_FreeMachinePage -- * - * Free an anonymous machine page allocated by - * HostIF_AllocMachinePage(). This page is not tracked in any + * Free an anonymous machine page allocated by + * HostIF_AllocMachinePage(). This page is not tracked in any * phystracker. * * Results: @@ -854,9 +772,8 @@ HostIF_FreeMachinePage(MPN mpn) // IN: * Alloc non-swappable memory. * * Results: - * negative value on complete failure - * non-negative value on partial/full completion, number of MPNs - * allocated & filled in pmpn returned. + * negative value on complete failure non-negative value on partial/full + * completion, number of allocated MPNs returned. * * Side effects: * Pages allocated. @@ -865,13 +782,11 @@ HostIF_FreeMachinePage(MPN mpn) // IN: */ int -HostIF_AllocLockedPages(VMDriver *vm, // IN: VM instance pointer - VA64 addr, // OUT: pointer to user or kernel buffer for MPNs - unsigned numPages, // IN: number of pages to allocate - Bool kernelMPNBuffer)// IN: is the MPN buffer in kernel or user address space? +HostIF_AllocLockedPages(VMDriver *vm, // IN: VM instance pointer + VA64 addr, // OUT: buffer address + unsigned numPages, // IN: number of pages to allocate + Bool kernelMPNBuffer) // IN: kernel vs user space { - MPN *pmpn = VA64ToPtr(addr); - VMHost *vmh = vm->vmhost; unsigned int cnt; int err = 0; @@ -886,19 +801,20 @@ HostIF_AllocLockedPages(VMDriver *vm, // IN: VM instance pointer pg = alloc_page(GFP_HIGHUSER); if (!pg) { err = -ENOMEM; - break; + break; } mpn = (MPN)page_to_pfn(pg); if (kernelMPNBuffer) { + MPN *pmpn = VA64ToPtr(addr); *pmpn = mpn; - } else if (HostIF_CopyToUser(pmpn, &mpn, sizeof *pmpn) != 0) { - __free_page(pg); - err = -EFAULT; - break; + } else if (HostIF_CopyToUser(addr, &mpn, sizeof mpn) != 0) { + __free_page(pg); + err = -EFAULT; + break; } - pmpn++; + addr += sizeof mpn; if (PhysTrack_Test(vmh->AWEPages, mpn)) { - Warning("%s: duplicate MPN %016" FMT64 "x\n", __func__, mpn); + Warning("%s: duplicate MPN %016" FMT64 "x\n", __func__, mpn); } PhysTrack_Add(vmh->AWEPages, mpn); } @@ -925,10 +841,10 @@ HostIF_AllocLockedPages(VMDriver *vm, // IN: VM instance pointer */ int -HostIF_FreeLockedPages(VMDriver *vm, // IN: VM instance pointer - VA64 addr, // IN: user or kernel array of MPNs - unsigned numPages, // IN: number of pages to free - Bool kernelMPNBuffer) // IN: is the MPN buffer in kernel or user address space? +HostIF_FreeLockedPages(VMDriver *vm, // IN: VM instance pointer + VA64 addr, // IN: array of MPNs + unsigned numPages, // IN: number of pages to free + Bool kernelMPNBuffer) // IN: kernel vs user address { const int MPN_BATCH = 64; MPN const *pmpn = VA64ToPtr(addr); @@ -953,7 +869,7 @@ HostIF_FreeLockedPages(VMDriver *vm, // IN: VM instance pointer return -EINVAL; } - if (HostIF_CopyFromUser(mpns, pmpn, numPages * sizeof *pmpn)) { + if (HostIF_CopyFromUser(mpns, addr, numPages * sizeof *pmpn)) { printk(KERN_DEBUG "Cannot read from process address space at %p\n", pmpn); HostIF_FreeKernelMem(mpns); @@ -1108,38 +1024,6 @@ HostIF_LookupUserMPN(VMDriver *vm, // IN: VMDriver } -/* - *---------------------------------------------------------------------- - * - * HostIF_InitFP -- - * - * masks IRQ13 if not previously the case. - * - * Results: - * prevents INTR #0x2d (IRQ 13) from being generated -- - * assume that Int16 works for interrupt reporting - * - * - * Side effects: - * PIC - * - *---------------------------------------------------------------------- - */ - -void -HostIF_InitFP(VMDriver *vm) // IN: -{ - int mask = (1 << (0xD - 0x8)); - - uint8 val = inb(0xA1); - - if (!(val & mask)) { - val = val | mask; - outb(val, 0xA1); - } -} - - /* *----------------------------------------------------------------------------- * @@ -1149,7 +1033,7 @@ HostIF_InitFP(VMDriver *vm) // IN: * If ppages is NULL, pages are only marked as dirty. * * Results: - * Zero on success, non-zero on failure. + * Zero on success, non-zero on failure. * * Side effects: * None @@ -1164,16 +1048,7 @@ HostIFGetUserPages(void *uvAddr, // IN { int retval; - down_read(¤t->mm->mmap_sem); -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) - retval = get_user_pages((unsigned long)uvAddr, numPages, 0, ppages, NULL); -#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) - retval = get_user_pages((unsigned long)uvAddr, numPages, 0, 0, ppages, NULL); -#else - retval = get_user_pages(current, current->mm, (unsigned long)uvAddr, - numPages, 0, 0, ppages, NULL); -#endif - up_read(¤t->mm->mmap_sem); + retval = get_user_pages_fast((unsigned long)uvAddr, numPages, 0, ppages); return retval != numPages; } @@ -1184,11 +1059,11 @@ HostIFGetUserPages(void *uvAddr, // IN * * HostIF_IsLockedByMPN -- * - * Checks if mpn was locked using allowMultipleMPNsPerVA. + * Checks if mpn was locked using allowMultipleMPNsPerVA. * * Results: * TRUE if mpn is present in the physTracker. - * + * * * Side effects: * None. @@ -1310,7 +1185,7 @@ HostIF_UnlockPage(VMDriver *vm, // IN: vpn = VA_2_VPN((VA)addr); e = MemTrack_LookupVPN(vm->memtracker, vpn); - + if (e == NULL) { return PAGE_UNLOCK_NOT_TRACKED; } @@ -1356,7 +1231,7 @@ HostIF_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver { void *va = VA64ToPtr(uAddr); MemTrackEntry *e; - + /* * Verify for debugging that VA and MPN make sense. * PgtblVa2MPN() can fail under high memory pressure. @@ -1374,7 +1249,7 @@ HostIF_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver } /* - * Verify that this MPN was locked with + * Verify that this MPN was locked with * HostIF_LockPage(allowMultipleMPNsPerVA = TRUE). * That means that this MPN should not be in the MemTracker. */ @@ -1387,7 +1262,7 @@ HostIF_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver return PAGE_UNLOCK_MISMATCHED_TYPE; } } -#endif +#endif HOST_UNLOCK_PFN_BYMPN(vm, mpn); @@ -1395,7 +1270,7 @@ HostIF_UnlockPageByMPN(VMDriver *vm, // IN: VMDriver } -static void +static void UnlockEntry(void *clientData, // IN: MemTrackEntry *entryPtr) // IN: { @@ -1441,10 +1316,10 @@ HostIF_FreeAllResources(VMDriver *vm) // IN } vm->vmhost->crosspagePagesCount = 0; if (vm->vmhost->hostAPICIsMapped) { - ASSERT(vm->hostAPIC.base != NULL); - iounmap((void*)vm->hostAPIC.base); - vm->hostAPIC.base = NULL; - vm->vmhost->hostAPICIsMapped = FALSE; + ASSERT(vm->hostAPIC.base != NULL); + iounmap((void*)vm->hostAPIC.base); + vm->hostAPIC.base = NULL; + vm->vmhost->hostAPICIsMapped = FALSE; } HostIF_FreeKernelMem(vm->vmhost); vm->vmhost = NULL; @@ -1458,11 +1333,11 @@ HostIF_FreeAllResources(VMDriver *vm) // IN * * HostIF_AllocKernelMem * - * Allocate some kernel memory for the driver. + * Allocate some kernel memory for the driver. * * Results: - * The address allocated or NULL on error. - * + * The address allocated or NULL on error. + * * * Side effects: * memory is malloced @@ -1474,8 +1349,8 @@ HostIF_AllocKernelMem(size_t size, // IN: int wired) // IN: { void * ptr = kmalloc(size, GFP_KERNEL); - - if (ptr == NULL) { + + if (ptr == NULL) { Warning("%s failed (size=%p)\n", __func__, (void*)size); } @@ -1503,7 +1378,7 @@ void * HostIF_AllocPage(void) { VA kvAddr; - + kvAddr = __get_free_page(GFP_KERNEL); if (kvAddr == 0) { Warning("%s: __get_free_page() failed\n", __func__); @@ -1518,7 +1393,7 @@ HostIF_AllocPage(void) * * HostIF_FreeKernelMem * - * Free kernel memory allocated for the driver. + * Free kernel memory allocated for the driver. * * Results: * None. @@ -1548,6 +1423,56 @@ HostIF_FreePage(void *ptr) // IN: } +/* + *---------------------------------------------------------------------- + * + * HostIF_MapPage -- + * + * Maps the specified MPN into the host kernel address space. + * returns the VPN of the mapping. + * + * Results: + * The VPN in the kernel address space of the new mapping, or 0 if + * the mapping failed. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +VPN +HostIF_MapPage(MPN mpn) // IN: +{ + struct page *p = pfn_to_page(mpn); + void *mappedAddr = vmap(&p, 1, VM_MAP, PAGE_KERNEL); + return mappedAddr == NULL ? 0 : VA_2_VPN((VA)mappedAddr); +} + + +/* + *---------------------------------------------------------------------- + * + * HostIF_UnmapPage -- + * + * Unmaps the specified VPN from the host kernel address space. + * + * Results: + * None. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +void +HostIF_UnmapPage(VPN vpn) // IN: +{ + vunmap((void *)VPN_2_VA(vpn)); +} + + /* *---------------------------------------------------------------------- * @@ -1557,7 +1482,7 @@ HostIF_FreePage(void *ptr) // IN: * from the kernel without causing the host to die or to be really upset. * * Results: - * The maximum number of pages that can be locked. + * The maximum number of pages that can be locked. * * Side effects: * none @@ -1567,7 +1492,7 @@ HostIF_FreePage(void *ptr) // IN: unsigned int HostIF_EstimateLockedPageLimit(const VMDriver* vm, // IN - unsigned int currentlyLockedPages) // IN + unsigned int currentlyLockedPages) // IN { /* * This variable is available and exported to modules, @@ -1578,16 +1503,13 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm, // IN unsigned int totalPhysicalPages = totalram_pages; -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28) - return MemDefaults_CalcMaxLockedPages(totalPhysicalPages); -#else /* * Use the memory information linux exports as of late for a more * precise estimate of locked memory. All kernel page-related structures * (slab, pagetable) are as good as locked. Unevictable includes things - * that are explicitly marked as such (like mlock()). Huge pages are - * also as good as locked, since we don't use them. Lastly, without - * available swap, anonymous pages become locked in memory as well. + * that are explicitly marked as such (like mlock()). Huge pages are + * also as good as locked, since we don't use them. Lastly, without + * available swap, anonymous pages become locked in memory as well. */ unsigned int forHost; @@ -1607,7 +1529,7 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm, // IN unsigned int swapPages = BYTES_2_PAGES(linuxState.swapSize); if (anonPages > swapPages) { - lockedPages += anonPages - swapPages; + lockedPages += anonPages - swapPages; } forHost = lockedPages + LOCKED_PAGE_SLACK; if (forHost > totalPhysicalPages) { @@ -1615,7 +1537,6 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm, // IN } return totalPhysicalPages - forHost; -#endif } @@ -1644,7 +1565,7 @@ HostIF_Wait(unsigned int timeoutMs) * Waits for pages to be available for allocation or locking. * * Results: - * New pages are likely to be available for allocation or locking. + * New pages are likely to be available for allocation or locking. * * Side effects: * none @@ -1652,7 +1573,7 @@ HostIF_Wait(unsigned int timeoutMs) *---------------------------------------------------------------------- */ -void +void HostIF_WaitForFreePages(unsigned int timeoutMs) // IN: { static unsigned count; @@ -1677,20 +1598,20 @@ HostIF_WaitForFreePages(unsigned int timeoutMs) // IN: * timeofday to have small drift (due to NTP rate correction, etc). * We handle this by rebasing the jiffies based monotonic clock * every second (see HostIFUptimeResyncMono). - * + * * Results: * The uptime, in units of UPTIME_FREQ. Also returns the jiffies * value that was used in the monotonic time calculation. * * Side effects: - * May reset the uptime base in the case gettimeofday warp was + * May reset the uptime base in the case gettimeofday warp was * detected. * *---------------------------------------------------------------------- */ static uint64 -HostIFReadUptimeWork(unsigned long *j) // OUT: current jiffies +HostIFReadUptimeWork(unsigned long *j) // OUT: current jiffies { struct timeval tv; uint64 monotime, uptime, upBase, monoBase; @@ -1712,14 +1633,14 @@ HostIFReadUptimeWork(unsigned long *j) // OUT: current jiffies do_gettimeofday(&tv); upBase = Atomic_Read64(&uptimeState.uptimeBase); - + monotime = (uint64)(jifs - jifBase) * (UPTIME_FREQ / HZ); monotime += monoBase; uptime = tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ; uptime += upBase; - - /* + + /* * Use the jiffies based monotonic time to sanity check gettimeofday. * If they differ by more than one second, assume the time of day has * been warped, and use the jiffies time to undo (most of) the warp. @@ -1731,7 +1652,7 @@ HostIFReadUptimeWork(unsigned long *j) // OUT: current jiffies uint64 newUpBase = monotime - (uptime - upBase); attempts++; - if (!Atomic_CMPXCHG64(&uptimeState.uptimeBase, &upBase, &newUpBase) && + if (!Atomic_CMPXCHG64(&uptimeState.uptimeBase, upBase, newUpBase) && attempts < 5) { /* Another thread updated uptimeBase. Recalculate uptime. */ goto retry; @@ -1772,7 +1693,7 @@ HostIFUptimeResyncMono(unsigned long data) // IN: ignored unsigned long jifs; uintptr_t flags; - /* + /* * Read the uptime and the corresponding jiffies value. This will * also correct the uptime (which is based on time of day) if needed * before we rebase monotonic time (which is based on jiffies). @@ -1780,7 +1701,7 @@ HostIFUptimeResyncMono(unsigned long data) // IN: ignored uint64 uptime = HostIFReadUptimeWork(&jifs); - /* + /* * Every second, recalculate monoBase and jiffiesBase to squash small * drift between gettimeofday and jiffies. Also, this prevents * (jiffies - jiffiesBase) wrap on 32-bits. @@ -1825,8 +1746,8 @@ HostIF_InitUptime(void) uptimeState.jiffiesBase = jiffies; do_gettimeofday(&tv); - Atomic_Write64(&uptimeState.uptimeBase, - -(tv.tv_usec * (UPTIME_FREQ / 1000000) + + Atomic_Write64(&uptimeState.uptimeBase, + -(tv.tv_usec * (UPTIME_FREQ / 1000000) + tv.tv_sec * UPTIME_FREQ)); init_timer(&uptimeState.timer); @@ -1927,11 +1848,11 @@ HostIF_UptimeFrequency(void) */ int -HostIF_CopyFromUser(void *dst, // OUT - const void *src, // IN - unsigned int len) // IN +HostIF_CopyFromUser(void *dst, // OUT + VA64 src, // IN + size_t len) // IN { - return copy_from_user(dst, src, len) ? -EFAULT : 0; + return copy_from_user(dst, VA64ToPtr(src), len) ? -EFAULT : 0; } @@ -1954,12 +1875,12 @@ HostIF_CopyFromUser(void *dst, // OUT *----------------------------------------------------------------------------- */ -int -HostIF_CopyToUser(void *dst, // OUT +int +HostIF_CopyToUser(VA64 dst, // OUT const void *src, // IN - unsigned int len) // IN + size_t len) // IN { - return copy_to_user(dst, src, len) ? -EFAULT : 0; + return copy_to_user(VA64ToPtr(dst), src, len) ? -EFAULT : 0; } @@ -1967,15 +1888,15 @@ HostIF_CopyToUser(void *dst, // OUT *----------------------------------------------------------------------------- * * HostIF_MapCrossPage -- - * - * Obtain kernel pointer to crosspage. * - * We must return a VA that is obtained through a kernel mapping, so that + * Obtain kernel pointer to crosspage. + * + * We must return a VA that is obtained through a kernel mapping, so that * the mapping never goes away (see bug 29753). * - * However, the LA corresponding to that VA must not overlap with the - * monitor (see bug 32922). The userland code ensures that by only - * allocating cross pages from low memory. For those pages, the kernel + * However, the LA corresponding to that VA must not overlap with the + * monitor (see bug 32922). The userland code ensures that by only + * allocating cross pages from low memory. For those pages, the kernel * uses a permanent mapping, instead of a temporary one with a high LA. * * Results: @@ -2020,18 +1941,15 @@ HostIF_MapCrossPage(VMDriver *vm, // IN /* *----------------------------------------------------------------------------- * - * HostIF_AllocCrossGDT -- + * HostIF_AllocKernelPages -- * - * Allocate the per-vmmon cross GDT page set. - * - * See bora/doc/worldswitch-pages.txt for the requirements on the cross - * GDT page set addresses. + * Allocates and maps a set of locked pages. * * Results: - * On success: Host kernel virtual address of the first cross GDT page. - * Use HostIF_FreeCrossGDT() with the same value to free. - * The 'crossGDTMPNs' array is filled with the MPNs of all the - * cross GDT pages. + * On success: Host kernel virtual address of the first page. + * Use HostIF_FreeKernelPages() with the same value to free. + * The 'mpns' array is filled with the MPNs of the + * allocated pages, in sequence. * On failure: NULL. * * Side effects: @@ -2041,64 +1959,40 @@ HostIF_MapCrossPage(VMDriver *vm, // IN */ void * -HostIF_AllocCrossGDT(uint32 numPages, // IN: Number of pages - MPN maxValidFirst, // IN: Highest valid MPN of first page - MPN *crossGDTMPNs) // OUT: Array of MPNs +HostIF_AllocKernelPages(unsigned numPages, // IN: Number of pages + MPN *mpns) // OUT: Array of MPNs { MPN startMPN; struct page *pages; - uint32 i; - void *crossGDT; + unsigned i; + void *ptr; - /* - * In practice, allocating a low page (MPN <= 0x100000 - 1) is equivalent to - * allocating a page with MPN <= 0xFEC00 - 1: - * - * o PC architecture guarantees that there is no RAM in top 16MB of 4GB - * range. - * - * o 0xFEC00000 is IOAPIC base. There could be RAM immediately below, - * but not above. - * - * How do we allocate a low page? We can safely use GFP_DMA32 when - * available. On 64bit kernels before GFP_DMA32 was introduced we - * fall back to DMA zone (which is not quite necessary for boxes - * with less than ~3GB of memory). On 32bit kernels we are using - * normal zone - which is usually 1GB, and at most 4GB (for 4GB/4GB - * kernels). And for 4GB/4GB kernels same restriction as for 64bit - * kernels applies - there is no RAM in top 16MB immediately below - * 4GB so alloc_pages() cannot return such page. - */ - - ASSERT(0xFEC00 - 1 <= maxValidFirst); for (i = 0; (1 << i) < numPages; i++) { } -#ifdef GFP_DMA32 - pages = alloc_pages(GFP_KERNEL | GFP_DMA32, i); -#else - pages = alloc_pages(GFP_KERNEL | GFP_DMA, i); -#endif - crossGDT = NULL; + /* Allocates physically contiguous pages. */ + pages = alloc_pages(GFP_KERNEL, i); + if (pages == NULL) { - Warning("%s: unable to alloc crossGDT (%u)\n", __func__, i); - } else { - startMPN = page_to_pfn(pages); - for (i = 0; i < numPages; i++) { - crossGDTMPNs[i] = startMPN + i; - } - crossGDT = (void *)page_address(pages); + return NULL; } - return crossGDT; + startMPN = page_to_pfn(pages); + + for (i = 0; i < numPages; i++) { + mpns[i] = startMPN + i; + } + ptr = (void *)page_address(pages); + ASSERT(!(PtrToVA64(ptr) & (PAGE_SIZE - 1))); /* Page-aligned */ + + return ptr; } /* *----------------------------------------------------------------------------- * - * HostIF_FreeCrossGDT -- + * HostIF_FreeKernelPages -- * - * Free the per-vmmon cross GDT page set allocated with - * HostIF_AllocCrossGDT(). + * Frees a set of pages allocated with HostIF_AllocKernelPages(). * * Results: * None @@ -2110,13 +2004,13 @@ HostIF_AllocCrossGDT(uint32 numPages, // IN: Number of pages */ void -HostIF_FreeCrossGDT(uint32 numPages, // IN: Number of pages - void *crossGDT) // IN: Kernel VA of first cross GDT page +HostIF_FreeKernelPages(unsigned numPages, // IN: Number of pages + void *ptr) // IN: Kernel VA of first page { - uint32 i; + unsigned i; for (i = 0; (1 << i) < numPages; i++) { } - free_pages((VA)crossGDT, i); + free_pages((VA)ptr, i); } @@ -2163,7 +2057,7 @@ HostIF_VMLock(VMDriver *vm, // IN * None * * Side effects: - * Can wake up the thread blocked on this lock. + * Can wake up the thread blocked on this lock. * *----------------------------------------------------------------------------- */ @@ -2186,7 +2080,7 @@ HostIF_VMUnlock(VMDriver *vm, // IN * HostIF_VMLockIsHeld -- * * Determine if the per-VM lock is held by the current thread. - * + * * Results: * TRUE if yes * FALSE if no @@ -2242,18 +2136,18 @@ HostIF_VMLockIsHeld(VMDriver *vm) // IN * *---------------------------------------------------------------------- */ - + static Bool isVAReadable(VA r) // IN: { mm_segment_t old_fs; uint32 dummy; int ret; - + old_fs = get_fs(); set_fs(get_ds()); r = APICR_TO_ADDR(r, APICR_VERSION); - ret = HostIF_CopyFromUser(&dummy, (void*)r, sizeof(dummy)); + ret = HostIF_CopyFromUser(&dummy, r, sizeof dummy); set_fs(old_fs); return ret == 0; @@ -2279,7 +2173,7 @@ isVAReadable(VA r) // IN: static void SetVMAPICAddr(VMDriver *vm, // IN/OUT: driver state - MA ma) // IN: host APIC's ma + MA ma) // IN: host APIC's ma { volatile void *hostapic; @@ -2287,11 +2181,11 @@ SetVMAPICAddr(VMDriver *vm, // IN/OUT: driver state hostapic = (volatile void *) ioremap_nocache(ma, PAGE_SIZE); if (hostapic) { if ((APIC_VERSIONREG(hostapic) & 0xF0) == 0x10) { - vm->hostAPIC.base = (volatile uint32 (*)[4]) hostapic; - ASSERT(vm->vmhost != NULL); - vm->vmhost->hostAPICIsMapped = TRUE; + vm->hostAPIC.base = (volatile uint32 (*)[4]) hostapic; + ASSERT(vm->vmhost != NULL); + vm->vmhost->hostAPICIsMapped = TRUE; } else { - iounmap((void*)hostapic); + iounmap((void*)hostapic); } } } @@ -2321,10 +2215,10 @@ SetVMAPICAddr(VMDriver *vm, // IN/OUT: driver state static Bool ProbeAPIC(VMDriver *vm, // IN/OUT: driver state - Bool setVMPtr) // IN: set a pointer to the APIC's virtual address + Bool setVMPtr) // IN: set a pointer to the APIC's virtual address { MA ma = APIC_GetMA(); - + if (ma == (MA)-1) { return FALSE; } @@ -2425,7 +2319,7 @@ HostIF_APICInit(VMDriver *vm, // IN: * Perform the semaphore wait (P) operation, possibly blocking. * * Result: - * 1 (which equals MX_WAITNORMAL) if success, + * 1 (which equals MX_WAITNORMAL) if success, * negated error code otherwise. * * Side-effects: @@ -2434,7 +2328,7 @@ HostIF_APICInit(VMDriver *vm, // IN: *----------------------------------------------------------------------------- */ -int +int HostIF_SemaphoreWait(VMDriver *vm, // IN: Vcpuid vcpuid, // IN: uint64 *args) // IN: @@ -2457,7 +2351,7 @@ HostIF_SemaphoreWait(VMDriver *vm, // IN: { struct poll_wqueues table; unsigned int mask; - + poll_initwait(&table); current->state = TASK_INTERRUPTIBLE; mask = file->f_op->poll(file, &table.pt); @@ -2532,7 +2426,7 @@ HostIF_SemaphoreWait(VMDriver *vm, // IN: *----------------------------------------------------------------------------- */ -void +void HostIF_SemaphoreForceWakeup(VMDriver *vm, // IN: const VCPUSet *vcs) // IN: { @@ -2620,34 +2514,6 @@ HostIF_SemaphoreSignal(uint64 *args) // IN: return res; } -#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)) || !defined(CONFIG_SMP)) -# define VMMON_USE_CALL_FUNC -#endif - -#if defined(VMMON_USE_CALL_FUNC) -/* - *---------------------------------------------------------------------- - * - * LinuxDriverIPIHandler -- - * - * Null IPI handler - for monitor to notice AIO completion - * - *---------------------------------------------------------------------- - */ -void -LinuxDriverIPIHandler(void *info) -{ - return; -} - -#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 17) -#define VMMON_CALL_FUNC_SYNC 0 // async; we've not seen any problems -#else -#define VMMON_CALL_FUNC_SYNC 1 // sync; insure no problems from old releases -#endif - -#endif - /* *---------------------------------------------------------------------- @@ -2677,17 +2543,8 @@ HostIF_IPI(VMDriver *vm, // IN: uint32 targetHostCpu = vm->currentHostCpu[v]; if (targetHostCpu != INVALID_PCPU) { ASSERT(targetHostCpu < MAX_PCPUS); -#if defined(VMMON_USE_CALL_FUNC) - /* older kernels IPI broadcast; use async when possible */ - (void) compat_smp_call_function(LinuxDriverIPIHandler, - NULL, VMMON_CALL_FUNC_SYNC); - mode = IPI_BROADCAST; - break; -#else - /* Newer kernels have (async) IPI targetting */ arch_send_call_function_single_ipi(targetHostCpu); - mode = IPI_UNICAST; -#endif + mode = IPI_UNICAST; } } ROF_EACH_VCPU_IN_SET(); @@ -2812,7 +2669,7 @@ HostIF_CallOnEachCPU(void (*func)(void*), // IN: function to call { preempt_disable(); (*func)(data); - (void)compat_smp_call_function(*func, data, 1); + (void)smp_call_function(*func, data, 1); preempt_enable(); } @@ -2865,6 +2722,13 @@ HostIFCheckTrackedMPN(VMDriver *vm, // IN: The VM instance return TRUE; } } + + if (vm->ptpTracker) { + if (MemTrack_LookupMPN(vm->ptpTracker, mpn) != NULL) { + HostIF_VMUnlock(vm, 32); + return TRUE; + } + } HostIF_VMUnlock(vm, 32); if (vmx86_debug) { @@ -2881,11 +2745,11 @@ HostIFCheckTrackedMPN(VMDriver *vm, // IN: The VM instance /* *---------------------------------------------------------------------- * - * HostIF_ReadPage -- + * HostIF_ReadPhysical -- * - * Reads one page of data from a machine page and returns it in the - * specified kernel or user buffer. The machine page must be owned by - * the specified VM. + * Reads bytes from a machine address and stores it in a kernel or + * user buffer. The address and number of bytes must describe + * memory on a single machine page owned by the specified VM. * * Results: * 0 on success @@ -2898,33 +2762,37 @@ HostIFCheckTrackedMPN(VMDriver *vm, // IN: The VM instance */ int -HostIF_ReadPage(VMDriver *vm, // IN: The VM instance - MPN mpn, // MPN of the page - VA64 addr, // buffer for data - Bool kernelBuffer) // is the buffer in kernel space? +HostIF_ReadPhysical(VMDriver *vm, // IN: The VM instance + MA ma, // MA to be read + VA64 addr, // dst for read data + Bool kernelBuffer, // is the buffer in kernel space? + size_t len) // number of bytes to read { - void *buf = VA64ToPtr(addr); int ret = 0; - const void* ptr; + void* ptr; struct page* page; + MPN mpn = MA_2_MPN(ma); + uint32 offset = ma & (PAGE_SIZE - 1); if (mpn == INVALID_MPN) { return -EFAULT; } - if (HostIFCheckTrackedMPN(vm, mpn) == FALSE) { + if (MA_2_MPN(ma + len - 1) != mpn) { + return -EFAULT; + } + if (!HostIFCheckTrackedMPN(vm, mpn)) { return -EFAULT; } - page = pfn_to_page(mpn); ptr = kmap(page); if (ptr == NULL) { return -ENOMEM; } - + if (kernelBuffer) { - memcpy(buf, ptr, PAGE_SIZE); + memcpy(VA64ToPtr(addr), ptr + offset, len); } else { - ret = HostIF_CopyToUser(buf, ptr, PAGE_SIZE); + ret = HostIF_CopyToUser(addr, ptr + offset, len); } kunmap(page); @@ -2935,10 +2803,11 @@ HostIF_ReadPage(VMDriver *vm, // IN: The VM instance /* *---------------------------------------------------------------------- * - * HostIF_WritePage -- + * HostIF_WritePhysical -- * - * Writes one page of data from a kernel or user buffer onto the specified - * machine page. The machine page must be owned by the specified VM. + * Writes bytes from a kernel or user mode buffer to a machine + * address. The address and number of bytes must describe memory on + * a single machine page owned by the specified VM. * * Results: * 0 on success @@ -2951,19 +2820,23 @@ HostIF_ReadPage(VMDriver *vm, // IN: The VM instance */ int -HostIFWritePageWork(MPN mpn, // MPN of the page - VA64 addr, // data to write to the page - Bool kernelBuffer) // is the buffer in kernel space? +HostIFWritePhysicalWork(MA ma, // MA to be written to + VA64 addr, // src data to write + Bool kernelBuffer, // is the buffer in kernel space? + size_t len) // number of bytes to write { - void const *buf = VA64ToPtr(addr); int ret = 0; void* ptr; struct page* page; + MPN mpn = MA_2_MPN(ma); + uint32 offset = ma & (PAGE_SIZE - 1); if (mpn == INVALID_MPN) { return -EFAULT; } - + if (MA_2_MPN(ma + len - 1) != mpn) { + return -EFAULT; + } page = pfn_to_page(mpn); ptr = kmap(page); if (ptr == NULL) { @@ -2971,9 +2844,9 @@ HostIFWritePageWork(MPN mpn, // MPN of the page } if (kernelBuffer) { - memcpy(ptr, buf, PAGE_SIZE); + memcpy(ptr + offset, VA64ToPtr(addr), len); } else { - ret = HostIF_CopyFromUser(ptr, buf, PAGE_SIZE); + ret = HostIF_CopyFromUser(ptr + offset, addr, len); } kunmap(page); @@ -2981,15 +2854,16 @@ HostIFWritePageWork(MPN mpn, // MPN of the page } int -HostIF_WritePage(VMDriver *vm, // IN: The VM instance - MPN mpn, // MPN of the page - VA64 addr, // data to write to the page - Bool kernelBuffer) // is the buffer in kernel space? +HostIF_WritePhysical(VMDriver *vm, // IN: The VM instance + MA ma, // MA to be written to + VA64 addr, // src data to write + Bool kernelBuffer, // is the buffer in kernel space? + size_t len) // number of bytes to write { - if (HostIFCheckTrackedMPN(vm, mpn) == FALSE) { + if (!HostIFCheckTrackedMPN(vm, MA_2_MPN(ma))) { return -EFAULT; } - return HostIFWritePageWork(mpn, addr, kernelBuffer); + return HostIFWritePhysicalWork(ma, addr, kernelBuffer, len); } @@ -3016,55 +2890,7 @@ int HostIF_WriteMachinePage(MPN mpn, // IN: MPN of the page VA64 addr) // IN: data to write to the page { - return HostIFWritePageWork(mpn, addr, TRUE); -} - - -/* - *---------------------------------------------------------------------- - * - * HostIF_GetLockedPageList -- - * - * puts MPNs of pages that were allocated by HostIF_AllocLockedPages() - * into user mode buffer. - * - * Results: - * non-negative number of the MPNs in the buffer on success. - * negative error code on error (-EFAULT) - * - * Side effects: - * none - * - *---------------------------------------------------------------------- - */ - -int -HostIF_GetLockedPageList(VMDriver* vm, // IN: VM instance pointer - VA64 uAddr, // OUT: user mode buffer for MPNs - unsigned int numPages) // IN: size of the buffer in MPNs -{ - MPN *mpns = VA64ToPtr(uAddr); - MPN mpn; - unsigned count; - - struct PhysTracker* AWEPages; - - if (!vm->vmhost || !vm->vmhost->AWEPages) { - return 0; - } - AWEPages = vm->vmhost->AWEPages; - - for (mpn = 0, count = 0; - (count < numPages) && - (INVALID_MPN != (mpn = PhysTrack_GetNext(AWEPages, mpn))); - count++) { - - if (HostIF_CopyToUser(&mpns[count], &mpn, sizeof *mpns) != 0) { - return -EFAULT; - } - } - - return count; + return HostIFWritePhysicalWork(MPN_2_MA(mpn), addr, TRUE, PAGE_SIZE); } @@ -3106,7 +2932,7 @@ HostIF_GetNextAnonPage(VMDriver *vm, MPN inMPN) * Side effects: * None. * - *---------------------------------------------------------------------- + *---------------------------------------------------------------------- */ uint32 @@ -3116,136 +2942,12 @@ HostIF_GetCurrentPCPU(void) } -#ifdef VMMON_USE_COMPAT_SCHEDULE_HRTIMEOUT -/* - *---------------------------------------------------------------------- - * - * HostIFWakeupClockThread -- - * - * Wake up the fast clock thread. Can't do this from the timer - * callback, because it holds locks that the scheduling code - * might take. - * - * Results: - * None. - * - * Side effects: - * None. - * - *---------------------------------------------------------------------- - */ - -static void -HostIFWakeupClockThread(unsigned long data) //IN: -{ - wake_up_process(linuxState.fastClockThread); -} - - -/* - *---------------------------------------------------------------------- - * - * HostIFTimerCallback -- - * - * Schedule a tasklet to wake up the fast clock thread. - * - * Results: - * Tell the kernel not to restart the timer. - * - * Side effects: - * None. - * - *---------------------------------------------------------------------- - */ - -static enum hrtimer_restart -HostIFTimerCallback(struct hrtimer *timer) //IN: -{ - tasklet_schedule(&timerTasklet); - - return HRTIMER_NORESTART; -} - - -/* - *---------------------------------------------------------------------- - * - * HostIFScheduleHRTimeout -- - * - * Schedule an hrtimer to wake up the fast clock thread. - * - * Results: - * None. - * - * Side effects: - * Sleep. - * - *---------------------------------------------------------------------- - */ - -static void -HostIFScheduleHRTimeout(ktime_t *expires) //IN: -{ - struct hrtimer t; - - if (expires && !expires->tv64) { - __set_current_state(TASK_RUNNING); - - return; - } - - hrtimer_init(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - t.function = HostIFTimerCallback; - hrtimer_start(&t, *expires, HRTIMER_MODE_REL); - - if (hrtimer_active(&t)) { - schedule(); - } - - hrtimer_cancel(&t); - __set_current_state(TASK_RUNNING); -} -#endif //VMMON_USE_COMPAT_SCHEDULE_HRTIMEOUT - - -#ifndef VMMON_USE_HIGH_RES_TIMERS -/* - *---------------------------------------------------------------------- - * - * HostIFDoIoctl -- - * - * Issue ioctl. Assume kernel is not locked. It is not true now, - * but it makes things easier to understand, and won't surprise us - * later when we get rid of kernel lock from our code. - * - * Results: - * Same as ioctl method. - * - * Side effects: - * none. - * - *---------------------------------------------------------------------- - */ - -static long -HostIFDoIoctl(struct file *filp, - u_int iocmd, - unsigned long ioarg) -{ - if (filp->f_op->unlocked_ioctl) { - return filp->f_op->unlocked_ioctl(filp, iocmd, ioarg); - } - return -ENOIOCTLCMD; -} -#endif //VMON_USE_HIGH_RES_TIMERS - - /* *---------------------------------------------------------------------- * * HostIFStartTimer -- * - * Starts the timer using either /dev/rtc or high-resolution timers. + * Starts the high-resolution timer. * * Results: * Returns 0 on success, -1 on failure. @@ -3257,83 +2959,26 @@ HostIFDoIoctl(struct file *filp, */ int -HostIFStartTimer(Bool rateChanged, //IN: Did rate change? - unsigned int rate, //IN: current clock rate - struct file *filp) //IN: /dev/rtc descriptor +HostIFStartTimer(Bool rateChanged, //IN: Did rate change? + unsigned int rate) //IN: current clock rate { -#ifdef VMMON_USE_HIGH_RES_TIMERS static unsigned long slack = 0; static ktime_t expires; int timerPeriod; if (rateChanged) { - timerPeriod = NSEC_PER_SEC / rate; + timerPeriod = NSEC_PER_SEC / rate; expires = ktime_set(0, timerPeriod); /* * Allow the kernel to expire the timer at its convenience. * ppoll() uses 0.1% of the timeout value. I think we can * tolerate 1%. */ - + slack = timerPeriod / 100; } set_current_state(TASK_INTERRUPTIBLE); -# ifdef VMMON_USE_SCHEDULE_HRTIMEOUT schedule_hrtimeout_range(&expires, slack, HRTIMER_MODE_REL); -# else - HostIFScheduleHRTimeout(&expires); -# endif -#else - unsigned p2rate; - int res; - unsigned long buf; - loff_t pos = 0; - - if (rateChanged) { - /* - * The host will already have HZ timer interrupts per second. So - * in order to satisfy the requested rate, we need up to (rate - - * HZ) additional interrupts generated by the RTC. That way, if - * the guest ask for a bit more than 1024 virtual interrupts per - * second (which is a common case for Windows with multimedia - * timers), we'll program the RTC to 1024 rather than 2048, which - * saves a considerable amount of CPU. PR 519228. - */ - if (rate > HZ) { - rate -= HZ; - } else { - rate = 0; - } - /* - * Don't set the RTC rate to 64 Hz or lower: some kernels have a - * bug in the HPET emulation of RTC that will cause the RTC - * frequency to get stuck at 64Hz. See PR 519228 comment #23. - */ - p2rate = 128; - // Hardware rate must be a power of 2 - while (p2rate < rate && p2rate < 8192) { - p2rate <<= 1; - } - - res = HostIFDoIoctl(filp, RTC_IRQP_SET, p2rate); - if (res < 0) { - Warning("/dev/rtc set rate %d failed: %d\n", p2rate, res); - - return -1; - } - if (kthread_should_stop()) { - return -1; - } - } - res = filp->f_op->read(filp, (void *) &buf, sizeof(buf), &pos); - if (res <= 0) { - if (res != -ERESTARTSYS) { - Log("/dev/rtc read failed: %d\n", res); - } - - return -1; - } -#endif return 0; } @@ -3360,9 +3005,8 @@ HostIFStartTimer(Bool rateChanged, //IN: Did rate change? */ static int -HostIFFastClockThread(void *data) // IN: +HostIFFastClockThread(void *unused) // IN: { - struct file *filp = (struct file *) data; int res; mm_segment_t oldFS; unsigned int rate = 0; @@ -3371,13 +3015,12 @@ HostIFFastClockThread(void *data) // IN: oldFS = get_fs(); set_fs(KERNEL_DS); allow_signal(SIGKILL); - set_user_nice(current, linuxState.fastClockPriority); while ((rate = linuxState.fastClockRate) > MIN_RATE) { if (kthread_should_stop()) { goto out; } - res = HostIFStartTimer(rate != prevRate, rate, filp); + res = HostIFStartTimer(rate != prevRate, rate); if (res < 0) { goto out; } @@ -3390,19 +3033,9 @@ HostIFFastClockThread(void *data) // IN: */ Vmx86_MonTimerIPI(); #endif - - /* - * Wake threads that are waiting for a fast poll timeout at - * userlevel. This is needed only on Linux. On Windows, - * we get shorter timeouts simply by increasing the host - * clock rate. - */ - - LinuxDriverWakeUp(TRUE); } out: - LinuxDriverWakeUp(TRUE); set_fs(oldFS); /* @@ -3475,34 +3108,15 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz. * system, whereas on a P4 system physical CPU 0 gets all of them. * * Long story short, unless the monitor requested rate is significantly - * higher than HZ, we don't need to send IPIs or exclusively grab /dev/rtc - * to periodically kick vCPU threads running in the monitor on all physical - * CPUs. + * higher than HZ, we don't need to send IPIs to periodically kick vCPU + * threads running in the monitor on all physical CPUs. */ if (rate > MIN_RATE) { if (!linuxState.fastClockThread) { struct task_struct *rtcTask; - struct file *filp = NULL; -#if !defined(VMMON_USE_HIGH_RES_TIMERS) - int res; - - filp = filp_open("/dev/rtc", O_RDONLY, 0); - if (IS_ERR(filp)) { - Warning("/dev/rtc open failed: %d\n", (int)(VA)filp); - - return -(int)(VA)filp; - } - res = HostIFDoIoctl(filp, RTC_PIE_ON, 0); - if (res < 0) { - Warning("/dev/rtc enable interrupt failed: %d\n", res); - filp_close(filp, current->files); - - return -res; - } -#endif - rtcTask = kthread_run(HostIFFastClockThread, filp, "vmware-rtc"); + rtcTask = kthread_run(HostIFFastClockThread, NULL, "vmware-clk"); if (IS_ERR(rtcTask)) { long err = PTR_ERR(rtcTask); @@ -3513,23 +3127,18 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz. */ if (err != -ERESTARTNOINTR) { - Warning("/dev/rtc cannot start watch thread: %ld\n", err); + Warning("vmmon cannot start hrtimer watch thread: %ld\n", err); } - close_rtc(filp, current->files); - return -err; } linuxState.fastClockThread = rtcTask; - linuxState.fastClockFile = filp; } } else { if (linuxState.fastClockThread) { force_sig(SIGKILL, linuxState.fastClockThread); kthread_stop(linuxState.fastClockThread); - close_rtc(linuxState.fastClockFile, current->files); linuxState.fastClockThread = NULL; - linuxState.fastClockFile = NULL; } } @@ -3542,7 +3151,7 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz. * * HostIF_MapUserMem -- * - * Obtain kernel pointer to user memory. The pages backing the user memory + * Obtain kernel pointer to user memory. The pages backing the user memory * address are locked into memory (this allows the pointer to be used in * contexts where paging is undesirable or impossible). * @@ -3552,7 +3161,7 @@ HostIF_SetFastClockRate(unsigned int rate) // IN: Frequency in Hz. * On failure, returns NULL. * * Side effects: - * Yes. + * Yes. * *----------------------------------------------------------------------------- */ @@ -3620,7 +3229,7 @@ HostIF_MapUserMem(VA addr, // IN: User memory virtual address printk(KERN_DEBUG "%s: p = 0x%p, offset = 0x%p, numPagesNeeded = %"FMTSZ"u," " handleSize = %"FMTSZ"u, mappedAddr = 0x%p\n", - __func__, p, (void *)offset, numPagesNeeded, handleSize, mappedAddr); + __func__, p, (void *)offset, numPagesNeeded, handleSize, mappedAddr); newHandle->numPages = numPagesNeeded; newHandle->addr = mappedAddr; @@ -3635,13 +3244,13 @@ HostIF_MapUserMem(VA addr, // IN: User memory virtual address * * HostIF_UnmapUserMem -- * - * Unmap user memory from HostIF_MapUserMem(). + * Unmap user memory from HostIF_MapUserMem(). * * Results: - * None. + * None. * * Side effects: - * Yes. + * Yes. * *----------------------------------------------------------------------------- */ @@ -3656,7 +3265,7 @@ HostIF_UnmapUserMem(VMMappedUserMem *handle) // IN: Handle to mapped memory } printk(KERN_DEBUG "%s: numPages = %"FMTSZ"u, addr = 0x%p\n", - __func__, handle->numPages, handle->addr); + __func__, handle->numPages, handle->addr); if (handle->numPages > 1) { vunmap(handle->addr); @@ -3682,7 +3291,7 @@ HostIF_UnmapUserMem(VMMappedUserMem *handle) // IN: Handle to mapped memory * 0 if successful, and MSR value is returned via *val. * * If the MSR is unimplemented, *val is set to 0, and a - * non-zero value is returned: -1 for Win32, -EFAULT for Linux, + * non-zero value is returned: -1 for Win32, -EIO for Linux, * and 1 for MacOS. * * Side effects: @@ -3694,18 +3303,12 @@ int HostIF_SafeRDMSR(unsigned int msr, // IN uint64 *val) // OUT: MSR value { - int ret; - unsigned low, high; - asm volatile("2: rdmsr ; xor %0,%0\n" - "1:\n\t" - ".section .fixup,\"ax\"\n\t" - "3: mov %4,%0 ; jmp 1b\n\t" - ".previous\n\t" - VMW_ASM_EXTABLE(2b, 3b) - : "=r"(ret), "=a"(low), "=d"(high) - : "c"(msr), "i"(-EFAULT), "1"(0), "2"(0)); // init eax/edx to 0 - *val = (low | ((u64)(high) << 32)); + int err; + u64 v; - return ret; + err = rdmsrl_safe(msr, &v); + *val = (err == 0) ? v : 0; // Linux corrupts 'v' on error + + return err; } diff --git a/vmmon-only/linux/hostif_priv.h b/vmmon-only/linux/hostif_priv.h index 8e731ba..fd79066 100644 --- a/vmmon-only/linux/hostif_priv.h +++ b/vmmon-only/linux/hostif_priv.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2009-2014 VMware, Inc. All rights reserved. + * Copyright (C) 2009-2015 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -25,14 +25,6 @@ #ifndef _HOSTIF_PRIV_H_ #define _HOSTIF_PRIV_H_ -/* Functions for transferring data to/from userspace. */ -EXTERN int HostIF_CopyFromUser(void *dst, const void *src, unsigned int len); -EXTERN int HostIF_CopyToUser(void *dst, const void *src, unsigned int len); - -/* Functions for dealing with the poll list lock. */ -EXTERN void HostIF_PollListLock(int callerID); -EXTERN void HostIF_PollListUnlock(int callerID); - /* Functions for mapping and unmapping userspace memory. */ struct VMMappedUserMem; EXTERN void *HostIF_MapUserMem(VA addr, size_t size, diff --git a/vmmon-only/linux/vmhost.h b/vmmon-only/linux/vmhost.h index 8041ba2..f686902 100644 --- a/vmmon-only/linux/vmhost.h +++ b/vmmon-only/linux/vmhost.h @@ -23,7 +23,7 @@ #define INCLUDE_ALLOW_VMCORE #include "includeCheck.h" -#include "compat_semaphore.h" +#include #ifdef VMX86_DEBUG @@ -73,7 +73,7 @@ typedef struct Mutex { */ typedef struct VMHost { - /* + /* * Used for shared modifications to VM's VMDriver data, mostly page locking. * It has higher rank than the global mutex. */ @@ -84,15 +84,15 @@ typedef struct VMHost { struct task_struct *vcpuSemaTask[MAX_INITBLOCK_CPUS]; /* * Pages that were allocated/mapped by VMX and locked by the driver and - * don't have a particular VA. + * don't have a particular VA. */ struct PhysTracker *lockedPages; /* - * Locked pages that were allocated by the driver and don't have + * Locked pages that were allocated by the driver and don't have * a particular VA. They are used as monitor anonymous pages or * as pages for "AWE" guest memory. */ - struct PhysTracker *AWEPages; + struct PhysTracker *AWEPages; /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */ Bool hostAPICIsMapped; } VMHost; diff --git a/vmmon-only/linux/vmmonInt.h b/vmmon-only/linux/vmmonInt.h deleted file mode 100644 index 4dc04e9..0000000 --- a/vmmon-only/linux/vmmonInt.h +++ /dev/null @@ -1,109 +0,0 @@ -/********************************************************* - * Copyright (C) 1998,2015 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef __VMMONINT_H__ -#define __VMMONINT_H__ - -#define INCLUDE_ALLOW_VMMON -#define INCLUDE_ALLOW_VMCORE -#include "includeCheck.h" - - -/* - * Hide all kernel compatibility stuff in these macros and functions. - */ - -#ifdef VMW_HAVE_SMP_CALL_3ARG -#define compat_smp_call_function(fn, info, wait) smp_call_function(fn, info, wait) -#else -#define compat_smp_call_function(fn, info, wait) smp_call_function(fn, info, 1, wait) -#endif - - -/* - *---------------------------------------------------------------------- - * - * compat_tsc_khz -- - * - * Compatibility wrapper for tsc_khz. - * - * Returns: - * - * Returns 0 if symbol is not exported by the kernel, else - * value of tsc_khz. - * - *---------------------------------------------------------------------- - */ - -static inline unsigned int -compat_tsc_khz(void) -{ -#if defined(VMW_HAVE_TSC_KHZ) - return tsc_khz; -#else - return 0; -#endif -} - - -/* - *---------------------------------------------------------------------- - * - * compat_smp_call_function_single -- - * - * Compatibility wrapper for calling smp_call_function_single. - * Versions prior to 2.6.20 did not export the symbol for both - * i386 and x86_64 kernels. - * - * Returns: - * - * Returns -ENOSYS if the host kernel does not implement or export - * the function, else returns the error status of - * smp_call_function_single. - * - *---------------------------------------------------------------------- - */ - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) -# ifdef VMW_HAVE_SMP_CALL_3ARG -# define compat_smp_call_function_single(cpu, fn, info, wait) \ - smp_call_function_single(cpu, fn, info, wait) -# else -# define compat_smp_call_function_single(cpu, fn, info, wait) \ - smp_call_function_single(cpu, fn, info, 1, wait) -# endif // VMW_HAVE_SMP_CALL_3ARG -#else // VERSION >= 2.6.20 -# define compat_smp_call_function_single(cpu, fn, info, wait) (-ENOSYS) -#endif - - -/* - * Although this is not really related to kernel-compatibility, I put this - * helper macro here for now for a lack of better place --hpreg - * - * The exit(2) path does, in this order: - * . set current->files to NULL - * . close all fds, which potentially calls LinuxDriver_Close() - * - * fget() requires current->files != NULL, so we must explicitely check --hpreg - */ -#define vmware_fget(_fd) (current->files ? fget(_fd) : NULL) - -extern void LinuxDriverWakeUp(Bool selective); - -#endif /* __VMMONINT_H__ */ diff --git a/vmmon-only/vmcore/moduleloop.c b/vmmon-only/vmcore/moduleloop.c index a6b2c1a..589b4bf 100644 --- a/vmmon-only/vmcore/moduleloop.c +++ b/vmmon-only/vmcore/moduleloop.c @@ -41,6 +41,7 @@ #include "memtrack.h" #include "driver_vmcore.h" #include "usercalldefs.h" +#include "cpuid.h" /* *---------------------------------------------------------------------- @@ -74,7 +75,7 @@ Vmx86_RunVM(VMDriver *vm, // IN: VMCrossPageData *crosspage = &vm->crosspage[vcpuid]->crosspageData; int bailValue = 0; - ASSERT(crosspage); + ASSERT(crosspage && CPUID_HostSupportsHV()); /* * Check if we were interrupted by signal. @@ -172,15 +173,13 @@ skipTaskSwitch:; } case MODULECALL_SEMAFORCEWAKEUP: { - ASSERT_ON_COMPILE(sizeof(VCPUSet) <= sizeof(crosspage->args)); - HostIF_SemaphoreForceWakeup(vm, (VCPUSet *) &crosspage->args[0]); + HostIF_SemaphoreForceWakeup(vm, &crosspage->vcpuSet); break; } case MODULECALL_IPI: { HostIFIPIMode mode; - ASSERT_ON_COMPILE(sizeof(VCPUSet) <= sizeof(crosspage->args)); - mode = HostIF_IPI(vm, (VCPUSet *) &crosspage->args[0]); + mode = HostIF_IPI(vm, &crosspage->vcpuSet); retval = (mode != IPI_NONE); break; } @@ -225,11 +224,9 @@ skipTaskSwitch:; } case MODULECALL_COSCHED: { - uint32 spinUS = (uint32)crosspage->args[2]; - ASSERT_ON_COMPILE(sizeof(VCPUSet) + sizeof(uint32) <= - sizeof(crosspage->args)); - Vmx86_YieldToSet(vm, vcpuid, (VCPUSet *) &crosspage->args[0], - spinUS, FALSE); + uint32 spinUS = (uint32)crosspage->args[0]; + ASSERT_ON_COMPILE(sizeof(uint32) <= sizeof(crosspage->args)); + Vmx86_YieldToSet(vm, vcpuid, &crosspage->vcpuSet, spinUS, FALSE); break; } @@ -251,6 +248,18 @@ skipTaskSwitch:; retval = crosspage->retval; } break; + case MODULECALL_VMCLEAR_VMCS_ALL_CPUS: { + MA vmcs = (MA)crosspage->args[0]; + Vmx86_FlushVMCSAllCPUs(vmcs); + } break; + + case MODULECALL_GET_PAGE_ROOT: { + MPN mpn; + Vcpuid targetVcpuid = (Vcpuid)crosspage->args[0]; + retval = Vmx86_GetPageRoot(vm, targetVcpuid, &mpn); + crosspage->args[0] = mpn; + } break; + default: Warning("ModuleCall %d not supported\n", crosspage->moduleCallType); } diff --git a/vmnet-only/Makefile b/vmnet-only/Makefile index 459846e..caab6b9 100644 --- a/vmnet-only/Makefile +++ b/vmnet-only/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f ########################################################## -# Copyright (C) 1998-2015 VMware, Inc. All rights reserved. +# Copyright (C) 1998-2016 VMware, Inc. All rights reserved. # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -84,8 +84,6 @@ endif ifneq ($(VM_KBUILD), no) -VMCCVER := $(shell $(CC) -dumpversion) - # If there is no version defined, we are in toplevel pass, not yet in kernel makefiles... ifeq ($(VERSION),) @@ -100,14 +98,6 @@ auto-build: $(DRIVER_KO) $(DRIVER): $(DRIVER_KO) if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi -# Pass gcc version down the chain, so we can detect if kernel attempts to use unapproved compiler -VM_CCVER := $(VMCCVER) -export VM_CCVER -VM_CC := $(CC) -export VM_CC - -MAKEOVERRIDES := $(filter-out CC=%,$(MAKEOVERRIDES)) - # # Define a setup target that gets built before the actual driver. # This target may not be used at all, but if it is then it will be defined diff --git a/vmnet-only/Makefile.kernel b/vmnet-only/Makefile.kernel index 2d8e6f6..09a1e3c 100644 --- a/vmnet-only/Makefile.kernel +++ b/vmnet-only/Makefile.kernel @@ -20,10 +20,6 @@ INCLUDE := -I$(SRCROOT) EXTRA_CFLAGS := $(CC_OPTS) $(INCLUDE) -EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/netdev_has_net.c,-DVMW_NETDEV_HAS_NET, ) -EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/netdev_has_dev_net.c,-DVMW_NETDEV_HAS_DEV_NET, ) -EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/nfhook_uses_skb.c,-DVMW_NFHOOK_USES_SKB, ) -EXTRA_CFLAGS += $(call vm_check_build, $(SRCROOT)/skblin.c, -DVMW_SKB_LINEARIZE_2618, ) obj-m += $(DRIVER).o diff --git a/vmnet-only/Makefile.normal b/vmnet-only/Makefile.normal index a218182..ab18108 100644 --- a/vmnet-only/Makefile.normal +++ b/vmnet-only/Makefile.normal @@ -46,10 +46,6 @@ INCLUDE := -I. -I$(HEADER_DIR) INCLUDE += $(shell $(CC) $(INCLUDE) -E geninclude.c \ | sed -n -e 's!^APATH!-I$(HEADER_DIR)/asm!p') -CC_OPTS += $(call vm_check_build, $(SRCROOT)/netdev_has_net.c,-DVMW_NETDEV_HAS_NET, ) -CC_OPTS += $(call vm_check_build, $(SRCROOT)/netdev_has_dev_net.c,-DVMW_NETDEV_HAS_DEV_NET, ) -CC_OPTS += $(call vm_check_build, $(SRCROOT)/skblin.c, -DVMW_SKB_LINEARIZE_2618, ) - CFLAGS := -O $(CC_WARNINGS) $(CC_OPTS) $(INCLUDE) $(GLOBAL_DEFS) OBJS := driver.o hub.o userif.o netif.o bridge.o procfs.o smac_compat.o \ diff --git a/vmnet-only/bridge.c b/vmnet-only/bridge.c index 8cdaf11..25a7442 100644 --- a/vmnet-only/bridge.c +++ b/vmnet-only/bridge.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -77,7 +78,6 @@ struct VNetBridge { struct sock *sk; // socket associated with skb's struct packet_type pt; // used to add packet handler Bool enabledPromisc; // track if promisc enabled - Bool warnPromisc; // tracks if warning has been logged Bool forceSmac; // whether to use smac unconditionally struct sk_buff *history[VNET_BRIDGE_HISTORY]; // avoid duplicate packets spinlock_t historyLock; // protects 'history' @@ -146,7 +146,6 @@ VNetBridgeStartPromisc(VNetBridge *bridge, // IN: if (!bridge->enabledPromisc && !bridge->wirelessAdapter) { dev_set_promiscuity(dev, 1); bridge->enabledPromisc = TRUE; - bridge->warnPromisc = FALSE; LOG(0, (KERN_NOTICE "bridge-%s: enabled promiscuous mode\n", bridge->name)); } @@ -213,11 +212,9 @@ static INLINE_SINGLE_CALLER int VNetBridgeDevCompatible(VNetBridge *bridge, // IN: Bridge struct net_device *net) // IN: Network device { -#ifdef VMW_NETDEV_HAS_NET - if (compat_dev_net(net) != &init_net) { + if (dev_net(net) != &init_net) { return 0; } -#endif return strcmp(net->name, bridge->name) == 0; } @@ -636,7 +633,7 @@ VNetBridgeReceiveFromVNet(VNetJack *this, // IN: jack unsigned long flags; int i; - atomic_inc(&clone->users); + clone = skb_get(clone); clone->dev = dev; clone->protocol = eth_type_trans(clone, dev); @@ -811,22 +808,10 @@ VNetBridgeIsBridged(VNetJack *this) // IN: jack static Bool VNetBridgeIsDeviceWireless(struct net_device *dev) //IN: sock { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22) -# if defined(CONFIG_WIRELESS_EXT) +#if defined(CONFIG_WIRELESS_EXT) return dev->ieee80211_ptr != NULL || dev->wireless_handlers != NULL; -# else - return dev->ieee80211_ptr != NULL; -# endif -#elif defined(CONFIG_WIRELESS_EXT) - return dev->wireless_handlers != NULL; -#elif !defined(CONFIG_NET_RADIO) - return FALSE; -#elif defined WIRELESS_EXT && WIRELESS_EXT > 19 - return dev->wireless_handlers != NULL; -#elif defined WIRELESS_EXT && WIRELESS_EXT > 12 - return dev->wireless_handlers != NULL || dev->get_wireless_stats != NULL; #else - return dev->get_wireless_stats != NULL; + return dev->ieee80211_ptr != NULL; #endif } @@ -911,7 +896,7 @@ VNetBridgeUp(VNetBridge *bridge, // IN: bridge struct */ dev_lock_list(); - bridge->dev = DEV_GET(bridge); + bridge->dev = __dev_get_by_name(&init_net, bridge->name); LOG(2, (KERN_DEBUG "bridge-%s: got dev %p\n", bridge->name, bridge->dev)); if (bridge->dev == NULL) { @@ -926,17 +911,10 @@ VNetBridgeUp(VNetBridge *bridge, // IN: bridge struct retval = -ENODEV; goto out; } - - /* - * At a minimum, the header size should be the same as ours. - * - * XXX we should either do header translation or ensure this - * is an Ethernet. - */ - - if (bridge->dev->hard_header_len != ETH_HLEN) { - LOG(1, (KERN_DEBUG "bridge-%s: can't bridge with %s, bad header length %d\n", - bridge->name, bridge->dev->name, bridge->dev->hard_header_len)); + if (bridge->dev->type != ARPHRD_ETHER) { + LOG(1, (KERN_DEBUG "bridge-%s: can't bridge with %s (header length %d, " + "type %d).\n", bridge->name, bridge->dev->name, + bridge->dev->hard_header_len, bridge->dev->type)); dev_unlock_list(); retval = -EINVAL; goto out; @@ -990,7 +968,6 @@ VNetBridgeUp(VNetBridge *bridge, // IN: bridge struct bridge->pt.af_packet_priv = bridge->sk; bridge->enabledPromisc = FALSE; - bridge->warnPromisc = FALSE; dev_add_pack(&bridge->pt); dev_unlock_list(); @@ -1406,154 +1383,6 @@ VNetBridgeComputeHeaderPos(struct sk_buff *skb) // IN: buffer to examine } -/* - * We deal with three types of kernels: - * New kernels: skb_shinfo() has gso_size member, and there is - * skb_gso_segment() helper to split GSO skb into flat ones. - * Older kernels: skb_shinfo() has tso_size member, and there is - * no helper. - * Oldest kernels: without any segmentation offload support. - */ -#if defined(NETIF_F_GSO) || LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18) -#define VNetBridgeIsGSO(skb) skb_shinfo(skb)->gso_size -#define VNetBridgeGSOSegment(skb) skb_gso_segment(skb, 0) -#elif defined(NETIF_F_TSO) -#define VNetBridgeIsGSO(skb) skb_shinfo(skb)->tso_size - - -/* - *---------------------------------------------------------------------- - * - * VNetBridgeGSOSegment -- - * - * Split a large TCP/IPv4 sk_buff into multiple sk_buffs of - * size skb_shinfo(skb)->tso_size - * Called from VNetBridgeSendLargePacket(). - * - * Results: - * List of skbs created. - * - * Side effects: - * The incoming packet is split into multiple packets. - * - *---------------------------------------------------------------------- - */ - -static struct sk_buff * -VNetBridgeGSOSegment(struct sk_buff *skb) // IN: packet to split -{ - struct sk_buff *segs = NULL; - struct sk_buff **next = &segs; - int bytesPerPacket, bytesLeft; - int macHdrLen, ipHdrLen, tcpHdrLen, allHdrLen; - int curByteOffset; - uint16 ipID; - uint32 seqNo; - - if (((struct ethhdr *)compat_skb_mac_header(skb))->h_proto != htons(ETH_P_IP)) { - return ERR_PTR(-EPFNOSUPPORT); - } - - if (compat_skb_ip_header(skb)->protocol != IPPROTO_TCP) { - return ERR_PTR(-EPROTONOSUPPORT); - } - - macHdrLen = compat_skb_network_header(skb) - compat_skb_mac_header(skb); - ipHdrLen = compat_skb_ip_header(skb)->ihl << 2; - tcpHdrLen = compat_skb_tcp_header(skb)->doff << 2; - allHdrLen = macHdrLen + ipHdrLen + tcpHdrLen; - - ipID = ntohs(compat_skb_ip_header(skb)->id); - seqNo = ntohl(compat_skb_tcp_header(skb)->seq); - - /* Host TCP stack populated this (MSS) for the host NIC driver */ - bytesPerPacket = skb_shinfo(skb)->tso_size; - - bytesLeft = skb->len - allHdrLen; - curByteOffset = allHdrLen; - - while (bytesLeft) { - struct sk_buff *newSkb; - int payloadSize = (bytesLeft < bytesPerPacket) ? bytesLeft : bytesPerPacket; - - newSkb = dev_alloc_skb(payloadSize + allHdrLen + NET_IP_ALIGN); - if (!newSkb) { - while (segs) { - newSkb = segs; - segs = segs->next; - newSkb->next = NULL; - dev_kfree_skb(newSkb); - } - return ERR_PTR(-ENOMEM); - } - skb_reserve(newSkb, NET_IP_ALIGN); - newSkb->dev = skb->dev; - newSkb->protocol = skb->protocol; - newSkb->pkt_type = skb->pkt_type; - newSkb->ip_summed = VM_TX_CHECKSUM_PARTIAL; - - /* - * MAC+IP+TCP copy - * This implies that ALL fields in the IP and TCP headers are copied from - * the original skb. This is convenient: we'll only fix up fields that - * need to be changed below - */ - memcpy(skb_put(newSkb, allHdrLen), skb->data, allHdrLen); - - /* Fix up pointers to different layers */ - compat_skb_reset_mac_header(newSkb); - compat_skb_set_network_header(newSkb, macHdrLen); - compat_skb_set_transport_header(newSkb, macHdrLen + ipHdrLen); - - /* Payload copy */ - skb_copy_bits(skb, curByteOffset, compat_skb_tail_pointer(newSkb), payloadSize); - skb_put(newSkb, payloadSize); - - curByteOffset+=payloadSize; - bytesLeft -= payloadSize; - - /* Fix up IP hdr */ - compat_skb_ip_header(newSkb)->tot_len = htons(payloadSize + tcpHdrLen + ipHdrLen); - compat_skb_ip_header(newSkb)->id = htons(ipID); - compat_skb_ip_header(newSkb)->check = 0; - /* Recompute new IP checksum */ - compat_skb_ip_header(newSkb)->check = - ip_fast_csum(compat_skb_network_header(newSkb), - compat_skb_ip_header(newSkb)->ihl); - - /* Fix up TCP hdr */ - compat_skb_tcp_header(newSkb)->seq = htonl(seqNo); - /* Clear FIN/PSH if not last packet */ - if (bytesLeft > 0) { - compat_skb_tcp_header(newSkb)->fin = 0; - compat_skb_tcp_header(newSkb)->psh = 0; - } - /* Recompute partial TCP checksum */ - compat_skb_tcp_header(newSkb)->check = - ~csum_tcpudp_magic(compat_skb_ip_header(newSkb)->saddr, - compat_skb_ip_header(newSkb)->daddr, - payloadSize+tcpHdrLen, IPPROTO_TCP, 0); - - /* Offset of field */ - newSkb->csum = offsetof(struct tcphdr, check); - - /* Join packet to the list of segments */ - *next = newSkb; - next = &newSkb->next; - - /* Bump up our counters */ - ipID++; - seqNo += payloadSize; - - } - return segs; -} -#else -#define VNetBridgeIsGSO(skb) (0) -#define VNetBridgeGSOSegment(skb) ERR_PTR(-ENOSYS) -#endif - - /* *---------------------------------------------------------------------- * @@ -1582,7 +1411,7 @@ VNetBridgeSendLargePacket(struct sk_buff *skb, // IN: packet to split { struct sk_buff *segs; - segs = VNetBridgeGSOSegment(skb); + segs = skb_gso_segment(skb, 0); dev_kfree_skb(skb); if (IS_ERR(segs)) { LOG(1, (KERN_DEBUG "bridge-%s: cannot segment packet: error %ld\n", @@ -1713,7 +1542,7 @@ VNetBridgeReceiveFromDev(struct sk_buff *skb, // IN: packet to receive /* * If this is a large packet, chop chop chop (if supported)... */ - if (VNetBridgeIsGSO(skb)) { + if (skb_shinfo(skb)->gso_size) { VNetBridgeSendLargePacket(skb, bridge); } else { VNetSend(&bridge->port.jack, skb); diff --git a/vmnet-only/community_source.h b/vmnet-only/community_source.h index 2d32d8f..754416e 100644 --- a/vmnet-only/community_source.h +++ b/vmnet-only/community_source.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2009-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2009-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmnet-only/compat_netdevice.h b/vmnet-only/compat_netdevice.h index e1a3b0e..0b34ccd 100644 --- a/vmnet-only/compat_netdevice.h +++ b/vmnet-only/compat_netdevice.h @@ -337,7 +337,9 @@ typedef netdev_features_t compat_netdev_features_t; typedef u32 compat_netdev_features_t; #endif -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) || \ + (defined(RHEL_RELEASE_CODE) && RHEL_RELEASE_CODE >= 0x0704) || \ + (defined(CONFIG_SUSE_KERNEL) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 73)) #define compat_netif_trans_update(d) netif_trans_update(d) #else #define compat_netif_trans_update(d) do { (d)->trans_start = jiffies; } while (0) diff --git a/vmnet-only/driver.c b/vmnet-only/driver.c index 7e7ad99..b3da8cb 100644 --- a/vmnet-only/driver.c +++ b/vmnet-only/driver.c @@ -1682,11 +1682,7 @@ VNetSnprintf(char *str, // OUT: resulting string va_start(args, format); -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 8) length = vsnprintf(str, size, format, args); -#else - length = vsprintf(str, format, args); -#endif va_end(args); diff --git a/vmnet-only/monitorAction_exported.h b/vmnet-only/monitorAction_exported.h deleted file mode 100644 index 28b57c2..0000000 --- a/vmnet-only/monitorAction_exported.h +++ /dev/null @@ -1,156 +0,0 @@ -/********************************************************* - * Copyright (C) 2010-2013 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef _MONITORACTION_EXPORTED_H_ -#define _MONITORACTION_EXPORTED_H_ - -#define INCLUDE_ALLOW_VMX -#define INCLUDE_ALLOW_VMCORE -#define INCLUDE_ALLOW_USERLEVEL -#define INCLUDE_ALLOW_MODULE -#define INCLUDE_ALLOW_VMMON -#define INCLUDE_ALLOW_VMKERNEL -#define INCLUDE_ALLOW_DISTRIBUTE -#include "includeCheck.h" - -#include "vm_assert.h" -#include "vm_atomic.h" -#include "vm_basic_types.h" - -/* - * Please bump the version number if your change will break the - * compatability to the drivers. - */ -#define ACTION_EXPORTED_VERSION 2 - -#define ACTION_WORD_SIZE (sizeof(uint64) * 8) -#define ACTION_NUM_WORDS (2) -#define ACTION_NUM_IDS (ACTION_NUM_WORDS * ACTION_WORD_SIZE) - -#define MONACTION_INVALID MAX_UINT32 - -typedef uint32 MonitorIdemAction; - -/* - * Representation of a set of actions. - */ -typedef struct MonitorActionSet { - volatile uint64 word[ACTION_NUM_WORDS]; -} MonitorActionSet; - -#ifndef __cplusplus -typedef enum MonitorActionSetName MonitorActionSetName; -#endif - -/* - * Summary of action and interrupt states. - */ -typedef struct MonitorActionIntr { - MonitorActionSet pendingSet; - volatile Bool action; - Bool intr; - Bool nmi; - Bool db; - uint32 _pad; -} MonitorActionIntr; - -/* - *------------------------------------------------------------------------ - * MonitorActionSet_AtomicInclude -- - * - * This function atomically adds an action to an action set. - * - * Results: - * TRUE if the action being added did not exist in the action set. - * FALSE otherwise. - * - * Side effects: - * The given action set will be updated. - *------------------------------------------------------------------------ - */ -static INLINE Bool -MonitorActionSet_AtomicInclude(MonitorActionSet *set, const uint32 actionID) -{ - Atomic_uint64 *atomicSet = - Atomic_VolatileToAtomic64(&set->word[actionID / ACTION_WORD_SIZE]); - uint64 mask = (uint64)1 << (actionID % ACTION_WORD_SIZE); - uint64 oldWord; - uint64 newWord; - - ASSERT_ON_COMPILE((ACTION_WORD_SIZE & (ACTION_WORD_SIZE - 1)) == 0); -#ifdef VMX86_DEBUG - /* If ASSERT is not desirable, do explicit check. Please see PR 567811. */ -#ifdef MODULE - if (UNLIKELY(actionID / ACTION_WORD_SIZE >= ACTION_NUM_WORDS)) { - return FALSE; - } -#else - ASSERT(actionID / ACTION_WORD_SIZE < ACTION_NUM_WORDS); -#endif // MODULE -#endif // VMX86_DEBUG - do { - oldWord = Atomic_Read64(atomicSet); - newWord = oldWord | mask; - } while (!Atomic_CMPXCHG64(atomicSet, &oldWord, &newWord)); - return (oldWord & mask) == 0; -} - - -/* - *---------------------------------------------------------------------------- - * MonitorAction_SetBits -- - * - * The core logic for posting an action. Update the set of pending - * actions of the target VCPU in the shared area to mark the action - * as present. Make sure the bit is set in the pendingSet first to - * avoid a race with the drain loop. - * - * It's the responsibility of the callers to ensure that the change - * to actionIntr->action is globally visible before any IPI is sent - * (the change to pendingSet is pushed out by the cmpxchg in - * MonitorActionSet_AtomicInclude). - * - * Results: - * TRUE if the action being posted was not pending before. - * FALSE otherwise (other threads could have posted the same action). - * - * Side effects: - * None. - *---------------------------------------------------------------------------- - */ -static INLINE Bool -MonitorAction_SetBits(MonitorActionIntr *actionIntr, MonitorIdemAction actionID) -{ - /* Careful if optimizing this: see PR70016. */ - Bool newAction = - MonitorActionSet_AtomicInclude(&actionIntr->pendingSet, actionID); - actionIntr->action = TRUE; - return newAction; -} - -/* - * C1 states entered by monitor while waiting for an action - */ -typedef enum { - VMM_C1_STATE_INVALID = 0, - VMM_C1_STATE_HLT, - VMM_C1_STATE_MWAIT, - VMM_C1_STATE_PAUSE -} vmmC1StateType; - -#endif // _MONITORACTION_EXPORTED_H_ diff --git a/vmnet-only/net.h b/vmnet-only/net.h index 28fd10d..c226f92 100644 --- a/vmnet-only/net.h +++ b/vmnet-only/net.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the diff --git a/vmnet-only/netdev_has_dev_net.c b/vmnet-only/netdev_has_dev_net.c deleted file mode 100644 index 7ae5a01..0000000 --- a/vmnet-only/netdev_has_dev_net.c +++ /dev/null @@ -1,37 +0,0 @@ -/********************************************************* - * Copyright (C) 2008 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * Detect whether there is dev_net accessor for dev->nd_net. - * It appeared between 2.6.25 and 2.6.26-rc1. - */ - -#include "compat_version.h" -#include "compat_autoconf.h" - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25) -# error This compile test intentionally fails. -#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26) -# include - -struct net * -vmware_dev_net(struct net_device *dev) -{ - return dev_net(dev); -} -#endif diff --git a/vmnet-only/netdev_has_net.c b/vmnet-only/netdev_has_net.c deleted file mode 100644 index 210c9de..0000000 --- a/vmnet-only/netdev_has_net.c +++ /dev/null @@ -1,43 +0,0 @@ -/********************************************************* - * Copyright (C) 2007 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * Detect whether there is separate net namespace. It got introduced after - * 2.6.23. If this builds, there are two arguments to __dev_get_by_name... - * For lower boundary use 2.6.23 - hopefully nobody crossports patch to - * older kernels. Note that this also affects sk_alloc interface - - * for that there are two versions: sk_alloc(net, family, gfp, proto, 1) for - * kernels 2.6.23 < x <= 2.6.24-rc1, and 4 argument version - * sk_alloc(net, family, gfp, proto) for 2.6.24-rc1 < x. We do ignore 2.6.24-rc1 - * as hopefully in few weeks all 2.6.24-rc1 users will be gone. - */ - -#include "compat_version.h" -#include "compat_autoconf.h" - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23) -# error This compile test intentionally fails. -#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24) -# include - -struct net_device * -vmware_get_by_name(void) -{ - return __dev_get_by_name(0, "dummy"); -} -#endif diff --git a/vmnet-only/nfhook_uses_skb.c b/vmnet-only/nfhook_uses_skb.c deleted file mode 100644 index fd61b67..0000000 --- a/vmnet-only/nfhook_uses_skb.c +++ /dev/null @@ -1,45 +0,0 @@ -/********************************************************* - * Copyright (C) 2007 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * Detect whether nf_hookfn takes struct sk_buff* skb, or struct sk_buff** pskb. - * Kernels before 2.6.23 take pskb, kernels since 2.6.24 take skb, and we - * are not sure about 2.6.23 itself, as change occured between 2.6.23 and - * 2.6.24-rc1. - */ - -#include "compat_version.h" -#include "compat_autoconf.h" - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23) -# error This compile test intentionally fails. -#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24) -# include - -nf_hookfn test_function; - -unsigned int -test_function(unsigned int hooknum, - struct sk_buff *skb, - const struct net_device *in, - const struct net_device *out, - int (*defn)(struct sk_buff*)) -{ - return 1234; -} -#endif diff --git a/vmnet-only/skblin.c b/vmnet-only/skblin.c deleted file mode 100644 index e3a6ccd..0000000 --- a/vmnet-only/skblin.c +++ /dev/null @@ -1,41 +0,0 @@ -/********************************************************* - * Copyright (C) 2006 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * Detect whether skb_linearize takes one or two arguments. - */ - -#include "compat_version.h" -#include "compat_autoconf.h" - -#if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 17) -/* - * Since 2.6.18 all kernels have single-argument skb_linearize. For - * older kernels use autodetection. Not using autodetection on newer - * kernels saves us from compile failure on some post 2.6.18 kernels - * which do not have selfcontained skbuff.h. - */ - -#include - -int test_skb_linearize(struct sk_buff *skb) -{ - return skb_linearize(skb); -} - -#endif diff --git a/vmnet-only/smac.c b/vmnet-only/smac.c index 71b711e..c1c8da8 100644 --- a/vmnet-only/smac.c +++ b/vmnet-only/smac.c @@ -2908,17 +2908,16 @@ SMAC_CheckPacketToHost(SMACState *state, // IN: pointer to state arpHeaderWord2)); return PacketStatusDropPacket; } else { // if EAPOL packet: typeClass == EthClassEAPOL - - /* - * Allow outgoing EAPOL packets to proceed unmolested provided the - * source address matches the hardware address. - */ - - if (!MAC_EQ(state->macAddress, eh->srcAddr)) { - VNETKdPrint((MODULE_NAME "ToHostEAPOL: outgoing request using " - "non-wireless-hardware-addr eth source MAC, dropping\n")); + if (!ClonePacket(packets)) { + VNETKdPrint((MODULE_NAME " ToHostEapol: couldn't clone packet\n")); return PacketStatusDropPacket; } + + /* For wireless, send EAPOL packets to host side. */ + CopyDataToClonedPacket(packets, state->macAddress, + ETH_ALEN /* offset for source MAC */, + ETH_ALEN /* length */); + return PacketStatusForwardPacket; } } diff --git a/vmnet-only/userif.c b/vmnet-only/userif.c index a7aee04..acc6ca6 100644 --- a/vmnet-only/userif.c +++ b/vmnet-only/userif.c @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2013 VMware, Inc. All rights reserved. + * Copyright (C) 1998,2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -36,9 +36,7 @@ #include #include #include -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) #include // For without version dependency -#endif #include #include @@ -50,7 +48,6 @@ #include "vmnetInt.h" #include "vm_atomic.h" #include "vm_assert.h" -#include "monitorAction_exported.h" typedef struct VNetUserIFStats { unsigned read; @@ -66,12 +63,9 @@ typedef struct VNetUserIF { VNetPort port; struct sk_buff_head packetQueue; Atomic_uint32 *pollPtr; - MonitorActionIntr *actionIntr; uint32 pollMask; - MonitorIdemAction actionID; uint32* recvClusterCount; wait_queue_head_t waitQueue; - struct page* actPage; struct page* pollPage; struct page* recvClusterPage; VNetUserIFStats stats; @@ -115,16 +109,7 @@ UserifLockPage(VA addr) // IN struct page *page = NULL; int retval; - down_read(¤t->mm->mmap_sem); -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) - retval = get_user_pages(addr, 1, FOLL_WRITE, &page, NULL); -#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) - retval = get_user_pages(addr, 1, 1, 0, &page, NULL); -#else - retval = get_user_pages(current, current->mm, addr, - 1, 1, 0, &page, NULL); -#endif - up_read(¤t->mm->mmap_sem); + retval = get_user_pages_fast(addr, 1, FOLL_WRITE, &page); if (retval != 1) { return NULL; @@ -185,15 +170,15 @@ VNetUserIfMapUint32Ptr(VA uAddr, // IN: pointer to user memory * * VNetUserIfSetupNotify -- * - * Sets up notification by filling in pollPtr, actPtr, and recvClusterCount + * Sets up notification by filling in pollPtr and recvClusterCount * fields. - * - * Results: + * + * Results: * 0 on success * < 0 on failure: the actual value determines the type of failure * * Side effects: - * Fields pollPtr, actPtr, recvClusterCount, pollPage, actPage, and + * Fields pollPtr, recvClusterCount, pollPage, and * recvClusterPage are filled in VNetUserIf structure. * *----------------------------------------------------------------------------- @@ -206,14 +191,12 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN unsigned long flags; struct sk_buff_head *q = &userIf->packetQueue; uint32 *pollPtr; - MonitorActionIntr *actionIntr; uint32 *recvClusterCount; struct page *pollPage = NULL; - struct page *actPage = NULL; struct page *recvClusterPage = NULL; int retval; - if (userIf->pollPtr || userIf->actionIntr || userIf->recvClusterCount) { + if (userIf->pollPtr || userIf->recvClusterCount) { LOG(0, (KERN_DEBUG "vmnet: Notification mechanism already active\n")); return -EBUSY; } @@ -230,12 +213,6 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN goto error_free; } - if ((retval = VNetUserIfMapPtr((VA)vn->actPtr, sizeof *actionIntr, - &actPage, - (void **)&actionIntr)) < 0) { - goto error_free; - } - if ((retval = VNetUserIfMapUint32Ptr((VA)vn->recvClusterPtr, &recvClusterPage, &recvClusterCount)) < 0) { @@ -243,7 +220,7 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN } spin_lock_irqsave(&q->lock, flags); - if (userIf->pollPtr || userIf->actionIntr || userIf->recvClusterCount) { + if (userIf->pollPtr || userIf->recvClusterCount) { spin_unlock_irqrestore(&q->lock, flags); retval = -EBUSY; LOG(0, (KERN_DEBUG "vmnet: Notification mechanism already active\n")); @@ -252,12 +229,9 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN userIf->pollPtr = (Atomic_uint32 *)pollPtr; userIf->pollPage = pollPage; - userIf->actionIntr = actionIntr; - userIf->actPage = actPage; userIf->recvClusterCount = recvClusterCount; userIf->recvClusterPage = recvClusterPage; userIf->pollMask = vn->pollMask; - userIf->actionID = vn->actionID; spin_unlock_irqrestore(&q->lock, flags); return 0; @@ -266,10 +240,6 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN kunmap(pollPage); put_page(pollPage); } - if (actPage) { - kunmap(actPage); - put_page(actPage); - } if (recvClusterPage) { kunmap(recvClusterPage); put_page(recvClusterPage); @@ -283,12 +253,12 @@ VNetUserIfSetupNotify(VNetUserIF *userIf, // IN * VNetUserIfUnsetupNotify -- * * Destroys permanent mapping for notify structure provided by user. - * - * Results: + * + * Results: * None. * * Side effects: - * Fields pollPtr, actPtr, recvClusterCount, etc. in VNetUserIf + * Fields pollPtr, recvClusterCount, etc. in VNetUserIf * structure are cleared. * *---------------------------------------------------------------------- @@ -299,7 +269,6 @@ VNetUserIfUnsetupNotify(VNetUserIF *userIf) // IN { unsigned long flags; struct page *pollPage = userIf->pollPage; - struct page *actPage = userIf->actPage; struct page *recvClusterPage = userIf->recvClusterPage; struct sk_buff_head *q = &userIf->packetQueue; @@ -307,12 +276,9 @@ VNetUserIfUnsetupNotify(VNetUserIF *userIf) // IN spin_lock_irqsave(&q->lock, flags); userIf->pollPtr = NULL; userIf->pollPage = NULL; - userIf->actionIntr = NULL; - userIf->actPage = NULL; userIf->recvClusterCount = NULL; userIf->recvClusterPage = NULL; userIf->pollMask = 0; - userIf->actionID = -1; spin_unlock_irqrestore(&q->lock, flags); /* Release */ @@ -320,10 +286,6 @@ VNetUserIfUnsetupNotify(VNetUserIF *userIf) // IN kunmap(pollPage); put_page(pollPage); } - if (actPage) { - kunmap(actPage); - put_page(actPage); - } if (recvClusterPage) { kunmap(recvClusterPage); put_page(recvClusterPage); @@ -436,9 +398,12 @@ VNetUserIfReceive(VNetJack *this, // IN __skb_queue_tail(&userIf->packetQueue, skb); if (userIf->pollPtr) { Atomic_Or(userIf->pollPtr, userIf->pollMask); +#if 0 + /* Potential optimization: avoid waking based on cluster size */ if (skb_queue_len(&userIf->packetQueue) >= (*userIf->recvClusterCount)) { - MonitorAction_SetBits(userIf->actionIntr, userIf->actionID); + // TODO } +#endif } spin_unlock_irqrestore(&userIf->packetQueue.lock, flags); @@ -866,14 +831,6 @@ VNetUserIfIoctl(VNetPort *port, // IN case SIOCSETNOTIFY: return -EINVAL; case SIOCSETNOTIFY2: -#ifdef VMX86_SERVER - /* - * This ioctl always return failure on ESX since we cannot map pages into - * the console os that are from the VMKernel address space which was the - * only case we used this. - */ - return -EINVAL; -#else // VMX86_SERVER /* * ORs pollMask into the integer pointed to by ptr if pending packet. Is * cleared when all packets are drained. @@ -886,11 +843,8 @@ VNetUserIfIoctl(VNetPort *port, // IN return -EFAULT; } - ASSERT_ON_COMPILE(VNET_NOTIFY_VERSION == 5); - ASSERT_ON_COMPILE(ACTION_EXPORTED_VERSION == 2); - if (vn.version != VNET_NOTIFY_VERSION || - vn.actionVersion != ACTION_EXPORTED_VERSION || - vn.actionID / ACTION_WORD_SIZE >= ACTION_NUM_WORDS) { + ASSERT_ON_COMPILE(VNET_NOTIFY_VERSION == 6); + if (vn.version != VNET_NOTIFY_VERSION) { return -ENOTTY; } @@ -901,10 +855,8 @@ VNetUserIfIoctl(VNetPort *port, // IN break; } -#endif // VMX86_SERVER case SIOCUNSETNOTIFY: if (!userIf->pollPtr) { - /* This should always happen on ESX. */ return -EINVAL; } VNetUserIfUnsetupNotify(userIf); @@ -1110,13 +1062,10 @@ VNetUserIf_Create(VNetPort **ret) // OUT userIf->port.jack.portsChanged = NULL; userIf->port.jack.isBridged = NULL; userIf->pollPtr = NULL; - userIf->actionIntr = NULL; userIf->recvClusterCount = NULL; userIf->pollPage = NULL; - userIf->actPage = NULL; userIf->recvClusterPage = NULL; userIf->pollMask = 0; - userIf->actionID = -1; userIf->port.exactFilterLen = 0; userIf->eventSender = NULL; diff --git a/vmnet-only/vm_assert.h b/vmnet-only/vm_assert.h index b368e1d..8cdbc93 100644 --- a/vmnet-only/vm_assert.h +++ b/vmnet-only/vm_assert.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -99,24 +99,13 @@ void Warning(const char *fmt, ...) PRINTF_DECL(1, 2); #if defined VMKPANIC void Panic_SaveRegs(void); -#ifdef VMX86_DEBUG -void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2); -#else NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2); -#endif - -NORETURN void Panic_NoSaveNoReturn(const char *fmt, ...) PRINTF_DECL(1, 2); #define Panic(fmt...) do { \ Panic_SaveRegs(); \ Panic_NoSave(fmt); \ } while(0) -#define Panic_NoReturn(fmt...) do { \ - Panic_SaveRegs(); \ - Panic_NoSaveNoReturn(fmt); \ -} while(0) - #else NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2); #endif @@ -176,8 +165,6 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3); #define ASSERT_NOT_IMPLEMENTED(cond) \ ASSERT_IFNOT(cond, NOT_IMPLEMENTED()) -#define ASSERT_NOT_IMPLEMENTED_BUG(bug, cond) \ - ASSERT_IFNOT(cond, NOT_IMPLEMENTED_BUG(bug)) #if defined VMKPANIC || defined VMM #define NOT_IMPLEMENTED() _ASSERT_PANIC_NORETURN(AssertNotImplemented) @@ -202,10 +189,8 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3); ASSERT_IFNOT(cond, _ASSERT_PANIC(AssertMemAlloc)) #ifdef VMX86_DEVEL -#define ASSERT_DEVEL(cond) ASSERT(cond) #define NOT_TESTED() Warning(_AssertNotTestedFmt "\n", __FILE__, __LINE__) #else -#define ASSERT_DEVEL(cond) ((void)0) #define NOT_TESTED() Log(_AssertNotTestedFmt "\n", __FILE__, __LINE__) #endif @@ -305,7 +290,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3); #define ASSERT_ON_COMPILE(e) \ do { \ _Static_assert(e, #e); \ - } while (0); + } while (0) #endif /* diff --git a/vmnet-only/vm_atomic.h b/vmnet-only/vm_atomic.h index 07f32ea..38d45fd 100644 --- a/vmnet-only/vm_atomic.h +++ b/vmnet-only/vm_atomic.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -32,7 +32,6 @@ #define _ATOMIC_H_ #define INCLUDE_ALLOW_USERLEVEL - #define INCLUDE_ALLOW_MODULE #define INCLUDE_ALLOW_VMMON #define INCLUDE_ALLOW_VMKDRIVERS @@ -45,6 +44,10 @@ #include "vm_basic_types.h" #include "vm_assert.h" +#if defined __cplusplus +extern "C" { +#endif + /* * In the Atomic_* definitions below, memory ordering and atomicity are somewhat * conflated in an inconsistent manner. First, we have Atomic_{Read,Write}, @@ -84,153 +87,15 @@ */ #ifdef VM_ARM_64 -#include "vm_basic_asm_arm64.h" - -#define ARM_64_ATOMIC_16_OPV_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint16 _sample; \ - __asm__ __volatile__ ( \ - "1: ldxrh %w0, [%2] \n\t" \ - #op " %w0, %w0, %w3 \n\t" \ - " stxrh %w1, %w0, [%2] \n\t" \ - " cbnz %w1, 1b \n\t" \ - : "=&r" (_sample), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ -}) - -#define ARM_64_ATOMIC_16_OPO_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint16 _newval; \ - uint16 _oldval; \ - __asm__ __volatile__ ( \ - "1: ldxrh %w0, [%3] \n\t" \ - #op " %w1, %w0, %w4 \n\t" \ - " stxrh %w2, %w1, [%3] \n\t" \ - " cbnz %w2, 1b \n\t" \ - : "=&r" (_oldval), \ - "=&r" (_newval), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ - _oldval; \ -}) - -#define ARM_64_ATOMIC_32_OPV_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint32 _sample; \ - __asm__ __volatile__ ( \ - "1: ldxr %w0, [%2] \n\t" \ - #op " %w0, %w0, %w3 \n\t" \ - " stxr %w1, %w0, [%2] \n\t" \ - " cbnz %w1, 1b \n\t" \ - : "=&r" (_sample), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ -}) - -#define ARM_64_ATOMIC_32_OPO_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint32 _newval; \ - uint32 _oldval; \ - __asm__ __volatile__ ( \ - "1: ldxr %w0, [%3] \n\t" \ - #op " %w1, %w0, %w4 \n\t" \ - " stxr %w2, %w1, [%3] \n\t" \ - " cbnz %w2, 1b \n\t" \ - : "=&r" (_oldval), \ - "=&r" (_newval), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ - _oldval; \ -}) - -#define ARM_64_ATOMIC_64_OPV_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint64 _sample; \ - __asm__ __volatile__ ( \ - "1: ldxr %x0, [%2] \n\t" \ - #op " %x0, %x0, %x3 \n\t" \ - " stxr %w1, %x0, [%2] \n\t" \ - " cbnz %w1, 1b \n\t" \ - : "=&r" (_sample), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ -}) - -#define ARM_64_ATOMIC_64_OPO_PRIVATE(atm,modval,op) ({ \ - uint32 _failed; \ - uint64 _newval; \ - uint64 _oldval; \ - __asm__ __volatile__ ( \ - "1: ldxr %x0, [%3] \n\t" \ - #op " %x1, %x0, %x4 \n\t" \ - " stxr %w2, %x1, [%3] \n\t" \ - " cbnz %w2, 1b \n\t" \ - : "=&r" (_oldval), \ - "=&r" (_newval), \ - "=&r" (_failed) \ - : "r" (atm), \ - "r" (modval) \ - : "memory"); \ - _oldval; \ -}) - -#define ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(atm,modval,op) ({ \ - DMB(); \ - ARM_64_ATOMIC_16_OPV_PRIVATE(atm,modval,op); \ - DMB(); \ -}) - -#define ARM_64_ATOMIC_16_OPO_PRIVATE_FENCED(atm,modval,op) ({ \ - uint16 _oldval_fenced; \ - DMB(); \ - _oldval_fenced = ARM_64_ATOMIC_16_OPO_PRIVATE(atm,modval,op); \ - DMB(); \ - _oldval_fenced; \ -}) - -#define ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(atm,modval,op) ({ \ - DMB(); \ - ARM_64_ATOMIC_32_OPV_PRIVATE(atm,modval,op); \ - DMB(); \ -}) - -#define ARM_64_ATOMIC_32_OPO_PRIVATE_FENCED(atm,modval,op) ({ \ - uint32 _oldval_fenced; \ - DMB(); \ - _oldval_fenced = ARM_64_ATOMIC_32_OPO_PRIVATE(atm,modval,op); \ - DMB(); \ - _oldval_fenced; \ -}) - -#define ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(atm,modval,op) ({ \ - DMB(); \ - ARM_64_ATOMIC_64_OPV_PRIVATE(atm,modval,op); \ - DMB(); \ -}) - -#define ARM_64_ATOMIC_64_OPO_PRIVATE_FENCED(atm,modval,op) ({ \ - uint64 _oldval_fenced; \ - DMB(); \ - _oldval_fenced = ARM_64_ATOMIC_64_OPO_PRIVATE(atm,modval,op); \ - DMB(); \ - _oldval_fenced; \ -}) - -#endif /* VM_ARM_64 */ +# include "vm_atomic_arm64_begin.h" +#endif -/* Basic atomic types: 16, 32 and 64 bits */ +/* Basic atomic types: 8, 16, 32, 64 and 128 bits */ +typedef struct Atomic_uint8 { + volatile uint8 value; +} Atomic_uint8 ALIGNED(1); + typedef struct Atomic_uint16 { volatile uint16 value; } Atomic_uint16 ALIGNED(2); @@ -239,10 +104,17 @@ typedef struct Atomic_uint32 { volatile uint32 value; } Atomic_uint32 ALIGNED(4); -typedef struct Atomic_uint64 { +typedef struct Atomic_uint64 { volatile uint64 value; } Atomic_uint64 ALIGNED(8); +#if defined __GNUC__ && defined VM_64BIT && \ + (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64) +typedef struct Atomic_uint128 { + volatile uint128 value; +} Atomic_uint128 ALIGNED(16); +#endif + /* * Prototypes for msft atomics. These are defined & inlined by the * compiler so no function definition is needed. The prototypes are @@ -258,7 +130,7 @@ typedef struct Atomic_uint64 { * Windows typedefs. This avoids having to include windows header files * to get to the windows types. */ -#if defined(_MSC_VER) && _MSC_VER >= 1310 && !defined(BORA_NO_WIN32_INTRINS) +#if defined _MSC_VER && _MSC_VER >= 1310 && !defined BORA_NO_WIN32_INTRINS #ifdef __cplusplus extern "C" { #endif @@ -267,10 +139,11 @@ long _InterlockedCompareExchange(long volatile*, long, long); long _InterlockedExchangeAdd(long volatile*, long); long _InterlockedDecrement(long volatile*); long _InterlockedIncrement(long volatile*); -void _ReadWriteBarrier(void); +__int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); #pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange) #pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement) -#pragma intrinsic(_InterlockedIncrement, _ReadWriteBarrier) +#pragma intrinsic(_InterlockedIncrement) +#pragma intrinsic(_InterlockedCompareExchange64) # if _MSC_VER >= 1600 char _InterlockedExchange8(char volatile *, char); @@ -278,7 +151,7 @@ char _InterlockedCompareExchange8(char volatile *, char, char); #pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8) #endif -#if defined(VM_X86_64) +#if defined VM_X86_64 long _InterlockedAnd(long volatile*, long); __int64 _InterlockedAnd64(__int64 volatile*, __int64); long _InterlockedOr(long volatile*, long); @@ -289,14 +162,12 @@ __int64 _InterlockedExchangeAdd64(__int64 volatile*, __int64); __int64 _InterlockedIncrement64(__int64 volatile*); __int64 _InterlockedDecrement64(__int64 volatile*); __int64 _InterlockedExchange64(__int64 volatile*, __int64); -__int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); -#if !defined(_WIN64) +#if !defined _WIN64 #pragma intrinsic(_InterlockedAnd, _InterlockedAnd64) #pragma intrinsic(_InterlockedOr, _InterlockedOr64) #pragma intrinsic(_InterlockedXor, _InterlockedXor64) #pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64) #pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64) -#pragma intrinsic(_InterlockedCompareExchange64) #endif /* !_WIN64 */ #endif /* __x86_64__ */ @@ -305,7 +176,7 @@ __int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); #endif #endif /* _MSC_VER */ -#if defined(__arm__) +#if defined __arm__ /* * LDREX without STREX or CLREX may cause problems in environments where the * context switch may not clear the reference monitor - according ARM manual @@ -313,8 +184,8 @@ __int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); * may not like Linux kernel's non-preemptive context switch path. So use of * ARM routines in kernel code may not be safe. */ -# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ - defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__) +# if defined __ARM_ARCH_7__ || defined __ARM_ARCH_7A__ || \ + defined __ARM_ARCH_7R__|| defined __ARM_ARCH_7M__ # define VM_ARM_V7 # ifdef __KERNEL__ # warning LDREX/STREX may not be safe in linux kernel, since it \ @@ -335,10 +206,12 @@ __int64 _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64); /* Convert a volatile uint32 to Atomic_uint32. */ static INLINE Atomic_uint32 * -Atomic_VolatileToAtomic(volatile uint32 *var) // IN: +Atomic_VolatileToAtomic32(volatile uint32 *var) // IN: { return (Atomic_uint32 *)var; } +#define Atomic_VolatileToAtomic Atomic_VolatileToAtomic32 + /* Convert a volatile uint64 to Atomic_uint64. */ static INLINE Atomic_uint64 * @@ -347,135 +220,6 @@ Atomic_VolatileToAtomic64(volatile uint64 *var) // IN: return (Atomic_uint64 *)var; } -/* - *----------------------------------------------------------------------------- - * - * Atomic_Init, Atomic_SetFence, AtomicUseFence -- - * - * Determine whether an lfence intruction is executed after - * every locked instruction. - * - * Certain AMD processors have a bug (see bug 107024) that - * requires an lfence after every locked instruction. - * - * The global variable AtomicUseFence controls whether lfence - * is used (see AtomicEpilogue). - * - * Atomic_SetFence sets AtomicUseFence to the given value. - * - * Atomic_Init computes and sets AtomicUseFence for x86. - * It does not take into account the number of processors. - * - * The rationale for all this complexity is that Atomic_Init - * is the easy-to-use interface. It can be called a number - * of times cheaply, and does not depend on other libraries. - * However, because the number of CPUs is difficult to compute, - * it does without it and always assumes there are more than one. - * - * For programs that care or have special requirements, - * Atomic_SetFence can be called directly, in addition to Atomic_Init. - * It overrides the effect of Atomic_Init, and can be called - * before, after, or between calls to Atomic_Init. - * - *----------------------------------------------------------------------------- - */ - -// The freebsd assembler doesn't know the lfence instruction -#if defined(__GNUC__) && \ - __GNUC__ >= 3 && \ - (defined(__VMKERNEL__) || !defined(__FreeBSD__)) && \ - (!defined(MODULE) || defined(__VMKERNEL_MODULE__)) && \ - !defined(__APPLE__) && \ - (defined(__i386__) || defined(__x86_64__)) /* PR136775 */ -#define ATOMIC_USE_FENCE -#endif - -/* - * Starting with vSphere 2014, we no longer support ESX on AMD Rev F. - * Thus, we can eliminate all dynamic checks for whether to enable - * the Errata 147 work-around when compiling many of our binaries. - * However, we use an opt-in approach here rather than assuming all - * parts of our builds are safe. For example, the "fdm" binary from - * a new build may time travel back to hosts running older versions - * of ESX on Rev F, so "fdm" continues to require the ability to - * dynamically enable the errata work-around. With vSphere 2017, - * this will no longer be required as the oldest version of ESX that - * VC 2017 will support is ESX 2014 (which won't run on Rev F). - * - * Modules may explicitly define MAY_NEED_AMD_REVF_WORKAROUND as 0 prior to - * inclusion of vm_atomic.h when they are safe on AMD Rev F with the elided - * lfence. - */ -#if !defined(MAY_NEED_AMD_REVF_WORKAROUND) -#if (!defined(VMX86_SERVER) || \ - (!defined(VMX86_VMX) && !defined(VMKERNEL) && \ - !defined(VMM) && !defined(VMCORE))) -#define MAY_NEED_AMD_REVF_WORKAROUND 1 -#else -#define MAY_NEED_AMD_REVF_WORKAROUND 0 -#endif -#endif - -#if MAY_NEED_AMD_REVF_WORKAROUND -#if defined(VMATOMIC_IMPORT_DLLDATA) -VMX86_EXTERN_DATA Bool AtomicUseFence; -#else -EXTERN Bool AtomicUseFence; -#endif -EXTERN Bool atomicFenceInitialized; -#else /* MAY_NEED_AMD_REVF_WORKAROUND */ -#define AtomicUseFence FALSE -#define atomicFenceInitialized TRUE -#endif /* MAY_NEED_AMD_REVF_WORKAROUND */ - - -void AtomicInitFence(void); - -static INLINE void -Atomic_Init(void) -{ -#ifdef ATOMIC_USE_FENCE - if (MAY_NEED_AMD_REVF_WORKAROUND && !atomicFenceInitialized) { - AtomicInitFence(); - } -#endif -} - -static INLINE void -Atomic_SetFence(Bool fenceAfterLock) // IN: -{ - (void)fenceAfterLock; /* Work around unused parameter. */ -#if MAY_NEED_AMD_REVF_WORKAROUND - AtomicUseFence = fenceAfterLock; - atomicFenceInitialized = TRUE; -#endif -} - - -/* Conditionally execute fence after interlocked instruction. */ -static INLINE void -AtomicEpilogue(void) -{ -#if MAY_NEED_AMD_REVF_WORKAROUND && defined(ATOMIC_USE_FENCE) -#ifdef VMM - /* The monitor conditionally patches out the lfence when not needed.*/ - /* Construct a MonitorPatchTextEntry in the .patchtext section. */ - asm volatile ("1:\n\t" - "lfence\n\t" - "2:\n\t" - ".pushsection .patchtext\n\t" - ".quad 1b\n\t" - ".quad 2b\n\t" - ".quad 0\n\t" - ".popsection\n\t" ::: "memory"); -#else - if (UNLIKELY(AtomicUseFence)) { - asm volatile ("lfence" ::: "memory"); - } -#endif -#endif -} - /* * All the assembly code is tricky and written conservatively. @@ -512,26 +256,86 @@ AtomicEpilogue(void) * - walken */ -#if defined(_MSC_VER) && _MSC_VER < 1600 && defined(__x86_64__) -Bool VMWInterlockedExchangeBool(Bool volatile *ptr, - Bool val); +#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__ +uint8 VMWInterlockedExchange8(uint8 volatile *ptr, + uint8 val); -Bool VMWInterlockedCompareExchangeBool(Bool volatile *ptr, - Bool newVal, - Bool oldVal); +uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr, + uint8 newVal, + uint8 oldVal); #endif -typedef struct Atomic_Bool { - volatile Bool value; -} Atomic_Bool; +#if defined __GNUC__ && defined VM_ARM_32 +/* Force the link step to fail for unimplemented functions. */ +extern int AtomicUndefined(void const *); +#endif -/* The ARM team can come along and add the code real soon now */ -extern Bool AtomicUndefinedOnARM(void); /* *----------------------------------------------------------------------------- * - * Atomic_ReadBool -- + * Atomic_ReadIfEqualWrite128 -- + * + * Compare and exchange a 16 byte tuple. + * + * Results: + * old value + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ +#if defined __GNUC__ && defined VM_64BIT && \ + (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64) +static INLINE uint128 +Atomic_ReadIfEqualWrite128(Atomic_uint128 *ptr, // IN/OUT + uint128 oldVal, // IN + uint128 newVal) // IN +{ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 + return __sync_val_compare_and_swap(&ptr->value, oldVal, newVal); +#elif defined VM_ARM_64 + union { + uint128 raw; + struct { + uint64 lo; + uint64 hi; + }; + } res, _old = { oldVal }, _new = { newVal }; + uint32 failed; + + SMP_RW_BARRIER_RW(); + __asm__ __volatile__( + "1: ldxp %x0, %x1, %3 \n\t" + " cmp %x0, %x4 \n\t" + " ccmp %x1, %x5, #0, eq \n\t" + " b.ne 2f \n\t" + " stxp %w2, %x6, %x7, %3 \n\t" + " cbnz %w2, 1b \n\t" + "2: \n\t" + : "=&r" (res.lo), + "=&r" (res.hi), + "=&r" (failed), + "+Q" (ptr->value) + : "r" (_old.lo), + "r" (_old.hi), + "r" (_new.lo), + "r" (_new.hi) + : "cc" + ); + SMP_RW_BARRIER_RW(); + + return res.raw; +#endif +} +#endif + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Read8 -- * * Read the value of the specified object atomically. * @@ -544,23 +348,25 @@ extern Bool AtomicUndefinedOnARM(void); *----------------------------------------------------------------------------- */ -static INLINE Bool -Atomic_ReadBool(Atomic_Bool const *var) // IN: +static INLINE uint8 +Atomic_Read8(Atomic_uint8 const *var) // IN: { - Bool val; + uint8 val; -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - val = AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#if defined __GNUC__ && defined VM_ARM_32 + val = AtomicUndefined(var); +#elif defined VM_ARM_64 + val = _VMATOM_X(R, 8, &var->value); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) __asm__ __volatile__( "movb %1, %0" : "=q" (val) : "m" (var->value) ); -#elif defined(_MSC_VER) +#elif defined _MSC_VER val = var->value; #else -#error No compiler defined for Atomic_ReadBool +#error No compiler defined for Atomic_Read8 #endif return val; @@ -570,7 +376,7 @@ Atomic_ReadBool(Atomic_Bool const *var) // IN: /* *----------------------------------------------------------------------------- * - * Atomic_ReadWriteBool -- + * Atomic_ReadWrite8 -- * * Read followed by write. * @@ -583,36 +389,37 @@ Atomic_ReadBool(Atomic_Bool const *var) // IN: *----------------------------------------------------------------------------- */ -static INLINE Bool -Atomic_ReadWriteBool(Atomic_Bool *var, // IN/OUT: - Bool val) // IN: +static INLINE uint8 +Atomic_ReadWrite8(Atomic_uint8 *var, // IN/OUT: + uint8 val) // IN: { -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - return AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#if defined __GNUC__ && defined VM_ARM_32 + return AtomicUndefined(var + val); +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 8, TRUE, &var->value, val); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) __asm__ __volatile__( "xchgb %0, %1" : "=q" (val), "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; -#elif defined(_MSC_VER) && _MSC_VER >= 1600 - return _InterlockedExchange8(&var->value, val); -#elif defined(_MSC_VER) && defined(__i386__) +#elif defined _MSC_VER && _MSC_VER >= 1600 + return _InterlockedExchange8((volatile char *)&var->value, val); +#elif defined _MSC_VER && defined __i386__ #pragma warning(push) #pragma warning(disable : 4035) // disable no-return warning { __asm movzx eax, val __asm mov ebx, var - __asm xchg [ebx]Atomic_Bool.value, al + __asm xchg [ebx]Atomic_uint8.value, al } #pragma warning(pop) -#elif defined(_MSC_VER) && defined(__x86_64__) - return VMWInterlockedExchangeBool(&var->value, val); +#elif defined _MSC_VER && defined __x86_64__ + return VMWInterlockedExchange8(&var->value, val); #else -#error No compiler defined for Atomic_ReadBool +#error No compiler defined for Atomic_ReadWrite8 #endif } @@ -620,7 +427,7 @@ Atomic_ReadWriteBool(Atomic_Bool *var, // IN/OUT: /* *----------------------------------------------------------------------------- * - * Atomic_WriteBool -- + * Atomic_Write8 -- * * Write the specified value to the specified object atomically. * @@ -634,21 +441,23 @@ Atomic_ReadWriteBool(Atomic_Bool *var, // IN/OUT: */ static INLINE void -Atomic_WriteBool(Atomic_Bool *var, // IN/OUT: - Bool val) // IN: +Atomic_Write8(Atomic_uint8 *var, // IN/OUT: + uint8 val) // IN: { -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#if defined __GNUC__ && defined VM_ARM_32 + AtomicUndefined(var + val); +#elif defined VM_ARM_64 + _VMATOM_X(W, 8, &var->value, val); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) __asm__ __volatile__( "movb %1, %0" : "=m" (var->value) : "qn" (val) ); -#elif defined(_MSC_VER) +#elif defined _MSC_VER var->value = val; #else -#error No compiler defined for Atomic_WriteBool +#error No compiler defined for Atomic_Write8 #endif } @@ -656,7 +465,7 @@ Atomic_WriteBool(Atomic_Bool *var, // IN/OUT: /* *----------------------------------------------------------------------------- * - * Atomic_ReadIfEqualWriteBool -- + * Atomic_ReadIfEqualWrite8 -- * * Compare exchange: Read variable, if equal to oldVal, write newVal. * @@ -669,15 +478,17 @@ Atomic_WriteBool(Atomic_Bool *var, // IN/OUT: *----------------------------------------------------------------------------- */ -static INLINE Bool -Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: - Bool oldVal, // IN: - Bool newVal) // IN: +static INLINE uint8 +Atomic_ReadIfEqualWrite8(Atomic_uint8 *var, // IN/OUT: + uint8 oldVal, // IN: + uint8 newVal) // IN: { -#if defined(__GNUC__) && (defined(VM_ARM_32) || defined(VM_ARM_64)) - return AtomicUndefinedOnARM(); -#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - Bool val; +#if defined __GNUC__ && defined VM_ARM_32 + return AtomicUndefined(var + oldVal + newVal); +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 8, TRUE, &var->value, oldVal, newVal); +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) + uint8 val; __asm__ __volatile__( "lock; cmpxchgb %2, %1" @@ -687,26 +498,27 @@ Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: "0" (oldVal) : "cc" ); - AtomicEpilogue(); + return val; -#elif defined(_MSC_VER) && _MSC_VER >= 1600 - return _InterlockedCompareExchange8(&var->value, newVal, oldVal); -#elif defined(_MSC_VER) && defined(__i386__) +#elif defined _MSC_VER && _MSC_VER >= 1600 + return _InterlockedCompareExchange8((volatile char *)&var->value, + newVal, oldVal); +#elif defined _MSC_VER && defined __i386__ #pragma warning(push) #pragma warning(disable : 4035) // disable no-return warning { __asm mov al, oldVal __asm mov ebx, var __asm mov cl, newVal - __asm lock cmpxchg [ebx]Atomic_Bool.value, cl + __asm lock cmpxchg [ebx]Atomic_uint8.value, cl __asm movzx eax, al // eax is the return value, this is documented to work - edward } #pragma warning(pop) -#elif defined(_MSC_VER) && defined(__x86_64__) - return VMWInterlockedCompareExchangeBool(&var->value, newVal, oldVal); +#elif defined _MSC_VER && defined __x86_64__ + return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal); #else -#error No compiler defined for Atomic_ReadIfEqualWriteBool +#error No compiler defined for Atomic_ReadIfEqualWrite8 #endif } @@ -714,7 +526,375 @@ Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: /* *----------------------------------------------------------------------------- * - * Atomic_Read -- + * Atomic_ReadAnd8 -- + * + * Atomic read (returned), bitwise AND with a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadAnd8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, and, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res & val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_And8 -- + * + * Atomic read, bitwise AND with a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_And8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, and, val); +#else + (void)Atomic_ReadAnd8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadOr8 -- + * + * Atomic read (returned), bitwise OR with a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadOr8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, orr, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res | val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Or8 -- + * + * Atomic read, bitwise OR with a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Or8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, orr, val); +#else + (void)Atomic_ReadOr8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadXor8 -- + * + * Atomic read (returned), bitwise XOR with a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadXor8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, eor, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res ^ val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Xor8 -- + * + * Atomic read, bitwise XOR with a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Xor8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, eor, val); +#else + (void)Atomic_ReadXor8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadAdd8 -- + * + * Atomic read (returned), add a value, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadAdd8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ + uint8 res; + +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 8, TRUE, &var->value, add, val); +#else + do { + res = Atomic_Read8(var); + } while (res != Atomic_ReadIfEqualWrite8(var, res, res + val)); +#endif + + return res; +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Add8 -- + * + * Atomic read, add a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Add8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, add, val); +#else + (void)Atomic_ReadAdd8(var, val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Sub8 -- + * + * Atomic read, subtract a value, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Sub8(Atomic_uint8 *var, // IN/OUT + uint8 val) // IN +{ +#if defined VM_ARM_64 + _VMATOM_X(OP, 8, TRUE, &var->value, sub, val); +#else + Atomic_Add8(var, -val); +#endif +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Inc8 -- + * + * Atomic read, increment, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Inc8(Atomic_uint8 *var) // IN/OUT +{ + Atomic_Add8(var, 1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Dec8 -- + * + * Atomic read, decrement, write. + * + * Results: + * None + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE void +Atomic_Dec8(Atomic_uint8 *var) // IN/OUT +{ + Atomic_Sub8(var, 1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadInc8 -- + * + * Atomic read (returned), increment, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadInc8(Atomic_uint8 *var) // IN/OUT +{ + return Atomic_ReadAdd8(var, 1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_ReadDec8 -- + * + * Atomic read (returned), decrement, write. + * + * Results: + * The value of the variable before the operation. + * + * Side effects: + * None + * + *----------------------------------------------------------------------------- + */ + +static INLINE uint8 +Atomic_ReadDec8(Atomic_uint8 *var) // IN/OUT +{ + return Atomic_ReadAdd8(var, (uint8)-1); +} + + +/* + *----------------------------------------------------------------------------- + * + * Atomic_Read32 -- * * Read * @@ -728,33 +908,29 @@ Atomic_ReadIfEqualWriteBool(Atomic_Bool *var, // IN/OUT: */ static INLINE uint32 -Atomic_Read(Atomic_uint32 const *var) // IN +Atomic_Read32(Atomic_uint32 const *var) // IN { uint32 value; -#if defined(VMM) +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT(((uintptr_t)var % 4) == 0); #endif -#if defined(__GNUC__) +#if defined __GNUC__ /* * Use inline assembler to force using a single load instruction to * ensure that the compiler doesn't split a transfer operation into multiple * instructions. */ -#if defined(VM_ARM_32) +#if defined VM_ARM_32 __asm__ __volatile__( "ldr %0, [%1]" : "=r" (value) : "r" (&var->value) ); -#elif defined(VM_ARM_64) - __asm__ __volatile__ ( - "ldr %w0, [%1]" - : "=r" (value) - : "r" (&var->value) - ); +#elif defined VM_ARM_64 + value = _VMATOM_X(R, 32, &var->value); #else __asm__ __volatile__( "mov %1, %0" @@ -762,7 +938,7 @@ Atomic_Read(Atomic_uint32 const *var) // IN : "m" (var->value) ); #endif -#elif defined(_MSC_VER) +#elif defined _MSC_VER /* * Microsoft docs guarantee simple reads and writes to properly * aligned 32-bit variables use only a single instruction. @@ -776,13 +952,13 @@ Atomic_Read(Atomic_uint32 const *var) // IN return value; } -#define Atomic_Read32 Atomic_Read +#define Atomic_Read Atomic_Read32 /* *----------------------------------------------------------------------------- * - * Atomic_ReadWrite -- + * Atomic_ReadWrite32 -- * * Read followed by write * @@ -796,10 +972,10 @@ Atomic_Read(Atomic_uint32 const *var) // IN */ static INLINE uint32 -Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_ReadWrite32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 retVal; register volatile uint32 res; @@ -819,22 +995,8 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint32 retVal; - register uint32 failed; - - __asm__ __volatile__( - " dmb sy \n\t" - "1: ldxr %w0, [%2] \n\t" - " stxr %w1, %w3, [%2] \n\t" - " cbnz %w1, 1b \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (val) - : "memory" - ); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 32, TRUE, &var->value, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -843,7 +1005,6 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; #endif /* VM_X86_ANY */ #elif defined _MSC_VER @@ -864,13 +1025,13 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_ReadWrite #endif // __GNUC__ } -#define Atomic_ReadWrite32 Atomic_ReadWrite +#define Atomic_ReadWrite Atomic_ReadWrite32 /* *----------------------------------------------------------------------------- * - * Atomic_Write -- + * Atomic_Write32 -- * * Write * @@ -884,16 +1045,21 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Write(Atomic_uint32 *var, // OUT - uint32 val) // IN +Atomic_Write32(Atomic_uint32 *var, // OUT + uint32 val) // IN { -#if defined(VMM) +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT(((uintptr_t)var % 4) == 0); #endif -#if defined(__GNUC__) -#if defined(VM_ARM_ANY) +#if defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(W, 32, &var->value, val); +#elif defined VM_ARM_32 /* + * Best left this way due to the intricacies of exclusive load/store + * operations on legacy (32-bit) ARM. + * * A3.4.1 ARM DDI 0406C: * * When a processor writes using any instruction other than a @@ -915,7 +1081,7 @@ Atomic_Write(Atomic_uint32 *var, // OUT * - it is not necessary for every LDREX to have a subsequent STREX. */ - Atomic_ReadWrite(var, val); + Atomic_ReadWrite32(var, val); #else /* * Use inline assembler to force using a single store instruction to @@ -929,7 +1095,7 @@ Atomic_Write(Atomic_uint32 *var, // OUT : "r" (val) ); #endif -#elif defined(_MSC_VER) +#elif defined _MSC_VER /* * Microsoft docs guarantee simple reads and writes to properly * aligned 32-bit variables use only a single instruction. @@ -941,13 +1107,13 @@ Atomic_Write(Atomic_uint32 *var, // OUT #error No compiler defined for Atomic_Write #endif } -#define Atomic_Write32 Atomic_Write +#define Atomic_Write Atomic_Write32 /* *----------------------------------------------------------------------------- * - * Atomic_ReadIfEqualWrite -- + * Atomic_ReadIfEqualWrite32 -- * * Compare exchange: Read variable, if equal to oldVal, write newVal * @@ -961,11 +1127,11 @@ Atomic_Write(Atomic_uint32 *var, // OUT */ static INLINE uint32 -Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT - uint32 oldVal, // IN - uint32 newVal) // IN +Atomic_ReadIfEqualWrite32(Atomic_uint32 *var, // IN/OUT + uint32 oldVal, // IN + uint32 newVal) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register uint32 retVal; register uint32 res; @@ -987,24 +1153,8 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint32 retVal; - register uint32 failed; - - __asm__ __volatile__ ( - " dmb sy \n\t" - "1: ldxr %w0, [%2] \n\t" - " cmp %w0, %w3 \n\t" - " b.ne 2f \n\t" - " stxr %w1, %w4, [%2] \n\t" - " cbnz %w1, 1b \n\t" - "2: clrex \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (oldVal), "r" (newVal) - : "cc", "memory"); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 32, TRUE, &var->value, oldVal, newVal); #else /* VM_X86_ANY */ uint32 val; @@ -1017,7 +1167,6 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT "0" (oldVal) : "cc" ); - AtomicEpilogue(); return val; #endif /* VM_X86_ANY */ #elif defined _MSC_VER @@ -1041,10 +1190,10 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_ReadIfEqualWrite #endif } -#define Atomic_ReadIfEqualWrite32 Atomic_ReadIfEqualWrite +#define Atomic_ReadIfEqualWrite Atomic_ReadIfEqualWrite32 -#if defined(VM_64BIT) || defined(VM_ARM_V7) +#if defined VM_64BIT || defined VM_ARM_V7 /* *----------------------------------------------------------------------------- * @@ -1066,7 +1215,7 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT uint64 oldVal, // IN uint64 newVal) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register uint64 retVal; register uint32 res; @@ -1087,7 +1236,7 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT * Furthermore, using a 32-bits register to store a * 64-bits value of an variable looks risky. */ -#if defined(__APPLE__) && __clang__ == 1 && __clang_major__ >= 5 +#if defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5 #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wasm-operand-widths" #endif @@ -1103,30 +1252,14 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal) : "cc" ); -#if defined(__APPLE__) && __clang__ == 1 && __clang_major__ >= 5 +#if defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5 #pragma clang diagnostic pop -#endif // defined(__APPLE__) && __clang__ == 1 && __clang_major__ >= 5 +#endif // defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5 dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint64 retVal; - register uint32 failed; - - __asm__ __volatile__ ( - " dmb sy \n\t" - "1: ldxr %x0, [%2] \n\t" - " cmp %x0, %x3 \n\t" - " b.ne 2f \n\t" - " stxr %w1, %x4, [%2] \n\t" - " cbnz %w1, 1b \n\t" - "2: clrex \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (oldVal), "r" (newVal) - : "cc", "memory"); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 64, TRUE, &var->value, oldVal, newVal); #else /* VM_X86_64 */ uint64 val; @@ -1139,7 +1272,6 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT "0" (oldVal) : "cc" ); - AtomicEpilogue(); return val; #endif //VM_ARM_V7 #elif defined _MSC_VER @@ -1156,7 +1288,7 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT /* *----------------------------------------------------------------------------- * - * Atomic_And -- + * Atomic_And32 -- * * Atomic read, bitwise AND with a value, write. * @@ -1170,10 +1302,10 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT */ static INLINE void -Atomic_And(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_And32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1192,8 +1324,8 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, and); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, and, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1202,10 +1334,9 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER -#if defined(__x86_64__) +#if defined __x86_64__ || defined VM_ARM_32 _InterlockedAnd((long *)&var->value, (long)val); #else __asm mov eax, val @@ -1216,13 +1347,13 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_And #endif } -#define Atomic_And32 Atomic_And +#define Atomic_And Atomic_And32 /* *----------------------------------------------------------------------------- * - * Atomic_Or -- + * Atomic_Or32 -- * * Atomic read, bitwise OR with a value, write. * @@ -1236,10 +1367,10 @@ Atomic_And(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Or(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Or32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1258,8 +1389,8 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, orr); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, orr, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1268,10 +1399,9 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER -#if defined(__x86_64__) +#if defined __x86_64__ || defined VM_ARM_32 _InterlockedOr((long *)&var->value, (long)val); #else __asm mov eax, val @@ -1282,13 +1412,13 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Or #endif } -#define Atomic_Or32 Atomic_Or +#define Atomic_Or Atomic_Or32 /* *----------------------------------------------------------------------------- * - * Atomic_Xor -- + * Atomic_Xor32 -- * * Atomic read, bitwise XOR with a value, write. * @@ -1302,10 +1432,10 @@ Atomic_Or(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Xor(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Xor32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1324,8 +1454,8 @@ Atomic_Xor(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, eor); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, eor, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1334,10 +1464,9 @@ Atomic_Xor(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER -#if defined(__x86_64__) +#if defined __x86_64__ || defined VM_ARM_32 _InterlockedXor((long *)&var->value, (long)val); #else __asm mov eax, val @@ -1348,10 +1477,10 @@ Atomic_Xor(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Xor #endif } -#define Atomic_Xor32 Atomic_Xor +#define Atomic_Xor Atomic_Xor32 -#if defined(VM_64BIT) +#if defined VM_64BIT /* *----------------------------------------------------------------------------- * @@ -1372,9 +1501,9 @@ static INLINE void Atomic_Xor64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__GNUC__) -#if defined(VM_ARM_64) - ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, eor); +#if defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, eor, val); #else /* VM_X86_64 */ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( @@ -1383,7 +1512,6 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #endif #elif defined _MSC_VER _InterlockedXor64((__int64 *)&var->value, (__int64)val); @@ -1397,7 +1525,7 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT /* *----------------------------------------------------------------------------- * - * Atomic_Add -- + * Atomic_Add32 -- * * Atomic read, add a value, write. * @@ -1411,10 +1539,10 @@ Atomic_Xor64(Atomic_uint64 *var, // IN/OUT */ static INLINE void -Atomic_Add(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Add32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1433,8 +1561,8 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, add, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1443,7 +1571,6 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1457,13 +1584,13 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Add #endif } -#define Atomic_Add32 Atomic_Add +#define Atomic_Add Atomic_Add32 /* *----------------------------------------------------------------------------- * - * Atomic_Sub -- + * Atomic_Sub32 -- * * Atomic read, subtract a value, write. * @@ -1477,10 +1604,10 @@ Atomic_Add(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Sub(Atomic_uint32 *var, // IN/OUT - uint32 val) // IN +Atomic_Sub32(Atomic_uint32 *var, // IN/OUT + uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 tmp; @@ -1499,8 +1626,8 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, sub); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 32, TRUE, &var->value, sub, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1509,7 +1636,6 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT : "ri" (val) : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1523,13 +1649,13 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT #error No compiler defined for Atomic_Sub #endif } -#define Atomic_Sub32 Atomic_Sub +#define Atomic_Sub Atomic_Sub32 /* *----------------------------------------------------------------------------- * - * Atomic_Inc -- + * Atomic_Inc32 -- * * Atomic read, increment, write. * @@ -1543,11 +1669,11 @@ Atomic_Sub(Atomic_uint32 *var, // IN/OUT */ static INLINE void -Atomic_Inc(Atomic_uint32 *var) // IN/OUT +Atomic_Inc32(Atomic_uint32 *var) // IN/OUT { #ifdef __GNUC__ -#if defined(VM_ARM_ANY) - Atomic_Add(var, 1); +#if defined VM_ARM_ANY + Atomic_Add32(var, 1); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1556,7 +1682,6 @@ Atomic_Inc(Atomic_uint32 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1569,13 +1694,13 @@ Atomic_Inc(Atomic_uint32 *var) // IN/OUT #error No compiler defined for Atomic_Inc #endif } -#define Atomic_Inc32 Atomic_Inc +#define Atomic_Inc Atomic_Inc32 /* *----------------------------------------------------------------------------- * - * Atomic_Dec -- + * Atomic_Dec32 -- * * Atomic read, decrement, write. * @@ -1589,11 +1714,11 @@ Atomic_Inc(Atomic_uint32 *var) // IN/OUT */ static INLINE void -Atomic_Dec(Atomic_uint32 *var) // IN/OUT +Atomic_Dec32(Atomic_uint32 *var) // IN/OUT { #ifdef __GNUC__ -#if defined(VM_ARM_ANY) - Atomic_Sub(var, 1); +#if defined VM_ARM_ANY + Atomic_Sub32(var, 1); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1602,7 +1727,6 @@ Atomic_Dec(Atomic_uint32 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #endif /* VM_X86_ANY */ #elif defined _MSC_VER #if _MSC_VER >= 1310 @@ -1615,7 +1739,7 @@ Atomic_Dec(Atomic_uint32 *var) // IN/OUT #error No compiler defined for Atomic_Dec #endif } -#define Atomic_Dec32 Atomic_Dec +#define Atomic_Dec Atomic_Dec32 /* @@ -1646,9 +1770,13 @@ Atomic_ReadOr32(Atomic_uint32 *var, // IN/OUT { uint32 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 32, TRUE, &var->value, orr, val); +#else do { - res = Atomic_Read(var); - } while (res != Atomic_ReadIfEqualWrite(var, res, res | val)); + res = Atomic_Read32(var); + } while (res != Atomic_ReadIfEqualWrite32(var, res, res | val)); +#endif return res; } @@ -1676,15 +1804,19 @@ Atomic_ReadAnd32(Atomic_uint32 *var, // IN/OUT { uint32 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 32, TRUE, &var->value, and, val); +#else do { - res = Atomic_Read(var); - } while (res != Atomic_ReadIfEqualWrite(var, res, res & val)); + res = Atomic_Read32(var); + } while (res != Atomic_ReadIfEqualWrite32(var, res, res & val)); +#endif return res; } -#if defined(VM_64BIT) +#if defined VM_64BIT /* *----------------------------------------------------------------------------- * @@ -1707,9 +1839,13 @@ Atomic_ReadOr64(Atomic_uint64 *var, // IN/OUT { uint64 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 64, TRUE, &var->value, orr, val); +#else do { res = var->value; } while (res != Atomic_ReadIfEqualWrite64(var, res, res | val)); +#endif return res; } @@ -1737,13 +1873,17 @@ Atomic_ReadAnd64(Atomic_uint64 *var, // IN/OUT { uint64 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 64, TRUE, &var->value, and, val); +#else do { res = var->value; } while (res != Atomic_ReadIfEqualWrite64(var, res, res & val)); +#endif return res; } -#endif /* defined(VM_64BIT) */ +#endif /* defined VM_64BIT */ /* @@ -1770,7 +1910,7 @@ static INLINE uint32 Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT uint32 val) // IN { -#if defined(__GNUC__) +#if defined __GNUC__ #ifdef VM_ARM_V7 register volatile uint32 res; register volatile uint32 retVal; @@ -1792,8 +1932,8 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - return ARM_64_ATOMIC_32_OPO_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + return _VMATOM_X(ROP, 32, TRUE, &var->value, add, val); #else /* VM_X86_ANY */ /* Checked against the Intel manual and GCC --walken */ __asm__ __volatile__( @@ -1803,7 +1943,6 @@ Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT : "0" (val) : "cc" ); - AtomicEpilogue(); return val; #endif /* VM_X86_ANY */ #elif defined _MSC_VER @@ -1871,51 +2010,6 @@ Atomic_ReadDec32(Atomic_uint32 *var) // IN/OUT } -#if defined VMKERNEL || defined VMM -#if !defined(VM_ARM_64) -/* - *----------------------------------------------------------------------------- - * - * CMPXCHG1B -- - * - * Compare and exchange a single byte. - * - * Results: - * The value read from ptr. - * - * Side effects: - * None - * - *----------------------------------------------------------------------------- - */ -static INLINE uint8 -CMPXCHG1B(volatile uint8 *ptr, // IN/OUT - uint8 oldVal, // IN - uint8 newVal) // IN -{ - uint8 val; - __asm__ __volatile__("lock; cmpxchgb %b2, %1" - : "=a" (val), - "+m" (*ptr) - : "r" (newVal), - "0" (oldVal) - : "cc"); - return val; -} -#endif /* !defined(VM_ARM_64) */ -#endif - - -/* - * Usage of this helper struct is strictly reserved to the following - * function. --hpreg - */ -typedef struct { - uint32 lowValue; - uint32 highValue; -} S_uint64; - - /* *----------------------------------------------------------------------------- * @@ -1923,12 +2017,6 @@ typedef struct { * * Compare exchange: Read variable, if equal to oldVal, write newVal * - * XXX: Ensure that if this function is to be inlined by gcc, it is - * compiled with -fno-strict-aliasing. Otherwise it will break. - * Unfortunately we know that gcc 2.95.3 (used to build the FreeBSD 3.2 - * Tools) does not honor -fno-strict-aliasing. As a workaround, we avoid - * inlining the function entirely for versions of gcc under 3.0. - * * Results: * TRUE if equal, FALSE if not equal * @@ -1938,23 +2026,19 @@ typedef struct { *----------------------------------------------------------------------------- */ -#if defined(__GNUC__) && __GNUC__ < 3 -static Bool -#else static INLINE Bool -#endif Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT - uint64 const *oldVal, // IN - uint64 const *newVal) // IN + uint64 oldVal, // IN + uint64 newVal) // IN { -#if defined(__GNUC__) -#if defined(VM_ARM_ANY) - return (Atomic_ReadIfEqualWrite64(var, *oldVal, *newVal) == *oldVal); +#if defined __GNUC__ +#if defined VM_ARM_ANY + return Atomic_ReadIfEqualWrite64(var, oldVal, newVal) == oldVal; #else /* VM_X86_ANY */ Bool equal; /* Checked against the Intel manual and GCC --walken */ -#if defined(__x86_64__) +#if defined __x86_64__ uint64 dummy; __asm__ __volatile__( "lock; cmpxchgq %3, %0" "\n\t" @@ -1962,11 +2046,16 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT : "+m" (*var), "=qm" (equal), "=a" (dummy) - : "r" (*newVal), - "2" (*oldVal) + : "r" (newVal), + "2" (oldVal) : "cc" ); #else /* 32-bit version for non-ARM */ + typedef struct { + uint32 lowValue; + uint32 highValue; + } S_uint64; + int dummy1, dummy2; # if defined __PIC__ /* @@ -1994,28 +2083,6 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT * where it temporarily modifies %ebx as small as possible, and should * prefer specific register assignments. */ -# if __GNUC__ < 3 // Part of #188541 - for RHL 6.2 etc. - __asm__ __volatile__( - "xchg %%ebx, %6" "\n\t" - "mov 4(%%ebx), %%ecx" "\n\t" - "mov (%%ebx), %%ebx" "\n\t" - "lock; cmpxchg8b (%3)" "\n\t" - "xchg %%ebx, %6" "\n\t" - "sete %0" - : "=a" (equal), - "=d" (dummy2), - "=D" (dummy1) - : /* - * See the "Rules for __asm__ statements in __PIC__ code" above: %3 - * must use a register class which does not contain %ebx. - */ - "S" (var), - "0" (((S_uint64 const *)oldVal)->lowValue), - "1" (((S_uint64 const *)oldVal)->highValue), - "D" (newVal) - : "ecx", "cc", "memory" - ); -# else __asm__ __volatile__( "xchgl %%ebx, %6" "\n\t" "lock; cmpxchg8b (%3)" "\n\t" @@ -2033,13 +2100,12 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT * stack, %ebp reserved for frame, %ebx reserved for PIC). */ "S" (var), - "1" (((S_uint64 const *)oldVal)->lowValue), - "2" (((S_uint64 const *)oldVal)->highValue), - "D" (((S_uint64 const *)newVal)->lowValue), - "c" (((S_uint64 const *)newVal)->highValue) + "1" (((S_uint64 *)&oldVal)->lowValue), + "2" (((S_uint64 *)&oldVal)->highValue), + "D" (((S_uint64 *)&newVal)->lowValue), + "c" (((S_uint64 *)&newVal)->highValue) : "cc", "memory" ); -# endif # else __asm__ __volatile__( "lock; cmpxchg8b %0" "\n\t" @@ -2048,40 +2114,20 @@ Atomic_CMPXCHG64(Atomic_uint64 *var, // IN/OUT "=qm" (equal), "=a" (dummy1), "=d" (dummy2) - : "2" (((S_uint64 const *)oldVal)->lowValue), - "3" (((S_uint64 const *)oldVal)->highValue), - "b" (((S_uint64 const *)newVal)->lowValue), - "c" (((S_uint64 const *)newVal)->highValue) + : "2" (((S_uint64 *)&oldVal)->lowValue), + "3" (((S_uint64 *)&oldVal)->highValue), + "b" (((S_uint64 *)&newVal)->lowValue), + "c" (((S_uint64 *)&newVal)->highValue) : "cc" ); # endif #endif - AtomicEpilogue(); return equal; #endif //VM_ARM_V7 #elif defined _MSC_VER -#if defined(__x86_64__) - return (__int64)*oldVal == _InterlockedCompareExchange64((__int64 *)&var->value, - (__int64)*newVal, - (__int64)*oldVal); -#else -#pragma warning(push) -#pragma warning(disable : 4035) // disable no-return warning - { - __asm mov esi, var - __asm mov edx, oldVal - __asm mov ecx, newVal - __asm mov eax, [edx]S_uint64.lowValue - __asm mov edx, [edx]S_uint64.highValue - __asm mov ebx, [ecx]S_uint64.lowValue - __asm mov ecx, [ecx]S_uint64.highValue - __asm lock cmpxchg8b [esi] - __asm sete al - __asm movzx eax, al - // eax is the return value, this is documented to work - edward - } -#pragma warning(pop) -#endif + return (__int64)oldVal == _InterlockedCompareExchange64((__int64 *)&var->value, + (__int64)newVal, + (__int64)oldVal); #else #error No compiler defined for Atomic_CMPXCHG64 #endif // !GNUC @@ -2109,9 +2155,9 @@ Atomic_CMPXCHG32(Atomic_uint32 *var, // IN/OUT uint32 oldVal, // IN uint32 newVal) // IN { -#if defined(__GNUC__) -#if defined(VM_ARM_ANY) - return (Atomic_ReadIfEqualWrite(var, oldVal, newVal) == oldVal); +#if defined __GNUC__ +#if defined VM_ARM_ANY + return Atomic_ReadIfEqualWrite32(var, oldVal, newVal) == oldVal; #else /* VM_X86_ANY */ Bool equal; uint32 dummy; @@ -2126,12 +2172,11 @@ Atomic_CMPXCHG32(Atomic_uint32 *var, // IN/OUT "2" (oldVal) : "cc" ); - AtomicEpilogue(); return equal; #endif /* VM_X86_ANY */ -#else // defined(__GNUC__) - return (Atomic_ReadIfEqualWrite(var, oldVal, newVal) == oldVal); -#endif // !defined(__GNUC__) +#else // defined __GNUC__ + return Atomic_ReadIfEqualWrite32(var, oldVal, newVal) == oldVal; +#endif // !defined __GNUC__ } @@ -2154,12 +2199,15 @@ Atomic_CMPXCHG32(Atomic_uint32 *var, // IN/OUT static INLINE uint64 Atomic_Read64(Atomic_uint64 const *var) // IN { -#if defined(__GNUC__) && defined(__x86_64__) +#if defined __GNUC__ uint64 value; +#endif -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 8 == 0); #endif + +#if defined __GNUC__ && defined __x86_64__ /* * Use asm to ensure we emit a single load. */ @@ -2168,9 +2216,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN : "=r" (value) : "m" (var->value) ); - return value; -#elif defined(__GNUC__) && defined(__i386__) - uint64 value; +#elif defined __GNUC__ && defined __i386__ /* * Since cmpxchg8b will replace the contents of EDX:EAX with the * value in memory if there is no match, we need only execute the @@ -2188,9 +2234,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN : "m" (*var) : "cc" ); - AtomicEpilogue(); - return value; -#elif defined (_MSC_VER) && defined(__x86_64__) +#elif defined _MSC_VER && defined __x86_64__ /* * Microsoft docs guarantee "Simple reads and writes to properly * aligned 64-bit variables are atomic on 64-bit Windows." @@ -2199,7 +2243,9 @@ Atomic_Read64(Atomic_uint64 const *var) // IN * XXX Verify that value is properly aligned. Bug 61315. */ return var->value; -#elif defined (_MSC_VER) && defined(__i386__) +#elif defined _MSC_VER && defined VM_ARM_32 + return _InterlockedAdd64((__int64 *)&var->value, 0); +#elif defined _MSC_VER && defined __i386__ # pragma warning(push) # pragma warning(disable : 4035) // disable no-return warning { @@ -2210,26 +2256,19 @@ Atomic_Read64(Atomic_uint64 const *var) // IN // edx:eax is the return value; this is documented to work. --mann } # pragma warning(pop) -#elif defined(__GNUC__) && defined (VM_ARM_V7) - uint64 value; - +#elif defined __GNUC__ && defined VM_ARM_V7 __asm__ __volatile__( "ldrexd %[value], %H[value], [%[var]] \n\t" : [value] "=&r" (value) : [var] "r" (&var->value) ); +#elif defined VM_ARM_64 + value = _VMATOM_X(R, 64, &var->value); +#endif +#if defined __GNUC__ return value; -#elif defined(__GNUC__) && defined(VM_ARM_64) - uint64 value; - - __asm__ __volatile__( - "ldr %0, [%1]" - : "=r" (value) - : "r" (&var->value)); - - return value; -#endif /* defined(__GNUC__) && defined(VM_ARM_64) */ +#endif } @@ -2241,7 +2280,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN * Atomically read a 64 bit integer, possibly misaligned. * This function can be *very* expensive, costing over 50 kcycles * on Nehalem. - * + * * Note that "var" needs to be writable, even though it will not * be modified. * @@ -2254,7 +2293,7 @@ Atomic_Read64(Atomic_uint64 const *var) // IN *---------------------------------------------------------------------- */ -#if defined(VM_64BIT) +#if defined VM_64BIT static INLINE uint64 Atomic_ReadUnaligned64(Atomic_uint64 const *var) // IN: { @@ -2283,11 +2322,11 @@ static INLINE uint64 Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(VM_ARM_64) - return ARM_64_ATOMIC_64_OPO_PRIVATE_FENCED(var, val, add); -#elif defined(__x86_64__) +#if defined VM_ARM_64 + return _VMATOM_X(ROP, 64, TRUE, &var->value, add, val); +#elif defined __x86_64__ -#if defined(__GNUC__) +#if defined __GNUC__ __asm__ __volatile__( "lock; xaddq %0, %1" : "=r" (val), @@ -2295,7 +2334,6 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT : "0" (val) : "cc" ); - AtomicEpilogue(); return val; #elif defined _MSC_VER return _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)val); @@ -2310,7 +2348,7 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal + val; - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); return oldVal; #endif @@ -2337,7 +2375,11 @@ static INLINE uint64 Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { +#if defined VM_ARM_64 + return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val); +#else return Atomic_ReadAdd64(var, -val); +#endif } @@ -2407,12 +2449,12 @@ static INLINE void Atomic_Add64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if !defined(VM_64BIT) +#if !defined VM_64BIT Atomic_ReadAdd64(var, val); /* Return value is unused. */ -#elif defined(__GNUC__) -#if defined(VM_ARM_64) - ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, add); -#else /* defined(VM_X86_64) */ +#elif defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, add, val); +#else /* defined VM_X86_64 */ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; addq %1, %0" @@ -2420,7 +2462,6 @@ Atomic_Add64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #endif #elif defined _MSC_VER _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)val); @@ -2450,11 +2491,11 @@ static INLINE void Atomic_Sub64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if !defined(__x86_64__) +#if !defined VM_64BIT Atomic_ReadSub64(var, val); /* Return value is unused. */ -#elif defined(__GNUC__) -#if defined(VM_ARM_64) - ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, sub); +#elif defined __GNUC__ +#if defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, sub, val); #else /* VM_X86_64 */ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( @@ -2463,7 +2504,6 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #endif #elif defined _MSC_VER _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)-val); @@ -2492,9 +2532,11 @@ Atomic_Sub64(Atomic_uint64 *var, // IN/OUT static INLINE void Atomic_Inc64(Atomic_uint64 *var) // IN/OUT { -#if !defined(__x86_64__) +#if defined VM_ARM_64 + Atomic_Add64(var, 1); +#elif !defined __x86_64__ Atomic_ReadInc64(var); /* Return value is unused. */ -#elif defined(__GNUC__) +#elif defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; incq %0" @@ -2502,7 +2544,6 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedIncrement64((__int64 *)&var->value); #else @@ -2530,9 +2571,11 @@ Atomic_Inc64(Atomic_uint64 *var) // IN/OUT static INLINE void Atomic_Dec64(Atomic_uint64 *var) // IN/OUT { -#if !defined(__x86_64__) +#if defined VM_ARM_64 + Atomic_Sub64(var, 1); +#elif !defined __x86_64__ Atomic_ReadDec64(var); /* Return value is unused. */ -#elif defined(__GNUC__) +#elif defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; decq %0" @@ -2540,7 +2583,6 @@ Atomic_Dec64(Atomic_uint64 *var) // IN/OUT : : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedDecrement64((__int64 *)&var->value); #else @@ -2569,8 +2611,8 @@ static INLINE uint64 Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "xchgq %0, %1" @@ -2578,19 +2620,20 @@ Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; #elif defined _MSC_VER return _InterlockedExchange64((__int64 *)&var->value, (__int64)val); #else #error No compiler defined for Atomic_ReadWrite64 #endif +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 64, TRUE, &var->value, val); #else uint64 oldVal; do { oldVal = var->value; - } while (!Atomic_CMPXCHG64(var, &oldVal, &val)); + } while (!Atomic_CMPXCHG64(var, oldVal, val)); return oldVal; #endif @@ -2617,12 +2660,12 @@ static INLINE void Atomic_Write64(Atomic_uint64 *var, // OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) - -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 8 == 0); #endif + +#if defined __x86_64__ +#if defined __GNUC__ /* * There is no move instruction for 64-bit immediate to memory, so unless * the immediate value fits in 32-bit (i.e. can be sign-extended), GCC @@ -2637,7 +2680,7 @@ Atomic_Write64(Atomic_uint64 *var, // OUT ); #elif defined _MSC_VER /* - * Microsoft docs guarantee "Simple reads and writes to properly aligned + * Microsoft docs guarantee "Simple reads and writes to properly aligned * 64-bit variables are atomic on 64-bit Windows." * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx * @@ -2648,7 +2691,9 @@ Atomic_Write64(Atomic_uint64 *var, // OUT #else #error No compiler defined for Atomic_Write64 #endif -#else /* defined(__x86_64__) */ +#elif defined VM_ARM_64 + _VMATOM_X(W, 64, &var->value, val); +#else (void)Atomic_ReadWrite64(var, val); #endif } @@ -2674,8 +2719,8 @@ static INLINE void Atomic_Or64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; orq %1, %0" @@ -2683,19 +2728,20 @@ Atomic_Or64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedOr64((__int64 *)&var->value, (__int64)val); #else #error No compiler defined for Atomic_Or64 #endif +#elif defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, orr, val); #else // __x86_64__ uint64 oldVal; uint64 newVal; do { oldVal = var->value; newVal = oldVal | val; - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2720,8 +2766,8 @@ static INLINE void Atomic_And64(Atomic_uint64 *var, // IN/OUT uint64 val) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ /* Checked against the AMD manual and GCC --hpreg */ __asm__ __volatile__( "lock; andq %1, %0" @@ -2729,19 +2775,20 @@ Atomic_And64(Atomic_uint64 *var, // IN/OUT : "re" (val) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER _InterlockedAnd64((__int64 *)&var->value, (__int64)val); #else #error No compiler defined for Atomic_And64 #endif +#elif defined VM_ARM_64 + _VMATOM_X(OP, 64, TRUE, &var->value, and, val); #else // __x86_64__ uint64 oldVal; uint64 newVal; do { oldVal = var->value; newVal = oldVal & val; - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2766,8 +2813,8 @@ static INLINE void Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT uint64 bit) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ ASSERT(bit <= 63); __asm__ __volatile__( "lock; bts %1, %0" @@ -2775,7 +2822,6 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT : "ri" (bit) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER uint64 oldVal; uint64 newVal; @@ -2783,7 +2829,7 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal | (CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #else #error No compiler defined for Atomic_SetBit64 #endif @@ -2794,7 +2840,7 @@ Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal | (CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2819,8 +2865,8 @@ static INLINE void Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT uint64 bit) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ ASSERT(bit <= 63); __asm__ __volatile__( "lock; btr %1, %0" @@ -2828,7 +2874,6 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT : "ri" (bit) : "cc" ); - AtomicEpilogue(); #elif defined _MSC_VER uint64 oldVal; uint64 newVal; @@ -2836,7 +2881,7 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal & ~(CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #else #error No compiler defined for Atomic_ClearBit64 #endif @@ -2847,7 +2892,7 @@ Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT do { oldVal = var->value; newVal = oldVal & ~(CONST64U(1) << bit); - } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal)); + } while (!Atomic_CMPXCHG64(var, oldVal, newVal)); #endif } @@ -2872,8 +2917,8 @@ static INLINE Bool Atomic_TestBit64(Atomic_uint64 *var, // IN uint64 bit) // IN { -#if defined(__x86_64__) -#if defined(__GNUC__) +#if defined __x86_64__ +#if defined __GNUC__ Bool out = FALSE; ASSERT(bit <= 63); __asm__ __volatile__( @@ -2897,7 +2942,7 @@ Atomic_TestBit64(Atomic_uint64 *var, // IN } -#if defined(__GNUC__) +#if defined __GNUC__ /* *----------------------------------------------------------------------------- * @@ -2919,19 +2964,19 @@ Atomic_Read16(Atomic_uint16 const *var) // IN { uint16 value; -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 2 == 0); #endif -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "movw %1, %0" : "=r" (value) : "m" (var->value) ); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 NOT_TESTED(); __asm__ __volatile__( @@ -2939,13 +2984,8 @@ Atomic_Read16(Atomic_uint16 const *var) // IN : "=r" (value) : "r" (&var->value) ); -#elif defined(VM_ARM_64) - __asm__ __volatile__ ( - "ldrh %w0, [%1]" - : "=r" (value) - : "r" (&var->value) - ); - +#elif defined VM_ARM_64 + value = _VMATOM_X(R, 16, &var->value); #else #error No 16-bits atomics. #endif @@ -2975,17 +3015,16 @@ static INLINE uint16 Atomic_ReadWrite16(Atomic_uint16 *var, // IN/OUT: uint16 val) // IN: { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "xchgw %0, %1" : "=r" (val), "+m" (var->value) : "0" (val) ); - AtomicEpilogue(); return val; -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 retVal; register volatile uint16 res; @@ -3006,22 +3045,8 @@ Atomic_ReadWrite16(Atomic_uint16 *var, // IN/OUT: dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint16 retVal; - register uint16 failed; - - __asm__ __volatile__( - " dmb sy \n\t" - "1: ldxrh %w0, [%2] \n\t" - " stxrh %w1, %w3, [%2] \n\t" - " cbnz %w1, 1b \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (val) - : "memory" - ); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RW, 16, TRUE, &var->value, val); #else #error No 16-bits atomics. #endif @@ -3049,19 +3074,24 @@ static INLINE void Atomic_Write16(Atomic_uint16 *var, // OUT: uint16 val) // IN: { -#ifdef VMM +#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE ASSERT((uintptr_t)var % 2 == 0); #endif -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) - +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "movw %1, %0" : "=m" (var->value) : "r" (val) ); -#elif defined(VM_ARM_ANY) +#elif defined VM_ARM_64 + _VMATOM_X(W, 16, &var->value, val); +#elif defined VM_ARM_32 + /* + * Best left this way due to the intricacies of exclusive load/store + * operations on legacy (32-bit) ARM. + */ Atomic_ReadWrite16(var, val); #else #error No 16-bits atomics. @@ -3091,8 +3121,8 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var, // IN/OUT uint16 oldVal, // IN uint16 newVal) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ uint16 val; __asm__ __volatile__( @@ -3103,9 +3133,8 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var, // IN/OUT "0" (oldVal) : "cc" ); - AtomicEpilogue(); return val; -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register uint16 retVal; register uint16 res; @@ -3128,24 +3157,8 @@ Atomic_ReadIfEqualWrite16(Atomic_uint16 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - register uint16 retVal; - register uint16 failed; - - __asm__ __volatile__ ( - " dmb sy \n\t" - "1: ldxrh %w0, [%2] \n\t" - " cmp %w0, %w3 \n\t" - " b.ne 2f \n\t" - " stxrh %w1, %w4, [%2] \n\t" - " cbnz %w1, 1b \n\t" - "2: clrex \n\t" - " dmb sy \n\t" - : "=&r" (retVal), "=&r" (failed) - : "r" (&var->value), "r" (oldVal), "r" (newVal) - : "cc", "memory"); - - return retVal; +#elif defined VM_ARM_64 + return _VMATOM_X(RIFEQW, 16, TRUE, &var->value, oldVal, newVal); #else #error No 16-bits atomics. #endif @@ -3173,16 +3186,15 @@ static INLINE void Atomic_And16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; andw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3202,8 +3214,8 @@ Atomic_And16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, and); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, and, val); #else #error No 16-bits atomics. #endif @@ -3231,16 +3243,15 @@ static INLINE void Atomic_Or16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; orw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3260,14 +3271,15 @@ Atomic_Or16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, orr); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, orr, val); #else #error No 16-bits atomics. #endif #endif } + /* *----------------------------------------------------------------------------- * @@ -3288,16 +3300,15 @@ static INLINE void Atomic_Xor16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; xorw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3317,8 +3328,8 @@ Atomic_Xor16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, eor); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, eor, val); #else #error No 16-bits atomics. #endif @@ -3346,16 +3357,15 @@ static INLINE void Atomic_Add16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; addw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3375,8 +3385,8 @@ Atomic_Add16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, add, val); #else #error No 16-bits atomics. #endif @@ -3404,16 +3414,15 @@ static INLINE void Atomic_Sub16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; subw %1, %0" : "+m" (var->value) : "re" (val) : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 tmp; @@ -3433,8 +3442,8 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT ); dmb(); -#elif defined(VM_ARM_64) - ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, sub); +#elif defined VM_ARM_64 + _VMATOM_X(OP, 16, TRUE, &var->value, sub, val); #else #error No 16-bits atomics. #endif @@ -3461,16 +3470,15 @@ Atomic_Sub16(Atomic_uint16 *var, // IN/OUT static INLINE void Atomic_Inc16(Atomic_uint16 *var) // IN/OUT { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; incw %0" : "+m" (var->value) : : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_ANY) +#elif defined VM_ARM_ANY Atomic_Add16(var, 1); #else #error No 16-bits atomics. @@ -3498,16 +3506,15 @@ Atomic_Inc16(Atomic_uint16 *var) // IN/OUT static INLINE void Atomic_Dec16(Atomic_uint16 *var) // IN/OUT { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; decw %0" : "+m" (var->value) : : "cc" ); - AtomicEpilogue(); -#elif defined(VM_ARM_ANY) +#elif defined VM_ARM_ANY Atomic_Sub16(var, 1); #else #error No 16-bits atomics. @@ -3538,9 +3545,13 @@ Atomic_ReadOr16(Atomic_uint16 *var, // IN/OUT { uint16 res; +#if defined VM_ARM_64 + res = _VMATOM_X(ROP, 16, TRUE, &var->value, orr, val); +#else do { res = var->value; } while (res != Atomic_ReadIfEqualWrite16(var, res, res | val)); +#endif return res; } @@ -3566,8 +3577,8 @@ static INLINE uint16 Atomic_ReadAdd16(Atomic_uint16 *var, // IN/OUT uint16 val) // IN: { -#if defined(__GNUC__) -#if (defined(__x86_64__) || defined(__i386__)) +#if defined __GNUC__ +#if defined __x86_64__ || defined __i386__ __asm__ __volatile__( "lock; xaddw %0, %1" : "=r" (val), @@ -3575,9 +3586,8 @@ Atomic_ReadAdd16(Atomic_uint16 *var, // IN/OUT : "0" (val) : "cc" ); - AtomicEpilogue(); return val; -#elif defined(VM_ARM_V7) +#elif defined VM_ARM_V7 register volatile uint16 res; register volatile uint16 retVal; register volatile uint16 tmp; @@ -3600,8 +3610,8 @@ Atomic_ReadAdd16(Atomic_uint16 *var, // IN/OUT dmb(); return retVal; -#elif defined(VM_ARM_64) - return ARM_64_ATOMIC_16_OPO_PRIVATE_FENCED(var, val, add); +#elif defined VM_ARM_64 + return _VMATOM_X(ROP, 16, TRUE, &var->value, add, val); #else #error No 16-bits atomics. #endif @@ -3631,8 +3641,8 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT return Atomic_ReadAdd16(var, 1); } -/* +/* *---------------------------------------------------------------------- * * Atomic_ReadDec16 -- @@ -3840,57 +3850,38 @@ Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT * Atomic_ReadAddInt -- * Atomic_ReadIncInt -- * Atomic_ReadDecInt -- + * + * Atomic_Bool + * Atomic_ReadBool -- + * Atomic_WriteBool -- + * Atomic_ReadWriteBool -- + * Atomic_ReadIfEqualWriteBool -- + * Atomic_AndBool -- + * Atomic_OrBool -- + * Atomic_XorBool -- + * Atomic_AddBool -- + * Atomic_SubBool -- + * Atomic_IncBool -- + * Atomic_DecBool -- + * Atomic_ReadOrBool -- + * Atomic_ReadAddBool -- + * Atomic_ReadIncBool -- + * Atomic_ReadDecBool -- */ -#if defined(VM_64BIT) +#if defined VM_64BIT MAKE_ATOMIC_TYPE(Ptr, 64, void const *, void *, uintptr_t) #else MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t) #endif MAKE_ATOMIC_TYPE(Int, 32, int, int, int) +MAKE_ATOMIC_TYPE(Bool, 8, Bool, Bool, Bool) - -/* Prevent the compiler from re-ordering memory references. */ -#ifdef __GNUC__ -#define ATOMIC_COMPILER_BARRIER() __asm__ __volatile__ ("": : :"memory") -#elif defined(_MSC_VER) -#define ATOMIC_COMPILER_BARRIER() _ReadWriteBarrier() -#else -#error No compiler defined for ATOMIC_COMPILER_BARRIER +#ifdef VM_ARM_64 +# include "vm_atomic_arm64_end.h" #endif - -/* - *----------------------------------------------------------------------------- - * - * Atomic_MFence -- - * - * Implements mfence in terms of a lock xor. The reason for implementing - * our own mfence is that not all of our supported cpus have an assembly - * mfence (P3, Athlon). We put it here to avoid duplicating code which is - * also why it is prefixed with "Atomic_". Moreover, this implementation - * performs slightly better than 'mfence' on modern CPUs (See PR 817059). - * - * Results: - * None. - * - * Side effects: - * Cause loads and stores prior to this to be globally - * visible. - * - *----------------------------------------------------------------------------- - */ - -static INLINE void -Atomic_MFence(void) -{ - Atomic_uint32 fence; - ATOMIC_COMPILER_BARRIER(); - Atomic_Xor(&fence, 0x1); - ATOMIC_COMPILER_BARRIER(); -} - -#ifdef ATOMIC_COMPILER_BARRIER -#undef ATOMIC_COMPILER_BARRIER -#endif /* ATOMIC_COMPILER_BARRIER */ +#if defined __cplusplus +} // extern "C" +#endif #endif // ifndef _ATOMIC_H_ diff --git a/vmnet-only/vm_basic_asm.h b/vmnet-only/vm_basic_asm.h index 0e4adae..11c3aba 100644 --- a/vmnet-only/vm_basic_asm.h +++ b/vmnet-only/vm_basic_asm.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -43,6 +43,10 @@ #elif defined VM_X86_32 #include "vm_basic_asm_x86_common.h" #include "vm_basic_asm_x86.h" +#elif defined VM_ARM_32 +#include "vm_basic_asm_arm32.h" +#define MUL64_NO_ASM 1 +#include "mul64.h" #elif defined VM_ARM_64 #include "arm64_basic_defs.h" #include "vm_basic_asm_arm64.h" @@ -51,6 +55,11 @@ #include "mul64.h" #endif +#if defined __cplusplus +extern "C" { +#endif + + /* * Locate most and least significant bit set functions. Use our own name * space to avoid namespace collisions. The new names follow a pattern, @@ -623,6 +632,11 @@ uint16set(void *dst, uint16 val, size_t count) { #ifdef VM_X86_64 __stosw((uint16*)dst, val, count); +#elif defined(VM_ARM_32) + size_t i; + for (i = 0; i < count; i++) { + ((uint16 *)dst)[i] = val; + } #else __asm { pushf; mov ax, val; @@ -641,6 +655,11 @@ uint32set(void *dst, uint32 val, size_t count) { #ifdef VM_X86_64 __stosd((unsigned long*)dst, (unsigned long)val, count); +#elif defined(VM_ARM_32) + size_t i; + for (i = 0; i < count; i++) { + ((uint32 *)dst)[i] = val; + } #else __asm { pushf; mov eax, val; @@ -673,7 +692,7 @@ static INLINE uint16 Bswap16(uint16 v) { #if defined(VM_ARM_64) - __asm__("rev16 %0, %0" : "+r"(v)); + __asm__("rev16 %w0, %w0" : "+r"(v)); return v; #else return ((v >> 8) & 0x00ff) | ((v << 8) & 0xff00); @@ -702,12 +721,12 @@ Bswap32(uint32 v) // IN : "0" (v) ); return v; -#elif defined(VM_ARM_32) && !defined(__ANDROID__) +#elif defined(VM_ARM_32) && !defined(__ANDROID__) && !defined(_MSC_VER) __asm__("rev %0, %0" : "+r"(v)); return v; #elif defined(VM_ARM_64) - __asm__("rev32 %0, %0" : "+r"(v)); - return v; + __asm__("rev32 %x0, %x0" : "+r"(v)); + return v; #else return (v >> 24) | ((v >> 8) & 0xFF00) @@ -740,63 +759,6 @@ Bswap64(uint64 v) // IN } -/* - * COMPILER_MEM_BARRIER prevents the compiler from re-ordering memory - * references accross the barrier. NOTE: It does not generate any - * instruction, so the CPU is free to do whatever it wants to... - */ -#ifdef __GNUC__ -#define COMPILER_MEM_BARRIER() __asm__ __volatile__ ("": : :"memory") -#define COMPILER_READ_BARRIER() COMPILER_MEM_BARRIER() -#define COMPILER_WRITE_BARRIER() COMPILER_MEM_BARRIER() -#elif defined(_MSC_VER) -#define COMPILER_MEM_BARRIER() _ReadWriteBarrier() -#define COMPILER_READ_BARRIER() _ReadBarrier() -#define COMPILER_WRITE_BARRIER() _WriteBarrier() -#endif - - -/* - *---------------------------------------------------------------------- - * - * COMPILER_FORCED_LOAD_AND_MEM_BARRIER -- - * - * This macro prevents the compiler from re-ordering memory references - * across the barrier. In addition it emits a forced load from the given - * memory reference. The memory reference has to be either 1, 2, 4 or 8 - * bytes wide. - * The forced load of a memory reference can be used exploit details of a - * given CPUs memory model. For example x86 CPUs won't reorder stores to - * a memory location x with loads from a memory location x. - * NOTE: It does not generate any fencing instruction, so the CPU is free - * to reorder instructions according to its memory model. - * - * Results: - * None - * - * Side Effects: - * None. - * - *---------------------------------------------------------------------- - */ - -#ifdef VM_X86_64 -#ifdef __GNUC__ - -#define COMPILER_FORCED_LOAD_AND_MEM_BARRIER(_memory_reference) \ - do { \ - typeof(_memory_reference) _dummy; \ - \ - asm volatile("mov %1, %0\n\t" \ - : "=r" (_dummy) /* Let compiler choose reg for _dummy */ \ - : "m" (_memory_reference) \ - : "memory"); \ - } while(0) - -#endif /* __GNUC__ */ -#endif /* VM_X86_64 */ - - /* * PAUSE is a P4 instruction that improves spinlock power+performance; * on non-P4 IA32 systems, the encoding is interpreted as a REPZ-NOP. @@ -804,7 +766,7 @@ Bswap64(uint64 v) // IN */ static INLINE void PAUSE(void) -#ifdef __GNUC__ +#if defined(__GNUC__) || defined(VM_ARM_32) { #ifdef VM_ARM_ANY /* @@ -878,6 +840,15 @@ RDTSC(void) { return __rdtsc(); } +#elif defined(VM_ARM_32) +{ + /* + * We need to do more inverstagetion here to find + * a microsoft equivalent of that code + */ + NOT_IMPLEMENTED(); + return 0; +} #else #pragma warning( disable : 4035) { @@ -1254,5 +1225,10 @@ RoundUpPow2_32(uint32 value) #endif } + +#if defined __cplusplus +} // extern "C" +#endif + #endif // _VM_BASIC_ASM_H_ diff --git a/vmnet-only/vm_basic_asm_x86.h b/vmnet-only/vm_basic_asm_x86.h index 2c4232c..b9c684f 100644 --- a/vmnet-only/vm_basic_asm_x86.h +++ b/vmnet-only/vm_basic_asm_x86.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -35,6 +35,11 @@ #define INCLUDE_ALLOW_VMCORE #include "includeCheck.h" +#if defined __cplusplus +extern "C" { +#endif + + #ifdef VM_X86_64 /* * The gcc inline asm uses the "A" constraint which differs in 32 & 64 @@ -545,4 +550,9 @@ Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift) #pragma warning(default: 4035) #endif + +#if defined __cplusplus +} // extern "C" #endif + +#endif // _VM_BASIC_ASM_X86_H_ diff --git a/vmnet-only/vm_basic_asm_x86_64.h b/vmnet-only/vm_basic_asm_x86_64.h index bef0056..4f9fd4f 100644 --- a/vmnet-only/vm_basic_asm_x86_64.h +++ b/vmnet-only/vm_basic_asm_x86_64.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -59,24 +59,23 @@ uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift); #if defined(__GNUC__) /* - * GET_CURRENT_RIP + * GET_CURRENT_PC * - * Returns the current instruction pointer. In the example below: + * Returns the current program counter (i.e. instruction pointer i.e. rip + * register on x86_64). In the example below: * * foo.c - * L123: Foo(GET_CURRENT_RIP()) + * L123: Foo(GET_CURRENT_PC()) * - * the return value from GET_CURRENT_RIP will point a debugger to L123. + * the return value from GET_CURRENT_PC will point a debugger to L123. */ -#define GET_CURRENT_RIP() ({ \ +#define GET_CURRENT_PC() ({ \ void *__rip; \ asm("lea 0(%%rip), %0;\n\t" \ : "=r" (__rip)); \ __rip; \ }) -#define GET_CURRENT_PC() GET_CURRENT_RIP() - /* * GET_CURRENT_LOCATION * diff --git a/vmnet-only/vm_basic_asm_x86_common.h b/vmnet-only/vm_basic_asm_x86_common.h index 6bbbc3e..06ae1b5 100644 --- a/vmnet-only/vm_basic_asm_x86_common.h +++ b/vmnet-only/vm_basic_asm_x86_common.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2013 VMware, Inc. All rights reserved. + * Copyright (C) 2013-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -89,6 +89,9 @@ void _mm_mfence(void); void _mm_lfence(void); #pragma intrinsic(_mm_mfence, _mm_lfence) +long _InterlockedXor(long volatile *, long); +#pragma intrinsic(_InterlockedXor) + unsigned int __getcallerseflags(void); #pragma intrinsic(__getcallerseflags) @@ -294,20 +297,150 @@ RDTSC_BARRIER(void) /* - * Compiler/CPU barriers. These take the form of __MEM_BARRIER, where is either LD (load), ST (store) or - * LDST (any). On x86 we only need to care specifically about store-load - * reordering on normal memory types and mfence, otherwise only a compiler - * barrier is needed. + * Memory Barriers + * =============== + * + * Terminology + * ----------- + * + * A compiler memory barrier prevents the compiler from re-ordering memory + * accesses accross the barrier. It is not a CPU instruction, it is a compiler + * directive (i.e. it does not emit any code). + * + * A CPU memory barrier prevents the CPU from re-ordering memory accesses + * accross the barrier. It is a CPU instruction. + * + * A memory barrier is the union of a compiler memory barrier and a CPU memory + * barrier. A compiler memory barrier is a useless construct by itself. It is + * only useful when combined with a CPU memory barrier, to implement a memory + * barrier. + * + * Semantics + * --------- + * + * At the time COMPILER_*_BARRIER were created (and references to them were + * added to the code), the code was only targetting x86. The intent of the code + * was really to use a memory barrier, but because x86 uses a strongly ordered + * memory model, the CPU would not re-order memory accesses, and the code could + * get away with using just a compiler memory barrier. So COMPILER_*_BARRIER + * were born and were implemented as compiler memory barriers _on x86_. But + * make no mistake, _the semantics that the code expects from + * COMPILER_*_BARRIER are that of a memory barrier_! + * + * DO NOT USE! + * ----------- + * + * On at least one non-x86 architecture, COMPILER_*_BARRIER are + * 1) Misnomers + * 2) Not fine-grained enough to provide the best performance. + * For the above two reasons, usage of COMPILER_*_BARRIER is now deprecated. + * _Do not add new references to COMPILER_*_BARRIER._ Instead, precisely + * document the intent of your code by using + * __BARRIER_. + * Existing references to COMPILER_*_BARRIER are being slowly but surely + * converted, and when no references are left, COMPILER_*_BARRIER will be + * retired. + * + * Thanks for pasting this whole comment into every architecture header. */ -#define LD_LD_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define LD_ST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define LD_LDST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define ST_LD_MEM_BARRIER() asm volatile ("mfence" ::: "memory") -#define ST_ST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define ST_LDST_MEM_BARRIER() ST_LD_MEM_BARRIER() -#define LDST_LD_MEM_BARRIER() ST_LD_MEM_BARRIER() -#define LDST_ST_MEM_BARRIER() COMPILER_MEM_BARRIER() -#define LDST_LDST_MEM_BARRIER() ST_LD_MEM_BARRIER() + +#if defined __GNUC__ +# define COMPILER_READ_BARRIER() COMPILER_MEM_BARRIER() +# define COMPILER_WRITE_BARRIER() COMPILER_MEM_BARRIER() +# define COMPILER_MEM_BARRIER() __asm__ __volatile__("" ::: "memory") +#elif defined _MSC_VER +# define COMPILER_READ_BARRIER() _ReadBarrier() +# define COMPILER_WRITE_BARRIER() _WriteBarrier() +# define COMPILER_MEM_BARRIER() _ReadWriteBarrier() +#endif + + +/* + * Memory barriers. These take the form of + * + * __BARRIER_ + * + * where: + * is either SMP, DMA, or MMIO. + * <*_access type> is either R(load), W(store) or RW(any). + * + * Above every use of these memory barriers in the code, there _must_ be a + * comment to justify the use, i.e. a comment which: + * + * 1) Precisely identifies which memory accesses must not be re-ordered across + * the memory barrier. + * 2) Explains why it is important that the memory accesses not be re-ordered. + * + * Thanks for pasting this whole comment into every architecture header. + * + * On x86, we only need to care specifically about store-load reordering on + * normal memory types. In other cases, only a compiler barrier is needed. The + * ST_LD barrier is implemented with a locked xor operation (instead of the + * mfence instruction) for performance reasons. See PR 1674199 for more + * details. + * + * On x64, special instructions are only provided for load-load (lfence) and + * store-store (sfence) ordering, and they don't apply to normal memory. + */ + + +static INLINE void +SMP_W_BARRIER_R(void) +{ + volatile long temp; + + COMPILER_MEM_BARRIER(); +#if defined __GNUC__ + __asm__ __volatile__ ( + "lock xorl $1, %0\n" + : "+m" (temp) + : /* no additional inputs */ + : "cc"); +#elif defined _MSC_VER + _InterlockedXor(&temp, 1); +#else +#error SMP_W_BARRIER_R not defined for this compiler +#endif + COMPILER_MEM_BARRIER(); +} + +#define SMP_R_BARRIER_R() COMPILER_READ_BARRIER() +#define SMP_R_BARRIER_W() COMPILER_MEM_BARRIER() +#define SMP_R_BARRIER_RW() COMPILER_MEM_BARRIER() +#define SMP_W_BARRIER_W() COMPILER_WRITE_BARRIER() +#define SMP_W_BARRIER_RW() SMP_W_BARRIER_R() +#define SMP_RW_BARRIER_R() SMP_W_BARRIER_R() +#define SMP_RW_BARRIER_W() COMPILER_MEM_BARRIER() +#define SMP_RW_BARRIER_RW() SMP_W_BARRIER_R() + +/* + * Like the above, only for use with observers other than CPUs, + * i.e. DMA masters. + */ + +#define DMA_R_BARRIER_R() SMP_R_BARRIER_R() +#define DMA_R_BARRIER_W() SMP_R_BARRIER_W() +#define DMA_R_BARRIER_RW() SMP_R_BARRIER_RW() +#define DMA_W_BARRIER_R() SMP_W_BARRIER_R() +#define DMA_W_BARRIER_W() SMP_W_BARRIER_W() +#define DMA_W_BARRIER_RW() SMP_W_BARRIER_RW() +#define DMA_RW_BARRIER_R() SMP_RW_BARRIER_R() +#define DMA_RW_BARRIER_W() SMP_RW_BARRIER_W() +#define DMA_RW_BARRIER_RW() SMP_RW_BARRIER_RW() + +/* + * And finally a set for use with MMIO accesses. + */ + +#define MMIO_R_BARRIER_R() SMP_R_BARRIER_R() +#define MMIO_R_BARRIER_W() SMP_R_BARRIER_W() +#define MMIO_R_BARRIER_RW() SMP_R_BARRIER_RW() +#define MMIO_W_BARRIER_R() SMP_W_BARRIER_R() +#define MMIO_W_BARRIER_W() SMP_W_BARRIER_W() +#define MMIO_W_BARRIER_RW() SMP_W_BARRIER_RW() +#define MMIO_RW_BARRIER_R() SMP_RW_BARRIER_R() +#define MMIO_RW_BARRIER_W() SMP_RW_BARRIER_W() +#define MMIO_RW_BARRIER_RW() SMP_RW_BARRIER_RW() + #endif // _VM_BASIC_ASM_X86_COMMON_H_ diff --git a/vmnet-only/vm_basic_defs.h b/vmnet-only/vm_basic_defs.h index efe492a..4702238 100644 --- a/vmnet-only/vm_basic_defs.h +++ b/vmnet-only/vm_basic_defs.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2003-2015 VMware, Inc. All rights reserved. + * Copyright (C) 2003-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -251,12 +251,16 @@ Max(int a, int b) #define PAGES_2_BYTES(_npages) (((uint64)(_npages)) << PAGE_SHIFT) #endif +#ifndef MBYTES_SHIFT +#define MBYTES_SHIFT 20 +#endif + #ifndef MBYTES_2_PAGES -#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (20 - PAGE_SHIFT)) +#define MBYTES_2_PAGES(_nbytes) ((_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT)) #endif #ifndef PAGES_2_MBYTES -#define PAGES_2_MBYTES(_npages) ((_npages) >> (20 - PAGE_SHIFT)) +#define PAGES_2_MBYTES(_npages) ((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT)) #endif #ifndef GBYTES_2_PAGES @@ -268,11 +272,11 @@ Max(int a, int b) #endif #ifndef BYTES_2_MBYTES -#define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> 20) +#define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT) #endif #ifndef MBYTES_2_BYTES -#define MBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << 20) +#define MBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << MBYTES_SHIFT) #endif #ifndef BYTES_2_GBYTES @@ -299,6 +303,26 @@ Max(int a, int b) #define VM_PAE_LARGE_2_SMALL_PAGES (BYTES_2_PAGES(VM_PAE_LARGE_PAGE_SIZE)) #endif +#ifndef VM_PAE_LARGE_2_BYTES +#define VM_PAE_LARGE_2_BYTES(_2mbytes) ((_2mbytes) << VM_PAE_LARGE_PAGE_SHIFT) +#endif + +#ifndef VM_1GB_PAGE_SHIFT +#define VM_1GB_PAGE_SHIFT 30 +#endif + +#ifndef VM_1GB_PAGE_SIZE +#define VM_1GB_PAGE_SIZE (1 << VM_1GB_PAGE_SHIFT) +#endif + +#ifndef VM_1GB_2_PAGES +#define VM_1GB_2_PAGES (BYTES_2_PAGES(VM_1GB_PAGE_SIZE)) +#endif + +#ifndef VM_1GB_2_PDIRS +#define VM_1GB_2_PDIRS (VM_1GB_PAGE_SIZE / VM_PAE_LARGE_PAGE_SIZE) +#endif + /* * Word operations */ @@ -394,9 +418,7 @@ void *_ReturnAddress(void); * guarantee. Bummer. --Jeremy. */ -#if defined(N_PLAT_NLM) -/* We do not have YIELD() as we do not need it yet... */ -#elif defined(_WIN32) +#if defined(_WIN32) # define YIELD() Sleep(0) #elif defined(VMKERNEL) /* We don't have a YIELD macro in the vmkernel */ @@ -412,7 +434,11 @@ void *_ReturnAddress(void); #ifdef _WIN32 // { +/* Conflict with definition of Visual Studio 2015 */ +#if (_MSC_VER < 1900) #define snprintf _snprintf +#endif + #define strtok_r strtok_s #if (_MSC_VER < 1500) @@ -652,14 +678,24 @@ typedef int pid_t; #define APPLE_ONLY(x) x #else #define vmx86_apple 0 -#define APPLE_ONLY(x) +#define APPLE_ONLY(x) +#endif + +#if defined(__APPLE__) && defined(VMW_APPLE_SANDBOX) +#define vmw_apple_sandbox 1 +#else +#define vmw_apple_sandbox 0 #endif #ifdef VMM #define VMM_ONLY(x) x -#define USER_ONLY(x) #else #define VMM_ONLY(x) +#endif + +#if defined(VMM) || defined(VMKERNEL) +#define USER_ONLY(x) +#else #define USER_ONLY(x) x #endif @@ -781,6 +817,22 @@ typedef int pid_t; #define SIZE_80BIT 10 #define SIZE_128BIT 16 #define SIZE_256BIT 32 +#define SIZE_512BIT 64 +/* + * Allocate a variable of type _type, aligned to _align bytes, returning a + * pointer to the variable in _var. Potentially _align - 1 bytes may be + * wasted. On x86, GCC 6.3.0 behaves sub-optimally when variables are declared + * on the stack using the aligned attribute, so this pattern is preferred. + * See PRs 1795155, 1819963. + */ +#define WITH_PTR_TO_ALIGNED_VAR(_type, _align, _var) \ + do { \ + uint8 _buf_##_var[sizeof(_type) + (_align) - 1]; \ + _type *_var = (_type *) ((uintptr_t)(_buf_##_var + (_align) - 1) & \ + ~((uintptr_t) ((_align) - 1))); + +#define END_PTR_TO_ALIGNED_VAR \ + } while (0) #endif // ifndef _VM_BASIC_DEFS_H_ diff --git a/vmnet-only/vm_basic_types.h b/vmnet-only/vm_basic_types.h index 17b11b9..01aa0c9 100644 --- a/vmnet-only/vm_basic_types.h +++ b/vmnet-only/vm_basic_types.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -40,7 +40,8 @@ /* STRICT ANSI means the Xserver build and X defines Bool differently. */ #if !defined(_XTYPEDEF_BOOL) && \ - (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || defined(__MINGW32__)) + (!defined(__STRICT_ANSI__) || defined(__FreeBSD__) || \ + defined(__MINGW32__) || defined(__APPLE__)) #define _XTYPEDEF_BOOL typedef char Bool; #endif @@ -53,8 +54,7 @@ typedef char Bool; #define TRUE 1 #endif -#define IsBool(x) (((x) & ~1) == 0) -#define IsBool2(x, y) ((((x) | (y)) & ~1) == 0) +#define IS_BOOL(x) (((x) & ~1) == 0) /* * Macros __i386__ and __ia64 are intrinsically defined by GCC @@ -312,6 +312,19 @@ typedef uint32 uintptr_t; #endif +#if defined(__GNUC__) && defined(__SIZEOF_INT128__) + +typedef unsigned __int128 uint128; +typedef __int128 int128; + +#define MIN_INT128 ((int128)1 << 127) +#define MAX_INT128 (~MIN_INT128) +#define MIN_UINT128 ((uint128)0) +#define MAX_UINT128 (~MIN_UINT128) + +#endif + + /* * Time * XXX These should be cleaned up. -- edward @@ -353,14 +366,7 @@ typedef int64 VmTimeVirtualClock; /* Virtual Clock kept in CPU cycles */ #define FMTH "" #elif __GNUC__ #define FMTH "" - #if defined(N_PLAT_NLM) || defined(sun) || \ - (defined(__FreeBSD__) && (__FreeBSD__ + 0) && ((__FreeBSD__ + 0) < 5)) - /* - * Why (__FreeBSD__ + 0)? See bug 141008. - * Yes, we really need to test both (__FreeBSD__ + 0) and - * ((__FreeBSD__ + 0) < 5). No, we can't remove "+ 0" from - * ((__FreeBSD__ + 0) < 5). - */ + #if defined(sun) #if defined(VM_X86_64) || defined(VM_ARM_64) #define FMTSZ "l" #define FMTPD "l" @@ -368,7 +374,8 @@ typedef int64 VmTimeVirtualClock; /* Virtual Clock kept in CPU cycles */ #define FMTSZ "" #define FMTPD "" #endif - #elif defined(__linux__) \ + #elif defined(__linux__) || \ + (defined(__FreeBSD__) && (__FreeBSD__ + 0))\ || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) \ || (defined(_POSIX_VERSION) && _POSIX_VERSION >= 200112L) \ || (defined(_POSIX2_VERSION) && _POSIX2_VERSION >= 200112L) @@ -543,6 +550,11 @@ typedef uint16 UReg16; typedef uint32 UReg32; typedef uint64 UReg64; +#if defined(__GNUC__) && defined(__SIZEOF_INT128__) +typedef int128 Reg128; +typedef uint128 UReg128; +#endif + #if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) || \ defined (VMKERNEL) || defined (VMKBOOT) typedef Reg64 Reg; @@ -673,10 +685,17 @@ typedef void * UserVA; #endif #define CONST const - #ifndef INLINE # ifdef _MSC_VER -# define INLINE __inline + /* + * On UWP(Universal Windows Platform), + * Only X86 32bit support '__inline' + */ +# if defined(VM_WIN_UWP) && !defined(VM_X86_32) +# define INLINE +# else +# define INLINE __inline +# endif # else # define INLINE inline # endif @@ -887,6 +906,8 @@ typedef void * UserVA; #ifndef UNUSED_PARAM # if defined(__GNUC__) # define UNUSED_PARAM(_parm) _parm __attribute__((__unused__)) +# elif defined _MSC_VER +# define UNUSED_PARAM(_parm) __pragma(warning(suppress:4100)) _parm # else # define UNUSED_PARAM(_parm) _parm # endif @@ -921,6 +942,21 @@ typedef void * UserVA; #define ALIGNED(n) #endif + +/* + * Encapsulate the syntactic differences between gcc and msvc alignment control. + * BOUNDARY must match in the prefix and suffix. + */ + +#ifdef _WIN32 +#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY)) +#define ALIGN_SUFFIX(BOUNDRY) +#else +#define ALIGN_PREFIX(BOUNDRY) +#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY))) +#endif + + /* * Once upon a time, this was used to silence compiler warnings that * get generated when the compiler thinks that a function returns @@ -1102,16 +1138,6 @@ typedef void * UserVA; #endif -/* - * Define MXSemaHandle here so both vmmon and vmx see this definition. - */ - -#ifdef _WIN32 -typedef uintptr_t MXSemaHandle; -#else -typedef int MXSemaHandle; -#endif - /* * Define type for poll device handles. */ diff --git a/vmnet-only/vm_device_version.h b/vmnet-only/vm_device_version.h index e2cb477..ab396bc 100644 --- a/vmnet-only/vm_device_version.h +++ b/vmnet-only/vm_device_version.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998,2005-2012,2014-2015 VMware, Inc. All rights reserved. + * Copyright (C) 1998,2005-2012,2014-2016 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -74,9 +74,19 @@ #define PCI_DEVICE_ID_VMWARE_PVSCSI 0x07C0 #define PCI_DEVICE_ID_VMWARE_82574 0x07D0 #define PCI_DEVICE_ID_VMWARE_AHCI 0x07E0 +#define PCI_DEVICE_ID_VMWARE_NVME 0x07F0 #define PCI_DEVICE_ID_VMWARE_HDAUDIO_CODEC 0x1975 #define PCI_DEVICE_ID_VMWARE_HDAUDIO_CONTROLLER 0x1977 +/* + * TXT vendor, device and revision ID. We are keeping vendor + * as Intel since tboot code does not like anything other + * than Intel in the SINIT ACM header. + */ +#define TXT_VENDOR_ID 0x8086 +#define TXT_DEVICE_ID 0xB002 +#define TXT_REVISION_ID 0x01 + /* The hypervisor device might grow. Please leave room * for 7 more subfunctions. */ @@ -90,8 +100,8 @@ #define PCI_DEVICE_ID_VMWARE_DUMMY 0x0809 -#define PCI_DEVICE_ID_VMWARE_NVDIMM 0x0810 #define PCI_DEVICE_ID_VMWARE_VRDMA 0x0820 +#define PCI_DEVICE_ID_VMWARE_VTPM 0x0830 /* * VMware Virtual Device Test Infrastructure (VDTI) devices @@ -136,11 +146,31 @@ #define PCI_DEVICE_ID_INTEL_HECI 0x2a74 #define PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI 0x1e31 -/* From drivers/usb/host/xhci-pci.c: +/* + * From drivers/usb/host/xhci-pci.c: * Intel XHCI (Lynx Point / Intel 8 Series) */ #define PCI_DEVICE_ID_INTEL_LYNXPOINT_XHCI 0x8c31 +/* + * Intel Quickassist (QAT) devices: + * - Gen 1: Cave Creek (CVC) and Coleto Creek (CLC) + * - Gen 2: Lewisburg (LBG) + */ +#define PCI_DEVICE_ID_INTEL_QAT_CVC 0x0434 +#define PCI_DEVICE_ID_INTEL_QAT_CLC 0x0435 +#define PCI_DEVICE_ID_INTEL_QAT_LBG 0x37c8 +#define PCI_DEVICE_ID_INTEL_QAT_CVC_VF 0x0442 +#define PCI_DEVICE_ID_INTEL_QAT_CLC_VF 0x0443 +#define PCI_DEVICE_ID_INTEL_QAT_LBG_VF 0x37c9 + +/* + * Intel FPGAs + */ + +#define PCI_DEVICE_ID_INTEL_FPGA_SKL_PF 0xbcc0 +#define PCI_DEVICE_ID_INTEL_FPGA_SKL_VF 0xbcc1 + #define E1000E_PCI_DEVICE_ID_CONFIG_STR "e1000e.pci.deviceID" #define E1000E_PCI_SUB_VENDOR_ID_CONFIG_STR "e1000e.pci.subVendorID" #define E1000E_PCI_SUB_DEVICE_ID_CONFIG_STR "e1000e.pci.subDeviceID" @@ -198,6 +228,13 @@ #define SCSI_CDROM_VENDOR_NAME COMPANY_NAME #define SCSI_CDROM_REV_LEVEL "1.0" +/************* NVME implementation limits ********************************/ +#define NVME_MAX_CONTROLLERS 4 +#define NVME_MIN_NAMESPACES 1 +#define NVME_MAX_NAMESPACES 15 /* We support only 15 namespaces same + * as SCSI devices. + */ + /************* SCSI implementation limits ********************************/ #define SCSI_MAX_CONTROLLERS 4 // Need more than 1 for MSCS clustering #define SCSI_MAX_DEVICES 16 // BT-958 emulates only 16 @@ -218,6 +255,7 @@ #define MAX_NUM_DISKS \ ((SATA_MAX_CONTROLLERS * SATA_MAX_DEVICES) + \ (SCSI_MAX_CONTROLLERS * SCSI_MAX_DEVICES) + \ + (NVME_MAX_CONTROLLERS * NVME_MAX_NAMESPACES) + \ (IDE_NUM_INTERFACES * IDE_DRIVES_PER_IF)) /* @@ -228,7 +266,17 @@ #define SCSI_IDE_CHANNEL SCSI_MAX_CONTROLLERS #define SCSI_IDE_HOSTED_CHANNEL (SCSI_MAX_CONTROLLERS + 1) #define SCSI_SATA_CHANNEL_FIRST (SCSI_IDE_HOSTED_CHANNEL + 1) -#define SCSI_MAX_CHANNELS (SCSI_SATA_CHANNEL_FIRST + SATA_MAX_CONTROLLERS) +#define SCSI_NVME_CHANNEL_FIRST (SCSI_SATA_CHANNEL_FIRST + \ + SATA_MAX_CONTROLLERS) +#define SCSI_MAX_CHANNELS (SCSI_NVME_CHANNEL_FIRST + \ + NVME_MAX_CONTROLLERS) + +/************* SCSI-NVME channel IDs *******************************/ +#define NVME_ID_TO_SCSI_ID(nvmeId) \ + (SCSI_NVME_CHANNEL_FIRST + (nvmeId)) + +#define SCSI_ID_TO_NVME_ID(scsiId) \ + ((scsiId) - SCSI_NVME_CHANNEL_FIRST) /************* SCSI-SATA channel IDs********************************/ #define SATA_ID_TO_SCSI_ID(sataId) \ @@ -264,10 +312,14 @@ #define MAX_USB_DEVICES_PER_HOST_CONTROLLER 127 /************* NVDIMM implementation limits ********************************/ +#define NVDIMM_MAX_CONTROLLERS 1 #define MAX_NVDIMM 64 /************* vRDMA implementation limits ******************************/ -#define MAX_VRDMA_DEVICES 2 +#define MAX_VRDMA_DEVICES 1 + +/************* QAT implementation limits ********************/ +#define MAX_QAT_PCI_DEVICES 4 /************* Strings for Host USB Driver *******************************/ @@ -306,4 +358,13 @@ DEFINE_GUID(GUID_CLASS_VMWARE_USB_DEVICES, #endif #endif +/* + * Our JEDEC 2 Manufacturer ID number is 2 in bank 10. Our number is nine + * bytes of continuation code (with an odd parity bit in bit 7) followed by the + * number itself. + * + */ +#define JEDEC_VENDOR_ID_VMWARE 0x289 +#define JEDEC_DEVICE_ID_VMWARE_NVDIMM 0x0 + #endif /* VM_DEVICE_VERSION_H */ diff --git a/vmnet-only/vm_oui.h b/vmnet-only/vm_oui.h index a67075a..83b8d73 100644 --- a/vmnet-only/vm_oui.h +++ b/vmnet-only/vm_oui.h @@ -26,6 +26,10 @@ #include "vm_basic_asm.h" +#if defined __cplusplus +extern "C" { +#endif + /* * Our own OUIs given by IEEE. */ @@ -204,4 +208,8 @@ VMX86_GENERATE_LEGACY_MAC(uint8 mac[6], //OUT: mac[5] = (suffix) & 0xff; } +#if defined __cplusplus +} // extern "C" +#endif + #endif diff --git a/vmnet-only/vmnetInt.h b/vmnet-only/vmnetInt.h index 0ee52ec..b49c399 100644 --- a/vmnet-only/vmnetInt.h +++ b/vmnet-only/vmnetInt.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998 VMware, Inc. All rights reserved. + * Copyright (C) 1998, 2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -25,27 +25,6 @@ #include "driver-config.h" -/* - * Hide all kernel compatibility stuff in those macros. This part of code - * is used only when building prebuilt modules, when autoconf code is disabled. - */ - -/* All kernels above 2.6.23 have net namespaces. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24) && !defined(VMW_NETDEV_HAS_NET) -# define VMW_NETDEV_HAS_NET -#endif - -/* All kernels above 2.6.23 have skb argument in nf_hookfn. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24) && !defined(VMW_NFHOOK_USES_SKB) -# define VMW_NFHOOK_USES_SKB -#endif - -/* All kernels above 2.6.25 have dev_net & friends. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26) && !defined(VMW_NETDEV_HAS_DEV_NET) -# define VMW_NETDEV_HAS_DEV_NET -#endif - - #ifdef skb_shinfo # define SKB_IS_CLONE_OF(clone, skb) ( \ skb_shinfo(clone) == skb_shinfo(skb) \ @@ -64,27 +43,15 @@ ) #define dev_lock_list() read_lock(&dev_base_lock) #define dev_unlock_list() read_unlock(&dev_base_lock) -#ifdef VMW_NETDEV_HAS_NET -# define DEV_GET(x) __dev_get_by_name(&init_net, (x)->name) -# ifdef VMW_NETDEV_HAS_DEV_NET -# define compat_dev_net(x) dev_net(x) -# else -# define compat_dev_net(x) (x)->nd_net -# endif -#else -# define DEV_GET(x) __dev_get_by_name((x)->name) -#endif extern struct proto vmnet_proto; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) || defined(sk_net_refcnt) # define compat_sk_alloc(_bri, _pri) sk_alloc(&init_net, \ PF_NETLINK, _pri, &vmnet_proto, 1) -#elif defined(VMW_NETDEV_HAS_NET) +#else # define compat_sk_alloc(_bri, _pri) sk_alloc(&init_net, \ PF_NETLINK, _pri, &vmnet_proto) -#else -# define compat_sk_alloc(_bri, _pri) sk_alloc(PF_NETLINK, _pri, &vmnet_proto, 1) #endif diff --git a/vmnet-only/vmware_pack_begin.h b/vmnet-only/vmware_pack_begin.h deleted file mode 100644 index d0eb4bd..0000000 --- a/vmnet-only/vmware_pack_begin.h +++ /dev/null @@ -1,43 +0,0 @@ -/********************************************************* - * Copyright (C) 2002-2015 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * vmware_pack_begin.h -- - * - * Begin of structure packing. See vmware_pack_init.h for details. - * - * Note that we do not use the following construct in this include file, - * because we want to emit the code every time the file is included --hpreg - * - * #ifndef foo - * # define foo - * ... - * #endif - * - */ - - -#include "vmware_pack_init.h" - - -#ifdef _MSC_VER -# pragma pack(push, 1) -#elif __GNUC__ -#else -# error Compiler packing... -#endif diff --git a/vmnet-only/vmware_pack_end.h b/vmnet-only/vmware_pack_end.h deleted file mode 100644 index f02e215..0000000 --- a/vmnet-only/vmware_pack_end.h +++ /dev/null @@ -1,44 +0,0 @@ -/********************************************************* - * Copyright (C) 2002-2015 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -/* - * vmware_pack_end.h -- - * - * End of structure packing. See vmware_pack_init.h for details. - * - * Note that we do not use the following construct in this include file, - * because we want to emit the code every time the file is included --hpreg - * - * #ifndef foo - * # define foo - * ... - * #endif - * - */ - - -#include "vmware_pack_init.h" - - -#ifdef _MSC_VER -# pragma pack(pop) -#elif __GNUC__ -__attribute__((__packed__)) -#else -# error Compiler packing... -#endif diff --git a/vmnet-only/vmware_pack_init.h b/vmnet-only/vmware_pack_init.h deleted file mode 100644 index c401d66..0000000 --- a/vmnet-only/vmware_pack_init.h +++ /dev/null @@ -1,65 +0,0 @@ -/********************************************************* - * Copyright (C) 2002-2015 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef __VMWARE_PACK_INIT_H__ -# define __VMWARE_PACK_INIT_H__ - - -/* - * vmware_pack_init.h -- - * - * Platform-independent code to make the compiler pack (i.e. have them - * occupy the smallest possible space) structure definitions. The following - * constructs are known to work --hpreg - * - * #include "vmware_pack_begin.h" - * struct foo { - * ... - * } - * #include "vmware_pack_end.h" - * ; - * - * typedef - * #include "vmware_pack_begin.h" - * struct foo { - * ... - * } - * #include "vmware_pack_end.h" - * foo; - */ - - -#ifdef _MSC_VER -/* - * MSVC 6.0 emits warning 4103 when the pack push and pop pragma pairing is - * not balanced within 1 included file. That is annoying because our scheme - * is based on the pairing being balanced between 2 included files. - * - * So we disable this warning, but this is safe because the compiler will also - * emit warning 4161 when there is more pops than pushes within 1 main - * file --hpreg - */ - -# pragma warning(disable:4103) -#elif __GNUC__ -#else -# error Compiler packing... -#endif - - -#endif /* __VMWARE_PACK_INIT_H__ */ diff --git a/vmnet-only/vnet.h b/vmnet-only/vnet.h index f6f45e7..4e2ba55 100644 --- a/vmnet-only/vnet.h +++ b/vmnet-only/vnet.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 1998-2014 VMware, Inc. All rights reserved. + * Copyright (C) 1998-2014,2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -26,7 +26,11 @@ #include "includeCheck.h" #include "vm_basic_types.h" #include "vm_atomic.h" -#include "monitorAction_exported.h" + +#if defined __cplusplus +extern "C" { +#endif + #define VNET_PVN_ABI_ID_LEN (256 / 8) // bytes used on ioctl() #define VNET_PVN_ID_LEN (160 / 8) // actual length used @@ -67,7 +71,7 @@ typedef struct VNet_Bind { #define SIOCGETAPIVERSION 0x99FC #define SIOCINJECTLINKSTATE 0x99FD -#define VNET_NOTIFY_VERSION 5 +#define VNET_NOTIFY_VERSION 6 #define VNET_LAST_CMD 0x99FD #if defined __linux__ || defined __APPLE__ @@ -121,7 +125,7 @@ enum VMNetSockOpt { }; /* - * This magic value is populated in VNet_Notify.actionID and VNet_Notify.pollMask + * This magic value is populated in VNet_Notify.pollMask * to request the driver to clear the Notify pollPtr if the receive queue is empty. */ #define VNET_NOTIFY_CLR_MAGIC 0xDECAFBAD @@ -155,7 +159,7 @@ typedef struct VNet_Read { * as well as new driver with old executable). */ -#ifdef linux +#ifdef __linux__ #define VNET_API_VERSION (3 << 16 | 0) #elif defined __APPLE__ #define VNET_API_VERSION (6 << 16 | 0) @@ -176,11 +180,11 @@ typedef struct VNet_SetMacAddrIOCTL { #pragma pack(push, 1) typedef struct VNet_Notify { uint32 version; - uint32 actionVersion; /* Version of monitor action logic */ - VA64 actPtr; /* User VA of a MonitorActionIntr */ + uint32 _unused0; + VA64 _unused1; VA64 pollPtr; /* User VA of a volatile uint32 */ VA64 recvClusterPtr; /* User VA of a uint32 */ - MonitorIdemAction actionID; + uint32 _unused2; uint32 pollMask; } VNet_Notify; #pragma pack(pop) @@ -447,4 +451,8 @@ exit_failure: #endif // __APPLE__ && ! KERNEL +#if defined __cplusplus +} // extern "C" #endif + +#endif // _VNET_H diff --git a/vmnet-only/vnetFilterInt.h b/vmnet-only/vnetFilterInt.h deleted file mode 100644 index e69de29..0000000 diff --git a/vmnet-only/x86cpuid.h b/vmnet-only/x86cpuid.h deleted file mode 100644 index a1412f3..0000000 --- a/vmnet-only/x86cpuid.h +++ /dev/null @@ -1,1815 +0,0 @@ -/********************************************************* - * Copyright (C) 1998-2016 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *********************************************************/ - -#ifndef _X86CPUID_H_ -#define _X86CPUID_H_ - -/* http://www.sandpile.org/ia32/cpuid.htm */ - -#define INCLUDE_ALLOW_USERLEVEL -#define INCLUDE_ALLOW_VMX - -#define INCLUDE_ALLOW_VMKERNEL -#define INCLUDE_ALLOW_MODULE -#define INCLUDE_ALLOW_DISTRIBUTE -#define INCLUDE_ALLOW_VMK_MODULE -#define INCLUDE_ALLOW_VMCORE -#define INCLUDE_ALLOW_VMMON -#include "includeCheck.h" - -#include "vm_basic_types.h" -#include "community_source.h" -#include "x86vendor.h" -#include "vm_assert.h" - -/* - * The linux kernel's ptrace.h stupidly defines the bare - * EAX/EBX/ECX/EDX, which wrecks havoc with our preprocessor tricks. - */ -#undef EAX -#undef EBX -#undef ECX -#undef EDX - -typedef struct CPUIDRegs { - uint32 eax, ebx, ecx, edx; -} CPUIDRegs; - -typedef union CPUIDRegsUnion { - uint32 array[4]; - CPUIDRegs regs; -} CPUIDRegsUnion; - -/* - * Results of calling cpuid(eax, ecx) on all host logical CPU. - */ -#ifdef _MSC_VER -#pragma warning (disable :4200) // non-std extension: zero-sized array in struct -#endif - -typedef -#include "vmware_pack_begin.h" -struct CPUIDReply { - /* - * Unique host logical CPU identifier. It does not change across queries, so - * we use it to correlate the replies of multiple queries. - */ - uint64 tag; // OUT - - CPUIDRegs regs; // OUT -} -#include "vmware_pack_end.h" -CPUIDReply; - -typedef -#include "vmware_pack_begin.h" -struct CPUIDQuery { - uint32 eax; // IN - uint32 ecx; // IN - uint32 numLogicalCPUs; // IN/OUT - CPUIDReply logicalCPUs[0]; // OUT -} -#include "vmware_pack_end.h" -CPUIDQuery; - -/* - * CPUID levels the monitor caches. - * - * The first parameter defines whether the level has its default masks - * generated from the values in this file. Any level which is marked - * as FALSE here *must* have all monitor support types set to NA. A - * static assert in lib/cpuidcompat/cpuidcompat.c will check this. - * - * The fourth parameter is a "sub leaf count", where 0 means that ecx - * is ignored, otherwise is the count of sub-leaves cached/supported. - */ - -#define CPUID_CACHED_LEVELS \ - CPUIDLEVEL(TRUE, 0, 0, 0) \ - CPUIDLEVEL(TRUE, 1, 1, 0) \ - CPUIDLEVEL(FALSE, 2, 2, 0) \ - CPUIDLEVEL(FALSE, 4, 4, 7) \ - CPUIDLEVEL(FALSE, 5, 5, 0) \ - CPUIDLEVEL(FALSE, 6, 6, 0) \ - CPUIDLEVEL(TRUE, 7, 7, 1) \ - CPUIDLEVEL(FALSE, A, 0xA, 0) \ - CPUIDLEVEL(FALSE, B, 0xB, 2) \ - CPUIDLEVEL(TRUE, D, 0xD, 4) \ - CPUIDLEVEL(FALSE, 12, 0x12, 4) \ - CPUIDLEVEL(FALSE, 400, 0x40000000, 0) \ - CPUIDLEVEL(FALSE, 401, 0x40000001, 0) \ - CPUIDLEVEL(FALSE, 402, 0x40000002, 0) \ - CPUIDLEVEL(FALSE, 403, 0x40000003, 0) \ - CPUIDLEVEL(FALSE, 404, 0x40000004, 0) \ - CPUIDLEVEL(FALSE, 405, 0x40000005, 0) \ - CPUIDLEVEL(FALSE, 406, 0x40000006, 0) \ - CPUIDLEVEL(FALSE, 410, 0x40000010, 0) \ - CPUIDLEVEL(FALSE, 80, 0x80000000, 0) \ - CPUIDLEVEL(TRUE, 81, 0x80000001, 0) \ - CPUIDLEVEL(FALSE, 82, 0x80000002, 0) \ - CPUIDLEVEL(FALSE, 83, 0x80000003, 0) \ - CPUIDLEVEL(FALSE, 84, 0x80000004, 0) \ - CPUIDLEVEL(FALSE, 85, 0x80000005, 0) \ - CPUIDLEVEL(FALSE, 86, 0x80000006, 0) \ - CPUIDLEVEL(FALSE, 87, 0x80000007, 0) \ - CPUIDLEVEL(FALSE, 88, 0x80000008, 0) \ - CPUIDLEVEL(TRUE, 8A, 0x8000000A, 0) \ - CPUIDLEVEL(FALSE, 819, 0x80000019, 0) \ - CPUIDLEVEL(FALSE, 81A, 0x8000001A, 0) \ - CPUIDLEVEL(FALSE, 81B, 0x8000001B, 0) \ - CPUIDLEVEL(FALSE, 81C, 0x8000001C, 0) \ - CPUIDLEVEL(FALSE, 81D, 0x8000001D, 5) \ - CPUIDLEVEL(FALSE, 81E, 0x8000001E, 0) - -#define CPUID_ALL_LEVELS CPUID_CACHED_LEVELS - -/* Define cached CPUID levels in the form: CPUID_LEVEL_ */ -typedef enum { -#define CPUIDLEVEL(t, s, v, c) CPUID_LEVEL_##s, - CPUID_CACHED_LEVELS -#undef CPUIDLEVEL - CPUID_NUM_CACHED_LEVELS -} CpuidCachedLevel; - -/* Enum to translate between shorthand name and actual CPUID level value. */ -enum { -#define CPUIDLEVEL(t, s, v, c) CPUID_LEVEL_VAL_##s = v, - CPUID_ALL_LEVELS -#undef CPUIDLEVEL -}; - - -/* Named feature leaves */ -#define CPUID_FEATURE_INFORMATION 0x01 -#define CPUID_PROCESSOR_TOPOLOGY 4 -#define CPUID_MWAIT_FEATURES 5 -#define CPUID_XSAVE_FEATURES 0xd -#define CPUID_HYPERVISOR_LEVEL_0 0x40000000 -#define CPUID_SVM_FEATURES 0x8000000a - - -/* - * CPUID result registers - */ - -#define CPUID_REGS \ - CPUIDREG(EAX, eax) \ - CPUIDREG(EBX, ebx) \ - CPUIDREG(ECX, ecx) \ - CPUIDREG(EDX, edx) - -typedef enum { -#define CPUIDREG(uc, lc) CPUID_REG_##uc, - CPUID_REGS -#undef CPUIDREG - CPUID_NUM_REGS -} CpuidReg; - -#define CPUID_INTEL_VENDOR_STRING "GenuntelineI" -#define CPUID_AMD_VENDOR_STRING "AuthcAMDenti" -#define CPUID_CYRIX_VENDOR_STRING "CyriteadxIns" -#define CPUID_VIA_VENDOR_STRING "CentaulsaurH" - -#define CPUID_HYPERV_HYPERVISOR_VENDOR_STRING "Microsoft Hv" -#define CPUID_KVM_HYPERVISOR_VENDOR_STRING "KVMKVMKVM\0\0\0" -#define CPUID_VMWARE_HYPERVISOR_VENDOR_STRING "VMwareVMware" -#define CPUID_XEN_HYPERVISOR_VENDOR_STRING "XenVMMXenVMM" - -#define CPUID_INTEL_VENDOR_STRING_FIXED "GenuineIntel" -#define CPUID_AMD_VENDOR_STRING_FIXED "AuthenticAMD" -#define CPUID_CYRIX_VENDOR_STRING_FIXED "CyrixInstead" -#define CPUID_VIA_VENDOR_STRING_FIXED "CentaurHauls" - -/* - * FIELD can be defined to process the CPUID information provided - * in the following CPUID_FIELD_DATA macro. The first parameter is - * the CPUID level of the feature (must be defined in - * CPUID_ALL_LEVELS, above. The second parameter is the CPUID result - * register in which the field is returned (defined in CPUID_REGS). - * The third field is the vendor(s) this feature applies to. "COMMON" - * means all vendors apply. UNKNOWN may not be used here. The fourth - * and fifth parameters are the bit position of the field and the - * width, respectively. The sixth is the text name of the field. - * - * The seventh parameters specifies the monitor support - * characteristics for this field. The value must be a valid - * CpuidFieldSupported value (omitting CPUID_FIELD_SUPPORT_ for - * convenience). The meaning of those values are described below. - * - * The eighth parameter describes whether the feature is capable of - * being used by usermode code (TRUE), or just CPL0 kernel code - * (FALSE). - * - * FLAG is defined identically to FIELD, but its accessors are more - * appropriate for 1-bit flags, and compile-time asserts enforce that - * the size is 1 bit wide. - */ - - -/* - * CpuidFieldSupported is made up of the following values: - * - * NO: A feature/field that IS NOT SUPPORTED by the monitor. Even - * if the host supports this feature, we will never expose it to - * the guest. - * - * YES: A feature/field that IS SUPPORTED by the monitor. If the - * host supports this feature, we will expose it to the guest. If - * not, then we will not set the feature. - * - * ANY: A feature/field that IS ALWAYS SUPPORTED by the monitor. - * Even if the host does not support the feature, the monitor can - * expose the feature to the guest. As with "YES", the guest cpuid - * value defaults to the host/evc cpuid value. But usually the - * guest cpuid value is recomputed at power on, ignoring the default - * value. - * - * - * NA: Only legal for levels not masked/tested by default (see - * above for this definition). Such fields must always be marked - * as NA. - * - * These distinctions, when combined with the feature's CPL3 - * properties can be translated into a common CPUID mask string as - * follows: - * - * NO + CPL3 --> "R" (Reserved). We don't support the feature, - * but we can't properly hide this from applications when using - * direct execution or HV with apps that do try/catch/fail, so we - * must still perform compatibility checks. - * - * NO + !CPL3 --> "0" (Masked). We can hide this from the guest. - * - * YES --> "H" (Host). We support the feature, so show it to the - * guest if the host has the feature. - * - * ANY/NA --> "X" (Ignore). By default, don't perform checks for - * this feature bit. Per-GOS masks may choose to set this bit in - * the guest. (e.g. the APIC feature bit is always set to 1.) - * - * See lib/cpuidcompat/cpuidcompat.c for any possible overrides to - * these defaults. - */ -typedef enum { - CPUID_FIELD_SUPPORTED_NO, - CPUID_FIELD_SUPPORTED_YES, - CPUID_FIELD_SUPPORTED_ANY, - CPUID_FIELD_SUPPORTED_NA, - CPUID_NUM_FIELD_SUPPORTEDS -} CpuidFieldSupported; - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_0 \ -FIELD( 0, 0, EAX, 0, 32, NUMLEVELS, ANY, FALSE) \ -FIELD( 0, 0, EBX, 0, 32, VENDOR1, YES, TRUE) \ -FIELD( 0, 0, ECX, 0, 32, VENDOR3, YES, TRUE) \ -FIELD( 0, 0, EDX, 0, 32, VENDOR2, YES, TRUE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_1 \ -FIELD( 1, 0, EAX, 0, 4, STEPPING, ANY, FALSE) \ -FIELD( 1, 0, EAX, 4, 4, MODEL, ANY, FALSE) \ -FIELD( 1, 0, EAX, 8, 4, FAMILY, YES, FALSE) \ -FIELD( 1, 0, EAX, 12, 2, TYPE, ANY, FALSE) \ -FIELD( 1, 0, EAX, 16, 4, EXTENDED_MODEL, ANY, FALSE) \ -FIELD( 1, 0, EAX, 20, 8, EXTENDED_FAMILY, YES, FALSE) \ -FIELD( 1, 0, EBX, 0, 8, BRAND_ID, ANY, FALSE) \ -FIELD( 1, 0, EBX, 8, 8, CLFL_SIZE, ANY, FALSE) \ -FIELD( 1, 0, EBX, 16, 8, LCPU_COUNT, ANY, FALSE) \ -FIELD( 1, 0, EBX, 24, 8, APICID, ANY, FALSE) \ -FLAG( 1, 0, ECX, 0, 1, SSE3, YES, TRUE) \ -FLAG( 1, 0, ECX, 1, 1, PCLMULQDQ, YES, TRUE) \ -FLAG( 1, 0, ECX, 2, 1, DTES64, NO, FALSE) \ -FLAG( 1, 0, ECX, 3, 1, MWAIT, YES, FALSE) \ -FLAG( 1, 0, ECX, 4, 1, DSCPL, NO, FALSE) \ -FLAG( 1, 0, ECX, 5, 1, VMX, YES, FALSE) \ -FLAG( 1, 0, ECX, 6, 1, SMX, NO, FALSE) \ -FLAG( 1, 0, ECX, 7, 1, EIST, NO, FALSE) \ -FLAG( 1, 0, ECX, 8, 1, TM2, NO, FALSE) \ -FLAG( 1, 0, ECX, 9, 1, SSSE3, YES, TRUE) \ -FLAG( 1, 0, ECX, 10, 1, CNXTID, NO, FALSE) \ -FLAG( 1, 0, ECX, 11, 1, SDBG, NO, FALSE) \ -FLAG( 1, 0, ECX, 12, 1, FMA, YES, TRUE) \ -FLAG( 1, 0, ECX, 13, 1, CMPXCHG16B, YES, TRUE) \ -FLAG( 1, 0, ECX, 14, 1, xTPR, NO, FALSE) \ -FLAG( 1, 0, ECX, 15, 1, PDCM, NO, FALSE) \ -FLAG( 1, 0, ECX, 17, 1, PCID, YES, FALSE) \ -FLAG( 1, 0, ECX, 18, 1, DCA, NO, FALSE) \ -FLAG( 1, 0, ECX, 19, 1, SSE41, YES, TRUE) \ -FLAG( 1, 0, ECX, 20, 1, SSE42, YES, TRUE) \ -FLAG( 1, 0, ECX, 21, 1, x2APIC, ANY, FALSE) \ -FLAG( 1, 0, ECX, 22, 1, MOVBE, YES, TRUE) \ -FLAG( 1, 0, ECX, 23, 1, POPCNT, YES, TRUE) \ -FLAG( 1, 0, ECX, 24, 1, TSC_DEADLINE, ANY, FALSE) \ -FLAG( 1, 0, ECX, 25, 1, AES, YES, TRUE) \ -FLAG( 1, 0, ECX, 26, 1, XSAVE, YES, FALSE) \ -FLAG( 1, 0, ECX, 27, 1, OSXSAVE, ANY, FALSE) \ -FLAG( 1, 0, ECX, 28, 1, AVX, YES, FALSE) \ -FLAG( 1, 0, ECX, 29, 1, F16C, YES, TRUE) \ -FLAG( 1, 0, ECX, 30, 1, RDRAND, YES, TRUE) \ -FLAG( 1, 0, ECX, 31, 1, HYPERVISOR, ANY, TRUE) \ -FLAG( 1, 0, EDX, 0, 1, FPU, YES, TRUE) \ -FLAG( 1, 0, EDX, 1, 1, VME, YES, FALSE) \ -FLAG( 1, 0, EDX, 2, 1, DE, YES, FALSE) \ -FLAG( 1, 0, EDX, 3, 1, PSE, YES, FALSE) \ -FLAG( 1, 0, EDX, 4, 1, TSC, YES, TRUE) \ -FLAG( 1, 0, EDX, 5, 1, MSR, YES, FALSE) \ -FLAG( 1, 0, EDX, 6, 1, PAE, YES, FALSE) \ -FLAG( 1, 0, EDX, 7, 1, MCE, YES, FALSE) \ -FLAG( 1, 0, EDX, 8, 1, CX8, YES, TRUE) \ -FLAG( 1, 0, EDX, 9, 1, APIC, ANY, FALSE) \ -FLAG( 1, 0, EDX, 11, 1, SEP, YES, TRUE) \ -FLAG( 1, 0, EDX, 12, 1, MTRR, YES, FALSE) \ -FLAG( 1, 0, EDX, 13, 1, PGE, YES, FALSE) \ -FLAG( 1, 0, EDX, 14, 1, MCA, YES, FALSE) \ -FLAG( 1, 0, EDX, 15, 1, CMOV, YES, TRUE) \ -FLAG( 1, 0, EDX, 16, 1, PAT, YES, FALSE) \ -FLAG( 1, 0, EDX, 17, 1, PSE36, YES, FALSE) \ -FLAG( 1, 0, EDX, 18, 1, PSN, YES, FALSE) \ -FLAG( 1, 0, EDX, 19, 1, CLFSH, YES, TRUE) \ -FLAG( 1, 0, EDX, 21, 1, DS, YES, FALSE) \ -FLAG( 1, 0, EDX, 22, 1, ACPI, ANY, FALSE) \ -FLAG( 1, 0, EDX, 23, 1, MMX, YES, TRUE) \ -FLAG( 1, 0, EDX, 24, 1, FXSR, YES, TRUE) \ -FLAG( 1, 0, EDX, 25, 1, SSE, YES, TRUE) \ -FLAG( 1, 0, EDX, 26, 1, SSE2, YES, TRUE) \ -FLAG( 1, 0, EDX, 27, 1, SS, YES, FALSE) \ -FLAG( 1, 0, EDX, 28, 1, HTT, ANY, FALSE) \ -FLAG( 1, 0, EDX, 29, 1, TM, NO, FALSE) \ -FLAG( 1, 0, EDX, 30, 1, IA64, NO, FALSE) \ -FLAG( 1, 0, EDX, 31, 1, PBE, NO, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_2 \ -FIELD( 2, 0, EAX, 0, 8, LEAF2_COUNT, NA, FALSE) \ -FIELD( 2, 0, EAX, 8, 8, LEAF2_CACHE1, NA, FALSE) \ -FIELD( 2, 0, EAX, 16, 8, LEAF2_CACHE2, NA, FALSE) \ -FIELD( 2, 0, EAX, 24, 8, LEAF2_CACHE3, NA, FALSE) \ -FIELD( 2, 0, EBX, 0, 8, LEAF2_CACHE4, NA, FALSE) \ -FIELD( 2, 0, EBX, 8, 8, LEAF2_CACHE5, NA, FALSE) \ -FIELD( 2, 0, EBX, 16, 8, LEAF2_CACHE6, NA, FALSE) \ -FIELD( 2, 0, EBX, 24, 8, LEAF2_CACHE7, NA, FALSE) \ -FIELD( 2, 0, ECX, 0, 8, LEAF2_CACHE8, NA, FALSE) \ -FIELD( 2, 0, ECX, 8, 8, LEAF2_CACHE9, NA, FALSE) \ -FIELD( 2, 0, ECX, 16, 8, LEAF2_CACHE10, NA, FALSE) \ -FIELD( 2, 0, ECX, 24, 8, LEAF2_CACHE11, NA, FALSE) \ -FIELD( 2, 0, EDX, 0, 8, LEAF2_CACHE12, NA, FALSE) \ -FIELD( 2, 0, EDX, 8, 8, LEAF2_CACHE13, NA, FALSE) \ -FIELD( 2, 0, EDX, 16, 8, LEAF2_CACHE14, NA, FALSE) \ -FIELD( 2, 0, EDX, 24, 8, LEAF2_CACHE15, NA, FALSE) \ - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_4 \ -FIELD( 4, 0, EAX, 0, 5, LEAF4_CACHE_TYPE, NA, FALSE) \ -FIELD( 4, 0, EAX, 5, 3, LEAF4_CACHE_LEVEL, NA, FALSE) \ -FLAG( 4, 0, EAX, 8, 1, LEAF4_CACHE_SELF_INIT, NA, FALSE) \ -FLAG( 4, 0, EAX, 9, 1, LEAF4_CACHE_FULLY_ASSOC, NA, FALSE) \ -FIELD( 4, 0, EAX, 14, 12, LEAF4_CACHE_NUMHT_SHARING, NA, FALSE) \ -FIELD( 4, 0, EAX, 26, 6, LEAF4_CORE_COUNT, NA, FALSE) \ -FIELD( 4, 0, EBX, 0, 12, LEAF4_CACHE_LINE, NA, FALSE) \ -FIELD( 4, 0, EBX, 12, 10, LEAF4_CACHE_PART, NA, FALSE) \ -FIELD( 4, 0, EBX, 22, 10, LEAF4_CACHE_WAYS, NA, FALSE) \ -FIELD( 4, 0, ECX, 0, 32, LEAF4_CACHE_SETS, NA, FALSE) \ -FLAG( 4, 0, EDX, 0, 1, LEAF4_CACHE_WBINVD_NOT_GUARANTEED, NA, FALSE) \ -FLAG( 4, 0, EDX, 1, 1, LEAF4_CACHE_IS_INCLUSIVE, NA, FALSE) \ -FLAG( 4, 0, EDX, 2, 1, LEAF4_CACHE_COMPLEX_INDEXING, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_5 \ -FIELD( 5, 0, EAX, 0, 16, MWAIT_MIN_SIZE, NA, FALSE) \ -FIELD( 5, 0, EBX, 0, 16, MWAIT_MAX_SIZE, NA, FALSE) \ -FLAG( 5, 0, ECX, 0, 1, MWAIT_EXTENSIONS, NA, FALSE) \ -FLAG( 5, 0, ECX, 1, 1, MWAIT_INTR_BREAK, NA, FALSE) \ -FIELD( 5, 0, EDX, 0, 4, MWAIT_C0_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 4, 4, MWAIT_C1_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 8, 4, MWAIT_C2_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 12, 4, MWAIT_C3_SUBSTATE, NA, FALSE) \ -FIELD( 5, 0, EDX, 16, 4, MWAIT_C4_SUBSTATE, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_6 \ -FLAG( 6, 0, EAX, 0, 1, THERMAL_SENSOR, NA, FALSE) \ -FLAG( 6, 0, EAX, 1, 1, TURBO_MODE, NA, FALSE) \ -FLAG( 6, 0, EAX, 2, 1, APIC_INVARIANT, NA, FALSE) \ -FLAG( 6, 0, EAX, 4, 1, PLN, NA, FALSE) \ -FLAG( 6, 0, EAX, 5, 1, ECMD, NA, FALSE) \ -FLAG( 6, 0, EAX, 6, 1, PTM, NA, FALSE) \ -FLAG( 6, 0, EAX, 7, 1, HWP, NA, FALSE) \ -FLAG( 6, 0, EAX, 8, 1, HWP_NOTIFICATION, NA, FALSE) \ -FLAG( 6, 0, EAX, 9, 1, HWP_ACTIVITY_WINDOW, NA, FALSE) \ -FLAG( 6, 0, EAX, 10, 1, HWP_ENERGY_PERFORMANCE_PREFERENCE, NA, FALSE) \ -FLAG( 6, 0, EAX, 11, 1, HWP_PACKAGE_LEVEL_REQUEST, NA, FALSE) \ -FLAG( 6, 0, EAX, 13, 1, HDC, NA, FALSE) \ -FIELD( 6, 0, EBX, 0, 4, NUM_INTR_THRESHOLDS, NA, FALSE) \ -FLAG( 6, 0, ECX, 0, 1, HW_COORD_FEEDBACK, NA, FALSE) \ -FLAG( 6, 0, ECX, 3, 1, ENERGY_PERF_BIAS, NA, FALSE) - - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_7 \ -FLAG( 7, 0, EBX, 0, 1, FSGSBASE, YES, FALSE) \ -FLAG( 7, 0, EBX, 1, 1, TSC_ADJUST, ANY, FALSE) \ -FLAG( 7, 0, EBX, 3, 1, BMI1, YES, TRUE) \ -FLAG( 7, 0, EBX, 2, 1, SGX, NO, FALSE) \ -FLAG( 7, 0, EBX, 4, 1, HLE, YES, TRUE) \ -FLAG( 7, 0, EBX, 5, 1, AVX2, YES, TRUE) \ -FLAG( 7, 0, EBX, 7, 1, SMEP, YES, FALSE) \ -FLAG( 7, 0, EBX, 8, 1, BMI2, YES, TRUE) \ -FLAG( 7, 0, EBX, 9, 1, ENFSTRG, YES, FALSE) \ -FLAG( 7, 0, EBX, 10, 1, INVPCID, YES, FALSE) \ -FLAG( 7, 0, EBX, 11, 1, RTM, YES, TRUE) \ -FLAG( 7, 0, EBX, 12, 1, PQM, NO, FALSE) \ -FLAG( 7, 0, EBX, 13, 1, FP_SEGMENT_ZERO, ANY, TRUE) \ -FLAG( 7, 0, EBX, 15, 1, PQE, NO, FALSE) \ -FLAG( 7, 0, EBX, 18, 1, RDSEED, YES, TRUE) \ -FLAG( 7, 0, EBX, 19, 1, ADX, YES, TRUE) \ -FLAG( 7, 0, EBX, 20, 1, SMAP, YES, FALSE) \ -FLAG( 7, 0, EBX, 25, 1, PT, NO, FALSE) \ -FLAG( 7, 0, ECX, 0, 1, PREFETCHWT1, NO, TRUE) - - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_A \ -FIELD( A, 0, EAX, 0, 8, PMC_VERSION, NA, FALSE) \ -FIELD( A, 0, EAX, 8, 8, PMC_NUM_GEN, NA, FALSE) \ -FIELD( A, 0, EAX, 16, 8, PMC_WIDTH_GEN, NA, FALSE) \ -FIELD( A, 0, EAX, 24, 8, PMC_EBX_LENGTH, NA, FALSE) \ -FLAG( A, 0, EBX, 0, 1, PMC_CORE_CYCLES, NA, FALSE) \ -FLAG( A, 0, EBX, 1, 1, PMC_INSTR_RETIRED, NA, FALSE) \ -FLAG( A, 0, EBX, 2, 1, PMC_REF_CYCLES, NA, FALSE) \ -FLAG( A, 0, EBX, 3, 1, PMC_LAST_LVL_CREF, NA, FALSE) \ -FLAG( A, 0, EBX, 4, 1, PMC_LAST_LVL_CMISS, NA, FALSE) \ -FLAG( A, 0, EBX, 5, 1, PMC_BR_INST_RETIRED, NA, FALSE) \ -FLAG( A, 0, EBX, 6, 1, PMC_BR_MISS_RETIRED, NA, FALSE) \ -FIELD( A, 0, EDX, 0, 5, PMC_NUM_FIXED, NA, FALSE) \ -FIELD( A, 0, EDX, 5, 8, PMC_WIDTH_FIXED, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_B \ -FIELD( B, 0, EAX, 0, 5, TOPOLOGY_MASK_WIDTH, NA, FALSE) \ -FIELD( B, 0, EBX, 0, 16, TOPOLOGY_CPUS_SHARING_LEVEL, NA, FALSE) \ -FIELD( B, 0, ECX, 0, 8, TOPOLOGY_LEVEL_NUMBER, NA, FALSE) \ -FIELD( B, 0, ECX, 8, 8, TOPOLOGY_LEVEL_TYPE, NA, FALSE) \ -FIELD( B, 0, EDX, 0, 32, TOPOLOGY_X2APIC_ID, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_D \ -FLAG( D, 0, EAX, 0, 1, XCR0_MASTER_LEGACY_FP, YES, FALSE) \ -FLAG( D, 0, EAX, 1, 1, XCR0_MASTER_SSE, YES, FALSE) \ -FLAG( D, 0, EAX, 2, 1, XCR0_MASTER_YMM_H, YES, FALSE) \ -FIELD( D, 0, EAX, 3, 29, XCR0_MASTER_LOWER, NO, FALSE) \ -FIELD( D, 0, EBX, 0, 32, XSAVE_ENABLED_SIZE, ANY, FALSE) \ -FIELD( D, 0, ECX, 0, 32, XSAVE_MAX_SIZE, YES, FALSE) \ -FIELD( D, 0, EDX, 0, 29, XCR0_MASTER_UPPER, NO, FALSE) \ -FLAG( D, 0, EDX, 30, 1, XCR0_MASTER_LWP, NO, FALSE) \ -FLAG( D, 0, EDX, 31, 1, XCR0_MASTER_EXTENDED_XSAVE, NO, FALSE) \ -FLAG( D, 1, EAX, 0, 1, XSAVEOPT, YES, FALSE) \ -FLAG( D, 1, EAX, 1, 1, XSAVEC, NO, FALSE) \ -FLAG( D, 1, EAX, 2, 1, XGETBV_ECX1, NO, FALSE) \ -FLAG( D, 1, EAX, 3, 1, XSAVES, NO, FALSE) \ -FIELD( D, 1, EBX, 0, 32, XSAVE_XSS_SIZE, NO, FALSE) \ -FIELD( D, 1, ECX, 0, 32, XSS_LOWER, NO, FALSE) \ -FIELD( D, 1, EDX, 0, 32, XSS_UPPER, NO, FALSE) \ -FIELD( D, 2, EAX, 0, 32, XSAVE_YMM_SIZE, YES, FALSE) \ -FIELD( D, 2, EBX, 0, 32, XSAVE_YMM_OFFSET, YES, FALSE) \ -FIELD( D, 2, ECX, 0, 32, XSAVE_YMM_RSVD1, YES, FALSE) \ -FIELD( D, 2, EDX, 0, 32, XSAVE_YMM_RSVD2, YES, FALSE) \ -FIELD( D, 62, EAX, 0, 32, XSAVE_LWP_SIZE, NO, FALSE) \ -FIELD( D, 62, EBX, 0, 32, XSAVE_LWP_OFFSET, NO, FALSE) \ -FIELD( D, 62, ECX, 0, 32, XSAVE_LWP_RSVD1, NO, FALSE) \ -FIELD( D, 62, EDX, 0, 32, XSAVE_LWP_RSVD2, NO, FALSE) - - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_12 \ -FLAG( 12, 0, EAX, 0, 1, SGX1, NA, FALSE) \ -FLAG( 12, 0, EAX, 1, 1, SGX2, NA, FALSE) \ -FLAG( 12, 0, EBX, 31, 1, SGX_MISCSELECT, NA, FALSE) \ -FIELD( 12, 0, EDX, 0, 8, MAX_ENCLAVE_SIZE_NOT64, NA, FALSE) \ -FIELD( 12, 0, EDX, 8, 8, MAX_ENCLAVE_SIZE_64, NA, FALSE) \ -FIELD( 12, 1, EAX, 0, 32, SECS_ATTRIBUTES0, NA, FALSE) \ -FIELD( 12, 1, EBX, 0, 32, SECS_ATTRIBUTES1, NA, FALSE) \ -FIELD( 12, 1, ECX, 0, 32, SECS_ATTRIBUTES2, NA, FALSE) \ -FIELD( 12, 1, EDX, 0, 32, SECS_ATTRIBUTES3, NA, FALSE) \ -FIELD( 12, 2, EAX, 0, 4, EPC00_VALID, NA, FALSE) \ -FIELD( 12, 2, EAX, 12, 20, EPC00_BASE_LOW, NA, FALSE) \ -FIELD( 12, 2, EBX, 0, 20, EPC00_BASE_HIGH, NA, FALSE) \ -FIELD( 12, 2, ECX, 0, 4, EPC00_PROTECTED, NA, FALSE) \ -FIELD( 12, 2, ECX, 12, 20, EPC00_SIZE_LOW, NA, FALSE) \ -FIELD( 12, 2, EDX, 0, 20, EPC00_SIZE_HIGH, NA, FALSE) \ -FIELD( 12, 3, EAX, 0, 4, EPC01_VALID, NA, FALSE) \ -FIELD( 12, 3, EAX, 12, 20, EPC01_BASE_LOW, NA, FALSE) \ -FIELD( 12, 3, EBX, 0, 20, EPC01_BASE_HIGH, NA, FALSE) \ -FIELD( 12, 3, ECX, 0, 4, EPC01_PROTECTED, NA, FALSE) \ -FIELD( 12, 3, ECX, 12, 20, EPC01_SIZE_LOW, NA, FALSE) \ -FIELD( 12, 3, EDX, 0, 20, EPC01_SIZE_HIGH, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_400 \ -FIELD(400, 0, EAX, 0, 32, MAX_HYP_LEVEL, NA, FALSE) \ -FIELD(400, 0, EBX, 0, 32, HYPERVISOR_VENDOR0, NA, FALSE) \ -FIELD(400, 0, ECX, 0, 32, HYPERVISOR_VENDOR1, NA, FALSE) \ -FIELD(400, 0, EDX, 0, 32, HYPERVISOR_VENDOR2, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_401 \ -FIELD(401, 0, EAX, 0, 32, HV_INTERFACE_SIGNATURE, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_402 \ -FIELD(402, 0, EAX, 0, 32, BUILD_NUMBER, NA, FALSE) \ -FIELD(402, 0, EBX, 0, 16, MINOR_VERSION, NA, FALSE) \ -FIELD(402, 0, EBX, 16, 16, MAJOR_VERSION, NA, FALSE) \ -FIELD(402, 0, ECX, 0, 32, SERVICE_PACK, NA, FALSE) \ -FIELD(402, 0, EDX, 0, 24, SERVICE_NUMBER, NA, FALSE) \ -FIELD(402, 0, EDX, 24, 8, SERVICE_BRANCH, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_403 \ -FLAG( 403, 0, EAX, 0, 1, VP_RUNTIME_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 1, 1, REF_COUNTER_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 2, 1, BASIC_SYNIC_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 3, 1, SYNTH_TIMER_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 4, 1, APIC_ACCESS_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 5, 1, HYPERCALL_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 6, 1, VP_INDEX_MSR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 7, 1, VIRT_RESET_MSR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 8, 1, STATS_PAGES_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 9, 1, REF_TSC_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 10, 1, GUEST_IDLE_MSR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 11, 1, FREQUENCY_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EAX, 12, 1, SYNTH_DEBUG_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EBX, 0, 1, CREATE_PARTITIONS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 1, 1, ACCESS_PARTITION_ID_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 2, 1, ACCESS_MEMORY_POOL_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 3, 1, ADJUST_MESSAGE_BUFFERS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 4, 1, POST_MESSAGES_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 5, 1, SIGNAL_EVENTS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 6, 1, CREATE_PORT_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 7, 1, CONNECT_PORT_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 8, 1, ACCESS_STATS_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 11, 1, DEBUGGING_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 12, 1, CPU_MANAGEMENT_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 13, 1, CONFIGURE_PROFILER_FLAG, NA, FALSE) \ -FLAG( 403, 0, EBX, 14, 1, ENABLE_EXPANDED_STACKWALKING_FLAG, NA, FALSE) \ -FIELD(403, 0, ECX, 0, 4, MAX_POWER_STATE, NA, FALSE) \ -FLAG( 403, 0, ECX, 4, 1, HPET_NEEDED_FOR_C3, NA, FALSE) \ -FLAG( 403, 0, EDX, 0, 1, MWAIT_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 1, 1, GUEST_DEBUGGING_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 2, 1, PERFORMANCE_MONITOR_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 3, 1, CPU_DYN_PARTITIONING_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 4, 1, XMM_REGISTERS_FOR_HYPERCALL_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 5, 1, GUEST_IDLE_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 6, 1, HYPERVISOR_SLEEP_STATE_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 7, 1, NUMA_DISTANCE_QUERY_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 8, 1, TIMER_FREQUENCY_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 9, 1, SYNTH_MACHINE_CHECK_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 10, 1, GUEST_CRASH_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 11, 1, DEBUG_MSRS_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 12, 1, NPIEP1_AVAIL, NA, FALSE) \ -FLAG( 403, 0, EDX, 13, 1, DISABLE_HYPERVISOR_AVAIL, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_404 \ -FLAG( 404, 0, EAX, 0, 1, USE_HYPERCALL_TO_SWITCH_ADDR_SPACE, NA, FALSE) \ -FLAG( 404, 0, EAX, 1, 1, USE_HYPERCALL_TO_FLUSH_TLB, NA, FALSE) \ -FLAG( 404, 0, EAX, 2, 1, USE_HYPERCALL_FOR_TLB_SHOOTDOWN, NA, FALSE) \ -FLAG( 404, 0, EAX, 3, 1, USE_MSRS_FOR_EOI_ICR_TPR, NA, FALSE) \ -FLAG( 404, 0, EAX, 4, 1, USE_MSR_FOR_RESET, NA, FALSE) \ -FLAG( 404, 0, EAX, 5, 1, USE_RELAXED_TIMING, NA, FALSE) \ -FLAG( 404, 0, EAX, 6, 1, USE_DMA_REMAPPING, NA, FALSE) \ -FLAG( 404, 0, EAX, 7, 1, USE_INTERRUPT_REMAPPING, NA, FALSE) \ -FLAG( 404, 0, EAX, 8, 1, USE_X2APIC, NA, FALSE) \ -FLAG( 404, 0, EAX, 9, 1, DEPRECATE_AUTOEOI, NA, FALSE) \ -FIELD(404, 0, EBX, 0, 32, SPINLOCK_RETRIES, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_405 \ -FIELD(405, 0, EAX, 0, 32, MAX_VCPU, NA, FALSE) \ -FIELD(405, 0, EBX, 0, 32, MAX_LCPU, NA, FALSE) \ -FIELD(405, 0, ECX, 0, 32, MAX_REMAPPABLE_VECTORS, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_406 \ -FLAG( 406, 0, EAX, 0, 1, APIC_OVERLAY_ASSIST, NA, FALSE) \ -FLAG( 406, 0, EAX, 1, 1, MSR_BITMAPS, NA, FALSE) \ -FLAG( 406, 0, EAX, 2, 1, ARCH_PMCS, NA, FALSE) \ -FLAG( 406, 0, EAX, 3, 1, SLAT, NA, FALSE) \ -FLAG( 406, 0, EAX, 4, 1, DMA_REMAPPING, NA, FALSE) \ -FLAG( 406, 0, EAX, 5, 1, INTERRUPT_REMAPPING, NA, FALSE) \ -FLAG( 406, 0, EAX, 6, 1, MEMORY_PATROL_SCRUBBER, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_410 \ -FIELD(410, 0, EAX, 0, 32, TSC_HZ, NA, FALSE) \ -FIELD(410, 0, EBX, 0, 32, ACPIBUS_HZ, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_80 \ -FIELD( 80, 0, EAX, 0, 32, NUM_EXT_LEVELS, NA, FALSE) \ -FIELD( 80, 0, EBX, 0, 32, LEAF80_VENDOR1, NA, FALSE) \ -FIELD( 80, 0, ECX, 0, 32, LEAF80_VENDOR3, NA, FALSE) \ -FIELD( 80, 0, EDX, 0, 32, LEAF80_VENDOR2, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_81 \ -FIELD( 81, 0, EAX, 0, 32, UNKNOWN81EAX, ANY, FALSE) \ -FIELD( 81, 0, EAX, 0, 4, LEAF81_STEPPING, ANY, FALSE) \ -FIELD( 81, 0, EAX, 4, 4, LEAF81_MODEL, ANY, FALSE) \ -FIELD( 81, 0, EAX, 8, 4, LEAF81_FAMILY, ANY, FALSE) \ -FIELD( 81, 0, EAX, 12, 2, LEAF81_TYPE, ANY, FALSE) \ -FIELD( 81, 0, EAX, 16, 4, LEAF81_EXTENDED_MODEL, ANY, FALSE) \ -FIELD( 81, 0, EAX, 20, 8, LEAF81_EXTENDED_FAMILY, ANY, FALSE) \ -FIELD( 81, 0, EBX, 0, 32, UNKNOWN81EBX, ANY, FALSE) \ -FIELD( 81, 0, EBX, 0, 16, LEAF81_BRAND_ID, ANY, FALSE) \ -FIELD( 81, 0, EBX, 16, 16, UNDEF, ANY, FALSE) \ -FLAG( 81, 0, ECX, 0, 1, LAHF64, YES, TRUE) \ -FLAG( 81, 0, ECX, 1, 1, CMPLEGACY, ANY, FALSE) \ -FLAG( 81, 0, ECX, 2, 1, SVM, YES, FALSE) \ -FLAG( 81, 0, ECX, 3, 1, EXTAPICSPC, YES, FALSE) \ -FLAG( 81, 0, ECX, 4, 1, CR8AVAIL, YES, FALSE) \ -FLAG( 81, 0, ECX, 5, 1, ABM, YES, TRUE) \ -FLAG( 81, 0, ECX, 6, 1, SSE4A, YES, TRUE) \ -FLAG( 81, 0, ECX, 7, 1, MISALIGNED_SSE, YES, TRUE) \ -FLAG( 81, 0, ECX, 8, 1, 3DNPREFETCH, YES, TRUE) \ -FLAG( 81, 0, ECX, 9, 1, OSVW, ANY, FALSE) \ -FLAG( 81, 0, ECX, 10, 1, IBS, NO, FALSE) \ -FLAG( 81, 0, ECX, 11, 1, XOP, YES, TRUE) \ -FLAG( 81, 0, ECX, 12, 1, SKINIT, NO, FALSE) \ -FLAG( 81, 0, ECX, 13, 1, WATCHDOG, NO, FALSE) \ -FLAG( 81, 0, ECX, 15, 1, LWP, NO, FALSE) \ -FLAG( 81, 0, ECX, 16, 1, FMA4, YES, TRUE) \ -FLAG( 81, 0, ECX, 17, 1, TCE, NO, FALSE) \ -FLAG( 81, 0, ECX, 19, 1, NODEID_MSR, NO, FALSE) \ -FLAG( 81, 0, ECX, 21, 1, TBM, YES, TRUE) \ -FLAG( 81, 0, ECX, 22, 1, TOPOLOGY, NO, FALSE) \ -FLAG( 81, 0, ECX, 23, 1, PERFCORE, ANY, TRUE) \ -FLAG( 81, 0, EDX, 0, 1, LEAF81_FPU, YES, TRUE) \ -FLAG( 81, 0, EDX, 1, 1, LEAF81_VME, YES, FALSE) \ -FLAG( 81, 0, EDX, 2, 1, LEAF81_DE, YES, FALSE) \ -FLAG( 81, 0, EDX, 3, 1, LEAF81_PSE, YES, FALSE) \ -FLAG( 81, 0, EDX, 4, 1, LEAF81_TSC, YES, TRUE) \ -FLAG( 81, 0, EDX, 5, 1, LEAF81_MSR, YES, FALSE) \ -FLAG( 81, 0, EDX, 6, 1, LEAF81_PAE, YES, FALSE) \ -FLAG( 81, 0, EDX, 7, 1, LEAF81_MCE, YES, FALSE) \ -FLAG( 81, 0, EDX, 8, 1, LEAF81_CX8, YES, TRUE) \ -FLAG( 81, 0, EDX, 9, 1, LEAF81_APIC, ANY, FALSE) \ -FLAG( 81, 0, EDX, 11, 1, SYSC, ANY, TRUE) \ -FLAG( 81, 0, EDX, 12, 1, LEAF81_MTRR, YES, FALSE) \ -FLAG( 81, 0, EDX, 13, 1, LEAF81_PGE, YES, FALSE) \ -FLAG( 81, 0, EDX, 14, 1, LEAF81_MCA, YES, FALSE) \ -FLAG( 81, 0, EDX, 15, 1, LEAF81_CMOV, YES, TRUE) \ -FLAG( 81, 0, EDX, 16, 1, LEAF81_PAT, YES, FALSE) \ -FLAG( 81, 0, EDX, 17, 1, LEAF81_PSE36, YES, FALSE) \ -FLAG( 81, 0, EDX, 20, 1, NX, YES, FALSE) \ -FLAG( 81, 0, EDX, 22, 1, MMXEXT, YES, TRUE) \ -FLAG( 81, 0, EDX, 23, 1, LEAF81_MMX, YES, TRUE) \ -FLAG( 81, 0, EDX, 24, 1, LEAF81_FXSR, YES, TRUE) \ -FLAG( 81, 0, EDX, 25, 1, FFXSR, YES, FALSE) \ -FLAG( 81, 0, EDX, 26, 1, PDPE1GB, YES, FALSE) \ -FLAG( 81, 0, EDX, 27, 1, RDTSCP, YES, TRUE) \ -FLAG( 81, 0, EDX, 29, 1, LM, YES, FALSE) \ -FLAG( 81, 0, EDX, 30, 1, 3DNOWPLUS, YES, TRUE) \ -FLAG( 81, 0, EDX, 31, 1, 3DNOW, YES, TRUE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_82 \ -FIELD( 82, 0, EAX, 0, 32, LEAF82_BRAND_STRING_EAX, NA, FALSE) \ -FIELD( 82, 0, EBX, 0, 32, LEAF82_BRAND_STRING_EBX, NA, FALSE) \ -FIELD( 82, 0, ECX, 0, 32, LEAF82_BRAND_STRING_ECX, NA, FALSE) \ -FIELD( 82, 0, EDX, 0, 32, LEAF82_BRAND_STRING_EDX, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_83 \ -FIELD( 83, 0, EAX, 0, 32, LEAF83_BRAND_STRING_EAX, NA, FALSE) \ -FIELD( 83, 0, EBX, 0, 32, LEAF83_BRAND_STRING_EBX, NA, FALSE) \ -FIELD( 83, 0, ECX, 0, 32, LEAF83_BRAND_STRING_ECX, NA, FALSE) \ -FIELD( 83, 0, EDX, 0, 32, LEAF83_BRAND_STRING_EDX, NA, FALSE) - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_84 \ -FIELD( 84, 0, EAX, 0, 32, LEAF84_BRAND_STRING_EAX, NA, FALSE) \ -FIELD( 84, 0, EBX, 0, 32, LEAF84_BRAND_STRING_EBX, NA, FALSE) \ -FIELD( 84, 0, ECX, 0, 32, LEAF84_BRAND_STRING_ECX, NA, FALSE) \ -FIELD( 84, 0, EDX, 0, 32, LEAF84_BRAND_STRING_EDX, NA, FALSE) - -#ifdef COMMUNITY_SOURCE_AMD_SECRET /* { */ -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_8A_EDX_11 \ -FLAG( 8A, 0, EDX, 11, 1, SKINIT_PATCH_RETAINED, NO, FALSE) -#define CPUID_8A_EDX_14_31 \ -FLAG( 8A, 0, EDX, 14, 1, SVM_X2APIC, NO, FALSE) \ -FIELD( 8A, 0, EDX, 15, 17, SVMEDX_RSVD2, NO, FALSE) -#else -#define CPUID_8A_EDX_11 \ -FLAG( 8A, 0, EDX, 11, 1, SVMEDX_RSVD1, NO, FALSE) -#define CPUID_8A_EDX_14_31 \ -FIELD( 8A, 0, EDX, 14, 18, SVMEDX_RSVD2, NO, FALSE) -#endif /* } COMMUNITY_SOURCE_AMD_SECRET */ - -/* LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_8x \ -FIELD( 85, 0, EAX, 0, 8, ITLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EAX, 8, 8, ITLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EAX, 16, 8, DTLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EAX, 24, 8, DTLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 0, 8, ITLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 8, 8, ITLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 16, 8, DTLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, EBX, 24, 8, DTLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 85, 0, ECX, 0, 8, L1_DCACHE_LINE_SIZE, NA, FALSE) \ -FIELD( 85, 0, ECX, 8, 8, L1_DCACHE_LINES_PER_TAG, NA, FALSE) \ -FIELD( 85, 0, ECX, 16, 8, L1_DCACHE_ASSOC, NA, FALSE) \ -FIELD( 85, 0, ECX, 24, 8, L1_DCACHE_SIZE, NA, FALSE) \ -FIELD( 85, 0, EDX, 0, 8, L1_ICACHE_LINE_SIZE, NA, FALSE) \ -FIELD( 85, 0, EDX, 8, 8, L1_ICACHE_LINES_PER_TAG, NA, FALSE) \ -FIELD( 85, 0, EDX, 16, 8, L1_ICACHE_ASSOC, NA, FALSE) \ -FIELD( 85, 0, EDX, 24, 8, L1_ICACHE_SIZE, NA, FALSE) \ -FIELD( 86, 0, EAX, 0, 12, L2_ITLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EAX, 12, 4, L2_ITLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EAX, 16, 12, L2_DTLB_ENTRIES_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EAX, 28, 4, L2_DTLB_ASSOC_2M4M_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 0, 12, L2_ITLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 12, 4, L2_ITLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 16, 12, L2_DTLB_ENTRIES_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, EBX, 28, 4, L2_DTLB_ASSOC_4K_PGS, NA, FALSE) \ -FIELD( 86, 0, ECX, 0, 8, L2CACHE_LINE, NA, FALSE) \ -FIELD( 86, 0, ECX, 8, 4, L2CACHE_LINE_PER_TAG, NA, FALSE) \ -FIELD( 86, 0, ECX, 12, 4, L2CACHE_WAYS, NA, FALSE) \ -FIELD( 86, 0, ECX, 16, 16, L2CACHE_SIZE, NA, FALSE) \ -FIELD( 86, 0, EDX, 0, 8, L3CACHE_LINE, NA, FALSE) \ -FIELD( 86, 0, EDX, 8, 4, L3CACHE_LINE_PER_TAG, NA, FALSE) \ -FIELD( 86, 0, EDX, 12, 4, L3CACHE_WAYS, NA, FALSE) \ -FIELD( 86, 0, EDX, 18, 14, L3CACHE_SIZE, NA, FALSE) \ -FLAG( 87, 0, EDX, 0, 1, TS, NA, FALSE) \ -FLAG( 87, 0, EDX, 1, 1, FID, NA, FALSE) \ -FLAG( 87, 0, EDX, 2, 1, VID, NA, FALSE) \ -FLAG( 87, 0, EDX, 3, 1, TTP, NA, FALSE) \ -FLAG( 87, 0, EDX, 4, 1, LEAF87_TM, NA, FALSE) \ -FLAG( 87, 0, EDX, 5, 1, STC, NA, FALSE) \ -FLAG( 87, 0, EDX, 6, 1, 100MHZSTEPS, NA, FALSE) \ -FLAG( 87, 0, EDX, 7, 1, HWPSTATE, NA, FALSE) \ -FLAG( 87, 0, EDX, 8, 1, TSC_INVARIANT, NA, FALSE) \ -FLAG( 87, 0, EDX, 9, 1, CORE_PERF_BOOST, NA, FALSE) \ -FIELD( 88, 0, EAX, 0, 8, PHYS_BITS, NA, FALSE) \ -FIELD( 88, 0, EAX, 8, 8, VIRT_BITS, NA, FALSE) \ -FIELD( 88, 0, EAX, 16, 8, GUEST_PHYS_ADDR_SZ, NA, FALSE) \ -FIELD( 88, 0, ECX, 0, 8, LEAF88_CORE_COUNT, NA, FALSE) \ -FIELD( 88, 0, ECX, 12, 4, APICID_COREID_SIZE, NA, FALSE) \ -FIELD( 8A, 0, EAX, 0, 8, SVM_REVISION, YES, FALSE) \ -FLAG( 8A, 0, EAX, 8, 1, SVM_HYPERVISOR, NO, FALSE) \ -FIELD( 8A, 0, EAX, 9, 23, SVMEAX_RSVD, NO, FALSE) \ -FIELD( 8A, 0, EBX, 0, 32, SVM_NUM_ASIDS, YES, FALSE) \ -FIELD( 8A, 0, ECX, 0, 32, SVMECX_RSVD, NO, FALSE) \ -FLAG( 8A, 0, EDX, 0, 1, SVM_NPT, YES, FALSE) \ -FLAG( 8A, 0, EDX, 1, 1, SVM_LBR, NO, FALSE) \ -FLAG( 8A, 0, EDX, 2, 1, SVM_LOCK, ANY, FALSE) \ -FLAG( 8A, 0, EDX, 3, 1, SVM_NRIP, YES, FALSE) \ -FLAG( 8A, 0, EDX, 4, 1, SVM_TSC_RATE_MSR, NO, FALSE) \ -FLAG( 8A, 0, EDX, 5, 1, SVM_VMCB_CLEAN, YES, FALSE) \ -FLAG( 8A, 0, EDX, 6, 1, SVM_FLUSH_BY_ASID, YES, FALSE) \ -FLAG( 8A, 0, EDX, 7, 1, SVM_DECODE_ASSISTS, YES, FALSE) \ -FIELD( 8A, 0, EDX, 8, 2, SVMEDX_RSVD0, NO, FALSE) \ -FLAG( 8A, 0, EDX, 10, 1, SVM_PAUSE_FILTER, NO, FALSE) \ -CPUID_8A_EDX_11 \ -FLAG( 8A, 0, EDX, 12, 1, SVM_PAUSE_THRESHOLD, NO, FALSE) \ -FLAG( 8A, 0, EDX, 13, 1, SVM_AVIC, NO, FALSE) \ -CPUID_8A_EDX_14_31 - -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define CPUID_FIELD_DATA_LEVEL_81x \ -FIELD(819, 0, EAX, 0, 12, L1_ITLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EAX, 12, 4, L1_ITLB_ASSOC_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EAX, 16, 12, L1_DTLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EAX, 28, 4, L1_DTLB_ASSOC_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 0, 12, L2_ITLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 12, 4, L2_ITLB_ASSOC_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 16, 12, L2_DTLB_ENTRIES_1G_PGS, NA, FALSE) \ -FIELD(819, 0, EBX, 28, 4, L2_DTLB_ASSOC_1G_PGS, NA, FALSE) \ -FLAG( 81A, 0, EAX, 0, 1, FP128, NA, FALSE) \ -FLAG( 81A, 0, EAX, 1, 1, MOVU, NA, FALSE) \ -FLAG( 81B, 0, EAX, 0, 1, IBS_FFV, NA, FALSE) \ -FLAG( 81B, 0, EAX, 1, 1, IBS_FETCHSAM, NA, FALSE) \ -FLAG( 81B, 0, EAX, 2, 1, IBS_OPSAM, NA, FALSE) \ -FLAG( 81B, 0, EAX, 3, 1, RW_OPCOUNT, NA, FALSE) \ -FLAG( 81B, 0, EAX, 4, 1, OPCOUNT, NA, FALSE) \ -FLAG( 81B, 0, EAX, 5, 1, BRANCH_TARGET_ADDR, NA, FALSE) \ -FLAG( 81B, 0, EAX, 6, 1, OPCOUNT_EXT, NA, FALSE) \ -FLAG( 81B, 0, EAX, 7, 1, RIP_INVALID_CHECK, NA, FALSE) \ -FLAG( 81C, 0, EAX, 0, 1, LWP_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 1, 1, LWP_VAL_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 2, 1, LWP_IRE_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 3, 1, LWP_BRE_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 4, 1, LWP_DME_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 5, 1, LWP_CNH_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 6, 1, LWP_RNH_AVAIL, NA, FALSE) \ -FLAG( 81C, 0, EAX, 31, 1, LWP_INT_AVAIL, NA, FALSE) \ -FIELD(81C, 0, EBX, 0, 8, LWP_CB_SIZE, NA, FALSE) \ -FIELD(81C, 0, EBX, 8, 8, LWP_EVENT_SIZE, NA, FALSE) \ -FIELD(81C, 0, EBX, 16, 8, LWP_MAX_EVENTS, NA, FALSE) \ -FIELD(81C, 0, EBX, 24, 8, LWP_EVENT_OFFSET, NA, FALSE) \ -FIELD(81C, 0, ECX, 0, 4, LWP_LATENCY_MAX, NA, FALSE) \ -FLAG( 81C, 0, ECX, 5, 1, LWP_DATA_ADDR_VALID, NA, FALSE) \ -FIELD(81C, 0, ECX, 6, 3, LWP_LATENCY_ROUND, NA, FALSE) \ -FIELD(81C, 0, ECX, 9, 7, LWP_VERSION, NA, FALSE) \ -FIELD(81C, 0, ECX, 16, 8, LWP_MIN_BUF_SIZE, NA, FALSE) \ -FLAG( 81C, 0, ECX, 28, 1, LWP_BRANCH_PRED, NA, FALSE) \ -FLAG( 81C, 0, ECX, 29, 1, LWP_IP_FILTERING, NA, FALSE) \ -FLAG( 81C, 0, ECX, 30, 1, LWP_CACHE_LEVEL, NA, FALSE) \ -FLAG( 81C, 0, ECX, 31, 1, LWP_CACHE_LATENCY, NA, FALSE) \ -FLAG( 81C, 0, EDX, 0, 1, LWP_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 1, 1, LWP_VAL_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 2, 1, LWP_IRE_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 3, 1, LWP_BRE_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 4, 1, LWP_DME_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 5, 1, LWP_CNH_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 6, 1, LWP_RNH_SUPPORTED, NA, FALSE) \ -FLAG( 81C, 0, EDX, 31, 1, LWP_INT_SUPPORTED, NA, FALSE) \ -FIELD(81D, 0, EAX, 0, 5, LEAF81D_CACHE_TYPE, NA, FALSE) \ -FIELD(81D, 0, EAX, 5, 3, LEAF81D_CACHE_LEVEL, NA, FALSE) \ -FLAG( 81D, 0, EAX, 8, 1, LEAF81D_CACHE_SELF_INIT, NA, FALSE) \ -FLAG( 81D, 0, EAX, 9, 1, LEAF81D_CACHE_FULLY_ASSOC, NA, FALSE) \ -FIELD(81D, 0, EAX, 14, 12, LEAF81D_NUM_SHARING_CACHE, NA, FALSE) \ -FIELD(81D, 0, EBX, 0, 12, LEAF81D_CACHE_LINE_SIZE, NA, FALSE) \ -FIELD(81D, 0, EBX, 12, 10, LEAF81D_CACHE_PHYS_PARTITIONS, NA, FALSE) \ -FIELD(81D, 0, EBX, 22, 10, LEAF81D_CACHE_WAYS, NA, FALSE) \ -FIELD(81D, 0, ECX, 0, 32, LEAF81D_CACHE_NUM_SETS, NA, FALSE) \ -FLAG( 81D, 0, EDX, 0, 1, LEAF81D_CACHE_WBINVD, NA, FALSE) \ -FLAG( 81D, 0, EDX, 1, 1, LEAF81D_CACHE_INCLUSIVE, NA, FALSE) \ -FIELD(81E, 0, EAX, 0, 32, EXTENDED_APICID, NA, FALSE) \ -FIELD(81E, 0, EBX, 0, 8, COMPUTE_UNIT_ID, NA, FALSE) \ -FIELD(81E, 0, EBX, 8, 2, CORES_PER_COMPUTE_UNIT, NA, FALSE) \ -FIELD(81E, 0, ECX, 0, 8, NODEID_VAL, NA, FALSE) \ -FIELD(81E, 0, ECX, 8, 3, NODES_PER_PKG, NA, FALSE) - -#define INTEL_CPUID_FIELD_DATA - -#ifdef COMMUNITY_SOURCE_AMD_SECRET -/* These CPUID bit definitions are not yet public either. */ -/* LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME, MON SUPP, CPL3 */ -#define AMD_CPUID_FIELD_DATA \ -FLAG( 81, 0, ECX, 14, 1, TBM0, NO, FALSE) \ -FLAG( 81, 0, ECX, 23, 1, PERFCTR_EXT_NB, NO, FALSE) \ -FLAG( 81, 0, ECX, 24, 1, PERFCTR_EXT_CORE, NO, FALSE) -#else -#define AMD_CPUID_FIELD_DATA -#endif - -#define CPUID_FIELD_DATA \ - CPUID_FIELD_DATA_LEVEL_0 \ - CPUID_FIELD_DATA_LEVEL_1 \ - CPUID_FIELD_DATA_LEVEL_2 \ - CPUID_FIELD_DATA_LEVEL_4 \ - CPUID_FIELD_DATA_LEVEL_5 \ - CPUID_FIELD_DATA_LEVEL_6 \ - CPUID_FIELD_DATA_LEVEL_7 \ - CPUID_FIELD_DATA_LEVEL_A \ - CPUID_FIELD_DATA_LEVEL_B \ - CPUID_FIELD_DATA_LEVEL_D \ - CPUID_FIELD_DATA_LEVEL_12 \ - CPUID_FIELD_DATA_LEVEL_400 \ - CPUID_FIELD_DATA_LEVEL_401 \ - CPUID_FIELD_DATA_LEVEL_402 \ - CPUID_FIELD_DATA_LEVEL_403 \ - CPUID_FIELD_DATA_LEVEL_404 \ - CPUID_FIELD_DATA_LEVEL_405 \ - CPUID_FIELD_DATA_LEVEL_406 \ - CPUID_FIELD_DATA_LEVEL_410 \ - CPUID_FIELD_DATA_LEVEL_80 \ - CPUID_FIELD_DATA_LEVEL_81 \ - CPUID_FIELD_DATA_LEVEL_82 \ - CPUID_FIELD_DATA_LEVEL_83 \ - CPUID_FIELD_DATA_LEVEL_84 \ - CPUID_FIELD_DATA_LEVEL_8x \ - CPUID_FIELD_DATA_LEVEL_81x \ - INTEL_CPUID_FIELD_DATA \ - AMD_CPUID_FIELD_DATA - -/* - * Define all field and flag values as an enum. The result is a full - * set of values taken from the table above in the form: - * - * CPUID__MASK == mask for feature/field - * CPUID__SHIFT == offset of field - * - * e.g. - CPUID_VIRT_BITS_MASK = 0xff00 - * - CPUID_VIRT_BITS_SHIFT = 8 - * - * Note: The MASK definitions must use some gymnastics to get - * around a warning when shifting left by 32. - */ -#define VMW_BIT_MASK(shift) (((1 << (shift - 1)) << 1) - 1) - -#define FIELD(lvl, ecxIn, reg, bitpos, size, name, s, c3) \ - CPUID_##name##_SHIFT = bitpos, \ - CPUID_##name##_MASK = VMW_BIT_MASK(size) << bitpos, \ - CPUID_INTERNAL_SHIFT_##name = bitpos, \ - CPUID_INTERNAL_MASK_##name = VMW_BIT_MASK(size) << bitpos, \ - CPUID_INTERNAL_REG_##name = CPUID_REG_##reg, \ - CPUID_INTERNAL_EAXIN_##name = CPUID_LEVEL_VAL_##lvl, \ - CPUID_INTERNAL_ECXIN_##name = ecxIn, - -#define FLAG FIELD - -enum { - /* Define data for every CPUID field we have */ - CPUID_FIELD_DATA -}; -#undef VMW_BIT_MASK -#undef FIELD -#undef FLAG - -/* - * Legal CPUID config file mask characters. For a description of the - * cpuid masking system, please see: - * - * http://vmweb.vmware.com/~mts/cgi-bin/view.cgi/Apps/CpuMigrationChecks - */ - -#define CPUID_MASK_HIDE_CHR '0' -#define CPUID_MASK_HIDE_STR "0" -#define CPUID_MASK_FORCE_CHR '1' -#define CPUID_MASK_FORCE_STR "1" -#define CPUID_MASK_PASS_CHR '-' -#define CPUID_MASK_PASS_STR "-" -#define CPUID_MASK_TRUE_CHR 'T' -#define CPUID_MASK_TRUE_STR "T" -#define CPUID_MASK_FALSE_CHR 'F' -#define CPUID_MASK_FALSE_STR "F" -#define CPUID_MASK_IGNORE_CHR 'X' -#define CPUID_MASK_IGNORE_STR "X" -#define CPUID_MASK_HOST_CHR 'H' -#define CPUID_MASK_HOST_STR "H" -#define CPUID_MASK_RSVD_CHR 'R' -#define CPUID_MASK_RSVD_STR "R" -#define CPUID_MASK_INSTALL_CHR 'I' -#define CPUID_MASK_INSTALL_STR "I" - -/* - * When LM is disabled, we overlay the following masks onto the - * guest's default masks. Any level that is not defined below should - * be treated as all "-"s - */ - -#define CPT_ID1ECX_LM_DISABLED "----:----:----:----:--0-:----:----:----" -#define CPT_ID81EDX_LM_DISABLED "--0-:----:----:----:----:----:----:----" -#define CPT_ID81ECX_LM_DISABLED "----:----:----:----:----:----:----:---0" - -#define CPT_GET_LM_DISABLED_MASK(lvl, reg) \ - ((lvl == 1 && reg == CPUID_REG_ECX) ? CPT_ID1ECX_LM_DISABLED : \ - (lvl == 0x80000001 && reg == CPUID_REG_ECX) ? CPT_ID81ECX_LM_DISABLED : \ - (lvl == 0x80000001 && reg == CPUID_REG_EDX) ? CPT_ID81EDX_LM_DISABLED : \ - NULL) - -/* - * CPUID_MASK -- - * CPUID_SHIFT -- - * CPUID_ISSET -- - * CPUID_GET -- - * CPUID_SET -- - * CPUID_CLEAR -- - * CPUID_SETTO -- - * - * Accessor macros for all CPUID consts/fields/flags. Level and reg are not - * required, but are used to force compile-time asserts which help verify that - * the flag is being used on the right CPUID input and result register. - * - * Note: ASSERT_ON_COMPILE is duplicated rather than factored into its own - * macro, because token concatenation does not work as expected if an input is - * #defined (e.g. APIC) when macros are nested. Also, compound statements - * within parenthes is a GCC extension, so we must use runtime asserts with - * other compilers. - */ - -#if defined(__GNUC__) && !defined(__clang__) - -#define CPUID_MASK(eaxIn, reg, flag) \ - ({ \ - ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag); \ - CPUID_INTERNAL_MASK_##flag; \ - }) - -#define CPUID_SHIFT(eaxIn, reg, flag) \ - ({ \ - ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag); \ - CPUID_INTERNAL_SHIFT_##flag; \ - }) - -#define CPUID_ISSET(eaxIn, reg, flag, data) \ - ({ \ - ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag); \ - (((data) & CPUID_INTERNAL_MASK_##flag) != 0); \ - }) - -#define CPUID_GET(eaxIn, reg, field, data) \ - ({ \ - ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##field && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field); \ - (((uint32)(data) & CPUID_INTERNAL_MASK_##field) >> \ - CPUID_INTERNAL_SHIFT_##field); \ - }) - -#else - -/* - * CPUIDCheck -- - * - * Return val after verifying parameters. - */ - -static INLINE uint32 -CPUIDCheck(int32 eaxIn, int32 eaxInCheck, - CpuidReg reg, CpuidReg regCheck, uint32 val) -{ - ASSERT(eaxIn == eaxInCheck && reg == regCheck); - return val; -} - -#define CPUID_MASK(eaxIn, reg, flag) \ - CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##flag, \ - CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag, \ - CPUID_INTERNAL_MASK_##flag) - -#define CPUID_SHIFT(eaxIn, reg, flag) \ - CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##flag, \ - CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag, \ - CPUID_INTERNAL_SHIFT_##flag) - -#define CPUID_ISSET(eaxIn, reg, flag, data) \ - (CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##flag, \ - CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag, \ - CPUID_INTERNAL_MASK_##flag & (data)) != 0) - -#define CPUID_GET(eaxIn, reg, field, data) \ - CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##field, \ - CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##field, \ - ((uint32)(data) & CPUID_INTERNAL_MASK_##field) >> \ - CPUID_INTERNAL_SHIFT_##field) - -#endif - - -#define CPUID_SET(eaxIn, reg, flag, dataPtr) \ - do { \ - ASSERT_ON_COMPILE( \ - (uint32)eaxIn == (uint32)CPUID_INTERNAL_EAXIN_##flag && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag); \ - *(dataPtr) |= CPUID_INTERNAL_MASK_##flag; \ - } while (0) - -#define CPUID_CLEAR(eaxIn, reg, flag, dataPtr) \ - do { \ - ASSERT_ON_COMPILE( \ - (uint32)eaxIn == (uint32)CPUID_INTERNAL_EAXIN_##flag && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag); \ - *(dataPtr) &= ~CPUID_INTERNAL_MASK_##flag; \ - } while (0) - -#define CPUID_SETTO(eaxIn, reg, field, dataPtr, val) \ - do { \ - uint32 _v = val; \ - uint32 *_d = dataPtr; \ - ASSERT_ON_COMPILE( \ - (uint32)eaxIn == (uint32)CPUID_INTERNAL_EAXIN_##field && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field); \ - *_d = (*_d & ~CPUID_INTERNAL_MASK_##field) | \ - (_v << CPUID_INTERNAL_SHIFT_##field); \ - ASSERT(_v == (*_d & CPUID_INTERNAL_MASK_##field) >> \ - CPUID_INTERNAL_SHIFT_##field); \ - } while (0) - -#define CPUID_SETTO_SAFE(eaxIn, reg, field, dataPtr, val) \ - do { \ - uint32 _v = val & \ - (CPUID_INTERNAL_MASK_##field >> CPUID_INTERNAL_SHIFT_##field); \ - uint32 *_d = dataPtr; \ - ASSERT_ON_COMPILE( \ - (uint32)eaxIn == (uint32)CPUID_INTERNAL_EAXIN_##field && \ - CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field); \ - *_d = (*_d & ~CPUID_INTERNAL_MASK_##field) | \ - (_v << CPUID_INTERNAL_SHIFT_##field); \ - } while (0) - - -/* - * Definitions of various fields' values and more complicated - * macros/functions for reading cpuid fields. - */ - -#define CPUID_FAMILY_EXTENDED 15 - -/* Effective Intel CPU Families */ -#define CPUID_FAMILY_486 4 -#define CPUID_FAMILY_P5 5 -#define CPUID_FAMILY_P6 6 -#define CPUID_FAMILY_P4 15 - -/* Effective AMD CPU Families */ -#define CPUID_FAMILY_5x86 0x4 -#define CPUID_FAMILY_K5 0x5 -#define CPUID_FAMILY_K6 0x5 -#define CPUID_FAMILY_K7 0x6 -#define CPUID_FAMILY_K8 0xf -#define CPUID_FAMILY_K8L 0x10 -#define CPUID_FAMILY_K8MOBILE 0x11 -#define CPUID_FAMILY_LLANO 0x12 -#define CPUID_FAMILY_BOBCAT 0x14 -#define CPUID_FAMILY_BULLDOZER 0x15 // Bulldozer Piledriver Steamroller -#define CPUID_FAMILY_KYOTO 0x16 // Note: Jaguar microarch - -/* Effective VIA CPU Families */ -#define CPUID_FAMILY_C7 6 - -/* Intel model information */ -#define CPUID_MODEL_PPRO 1 -#define CPUID_MODEL_PII_03 3 -#define CPUID_MODEL_PII_05 5 -#define CPUID_MODEL_CELERON_06 6 -#define CPUID_MODEL_PM_09 9 -#define CPUID_MODEL_PM_0D 13 -#define CPUID_MODEL_PM_0E 14 // Yonah / Sossaman -#define CPUID_MODEL_CORE_0F 15 // Conroe / Merom -#define CPUID_MODEL_CORE_17 0x17 // Penryn -#define CPUID_MODEL_NEHALEM_1A 0x1a // Nehalem / Gainestown -#define CPUID_MODEL_ATOM_1C 0x1c // Silverthorne / Diamondville -#define CPUID_MODEL_CORE_1D 0x1d // Dunnington -#define CPUID_MODEL_NEHALEM_1E 0x1e // Lynnfield -#define CPUID_MODEL_NEHALEM_1F 0x1f // Havendale -#define CPUID_MODEL_NEHALEM_25 0x25 // Westmere / Clarkdale -#define CPUID_MODEL_ATOM_26 0x26 // Lincroft -#define CPUID_MODEL_ATOM_27 0x27 // Saltwell -#define CPUID_MODEL_SANDYBRIDGE_2A 0x2a // Sandybridge (desktop/mobile) -#define CPUID_MODEL_NEHALEM_2C 0x2c // Westmere-EP -#define CPUID_MODEL_SANDYBRIDGE_2D 0x2d // Sandybridge-EP -#define CPUID_MODEL_NEHALEM_2E 0x2e // Nehalem-EX -#define CPUID_MODEL_NEHALEM_2F 0x2f // Westmere-EX -#define CPUID_MODEL_ATOM_35 0x35 // Cloverview -#define CPUID_MODEL_ATOM_36 0x36 // Cedarview -#define CPUID_MODEL_ATOM_37 0x37 // Bay Trail -#define CPUID_MODEL_SANDYBRIDGE_3A 0x3a // Ivy Bridge -#define CPUID_MODEL_HASWELL_3C 0x3c // Haswell DT -#define CPUID_MODEL_BROADWELL_3D 0x3d // Broadwell-Ult -#define CPUID_MODEL_SANDYBRIDGE_3E 0x3e // Ivy Bridge-EP -#define CPUID_MODEL_HASWELL_3F 0x3f // Haswell EP/EN/EX -#define CPUID_MODEL_HASWELL_45 0x45 // Haswell Ultrathin -#define CPUID_MODEL_HASWELL_46 0x46 // Haswell (Crystal Well) -#define CPUID_MODEL_BROADWELL_47 0x47 // Broadwell (Denlow) -#define CPUID_MODEL_ATOM_4A 0x4a // Future Silvermont -#define CPUID_MODEL_ATOM_4C 0x4c // Airmont -#define CPUID_MODEL_ATOM_4D 0x4d // Avoton -#define CPUID_MODEL_SKYLAKE_4E 0x4e // Skylake-Y -#define CPUID_MODEL_BROADWELL_4F 0x4f // Broadwell EP/EN/EX -#define CPUID_MODEL_BROADWELL_56 0x56 // Broadwell DE -#define CPUID_MODEL_ATOM_5A 0x5a // Future Silvermont -#define CPUID_MODEL_ATOM_5D 0x5d // Future Silvermont -#define CPUID_MODEL_SKYLAKE_5E 0x5e // Skylake-S - -#define CPUID_MODEL_PIII_07 7 -#define CPUID_MODEL_PIII_08 8 -#define CPUID_MODEL_PIII_0A 10 - -/* AMD model information */ -#define CPUID_MODEL_BARCELONA_02 0x02 // Barcelona (Opteron & Phenom) -#define CPUID_MODEL_SHANGHAI_04 0x04 // Shanghai RB -#define CPUID_MODEL_SHANGHAI_05 0x05 // Shanghai BL -#define CPUID_MODEL_SHANGHAI_06 0x06 // Shanghai DA -#define CPUID_MODEL_ISTANBUL_MAGNY_08 0x08 // Istanbul (6 core) & Magny-cours (12) HY -#define CPUID_MODEL_ISTANBUL_MAGNY_09 0x09 // HY - G34 package -#define CPUID_MODEL_PHAROAH_HOUND_0A 0x0A // Pharoah Hound -#define CPUID_MODEL_PILEDRIVER_1F 0x1F // Max piledriver model defined in BKDG -#define CPUID_MODEL_PILEDRIVER_10 0x10 // family == CPUID_FAMILY_BULLDOZER -#define CPUID_MODEL_PILEDRIVER_02 0x02 // family == CPUID_FAMILY_BULLDOZER -#define CPUID_MODEL_OPTERON_REVF_41 0x41 // family == CPUID_FAMILY_K8 -#define CPUID_MODEL_KYOTO_00 0x00 // family == CPUID_FAMILY_KYOTO -#ifdef COMMUNITY_SOURCE_AMD_SECRET /* { */ -#define CPUID_MODEL_STEAMROLLER_3F 0x3F // Max Steamroller model defined in BKDG -#define CPUID_MODEL_STEAMROLLER_30 0x30 // family == CPUID_FAMILY_BULLDOZER -#endif /* } COMMUNITY_SOURCE_AMD_SECRET */ - -/* VIA model information */ -#define CPUID_MODEL_NANO 15 // Isaiah - -/* - *---------------------------------------------------------------------- - * - * CPUID_IsVendor{AMD,Intel,VIA} -- - * - * Determines if the vendor string in cpuid id0 is from {AMD,Intel,VIA}. - * - * Results: - * True iff vendor string is CPUID_{AMD,INTEL,VIA}_VENDOR_STRING - * - * Side effects: - * None. - * - *---------------------------------------------------------------------- - */ -static INLINE Bool -CPUID_IsRawVendor(CPUIDRegs *id0, const char* vendor) -{ - // hard to get strcmp() in some environments, so do it in the raw - return (id0->ebx == *(const uint32 *) (vendor + 0) && - id0->ecx == *(const uint32 *) (vendor + 4) && - id0->edx == *(const uint32 *) (vendor + 8)); -} - -static INLINE Bool -CPUID_IsVendorAMD(CPUIDRegs *id0) -{ - return CPUID_IsRawVendor(id0, CPUID_AMD_VENDOR_STRING); -} - -static INLINE Bool -CPUID_IsVendorIntel(CPUIDRegs *id0) -{ - return CPUID_IsRawVendor(id0, CPUID_INTEL_VENDOR_STRING); -} - -static INLINE Bool -CPUID_IsVendorVIA(CPUIDRegs *id0) -{ - return CPUID_IsRawVendor(id0, CPUID_VIA_VENDOR_STRING); -} - -static INLINE uint32 -CPUID_EFFECTIVE_FAMILY(uint32 v) /* %eax from CPUID with %eax=1. */ -{ - uint32 f = CPUID_GET(1, EAX, FAMILY, v); - return f != CPUID_FAMILY_EXTENDED ? f : f + - CPUID_GET(1, EAX, EXTENDED_FAMILY, v); -} - -/* Normally only used when FAMILY==CPUID_FAMILY_EXTENDED, but Intel is - * now using the extended model field for FAMILY==CPUID_FAMILY_P6 to - * refer to the newer Core2 CPUs - */ -static INLINE uint32 -CPUID_EFFECTIVE_MODEL(uint32 v) /* %eax from CPUID with %eax=1. */ -{ - uint32 m = CPUID_GET(1, EAX, MODEL, v); - uint32 em = CPUID_GET(1, EAX, EXTENDED_MODEL, v); - return m + (em << 4); -} - -/* - * Notice that CPUID families for Intel and AMD overlap. The following macros - * should only be used AFTER the manufacturer has been established (through - * the use of CPUID standard function 0). - */ -static INLINE Bool -CPUID_FAMILY_IS_486(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_486; -} - -static INLINE Bool -CPUID_FAMILY_IS_P5(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_P5; -} - -static INLINE Bool -CPUID_FAMILY_IS_P6(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_P6; -} - -static INLINE Bool -CPUID_FAMILY_IS_PENTIUM4(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_P4; -} - -/* - * Intel Pentium M processors are Yonah/Sossaman or an older P-M - */ -static INLINE Bool -CPUID_UARCH_IS_PENTIUM_M(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && - (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_PM_09 || - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_PM_0D || - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_PM_0E); -} - -/* - * Intel Core processors are Merom, Conroe, Woodcrest, Clovertown, - * Penryn, Dunnington, Kentsfield, Yorktown, Harpertown, ........ - */ -static INLINE Bool -CPUID_UARCH_IS_CORE(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - uint32 model = CPUID_EFFECTIVE_MODEL(v); - /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && - model >= CPUID_MODEL_CORE_0F && - (model < CPUID_MODEL_NEHALEM_1A || - model == CPUID_MODEL_CORE_1D); -} - -/* - * Intel Nehalem processors are: Nehalem, Gainestown, Lynnfield, Clarkdale. - */ -static INLINE Bool -CPUID_UARCH_IS_NEHALEM(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v); - - return CPUID_FAMILY_IS_P6(v) && - (effectiveModel == CPUID_MODEL_NEHALEM_1A || - effectiveModel == CPUID_MODEL_NEHALEM_1E || - effectiveModel == CPUID_MODEL_NEHALEM_1F || - effectiveModel == CPUID_MODEL_NEHALEM_25 || - effectiveModel == CPUID_MODEL_NEHALEM_2C || - effectiveModel == CPUID_MODEL_NEHALEM_2E || - effectiveModel == CPUID_MODEL_NEHALEM_2F); -} - - -static INLINE Bool -CPUID_UARCH_IS_SANDYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v); - - return CPUID_FAMILY_IS_P6(v) && - (effectiveModel == CPUID_MODEL_SANDYBRIDGE_2A || - effectiveModel == CPUID_MODEL_SANDYBRIDGE_2D || - effectiveModel == CPUID_MODEL_SANDYBRIDGE_3E || - effectiveModel == CPUID_MODEL_SANDYBRIDGE_3A); -} - - -static INLINE Bool -CPUID_MODEL_IS_BROADWELL(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v); - - return CPUID_FAMILY_IS_P6(v) && - (effectiveModel == CPUID_MODEL_BROADWELL_3D || - effectiveModel == CPUID_MODEL_BROADWELL_47 || - effectiveModel == CPUID_MODEL_BROADWELL_4F || - effectiveModel == CPUID_MODEL_BROADWELL_56); -} - - -static INLINE Bool -CPUID_MODEL_IS_HASWELL(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v); - - return CPUID_FAMILY_IS_P6(v) && - (effectiveModel == CPUID_MODEL_HASWELL_3C || - effectiveModel == CPUID_MODEL_HASWELL_3F || - effectiveModel == CPUID_MODEL_HASWELL_45 || - effectiveModel == CPUID_MODEL_HASWELL_46); -} - - -static INLINE Bool -CPUID_MODEL_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && - (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_5E || - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_4E); -} - - -static INLINE Bool -CPUID_UARCH_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && CPUID_MODEL_IS_SKYLAKE(v); -} - - -static INLINE Bool -CPUID_UARCH_IS_HASWELL(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && - (CPUID_MODEL_IS_BROADWELL(v) || CPUID_MODEL_IS_HASWELL(v)); -} - - -static INLINE Bool -CPUID_MODEL_IS_CENTERTON(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_1C; -} - -static INLINE Bool -CPUID_MODEL_IS_AVOTON(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - return CPUID_FAMILY_IS_P6(v) && - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_4D; -} - -static INLINE Bool -CPUID_MODEL_IS_WESTMERE(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v); - - return CPUID_FAMILY_IS_P6(v) && - (effectiveModel == CPUID_MODEL_NEHALEM_25 || // Clarkdale - effectiveModel == CPUID_MODEL_NEHALEM_2C || // Westmere-EP - effectiveModel == CPUID_MODEL_NEHALEM_2F); // Westmere-EX -} - - -static INLINE Bool -CPUID_MODEL_IS_SANDYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v); - - return CPUID_FAMILY_IS_P6(v) && - (effectiveModel == CPUID_MODEL_SANDYBRIDGE_2A || - effectiveModel == CPUID_MODEL_SANDYBRIDGE_2D); -} - - -static INLINE Bool -CPUID_MODEL_IS_IVYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is Intel. */ - uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v); - - return CPUID_FAMILY_IS_P6(v) && ( - effectiveModel == CPUID_MODEL_SANDYBRIDGE_3E || - effectiveModel == CPUID_MODEL_SANDYBRIDGE_3A); -} - - -static INLINE Bool -CPUID_FAMILY_IS_K7(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K7; -} - -static INLINE Bool -CPUID_FAMILY_IS_K8(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K8; -} - -static INLINE Bool -CPUID_FAMILY_IS_K8EXT(uint32 eax) -{ - /* - * We check for this pattern often enough that it's - * worth a separate function, for syntactic sugar. - */ - return CPUID_FAMILY_IS_K8(eax) && - CPUID_GET(1, EAX, EXTENDED_MODEL, eax) != 0; -} - -static INLINE Bool -CPUID_FAMILY_IS_K8L(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K8L || - CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_LLANO; -} - -static INLINE Bool -CPUID_FAMILY_IS_LLANO(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_LLANO; -} - -static INLINE Bool -CPUID_FAMILY_IS_K8MOBILE(uint32 eax) -{ - /* Essentially a K8 (not K8L) part, but with mobile features. */ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K8MOBILE; -} - -static INLINE Bool -CPUID_FAMILY_IS_K8STAR(uint32 eax) -{ - /* - * Read function name as "K8*", as in wildcard. - * Matches K8 or K8L or K8MOBILE - */ - return CPUID_FAMILY_IS_K8(eax) || CPUID_FAMILY_IS_K8L(eax) || - CPUID_FAMILY_IS_K8MOBILE(eax); -} - -static INLINE Bool -CPUID_FAMILY_IS_BOBCAT(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BOBCAT; -} - -static INLINE Bool -CPUID_FAMILY_IS_BULLDOZER(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER; -} - -static INLINE Bool -CPUID_FAMILY_IS_KYOTO(uint32 eax) -{ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_KYOTO; -} - -/* - * AMD Barcelona (of either Opteron or Phenom kind). - */ -static INLINE Bool -CPUID_MODEL_IS_BARCELONA(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is AMD. */ - return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L && - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_BARCELONA_02; -} - - -static INLINE Bool -CPUID_MODEL_IS_SHANGHAI(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is AMD. */ - return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L && - (CPUID_MODEL_SHANGHAI_04 <= CPUID_EFFECTIVE_MODEL(v) && - CPUID_EFFECTIVE_MODEL(v) <= CPUID_MODEL_SHANGHAI_06); -} - - -static INLINE Bool -CPUID_MODEL_IS_ISTANBUL_MAGNY(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is AMD. */ - return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L && - (CPUID_MODEL_ISTANBUL_MAGNY_08 <= CPUID_EFFECTIVE_MODEL(v) && - CPUID_EFFECTIVE_MODEL(v) <= CPUID_MODEL_ISTANBUL_MAGNY_09); -} - - -static INLINE Bool -CPUID_MODEL_IS_PHAROAH_HOUND(uint32 v) // IN: %eax from CPUID with %eax=1. -{ - /* Assumes the CPU manufacturer is AMD. */ - return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L && - CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_PHAROAH_HOUND_0A; -} - - -static INLINE Bool -CPUID_MODEL_IS_BULLDOZER(uint32 eax) -{ - /* - * Bulldozer is models of family 0x15 that are below 10 excluding - * Piledriver 02. - */ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER && - CPUID_EFFECTIVE_MODEL(eax) < CPUID_MODEL_PILEDRIVER_10 && - CPUID_EFFECTIVE_MODEL(eax) != CPUID_MODEL_PILEDRIVER_02; -} - - -static INLINE Bool -CPUID_MODEL_IS_PILEDRIVER(uint32 eax) -{ - /* Piledriver is models 0x02 & 0x10 of family 0x15 (so far). */ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER && - ((CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_PILEDRIVER_10 && - CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_PILEDRIVER_1F) || - CPUID_EFFECTIVE_MODEL(eax) == CPUID_MODEL_PILEDRIVER_02); -} - - -#ifdef COMMUNITY_SOURCE_AMD_SECRET /* { */ -static INLINE Bool -CPUID_MODEL_IS_STEAMROLLER(uint32 eax) -{ - /* Steamroller is model 0x30 of family 0x15 (so far). */ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER && - (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_STEAMROLLER_30 && - CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_STEAMROLLER_3F); -} -#endif /* } COMMUNITY_SOURCE_AMD_SECRET */ - - -static INLINE Bool -CPUID_MODEL_IS_KYOTO(uint32 eax) -{ - /* Kyoto is models 0x00 of family 0x16 (so far). */ - return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_KYOTO && - CPUID_EFFECTIVE_MODEL(eax) == CPUID_MODEL_KYOTO_00; -} - -#define CPUID_TYPE_PRIMARY 0 -#define CPUID_TYPE_OVERDRIVE 1 -#define CPUID_TYPE_SECONDARY 2 - -#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_NULL 0 -#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_DATA 1 -#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_INST 2 -#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_UNIF 3 -#define CPUID_LEAF4_CACHE_TYPE_NULL 0 -#define CPUID_LEAF4_CACHE_TYPE_DATA 1 -#define CPUID_LEAF4_CACHE_TYPE_INST 2 -#define CPUID_LEAF4_CACHE_TYPE_UNIF 3 -#define CPUID_LEAF4_CACHE_INDEXING_DIRECT 0 -#define CPUID_LEAF4_CACHE_INDEXING_COMPLEX 1 - -#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_SELF_INIT 0x00000100 -#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_FULLY_ASSOC 0x00000200 -#define CPUID_LEAF4_CACHE_SELF_INIT 0x00000100 -#define CPUID_LEAF4_CACHE_FULLY_ASSOC 0x00000200 - -#define CPUID_INTEL_IDBECX_LEVEL_TYPE_INVALID 0 -#define CPUID_INTEL_IDBECX_LEVEL_TYPE_SMT 1 -#define CPUID_INTEL_IDBECX_LEVEL_TYPE_CORE 2 -#define CPUID_TOPOLOGY_LEVEL_TYPE_INVALID 0 -#define CPUID_TOPOLOGY_LEVEL_TYPE_SMT 1 -#define CPUID_TOPOLOGY_LEVEL_TYPE_CORE 2 - - -/* - * For certain AMD processors, an lfence instruction is necessary at various - * places to ensure ordering. - */ - -static INLINE Bool -CPUID_VendorRequiresFence(CpuidVendor vendor) -{ - return vendor == CPUID_VENDOR_AMD; -} - -static INLINE Bool -CPUID_VersionRequiresFence(uint32 version) -{ - return CPUID_EFFECTIVE_FAMILY(version) == CPUID_FAMILY_K8 && - CPUID_EFFECTIVE_MODEL(version) < 0x40; -} - -static INLINE Bool -CPUID_ID0RequiresFence(CPUIDRegs *id0) -{ - if (id0->eax == 0) { - return FALSE; - } - return CPUID_IsVendorAMD(id0); -} - -static INLINE Bool -CPUID_ID1RequiresFence(CPUIDRegs *id1) -{ - return CPUID_VersionRequiresFence(id1->eax); -} - -static INLINE Bool -CPUID_RequiresFence(CpuidVendor vendor, // IN - uint32 version) // IN: %eax from CPUID with %eax=1. -{ - return CPUID_VendorRequiresFence(vendor) && - CPUID_VersionRequiresFence(version); -} - - -/* - * The following low-level functions compute the number of - * cores per cpu. They should be used cautiously because - * they do not necessarily work on all types of CPUs. - * High-level functions that are correct for all CPUs are - * available elsewhere: see lib/cpuidInfo/cpuidInfo.c. - */ - -static INLINE uint32 -CPUID_IntelCoresPerPackage(uint32 v) /* %eax from CPUID with %eax=4 and %ecx=0. */ -{ - // Note: This is not guaranteed to work on older Intel CPUs. - return 1 + CPUID_GET(4, EAX, LEAF4_CORE_COUNT, v); -} - - -static INLINE uint32 -CPUID_AMDCoresPerPackage(uint32 v) /* %ecx from CPUID with %eax=0x80000008. */ -{ - // Note: This is not guaranteed to work on older AMD CPUs. - return 1 + CPUID_GET(0x80000008, ECX, LEAF88_CORE_COUNT, v); -} - - -/* - * Hypervisor CPUID space is 0x400000XX. - */ -static INLINE Bool -CPUID_IsHypervisorLevel(uint32 level) -{ - return (level & 0xffffff00) == 0x40000000; -} - -/* - *---------------------------------------------------------------------- - * - * CPUID_LevelUsesEcx -- - * - * Returns TRUE for leaves that support input ECX != 0 (subleaves). - * - *---------------------------------------------------------------------- - */ - -static INLINE Bool -CPUID_LevelUsesEcx(uint32 level) { - return level == 4 || level == 7 || level == 0xb || level == 0xd || - level == 0x8000001d; -} - -/* - *---------------------------------------------------------------------- - * - * CPUID_IsValid*Subleaf -- - * - * Functions to determine the last subleaf for the level specified - * - *---------------------------------------------------------------------- - */ - -static INLINE Bool -CPUID_IsValidBSubleaf(uint32 ebx) // IN: %ebx = cpuid.b.sublevel.ebx -{ - return ebx != 0; -} - -static INLINE Bool -CPUID_IsValid4Subleaf(uint32 eax) // IN: %eax = cpuid.4.sublevel.eax -{ - return eax != 0; -} - -static INLINE Bool -CPUID_IsValid7Subleaf(uint32 eax, uint32 subleaf) // IN: %eax = cpuid.7.0.eax -{ - /* - * cpuid.7.0.eax is the max ecx (subleaf) index - */ - return subleaf <= eax; -} - -/* - *---------------------------------------------------------------------- - * - * CPUID_IsValidDSubleaf -- - * - * It is the caller's repsonsibility to determine if the processor - * supports XSAVE and therefore has D sub-leaves. - * - *---------------------------------------------------------------------- - */ -static INLINE Bool -CPUID_IsValidDSubleaf(uint32 subleaf) // IN: subleaf to check -{ - return subleaf <= 63; -} - -/* - *---------------------------------------------------------------------- - * - * CPUID_SupportsMsrPlatformInfo -- - * - * Uses vendor and cpuid.1.0.eax to determine if the processor - * supports MSR_PLATFORM_INFO. - * - *---------------------------------------------------------------------- - */ -static INLINE Bool -CPUID_SupportsMsrPlatformInfo(CpuidVendor vendor, uint32 version) -{ - return vendor == CPUID_VENDOR_INTEL && - (CPUID_UARCH_IS_NEHALEM(version) || - CPUID_UARCH_IS_SANDYBRIDGE(version) || - CPUID_UARCH_IS_HASWELL(version) || - CPUID_UARCH_IS_SKYLAKE(version) || - CPUID_MODEL_IS_AVOTON(version)); -} - -#endif