vmmon: use standard definition of MSR_TSX_FORCE_ABORT if available

The MSR_TSX_FORCE_ABORT macro is defined in mainline since commit 52f64909409c ("x86: Add TSX Force Abort CPUID/MSR") in v5.1-rc1.
vmnet: convert VNetBridge debugging code to ktime_get_ns()
2020-05-30 02:48:55 +02:00 · 2020-05-30 02:48:43 +02:00 · 2020-05-30 02:48:11 +02:00 · 2020-05-30 02:47:57 +02:00 · 2020-05-30 02:47:50 +02:00 · 2020-05-30 02:47:42 +02:00
40 changed files with 862 additions and 1170 deletions
@@ -136,9 +136,6 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

-# Add Spectre options when available
-CC_OPTS += $(call vm_check_gcc,-mindirect-branch=thunk -mindirect-branch-register,)
-
 include $(SRCROOT)/Makefile.kernel

 else
@@ -31,7 +31,7 @@ $(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, \
 		$(SRCROOT)/bootstrap/*.c)))

 clean:
-	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \
+	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions .cache.mk \
 		Module.symvers Modules.symvers Module.markers modules.order \
 		$(foreach dir,linux/ common/ vmcore/ bootstrap/ \
 		./,$(addprefix $(dir),.*.cmd .*.o.flags *.o)))
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,6 +59,13 @@

 #define CROSSCALL_SLEEP_US 1000

+typedef struct HostIFContigMemMap {
+   MPN mpn;
+   void *addr;
+   PageCnt pages;
+   struct HostIFContigMemMap *next;
+} HostIFContigMemMap;
+
 EXTERN Bool  HostIF_Init(VMDriver *vm, uint32 numVCPUs);
 EXTERN int   HostIF_LookupUserMPN(VMDriver *vm, VA64 uAddr, MPN *mpn);
 EXTERN void *HostIF_MapCrossPage(VMDriver *vm, VA64 uAddr);
@@ -82,13 +89,16 @@ EXTERN void  HostIF_Wait(unsigned int timeoutMs);
 EXTERN void  HostIF_WaitForFreePages(unsigned int timeoutMs);
 EXTERN void *HostIF_AllocKernelPages(PageCnt numPages, MPN *mpns);
 EXTERN void  HostIF_FreeKernelPages(PageCnt numPages, void *ptr);
+EXTERN HostIFContigMemMap *HostIF_AllocContigPages(VMDriver *vm,
+                                                   PageCnt numPages);
+EXTERN void  HostIF_FreeContigPages(VMDriver *vm, HostIFContigMemMap *mapping);
 EXTERN void  HostIF_VMLock(VMDriver *vm, int callerID);
 EXTERN void  HostIF_VMUnlock(VMDriver *vm, int callerID);
 #ifdef VMX86_DEBUG
 EXTERN Bool HostIF_VMLockIsHeld(VMDriver *vm);
 #endif

-EXTERN Bool  HostIF_APICInit(VMDriver *vm, Bool setVMPtr, Bool probe);
+EXTERN void  HostIF_APICInit(VMDriver *vm);
 EXTERN uint8 HostIF_GetMonitorIPIVector(void);
 EXTERN uint8 HostIF_GetHVIPIVector(void);
 EXTERN void  HostIF_GetTimerVectors(uint8 *v0, uint8 *v1);
@@ -2307,12 +2307,23 @@ TaskSwitchToMonitor(VMCrossPage *crosspage)
   {
      uint64 raxGetsWiped, rcxGetsWiped;

+#ifdef CALL_NOSPEC
+      __asm__ __volatile__(CALL_NOSPEC
+                           : "=a" (raxGetsWiped),
+                             "=c" (rcxGetsWiped)
+                           : "0" (codePtr),
+                             "1" (crosspage),
+                             THUNK_TARGET(codePtr)
+                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
+#else
      __asm__ __volatile__("call *%%rax"
                           : "=a" (raxGetsWiped),
                             "=c" (rcxGetsWiped)
                           : "0" (codePtr),
                             "1" (crosspage)
                           : "rdx", "r8", "r9", "r10", "r11", "cc", "memory");
+#endif
+
   }
 #elif defined(_MSC_VER)
   /*
@@ -108,6 +108,9 @@ static int vmIDsUnused;
 /* Max rate requested for fast clock by any virtual machine. */
 static unsigned globalFastClockRate;

+/* 3 physically contiguous pages for the I/O bitmap.  SVM only. */
+HostIFContigMemMap *hvIOBitmap;
+
 typedef struct {
   Atomic_uint32 index;
   MSRQuery *query;
@@ -687,6 +690,38 @@ Vmx86_CleanupVMMPages(VMDriver *vm)
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86CleanupContigMappings --
+ *
+ *     Frees all allocations from HostIF_AllocContigPages that are associated
+ *     with the given vm.
+ *
+ * Results:
+ *      None
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+static void
+Vmx86CleanupContigMappings(VMDriver *vm)
+{
+   HostIFContigMemMap *m, *next;
+
+   HostIF_VMLock(vm, 48);
+   for (m = vm->contigMappings; m != NULL; m = next) {
+      next = m->next;
+      HostIF_FreeContigPages(vm, m);
+   }
+   HostIF_VMUnlock(vm, 48);
+   vm->contigMappings = NULL;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -732,7 +767,9 @@ Vmx86FreeAllVMResources(VMDriver *vm)
         StatVarsVmmon_Cleanup(vm->statVars);
         vm->statVars = NULL;
      }
-
+      if (vm->contigMappings != NULL) {
+         Vmx86CleanupContigMappings(vm);
+      }
      HostIF_FreeAllResources(vm);

      Vmx86FreeVMDriver(vm);
@@ -2042,7 +2079,9 @@ Vmx86_AllocLowPage(VMDriver *vm,      // IN: VMDriver
      return INVALID_MPN;
   }

+   HostIF_VMLock(vm, 49);
   mpn = HostIF_AllocLowPage(vm);
+   HostIF_VMUnlock(vm, 49);

   if (mpn == INVALID_MPN) {
      Vmx86UnreserveFreePages(vm, 1);
@@ -3393,3 +3432,65 @@ Vmx86_GetMonitorContext(VMDriver *vm,       // IN: The VM instance.
   context->rip = cpData->monRIP;
   return TRUE;
 }
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_CleanupHVIOBitmap --
+ *
+ *      Free any resources that were allocated for the HV I/O bitmap.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+Vmx86_CleanupHVIOBitmap(void)
+{
+   if (hvIOBitmap != NULL) {
+      HostIF_FreeContigPages(NULL, hvIOBitmap);
+      hvIOBitmap = NULL;
+   }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Vmx86_CreateHVIOBitmap --
+ *
+ *      Called on driver load to create and initialize the host wide SVM I/O
+ *      bitmap.  This item is a physically contiguous region of
+ *      SVM_VMCB_IO_BITMAP_PAGES pages and is initialized to all-bits-set.
+ *
+ * Results:
+ *      TRUE on success or FALSE on failure.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool
+Vmx86_CreateHVIOBitmap(void)
+{
+   if (!CPUID_HostSupportsSVM()) {
+      return TRUE;
+   }
+   if (vmx86_apple) {
+      /*
+       * This function is not called on MacOS.  No supported MacOS system is
+       * available for AMD so that platform has no need to create the SVM I/O
+       * bitmap.
+       */
+      return TRUE;
+   }
+   hvIOBitmap = HostIF_AllocContigPages(NULL, SVM_VMCB_IO_BITMAP_PAGES);
+   if (hvIOBitmap == NULL) {
+      Warning("Failed to allocate SVM I/O bitmap.\n");
+      return FALSE;
+   }
+   memset(hvIOBitmap->addr, 0xff, SVM_VMCB_IO_BITMAP_SIZE);
+   return TRUE;
+}
@@ -48,6 +48,10 @@ typedef struct TSCDelta {

 struct VmmBlobInfo;

+struct HostIFContigMemMap;
+
+extern struct HostIFContigMemMap *hvIOBitmap;
+
 /*
 * VMDriver - the main data structure for the driver side of a
 *            virtual machine.
@@ -79,6 +83,12 @@ typedef struct VMDriver {
   int64                  *ptscOffsets;      /* numVCPUs-sized array. */
   Atomic_uint32          *currentHostCpu;   /* numVCPUs-sized array. */
   PageCnt                 numPTPPages;      /* Num PTP pages allocated. */
+   /*
+    * List of physically contiguous allocations associated with this VM.
+    * Access is protected by the VM lock.
+    */
+   struct HostIFContigMemMap     *contigMappings;
+
 } VMDriver;

 typedef struct MonLoaderArgs {
@@ -168,6 +178,8 @@ extern Bool Vmx86_GetAllMSRs(MSRQuery *query);
 extern void Vmx86_FlushVMCSAllCPUs(MA vmcs);
 extern void Vmx86_MonTimerIPI(void);
 extern void Vmx86_InitIDList(void);
+extern Bool Vmx86_CreateHVIOBitmap(void);
+extern void Vmx86_CleanupHVIOBitmap(void);
 extern Bool Vmx86_GetPageRoot(VMDriver *vm, Vcpuid vcpuid, MPN *mpn);
 extern void Vmx86_Open(void);
 extern void Vmx86_Close(void);
@@ -0,0 +1,30 @@
+#ifndef __COMPAT_POLL_H__
+#define __COMPAT_POLL_H__
+
+#include <linux/poll.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0))
+
+#ifndef __poll_t
+typedef unsigned int __poll_t;
+#endif
+
+static inline __poll_t compat_vfs_poll(struct file *file,
+				       struct poll_table_struct *pt)
+{
+	if (unlikely(!file->f_op->poll))
+		return DEFAULT_POLLMASK;
+	return file->f_op->poll(file, pt);
+}
+
+#else
+
+static inline __poll_t compat_vfs_poll(struct file *file,
+				       struct poll_table_struct *pt)
+{
+	return vfs_poll(file, pt);
+}
+
+#endif
+
+#endif /* __COMPAT_POLL_H__ */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -143,9 +143,12 @@ PtrToVA64(void const *ptr) // IN
 * does not break vmcore compatibility.
 *
 * See bora/doc/vmcore details.
+ *
+ * If you increment VMMON_VERSION, you must also increment
+ * the NT specific VMX86_DRIVER_VERSION.
 */

-#define VMMON_VERSION           (385 << 16 | 0)
+#define VMMON_VERSION           (392 << 16 | 0)
 #define VMMON_VERSION_MAJOR(v)  ((uint32) (v) >> 16)
 #define VMMON_VERSION_MINOR(v)  ((uint16) (v))

@@ -253,18 +256,10 @@ enum IOCTLCmd {

 #if defined __linux__
   IOCTLCMD(SET_UID),		// VMX86_DEVEL only
-#endif
-
-#if defined __linux__ || defined __APPLE__
   IOCTLCMD(GET_ALL_CPUID),
 #endif

-#if defined _WIN32 || defined __APPLE__
-   IOCTLCMD(ALLOC_CONTIG_PAGES),
-#endif
-
 #if defined _WIN32
-   IOCTLCMD(FREE_CONTIG_PAGES),
   IOCTLCMD(HARD_LIMIT_MONITOR_STATUS), // used by vmauthd on Windows
   IOCTLCMD(CHANGE_HARD_LIMIT),         // used by vmauthd on Windows
   IOCTLCMD(READ_DISASM_PROC_BINARY),
@@ -272,6 +267,7 @@ enum IOCTLCmd {
   IOCTLCMD(SET_MEMORY_PARAMS),
   IOCTLCMD(REMEMBER_KHZ_ESTIMATE),
   IOCTLCMD(REMAP_SCATTER_LIST),
+   IOCTLCMD(REMAP_SCATTER_LIST_RO),     // map the list as read-only
   IOCTLCMD(UNMAP_SCATTER_LIST),
 #endif

@@ -281,10 +277,6 @@ enum IOCTLCmd {
   IOCTLCMD(BLUEPILL),
 #endif

-#if defined __linux__
-   IOCTLCMD(SET_HOST_SWAP_SIZE),
-#endif
-
   IOCTLCMD(GET_UNAVAIL_PERF_CTRS),
   IOCTLCMD(GET_MONITOR_CONTEXT),
   // Must be last.
@@ -336,8 +328,6 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_UPDATE_MEM_INFO     VMIOCTL_BUFFERED(UPDATE_MEM_INFO)
 #define IOCTL_VMX86_HARD_LIMIT_MONITOR_STATUS   VMIOCTL_BUFFERED(HARD_LIMIT_MONITOR_STATUS)
 #define IOCTL_VMX86_CHANGE_HARD_LIMIT   VMIOCTL_BUFFERED(CHANGE_HARD_LIMIT)
-#define IOCTL_VMX86_ALLOC_CONTIG_PAGES  VMIOCTL_BUFFERED(ALLOC_CONTIG_PAGES)
-#define IOCTL_VMX86_FREE_CONTIG_PAGES   VMIOCTL_BUFFERED(FREE_CONTIG_PAGES)

 #define IOCTL_VMX86_GET_TOTAL_MEM_USAGE	VMIOCTL_BUFFERED(GET_TOTAL_MEM_USAGE)
 #define IOCTL_VMX86_GET_KHZ_ESTIMATE    VMIOCTL_BUFFERED(GET_KHZ_ESTIMATE)
@@ -367,20 +357,14 @@ enum IOCTLCmd {
 #define IOCTL_VMX86_GET_PSEUDO_TSC           VMIOCTL_NEITHER(GET_PSEUDO_TSC)
 #define IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS    VMIOCTL_NEITHER(GET_UNAVAIL_PERF_CTRS)
 #define IOCTL_VMX86_GET_MONITOR_CONTEXT      VMIOCTL_BUFFERED(GET_MONITOR_CONTEXT)
-#define IOCTL_VMX86_REMAP_SCATTER_LIST VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
-#define IOCTL_VMX86_UNMAP_SCATTER_LIST VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST    VMIOCTL_BUFFERED(REMAP_SCATTER_LIST)
+#define IOCTL_VMX86_REMAP_SCATTER_LIST_RO VMIOCTL_BUFFERED(REMAP_SCATTER_LIST_RO)
+#define IOCTL_VMX86_UNMAP_SCATTER_LIST    VMIOCTL_BUFFERED(UNMAP_SCATTER_LIST)
 #endif


 #define INIT_BLOCK_MAGIC     (0x1789 + 14)

-/*
- * Flags sent into APICBASE ioctl
- */
-
-#define APIC_FLAG_DISABLE_NMI       0x00000001
-#define APIC_FLAG_PROBE             0x00000002
-#define APIC_FLAG_FORCE_ENABLE      0x00000004

 typedef
 #include "vmware_pack_begin.h"
@@ -409,10 +393,6 @@ union {
 #include "vmware_pack_end.h"
 VcpuPageRoot;

-typedef struct VMAPICInfo {
-   uint32 flags;
-} VMAPICInfo;
-
 #define VMX86_DRIVER_VCPUID_OFFSET 1000


@@ -614,39 +594,7 @@ typedef union {
   Context64 context; // OUT
 } VMMonContext;

-#if defined __linux__
-
-/*
- * Linux uses mmap(2) to allocate contiguous locked pages, and uses these
- * macros to marshall real arguments to mmap's made-up 'offset' argument.
- */
-
-#define VMMON_MAP_OFFSET_SHIFT  0
-#define VMMON_MAP_OFFSET_MASK   0x00000FFF
-#define VMMON_MAP_ORDER_SHIFT   12
-#define VMMON_MAP_ORDER_MASK    0xF
-#define VMMON_MAP_RSVD_SHIFT    16
-
-#define VMMON_MAP_RSVD(base)    \
-                ((base) >> VMMON_MAP_RSVD_SHIFT)
-#define VMMON_MAP_ORDER(base)   \
-                (((base) >> VMMON_MAP_ORDER_SHIFT) & VMMON_MAP_ORDER_MASK)
-#define VMMON_MAP_OFFSET(base)  \
-                (((base) >> VMMON_MAP_OFFSET_SHIFT) & VMMON_MAP_OFFSET_MASK)
-
-#define VMMON_MAP_BASE(order)   ((order) << VMMON_MAP_ORDER_SHIFT)
-
-#elif defined _WIN32
-/*
- * Windows uses an ioctl to allocate contiguous locked pages.
- */
-
-typedef struct VMAllocContiguousMem {
-   VA64   mpnList;  // IN: User VA of an array of 64-bit MPNs.
-   uint32 mpnCount; // IN
-   uint32 order;    // IN
-} VMAllocContiguousMem;
-#elif defined __APPLE__
+#if defined __APPLE__
 #   include "iocontrolsMacos.h"
 #endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -57,6 +57,7 @@
   MC(RELEASE_ANON_PAGES)                                                     \
   MC(LOOKUP_MPN)                                                             \
   MC(COSCHED)                                                                \
+   MC(ALLOC_CONTIG_PAGES)                                                     \
   MC(ALLOC_VMX_PAGE)                                                         \
   MC(ALLOC_TMP_GDT)                                                          \
   MC(PIN_MPN)                                                                \
@@ -69,7 +70,8 @@
   MC(BOOTSTRAP_CLEANUP)                                                      \
   MC(GET_SHARED_AREA)                                                        \
   MC(GET_STAT_VARS)                                                          \
-   MC(GET_NUM_PTP_PAGES)
+   MC(GET_NUM_PTP_PAGES)                                                      \
+   MC(GET_HV_IO_BITMAP)                                                       \

 /*
 *----------------------------------------------------------------------
@@ -393,7 +395,6 @@ struct VMCrossPageData {
   uint8         _pad7[4];
   uint64        wsUD2;                       // IP of ud2 instr or 0 if unset.
   uint64        specCtrl; /* host MSR_SPEC_CTRL value before world switch. */
-   uint8         _pad8[8];
 }
 #include "vmware_pack_end.h"
 VMCrossPageData;
@@ -450,7 +451,7 @@ struct VMCrossPage {
 #include "vmware_pack_end.h"
 VMCrossPage;

-#define CROSSPAGE_VERSION_BASE 0xc0c /* increment by 1 */
+#define CROSSPAGE_VERSION_BASE 0xc0e /* increment by 1 */
 #define CROSSPAGE_VERSION    ((CROSSPAGE_VERSION_BASE << 1) + WS_INTR_STRESS)

 #if !defined(VMX86_SERVER) && defined(VMM)
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2006,2009-2011,2013-2015 VMware, Inc. All rights reserved.
+ * Copyright (C) 2006,2009-2011,2013-2015,2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -66,8 +66,7 @@
   SWMSR(MSR_CSTAR)         \
   SWMSR(MSR_SFMASK)        \
   SWMSR(MSR_TSC_AUX)       \
-   SWMSR(MSR_BD_TSC_RATIO)  \
-   SWMSR(MSR_BNDCFGS)
+   SWMSR(MSR_BD_TSC_RATIO)

 /*
 *      Data structures for dealing with the context-switched MSRs that need
@@ -94,7 +93,6 @@ typedef struct SwitchedMSRValues {
 typedef struct SwitchedMSRState {
   SwitchedMSRValues smv;
   uint8             flags[NUM_SWITCHED_MSRS];
-   uint32            _pad;
 } SwitchedMSRState;

 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2016-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -59,6 +59,9 @@ typedef uint32 Vcpuid;                 // VCPU number
 */
 #define MAX_SMALL_VM_VCPUS 128

+/* Supported limit. */
+#define MAX_SUPPORTED_VCPUS   256
+
 #if defined __cplusplus
 } // extern "C"
 #endif
@@ -155,7 +155,6 @@ _GET_LDT(Selector * const result)
   } while (0)


-/* Checked against the Intel manual and GCC --thutt */
 #define _BUILD_SET_R(func, reg)        \
   static INLINE void                  \
   func(uintptr_t r)                   \
@@ -166,24 +165,20 @@ _GET_LDT(Selector * const result)
              : "memory");             \
   }

-/* Not yet checked against the Intel manual and GCC --slava
- *
- * 'volatile' because CRs and DRs can change without the compiler
- * knowing it (when there is a page fault, when a breakpoint occurs,
- * and moreover it seems there is no way to teach gcc that smsw
- * clobbers cr0 for example).
- *
- * The parameter is a 'uintptr_t *' so that the size of the actual
- * parameter must exactly match the size of the hardware register.
- * This prevents the use of 32-bit variables when building 64-bit
- * code.
+/*
+ * The inline asm is marked 'volatile' because CRs and DRs can change
+ * without the compiler knowing it (when there is a page fault, when a
+ * breakpoint occurs, and moreover it seems there is no way to teach
+ * gcc that smsw clobbers cr0 for example).
 */
 #define _BUILD_GET_R(func, reg)                         \
-   static INLINE void                                   \
-   func(uintptr_t *result)                              \
+   static INLINE uintptr_t                              \
+   func(void)                                           \
   {                                                    \
+      uintptr_t result;                                 \
      __asm__ __volatile__("mov %%" #reg ", %0"         \
-                           : "=r" (*result));           \
+                           : "=r" (result));            \
+      return result;                                    \
   }

 _BUILD_SET_R(_SET_CR0, cr0)
@@ -374,10 +369,10 @@ CLTS(void)
   }

 #define _BUILD_GET_DR(func, reg)                      \
-   static INLINE void                                 \
-   func(uintptr_t *result)                            \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
   {                                                  \
-      *result = __readdr(reg);                        \
+      return __readdr(reg);                           \
   }

 #define _BUILD_SET_CR(func, reg)                      \
@@ -388,10 +383,10 @@ CLTS(void)
   }

 #define _BUILD_GET_CR(func, reg)                      \
-   static INLINE void                                 \
-   func(uintptr_t *result)                            \
+   static INLINE uintptr_t                            \
+   func(void)                                         \
   {                                                  \
-      *result = __readcr##reg();                      \
+      return __readcr##reg();                         \
   }

 _BUILD_SET_DR(_SET_DR0, 0)
@@ -618,7 +613,7 @@ _Get_TR(void)

 #define GET_CR_DR(regType, regNum, var) \
   do {                                 \
-      _GET_##regType##regNum(&(var));   \
+      var = _GET_##regType##regNum();   \
   } while (0)

 #define SET_DR0(expr) SET_CR_DR(DR, 0, expr)
@@ -648,6 +643,11 @@ _Get_TR(void)
 #define SET_CR4(expr) SET_CR_DR(CR, 4, expr)
 #define SET_CR8(expr) SET_CR_DR(CR, 8, expr)

+/*
+ * When the ULM macro-defines INTERRUPTS_ENABLED, attempting to define
+ * it as a function produces hard-to-diagnose compile-time errors.
+ */
+#if !defined(ULM) && !defined(INTERRUPTS_ENABLED)
 static INLINE Bool
 INTERRUPTS_ENABLED(void)
 {
@@ -655,6 +655,7 @@ INTERRUPTS_ENABLED(void)
   SAVE_FLAGS(flags);
   return ((flags & EFLAGS_IF) != 0);
 }
+#endif

 /*
 * [GS]ET_[GI]DT() are defined as macros wrapping a function
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -67,6 +67,7 @@ extern "C" {
 #if defined (VMKPANIC) 
 #include "vmk_assert.h"
 #else /* !VMKPANIC */
+#include <linux/kernel.h>
 #define _ASSERT_PANIC(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
 #define _ASSERT_PANIC_BUG(bug, name) \
@@ -107,7 +108,7 @@ NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
 } while(0)

 #else
-NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+#define Panic panic
 #endif

 void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
@@ -207,7 +208,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +315,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -122,27 +122,27 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

 /*
@@ -3584,7 +3584,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3607,7 +3607,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -48,7 +48,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -1180,6 +1179,9 @@ RoundUpPow2_32(uint32 value)
 static INLINE unsigned
 PopCount32(uint32 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
   /*
    * Attribution:
    *     This algorithm was copied from:
@@ -1223,6 +1225,7 @@ PopCount32(uint32 value)
   value += (value >> 8);
   value += (value >> 16);
   return value & 0x0000003f;
+#endif
 }


@@ -1245,6 +1248,13 @@ PopCount32(uint32 value)
 static INLINE unsigned
 PopCount64(uint64 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
   value -= (value >> 1) & 0x5555555555555555ULL;
   value = ((value >> 2) & 0x3333333333333333ULL) +
           (value & 0x3333333333333333ULL);
@@ -1253,6 +1263,7 @@ PopCount64(uint64 value)
   value += value >> 16;
   value += value >> 32;
   return (unsigned) (value & 0xff);
+#endif
 }


@@ -438,10 +438,6 @@ void *_ReturnAddress(void);

 #define strtok_r  strtok_s

-#if (_MSC_VER < 1500)
-#define	vsnprintf _vsnprintf
-#endif
-
 typedef int uid_t;
 typedef int gid_t;

@@ -465,20 +461,9 @@ typedef int pid_t;
 #define       W_OK          2
 #define       R_OK          4

-#endif // }
+#endif // } _WIN32

-/*
- * Macro for username comparison.
- */
-
-#ifdef _WIN32 // {
-#define USERCMP(x,y)  Str_Strcasecmp(x,y)
-#else
-#define USERCMP(x,y)  strcmp(x,y)
-#endif // }
-
-
-#endif // }
+#endif // } USERLEVEL

 #ifndef va_copy

@@ -657,8 +642,10 @@ typedef int pid_t;
 #endif

 #ifdef VMM
+#define vmx86_vmm 1
 #define VMM_ONLY(x) x
 #else
+#define vmx86_vmm 0
 #define VMM_ONLY(x)
 #endif

@@ -699,7 +686,6 @@ typedef int pid_t;
 * display/printer drivers only.
 */
 #ifdef _WIN32
-#ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
 #ifdef _WIN64
 #define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
@@ -711,7 +697,6 @@ typedef int pid_t;
 #define WinDrvPrint(arg, ...)
 #define WinDrvEngPrint(arg, ...)
 #endif
-#endif
 #endif // _WIN32

 /*
@@ -801,4 +786,30 @@ typedef int pid_t;
 #define END_PTR_TO_ALIGNED_VAR \
   } while (0)

+
+/*
+ * -Wswitch means that when you pass switch an enum that it's looking for
+ * all values from that enum, and only that enum, to be accounted for.
+ * "default:;" is fine for catching values you don't care about. But today
+ * we have a bunch of code that uses internal and external enum values, or
+ * in other words combines two enums into a single variable. This cast is
+ * the workaround, but we really need to fix this mess.
+ */
+#define UNCHECKED_SWITCH__FIXME(x) switch ((uint64)(x))
+
+
+/*
+ * When clang static analyzer parses source files, it implicitly defines
+ * __clang_analyzer__ macro. We use this to define our custom macro to stop
+ * its execution for the current path of analysis by calling a function that
+ * doesn't return, making it think that it hit a failed assertion.
+ *
+ * DO NOT use to silence the analyzer! See PR2447238.
+ */
+#ifdef __clang_analyzer__
+#define VMW_CLANG_ANALYZER_NORETURN() Panic("Disable Clang static analyzer")
+#else
+#define VMW_CLANG_ANALYZER_NORETURN() ((void)0)
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -101,6 +101,7 @@

 #pragma warning (3 :4505) // unreferenced local function
 #pragma warning (disable :4018) // signed/unsigned mismatch
+#pragma warning (suppress:4619) // suppress warning next line (C4761 was removed in vs2019u4)
 #pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
 #pragma warning (disable :4305) // truncation from 'const int' to 'short'
 #pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
@@ -698,11 +699,11 @@ typedef void * UserVA;
 #endif

 /*
- * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Similarly, we require a compiler that is at least vc90 (vs2008).
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1400
-#error "cl.exe version is too old, need vc80 or better"
+#if defined _MSC_VER && _MSC_VER < 1500
+#error "cl.exe version is too old, need vc90 or better"
 #endif


@@ -864,12 +865,10 @@ typedef void * UserVA;
 #endif

 #ifndef UNUSED_TYPE
-// XXX _Pragma would better but doesn't always work right now.
 #  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
 #endif

 #ifndef UNUSED_VARIABLE
-// XXX is there a better way?
 #  define UNUSED_VARIABLE(_var) (void)_var
 #endif

@@ -884,29 +883,21 @@ typedef void * UserVA;

 /*
 * ALIGNED specifies minimum alignment in "n" bytes.
+ *
+ * NOTE: __declspec(align) has limited syntax; it must essentially be
+ *       an integer literal.  Expressions, such as sizeof(), do not
+ *       work.
 */

 #ifdef __GNUC__
 #define ALIGNED(n) __attribute__((__aligned__(n)))
+#elif defined(_MSC_VER)
+#define ALIGNED(n) __declspec(align(n))
 #else
 #define ALIGNED(n)
 #endif


-/*
- * Encapsulate the syntactic differences between gcc and msvc alignment control.
- * BOUNDARY must match in the prefix and suffix.
- */
-
-#ifdef _WIN32
-#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
-#define ALIGN_SUFFIX(BOUNDRY)
-#else
-#define ALIGN_PREFIX(BOUNDRY)
-#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
-#endif
-
-
 /*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000-2015,2017-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2015,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -46,6 +46,7 @@
 #define VMMEM_ANON_IOABLE_PAGE    VMMEM_FLAG_BIT(4)
 #define VMMEM_ANON_ALL_FLAGS      MASK(5)

+#define VMMEM_GUEST_NO_FLAGS      0
 #define VMMEM_GUEST_WRITEABLE     VMMEM_FLAG_BIT(0)
 #define VMMEM_GUEST_BREAKCOW      VMMEM_FLAG_BIT(1)
 #define VMMEM_GUEST_2M_PAGE       VMMEM_FLAG_BIT(2)
@@ -35,6 +35,8 @@
 #define INCLUDE_ALLOW_VMCORE
 #include "includeCheck.h"

+#include <asm/processor-flags.h>
+
 #define X86_MAX_INSTR_LEN  15   /* Max byte length of an x86 instruction. */

 #define NUM_IDT_VECTORS 256
@@ -75,7 +77,9 @@
 #define CR3_PDB_MASK   0xfffff000
 #define CR3_IGNORE     0xFFF
 #define PAE_CR3_IGNORE 0x1F
+#ifndef CR3_PCID_MASK
 #define CR3_PCID_MASK  0xFFF
+#endif
 #define CR3_NO_FLUSH   (1ULL << 63)

 #define CR4_VME        0x00000001
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -58,6 +58,7 @@ typedef struct CPUIDRegs {
 typedef union CPUIDRegsUnion {
   uint32 array[4];
   CPUIDRegs regs;
+   uint64 force8byteAlign[2]; /* See CpuidInfoNodePtr (needed on Apple Mac). */
 } CPUIDRegsUnion;

 /*
@@ -117,49 +118,57 @@ CPUIDQuery;
 * this level or parts of it to guest.
 */

-#define CPUID_CACHED_LEVELS                         \
-   CPUIDLEVEL(TRUE,  0,   0,          0,  0)        \
-   CPUIDLEVEL(TRUE,  1,   1,          0,  0)        \
-   CPUIDLEVEL(FALSE, 2,   2,          0,  0)        \
-   CPUIDLEVEL(FALSE, 4,   4,          7,  0)        \
-   CPUIDLEVEL(FALSE, 5,   5,          0,  0)        \
-   CPUIDLEVEL(TRUE,  6,   6,          0,  0)        \
-   CPUIDLEVEL(TRUE,  7,   7,          1,  0)        \
-   CPUIDLEVEL(FALSE, A,   0xA,        0,  0)        \
-   CPUIDLEVEL(FALSE, B,   0xB,        2,  0)        \
-   CPUIDLEVEL(TRUE,  D,   0xD,       10,  0)        \
-   CPUIDLEVEL(TRUE,  F,   0xF,        2, 13)        \
-   CPUIDLEVEL(TRUE,  10,  0x10,       2, 13)        \
-   CPUIDLEVEL(TRUE,  12,  0x12,       4, 13)        \
-   CPUIDLEVEL(TRUE,  14,  0x14,       2, 13)        \
-   CPUIDLEVEL(TRUE,  15,  0x15,       0, 13)        \
-   CPUIDLEVEL(TRUE,  16,  0x16,       0, 13)        \
-   CPUIDLEVEL(TRUE,  17,  0x17,       4, 14)        \
-   CPUIDLEVEL(FALSE, 400, 0x40000000, 0,  0)        \
-   CPUIDLEVEL(FALSE, 401, 0x40000001, 0,  0)        \
-   CPUIDLEVEL(FALSE, 402, 0x40000002, 0,  0)        \
-   CPUIDLEVEL(FALSE, 403, 0x40000003, 0,  0)        \
-   CPUIDLEVEL(FALSE, 404, 0x40000004, 0,  0)        \
-   CPUIDLEVEL(FALSE, 405, 0x40000005, 0,  0)        \
-   CPUIDLEVEL(FALSE, 406, 0x40000006, 0,  0)        \
-   CPUIDLEVEL(FALSE, 410, 0x40000010, 0,  0)        \
-   CPUIDLEVEL(FALSE, 80,  0x80000000, 0,  0)        \
-   CPUIDLEVEL(TRUE,  81,  0x80000001, 0,  0)        \
-   CPUIDLEVEL(FALSE, 82,  0x80000002, 0,  0)        \
-   CPUIDLEVEL(FALSE, 83,  0x80000003, 0,  0)        \
-   CPUIDLEVEL(FALSE, 84,  0x80000004, 0,  0)        \
-   CPUIDLEVEL(FALSE, 85,  0x80000005, 0,  0)        \
-   CPUIDLEVEL(FALSE, 86,  0x80000006, 0,  0)        \
-   CPUIDLEVEL(FALSE, 87,  0x80000007, 0,  0)        \
-   CPUIDLEVEL(TRUE,  88,  0x80000008, 0,  0)        \
-   CPUIDLEVEL(TRUE,  8A,  0x8000000A, 0,  0)        \
-   CPUIDLEVEL(FALSE, 819, 0x80000019, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81A, 0x8000001A, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81B, 0x8000001B, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81C, 0x8000001C, 0,  0)        \
-   CPUIDLEVEL(FALSE, 81D, 0x8000001D, 5,  0)        \
-   CPUIDLEVEL(FALSE, 81E, 0x8000001E, 0,  0)        \
-   CPUIDLEVEL(TRUE,  81F, 0x8000001F, 0, 14)
+/*            MASKS, LVL, VAL,      CNT, HWV */
+#define CPUID_CACHED_LEVELS                  \
+   CPUIDLEVEL(TRUE,  0,   0x0,        0,  0) \
+   CPUIDLEVEL(TRUE,  1,   0x1,        0,  0) \
+   CPUIDLEVEL(FALSE, 2,   0x2,        0,  0) \
+   CPUIDLEVEL(FALSE, 4,   0x4,        7,  0) \
+   CPUIDLEVEL(FALSE, 5,   0x5,        0,  0) \
+   CPUIDLEVEL(TRUE,  6,   0x6,        0,  0) \
+   CPUIDLEVEL(TRUE,  7,   0x7,        2,  0) \
+   CPUIDLEVEL(TRUE,  9,   0x9,        0, 17) \
+   CPUIDLEVEL(FALSE, A,   0xa,        0,  0) \
+   CPUIDLEVEL(FALSE, B,   0xb,        3,  0) \
+   CPUIDLEVEL(TRUE,  D,   0xd,       10,  0) \
+   CPUIDLEVEL(TRUE,  F,   0xf,        2, 13) \
+   CPUIDLEVEL(TRUE,  10,  0x10,       4, 13) \
+   CPUIDLEVEL(TRUE,  12,  0x12,       4, 13) \
+   CPUIDLEVEL(TRUE,  14,  0x14,       2, 13) \
+   CPUIDLEVEL(TRUE,  15,  0x15,       0, 13) \
+   CPUIDLEVEL(TRUE,  16,  0x16,       0, 13) \
+   CPUIDLEVEL(TRUE,  17,  0x17,       4, 14) \
+   CPUIDLEVEL(TRUE,  18,  0x18,       8, 17) \
+   CPUIDLEVEL(TRUE,  1A,  0x1a,       0, 17) \
+   CPUIDLEVEL(TRUE,  1B,  0x1b,       2, 17) \
+   CPUIDLEVEL(TRUE,  1F,  0x1f,       6, 17) \
+   CPUIDLEVEL(FALSE, 400, 0x40000000, 0,  0) \
+   CPUIDLEVEL(FALSE, 401, 0x40000001, 0,  0) \
+   CPUIDLEVEL(FALSE, 402, 0x40000002, 0,  0) \
+   CPUIDLEVEL(FALSE, 403, 0x40000003, 0,  0) \
+   CPUIDLEVEL(FALSE, 404, 0x40000004, 0,  0) \
+   CPUIDLEVEL(FALSE, 405, 0x40000005, 0,  0) \
+   CPUIDLEVEL(FALSE, 406, 0x40000006, 0,  0) \
+   CPUIDLEVEL(FALSE, 410, 0x40000010, 0,  0) \
+   CPUIDLEVEL(FALSE, 80,  0x80000000, 0,  0) \
+   CPUIDLEVEL(TRUE,  81,  0x80000001, 0,  0) \
+   CPUIDLEVEL(FALSE, 82,  0x80000002, 0,  0) \
+   CPUIDLEVEL(FALSE, 83,  0x80000003, 0,  0) \
+   CPUIDLEVEL(FALSE, 84,  0x80000004, 0,  0) \
+   CPUIDLEVEL(FALSE, 85,  0x80000005, 0,  0) \
+   CPUIDLEVEL(FALSE, 86,  0x80000006, 0,  0) \
+   CPUIDLEVEL(FALSE, 87,  0x80000007, 0,  0) \
+   CPUIDLEVEL(TRUE,  88,  0x80000008, 0,  0) \
+   CPUIDLEVEL(TRUE,  8A,  0x8000000a, 0,  0) \
+   CPUIDLEVEL(FALSE, 819, 0x80000019, 0,  0) \
+   CPUIDLEVEL(FALSE, 81A, 0x8000001a, 0,  0) \
+   CPUIDLEVEL(FALSE, 81B, 0x8000001b, 0,  0) \
+   CPUIDLEVEL(FALSE, 81C, 0x8000001c, 0,  0) \
+   CPUIDLEVEL(FALSE, 81D, 0x8000001d, 5,  0) \
+   CPUIDLEVEL(FALSE, 81E, 0x8000001e, 0,  0) \
+   CPUIDLEVEL(TRUE,  81F, 0x8000001f, 0, 14) \
+   CPUIDLEVEL(TRUE,  820, 0x80000020, 2, 17) \
+   CPUIDLEVEL(TRUE,  821, 0x80000021, 0, 17)

 #define CPUID_ALL_LEVELS CPUID_CACHED_LEVELS

@@ -183,6 +192,7 @@ enum {
 #define CPUID_FEATURE_INFORMATION  0x01
 #define CPUID_PROCESSOR_TOPOLOGY   4
 #define CPUID_MWAIT_FEATURES       5
+#define CPUID_PMC_FEATURES         0xa
 #define CPUID_XSAVE_FEATURES       0xd
 #define CPUID_SGX_FEATURES         0x12
 #define CPUID_PT_FEATURES          0x14
@@ -453,7 +463,13 @@ FLAG(   6,  0, EAX,  9,  1, HWP_ACTIVITY_WINDOW,                 NO,    0 ) \
 FLAG(   6,  0, EAX, 10,  1, HWP_ENERGY_PERFORMANCE_PREFERENCE,   NO,    0 ) \
 FLAG(   6,  0, EAX, 11,  1, HWP_PACKAGE_LEVEL_REQUEST,           NO,    0 ) \
 FLAG(   6,  0, EAX, 13,  1, HDC,                                 NO,    0 ) \
+FLAG(   6,  0, EAX, 14,  1, TURBO_BOOST_MAX_3,                   NO,    0 ) \
+FLAG(   6,  0, EAX, 15,  1, HWP_CAPABILITIES,                    NO,    0 ) \
+FLAG(   6,  0, EAX, 16,  1, HWP_PECI,                            NO,    0 ) \
+FLAG(   6,  0, EAX, 17,  1, HWP_FLEXIBLE,                        NO,    0 ) \
+FLAG(   6,  0, EAX, 18,  1, HWP_FAST_ACCESS,                     NO,    0 ) \
 FLAG(   6,  0, EAX, 19,  1, HW_FEEDBACK,                         NO,    0 ) \
+FLAG(   6,  0, EAX, 20,  1, HWP_IGNORE_IDLE_REQUEST,             NO,    0 ) \
 FIELD(  6,  0, EBX,  0,  4, NUM_INTR_THRESHOLDS,                 NO,    0 ) \
 FLAG(   6,  0, ECX,  0,  1, HW_COORD_FEEDBACK,                   NO,    0 ) \
 FLAG(   6,  0, ECX,  1,  1, ACNT2,                               ANY,  13 ) \
@@ -463,12 +479,9 @@ FLAG(   6,  0, EDX,  1,  1, ENERGY_CAP_REPORTING,                NO,    0 ) \
 FIELD(  6,  0, EDX,  8,  4, HW_FEEDBACK_SIZE,                    NO,    0 ) \
 FIELD(  6,  0, EDX, 16, 16, HW_FEEDBACK_INDEX,                   NO,    0 )

-/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
-#define CPUID_7_EDX_10 \
-FLAG(   7,  0, EDX, 10,  1, LEAF7_RSVD,                          NO,    0 )
-
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_7                                            \
+FIELD(  7,  0, EAX,  0, 32, LEAF_7_MAX_SUBLEVEL,                 YES, FUT ) \
 FLAG(   7,  0, EBX,  0,  1, FSGSBASE,                            YES,   9 ) \
 FLAG(   7,  0, EBX,  1,  1, TSC_ADJUST,                          ANY,  11 ) \
 FLAG(   7,  0, EBX,  2,  1, SGX,                                 ANY,  17 ) \
@@ -507,6 +520,7 @@ FLAG(   7,  0, ECX,  3,  1, PKU,                                 YES,  13 ) \
 FLAG(   7,  0, ECX,  4,  1, OSPKE,                               ANY,  13 ) \
 FLAG(   7,  0, ECX,  5,  1, WAITPKG,                             NO,    0 ) \
 FLAG(   7,  0, ECX,  6,  1, AVX512VBMI2,                         YES,  17 ) \
+FLAG(   7,  0, ECX,  7,  1, CET_SS,                              NO,    0 ) \
 FLAG(   7,  0, ECX,  8,  1, GFNI,                                YES,  17 ) \
 FLAG(   7,  0, ECX,  9,  1, VAES,                                YES,  17 ) \
 FLAG(   7,  0, ECX, 10,  1, VPCLMULQDQ,                          YES,  17 ) \
@@ -516,21 +530,31 @@ FLAG(   7,  0, ECX, 14,  1, AVX512VPOPCNTDQ,                     YES,  16 ) \
 FLAG(   7,  0, ECX, 16,  1, VA57,                                NO,    0 ) \
 FIELD(  7,  0, ECX, 17,  5, MAWA,                                NO,    0 ) \
 FLAG(   7,  0, ECX, 22,  1, RDPID,                               YES,  17 ) \
-FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            NO,    0 ) \
-FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             NO,    0 ) \
-FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           NO,    0 ) \
+FLAG(   7,  0, ECX, 25,  1, CLDEMOTE,                            YES, FUT ) \
+FLAG(   7,  0, ECX, 27,  1, MOVDIRI,                             YES, FUT ) \
+FLAG(   7,  0, ECX, 28,  1, MOVDIR64B,                           YES, FUT ) \
+FLAG(   7,  0, ECX, 29,  1, ENQCMD,                              NO,    0 ) \
 FLAG(   7,  0, ECX, 30,  1, SGX_LC,                              ANY,  17 ) \
 FLAG(   7,  0, EDX,  2,  1, AVX512QVNNIW,                        YES,  16 ) \
 FLAG(   7,  0, EDX,  3,  1, AVX512QFMAPS,                        YES,  16 ) \
-FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   NO,    0 ) \
-CPUID_7_EDX_10 \
+FLAG(   7,  0, EDX,  4,  1, FAST_SHORT_REPMOV,                   YES, FUT ) \
+FLAG(   7,  0, EDX,  8,  1, AVX512VP2INTERSECT,                  YES, FUT ) \
+FLAG(   7,  0, EDX, 10,  1, MDCLEAR,                             YES,   9 ) \
 FLAG(   7,  0, EDX, 13,  1, TSX_MICROCODE_UPDATE,                NO,    0 ) \
+FLAG(   7,  0, EDX, 15,  1, HYBRID,                              NO,    0 ) \
 FLAG(   7,  0, EDX, 18,  1, PCONFIG,                             NO,    0 ) \
+FLAG(   7,  0, EDX, 20,  1, CET_IBT,                             NO,    0 ) \
 FLAG(   7,  0, EDX, 26,  1, IBRSIBPB,                            ANY,   9 ) \
 FLAG(   7,  0, EDX, 27,  1, STIBP,                               YES,   9 ) \
 FLAG(   7,  0, EDX, 28,  1, FCMD,                                YES,   9 ) \
 FLAG(   7,  0, EDX, 29,  1, ARCH_CAPABILITIES,                   ANY,   9 ) \
-FLAG(   7,  0, EDX, 31,  1, SSBD,                                YES,   9 )
+FLAG(   7,  0, EDX, 30,  1, CORE_CAPABILITIES,                   NO,    0 ) \
+FLAG(   7,  0, EDX, 31,  1, SSBD,                                YES,   9 ) \
+FLAG(   7,  1, EAX,  5,  1, AVX512BF16,                          YES, FUT )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_9                                            \
+FIELD(  9,  0, EAX,  0, 32, IA32_PLATFORM_DCA_CAP_VAL,           NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_A                                            \
@@ -574,7 +598,7 @@ FLAG(   D,  0, EAX,  6,  1, XCR0_MASTER_ZMM_H,                   YES,  13 ) \
 FLAG(   D,  0, EAX,  7,  1, XCR0_MASTER_HI16_ZMM,                YES,  13 ) \
 FLAG(   D,  0, EAX,  8,  1, XCR0_MASTER_XSS,                     NO,    0 ) \
 FLAG(   D,  0, EAX,  9,  1, XCR0_MASTER_PKRU,                    YES,  13 ) \
-FIELD(  D,  0, EAX,  10,22, XCR0_MASTER_LOWER,                   NO,    0 ) \
+FIELD(  D,  0, EAX, 10, 22, XCR0_MASTER_LOWER,                   NO,    0 ) \
 FIELD(  D,  0, EBX,  0, 32, XSAVE_ENABLED_SIZE,                  ANY,   8 ) \
 FIELD(  D,  0, ECX,  0, 32, XSAVE_MAX_SIZE,                      YES,   8 ) \
 FIELD(  D,  0, EDX,  0, 29, XCR0_MASTER_UPPER,                   NO,    0 ) \
@@ -588,7 +612,7 @@ FIELD(  D,  1, EBX,  0, 32, XSAVES_ENABLED_SIZE,                 ANY,  13 ) \
 FIELD(  D,  1, ECX,  0,  7, XSS_XCR0_USED0,                      NO,    0 ) \
 FLAG(   D,  1, ECX,  8,  1, XSS_PT,                              NO,    0 ) \
 FIELD(  D,  1, ECX,  9,  1, XSS_XCR0_USED1,                      NO,    0 ) \
-FIELD(  D,  1, ECX,  10,22, XSS_RSVD0,                           NO,    0 ) \
+FIELD(  D,  1, ECX, 10, 22, XSS_RSVD0,                           NO,    0 ) \
 FIELD(  D,  1, EDX,  0, 32, XSS_RSVD1,                           NO,    0 ) \
 FIELD(  D,  2, EAX,  0, 32, XSAVE_YMM_SIZE,                      YES,   8 ) \
 FIELD(  D,  2, EBX,  0, 32, XSAVE_YMM_OFFSET,                    YES,   8 ) \
@@ -637,11 +661,8 @@ FIELD(  D,  9, EBX,  0, 32, XSAVE_PKRU_OFFSET,                   YES,  13 ) \
 FLAG(   D,  9, ECX,  0,  1, XSAVE_PKRU_SUP_BY_XSS,               NO,    0 ) \
 FLAG(   D,  9, ECX,  1,  1, XSAVE_PKRU_ALIGN,                    YES,  13 ) \
 FIELD(  D,  9, ECX,  2, 30, XSAVE_PKRU_RSVD1,                    NO,    0 ) \
-FIELD(  D,  9, EDX,  0, 32, XSAVE_PKRU_RSVD2,                    NO,    0 ) \
-FIELD(  D, 62, EAX,  0, 32, XSAVE_LWP_SIZE,                      NO,    0 ) \
-FIELD(  D, 62, EBX,  0, 32, XSAVE_LWP_OFFSET,                    NO,    0 ) \
-FIELD(  D, 62, ECX,  0, 32, XSAVE_LWP_RSVD1,                     NO,    0 ) \
-FIELD(  D, 62, EDX,  0, 32, XSAVE_LWP_RSVD2,                     NO,    0 )
+FIELD(  D,  9, EDX,  0, 32, XSAVE_PKRU_RSVD2,                    NO,    0 )
+/* D, 62: AMD LWP leaf on BD, PD, SR. Dropped in Zen. Never referenced. */

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_F                                            \
@@ -675,10 +696,10 @@ FIELD( 12,  1, EAX,  0, 32, SECS_ATTRIBUTES0,                    ANY,  17 ) \
 FIELD( 12,  1, EBX,  0, 32, SECS_ATTRIBUTES1,                    ANY,  17 ) \
 FIELD( 12,  1, ECX,  0, 32, SECS_ATTRIBUTES2,                    ANY,  17 ) \
 FIELD( 12,  1, EDX,  0, 32, SECS_ATTRIBUTES3,                    ANY,  17 ) \
-FIELD( 12,  2, EAX,  0, 15, EPC00_VALID,                         ANY,  17 ) \
+FIELD( 12,  2, EAX,  0,  4, EPC00_VALID,                         ANY,  17 ) \
 FIELD( 12,  2, EAX, 12, 20, EPC00_BASE_LOW,                      ANY,  17 ) \
 FIELD( 12,  2, EBX,  0, 20, EPC00_BASE_HIGH,                     ANY,  17 ) \
-FIELD( 12,  2, ECX,  0, 15, EPC00_PROTECTED,                     ANY,  17 ) \
+FIELD( 12,  2, ECX,  0,  4, EPC00_PROTECTED,                     ANY,  17 ) \
 FIELD( 12,  2, ECX, 12, 20, EPC00_SIZE_LOW,                      ANY,  17 ) \
 FIELD( 12,  2, EDX,  0, 20, EPC00_SIZE_HIGH,                     ANY,  17 ) \
 FIELD( 12,  3, EAX,  0,  4, EPC01_VALID,                         NO,    0 ) \
@@ -711,6 +732,7 @@ FIELD( 14,  1, EBX, 16, 16, PT_AVAIL_PSB_FREQ_ENCS,             YES, FUT ) \
 #define CPUID_FIELD_DATA_LEVEL_15                                           \
 FIELD( 15,  0, EAX,  0, 32, DENOM_TSC_TO_CORE_CRYSTAL_CLK,       NO,    0 ) \
 FIELD( 15,  0, EBX,  0, 32, NUMER_TSC_TO_CORE_CRYSTAL_CLK,       NO,    0 ) \
+FIELD( 15,  0, ECX,  0, 32, CORE_CRYSTAL_CLK_FREQ,               NO,    0 ) \

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_16                                           \
@@ -739,7 +761,42 @@ FIELD( 17,  3, ECX,  0, 32, SOC_VENDOR_BRAND_STRING_3_2,         NO,    0 ) \
 FIELD( 17,  3, EDX,  0, 32, SOC_VENDOR_BRAND_STRING_3_3,         NO,    0 ) \

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
-#define CPUID_FIELD_DATA_LEVEL_400                                         \
+#define CPUID_FIELD_DATA_LEVEL_18                                           \
+FIELD( 18,  0, EAX,  0, 32, TLB_INFO_MAX_SUBLEAF,                NO,    0 ) \
+FLAG(  18,  0, EBX,  0,  1, TLB_INFO_LEVEL_SIZE_4K,              NO,    0 ) \
+FLAG(  18,  0, EBX,  1,  1, TLB_INFO_LEVEL_SIZE_2M,              NO,    0 ) \
+FLAG(  18,  0, EBX,  2,  1, TLB_INFO_LEVEL_SIZE_4M,              NO,    0 ) \
+FLAG(  18,  0, EBX,  3,  1, TLB_INFO_LEVEL_SIZE_1G,              NO,    0 ) \
+FIELD( 18,  0, EBX,  8,  3, TLB_INFO_PARTITIONING,               NO,    0 ) \
+FIELD( 18,  0, EBX, 16, 16, TLB_INFO_NUM_WAYS,                   NO,    0 ) \
+FIELD( 18,  0, ECX,  0, 32, TLB_INFO_NUM_SETS,                   NO,    0 ) \
+FIELD( 18,  0, EDX,  0,  5, TLB_INFO_TYPE,                       NO,    0 ) \
+FIELD( 18,  0, EDX,  5,  3, TLB_INFO_LEVEL,                      NO,    0 ) \
+FLAG(  18,  0, EDX,  8,  1, TLB_INFO_FULLY_ASSOCIATIVE,          NO,    0 ) \
+FIELD( 18,  0, EDX, 14, 12, TLB_INFO_MAX_ADDRESSABLE_IDS,        NO,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_1A                                           \
+FIELD( 1A,  0, EAX,  0, 24, NATIVE_MODEL_ID,                     NO,    0 ) \
+FIELD( 1A,  0, EAX, 24,  8, CORE_TYPE,                           NO,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_1B                                           \
+FIELD( 1B,  0, EAX,  0, 12, PCONFIG_SUBLEAF_TYPE,                NO,    0 ) \
+FIELD( 1B,  0, EBX,  0, 32, PCONFIG_TARGET_ID1,                  NO,    0 ) \
+FIELD( 1B,  0, ECX,  0, 32, PCONFIG_TARGET_ID2,                  NO,    0 ) \
+FIELD( 1B,  0, EDX,  0, 32, PCONFIG_TARGET_ID3,                  NO,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_1F                                           \
+FIELD( 1F,  0, EAX,  0,  5, TOPOLOGY_V2_MASK_WIDTH,              NO,    0 ) \
+FIELD( 1F,  0, EBX,  0, 16, TOPOLOGY_V2_CPUS_SHARING_LEVEL,      NO,    0 ) \
+FIELD( 1F,  0, ECX,  0,  8, TOPOLOGY_V2_LEVEL_NUMBER,            NO,    0 ) \
+FIELD( 1F,  0, ECX,  8,  8, TOPOLOGY_V2_LEVEL_TYPE,              NO,    0 ) \
+FIELD( 1F,  0, EDX,  0, 32, TOPOLOGY_V2_X2APIC_ID,               NO,    0 )
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_400                                          \
 FIELD(400,  0, EAX,  0, 32, MAX_HYP_LEVEL,                       NA,    0 ) \
 FIELD(400,  0, EBX,  0, 32, HYPERVISOR_VENDOR0,                  NA,    0 ) \
 FIELD(400,  0, ECX,  0, 32, HYPERVISOR_VENDOR1,                  NA,    0 ) \
@@ -850,16 +907,17 @@ FIELD( 80,  0, EBX,  0, 32, LEAF80_VENDOR1,                      NA,    0 ) \
 FIELD( 80,  0, ECX,  0, 32, LEAF80_VENDOR3,                      NA,    0 ) \
 FIELD( 80,  0, EDX,  0, 32, LEAF80_VENDOR2,                      NA,    0 )

+#define CPUID_81_ECX_14 \
+FLAG(  81,  0, ECX, 14,  1, LEAF81ECX_RSVD1,                     NO,    0 )
+
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_81                                           \
-FIELD( 81,  0, EAX,  0, 32, UNKNOWN81EAX,                        ANY,   4 ) \
 FIELD( 81,  0, EAX,  0,  4, LEAF81_STEPPING,                     ANY,   4 ) \
 FIELD( 81,  0, EAX,  4,  4, LEAF81_MODEL,                        ANY,   4 ) \
 FIELD( 81,  0, EAX,  8,  4, LEAF81_FAMILY,                       ANY,   4 ) \
 FIELD( 81,  0, EAX, 12,  2, LEAF81_TYPE,                         ANY,   4 ) \
 FIELD( 81,  0, EAX, 16,  4, LEAF81_EXTENDED_MODEL,               ANY,   4 ) \
 FIELD( 81,  0, EAX, 20,  8, LEAF81_EXTENDED_FAMILY,              ANY,   4 ) \
-FIELD( 81,  0, EBX,  0, 32, UNKNOWN81EBX,                        ANY,   4 ) \
 FIELD( 81,  0, EBX,  0, 16, LEAF81_BRAND_ID,                     ANY,   4 ) \
 FIELD( 81,  0, EBX, 16, 16, UNDEF,                               ANY,   4 ) \
 FLAG(  81,  0, ECX,  0,  1, LAHF64,                              YES,   4 ) \
@@ -876,6 +934,7 @@ FLAG(  81,  0, ECX, 10,  1, IBS,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 11,  1, XOP,                                 YES,   8 ) \
 FLAG(  81,  0, ECX, 12,  1, SKINIT,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 13,  1, WATCHDOG,                            NO,    0 ) \
+CPUID_81_ECX_14 \
 FLAG(  81,  0, ECX, 15,  1, LWP,                                 NO,    0 ) \
 FLAG(  81,  0, ECX, 16,  1, FMA4,                                YES,   8 ) \
 FLAG(  81,  0, ECX, 17,  1, TCE,                                 NO,    0 ) \
@@ -888,6 +947,7 @@ FLAG(  81,  0, ECX, 26,  1, DATABK,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 27,  1, PERFTSC,                             NO,    0 ) \
 FLAG(  81,  0, ECX, 28,  1, PERFL3,                              NO,    0 ) \
 FLAG(  81,  0, ECX, 29,  1, MWAITX,                              NO,    0 ) \
+FLAG(  81,  0, ECX, 30,  1, ADDR_MASK_EXT,                       NO,    0 ) \
 FLAG(  81,  0, EDX,  0,  1, LEAF81_FPU,                          YES,   4 ) \
 FLAG(  81,  0, EDX,  1,  1, LEAF81_VME,                          YES,   4 ) \
 FLAG(  81,  0, EDX,  2,  1, LEAF81_DE,                           YES,   4 ) \
@@ -982,6 +1042,7 @@ FLAG(  87,  0, EBX,  1,  1, SUCCOR,                              NA,    0 ) \
 FLAG(  87,  0, EBX,  2,  1, HWA,                                 NA,    0 ) \
 FLAG(  87,  0, EBX,  3,  1, SCALABLE_MCA,                        NA,    0 ) \
 FLAG(  87,  0, EBX,  4,  1, PFEH_SUPPORT_PRESENT,                NA,    0 ) \
+FIELD( 87,  0, ECX,  0, 32, POWER_SAMPLE_TIME_RATIO,             NA,    0 ) \
 FLAG(  87,  0, EDX,  0,  1, TS,                                  NA,    0 ) \
 FLAG(  87,  0, EDX,  1,  1, FID,                                 NA,    0 ) \
 FLAG(  87,  0, EDX,  2,  1, VID,                                 NA,    0 ) \
@@ -991,7 +1052,12 @@ FLAG(  87,  0, EDX,  5,  1, STC,                                 NA,    0 ) \
 FLAG(  87,  0, EDX,  6,  1, 100MHZSTEPS,                         NA,    0 ) \
 FLAG(  87,  0, EDX,  7,  1, HWPSTATE,                            NA,    0 ) \
 FLAG(  87,  0, EDX,  8,  1, TSC_INVARIANT,                       NA,    0 ) \
-FLAG(  87,  0, EDX,  9,  1, CORE_PERF_BOOST,                     NA,    0 )
+FLAG(  87,  0, EDX,  9,  1, CORE_PERF_BOOST,                     NA,    0 ) \
+FLAG(  87,  0, EDX, 10,  1, EFFECTIVE_FREQUENCY,                 NA,    0 ) \
+FLAG(  87,  0, EDX, 11,  1, PROC_FEEDBACK_INTERFACE,             NA,    0 ) \
+FLAG(  87,  0, EDX, 12,  1, PROC_POWER_REPORTING,                NA,    0 ) \
+FLAG(  87,  0, EDX, 13,  1, CONNECTED_STANDBY,                   NA,    0 ) \
+FLAG(  87,  0, EDX, 14,  1, RAPL,                                NA,    0 )

 /*    LEVEL, REG, POS, SIZE, NAME,                          MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_88                                           \
@@ -1001,7 +1067,9 @@ FIELD( 88,  0, EAX, 16,  8, GUEST_PHYS_ADDR_SZ,                  YES,   8 ) \
 FLAG(  88,  0, EBX,  0,  1, CLZERO,                              YES,  14 ) \
 FLAG(  88,  0, EBX,  1,  1, IRPERF,                              NO,    0 ) \
 FLAG(  88,  0, EBX,  2,  1, XSAVE_ERR_PTR,                       NO,    0 ) \
-FLAG(  88,  0, EBX,  9,  1, WBNOINVD,                            YES, FUT ) \
+FLAG(  88,  0, EBX,  4,  1, RDPRU,                               NO,    0 ) \
+FLAG(  88,  0, EBX,  6,  1, MBE,                                 NO,    0 ) \
+FLAG(  88,  0, EBX,  9,  1, WBNOINVD,                            YES,  17 ) \
 FLAG(  88,  0, EBX, 12,  1, LEAF88_IBPB,                         ANY,   9 ) \
 FLAG(  88,  0, EBX, 14,  1, LEAF88_IBRS,                         NO,    0 ) \
 FLAG(  88,  0, EBX, 15,  1, LEAF88_STIBP,                        NO,    0 ) \
@@ -1013,14 +1081,13 @@ FLAG(  88,  0, EBX, 25,  1, LEAF88_SSBD_VIRT_SPEC_CTRL,          ANY,   9 ) \
 FLAG(  88,  0, EBX, 26,  1, LEAF88_SSBD_NOT_NEEDED,              NO,    0 ) \
 FIELD( 88,  0, ECX,  0,  8, LEAF88_CORE_COUNT,                   YES,   4 ) \
 FIELD( 88,  0, ECX, 12,  4, APICID_COREID_SIZE,                  YES,   7 ) \
-FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                        NO,    0 )
+FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                        NO,    0 ) \
+FIELD( 88,  0, EDX, 16,  8, RDPRU_MAX,                           NO,    0 )

 #define CPUID_8A_EDX_11 \
 FLAG(  8A,  0, EDX, 11,  1, SVMEDX_RSVD1,                        NO,    0 )
 #define CPUID_8A_EDX_14 \
 FLAG(  8A,  0, EDX, 14,  1, SVMEDX_RSVD2,                        NO,    0 )
-#define CPUID_8A_EDX_17 \
-FLAG(  8A,  0, EDX, 17,  1, SVMEDX_RSVD3,                        NO,    0 )

 /*    LEVEL, REG, POS, SIZE, NAME,                          MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_8A                                           \
@@ -1045,8 +1112,10 @@ FLAG(  8A,  0, EDX, 13,  1, SVM_AVIC,                            NO,    0 ) \
 CPUID_8A_EDX_14 \
 FLAG(  8A,  0, EDX, 15,  1, SVM_V_VMSAVE_VMLOAD,                 NO,    0 ) \
 FLAG(  8A,  0, EDX, 16,  1, SVM_VGIF,                            NO,    0 ) \
-CPUID_8A_EDX_17 \
-FIELD( 8A,  0, EDX, 18, 14, SVMEDX_RSVD,                         NO,    0 )
+FLAG(  8A,  0, EDX, 17,  1, SVM_GMET,                            YES,  17 ) \
+FIELD( 8A,  0, EDX, 18,  2, SVMEDX_RSVD3,                        NO,    0 ) \
+FLAG(  8A,  0, EDX, 20,  1, SVM_GUEST_SPEC_CTRL,                 NO,    0 ) \
+FIELD( 8A,  0, EDX, 21, 11, SVMEDX_RSVD4,                        NO,    0 )

 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_819                                          \
@@ -1137,20 +1206,28 @@ FIELD(81E,  0, EBX,  8,  2, CORES_PER_COMPUTE_UNIT,              NA,    0 ) \
 FIELD(81E,  0, ECX,  0,  8, NODEID_VAL,                          NA,    0 ) \
 FIELD(81E,  0, ECX,  8,  3, NODES_PER_PKG,                       NA,    0 )

+#define CPUID_81F_EAX_10 \
+FLAG( 81F,  0, EAX, 10,  1, SVMEAX_RSVD1,                        NO,    0 )
+
 /*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
 #define CPUID_FIELD_DATA_LEVEL_81F                                          \
 FLAG( 81F,  0, EAX,  0,  1, SME,                                 NO,    0 ) \
 FLAG( 81F,  0, EAX,  1,  1, SEV,                                 YES,  17 ) \
 FLAG( 81F,  0, EAX,  2,  1, PAGE_FLUSH_MSR,                      NO,    0 ) \
 FLAG( 81F,  0, EAX,  3,  1, SEV_ES,                              YES,  17 ) \
+CPUID_81F_EAX_10 \
 FIELD(81F,  0, EBX,  0,  6, SME_PAGE_TABLE_BIT_NUM,              YES,  17 ) \
 FIELD(81F,  0, EBX,  6,  6, SME_PHYS_ADDR_SPACE_REDUCTION,       NO,    0 ) \
 FIELD(81F,  0, ECX,  0, 32, NUM_ENCRYPTED_GUESTS,                NO,    0 ) \
 FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )

-#define INTEL_CPUID_FIELD_DATA
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,               MON SUPP, HWV  */
+#define CPUID_FIELD_DATA_LEVEL_820                                          \
+FLAG( 820,  0, EBX,  1,  1, LEAF820_MBE,                         NO,    0 ) \
+FIELD(820,  1, EAX,  0, 32, CAPACITY_MASK_LEN,                   NO,    0 ) \
+FIELD(820,  1, EDX,  0, 32, NUM_SERVICE_CLASSES,                 NO,    0 )

-#define AMD_CPUID_FIELD_DATA
+#define CPUID_FIELD_DATA_LEVEL_821

 #define CPUID_FIELD_DATA                                              \
   CPUID_FIELD_DATA_LEVEL_0                                           \
@@ -1160,6 +1237,7 @@ FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )
   CPUID_FIELD_DATA_LEVEL_5                                           \
   CPUID_FIELD_DATA_LEVEL_6                                           \
   CPUID_FIELD_DATA_LEVEL_7                                           \
+   CPUID_FIELD_DATA_LEVEL_9                                           \
   CPUID_FIELD_DATA_LEVEL_A                                           \
   CPUID_FIELD_DATA_LEVEL_B                                           \
   CPUID_FIELD_DATA_LEVEL_D                                           \
@@ -1170,6 +1248,10 @@ FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )
   CPUID_FIELD_DATA_LEVEL_15                                          \
   CPUID_FIELD_DATA_LEVEL_16                                          \
   CPUID_FIELD_DATA_LEVEL_17                                          \
+   CPUID_FIELD_DATA_LEVEL_18                                          \
+   CPUID_FIELD_DATA_LEVEL_1A                                          \
+   CPUID_FIELD_DATA_LEVEL_1B                                          \
+   CPUID_FIELD_DATA_LEVEL_1F                                          \
   CPUID_FIELD_DATA_LEVEL_400                                         \
   CPUID_FIELD_DATA_LEVEL_401                                         \
   CPUID_FIELD_DATA_LEVEL_402                                         \
@@ -1195,8 +1277,8 @@ FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                        NO,    0 )
   CPUID_FIELD_DATA_LEVEL_81D                                         \
   CPUID_FIELD_DATA_LEVEL_81E                                         \
   CPUID_FIELD_DATA_LEVEL_81F                                         \
-   INTEL_CPUID_FIELD_DATA                                             \
-   AMD_CPUID_FIELD_DATA
+   CPUID_FIELD_DATA_LEVEL_820                                         \
+   CPUID_FIELD_DATA_LEVEL_821

 /*
 * Define all field and flag values as an enum.  The result is a full
@@ -1376,6 +1458,7 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_FAMILY_BULLDOZER       0x15  // BD PD SR EX
 #define CPUID_FAMILY_KYOTO           0x16  // Note: Jaguar microarch
 #define CPUID_FAMILY_ZEN             0x17
+#define CPUID_FAMILY_ZEN3            0x19

 /* Effective VIA CPU Families */
 #define CPUID_FAMILY_C7               6
@@ -1429,10 +1512,13 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_MODEL_ATOM_5D        0x5d  // Future Silvermont
 #define CPUID_MODEL_SKYLAKE_5E     0x5e  // Skylake-S / Kaby Lake S/H ES
 #define CPUID_MODEL_ATOM_5F        0x5f  // Denverton
+#define CPUID_MODEL_ATOM_86        0x86  // Snow Ridge
 #define CPUID_MODEL_CANNONLAKE_66  0x66  // Cannon Lake
 #define CPUID_MODEL_KNM_85         0x85  // Knights Mill
 #define CPUID_MODEL_KABYLAKE_8E    0x8e  // Kaby Lake U/Y QS
 #define CPUID_MODEL_KABYLAKE_9E    0x9e  // Kaby Lake S/H QS
+#define CPUID_MODEL_COMETLAKE_A5   0xa5  // Comet Lake S
+#define CPUID_MODEL_COMETLAKE_A6   0xa6  // Comet Lake U

 /* Intel stepping information */
 #define CPUID_STEPPING_KABYLAKE_ES     0x8  // Kaby Lake S/H/U/Y ES
@@ -1441,6 +1527,7 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_STEPPING_CASCADELAKE_A   0x5  // Cascade Lake A-step
 #define CPUID_STEPPING_CASCADELAKE_B1  0x7  // Cascade Lake B1-step
 #define CPUID_STEPPING_WHISKEYLAKE     0xB  // Whiskey Lake U
+#define CPUID_STEPPING_AMBERLAKE       0xC  // Amber Lake Y

 #define CPUID_MODEL_PIII_07    7
 #define CPUID_MODEL_PIII_08    8
@@ -1468,6 +1555,10 @@ CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
 #define CPUID_MODEL_ZEN_1F            0x1F // Max Zen model defined in BKDG
 #define CPUID_MODEL_ZEN2_30           0x30 // family == CPUID_FAMILY_ZEN
 #define CPUID_MODEL_ZEN2_3F           0x3F // Max Zen2 model
+#define CPUID_MODEL_ZEN2_70           0x70 // Ryzen3: family Zen, model Zen2
+#define CPUID_MODEL_ZEN2_7F           0x7F // Ryzen3: max model
+#define CPUID_MODEL_ZEN3_00           0x00 // family == CPUID_FAMILY_ZEN3
+#define CPUID_MODEL_ZEN3_0F           0x0F // Max Zen3 model

 /* AMD stepping information */
 #define CPUID_STEPPING_ZEN_NAPLES_B2  0x02 // Zen Naples ZP-B2
@@ -1716,6 +1807,24 @@ CPUID_MODEL_IS_WHISKEYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
          CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_WHISKEYLAKE;
 }

+static INLINE Bool
+CPUID_MODEL_IS_COMETLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_COMETLAKE_A5 ||
+           CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_COMETLAKE_A6);
+}
+
+static INLINE Bool
+CPUID_MODEL_IS_AMBERLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_8E &&
+          CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_AMBERLAKE;
+}
+
 static INLINE Bool
 CPUID_MODEL_IS_KABYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
 {
@@ -1747,6 +1856,8 @@ CPUID_UARCH_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
          CPUID_MODEL_IS_KABYLAKE(v)    ||
          CPUID_MODEL_IS_COFFEELAKE(v)  ||
          CPUID_MODEL_IS_WHISKEYLAKE(v) ||
+          CPUID_MODEL_IS_COMETLAKE(v)   ||
+          CPUID_MODEL_IS_AMBERLAKE(v)   ||
          CPUID_MODEL_IS_CASCADELAKE(v) ||
          CPUID_MODEL_IS_CANNONLAKE(v);
 }
@@ -1801,6 +1912,21 @@ CPUID_MODEL_IS_DENVERTON(uint32 v) // IN: %eax from CPUID with %eax=1.
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_5F;
 }

+static INLINE Bool
+CPUID_MODEL_IS_SNOWRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) &&
+          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_86;
+}
+
+static INLINE Bool
+CPUID_UARCH_IS_TREMONT(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) && CPUID_MODEL_IS_SNOWRIDGE(v);
+}
+
 static INLINE Bool
 CPUID_MODEL_IS_WESTMERE(uint32 v) // IN: %eax from CPUID with %eax=1.
 {
@@ -1938,6 +2064,12 @@ CPUID_FAMILY_IS_ZEN(uint32 eax)
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN;
 }

+static INLINE Bool
+CPUID_FAMILY_IS_ZEN3(uint32 eax)
+{
+   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN3;
+}
+
 /*
 * AMD Barcelona (of either Opteron or Phenom kind).
 */
@@ -2044,8 +2176,10 @@ static INLINE Bool
 CPUID_MODEL_IS_ZEN2(uint32 eax)
 {
  return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN &&
-         (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_ZEN2_30 &&
-          CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN2_3F);
+         ((CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_ZEN2_30 &&
+           CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN2_3F) ||
+          (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_ZEN2_70 &&
+           CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN2_7F));
 }


@@ -2064,14 +2198,18 @@ CPUID_MODEL_IS_DHYANA_A(uint32 eax)
 }


+static INLINE Bool
+CPUID_MODEL_IS_ZEN3(uint32 eax)
+{
+  return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN3 &&
+         CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN3_0F;
+}
+
+
 #define CPUID_TYPE_PRIMARY     0
 #define CPUID_TYPE_OVERDRIVE   1
 #define CPUID_TYPE_SECONDARY   2

-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_NULL      0
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_DATA      1
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_INST      2
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_UNIF      3
 #define CPUID_LEAF4_CACHE_TYPE_NULL      0
 #define CPUID_LEAF4_CACHE_TYPE_DATA      1
 #define CPUID_LEAF4_CACHE_TYPE_INST      2
@@ -2079,17 +2217,15 @@ CPUID_MODEL_IS_DHYANA_A(uint32 eax)
 #define CPUID_LEAF4_CACHE_INDEXING_DIRECT  0
 #define CPUID_LEAF4_CACHE_INDEXING_COMPLEX 1

-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_SELF_INIT      0x00000100
-#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_FULLY_ASSOC    0x00000200
 #define CPUID_LEAF4_CACHE_SELF_INIT      0x00000100
 #define CPUID_LEAF4_CACHE_FULLY_ASSOC    0x00000200

-#define CPUID_INTEL_IDBECX_LEVEL_TYPE_INVALID   0
-#define CPUID_INTEL_IDBECX_LEVEL_TYPE_SMT       1
-#define CPUID_INTEL_IDBECX_LEVEL_TYPE_CORE      2
 #define CPUID_TOPOLOGY_LEVEL_TYPE_INVALID   0
 #define CPUID_TOPOLOGY_LEVEL_TYPE_SMT       1
 #define CPUID_TOPOLOGY_LEVEL_TYPE_CORE      2
+#define CPUID_TOPOLOGY_LEVEL_TYPE_MODULE    3
+#define CPUID_TOPOLOGY_LEVEL_TYPE_TILE      4
+#define CPUID_TOPOLOGY_LEVEL_TYPE_DIE       5

 #define CPUID_AMD_LEAF85_L1_CACHE_FULLY_ASSOC     0xff
 #define CPUID_AMD_LEAF86_L2_L3_CACHE_FULLY_ASSOC  0x0f
@@ -2103,64 +2239,13 @@ CPUID_MODEL_IS_DHYANA_A(uint32 eax)
 * places to ensure ordering.
 */

-static INLINE Bool
-CPUID_VendorRequiresFence(CpuidVendor vendor)
-{
-   return vendor == CPUID_VENDOR_AMD;
-}
-
-static INLINE Bool
-CPUID_VersionRequiresFence(uint32 version)
-{
-   return CPUID_EFFECTIVE_FAMILY(version) == CPUID_FAMILY_K8 &&
-          CPUID_EFFECTIVE_MODEL(version) < 0x40;
-}
-
-static INLINE Bool
-CPUID_ID0RequiresFence(CPUIDRegs *id0)
-{
-   if (id0->eax == 0) {
-      return FALSE;
-   }
-   return CPUID_IsVendorAMD(id0);
-}
-
-static INLINE Bool
-CPUID_ID1RequiresFence(CPUIDRegs *id1)
-{
-   return CPUID_VersionRequiresFence(id1->eax);
-}
-
 static INLINE Bool
 CPUID_RequiresFence(CpuidVendor vendor, // IN
-                    uint32 version)      // IN: %eax from CPUID with %eax=1.
+                    uint32 version)     // IN: %eax from CPUID with %eax=1.
 {
-   return CPUID_VendorRequiresFence(vendor) &&
-          CPUID_VersionRequiresFence(version);
-}
-
-
-/*
- * The following low-level functions compute the number of
- * cores per cpu.  They should be used cautiously because
- * they do not necessarily work on all types of CPUs.
- * High-level functions that are correct for all CPUs are
- * available elsewhere: see lib/cpuidInfo/cpuidInfo.c.
- */
-
-static INLINE uint32
-CPUID_IntelCoresPerPackage(uint32 v) /* %eax from CPUID with %eax=4 and %ecx=0. */
-{
-   // Note: This is not guaranteed to work on older Intel CPUs.
-   return 1 + CPUID_GET(4, EAX, LEAF4_CORE_COUNT, v);
-}
-
-
-static INLINE uint32
-CPUID_AMDCoresPerPackage(uint32 v) /* %ecx from CPUID with %eax=0x80000008. */
-{
-   // Note: This is not guaranteed to work on older AMD CPUs.
-   return 1 + CPUID_GET(0x80000008, ECX, LEAF88_CORE_COUNT, v);
+   return vendor == CPUID_VENDOR_AMD &&
+          CPUID_EFFECTIVE_FAMILY(version) == CPUID_FAMILY_K8 &&
+          CPUID_EFFECTIVE_MODEL(version) < 0x40;
 }


@@ -2188,8 +2273,8 @@ CPUID_LevelUsesEcx(uint32 level) {
   switch (level)
   {

-#define CPUIDLEVEL(t, s, v, c, h)   \
-      case v:                       \
+#define CPUIDLEVEL(t, s, v, c, h)     \
+      case v:                         \
         return c != 0;

      CPUID_ALL_LEVELS
@@ -2201,37 +2286,6 @@ CPUID_LevelUsesEcx(uint32 level) {
   }
 }

-/*
- *----------------------------------------------------------------------
- *
- * CPUID_IsValid*Subleaf --
- *
- *      Functions to determine the last subleaf for the level specified
- *
- *----------------------------------------------------------------------
- */
-
-static INLINE Bool
-CPUID_IsValidBSubleaf(uint32 ebx)  // IN: %ebx = cpuid.b.sublevel.ebx
-{
-   return ebx != 0;
-}
-
-static INLINE Bool
-CPUID_IsValid4Subleaf(uint32 eax)  // IN: %eax = cpuid.4.sublevel.eax
-{
-   return eax != 0;
-}
-
-static INLINE Bool
-CPUID_IsValid7Subleaf(uint32 eax, uint32 subleaf)  // IN: %eax = cpuid.7.0.eax
-{
-   /*
-    * cpuid.7.0.eax is the max ecx (subleaf) index
-    */
-   return subleaf <= eax;
-}
-
 /*
 *----------------------------------------------------------------------
 *
@@ -2251,8 +2305,9 @@ CPUID_SupportsMsrPlatformInfo(CpuidVendor vendor, uint32 version)
           CPUID_UARCH_IS_HASWELL(version)     ||
           CPUID_UARCH_IS_SKYLAKE(version)     ||
           CPUID_MODEL_IS_KNIGHTS_LANDING(version) ||
-           CPUID_MODEL_IS_DENVERTON(version) ||
-           CPUID_UARCH_IS_SILVERMONT(version));
+           CPUID_MODEL_IS_DENVERTON(version)   ||
+           CPUID_UARCH_IS_SILVERMONT(version)  ||
+           CPUID_UARCH_IS_TREMONT(version));
 }

 #ifdef _MSC_VER
@@ -40,15 +40,7 @@


 /*
- * x86-64 windows doesn't support inline asm so we have to use these
- * intrinsic functions defined in the compiler.  Not all of these are well
- * documented.  There is an array in the compiler dll (c1.dll) which has
- * an array of the names of all the intrinsics minus the leading
- * underscore.  Searching around in the ntddk.h file can also be helpful.
- *
- * The declarations for the intrinsic functions were taken from the DDK.
- * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
- * will complain about second linkage of the intrinsic functions.
+ * The declarations for the intrinsic functions were taken from MSDN.
 * We define the intrinsic using the basic types corresponding to the
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
@@ -57,21 +49,14 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#ifdef VM_X86_64
-/*
- * intrinsic functions only supported by x86-64 windows as of 2k3sp1
- */
-void __cpuid(int regs[4], int eax);
-#pragma intrinsic(__cpuid)
-
 /*
+ * __cpuid has been supported since VS2003
 * __cpuidex has been supported since VS2008
 */
-#if _MSC_VER >= 1500
+void __cpuid(int regs[4], int eax);
 void __cpuidex(int regs[4], int eax, int ecx);
+#pragma intrinsic(__cpuid)
 #pragma intrinsic(__cpuidex)
-#endif /* _MSC_VER >= 1500 */
-#endif /* VM_X86_64 */

 #ifdef __cplusplus
 }
@@ -89,34 +74,16 @@ void __cpuidex(int regs[4], int eax, int ecx);
 *
 */

-/*
- * %ebx is reserved on i386 PIC.  Apple's gcc-5493 (gcc 4.0) compiling
- * for x86_64 incorrectly errors out saying %ebx is reserved.  This is
- * Apple bug 7304232.
- */
-#if vm_x86_64 ? (defined __APPLE_CC__ && __APPLE_CC__ == 5493) : defined __PIC__
-#if vm_x86_64
-/*
- * Note that this generates movq %rbx,%rbx; cpuid; xchgq %rbx,%rbx ...
- * Unfortunately Apple's assembler does not have .ifnes, and I cannot
- * figure out how to do that with .if.   If we ever enable this code
- * on other 64bit systems, both movq & xchgq should be surrounded by
- * .ifnes \"%%rbx\", \"%q1\" & .endif
- */
-#define VM_CPUID_BLOCK  "movq %%rbx, %q1\n\t" \
-                        "cpuid\n\t"           \
-                        "xchgq %%rbx, %q1\n\t"
-#define VM_EBX_OUT(reg) "=&r"(reg)
+#if defined VM_X86_64 || !defined __PIC__
+#define VM_CPUID_BLOCK  "cpuid"
+#define VM_EBX_OUT(reg) "=b"(reg)
 #else
+/* %ebx is reserved on i386 PIC. */
 #define VM_CPUID_BLOCK  "movl %%ebx, %1\n\t" \
                        "cpuid\n\t"          \
                        "xchgl %%ebx, %1\n\t"
 #define VM_EBX_OUT(reg) "=&rm"(reg)
 #endif
-#else
-#define VM_CPUID_BLOCK  "cpuid"
-#define VM_EBX_OUT(reg) "=b"(reg)
-#endif

 static INLINE void
 __GET_CPUID(uint32     eax,  // IN
@@ -217,22 +184,6 @@ __GET_EDX_FROM_CPUID(uint32 eax) // IN
 }


-static INLINE uint32
-__GET_EAX_FROM_CPUID4(uint32 ecx) // IN
-{
-   uint32 eax;
-   uint32 ebx;
-
-   __asm__ __volatile__(
-      VM_CPUID_BLOCK
-      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
-      : "a" (4), "c" (ecx)
-      : "memory", "%edx"
-   );
-
-   return eax;
-}
-
 #undef VM_CPUID_BLOCK
 #undef VM_EBX_OUT

@@ -241,247 +192,51 @@ __GET_EAX_FROM_CPUID4(uint32 ecx) // IN
 static INLINE void
 __GET_CPUID(uint32 input, CPUIDRegs *regs)
 {
-#ifdef VM_X86_64
   __cpuid((int *)regs, input);
-#else
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-#endif
 }

-#ifdef VM_X86_64
-
-#if _MSC_VER >= 1500
-
-/*
- * __cpuidex has been supported since VS2008
- */
-
 static INLINE void
 __GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
 {
   __cpuidex((int *)regs, inputEax, inputEcx);
 }

-#else // _MSC_VER >= 1500
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern void
-__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs);
-#endif // _MSC_VER >= 1500
-
-#else // VM_X86_64
-
-static INLINE void
-__GET_CPUID2(uint32 inputEax, uint32 inputEcx, CPUIDRegs *regs)
-{
-   __asm push esi
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, inputEax
-   __asm mov  ecx, inputEcx
-   __asm mov  esi, regs
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov 0x0[esi], eax
-   __asm mov 0x4[esi], ebx
-   __asm mov 0x8[esi], ecx
-   __asm mov 0xC[esi], edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-   __asm pop esi
-}
-#endif
-
 static INLINE uint32
 __GET_EAX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.eax;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
 __GET_EBX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ebx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ebx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
 __GET_ECX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ecx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, ecx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

 static INLINE uint32
 __GET_EDX_FROM_CPUID(uint32 input)
 {
-#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.edx;
-#else
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, input
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, edx
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-#endif
 }

-#ifdef VM_X86_64
-
-/*
- * No inline assembly in Win64. Implemented in bora/lib/misc in
- * cpuidMasm64.asm.
- */
-
-extern uint32
-__GET_EAX_FROM_CPUID4(uint32 inputEcx);
-
-#else // VM_X86_64
-
-static INLINE uint32
-__GET_EAX_FROM_CPUID4(uint32 inputEcx)
-{
-   uint32 output;
-
-   //NOT_TESTED();
-   __asm push ebx
-   __asm push ecx
-   __asm push edx
-
-   __asm mov  eax, 4
-   __asm mov  ecx, inputEcx
-   __asm _emit 0x0f __asm _emit 0xa2
-   __asm mov  output, eax
-
-   __asm pop edx
-   __asm pop ecx
-   __asm pop ebx
-
-   return output;
-}
-
-#endif // VM_X86_64
-
 #else // }
 #error
 #endif

 #define CPUID_FOR_SIDE_EFFECTS() ((void)__GET_EAX_FROM_CPUID(0))

-/* The first parameter is used as an rvalue and then as an lvalue. */
-#define GET_CPUID(_ax, _bx, _cx, _dx) { \
-   CPUIDRegs regs;                      \
-   __GET_CPUID(_ax, &regs);             \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
-#define GET_CPUID2(_ax, _bx, _cx, _dx) {\
-   CPUIDRegs regs;                      \
-   __GET_CPUID2(_ax, _cx, &regs);       \
-   _ax = regs.eax;                      \
-   _bx = regs.ebx;                      \
-   _cx = regs.ecx;                      \
-   _dx = regs.edx;                      \
-}
-
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -24,6 +24,7 @@

 #ifndef _X86MSR_H_
 #define _X86MSR_H_
+#include <asm/msr-index.h>
 #define INCLUDE_ALLOW_USERLEVEL
 #define INCLUDE_ALLOW_VMX

@@ -120,6 +121,9 @@ MSRQuery;
 #define MSR_ARCH_CAPABILITIES_NOL1F_VMENTRY       (1ULL << 3)
 #define MSR_ARCH_CAPABILITIES_SSB_NO              (1ULL << 4)
 #define MSR_ARCH_CAPABILITIES_MDS_NO              (1ULL << 5)
+#define MSR_ARCH_CAPABILITIES_IF_PSCHANGE_MC_NO   (1ULL << 6)
+#define MSR_ARCH_CAPABILITIES_TSX_CTRL            (1ULL << 7)
+#define MSR_ARCH_CAPABILITIES_TAA_NO              (1ULL << 8)

 #define MSR_FLUSH_CMD                        0x10b
 #define MSR_FLUSH_CMD_FLUSH_L1D                   (1ULL << 0)
@@ -130,8 +134,13 @@ MSRQuery;

 #define MSR_PRED_CMD_IBPB                         (1UL << 0)

+#define MSR_TSX_CTRL                         0x122
+#define MSR_TSX_CTRL_RTM_DISABLE                  (1ULL << 0)
+#define MSR_TSX_CTRL_CPUID_CLEAR                  (1ULL << 1)

+#ifndef MSR_MISC_FEATURES_ENABLES
 #define MSR_MISC_FEATURES_ENABLES            0x140
+#endif

 /* Intel Core Architecture and later: use only architected counters. */
 #define IA32_MSR_PERF_CAPABILITIES                   0x345
@@ -148,14 +157,6 @@ MSRQuery;

 #define IA32_MSR_PEBS_ENABLE                      0x3f1

-typedef enum {
-   SL_PMC_FLAGS_NONE             = 0x00, /* No flags.                      */
-   SL_PMC_FLAGS_LBR_VA32         = 0x01, /* LBR format: 32-bit VA.         */
-   SL_PMC_FLAGS_LBR_LA64         = 0x02, /* LBR format: 64-bit LA.         */
-   SL_PMC_FLAGS_LBR_VA64         = 0x04, /* LBR format: 64-bit VA.         */
-   SL_PMC_FLAGS_LBR_PACKED_VA32  = 0x08, /* LBR format: 2x32-bit VAs.      */
-} StateLoggerPMCFlags;
-
 #define MSR_MTRR_BASE0        0x00000200
 #define MSR_MTRR_MASK0        0x00000201
 #define MSR_MTRR_BASE1        0x00000202
@@ -452,7 +453,6 @@ typedef enum {

 /* MSR_EFER bits. */
 #define MSR_EFER_SCE         0x0000000000000001ULL  /* Sys call ext'ns:  r/w */
-#define MSR_EFER_RAZ         0x00000000000000feULL  /* Read as zero          */
 #define MSR_EFER_LME         0x0000000000000100ULL  /* Long mode enable: r/w */
 #define MSR_EFER_LMA         0x0000000000000400ULL  /* Long mode active: r/o */
 #define MSR_EFER_NXE         0x0000000000000800ULL  /* No-exec enable:   r/w */
@@ -460,7 +460,11 @@ typedef enum {
 #define MSR_EFER_LMSLE       0x0000000000002000ULL  /* LM seg lim enable:r/w */
 #define MSR_EFER_FFXSR       0x0000000000004000ULL  /* Fast FXSAVE:      r/w */
 #define MSR_EFER_TCE         0x0000000000008000ULL  /* Trans. cache ext. r/w */
-#define MSR_EFER_MBZ         0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+/* Vendor specific EFER bits */
+#define MSR_EFER_INTEL_MBZ   0xffffffffffff02feULL  /* Must be zero (resrvd) */
+#define MSR_EFER_INTEL_RAZ   0x0000000000000000ULL  /* Read as zero          */
+#define MSR_EFER_AMD_MBZ     0xffffffffffff0200ULL  /* Must be zero (resrvd) */
+#define MSR_EFER_AMD_RAZ     0x00000000000000feULL  /* Read as zero          */

 #define MSR_AMD_PATCH_LOADER 0xc0010020

@@ -471,13 +475,17 @@ typedef enum {
 #define MSR_K7_HWCR_SSEDIS         0x00008000ULL // Disable SSE bit
 #define MSR_K7_HWCR_MONMWAITUSEREN 0x00000400ULL // Enable MONITOR/MWAIT CPL>0
 #define MSR_K7_HWCR_TLBFFDIS       0x00000040ULL // Disable TLB Flush Filter
+#ifndef MSR_K7_HWCR_SMMLOCK
 #define MSR_K7_HWCR_SMMLOCK        0x00000001ULL // Lock SMM environment
+#endif

 #ifndef MSR_K8_SYSCFG
 #define MSR_K8_SYSCFG        0xc0010010
 #endif
 #define MSR_K8_SYSCFG_MTRRTOM2EN         (1ULL<<21)
 #define MSR_K8_SYSCFG_TOM2FORCEMEMTYPEWB (1ULL<<22)
+#define MSR_K8_SYSCFG_SMEE               (1ULL<<23)
+
 #define MSR_K8_TOPMEM2       0xc001001d

 /* AMD "Greyhound" MSRs */
@@ -630,7 +638,9 @@ typedef enum {
 #define MSR_HYPERV_GUESTOSID_OS_WINNT_DERIVATIVE 4ULL

 /* MSR for forcing RTM abort to recover PMC3 (see PR 2333817) */
+#ifndef MSR_TSX_FORCE_ABORT
 #define MSR_TSX_FORCE_ABORT                      0x0000010f
+#endif
 #define MSR_TSX_FORCE_ABORT_RTM_BIT_INDEX        0

 /*
@@ -684,7 +694,11 @@ typedef unsigned char MTRRType;
 /*
 * MISC_FEATURES_ENABLES bits
 */
+#ifdef MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
+#else
 #define MSR_MISC_FEATURES_ENABLES_CPUID_FAULTING 1
+#endif



@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2005-2014,2017-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 2005-2014,2017-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -141,6 +141,7 @@
 #define SVM_VMCB_NPTCTL_NP_ENABLE          (1 << 0)
 #define SVM_VMCB_NPTCTL_SEV_ENABLE         (1 << 1)
 #define SVM_VMCB_NPTCTL_SEV_ES_ENABLE      (1 << 2)
+#define SVM_VMCB_NPTCTL_GMET_ENABLE        (1 << 3)

 /* VMCB.virtExt */
 #define SVM_VMCB_VIRTEXT_LBR_ENABLE        (1 << 0)
@@ -251,6 +252,7 @@
 #define SVM_EXITCODE_MMIO_READ           0x80000001   // SW only
 #define SVM_EXITCODE_MMIO_WRITE          0x80000002   // SW only
 #define SVM_EXITCODE_NMI_COMPLETE        0x80000003   // SW only
+#define SVM_EXITCODE_AP_JUMP_TABLE       0x80000005   // SW only
 #define SVM_EXITCODE_UNSUPPORTED         0x8000FFFF   // SW only
 #define SVM_EXITCODE_INVALID             (-1ULL)

@@ -313,6 +315,10 @@
 #define SVM_NPFEXIT_FINAL_ADDR     (1ULL << 32)    /* Rev. C */
 #define SVM_NPFEXIT_PTE_ACCESS     (1ULL << 33)    /* Rev. C */

+/* ExitInfo1 for AP jump table exits */
+#define SVM_APEXIT_SET            0x0
+#define SVM_APEXIT_GET            0x1
+
 /* Event Injection */
 #define SVM_INTINFO_VECTOR_MASK   0x000000ff
 #define SVM_INTINFO_TYPE_SHIFT    8
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2004-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2004-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -173,8 +173,11 @@ enum {
 /*
 * Sizes of referenced fields
 */
-#define VT_VMCS_IO_BITMAP_SIZE    (2 * PAGE_SIZE)
-#define VT_VMCS_MSR_BITMAP_SIZE   (1 * PAGE_SIZE)
+#define VT_VMCS_IO_BITMAP_PAGES   (2)
+#define VT_VMCS_IO_BITMAP_SIZE    PAGES_2_BYTES(VT_VMCS_IO_BITMAP_PAGES)
+#define VT_VMCS_MSR_BITMAP_PAGES  (1)
+#define VT_VMCS_MSR_BITMAP_SIZE   PAGES_2_BYTES(VT_VMCS_MSR_BITMAP_PAGES)
+

 /*
 * Execution control bit capabilities come in pairs: a "required" bit in
@@ -552,7 +555,7 @@ enum {
 #define VT_VMCS_PENDDBG_MBZ        0xfffeaff0

 /* Exception error must-be-zero bits for VMEntry */
-#define VT_XCP_ERR_MBZ             0xffff8000
+#define VT_XCP_ERR_MBZ             0xffff0000


 /* Exit reasons. */
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -68,8 +68,6 @@ static void LinuxDriverQueue(Device *device);
 static void LinuxDriverDequeue(Device *device);
 static Bool LinuxDriverCheckPadding(void);

-#define VMMON_UNKNOWN_SWAP_SIZE -1ULL
-
 struct VMXLinuxState linuxState;


@@ -95,21 +93,9 @@ long LinuxDriver_Ioctl(struct file *filp, u_int iocmd,
                       unsigned long ioarg);

 static int LinuxDriver_Close(struct inode *inode, struct file *filp);
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
-static vm_fault_t LinuxDriverFault(struct vm_fault *fault);
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-static int LinuxDriverFault(struct vm_fault *fault);
-#else
-static int LinuxDriverFault(struct vm_area_struct *vma, struct vm_fault *fault);
-#endif
-static int LinuxDriverMmap(struct file *filp, struct vm_area_struct *vma);

 static unsigned int LinuxDriverEstimateTSCkHz(void);

-static struct vm_operations_struct vmuser_mops = {
-        .fault  = LinuxDriverFault
-};
-
 static struct file_operations vmuser_fops;
 static struct timer_list tscTimer;
 static Atomic_uint32 tsckHz;
@@ -291,9 +277,11 @@ init_module(void)
      return -ENOEXEC;
   }

+   if (!Vmx86_CreateHVIOBitmap()) {
+      return -ENOMEM;
+   }
   linuxState.fastClockThread = NULL;
   linuxState.fastClockRate = 0;
-   linuxState.swapSize = VMMON_UNKNOWN_SWAP_SIZE;

   /*
    * Initialize the file_operations structure. Because this code is always
@@ -307,7 +295,6 @@ init_module(void)
   vmuser_fops.compat_ioctl = LinuxDriver_Ioctl;
   vmuser_fops.open = LinuxDriver_Open;
   vmuser_fops.release = LinuxDriver_Close;
-   vmuser_fops.mmap = LinuxDriverMmap;

 #ifdef VMX86_DEVEL
   devel_init_module();
@@ -328,7 +315,7 @@ init_module(void)
   if (retval) {
      Warning("Module %s: error registering with major=%d minor=%d\n",
              linuxState.deviceName, linuxState.major, linuxState.minor);
-
+      Vmx86_CleanupHVIOBitmap();
      return -ENOENT;
   }
   Log("Module %s: registered with major=%d minor=%d\n",
@@ -377,6 +364,7 @@ cleanup_module(void)

   del_timer_sync(&tscTimer);

+   Vmx86_CleanupHVIOBitmap();
   Task_Terminate();
   // Make sure fastClockThread is dead
   HostIF_FastClockLock(1);
@@ -414,7 +402,6 @@ LinuxDriver_Open(struct inode *inode, // IN
   }
   memset(device, 0, sizeof *device);

-   sema_init(&device->lock4Gb, 1);
   init_rwsem(&device->vmDriverRWSema);

   filp->private_data = device;
@@ -426,104 +413,6 @@ LinuxDriver_Open(struct inode *inode, // IN
 }


-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocPages --
- *
- *    Allocate physically contiguous block of memory with specified order.
- *    Pages in the allocated block are configured so that caller can pass
- *    independent pages to the VM.
- *
- * Results:
- *    Zero on success, non-zero (error code) on failure.
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverAllocPages(unsigned int order,   // IN
-                      struct page **pg,     // OUT
-                      PageCnt size)         // IN
-{
-   struct page* page = alloc_pages(GFP_HIGHUSER, order);
-   if (page) {
-      /*
-       * Grab an extra reference on all pages except first one - first
-       * one was already refcounted by alloc_pages.
-       *
-       * Under normal situation all pages except first one in the block
-       * have refcount zero.  As we pass these pages to the VM, we must
-       * bump their count, otherwise VM will release these pages every
-       * time they would be unmapped from user's process, causing crash.
-       *
-       * Note that this depends on Linux VM internals.  It works on all
-       * kernels we care about.
-       */
-      PageCnt i, orderToNumPages = 1 << order;
-      for (i = 0; i < orderToNumPages; i++) {
-         if (i) {
-            /*
-             * Debug kernels assert that page->_count is not zero when
-             * calling get_page. We use init_page_count as a temporary
-             * workaround. PR 894174
-             */
-#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 16)
-            ASSERT(page_count(page) == 0);
-            init_page_count(page);
-#else
-            get_page(page);
-#endif
-         }
-         if (i >= size) {
-            put_page(page);
-         } else {
-            void *addr = kmap(page);
-            memset(addr, 0, PAGE_SIZE);
-            kunmap(page);
-            *pg++ = page;
-         }
-         page++;
-      }
-      return 0;
-   }
-   return -ENOMEM;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverDestructor4Gb --
- *
- *    Deallocate all directly mappable memory.
- *
- * Results:
- *    None
- *
- * Side effects:
- *    None
- *
- *-----------------------------------------------------------------------------
- */
-
-static void
-LinuxDriverDestructor4Gb(Device *device) // IN
-{
-   PageCnt pg;
-   if (!device->size4Gb) {
-      return;
-   }
-   for (pg = 0; pg < device->size4Gb; pg++) {
-      put_page(device->pages4Gb[pg]);
-   }
-   device->size4Gb = 0;
-}
-
-
 /*
 *----------------------------------------------------------------------
 *
@@ -555,194 +444,12 @@ LinuxDriver_Close(struct inode *inode, // IN

   Vmx86_Close();

-   /*
-    * Destroy all low memory allocations.
-    * We are closing the struct file here, so clearly no other process
-    * uses it anymore, and we do not need to hold the semaphore.
-    */
-
-   LinuxDriverDestructor4Gb(device);
-
   kfree(device);
   filp->private_data = NULL;

   return 0;
 }

-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverFault --
- *
- *      Callback for returning allocated page for memory mapping
- *
- * Results:
- *    NoPage:
- *      Page or page address on success, NULL or 0 on failure.
- *    Fault:
- *      Error code; 0, minor page fault.
- *
- * Side effects:
- *      None.
- *
- *-----------------------------------------------------------------------------
- */
-
-static
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
-vm_fault_t
-#else
-int
-#endif
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-LinuxDriverFault(struct vm_fault *fault)     //IN/OUT
-#else
-LinuxDriverFault(struct vm_area_struct *vma, //IN
-                 struct vm_fault *fault)     //IN/OUT
-#endif
-{
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-   struct vm_area_struct *vma = fault->vma;
-#endif
-   Device *device = (Device *) vma->vm_file->private_data;
-   PageNum pg;
-   struct page* page;
-
-   pg = fault->pgoff;
-   pg = VMMON_MAP_OFFSET(pg);
-   if (pg >= device->size4Gb) {
-      return VM_FAULT_SIGBUS;
-   }
-   page = device->pages4Gb[pg];
-   get_page(page);
-   fault->page = page;
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverAllocContig --
- *
- *      Create mapping for contiguous memory areas.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int LinuxDriverAllocContig(Device *device,
-                                  struct vm_area_struct *vma,
-                                  unsigned long off,
-                                  PageCnt size)
-{
-   unsigned long vmaOrder = VMMON_MAP_ORDER(off);
-   PageCnt vmaAllocSize, i;
-
-   if (VMMON_MAP_RSVD(off)) {
-      /* Reserved bits set... */
-      return -EINVAL;
-   }
-   if (VMMON_MAP_OFFSET(off)) {
-      /* We do not need non-zero offsets... */
-      return -EINVAL;
-   }
-   if (size > VMMON_MAP_OFFSET_MASK + 1) {
-      /* Size is too big to fit to our window. */
-      return -ENOMEM;
-   }
-
-   /* 16 pages looks like a good limit... */
-   if (size > VMMON_MAX_LOWMEM_PAGES) {
-      return -ENOMEM;
-   }
-   /* Sorry. Only one mmap per one open. */
-   down(&device->lock4Gb);
-   if (device->size4Gb) {
-      up(&device->lock4Gb);
-      return -EINVAL;
-   }
-   vmaAllocSize = 1 << vmaOrder;
-   for (i = 0; i < size; i += vmaAllocSize) {
-      int err = LinuxDriverAllocPages(vmaOrder, device->pages4Gb + i, size - i);
-      if (err) {
-         while (i > 0) {
-            put_page(device->pages4Gb[--i]);
-         }
-         up(&device->lock4Gb);
-         return err;
-      }
-   }
-   device->size4Gb = size;
-   up(&device->lock4Gb);
-   vma->vm_ops = &vmuser_mops;
-   return 0;
-}
-
-
-/*
- *-----------------------------------------------------------------------------
- *
- * LinuxDriverMmap --
- *
- *      Create mapping for lowmem or locked memory.
- *
- * Results:
- *
- *      0       on success,
- *      -EINVAL on invalid arguments or
- *      -ENOMEM on out of memory
- *
- * Side effects:
- *      Pages for mapping are allocated.
- *
- *-----------------------------------------------------------------------------
- */
-
-static int
-LinuxDriverMmap(struct file *filp,
-                struct vm_area_struct *vma)
-{
-   Device *device = (Device *) filp->private_data;
-   PageCnt size;
-   int err;
-
-   /* Only shared mappings */
-   if (!(vma->vm_flags & VM_SHARED)) {
-      return -EINVAL;
-   }
-   if ((vma->vm_end | vma->vm_start) & (PAGE_SIZE - 1)) {
-      return -EINVAL;
-   }
-   size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
-   if (size < 1) {
-      return -EINVAL;
-   }
-   down_read(&device->vmDriverRWSema);
-   if (device->vm) {
-      err = -EINVAL;
-   } else {
-      err = LinuxDriverAllocContig(device, vma, vma->vm_pgoff, size);
-   }
-   up_read(&device->vmDriverRWSema);
-   if (err) {
-      return err;
-   }
-   /* Clear VM_IO, otherwise SuSE's kernels refuse to do get_user_pages */
-   vma->vm_flags &= ~VM_IO;
-   return 0;
-}
-
-
 typedef Bool (*SyncFunc)(void *data, unsigned cpu);

 typedef struct {
@@ -957,7 +664,7 @@ LinuxDriverReadTSC(void *data,   // OUT: TSC values
 *-----------------------------------------------------------------------------
 */

-__attribute__((always_inline)) static Bool
+__always_inline static Bool
 LinuxDriverSyncReadTSCs(uint64 *delta) // OUT: TSC max - TSC min
 {
   TSCDelta tscDelta;
@@ -1328,25 +1035,12 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
   }

   case IOCTL_VMX86_APIC_INIT: {
-      VMAPICInfo info;
-      Bool setVMPtr;
-      Bool probe;
-
-      retval = HostIF_CopyFromUser(&info, ioarg, sizeof info);
-      if (retval != 0) {
-         break;
-      }
-      setVMPtr = ((info.flags & APIC_FLAG_DISABLE_NMI) != 0);
-      probe = ((info.flags & APIC_FLAG_PROBE) != 0);
-
      /*
-       * Kernel uses NMIs for deadlock detection - set APIC VMptr so that
-       * NMIs get disabled in the monitor.
+       * Kernel uses NMIs for deadlock detection - so we always have to find
+       * APIC so that NMIs get disabled in the monitor.
       */
-
-      setVMPtr = TRUE;
-
-      retval = HostIF_APICInit(vm, setVMPtr, probe) ? 0 : -ENODEV;
+      HostIF_APICInit(vm);
+      retval = 0;
      break;
   }

@@ -1533,7 +1227,7 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      }

   case IOCTL_VMX86_WRITE_PAGE: {
-#if VMX86_DEVEL
+#ifdef VMX86_DEVEL
         VMMReadWritePage req;

         retval = HostIF_CopyFromUser(&req, ioarg, sizeof req);
@@ -1599,17 +1293,6 @@ LinuxDriver_Ioctl(struct file *filp,    // IN:
      break;
   }

-   case IOCTL_VMX86_SET_HOST_SWAP_SIZE: {
-      uint64 swapSize;
-      retval = HostIF_CopyFromUser(&swapSize, ioarg, sizeof swapSize);
-      if (retval != 0) {
-         Warning("Could not copy swap size from user, status %ld\n", retval);
-         break;
-      }
-      linuxState.swapSize = swapSize;
-      break;
-   }
-
   case IOCTL_VMX86_GET_UNAVAIL_PERF_CTRS: {
      uint64 ctrs = Vmx86_GetUnavailablePerfCtrs();
      retval = HostIF_CopyToUser(ioarg, &ctrs, sizeof ctrs);
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998,2018-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2011,2014-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -37,9 +37,6 @@
 /* Per-instance driver state */
 struct VMDriver;

-/* 16 pages (64KB) looks as a good limit for one allocation */
-#define VMMON_MAX_LOWMEM_PAGES  16
-
 typedef struct Device {
   struct Device   *next;
   struct VMDriver *vm;
@@ -48,19 +45,10 @@ typedef struct Device {
    * avoid racing between various ioctls, and the creation
    * and removal of the VM in question. The lock is read-acquired
    * by ioctls that reference the VMDriver, and write-acquired by
-    * ioctls or device callbacks that allocate or destory the
+    * ioctls or device callbacks that allocate or destroy the
    * VMDriver.
    */
   struct rw_semaphore vmDriverRWSema;
-   /*
-    * The semaphore protect accesses to size4Gb and pages4Gb
-    * in mmap(). mmap() may happen only once, and all other
-    * accesses except cleanup are read-only, and may happen
-    * only after successful mmap.
-    */
-   struct semaphore lock4Gb;
-   PageCnt size4Gb;
-   struct page *pages4Gb[VMMON_MAX_LOWMEM_PAGES];
 } Device;


@@ -81,11 +69,8 @@ typedef struct VMXLinuxState {

   struct task_struct *fastClockThread;
   unsigned fastClockRate;
-   uint64 swapSize;
 } VMXLinuxState;

 extern VMXLinuxState linuxState;
-extern uint8 monitorIPIVector;
-extern uint8 hvIPIVector;

 #endif
@@ -71,9 +71,11 @@
 #include "apic.h"
 #include "memDefaults.h"
 #include "vcpuid.h"
+#include "x86svm.h"

 #include "pgtbl.h"
 #include "versioned_atomic.h"
+#include "compat_poll.h"

 #if !defined(CONFIG_HIGH_RES_TIMERS)
 #error CONFIG_HIGH_RES_TIMERS required for acceptable performance
@@ -179,8 +181,22 @@ static struct {

 static void UnlockEntry(void *clientData, MemTrackEntry *entryPtr);

-uint8 monitorIPIVector;
-uint8 hvIPIVector;
+/*
+ * SPURIOUS_APIC_VECTOR is not defined on kernels built without APIC
+ * support.
+ */
+#ifndef SPURIOUS_APIC_VECTOR
+#define SPURIOUS_APIC_VECTOR 0
+#endif
+static const uint8 monitorIPIVector = SPURIOUS_APIC_VECTOR;
+/*
+ * POSTED_INTR_VECTOR is defined on kernels after 3.10 when built with
+ * KVM support.
+ */
+#ifndef POSTED_INTR_VECTOR
+#define POSTED_INTR_VECTOR 0
+#endif
+static const uint8 hvIPIVector = POSTED_INTR_VECTOR;

 /*
 *-----------------------------------------------------------------------------
@@ -923,7 +939,6 @@ HostIF_AllocLowPage(VMDriver *vm) //  IN: VM instance pointer

      mpn = (MPN)page_to_pfn(pg);

-      HostIF_VMLock(vm, 40);
      vmh = vm->vmhost;
      if (vmh->AWEPages != NULL) {
         if (PhysTrack_Test(vmh->AWEPages, mpn)) {
@@ -934,7 +949,6 @@ HostIF_AllocLowPage(VMDriver *vm) //  IN: VM instance pointer
         __free_page(pg);
         mpn = INVALID_MPN;
      }
-      HostIF_VMUnlock(vm, 40);
   }
   return mpn;
 }
@@ -962,8 +976,7 @@ HostIFFreeVMHost(VMHost *vmhost) // IN:
 {
   ASSERT(vmhost->lockedPages == NULL &&
          vmhost->AWEPages    == NULL &&
-          vmhost->crosspagePagesCount == 0 &&
-          !vmhost->hostAPICIsMapped);
+          vmhost->crosspagePagesCount == 0);
   Vmx86_Free(vmhost->crosspagePages);
   Vmx86_Free(vmhost->vcpuSemaTask);
   vmhost->crosspagePages = NULL;
@@ -1006,6 +1019,108 @@ HostIFAllocVMHost(uint32 numVCPUs) // IN:
 }


+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_FreeContigPages --
+ *
+ *      Free an allocated memory block returned from an earlier call to
+ *      HostIF_AllocContigPages.
+ *
+ * Results:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+HostIF_FreeContigPages(VMDriver *vm,             // IN: Driver state
+                       HostIFContigMemMap *map)  // IN: Object to free
+{
+   unsigned order = 0;
+
+   /*
+    * The only users of this API are the I/O and MSR bitmap allocations.
+    * Of those, the largest is the SVM I/O bitmap at 3 pages.
+    */
+   ASSERT(map->pages <= SVM_VMCB_IO_BITMAP_PAGES);
+   if (vm != NULL) {
+      PageCnt i;
+
+      for (i = 0; i < map->pages; i++) {
+         PhysTrack_Remove(vm->vmhost->AWEPages, map->mpn + i);
+      }
+   }
+   while (1u << order < map->pages) {
+      order++;
+   }
+   free_pages(PtrToVA64(map->addr), order);
+   HostIF_FreeKernelMem(map);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * HostIF_AllocContigPages --
+ *
+ *      Allocate a range of physically contiguous memory.  Also add the pages
+ *      to the VmAnon PhysTracker if 'vm' is non-NULL.  This allows the pages
+ *      to be written to a coredump when needed.
+ *
+ * Results:
+ *      A HostIFContigMemMap object containing the start MPN of the allocated
+ *      range, or NULL on failure.
+ *
+ *----------------------------------------------------------------------
+ */
+
+HostIFContigMemMap *
+HostIF_AllocContigPages(VMDriver *vm,      // IN/OUT: driver state
+                        PageCnt numPages)  // IN: Number of pages
+{
+   HostIFContigMemMap *map;
+   unsigned order = 0;
+   /*
+    * The only users of this API are the I/O and MSR bitmap allocations.
+    * Of those, the largest is the SVM I/O bitmap at 3 pages.
+    */
+   if (numPages == 0 || numPages > SVM_VMCB_IO_BITMAP_PAGES) {
+      return NULL;
+   }
+   map = HostIF_AllocKernelMem(sizeof *map, FALSE);
+   if (map == NULL) {
+      return NULL;
+   }
+   memset(map, 0, sizeof *map);
+   while (1u << order < numPages) {
+      order++;
+   }
+   map->addr = VA64ToPtr(__get_free_pages(GFP_HIGHUSER, order));
+   if (map->addr == NULL) {
+      HostIF_FreeKernelMem(map);
+      return NULL;
+   }
+   map->pages = numPages;
+   map->mpn = MA_2_MPN(virt_to_phys(map->addr));
+
+   /*
+    * Add the allocated pages to the VMM anonymous memory phystracker.
+    * This is to ensure the pages will be written out during a coredump.
+    */
+   if (vm != NULL) {
+      PageCnt i;
+
+      ASSERT(HostIF_VMLockIsHeld(vm));
+      for (i = 0; i < map->pages; i++) {
+         PhysTrack_Add(vm->vmhost->AWEPages, map->mpn + i);
+      }
+   }
+
+   return map;
+}
+
+
 /*
 *----------------------------------------------------------------------
 *
@@ -1419,12 +1534,6 @@ HostIF_FreeAllResources(VMDriver *vm) // IN
         UnmapCrossPage(p, vm->crosspage[cnt]);
      }
      vm->vmhost->crosspagePagesCount = 0;
-      if (vm->vmhost->hostAPICIsMapped) {
-         ASSERT(vm->hostAPIC.base != NULL);
-         iounmap((void*)vm->hostAPIC.base);
-         vm->hostAPIC.base = NULL;
-         vm->vmhost->hostAPICIsMapped = FALSE;
-      }
      HostIFFreeVMHost(vm->vmhost);
      vm->vmhost = NULL;
   }
@@ -1623,7 +1732,6 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
                                    : BYTES_2_PAGES(vm->memInfo.hugePageBytes);
   PageCnt lockedPages = hugePages + reservedPages;
   PageCnt anonPages;
-   PageCnt swapPages = BYTES_2_PAGES(linuxState.swapSize);

   /* global_page_state is global_zone_page_state in 4.14. */
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
@@ -1650,9 +1758,6 @@ HostIF_EstimateLockedPageLimit(const VMDriver* vm,                // IN
   anonPages = global_page_state(NR_ANON_PAGES);
 #endif

-   if (anonPages > swapPages) {
-      lockedPages += anonPages - swapPages;
-   }
   forHost = lockedPages + LOCKED_PAGE_SLACK;
   if (forHost > totalPhysicalPages) {
      forHost = totalPhysicalPages;
@@ -2297,85 +2402,6 @@ isVAReadable(VA r)  // IN:

   return ret == 0;
 }
-
-
-/*
- *----------------------------------------------------------------------
- *
- * SetVMAPICAddr --
- *
- *      Maps the host cpu's APIC.  The virtual address is stashed in
- *      the VMDriver structure.
- *
- * Results:
- *      None.
- *
- * Side effects:
- *      The VMDriver structure is updated.
- *
- *----------------------------------------------------------------------
- */
-
-static void
-SetVMAPICAddr(VMDriver *vm, // IN/OUT: driver state
-              MA ma)        // IN: host APIC's ma
-{
-   volatile void *hostapic;
-
-   ASSERT_ON_COMPILE(APICR_SIZE <= PAGE_SIZE);
-   hostapic = (volatile void *) ioremap(ma, PAGE_SIZE);
-   if (hostapic) {
-      if ((APIC_VERSIONREG(hostapic) & 0xF0) == 0x10) {
-         vm->hostAPIC.base = (volatile uint32 (*)[4]) hostapic;
-         ASSERT(vm->vmhost != NULL);
-         vm->vmhost->hostAPICIsMapped = TRUE;
-      } else {
-         iounmap((void*)hostapic);
-      }
-   }
-}
-
-
-/*
- *----------------------------------------------------------------------
- *
- * ProbeAPIC --
- *
- *      Attempts to map the host APIC.
- *
- *      Most versions of Linux already provide access to a mapped
- *      APIC.  This function is just a backup.
- *
- *      Caveat: We assume that the APIC physical address is the same
- *      on all host cpus.
- *
- * Results:
- *      TRUE if APIC was found, FALSE if not.
- *
- * Side effects:
- *      May map the APIC.
- *
- *----------------------------------------------------------------------
- */
-
-static Bool
-ProbeAPIC(VMDriver *vm,   // IN/OUT: driver state
-          Bool setVMPtr)  // IN: set a pointer to the APIC's virtual address
-{
-   MA ma = APIC_GetMA();
-
-   if (ma == (MA)-1) {
-      return FALSE;
-   }
-
-   if (setVMPtr) {
-      SetVMAPICAddr(vm, ma);
-   } else {
-      vm->hostAPIC.base = NULL;
-   }
-
-   return TRUE;
-}
 #endif


@@ -2384,75 +2410,50 @@ ProbeAPIC(VMDriver *vm,   // IN/OUT: driver state
 *
 * HostIF_APICInit --
 *
- *      Initialize APIC behavior.
- *      Attempts to map the host APIC into vm->hostAPIC.
+ *      Check if APIC is present, and is regular or x2 APIC.
 *
- *      We don't attempt to refresh the mapping after a host cpu
- *      migration.  Fortunately, hosts tend to use the same address
- *      for all APICs.
- *
- *      Most versions of Linux already provide a mapped APIC.  We
- *      have backup code to read APIC_BASE and map it, if needed.
+ *      We check for X2 APIC by checking X2APIC enable bit, and
+ *      for regular APIC by checking if APIC registers are mapped
+ *      as readable page (vs. not present for no APIC).
 *
 * Results:
- *      TRUE
+ *      None.
 *
 * Side effects:
- *      May map the host APIC.
+ *      None.
 *
 *----------------------------------------------------------------------
 */
-Bool
-HostIF_APICInit(VMDriver *vm,   // IN:
-                Bool setVMPtr,  // IN:
-                Bool probe)     // IN: force probing
+
+void
+HostIF_APICInit(VMDriver *vm)   // IN:
 {
 #if defined(CONFIG_SMP)         || defined(CONFIG_X86_UP_IOAPIC) || \
    defined(CONFIG_X86_UP_APIC) || defined(CONFIG_X86_LOCAL_APIC)
-   static Bool apicIPILogged = FALSE;
-   VA kAddr;
+   if (vm->hostAPIC.base == NULL && !vm->hostAPIC.isX2) {
+      static Bool apicIPILogged = FALSE;

-   monitorIPIVector = SPURIOUS_APIC_VECTOR;
-#if defined(POSTED_INTR_VECTOR)
-   hvIPIVector      = POSTED_INTR_VECTOR;
-#else
-   hvIPIVector      = 0;
-#endif
-
-
-   if (!apicIPILogged) {
-      Log("Monitor IPI vector: %x\n", monitorIPIVector);
-      Log("HV      IPI vector: %x\n", hvIPIVector);
-      apicIPILogged = TRUE;
-   }
-
-   if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
-      if (setVMPtr) {
-         vm->hostAPIC.base = NULL;
-         vm->vmhost->hostAPICIsMapped = FALSE;
-         vm->hostAPIC.isX2 = TRUE;
+      if (!apicIPILogged) {
+         Log("Monitor IPI vector: %x\n", monitorIPIVector);
+         Log("HV      IPI vector: %x\n", hvIPIVector);
+         apicIPILogged = TRUE;
      }
-      return TRUE;
-   }

-   if (probe && ProbeAPIC(vm, setVMPtr)) {
-      return TRUE;
-   }
+      if ((X86MSR_GetMSR(MSR_APIC_BASE) & APIC_MSR_X2APIC_ENABLED) != 0) {
+         vm->hostAPIC.isX2 = TRUE;
+      } else {
+         VA kAddr;

-   /*
-    * Normal case: use Linux's pre-mapped APIC.
-    */
-   kAddr = __fix_to_virt(FIX_APIC_BASE);
-   if (!isVAReadable(kAddr)) {
-      return TRUE;
-   }
-   if (setVMPtr) {
-      vm->hostAPIC.base = (void *)kAddr;
-   } else {
-      vm->hostAPIC.base = NULL;
+         /*
+          * Old APIC: use Linux's pre-mapped APIC.
+          */
+         kAddr = __fix_to_virt(FIX_APIC_BASE);
+         if (isVAReadable(kAddr)) {
+            vm->hostAPIC.base = (void *)kAddr;
+         }
+      }
   }
 #endif
-   return TRUE;
 }


@@ -2575,7 +2576,7 @@ HostIF_SemaphoreWait(VMDriver *vm,   // IN:

      poll_initwait(&table);
      current->state = TASK_INTERRUPTIBLE;
-      mask = file->f_op->poll(file, &table.pt);
+      mask = compat_vfs_poll(file, &table.pt);
      if (!(mask & (POLLIN | POLLERR | POLLHUP))) {
         vm->vmhost->vcpuSemaTask[vcpuid] = current;
         schedule_timeout(timeoutms * HZ / 1000);  // convert to Hz
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2002-2018 VMware, Inc. All rights reserved.
+ * Copyright (C) 2002-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -94,8 +94,6 @@ typedef struct VMHost {
    * as pages for "AWE" guest memory.
    */
   struct PhysTracker  *AWEPages;
-   /* Is VMDriver.hostAPIC mapped or is from __fix_to_virt(FIX_APIC_BASE)? */
-   Bool                 hostAPICIsMapped;
 } VMHost;

 #endif
@@ -316,13 +316,34 @@ skipTaskSwitch:;
         crosspage->args[1] = vm->numPTPPages;
      } break;

+      case MODULECALL_GET_HV_IO_BITMAP: {
+         crosspage->args[0] = hvIOBitmap == NULL ? INVALID_MPN :
+                                                   hvIOBitmap->mpn;
+      } break;
+
+      case MODULECALL_ALLOC_CONTIG_PAGES: {
+         PageCnt pages = crosspage->args[0];
+         HostIFContigMemMap *alloc;
+
+         HostIF_VMLock(vm, 47);
+         alloc = HostIF_AllocContigPages(vm, pages);
+         if (alloc != NULL) {
+            alloc->next = vm->contigMappings;
+            vm->contigMappings = alloc;
+            crosspage->args[1] = alloc->mpn;
+         } else {
+            crosspage->args[1] = INVALID_MPN;
+         }
+         HostIF_VMUnlock(vm, 47);
+      } break;
+
      default:
         Warning("ModuleCall %d not supported\n", crosspage->moduleCallType);
      }
      ASSERT(retval == (uint64)((uint32)retval));
      crosspage->retval = (uint32)retval;
 #if defined(__linux__)
-      cond_resched(); // Other kernels are preemptable
+      cond_resched(); // Other kernels are preemptible
 #endif
   }
 bailOut:
@@ -136,9 +136,6 @@ ifdef VMX86_DEBUG
 CC_OPTS += -DVMX86_DEBUG
 endif

-# Add Spectre options when available
-CC_OPTS += $(call vm_check_gcc,-mindirect-branch=thunk -mindirect-branch-register,)
-
 include $(SRCROOT)/Makefile.kernel

 else
@@ -32,6 +32,6 @@ $(DRIVER)-y := driver.o hub.o userif.o netif.o bridge.o procfs.o smac_compat.o \
 ####

 clean:
-	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko \
+	rm -rf $(DRIVER).o $(DRIVER).mod.o $(DRIVER).mod.c $(DRIVER).ko .cache.mk \
 	       .tmp_versions Module.symvers Modules.symvers Module.markers \
 	       modules.order $($(DRIVER)-y) .*.cmd .*.o.flags
@@ -66,7 +66,7 @@
 #endif

 #if LOGLEVEL >= 4
-static struct timeval vnetTime;
+static u64 vnetTime;
 #endif

 typedef struct VNetBridge VNetBridge;
@@ -693,7 +693,7 @@ VNetBridgeReceiveFromVNet(VNetJack        *this, // IN: jack

 	 netif_rx_ni(clone);
 #	 if LOGLEVEL >= 4
-	 do_gettimeofday(&vnetTime);
+	 vnetTime = ktime_get_ns();
 #	 endif
      }
   }
@@ -1490,12 +1490,11 @@ VNetBridgeReceiveFromDev(struct sk_buff *skb,         // IN: packet to receive

 #  if LOGLEVEL >= 4
   {
-      struct timeval now;
-      do_gettimeofday(&now);
+      u64 now;
+
+      now = ktime_get_ns();
      LOG(3, (KERN_DEBUG "bridge-%s: time %d\n",
-	      bridge->name,
-	      (int)((now.tv_sec * 1000000 + now.tv_usec)
-                    - (vnetTime.tv_sec * 1000000 + vnetTime.tv_usec))));
+	      bridge->name, (int)((now - vnetTime) / NSEC_PER_USEC)));
   }
 #  endif

@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2016,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -200,7 +200,7 @@ Net_GetNumAdapters(Net_AdapterCount *counts)
      } else if (Str_Strcasecmp(adapterStr, "e1000e") == 0) {
         counts->e1000e++;
      } else {
-         LOG_ONCE(("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr));
+         LOG_ONCE("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr);
      }
      free(adapterStr);
   }
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017,2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -207,7 +207,7 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      count = (count + 1) & 1023;                                       \
   } while (0)

-#define LOG_ONCE(_s) DO_ONCE(Log _s)
+#define LOG_ONCE(...) DO_ONCE(Log(__VA_ARGS__))


 /*
@@ -314,6 +314,17 @@ void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
      assertions                     \
   }

+/*
+ * Avoid generating extra code due to asserts which are required by
+ * Clang static analyzer, e.g. right before a statement would fail, using
+ * the __clang_analyzer__ macro defined only when clang SA is parsing files.
+ */
+#ifdef __clang_analyzer__
+#define ANALYZER_ASSERT(cond) ASSERT(cond)
+#else
+#define ANALYZER_ASSERT(cond) ((void)0)
+#endif
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -122,27 +122,27 @@ extern "C" {


 /* Basic atomic types: 8, 16, 32, 64 and 128 bits */
-typedef struct Atomic_uint8 {
+typedef ALIGNED(1) struct Atomic_uint8 {
   volatile uint8 value;
-} Atomic_uint8 ALIGNED(1);
+} Atomic_uint8;

-typedef struct Atomic_uint16 {
+typedef ALIGNED(2) struct Atomic_uint16 {
   volatile uint16 value;
-} Atomic_uint16 ALIGNED(2);
+} Atomic_uint16;

-typedef struct Atomic_uint32 {
+typedef ALIGNED(4) struct Atomic_uint32 {
   volatile uint32 value;
-} Atomic_uint32 ALIGNED(4);
+} Atomic_uint32;

-typedef struct Atomic_uint64 {
+typedef ALIGNED(8) struct Atomic_uint64 {
   volatile uint64 value;
-} Atomic_uint64 ALIGNED(8);
+} Atomic_uint64;

 #if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
-typedef struct Atomic_uint128 {
+typedef ALIGNED(16) struct Atomic_uint128 {
   volatile uint128 value;
-} Atomic_uint128 ALIGNED(16);
+} Atomic_uint128;
 #endif

 /*
@@ -3584,7 +3584,7 @@ Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
 *
 * Atomic_ReadInc16 --
 *
- *      Atomically increments a 64-bit integer
+ *      Atomically increments a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
@@ -3607,7 +3607,7 @@ Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
 *
 * Atomic_ReadDec16 --
 *
- *      Atomically decrements a 64-bit integer
+ *      Atomically decrements a 16-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
@@ -48,7 +48,6 @@
 #define MUL64_NO_ASM 1
 #include "mul64.h"
 #elif defined VM_ARM_64
-#include "arm64_basic_defs.h"
 #include "vm_basic_asm_arm64.h"
 #else
 #define MUL64_NO_ASM 1
@@ -1180,6 +1179,9 @@ RoundUpPow2_32(uint32 value)
 static INLINE unsigned
 PopCount32(uint32 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+   return __builtin_popcount(value);
+#else
   /*
    * Attribution:
    *     This algorithm was copied from:
@@ -1223,6 +1225,7 @@ PopCount32(uint32 value)
   value += (value >> 8);
   value += (value >> 16);
   return value & 0x0000003f;
+#endif
 }


@@ -1245,6 +1248,13 @@ PopCount32(uint32 value)
 static INLINE unsigned
 PopCount64(uint64 value)
 {
+#if defined(__GNUC__) && !defined(FEWER_BUILTINS) && defined(__POPCNT__)
+#if defined(VM_X86_64)
+   return __builtin_popcountll(value);
+#else
+   return PopCount32(value) + PopCount32(value >> 32);
+#endif
+#else
   value -= (value >> 1) & 0x5555555555555555ULL;
   value = ((value >> 2) & 0x3333333333333333ULL) +
           (value & 0x3333333333333333ULL);
@@ -1253,6 +1263,7 @@ PopCount64(uint64 value)
   value += value >> 16;
   value += value >> 32;
   return (unsigned) (value & 0xff);
+#endif
 }


@@ -438,10 +438,6 @@ void *_ReturnAddress(void);

 #define strtok_r  strtok_s

-#if (_MSC_VER < 1500)
-#define	vsnprintf _vsnprintf
-#endif
-
 typedef int uid_t;
 typedef int gid_t;

@@ -465,20 +461,9 @@ typedef int pid_t;
 #define       W_OK          2
 #define       R_OK          4

-#endif // }
+#endif // } _WIN32

-/*
- * Macro for username comparison.
- */
-
-#ifdef _WIN32 // {
-#define USERCMP(x,y)  Str_Strcasecmp(x,y)
-#else
-#define USERCMP(x,y)  strcmp(x,y)
-#endif // }
-
-
-#endif // }
+#endif // } USERLEVEL

 #ifndef va_copy

@@ -657,8 +642,10 @@ typedef int pid_t;
 #endif

 #ifdef VMM
+#define vmx86_vmm 1
 #define VMM_ONLY(x) x
 #else
+#define vmx86_vmm 0
 #define VMM_ONLY(x)
 #endif

@@ -699,7 +686,6 @@ typedef int pid_t;
 * display/printer drivers only.
 */
 #ifdef _WIN32
-#ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
 #ifdef _WIN64
 #define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
@@ -711,7 +697,6 @@ typedef int pid_t;
 #define WinDrvPrint(arg, ...)
 #define WinDrvEngPrint(arg, ...)
 #endif
-#endif
 #endif // _WIN32

 /*
@@ -801,4 +786,30 @@ typedef int pid_t;
 #define END_PTR_TO_ALIGNED_VAR \
   } while (0)

+
+/*
+ * -Wswitch means that when you pass switch an enum that it's looking for
+ * all values from that enum, and only that enum, to be accounted for.
+ * "default:;" is fine for catching values you don't care about. But today
+ * we have a bunch of code that uses internal and external enum values, or
+ * in other words combines two enums into a single variable. This cast is
+ * the workaround, but we really need to fix this mess.
+ */
+#define UNCHECKED_SWITCH__FIXME(x) switch ((uint64)(x))
+
+
+/*
+ * When clang static analyzer parses source files, it implicitly defines
+ * __clang_analyzer__ macro. We use this to define our custom macro to stop
+ * its execution for the current path of analysis by calling a function that
+ * doesn't return, making it think that it hit a failed assertion.
+ *
+ * DO NOT use to silence the analyzer! See PR2447238.
+ */
+#ifdef __clang_analyzer__
+#define VMW_CLANG_ANALYZER_NORETURN() Panic("Disable Clang static analyzer")
+#else
+#define VMW_CLANG_ANALYZER_NORETURN() ((void)0)
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2019 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2020 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
@@ -101,6 +101,7 @@

 #pragma warning (3 :4505) // unreferenced local function
 #pragma warning (disable :4018) // signed/unsigned mismatch
+#pragma warning (suppress:4619) // suppress warning next line (C4761 was removed in vs2019u4)
 #pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
 #pragma warning (disable :4305) // truncation from 'const int' to 'short'
 #pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
@@ -698,11 +699,11 @@ typedef void * UserVA;
 #endif

 /*
- * Similarly, we require a compiler that is at least vc80 (vs2005).
+ * Similarly, we require a compiler that is at least vc90 (vs2008).
 * Enforce this here.
 */
-#if defined _MSC_VER && _MSC_VER < 1400
-#error "cl.exe version is too old, need vc80 or better"
+#if defined _MSC_VER && _MSC_VER < 1500
+#error "cl.exe version is too old, need vc90 or better"
 #endif


@@ -864,12 +865,10 @@ typedef void * UserVA;
 #endif

 #ifndef UNUSED_TYPE
-// XXX _Pragma would better but doesn't always work right now.
 #  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
 #endif

 #ifndef UNUSED_VARIABLE
-// XXX is there a better way?
 #  define UNUSED_VARIABLE(_var) (void)_var
 #endif

@@ -884,29 +883,21 @@ typedef void * UserVA;

 /*
 * ALIGNED specifies minimum alignment in "n" bytes.
+ *
+ * NOTE: __declspec(align) has limited syntax; it must essentially be
+ *       an integer literal.  Expressions, such as sizeof(), do not
+ *       work.
 */

 #ifdef __GNUC__
 #define ALIGNED(n) __attribute__((__aligned__(n)))
+#elif defined(_MSC_VER)
+#define ALIGNED(n) __declspec(align(n))
 #else
 #define ALIGNED(n)
 #endif


-/*
- * Encapsulate the syntactic differences between gcc and msvc alignment control.
- * BOUNDARY must match in the prefix and suffix.
- */
-
-#ifdef _WIN32
-#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
-#define ALIGN_SUFFIX(BOUNDRY)
-#else
-#define ALIGN_PREFIX(BOUNDRY)
-#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
-#endif
-
-
 /*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
@@ -35,6 +35,8 @@
 #endif
 #endif

+#include <linux/pci_ids.h>
+
 /* LSILogic 53C1030 Parallel SCSI controller
 * LSILogic SAS1068 SAS controller
 */
@@ -53,7 +55,10 @@
 *    VMware HD Audio codec
 *    VMware HD Audio controller
 */
+#ifndef PCI_VENDOR_ID_VMWARE
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
+#endif
+
 #define PCI_DEVICE_ID_VMWARE_SVGA3              0x0406
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
@@ -71,7 +76,11 @@
 #define PCI_DEVICE_ID_VMWARE_1394               0x0780
 #define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
 #define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
+
+#ifndef PCI_DEVICE_ID_VMWARE_VMXNET3
 #define PCI_DEVICE_ID_VMWARE_VMXNET3            0x07B0
+#endif
+
 #define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
 #define PCI_DEVICE_ID_VMWARE_82574              0x07D0
 #define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
Author	SHA1	Message	Date
Michal Kubecek	e02b071950	vmmon: use standard definition of MSR_TSX_FORCE_ABORT if available The MSR_TSX_FORCE_ABORT macro is defined in mainline since commit 52f64909409c ("x86: Add TSX Force Abort CPUID/MSR") in v5.1-rc1.	2020-05-30 02:48:55 +02:00
Michal Kubecek	f98d4d9c34	vmnet: convert VNetBridge debugging code to ktime_get_ns() The do_gettimeofday() helper was removed by commit e4b92b108c6c ("timekeeping: remove obsolete time accessors") in v5.0-rc1 and since commit c766d1472c70 ("y2038: hide timeval/timespec/itimerval/itimerspec types") in v5.6-rc3, struct timeval should no longer be used in kernel code. Convert the do_gettimeofday() relics in VNetBridge (which are only compiled with LOGLEVEL >= 4) completely to ktime based interface.	2020-05-30 02:48:43 +02:00
Michal Kubecek	c21a12d6ff	vmmon: check presence of file_operations::poll() Commit 11c5ad0ec441 ("eventpoll: switch to ->poll_mask") in v4.18-rc1, made eventpoll switch from ->poll() to ->poll_mask(), resulting in a null pointer dereference as vmmon calls the callback directly without checking its availability. Rather than calling the callback directly (which would result in null pointer dereference), use vfs_poll() wrapper. As this wrapper is only available since 4.18-rc1 cycle as well, provide a copy to use when building against older kernels. Note: even if the commit revealing the problem was reverted in 4.18-rc3 (and is unlikely to go back in the same form) so that vmmon no longer crashes with 4.18 kernel, calling the callback without a check is a bad practice so let's keep the patch in place.	2020-05-30 02:48:11 +02:00
Michal Kubecek	2bb5a1991c	vmmon: fix indirect call with retpoline build Build against kernel with retpoline support issues warning objtool: Task_Switch()+0x425: indirect call found in RETPOLINE build This is because an indirect call in TaskSwitchToMonitor() is encoded using inline assembler so that it bypasses retpoline generation. For this purpose, macro CALL_NOSPEC exists since v4.15-rc8 (and has been backported into some distribution kernels with the rest of retpoline support). Use the macro if available and fallback to the original code if not.	2020-05-30 02:47:57 +02:00
Michal Kubecek	e384440ab1	vmmon: fix always_inline attribute usage Function declared with __attribute__((always_inline)) should also be declared as inline, otherwise gcc issues a warning "always_inline function might not be inlinable". It's just cosmetic but getting rid of known harmless warnings makes it easier to spot actual problems. Use the __always_inline macro for LinuxDriverSyncReadTSCs() as this is how always inline functions should be declared in kernel code.	2020-05-30 02:47:50 +02:00
Michal Kubecek	05049f29d7	vmmon: use standard definition of MSR_K7_HWCR_SMMLOCK if available The MSR_K7_HWCR_SMMLOCK macro is defined in mainline since commit 18c71ce9c882 ("x86/CPU/AMD: Add the Secure Encrypted Virtualization CPU feature") in v4.16-rc1.	2020-05-30 02:47:42 +02:00
Theo Bertacchini	240df2c940	modules: remove .cache.mk on make clean Remove .cache.mk inside vmmon-only and vmnet-only when executing "make clean". This file can cause issues when upgrading gcc as make will still look for includes inside older gcc includes directory. File .cache.mk with cache of generated variables was created by build since kernel v4.15-rc1, commit 3298b690b21c ("kbuild: Add a cache for generated variables") until the feature was removed in v4.18-rc1, commit e08d6de4e532 ("kbuild: remove kbuild cache").	2020-05-30 02:46:10 +02:00
Michal Kubecek	5d69da38fe	vmmon: quick workaround for objtool warnings As discussed in https://bugzilla.suse.com/show_bug.cgi?id=1059674 the reason for multiple objtool warnings is the fact that vmmon module defines its own Panic() function which never returns. While it is marked as such which is used by the compiler for optimization, there is no way to find this from object file. While this seems innocuous, it might result in problems with unwinder later. The quickest way around is to replace vmmon's own Panic() with standard kernel panic() until a cleaner solution is found.	2020-05-30 02:46:01 +02:00
Michal Kubecek	48563763f7	vmmon: use standard definition of CR3_PCID_MASK if available The CR3_PCID_MASK is defined in mainline asm/processor-flags.h since commit 6c690ee1039b ("x86/mm: Split read_cr3() into read_cr3_pa() and __read_cr3()") in v4.13-rc1.	2020-05-30 02:45:45 +02:00
Michal Kubecek	e8af2c1e83	vmmon: use standard definition of MSR_MISC_FEATURES_ENABLES if available The MSR_MISC_FEATURES_ENABLES macro is defined in mainline since commit ab6d9468631a ("x86/msr: Rename MISC_FEATURE_ENABLES to MISC_FEATURES_ENABLES") in v4.12-rc1. The MSR_MISC_FEATURES_ENABLES_CPUID_FAULT macro is defined in mainline since commit e9ea1e7f53b8 ("x86/arch_prctl: Add ARCH_[GET\|SET]_CPUID") in v4.12-rc1.	2020-05-30 02:45:35 +02:00
Michal Kubecek	bad750a0d8	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE_VMXNET3 if available The PCI_DEVICE_ID_VMWARE_VMXNET3 macro is defined in mainline pci_ids.h since commit b1226c7db1d9 ("vmxnet3: Move PCI Id to pci_ids.h") in v4.10-rc1.	2020-05-30 02:44:33 +02:00
Michal Kubecek	c1263a00b2	vmnet: use standard definition of PCI_VENDOR_ID_VMWARE if available The PCI_VENDOR_ID_VMWARE macro is defined in mainline pci_ids.h since commit 94e57fea6202 ("PCI: Move PCI_VENDOR_ID_VMWARE to pci_ids.h") in v3.18-rc1.	2020-05-30 02:44:04 +02:00
Michal Kubecek	992ae1d5a1	modules: do not override CC command line options Workstation/Player 15.5.0 tries to guess the right retpoline (Spectre) related compiler command line options, resulting in objtool warnings like warning: objtool: .text.__x86_indirect_thunk_rbp+0x0: unsupported intra-function call warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate e.g. in vmmon-only/common/memtrack.o (but also many others). This seems wrong; out-of-tree modules should be built with the same compiler options that were used to build the kernel they are built against. Distribution kernel devel packages usually make sure this is the way so let them do their job.	2020-05-30 02:43:52 +02:00
Michal Kubecek	06283a2606	vmmon: fix wrong VMX86_DEVEL test If VMX86_DEVEL macro is not defined, preprocessor warns when it is tested with "#if" in vmmon-only/linux/driver.c. As VMX86_DEVEL is tested with "#ifdef" everywhere else, change this one test to match the rest.	2020-05-30 02:43:42 +02:00
Michal Kubecek	dec5bab6a0	import Workstation 15.5.5 module sources	2020-05-30 02:35:30 +02:00