Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/dev/nvmm/x86 Optimize nvmm-intel, use inlined GCC assemb...



details:   https://anonhg.NetBSD.org/src/rev/c630d18d30b5
branches:  trunk
changeset: 998707:c630d18d30b5
user:      maxv <maxv%NetBSD.org@localhost>
date:      Sat Apr 27 08:16:19 2019 +0000

description:
Optimize nvmm-intel, use inlined GCC assembly rather than function calls.

diffstat:

 sys/dev/nvmm/x86/nvmm_x86_vmx.c     |  293 +++++++++++++++++++++--------------
 sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S |  133 ++--------------
 2 files changed, 192 insertions(+), 234 deletions(-)

diffs (truncated from 747 to 300 lines):

diff -r dacda40641c6 -r c630d18d30b5 sys/dev/nvmm/x86/nvmm_x86_vmx.c
--- a/sys/dev/nvmm/x86/nvmm_x86_vmx.c   Sat Apr 27 06:18:15 2019 +0000
+++ b/sys/dev/nvmm/x86/nvmm_x86_vmx.c   Sat Apr 27 08:16:19 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: nvmm_x86_vmx.c,v 1.27 2019/04/24 18:19:28 maxv Exp $   */
+/*     $NetBSD: nvmm_x86_vmx.c,v 1.28 2019/04/27 08:16:19 maxv Exp $   */
 
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.27 2019/04/24 18:19:28 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.28 2019/04/27 08:16:19 maxv Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -56,13 +56,6 @@
 
 int _vmx_vmxon(paddr_t *pa);
 int _vmx_vmxoff(void);
-int _vmx_invept(uint64_t op, void *desc);
-int _vmx_invvpid(uint64_t op, void *desc);
-int _vmx_vmread(uint64_t op, uint64_t *val);
-int _vmx_vmwrite(uint64_t op, uint64_t val);
-int _vmx_vmptrld(paddr_t *pa);
-int _vmx_vmptrst(paddr_t *pa);
-int _vmx_vmclear(paddr_t *pa);
 int vmx_vmlaunch(uint64_t *gprs);
 int vmx_vmresume(uint64_t *gprs);
 
@@ -74,34 +67,113 @@
        if (__predict_false(_vmx_vmxoff() != 0)) { \
                panic("%s: VMXOFF failed", __func__); \
        }
-#define vmx_invept(a, b) \
-       if (__predict_false(_vmx_invept(a, b) != 0)) { \
-               panic("%s: INVEPT failed", __func__); \
-       }
-#define vmx_invvpid(a, b) \
-       if (__predict_false(_vmx_invvpid(a, b) != 0)) { \
-               panic("%s: INVVPID failed", __func__); \
-       }
-#define vmx_vmread(a, b) \
-       if (__predict_false(_vmx_vmread(a, b) != 0)) { \
-               panic("%s: VMREAD failed", __func__); \
-       }
-#define vmx_vmwrite(a, b) \
-       if (__predict_false(_vmx_vmwrite(a, b) != 0)) { \
-               panic("%s: VMWRITE failed", __func__); \
-       }
-#define vmx_vmptrld(a) \
-       if (__predict_false(_vmx_vmptrld(a) != 0)) { \
-               panic("%s: VMPTRLD failed", __func__); \
-       }
-#define vmx_vmptrst(a) \
-       if (__predict_false(_vmx_vmptrst(a) != 0)) { \
-               panic("%s: VMPTRST failed", __func__); \
-       }
-#define vmx_vmclear(a) \
-       if (__predict_false(_vmx_vmclear(a) != 0)) { \
-               panic("%s: VMCLEAR failed", __func__); \
-       }
+
+struct ept_desc {
+       uint64_t eptp;
+       uint64_t mbz;
+} __packed;
+
+struct vpid_desc {
+       uint64_t vpid;
+       uint64_t addr;
+} __packed;
+
+static inline void
+vmx_invept(uint64_t op, struct ept_desc *desc)
+{
+       asm volatile (
+               "invept         %[desc],%[op];"
+               "jz             vmx_insn_failvalid;"
+               "jc             vmx_insn_failinvalid;"
+               :
+               : [desc] "m" (*desc), [op] "r" (op)
+               : "memory", "cc"
+       );
+}
+
+static inline void
+vmx_invvpid(uint64_t op, struct vpid_desc *desc)
+{
+       asm volatile (
+               "invvpid        %[desc],%[op];"
+               "jz             vmx_insn_failvalid;"
+               "jc             vmx_insn_failinvalid;"
+               :
+               : [desc] "m" (*desc), [op] "r" (op)
+               : "memory", "cc"
+       );
+}
+
+static inline uint64_t
+vmx_vmread(uint64_t field)
+{
+       uint64_t value;
+
+       asm volatile (
+               "vmread         %[field],%[value];"
+               "jz             vmx_insn_failvalid;"
+               "jc             vmx_insn_failinvalid;"
+               : [value] "=r" (value)
+               : [field] "r" (field)
+               : "cc"
+       );
+
+       return value;
+}
+
+static inline void
+vmx_vmwrite(uint64_t field, uint64_t value)
+{
+       asm volatile (
+               "vmwrite        %[value],%[field];"
+               "jz             vmx_insn_failvalid;"
+               "jc             vmx_insn_failinvalid;"
+               :
+               : [field] "r" (field), [value] "r" (value)
+               : "cc"
+       );
+}
+
+static inline paddr_t
+vmx_vmptrst(void)
+{
+       paddr_t pa;
+
+       asm volatile (
+               "vmptrst        %[pa];"
+               :
+               : [pa] "m" (*(paddr_t *)&pa)
+               : "memory"
+       );
+
+       return pa;
+}
+
+static inline void
+vmx_vmptrld(paddr_t *pa)
+{
+       asm volatile (
+               "vmptrld        %[pa];"
+               "jz             vmx_insn_failvalid;"
+               "jc             vmx_insn_failinvalid;"
+               :
+               : [pa] "m" (*pa)
+               : "memory", "cc"
+       );
+}
+
+static inline void
+vmx_vmclear(paddr_t *pa)
+{
+       asm volatile (
+               "vmclear        %[pa];"
+               "jz             vmx_insn_failvalid;"
+               "jc             vmx_insn_failinvalid;"
+               :
+               : [pa] "m" (*pa)
+               : "memory", "cc"
+       );
+}
 
 #define MSR_IA32_FEATURE_CONTROL       0x003A
 #define                IA32_FEATURE_CONTROL_LOCK       __BIT(0)
@@ -526,16 +598,6 @@
        uint64_t val;
 } __packed;
 
-struct ept_desc {
-       uint64_t eptp;
-       uint64_t mbz;
-} __packed;
-
-struct vpid_desc {
-       uint64_t vpid;
-       uint64_t addr;
-} __packed;
-
 #define VPID_MAX       0xFFFF
 
 /* Make sure we never run out of VPIDs. */
@@ -805,7 +867,7 @@
        if (cpudata->vmcs_refcnt > 1) {
 #ifdef DIAGNOSTIC
                KASSERT(kpreempt_disabled());
-               vmx_vmptrst(&oldpa);
+               oldpa = vmx_vmptrst();
                KASSERT(oldpa == cpudata->vmcs_pa);
 #endif
                return;
@@ -835,12 +897,10 @@
 vmx_vmcs_leave(struct nvmm_cpu *vcpu)
 {
        struct vmx_cpudata *cpudata = vcpu->cpudata;
-       paddr_t oldpa __diagused;
 
        KASSERT(kpreempt_disabled());
 #ifdef DIAGNOSTIC
-       vmx_vmptrst(&oldpa);
-       KASSERT(oldpa == cpudata->vmcs_pa);
+       KASSERT(vmx_vmptrst() == cpudata->vmcs_pa);
 #endif
        KASSERT(cpudata->vmcs_refcnt > 0);
        cpudata->vmcs_refcnt--;
@@ -857,12 +917,10 @@
 vmx_vmcs_destroy(struct nvmm_cpu *vcpu)
 {
        struct vmx_cpudata *cpudata = vcpu->cpudata;
-       paddr_t oldpa __diagused;
 
        KASSERT(kpreempt_disabled());
 #ifdef DIAGNOSTIC
-       vmx_vmptrst(&oldpa);
-       KASSERT(oldpa == cpudata->vmcs_pa);
+       KASSERT(vmx_vmptrst() == cpudata->vmcs_pa);
 #endif
        KASSERT(cpudata->vmcs_refcnt == 1);
        cpudata->vmcs_refcnt--;
@@ -879,7 +937,7 @@
        struct vmx_cpudata *cpudata = vcpu->cpudata;
        uint64_t ctls1;
 
-       vmx_vmread(VMCS_PROCBASED_CTLS, &ctls1);
+       ctls1 = vmx_vmread(VMCS_PROCBASED_CTLS);
 
        if (nmi) {
                // XXX INT_STATE_NMI?
@@ -899,7 +957,7 @@
        struct vmx_cpudata *cpudata = vcpu->cpudata;
        uint64_t ctls1;
 
-       vmx_vmread(VMCS_PROCBASED_CTLS, &ctls1);
+       ctls1 = vmx_vmread(VMCS_PROCBASED_CTLS);
 
        if (nmi) {
                ctls1 &= ~PROC_CTLS_NMI_WINDOW_EXITING;
@@ -950,7 +1008,7 @@
                if (event->vector == 2) {
                        type = INTR_TYPE_NMI;
                }
-               vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate);
+               intstate = vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY);
                if (type == INTR_TYPE_NMI) {
                        if (cpudata->nmi_window_exit) {
                                ret = EAGAIN;
@@ -958,7 +1016,7 @@
                        }
                        vmx_event_waitexit_enable(vcpu, true);
                } else {
-                       vmx_vmread(VMCS_GUEST_RFLAGS, &rflags);
+                       rflags = vmx_vmread(VMCS_GUEST_RFLAGS);
                        if ((rflags & PSL_I) == 0 ||
                            (intstate & (INT_STATE_STI|INT_STATE_MOVSS)) != 0) {
                                vmx_event_waitexit_enable(vcpu, false);
@@ -1041,10 +1099,10 @@
         * Matters for guest-ring3, because it can execute 'cpuid' under a
         * debugger.
         */
-       vmx_vmread(VMCS_EXIT_INSTRUCTION_LENGTH, &inslen);
-       vmx_vmread(VMCS_GUEST_RIP, &rip);
+       inslen = vmx_vmread(VMCS_EXIT_INSTRUCTION_LENGTH);
+       rip = vmx_vmread(VMCS_GUEST_RIP);
        vmx_vmwrite(VMCS_GUEST_RIP, rip + inslen);
-       vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate);
+       intstate = vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY);
        vmx_vmwrite(VMCS_GUEST_INTERRUPTIBILITY,
            intstate & ~(INT_STATE_STI|INT_STATE_MOVSS));
 }
@@ -1055,7 +1113,7 @@
 {
        uint64_t qual;
 
-       vmx_vmread(VMCS_EXIT_INTR_INFO, &qual);
+       qual = vmx_vmread(VMCS_EXIT_INTR_INFO);
 
        if ((qual & INTR_INFO_VALID) == 0) {
                goto error;
@@ -1091,7 +1149,7 @@
                cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx;
 
                /* CPUID2_OSXSAVE depends on CR4. */
-               vmx_vmread(VMCS_GUEST_CR4, &cr4);
+               cr4 = vmx_vmread(VMCS_GUEST_CR4);
                if (!(cr4 & CR4_OSXSAVE)) {
                        cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE;
                }
@@ -1207,7 +1265,7 @@
        uint64_t rflags;
 
        if (cpudata->int_window_exit) {



Home | Main Index | Thread Index | Old Index