[src/trunk]: src Change the NVMM API to reduce data movements. Sent to tech-k...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/53bf054856f3
branches:  trunk
changeset: 451855:53bf054856f3
user:      maxv <maxv%NetBSD.org@localhost>
date:      Sat Jun 08 07:27:44 2019 +0000

description:
Change the NVMM API to reduce data movements. Sent to tech-kern@.

diffstat:

 lib/libnvmm/libnvmm.3            |  114 ++++++++++++++++++--------
 lib/libnvmm/libnvmm.c            |   62 ++++++--------
 lib/libnvmm/libnvmm_x86.c        |  161 +++++++++++++++-----------------------
 lib/libnvmm/nvmm.h               |   30 ++++--
 tests/lib/libnvmm/h_io_assist.c  |   70 ++++++++--------
 tests/lib/libnvmm/h_mem_assist.c |   70 ++++++++--------
 6 files changed, 258 insertions(+), 249 deletions(-)

diffs (truncated from 1173 to 300 lines):

diff -r 43934e0ecd0c -r 53bf054856f3 lib/libnvmm/libnvmm.3
--- a/lib/libnvmm/libnvmm.3     Fri Jun 07 23:35:52 2019 +0000
+++ b/lib/libnvmm/libnvmm.3     Sat Jun 08 07:27:44 2019 +0000
@@ -1,4 +1,4 @@
-.\"    $NetBSD: libnvmm.3,v 1.18 2019/05/11 07:44:00 maxv Exp $
+.\"    $NetBSD: libnvmm.3,v 1.19 2019/06/08 07:27:44 maxv Exp $
 .\"
 .\" Copyright (c) 2018, 2019 The NetBSD Foundation, Inc.
 .\" All rights reserved.
@@ -27,7 +27,7 @@
 .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 .\" POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd May 11, 2019
+.Dd May 30, 2019
 .Dt LIBNVMM 3
 .Os
 .Sh NAME
@@ -47,21 +47,20 @@
 .Fn nvmm_machine_configure "struct nvmm_machine *mach" "uint64_t op" \
     "void *conf"
 .Ft int
-.Fn nvmm_vcpu_create "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid"
+.Fn nvmm_vcpu_create "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
+    "struct nvmm_vcpu *vcpu"
 .Ft int
-.Fn nvmm_vcpu_destroy "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid"
-.Ft int
-.Fn nvmm_vcpu_getstate "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
-    "void *state" "uint64_t flags"
+.Fn nvmm_vcpu_destroy "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu"
 .Ft int
-.Fn nvmm_vcpu_setstate "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
-    "void *state" "uint64_t flags"
+.Fn nvmm_vcpu_getstate "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" \
+    "uint64_t flags"
 .Ft int
-.Fn nvmm_vcpu_inject "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
-    "struct nvmm_event *event"
+.Fn nvmm_vcpu_setstate "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" \
+    "uint64_t flags"
 .Ft int
-.Fn nvmm_vcpu_run "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
-    "struct nvmm_exit *exit"
+.Fn nvmm_vcpu_inject "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu"
+.Ft int
+.Fn nvmm_vcpu_run "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu"
 .Ft int
 .Fn nvmm_hva_map "struct nvmm_machine *mach" "uintptr_t hva" "size_t size"
 .Ft int
@@ -73,17 +72,15 @@
 .Fn nvmm_gpa_unmap "struct nvmm_machine *mach" "uintptr_t hva" "gpaddr_t gpa" \
     "size_t size"
 .Ft int
-.Fn nvmm_gva_to_gpa "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
+.Fn nvmm_gva_to_gpa "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" \
     "gvaddr_t gva" "gpaddr_t *gpa" "nvmm_prot_t *prot"
 .Ft int
 .Fn nvmm_gpa_to_hva "struct nvmm_machine *mach" "gpaddr_t gpa" \
     "uintptr_t *hva" "nvmm_prot_t *prot"
 .Ft int
-.Fn nvmm_assist_io "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
-    "struct nvmm_exit *exit"
+.Fn nvmm_assist_io "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu"
 .Ft int
-.Fn nvmm_assist_mem "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \
-    "struct nvmm_exit *exit"
+.Fn nvmm_assist_mem "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu"
 .Sh DESCRIPTION
 .Nm
 provides a library for emulator software to handle hardware-accelerated virtual
@@ -95,6 +92,8 @@
 should use the API provided by
 .Nm
 to manage virtual machines.
+A virtual CPU is described by a public structure,
+.Cd nvmm_vcpu .
 .Pp
 .Fn nvmm_capability
 gets the capabilities of NVMM.
@@ -124,61 +123,61 @@
 creates a virtual CPU in the machine
 .Fa mach ,
 giving it the CPU id
-.Fa cpuid .
+.Fa cpuid ,
+and initializes
+.Fa vcpu .
 .Pp
 .Fn nvmm_vcpu_destroy
 destroys the virtual CPU identified by
-.Fa cpuid
+.Fa vcpu
 in the machine
 .Fa mach .
 .Pp
 .Fn nvmm_vcpu_getstate
 gets the state of the virtual CPU identified by
-.Fa cpuid
+.Fa vcpu
 in the machine
 .Fa mach .
-The
-.Fa state
-argument is the address of a state area, and
 .Fa flags
 is the bitmap of the components that are to be retrieved.
+The components are located in
+.Fa vcpu->state .
 See
 .Sx VCPU State Area
 below for details.
 .Pp
 .Fn nvmm_vcpu_setstate
 sets the state of the virtual CPU identified by
-.Fa cpuid
+.Fa vcpu
 in the machine
 .Fa mach .
-The
-.Fa state
-argument is the address of a state area, and
 .Fa flags
 is the bitmap of the components that are to be set.
+The components are located in
+.Fa vcpu->state .
 See
 .Sx VCPU State Area
 below for details.
 .Pp
 .Fn nvmm_vcpu_inject
 injects into the CPU identified by
-.Fa cpuid
+.Fa vcpu
 of the machine
 .Fa mach
 an event described by
-.Fa event .
+.Fa vcpu->event .
 See
 .Sx Event Injection
 below for details.
 .Pp
 .Fn nvmm_vcpu_run
 runs the CPU identified by
-.Fa cpuid
+.Fa vcpu
 in the machine
 .Fa mach ,
 until a VM exit is triggered.
 The
-.Fa exit
+.Fa vcpu->exit
 structure is filled to indicate the exit reason, and the associated parameters
 if any.
 .Pp
@@ -220,7 +219,7 @@
 .Pp
 .Fn nvmm_gva_to_gpa
 translates, on the CPU
-.Fa cpuid
+.Fa vcpu
 from the machine
 .Fa mach ,
 the guest virtual address given in
@@ -246,9 +245,9 @@
 .Pp
 .Fn nvmm_assist_io
 emulates the I/O operation described in
-.Fa exit
+.Fa vcpu->exit
 on CPU
-.Fa cpuid
+.Fa vcpu
 from machine
 .Fa mach .
 See
@@ -257,9 +256,9 @@
 .Pp
 .Fn nvmm_assist_mem
 emulates the Mem operation described in
-.Fa exit
+.Fa vcpu->exit
 on CPU
-.Fa cpuid
+.Fa vcpu
 from machine
 .Fa mach .
 See
@@ -364,6 +363,7 @@
        struct nvmm_x64_state_intr intr;
        struct fxsave fpu;
 };
+#define nvmm_vcpu_state nvmm_x64_state
 .Ed
 .Pp
 Refer to functional examples to see precisely how to use this structure.
@@ -379,6 +379,46 @@
 see
 .Sx Exit Reasons
 below for details.
+.Ss VCPU Programming Model
+A VCPU is described by a public structure,
+.Cd nvmm_vcpu :
+.Bd -literal
+struct nvmm_vcpu {
+       nvmm_cpuid_t cpuid;
+       struct nvmm_vcpu_state *state;
+       struct nvmm_event *event;
+       struct nvmm_exit *exit;
+};
+.Ed
+.Pp
+This structure is used both publicly by emulator software and internally by
+.Nm .
+Emulator software should not modify the pointers of this structure, because
+they are initialized to special values by
+.Nm .
+.Pp
+A call to
+.Fn nvmm_vcpu_getstate
+will fetch the relevant parts of the VCPU state and put them in
+.Fa vcpu->state .
+A call to
+.Fn nvmm_vcpu_setstate
+will install in the VCPU the relevant parts of
+.Fa vcpu->state .
+A call to
+.Fn nvmm_vcpu_inject
+will inject in the VCPU the event in
+.Fa vcpu->event .
+A call to
+.Fn nvmm_vcpu_run
+will fill
+.Fa vcpu->exit
+with the VCPU exit information.
+.Pp
+If emulator software uses several threads, a VCPU should be associated with
+only one thread, and only this thread should perform VCPU modifications.
+Emulator software should not modify the state of a VCPU with several
+different threads.
 .Ss Exit Reasons
 The
 .Cd nvmm_exit
diff -r 43934e0ecd0c -r 53bf054856f3 lib/libnvmm/libnvmm.c
--- a/lib/libnvmm/libnvmm.c     Fri Jun 07 23:35:52 2019 +0000
+++ b/lib/libnvmm/libnvmm.c     Sat Jun 08 07:27:44 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: libnvmm.c,v 1.13 2019/05/11 07:31:57 maxv Exp $        */
+/*     $NetBSD: libnvmm.c,v 1.14 2019/06/08 07:27:44 maxv Exp $        */
 
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -224,7 +224,6 @@
        memset(mach, 0, sizeof(*mach));
        mach->machid = args.machid;
        mach->pages = pages;
-       mach->npages = __capability.max_vcpus;
        mach->areas = areas;
 
        return 0;
@@ -272,7 +271,8 @@
 }
 
 int
-nvmm_vcpu_create(struct nvmm_machine *mach, nvmm_cpuid_t cpuid)
+nvmm_vcpu_create(struct nvmm_machine *mach, nvmm_cpuid_t cpuid,
+    struct nvmm_vcpu *vcpu)
 {
        struct nvmm_ioc_vcpu_create args;
        struct nvmm_comm_page *comm;
@@ -292,41 +292,42 @@
 
        mach->pages[cpuid] = comm;
 
+       vcpu->cpuid = cpuid;
+       vcpu->state = &comm->state;
+       vcpu->event = &comm->event;
+       vcpu->exit = malloc(sizeof(*vcpu->exit));
+
        return 0;
 }
 
 int
-nvmm_vcpu_destroy(struct nvmm_machine *mach, nvmm_cpuid_t cpuid)
+nvmm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
 {
        struct nvmm_ioc_vcpu_destroy args;
        struct nvmm_comm_page *comm;
        int ret;
 
        args.machid = mach->machid;
-       args.cpuid = cpuid;
+       args.cpuid = vcpu->cpuid;
 
        ret = ioctl(nvmm_fd, NVMM_IOC_VCPU_DESTROY, &args);
        if (ret == -1)
                return -1;
 
-       comm = mach->pages[cpuid];