qemu-nvmm: improvements

[Maxime Villard <max%m00nbsd.net@localhost>]

Module Name:	pkgsrc-wip
Committed By:	Maxime Villard <max%m00nbsd.net@localhost>
Pushed By:	maxv
Date:		Thu Jan 3 09:57:36 2019 +0100
Changeset:	209a90459c382760fd84cafded5d30eeedf72ef7

Modified Files:
	qemu-nvmm/distinfo
	qemu-nvmm/patches/patch-nvmm-support

Log Message:
qemu-nvmm: improvements

 - Force vcpu_dirty to false in the assist callbacks, Qemu sets it because
   it fetched the state, but it doesn't modify it; we don't want to
   install this state when resuming, because the guest RIP update done by
   libnvmm would be cancelled, causing an infinite loop.
 - Install an IPI handler. vcpu_stop is touched in the thread context, so
   it doesn't need to be atomic.
 - Don't forget to call x86_update_hflags when the state is updated.
 - De-dup some code.
 - Remove nvmm_vcpu_kick, unused.

To see a diff of this commit:
https://wip.pkgsrc.org/cgi-bin/gitweb.cgi?p=pkgsrc-wip.git;a=commitdiff;h=209a90459c382760fd84cafded5d30eeedf72ef7

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

diffstat:
 qemu-nvmm/distinfo                   |  2 +-
 qemu-nvmm/patches/patch-nvmm-support | 93 +++++++++++++++++++++++++-----------
 2 files changed, 65 insertions(+), 30 deletions(-)

diffs:
diff --git a/qemu-nvmm/distinfo b/qemu-nvmm/distinfo
index 0cf87dddaf..0a3fea5218 100644
--- a/qemu-nvmm/distinfo
+++ b/qemu-nvmm/distinfo
@@ -13,5 +13,5 @@ SHA1 (patch-hw_display_omap__dss.c) = 6b13242f28e32346bc70548c216c578d98fd3420
 SHA1 (patch-hw_net_etraxfs__eth.c) = e5dd1661d60dbcd27b332403e0843500ba9544bc
 SHA1 (patch-hw_net_xilinx__axienet.c) = ebcd2676d64ce6f31e4a8c976d4fdf530ad5e8b7
 SHA1 (patch-hw_usb_dev-mtp.c) = 66543b5559d92f8e2fa9a6eb85e5dfe7c1ad3339
-SHA1 (patch-nvmm-support) = 434efdf8ec5ceb05f42318324d7c3df59a124cfa
+SHA1 (patch-nvmm-support) = efb5cf245a9ea19bd392156979a009e3eca2b5af
 SHA1 (patch-tests_Makefile.include) = 42345d697cb2e324dccf1d68bd8d61e8001c6162
diff --git a/qemu-nvmm/patches/patch-nvmm-support b/qemu-nvmm/patches/patch-nvmm-support
index f3d3220b9a..e37050ae8f 100644
--- a/qemu-nvmm/patches/patch-nvmm-support
+++ b/qemu-nvmm/patches/patch-nvmm-support
@@ -12,8 +12,8 @@ Add NVMM support.
  obj-$(call lnot,$(CONFIG_KVM))  += kvm-stub.o
  obj-$(call lnot,$(CONFIG_TCG))  += tcg-stub.o
 --- accel/stubs/nvmm-stub.c	1970-01-01 01:00:00.000000000 +0100
-+++ accel/stubs/nvmm-stub.c	2018-10-28 15:27:35.991719000 +0100
-@@ -0,0 +1,71 @@
++++ accel/stubs/nvmm-stub.c	2019-01-02 19:21:03.914538149 +0100
+@@ -0,0 +1,67 @@
 +/*
 + * Copyright (c) 2018 The NetBSD Foundation, Inc.
 + * All rights reserved.
@@ -66,10 +66,6 @@ Add NVMM support.
 +{
 +}
 +
-+void nvmm_vcpu_kick(CPUState *cpu)
-+{
-+}
-+
 +void nvmm_cpu_synchronize_state(CPUState *cpu)
 +{
 +}
@@ -180,7 +176,7 @@ Add NVMM support.
    echo "TARGET_WORDS_BIGENDIAN=y" >> $config_target_mak
  fi
 --- cpus.c	2018-08-14 21:10:34.000000000 +0200
-+++ cpus.c	2018-10-30 11:00:23.590887374 +0100
++++ cpus.c	2019-01-02 19:01:47.486471540 +0100
 @@ -40,6 +40,7 @@
  #include "sysemu/hax.h"
  #include "sysemu/hvf.h"
@@ -189,7 +185,7 @@ Add NVMM support.
  #include "exec/exec-all.h"
  
  #include "qemu/thread.h"
-@@ -1616,6 +1617,49 @@
+@@ -1616,6 +1617,48 @@
      return NULL;
  }
  
@@ -198,6 +194,8 @@ Add NVMM support.
 +    CPUState *cpu = arg;
 +    int r;
 +
++    assert(nvmm_enabled());
++
 +    rcu_register_thread();
 +
 +    qemu_mutex_lock_iothread();
@@ -222,10 +220,7 @@ Add NVMM support.
 +                cpu_handle_guest_debug(cpu);
 +            }
 +        }
-+        while (cpu_thread_is_idle(cpu)) {
-+            qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
-+        }
-+        qemu_wait_io_event_common(cpu);
++        qemu_wait_io_event(cpu);
 +    } while (!cpu->unplug || cpu_can_run(cpu));
 +
 +    nvmm_destroy_vcpu(cpu);
@@ -239,7 +234,7 @@ Add NVMM support.
  #ifdef _WIN32
  static void CALLBACK dummy_apc_func(ULONG_PTR unused)
  {
-@@ -1970,6 +2014,19 @@
+@@ -1970,6 +2013,19 @@
  #endif
  }
  
@@ -259,7 +254,7 @@ Add NVMM support.
  static void qemu_dummy_start_vcpu(CPUState *cpu)
  {
      char thread_name[VCPU_THREAD_NAME_SIZE];
-@@ -2007,6 +2064,8 @@
+@@ -2007,6 +2063,8 @@
          qemu_tcg_init_vcpu(cpu);
      } else if (whpx_enabled()) {
          qemu_whpx_start_vcpu(cpu);
@@ -319,8 +314,8 @@ Add NVMM support.
  
  #endif /* QEMU_HW_ACCEL_H */
 --- include/sysemu/nvmm.h	1970-01-01 01:00:00.000000000 +0100
-+++ include/sysemu/nvmm.h	2018-10-28 15:32:34.120605000 +0100
-@@ -0,0 +1,60 @@
++++ include/sysemu/nvmm.h	2019-01-02 19:21:06.862479422 +0100
+@@ -0,0 +1,59 @@
 +/*
 + * Copyright (c) 2018 The NetBSD Foundation, Inc.
 + * All rights reserved.
@@ -363,7 +358,6 @@ Add NVMM support.
 +int nvmm_init_vcpu(CPUState *);
 +int nvmm_vcpu_exec(CPUState *);
 +void nvmm_destroy_vcpu(CPUState *);
-+void nvmm_vcpu_kick(CPUState *);
 +
 +void nvmm_cpu_synchronize_state(CPUState *);
 +void nvmm_cpu_synchronize_post_reset(CPUState *);
@@ -427,8 +421,8 @@ Add NVMM support.
  obj-$(CONFIG_WHPX) += whpx-all.o
 +obj-$(CONFIG_NVMM) += nvmm-all.o
 --- target/i386/nvmm-all.c	1970-01-01 01:00:00.000000000 +0100
-+++ target/i386/nvmm-all.c	2018-12-29 16:39:37.626985638 +0100
-@@ -0,0 +1,1081 @@
++++ target/i386/nvmm-all.c	2019-01-03 09:42:03.800979637 +0100
+@@ -0,0 +1,1122 @@
 +/*
 + * Copyright (c) 2018 The NetBSD Foundation, Inc.
 + * All rights reserved.
@@ -459,7 +453,7 @@ Add NVMM support.
 + */
 +
 +/*
-+ * QEMU NetBSD Virtual Machine Monitor accelerator (NVMM)
++ * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU.
 + */
 +
 +#include "qemu/osdep.h"
@@ -488,6 +482,7 @@ Add NVMM support.
 +	bool int_waiting;
 +	bool nmi_waiting;
 +	uint8_t tpr;
++	bool stop;
 +};
 +
 +static struct {
@@ -773,6 +768,8 @@ Add NVMM support.
 +	env->sysenter_eip = state.msrs[NVMM_X64_MSR_SYSENTER_ESP];
 +	env->sysenter_esp = state.msrs[NVMM_X64_MSR_SYSENTER_EIP];
 +	env->pat = state.msrs[NVMM_X64_MSR_PAT];
++
++	x86_update_hflags(env);
 +}
 +
 +/*
@@ -909,12 +906,18 @@ Add NVMM support.
 +		    "[%s, port=%lu, size=%zu]", (io->in ? "in" : "out"),
 +		    io->port, io->size);
 +	}
++
++	/* XXX Needed, otherwise infinite loop. */
++	current_cpu->vcpu_dirty = false;
 +}
 +
 +static void
 +nvmm_mem_callback(struct nvmm_mem *mem)
 +{
 +	cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write);
++
++	/* XXX Needed, otherwise infinite loop. */
++	current_cpu->vcpu_dirty = false;
 +}
 +
 +static const struct nvmm_callbacks nvmm_callbacks = {
@@ -1032,8 +1035,8 @@ Add NVMM support.
 +	 * VCPU loop. They are handled here.
 +	 */
 +	if (cpu->interrupt_request & CPU_INTERRUPT_INIT) {
++		nvmm_cpu_synchronize_state(cpu);
 +		do_cpu_init(x86_cpu);
-+		cpu->vcpu_dirty = true;
 +		vcpu->int_waiting = false;
 +		vcpu->nmi_waiting = false;
 +	}
@@ -1047,16 +1050,12 @@ Add NVMM support.
 +		cpu->halted = false;
 +	}
 +	if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
-+		if (!cpu->vcpu_dirty) {
-+			nvmm_get_registers(cpu);
-+		}
++		nvmm_cpu_synchronize_state(cpu);
 +		do_cpu_sipi(x86_cpu);
 +	}
 +	if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
 +		cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
-+		if (!cpu->vcpu_dirty) {
-+			nvmm_get_registers(cpu);
-+		}
++		nvmm_cpu_synchronize_state(cpu);
 +		apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
 +		    env->tpr_access_type);
 +	}
@@ -1079,11 +1078,17 @@ Add NVMM support.
 +			cpu->vcpu_dirty = false;
 +		}
 +
++		if (vcpu->stop) {
++			cpu->exception_index = EXCP_INTERRUPT;
++			vcpu->stop = false;
++			ret = 1;
++			break;
++		}
++
 +		nvmm_vcpu_pre_run(cpu);
 +
 +		if (atomic_read(&cpu->exit_request)) {
-+			ret = 1;
-+			break;
++			qemu_cpu_kick_self();
 +		}
 +
 +		ret = nvmm_vcpu_run(mach, vcpu->cpuid, &exit);
@@ -1199,6 +1204,34 @@ Add NVMM support.
 +
 +static Error *nvmm_migration_blocker;
 +
++static void
++nvmm_ipi_signal(int sigcpu)
++{
++	struct nvmm_vcpu *vcpu;
++
++	if (current_cpu) {
++		vcpu = get_nvmm_vcpu(current_cpu);
++		vcpu->stop = true;
++	}
++}
++
++static void
++nvmm_init_cpu_signals(void)
++{
++	struct sigaction sigact;
++	sigset_t set;
++
++	/* Install the IPI handler. */
++	memset(&sigact, 0, sizeof(sigact));
++	sigact.sa_handler = nvmm_ipi_signal;
++	sigaction(SIG_IPI, &sigact, NULL);
++
++	/* Allow IPIs on the current thread. */
++	sigprocmask(SIG_BLOCK, NULL, &set);
++	sigdelset(&set, SIG_IPI);
++	pthread_sigmask(SIG_SETMASK, &set, NULL);
++}
++
 +int
 +nvmm_init_vcpu(CPUState *cpu)
 +{
@@ -1207,6 +1240,8 @@ Add NVMM support.
 +	struct nvmm_vcpu *vcpu;
 +	int ret;
 +
++	nvmm_init_cpu_signals();
++
 +	if (nvmm_migration_blocker == NULL) {
 +		error_setg(&nvmm_migration_blocker,
 +		    "NVMM: Migration not supported");