Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/bouyer-xenpvh]: src/sys/arch Add PVHVM multiprocessor support:



details:   https://anonhg.NetBSD.org/src/rev/8d6997909154
branches:  bouyer-xenpvh
changeset: 982947:8d6997909154
user:      bouyer <bouyer%NetBSD.org@localhost>
date:      Sat Apr 18 15:06:18 2020 +0000

description:
Add PVHVM multiprocessor support:
We need the hypervisor to be set up before cpus attaches.
Move hypervisor setup to a new function xen_hvm_init(), called at the
beggining of mainbus_attach(). This function searches the cfdata[] array
to see if the hypervisor device is enabled (so you can disable PV
support with
disable hypervisor
from userconf).
For HVM, ci_cpuid doens't match the virtual CPU index needed by Xen.
Introduce ci_vcpuid to cpu_info. Introduce xen_hvm_init_cpu(), to be
called for each CPU in in its context, which initialize ci_vcpuid and
ci_vcpu, and setup the event callback.
Change Xen code to use ci_vcpuid.

Do not call lapic_calibrate_timer() for VM_GUEST_XENPVHVM, we will use
Xen timers.

Don't call lapic_initclocks() from cpu_hatch(); instead set
x86_cpu_initclock_func to lapic_initclocks() in lapic_calibrate_timer(),
and call *(x86_cpu_initclock_func)() from cpu_hatch().
Also call x86_cpu_initclock_func from cpu_attach() for the boot CPU.
As x86_cpu_initclock_func is called for all CPUs, x86_initclock_func can
be a NOP for lapic timer.

Reorganize Xen code for x86_initclock_func/x86_cpu_initclock_func.
Move x86_cpu_idle_xen() to hypervisor_machdep.c

diffstat:

 sys/arch/x86/include/cpu.h            |    4 +-
 sys/arch/x86/include/cpuvar.h         |    5 +-
 sys/arch/x86/x86/cpu.c                |   25 ++-
 sys/arch/x86/x86/mainbus.c            |   12 +-
 sys/arch/xen/include/hypervisor.h     |    6 +-
 sys/arch/xen/include/xen.h            |    4 +-
 sys/arch/xen/x86/cpu.c                |   34 +--
 sys/arch/xen/x86/hypervisor_machdep.c |   44 +++-
 sys/arch/xen/x86/xen_ipi.c            |   10 +-
 sys/arch/xen/x86/xen_mainbus.c        |    6 +-
 sys/arch/xen/xen/evtchn.c             |   14 +-
 sys/arch/xen/xen/hypervisor.c         |  329 ++++++++++++++++++---------------
 sys/arch/xen/xen/xen_clock.c          |   56 ++--
 13 files changed, 312 insertions(+), 237 deletions(-)

diffs (truncated from 1115 to 300 lines):

diff -r 98f2082ee15b -r 8d6997909154 sys/arch/x86/include/cpu.h
--- a/sys/arch/x86/include/cpu.h        Sat Apr 18 14:49:57 2020 +0000
+++ b/sys/arch/x86/include/cpu.h        Sat Apr 18 15:06:18 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu.h,v 1.117.4.5 2020/04/16 17:44:54 bouyer Exp $     */
+/*     $NetBSD: cpu.h,v 1.117.4.6 2020/04/18 15:06:18 bouyer Exp $     */
 
 /*
  * Copyright (c) 1990 The Regents of the University of California.
@@ -223,6 +223,7 @@
        uint32_t        ci_flags __aligned(64);/* general flags */
        uint32_t        ci_acpiid;      /* our ACPI/MADT ID */
        uint32_t        ci_initapicid;  /* our initial APIC ID */
+       uint32_t        ci_vcpuid;      /* our CPU id for hypervisor */
        cpuid_t         ci_cpuid;       /* our CPU ID */
        struct cpu_info *ci_next;       /* next cpu */
 
@@ -530,6 +531,7 @@
 void   xen_startrtclock(void);
 void   xen_delay(unsigned int);
 void   xen_initclocks(void);
+void   xen_cpu_initclocks(void);
 void   xen_suspendclocks(struct cpu_info *);
 void   xen_resumeclocks(struct cpu_info *);
 #endif /* XEN */
diff -r 98f2082ee15b -r 8d6997909154 sys/arch/x86/include/cpuvar.h
--- a/sys/arch/x86/include/cpuvar.h     Sat Apr 18 14:49:57 2020 +0000
+++ b/sys/arch/x86/include/cpuvar.h     Sat Apr 18 15:06:18 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpuvar.h,v 1.51 2019/02/11 14:59:32 cherry Exp $ */
+/*     $NetBSD: cpuvar.h,v 1.51.10.1 2020/04/18 15:06:18 bouyer Exp $ */
 
 /*-
  * Copyright (c) 2000, 2007 The NetBSD Foundation, Inc.
@@ -99,6 +99,7 @@
 #include <sys/kcpuset.h>
 #if defined(_KERNEL_OPT)
 #include "opt_multiprocessor.h"
+#include "opt_xen.h"
 #endif /* defined(_KERNEL_OPT) */
 
 extern int (*x86_ipi)(int, int, int);
@@ -115,7 +116,7 @@
 void x86_cpu_idle_init(void);
 void x86_cpu_idle_halt(void);
 void x86_cpu_idle_mwait(void);
-#ifdef XENPV
+#ifdef XEN
 void x86_cpu_idle_xen(void);
 #endif
 
diff -r 98f2082ee15b -r 8d6997909154 sys/arch/x86/x86/cpu.c
--- a/sys/arch/x86/x86/cpu.c    Sat Apr 18 14:49:57 2020 +0000
+++ b/sys/arch/x86/x86/cpu.c    Sat Apr 18 15:06:18 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu.c,v 1.181.4.2 2020/04/16 09:45:56 bouyer Exp $     */
+/*     $NetBSD: cpu.c,v 1.181.4.3 2020/04/18 15:06:18 bouyer Exp $     */
 
 /*
  * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.181.4.2 2020/04/16 09:45:56 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.181.4.3 2020/04/18 15:06:18 bouyer Exp $");
 
 #include "opt_ddb.h"
 #include "opt_mpbios.h"                /* for MPDEBUG */
@@ -89,6 +89,7 @@
 
 #include "acpica.h"            /* for NACPICA, for mp_verbose */
 
+#include <x86/machdep.h>
 #include <machine/cpufunc.h>
 #include <machine/cpuvar.h>
 #include <machine/pmap.h>
@@ -130,6 +131,10 @@
 #endif
 #endif
 
+#ifdef XEN
+#include <xen/hypervisor.h>
+#endif
+
 static int     cpu_match(device_t, cfdata_t, void *);
 static void    cpu_attach(device_t, device_t, void *);
 static void    cpu_defer(device_t);
@@ -442,7 +447,8 @@
                        /* Enable lapic. */
                        lapic_enable();
                        lapic_set_lvt();
-                       lapic_calibrate_timer(ci);
+                       if (vm_guest != VM_GUEST_XENPVHVM)
+                               lapic_calibrate_timer(ci);
                }
 #endif
                /* Make sure DELAY() is initialized. */
@@ -459,6 +465,10 @@
                cpu_identify(ci);
                x86_errata();
                x86_cpu_idle_init();
+               (*x86_cpu_initclock_func)();
+#ifdef XENPVHVM
+               xen_hvm_init_cpu(ci);
+#endif
                break;
 
        case CPU_ROLE_BP:
@@ -466,6 +476,10 @@
                cpu_identify(ci);
                x86_errata();
                x86_cpu_idle_init();
+#ifdef XENPVHVM
+               xen_hvm_init_cpu(ci);
+#endif
+               (*x86_cpu_initclock_func)();
                break;
 
 #ifdef MULTIPROCESSOR
@@ -971,7 +985,6 @@
 #if NLAPIC > 0
        lapic_enable();
        lapic_set_lvt();
-       lapic_initclocks();
 #endif
 
        fpuinit(ci);
@@ -984,6 +997,10 @@
         * above.
         */
        cpu_init(ci);
+#ifdef XENPVHVM
+       xen_hvm_init_cpu(ci);
+#endif
+       (*x86_cpu_initclock_func)();
        cpu_get_tsc_freq(ci);
 
        s = splhigh();
diff -r 98f2082ee15b -r 8d6997909154 sys/arch/x86/x86/mainbus.c
--- a/sys/arch/x86/x86/mainbus.c        Sat Apr 18 14:49:57 2020 +0000
+++ b/sys/arch/x86/x86/mainbus.c        Sat Apr 18 15:06:18 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: mainbus.c,v 1.3.12.2 2020/04/16 08:46:35 bouyer Exp $ */
+/* $NetBSD: mainbus.c,v 1.3.12.3 2020/04/18 15:06:18 bouyer Exp $ */
 
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -28,7 +28,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: mainbus.c,v 1.3.12.2 2020/04/16 08:46:35 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: mainbus.c,v 1.3.12.3 2020/04/18 15:06:18 bouyer Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -217,6 +217,10 @@
        aprint_naive("\n");
        aprint_normal("\n");
 
+#if defined(XENPVHVM)
+       xen_hvm_init(); /* before attaching CPUs */
+#endif
+
 #if defined(XENPV)
        if (xendomain_is_dom0()) {
 #endif /* XENPV */
@@ -226,6 +230,10 @@
        }
 #endif /* XENPV */
 #if defined(XEN)
+       /*
+        * before isa/pci probe, so that PV devices are not probed again
+        * as emulated
+        */
        xen_mainbus_attach(parent, self, aux);
 #endif
 #if defined(__i386__) && !defined(XENPV)
diff -r 98f2082ee15b -r 8d6997909154 sys/arch/xen/include/hypervisor.h
--- a/sys/arch/xen/include/hypervisor.h Sat Apr 18 14:49:57 2020 +0000
+++ b/sys/arch/xen/include/hypervisor.h Sat Apr 18 15:06:18 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: hypervisor.h,v 1.49.10.2 2020/04/16 08:46:35 bouyer Exp $      */
+/*     $NetBSD: hypervisor.h,v 1.49.10.3 2020/04/18 15:06:18 bouyer Exp $      */
 
 /*
  * Copyright (c) 2006 Manuel Bouyer.
@@ -58,6 +58,10 @@
 #include "isa.h"
 #include "pci.h"
 
+struct cpu_info;
+
+int xen_hvm_init(void);
+int xen_hvm_init_cpu(struct cpu_info *);
 void xen_mainbus_attach(device_t, device_t, void *);
 
 struct hypervisor_attach_args {
diff -r 98f2082ee15b -r 8d6997909154 sys/arch/xen/include/xen.h
--- a/sys/arch/xen/include/xen.h        Sat Apr 18 14:49:57 2020 +0000
+++ b/sys/arch/xen/include/xen.h        Sat Apr 18 15:06:18 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: xen.h,v 1.44 2019/05/09 17:09:50 bouyer Exp $  */
+/*     $NetBSD: xen.h,v 1.44.8.1 2020/04/18 15:06:18 bouyer Exp $      */
 
 /*
  *
@@ -70,8 +70,6 @@
 void   xenevt_setipending(int, int);
 void   xenevt_notify(void);
 
-void   idle_block(void);
-
 /* xen_machdep.c */
 void   sysctl_xen_suspend_setup(void);
 
diff -r 98f2082ee15b -r 8d6997909154 sys/arch/xen/x86/cpu.c
--- a/sys/arch/xen/x86/cpu.c    Sat Apr 18 14:49:57 2020 +0000
+++ b/sys/arch/xen/x86/cpu.c    Sat Apr 18 15:06:18 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu.c,v 1.133 2020/02/24 12:20:29 rin Exp $    */
+/*     $NetBSD: cpu.c,v 1.133.4.1 2020/04/18 15:06:18 bouyer Exp $     */
 
 /*-
  * Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -65,7 +65,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.133 2020/02/24 12:20:29 rin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.133.4.1 2020/04/18 15:06:18 bouyer Exp $");
 
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
@@ -317,7 +317,7 @@
 vcpu_is_up(struct cpu_info *ci)
 {
        KASSERT(ci != NULL);
-       return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_cpuid, NULL);
+       return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_vcpuid, NULL);
 }
 
 static void
@@ -390,6 +390,7 @@
        sc->sc_info = ci;
        ci->ci_dev = self;
        ci->ci_cpuid = cpunum;
+       ci->ci_vcpuid = cpunum;
 
        KASSERT(HYPERVISOR_shared_info != NULL);
        KASSERT(cpunum < XEN_LEGACY_MAX_VCPUS);
@@ -455,12 +456,14 @@
                atomic_or_32(&ci->ci_flags, CPUF_SP);
                cpu_identify(ci);
                x86_cpu_idle_init();
+               xen_cpu_initclocks();
                break;
 
        case CPU_ROLE_BP:
                atomic_or_32(&ci->ci_flags, CPUF_BSP);
                cpu_identify(ci);
                x86_cpu_idle_init();
+               xen_cpu_initclocks();
                break;
 
        case CPU_ROLE_AP:
@@ -723,7 +726,7 @@
 
        xen_ipi_init();
 
-       xen_initclocks();
+       xen_cpu_initclocks();
 
 #ifdef __x86_64__
        fpuinit(ci);
@@ -764,7 +767,7 @@
                db_printf("%p   %s      %ld     %x      %x      %10p\n",
                    ci,
                    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
-                   (long)ci->ci_cpuid,
+                   (long)ci->ci_vcpuid,
                    ci->ci_flags, ci->ci_ipis,
                    ci->ci_curlwp);
        }
@@ -1011,7 +1014,7 @@
 #endif
 
        /* Initialise the given vcpu to execute cpu_hatch(ci); */
-       if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_cpuid, &vcpuctx))) {
+       if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_vcpuid, &vcpuctx))) {
                aprint_error(": context initialisation failed. errno = %d\n", hyperror);
                return hyperror;
        }
@@ -1019,12 +1022,12 @@
        /* Start it up */
 
        /* First bring it down */
-       if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_cpuid, NULL))) {
+       if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_vcpuid, NULL))) {



Home | Main Index | Thread Index | Old Index