[src/trunk]: src/sys/arch MP probing and startup code

[cherry <cherry%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/5042770203b9
branches:  trunk
changeset: 768264:5042770203b9
user:      cherry <cherry%NetBSD.org@localhost>
date:      Sat Aug 13 12:37:30 2011 +0000

description:
MP probing and startup code

diffstat:

 sys/arch/x86/include/cpuvar.h |    6 +-
 sys/arch/xen/x86/cpu.c        |  619 ++++++++++++++++++++++++-----------------
 sys/arch/xen/xen/hypervisor.c |   44 ++-
 3 files changed, 402 insertions(+), 267 deletions(-)

diffs (truncated from 1057 to 300 lines):

diff -r 51cd9699b72a -r 5042770203b9 sys/arch/x86/include/cpuvar.h
--- a/sys/arch/x86/include/cpuvar.h     Sat Aug 13 12:18:54 2011 +0000
+++ b/sys/arch/x86/include/cpuvar.h     Sat Aug 13 12:37:30 2011 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpuvar.h,v 1.44 2011/06/12 03:35:50 rmind Exp $ */
+/*     $NetBSD: cpuvar.h,v 1.45 2011/08/13 12:37:30 cherry Exp $ */
 
 /*-
  * Copyright (c) 2000, 2007 The NetBSD Foundation, Inc.
@@ -67,7 +67,11 @@
 #define        _X86_CPUVAR_H_
 
 struct cpu_functions {
+#ifndef XEN
        int (*start)(struct cpu_info *, paddr_t);
+#else /* XEN */
+       int (*start)(struct cpu_info *, vaddr_t);
+#endif /* XEN */
        int (*stop)(struct cpu_info *);
        void (*cleanup)(struct cpu_info *);
 };
diff -r 51cd9699b72a -r 5042770203b9 sys/arch/xen/x86/cpu.c
--- a/sys/arch/xen/x86/cpu.c    Sat Aug 13 12:18:54 2011 +0000
+++ b/sys/arch/xen/x86/cpu.c    Sat Aug 13 12:37:30 2011 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu.c,v 1.61 2011/08/11 18:11:17 cherry Exp $  */
+/*     $NetBSD: cpu.c,v 1.62 2011/08/13 12:37:30 cherry Exp $  */
 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp  */
 
 /*-
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.61 2011/08/11 18:11:17 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.62 2011/08/13 12:37:30 cherry Exp $");
 
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
@@ -85,6 +85,7 @@
 #include <sys/cpu.h>
 #include <sys/atomic.h>
 #include <sys/reboot.h>
+#include <sys/idle.h>
 
 #include <uvm/uvm.h>
 
@@ -100,6 +101,14 @@
 #include <machine/mtrr.h>
 #include <machine/pio.h>
 
+#ifdef i386
+#include <machine/npx.h>
+#else
+#include <machine/fpu.h>
+#endif
+
+#include <xen/xen.h>
+#include <xen/xen3-public/vcpu.h>
 #include <xen/vcpuvar.h>
 
 #if NLAPIC > 0
@@ -131,7 +140,7 @@
        bool sc_wasonline;
 };
 
-int mp_cpu_start(struct cpu_info *, paddr_t);
+int mp_cpu_start(struct cpu_info *, vaddr_t);
 void mp_cpu_start_cleanup(struct cpu_info *);
 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
                                      mp_cpu_start_cleanup };
@@ -171,8 +180,6 @@
 struct cpu_info *cpu_info_list = &cpu_info_primary;
 struct cpu_info *phycpu_info_list = &phycpu_info_primary;
 
-static void    cpu_set_tss_gates(struct cpu_info *ci);
-
 uint32_t cpus_attached = 1;
 uint32_t cpus_running = 1;
 
@@ -194,21 +201,6 @@
 void           cpu_hatch(void *);
 static void            cpu_boot_secondary(struct cpu_info *ci);
 static void            cpu_start_secondary(struct cpu_info *ci);
-static void    cpu_copy_trampoline(void);
-
-/*
- * Runs once per boot once multiprocessor goo has been detected and
- * the local APIC on the boot processor has been mapped.
- *
- * Called from lapic_boot_init() (from mpbios_scan()).
- */
-void
-cpu_init_first(void)
-{
-
-       cpu_info_primary.ci_cpuid = lapic_cpu_number();
-       cpu_copy_trampoline();
-}
 #endif /* MULTIPROCESSOR */
 
 static int
@@ -317,9 +309,23 @@
 vcpu_match(device_t parent, cfdata_t match, void *aux)
 {
        struct vcpu_attach_args *vcaa = aux;
+       struct vcpu_runstate_info vcr;
+       int error;
 
-       if (strcmp(vcaa->vcaa_name, match->cf_name) == 0)
-               return 1;
+       if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) {
+               error = HYPERVISOR_vcpu_op(VCPUOP_get_runstate_info,
+                                          vcaa->vcaa_caa.cpu_number,
+                                          &vcr);
+               switch (error) {
+               case 0:
+                       return 1;
+               case -ENOENT:
+                       return 0;
+               default:
+                       panic("Unknown hypervisor error %d returned on vcpu runstate probe\n", error);
+               }
+       }
+
        return 0;
 }
 
@@ -328,9 +334,18 @@
 {
        struct vcpu_attach_args *vcaa = aux;
 
+       KASSERT(vcaa->vcaa_caa.cpu_func == NULL);
+       vcaa->vcaa_caa.cpu_func = &mp_cpu_funcs;
        cpu_attach_common(parent, self, &vcaa->vcaa_caa);
 }
 
+static int
+vcpu_is_up(struct cpu_info *ci)
+{
+       KASSERT(ci != NULL);
+       return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_cpuid, NULL);
+}
+
 static void
 cpu_vm_init(struct cpu_info *ci)
 {
@@ -395,22 +410,6 @@
                aprint_naive(": %s Processor\n",
                    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
                ci = &cpu_info_primary;
-#if NLAPIC > 0
-               if (cpunum != lapic_cpu_number()) {
-                       /* XXX should be done earlier */
-                       uint32_t reg;
-                       aprint_verbose("\n");
-                       aprint_verbose_dev(self, "running CPU at apic %d"
-                           " instead of at expected %d", lapic_cpu_number(),
-                           cpunum);
-                       reg = i82489_readreg(LAPIC_ID);
-                       i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
-                           (cpunum << LAPIC_ID_SHIFT));
-               }
-               if (cpunum != lapic_cpu_number()) {
-                       aprint_error_dev(self, "unable to reset apic id\n");
-               }
-#endif
        }
 
        ci->ci_self = ci;
@@ -421,6 +420,7 @@
        KASSERT(HYPERVISOR_shared_info != NULL);
        ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
 
+       KASSERT(ci->ci_func == 0);
        ci->ci_func = caa->cpu_func;
 
        /* Must be called before mi_cpu_attach(). */
@@ -430,12 +430,15 @@
                int error;
 
                error = mi_cpu_attach(ci);
+
+               KASSERT(ci->ci_data.cpu_idlelwp != NULL);
                if (error != 0) {
                        aprint_normal("\n");
                        aprint_error_dev(self,
                            "mi_cpu_attach failed with %d\n", error);
                        return;
                }
+
        } else {
                KASSERT(ci->ci_data.cpu_idlelwp != NULL);
        }
@@ -455,15 +458,12 @@
                cpu_intr_init(ci);
                cpu_get_tsc_freq(ci);
                cpu_init(ci);
-               cpu_set_tss_gates(ci);
-#if NLAPIC > 0
-               if (caa->cpu_role != CPU_ROLE_SP) {
-                       /* Enable lapic. */
-                       lapic_enable();
-                       lapic_set_lvt();
-                       lapic_calibrate_timer();
-               }
-#endif
+               pmap_cpu_init_late(ci); /* XXX: cosmetic */
+
+               /* Every processor needs to init it's own ipi h/w (similar to lapic) */
+               xen_ipi_init();
+               /* XXX: clock_init() */
+
                /* Make sure DELAY() is initialized. */
                DELAY(1);
                again = true;
@@ -479,6 +479,7 @@
                x86_errata();
 #endif
                x86_cpu_idle_init();
+
                break;
 
        case CPU_ROLE_BP:
@@ -489,22 +490,30 @@
                x86_errata();
 #endif
                x86_cpu_idle_init();
+
                break;
 
        case CPU_ROLE_AP:
+               atomic_or_32(&ci->ci_flags, CPUF_AP);
+
                /*
                 * report on an AP
                 */
 
 #if defined(MULTIPROCESSOR)
+               /* interrupt handler stack */
                cpu_intr_init(ci);
+
+               /* Setup per-cpu memory for gdt */
                gdt_alloc_cpu(ci);
-               cpu_set_tss_gates(ci);
+
+               pmap_cpu_init_late(ci);
                cpu_start_secondary(ci);
+
                if (ci->ci_flags & CPUF_PRESENT) {
                        struct cpu_info *tmp;
 
-                       identifycpu(ci);
+                       cpu_identify(ci);
                        tmp = cpu_info_list;
                        while (tmp->ci_next)
                                tmp = tmp->ci_next;
@@ -512,7 +521,7 @@
                        tmp->ci_next = ci;
                }
 #else
-               aprint_error_dev(self, "not started\n");
+               aprint_error(": not started\n");
 #endif
                break;
 
@@ -529,7 +538,7 @@
                aprint_error_dev(self, "couldn't establish power handler\n");
 #endif
 
-#if defined(MULTIPROCESSOR)
+#ifdef MPVERBOSE
        if (mp_verbose) {
                struct lwp *l = ci->ci_data.cpu_idlelwp;
                struct pcb *pcb = lwp_getpcb(l);
@@ -539,13 +548,13 @@
                    l,
 #ifdef i386
                    (void *)pcb->pcb_esp
-#else
+#else /* i386 */
                    (void *)pcb->pcb_rsp
-#endif
+#endif /* i386 */
                );
                
        }
-#endif
+#endif /* MPVERBOSE */
 }
 
 /*
@@ -593,16 +602,18 @@
 
        atomic_or_32(&cpus_running, ci->ci_cpumask);
        atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
+
+       /* XXX: register vcpu_register_runstate_memory_area, and figure out how to make sure this VCPU is running ? */
 }