[src/trunk]: src/sys/arch/xen PVH and HVM guests can easily have more than XE...

[bouyer <bouyer%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/37e8d80dc58c
branches:  trunk
changeset: 366402:37e8d80dc58c
user:      bouyer <bouyer%NetBSD.org@localhost>
date:      Wed May 25 14:35:15 2022 +0000

description:
PVH and HVM guests can easily have more than XEN_LEGACY_MAX_VCPUS (32) cpus.
Support up to HVM_MAX_VCPUS (256). This requires resizing a few arrays in
evtchn.c, and using
VCPUOP_register_vcpu_info for vcpuid >= XEN_LEGACY_MAX_VCPUS
Tested with 96 vCPUs.

diffstat:

 sys/arch/xen/include/hypervisor.h |   3 +-
 sys/arch/xen/xen/evtchn.c         |  20 ++++++++++----
 sys/arch/xen/xen/hypervisor.c     |  52 ++++++++++++++++++++++++++++++++++++--
 3 files changed, 65 insertions(+), 10 deletions(-)

diffs (153 lines):

diff -r cb6aa64d0b28 -r 37e8d80dc58c sys/arch/xen/include/hypervisor.h
--- a/sys/arch/xen/include/hypervisor.h Wed May 25 12:47:40 2022 +0000
+++ b/sys/arch/xen/include/hypervisor.h Wed May 25 14:35:15 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: hypervisor.h,v 1.53 2022/05/19 09:54:27 bouyer Exp $   */
+/*     $NetBSD: hypervisor.h,v 1.54 2022/05/25 14:35:15 bouyer Exp $   */
 
 /*
  * Copyright (c) 2006 Manuel Bouyer.
@@ -208,5 +208,6 @@
 }
 
 void xen_init_ksyms(void);
+void xen_map_vcpu(struct cpu_info *);
 
 #endif /* _XEN_HYPERVISOR_H_ */
diff -r cb6aa64d0b28 -r 37e8d80dc58c sys/arch/xen/xen/evtchn.c
--- a/sys/arch/xen/xen/evtchn.c Wed May 25 12:47:40 2022 +0000
+++ b/sys/arch/xen/xen/evtchn.c Wed May 25 14:35:15 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: evtchn.c,v 1.98 2022/05/24 15:55:19 bouyer Exp $       */
+/*     $NetBSD: evtchn.c,v 1.99 2022/05/25 14:35:15 bouyer Exp $       */
 
 /*
  * Copyright (c) 2006 Manuel Bouyer.
@@ -54,7 +54,7 @@
 
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.98 2022/05/24 15:55:19 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.99 2022/05/25 14:35:15 bouyer Exp $");
 
 #include "opt_xen.h"
 #include "isa.h"
@@ -81,6 +81,14 @@
 #include <xen/evtchn.h>
 #include <xen/xenfunc.h>
 
+/* maximum number of (v)CPUs supported */
+#ifdef XENPV
+#define NBSD_XEN_MAX_VCPUS XEN_LEGACY_MAX_VCPUS
+#else
+#include <xen/include/public/hvm/hvm_info_table.h>
+#define NBSD_XEN_MAX_VCPUS HVM_MAX_VCPUS
+#endif
+
 #define        NR_PIRQS        NR_EVENT_CHANNELS
 
 /*
@@ -96,10 +104,10 @@
 static uint8_t evtch_bindcount[NR_EVENT_CHANNELS];
 
 /* event-channel <-> VCPU mapping for IPIs. XXX: redo for SMP. */
-static evtchn_port_t vcpu_ipi_to_evtch[XEN_LEGACY_MAX_VCPUS];
+static evtchn_port_t vcpu_ipi_to_evtch[NBSD_XEN_MAX_VCPUS];
 
 /* event-channel <-> VCPU mapping for VIRQ_TIMER.  XXX: redo for SMP. */
-static int virq_timer_to_evtch[XEN_LEGACY_MAX_VCPUS];
+static int virq_timer_to_evtch[NBSD_XEN_MAX_VCPUS];
 
 /* event-channel <-> VIRQ mapping. */
 static int virq_to_evtch[NR_VIRQS];
@@ -226,11 +234,11 @@
        int i;
 
        /* No VCPU -> event mappings. */
-       for (i = 0; i < XEN_LEGACY_MAX_VCPUS; i++)
+       for (i = 0; i < NBSD_XEN_MAX_VCPUS; i++)
                vcpu_ipi_to_evtch[i] = -1;
 
        /* No VIRQ_TIMER -> event mappings. */
-       for (i = 0; i < XEN_LEGACY_MAX_VCPUS; i++)
+       for (i = 0; i < NBSD_XEN_MAX_VCPUS; i++)
                virq_timer_to_evtch[i] = -1;
 
        /* No VIRQ -> event mappings. */
diff -r cb6aa64d0b28 -r 37e8d80dc58c sys/arch/xen/xen/hypervisor.c
--- a/sys/arch/xen/xen/hypervisor.c     Wed May 25 12:47:40 2022 +0000
+++ b/sys/arch/xen/xen/hypervisor.c     Wed May 25 14:35:15 2022 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: hypervisor.c,v 1.93 2021/12/05 02:59:50 msaitoh Exp $ */
+/* $NetBSD: hypervisor.c,v 1.94 2022/05/25 14:35:15 bouyer Exp $ */
 
 /*
  * Copyright (c) 2005 Manuel Bouyer.
@@ -53,7 +53,7 @@
 
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hypervisor.c,v 1.93 2021/12/05 02:59:50 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hypervisor.c,v 1.94 2022/05/25 14:35:15 bouyer Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -476,7 +476,7 @@
                ci->ci_vcpuid = ci->ci_acpiid;
        }
 
-       ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[ci->ci_vcpuid];
+       xen_map_vcpu(ci);
 
        /* Register event callback handler. */
 
@@ -814,3 +814,49 @@
        kernxen_pkt = KERNFS_ENTOPARENTDIR(dkt);
 #endif
 }
+
+/*
+ * setup Xen's vcpu_info. requires ci_vcpuid to be initialized.
+ */
+void
+xen_map_vcpu(struct cpu_info *ci)
+{
+       int size;
+       uintptr_t ptr;
+       struct vcpu_register_vcpu_info vcpu_info_op;
+       paddr_t ma;
+       int ret;
+
+       if (ci->ci_vcpuid < XEN_LEGACY_MAX_VCPUS) {
+               ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[ci->ci_vcpuid];
+               return;
+       }
+
+       /*
+        * need to map it via VCPUOP_register_vcpu_info
+        * aligning to the smallest power-of-2 size which can contain
+        * vcpu_info ensures this. Also make sure it's cache-line aligned,
+        * for performances.
+        */
+       size = CACHE_LINE_SIZE;
+       while (size < sizeof(struct vcpu_info)) {
+               size = size << 1;
+       }
+       ptr = (uintptr_t)uvm_km_alloc(kernel_map,
+                   sizeof(struct vcpu_info) + size - 1, 0,
+                   UVM_KMF_WIRED|UVM_KMF_ZERO);
+       ptr = roundup2(ptr, size);
+       ci->ci_vcpu = (struct vcpu_info *)ptr;
+
+       pmap_extract_ma(pmap_kernel(), (ptr & ~PAGE_MASK), &ma);
+       vcpu_info_op.mfn = ma >> PAGE_SHIFT;
+       vcpu_info_op.offset = (ptr & PAGE_MASK);
+       vcpu_info_op.rsvd = 0;
+
+       ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info,
+           ci->ci_vcpuid, &vcpu_info_op);
+       if (ret) {
+               panic("VCPUOP_register_vcpu_info for %d failed: %d",
+                   ci->ci_vcpuid, ret);
+       }
+}