[src/trunk]: src/sys/arch Rework Arm (32bit and 64bit) AP startup so that cpu...

[skrll <skrll%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/40920aaad5c4
branches:  trunk
changeset: 990626:40920aaad5c4
user:      skrll <skrll%NetBSD.org@localhost>
date:      Sun Oct 31 16:23:47 2021 +0000

description:
Rework Arm (32bit and 64bit) AP startup so that cpu_hatch doesn't sleep.

The AP initialisation code in cpu_init_secondary_processor will read and
initialise the required system registers and state for the BP to attach
and report.

Rework the interrupt handler code for this new sequence. Thankfully,
this removes a bunch of code for bcm2836mp.

The VFP detection handler on <= armv7 relies on the global undefined
handler being in place until the BP attaches vfp. That is, after the
APs have been spun up.

gicv3_its.c has a serialisation issue which is protected against in
the gicv3_its_cpu_init, which is called from cpu_hatch, with a spin
lock. The serialisation issue needs addressing more completely.

Tested on RPI3, Apple M1, QEMU, and lx2k

Fixes PR port-arm/56264:
   diagnostic assertion "l->l_stat == LSONPROC" failed on RPI3

diffstat:

 sys/arch/aarch64/aarch64/aarch64_machdep.c |    6 +-
 sys/arch/aarch64/aarch64/cpu.c             |  103 ++++++++++-------
 sys/arch/aarch64/aarch64/cpufunc.c         |   93 +++++++---------
 sys/arch/aarch64/aarch64/db_machdep.c      |   23 +++-
 sys/arch/aarch64/aarch64/locore.S          |   60 +++-------
 sys/arch/aarch64/include/cpu.h             |   32 +++++-
 sys/arch/aarch64/include/cpufunc.h         |   33 +----
 sys/arch/aarch64/include/db_machdep.h      |    5 +-
 sys/arch/arm/apple/apple_intc.c            |   91 +++++++--------
 sys/arch/arm/arm/cpu_subr.c                |   11 +-
 sys/arch/arm/arm/undefined.c               |   40 ++++++-
 sys/arch/arm/arm32/arm32_boot.c            |   27 +---
 sys/arch/arm/arm32/arm32_machdep.c         |   16 ++-
 sys/arch/arm/arm32/cpu.c                   |   56 +++++----
 sys/arch/arm/broadcom/bcm2835_intr.c       |  118 +++----------------
 sys/arch/arm/cortex/gicv3_its.c            |   20 ++-
 sys/arch/arm/cortex/gicv3_its.h            |    5 +-
 sys/arch/arm/cortex/gtmr.c                 |    5 +-
 sys/arch/arm/include/cpu.h                 |   16 +-
 sys/arch/arm/include/locore.h              |    3 +-
 sys/arch/arm/include/undefined.h           |    7 +-
 sys/arch/arm/pic/pic.c                     |    6 +-
 sys/arch/arm/vfp/vfp_init.c                |  164 ++++++++++++----------------
 23 files changed, 452 insertions(+), 488 deletions(-)

diffs (truncated from 1996 to 300 lines):

diff -r 5eb5ac1dea5c -r 40920aaad5c4 sys/arch/aarch64/aarch64/aarch64_machdep.c
--- a/sys/arch/aarch64/aarch64/aarch64_machdep.c        Sun Oct 31 15:32:14 2021 +0000
+++ b/sys/arch/aarch64/aarch64/aarch64_machdep.c        Sun Oct 31 16:23:47 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: aarch64_machdep.c,v 1.62 2021/10/08 21:41:29 ryo Exp $ */
+/* $NetBSD: aarch64_machdep.c,v 1.63 2021/10/31 16:23:47 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aarch64_machdep.c,v 1.62 2021/10/08 21:41:29 ryo Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aarch64_machdep.c,v 1.63 2021/10/31 16:23:47 skrll Exp $");
 
 #include "opt_arm_debug.h"
 #include "opt_cpuoptions.h"
@@ -359,7 +359,7 @@
            VM_MAX_KERNEL_ADDRESS);
 
 #ifdef DDB
-       db_machdep_init();
+       db_machdep_cpu_init();
 #endif
 
        uvm_md_init();
diff -r 5eb5ac1dea5c -r 40920aaad5c4 sys/arch/aarch64/aarch64/cpu.c
--- a/sys/arch/aarch64/aarch64/cpu.c    Sun Oct 31 15:32:14 2021 +0000
+++ b/sys/arch/aarch64/aarch64/cpu.c    Sun Oct 31 16:23:47 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.66 2021/10/30 10:47:03 skrll Exp $ */
+/* $NetBSD: cpu.c,v 1.67 2021/10/31 16:23:47 skrll Exp $ */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.66 2021/10/30 10:47:03 skrll Exp $");
+__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.67 2021/10/31 16:23:47 skrll Exp $");
 
 #include "locators.h"
 #include "opt_arm_debug.h"
@@ -72,12 +72,13 @@
 #endif
 
 void cpu_attach(device_t, cpuid_t);
+void cpu_setup_id(struct cpu_info *);
+
 static void identify_aarch64_model(uint32_t, char *, size_t);
 static void cpu_identify(device_t self, struct cpu_info *);
 static void cpu_identify1(device_t self, struct cpu_info *);
 static void cpu_identify2(device_t self, struct cpu_info *);
 static void cpu_init_counter(struct cpu_info *);
-static void cpu_setup_id(struct cpu_info *);
 static void cpu_setup_sysctl(device_t, struct cpu_info *);
 static void cpu_setup_rng(device_t, struct cpu_info *);
 static void cpu_setup_aes(device_t, struct cpu_info *);
@@ -109,7 +110,6 @@
        if (unit == 0) {
                ci = curcpu();
                ci->ci_cpuid = id;
-               cpu_setup_id(ci);
        } else {
 #ifdef MULTIPROCESSOR
                if ((boothowto & RB_MD1) != 0) {
@@ -150,26 +150,31 @@
        arm_cpu_do_topology(ci);
        cpu_identify(dv, ci);
 
+       cpu_setup_sysctl(dv, ci);
+
 #ifdef MULTIPROCESSOR
        if (unit != 0) {
                mi_cpu_attach(ci);
                pmap_tlb_info_attach(&pmap_tlb0_info, ci);
-               return;
+               aarch64_parsecacheinfo(ci);
        }
 #endif /* MULTIPROCESSOR */
 
-       set_cpufuncs();
        fpu_attach(ci);
 
        cpu_identify1(dv, ci);
+       aarch64_printcacheinfo(dv, ci);
+       cpu_identify2(dv, ci);
 
-       /* aarch64_getcacheinfo(0) was called by locore.S */
-       aarch64_printcacheinfo(dv);
-       cpu_identify2(dv, ci);
+       if (unit != 0) {
+           return;
+       }
+
+       db_machdep_init(ci);
 
        cpu_init_counter(ci);
 
-       cpu_setup_sysctl(dv, ci);
+       /* These currently only check the BP. */
        cpu_setup_rng(dv, ci);
        cpu_setup_aes(dv, ci);
        cpu_setup_chacha(dv, ci);
@@ -251,10 +256,9 @@
 static void
 cpu_identify1(device_t self, struct cpu_info *ci)
 {
-       uint64_t ctr, clidr, sctlr;     /* for cache */
+       struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
+       uint64_t sctlr = ci->ci_sctlr_el1;
 
-       /* SCTLR - System Control Register */
-       sctlr = reg_sctlr_el1_read();
        if (sctlr & SCTLR_I)
                aprint_verbose_dev(self, "IC enabled");
        else
@@ -288,8 +292,8 @@
        /*
         * CTR - Cache Type Register
         */
-       ctr = reg_ctr_el0_read();
-       clidr = reg_clidr_el1_read();
+       const uint64_t ctr = id->ac_ctr;
+       const uint64_t clidr = id->ac_clidr;
        aprint_verbose_dev(self, "Cache Writeback Granule %" PRIu64 "B,"
            " Exclusives Reservation Granule %" PRIu64 "B\n",
            __SHIFTOUT(ctr, CTR_EL0_CWG_LINE) * 4,
@@ -313,22 +317,14 @@
 static void
 cpu_identify2(device_t self, struct cpu_info *ci)
 {
-       struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
-       uint64_t dfr0;
-
-       if (!CPU_IS_PRIMARY(ci)) {
-               cpu_setup_id(ci);
-               cpu_setup_sysctl(self, ci);
-       }
-
-       dfr0 = reg_id_aa64dfr0_el1_read();
+       struct aarch64_sysctl_cpu_id * const id = &ci->ci_id;
 
        aprint_debug_dev(self, "midr=0x%" PRIx32 " mpidr=0x%" PRIx32 "\n",
-           (uint32_t)ci->ci_id.ac_midr, (uint32_t)ci->ci_id.ac_mpidr);
+           (uint32_t)id->ac_midr, (uint32_t)id->ac_mpidr);
        aprint_verbose_dev(self, "revID=0x%" PRIx64, id->ac_revidr);
 
        /* ID_AA64DFR0_EL1 */
-       switch (__SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER)) {
+       switch (__SHIFTOUT(id->ac_aa64dfr0, ID_AA64DFR0_EL1_PMUVER)) {
        case ID_AA64DFR0_EL1_PMUVER_V3:
                aprint_verbose(", PMCv3");
                break;
@@ -501,13 +497,16 @@
 }
 
 /*
- * Fill in this CPUs id data.  Must be called from hatched cpus.
+ * Fill in this CPUs id data.  Must be called on all cpus.
  */
-static void
+void __noasan
 cpu_setup_id(struct cpu_info *ci)
 {
        struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
 
+       /* SCTLR - System Control Register */
+       ci->ci_sctlr_el1 = reg_sctlr_el1_read();
+
        memset(id, 0, sizeof *id);
 
        id->ac_midr      = reg_midr_el1_read();
@@ -611,10 +610,6 @@
 {
        struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
 
-       /* Probably shared between cores.  */
-       if (!CPU_IS_PRIMARY(ci))
-               return;
-
        /* Verify that it is supported.  */
        switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) {
        case ID_AA64ISAR0_EL1_RNDR_RNDRRS:
@@ -676,25 +671,47 @@
 }
 
 #ifdef MULTIPROCESSOR
+/*
+ * Initialise a secondary processor.
+ *
+ * printf isn't available as kmutex(9) relies on curcpu which isn't setup yet.
+ *
+ */
+void __noasan
+cpu_init_secondary_processor(int cpuindex)
+{
+       struct cpu_info * ci = &cpu_info_store[cpuindex];
+       struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
+
+       aarch64_setcpufuncs(ci);
+
+       /* Sets ci->ci_{sctlr,midr,mpidr}, etc */
+       cpu_setup_id(ci);
+
+       arm_cpu_topology_set(ci, id->ac_mpidr);
+       aarch64_getcacheinfo(ci);
+
+       cpu_set_hatched(cpuindex);
+
+       /*
+        * return to assembly to wait for cpu_boot_secondary_processors
+        */
+}
+
+
+/*
+ * When we are called, the MMU and caches are on and we are running on the stack
+ * of the idlelwp for this cpu.
+ */
 void
 cpu_hatch(struct cpu_info *ci)
 {
        KASSERT(curcpu() == ci);
        KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0);
 
-       mutex_enter(&cpu_hatch_lock);
-
-       set_cpufuncs();
-       fpu_attach(ci);
-
-       cpu_identify1(ci->ci_dev, ci);
-       aarch64_getcacheinfo(device_unit(ci->ci_dev));
-       aarch64_printcacheinfo(ci->ci_dev);
-       cpu_identify2(ci->ci_dev, ci);
 #ifdef DDB
-       db_machdep_init();
+       db_machdep_cpu_init();
 #endif
-       mutex_exit(&cpu_hatch_lock);
 
        cpu_init_counter(ci);
 
diff -r 5eb5ac1dea5c -r 40920aaad5c4 sys/arch/aarch64/aarch64/cpufunc.c
--- a/sys/arch/aarch64/aarch64/cpufunc.c        Sun Oct 31 15:32:14 2021 +0000
+++ b/sys/arch/aarch64/aarch64/cpufunc.c        Sun Oct 31 16:23:47 2021 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc.c,v 1.31 2021/10/31 07:56:55 skrll Exp $       */
+/*     $NetBSD: cpufunc.c,v 1.32 2021/10/31 16:23:47 skrll Exp $       */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -30,7 +30,7 @@
 #include "opt_multiprocessor.h"
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.31 2021/10/31 07:56:55 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.32 2021/10/31 16:23:47 skrll Exp $");
 
 #include <sys/param.h>
 #include <sys/types.h>
@@ -53,13 +53,7 @@
 int aarch64_pan_enabled __read_mostly;
 int aarch64_pac_enabled __read_mostly;
 
-/* cache info per cluster. the same cluster has the same cache configuration? */
-#define MAXCPUPACKAGES MAXCPUS         /* maximum of ci->ci_package_id */
-static struct aarch64_cache_info *aarch64_cacheinfo[MAXCPUPACKAGES];
-static struct aarch64_cache_info aarch64_cacheinfo0[MAX_CACHE_LEVEL];
-
-
-static void
+static void __noasan
 extract_cacheunit(int level, bool insn, int cachetype,
     struct aarch64_cache_info *cacheinfo)
 {
@@ -101,35 +95,14 @@
        cunit->cache_size = cunit->cache_way_size * cunit->cache_ways;
 }
 
-void
-aarch64_getcacheinfo(int unit)
+
+/* Must be called on each processor */
+void __noasan
+aarch64_getcacheinfo(struct cpu_info *ci)
 {
-       struct cpu_info * const ci = curcpu();
+       struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo;
        uint32_t clidr, ctr;
-       u_int vindexsize;
        int level, cachetype;
-       struct aarch64_cache_info *cinfo = NULL;
-
-       if (cputype == 0)
-               cputype = aarch64_cpuid();
-
-       /* already extract about this cluster? */