Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/arch Implement gic_splraise and the gic_splx fast path i...



details:   https://anonhg.NetBSD.org/src/rev/ff077db7d647
branches:  trunk
changeset: 1024666:ff077db7d647
user:      jmcneill <jmcneill%NetBSD.org@localhost>
date:      Sat Oct 30 18:44:24 2021 +0000

description:
Implement gic_splraise and the gic_splx fast path in asm (armv8).

diffstat:

 sys/arch/aarch64/conf/files.aarch64      |    5 +-
 sys/arch/arm/cortex/gic_splfuncs.c       |   56 ++------------
 sys/arch/arm/cortex/gic_splfuncs_armv8.S |  118 +++++++++++++++++++++++++++++++
 3 files changed, 132 insertions(+), 47 deletions(-)

diffs (239 lines):

diff -r bfcd9bbfe5ad -r ff077db7d647 sys/arch/aarch64/conf/files.aarch64
--- a/sys/arch/aarch64/conf/files.aarch64       Sat Oct 30 18:23:17 2021 +0000
+++ b/sys/arch/aarch64/conf/files.aarch64       Sat Oct 30 18:44:24 2021 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: files.aarch64,v 1.34 2021/10/10 07:15:25 skrll Exp $
+#      $NetBSD: files.aarch64,v 1.35 2021/10/30 18:44:24 jmcneill Exp $
 
 defflag opt_cpuoptions.h       AARCH64_ALIGNMENT_CHECK
 defflag opt_cpuoptions.h       AARCH64_EL0_STACK_ALIGNMENT_CHECK
@@ -119,6 +119,9 @@
 file   uvm/pmap/pmap_tlb.c
 file   uvm/pmap/pmap_pvt.c
 
+# GIC
+file   arch/arm/cortex/gic_splfuncs_armv8.S            gic_splfuncs
+
 # EFI runtime (machdep)
 file   arch/aarch64/aarch64/efi_machdep.c              efi_runtime
 
diff -r bfcd9bbfe5ad -r ff077db7d647 sys/arch/arm/cortex/gic_splfuncs.c
--- a/sys/arch/arm/cortex/gic_splfuncs.c        Sat Oct 30 18:23:17 2021 +0000
+++ b/sys/arch/arm/cortex/gic_splfuncs.c        Sat Oct 30 18:44:24 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: gic_splfuncs.c,v 1.4 2021/09/26 20:55:15 jmcneill Exp $ */
+/* $NetBSD: gic_splfuncs.c,v 1.5 2021/10/30 18:44:24 jmcneill Exp $ */
 
 /*-
  * Copyright (c) 2021 Jared McNeill <jmcneill%invisible.ca@localhost>
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.4 2021/09/26 20:55:15 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.5 2021/10/30 18:44:24 jmcneill Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -41,16 +41,12 @@
 
 #include <arm/cortex/gic_splfuncs.h>
 
-static int
-gic_splraise(int newipl)
-{
-       struct cpu_info * const ci = curcpu();
-       const int oldipl = ci->ci_cpl;
-       if (__predict_true(newipl > oldipl)) {
-               ci->ci_cpl = newipl;
-       }
-       return oldipl;
-}
+/* Prototypes for functions in gic_splfuncs_<arch>.S */
+int    gic_splraise(int);
+void   gic_splx(int);
+
+/* Local functions */
+void   Xgic_splx(int); 
 
 static int
 gic_spllower(int newipl)
@@ -72,8 +68,8 @@
        return oldipl;
 }
 
-static void
-gic_splx(int newipl)
+void
+Xgic_splx(int newipl)
 {
        struct cpu_info *ci = curcpu();
        register_t psw;
@@ -82,37 +78,6 @@
                return;
        }
 
-       /*
-        * Try to avoid touching any hardware registers (DAIF, PMR) as an
-        * optimization for the common case of splraise followed by splx
-        * with no interrupts in between.
-        *
-        * If an interrupt fires in this critical section, the vector
-        * handler is responsible for returning to the address pointed
-        * to by ci_splx_restart to restart the sequence.
-        */
-       if (__predict_true(ci->ci_intr_depth == 0)) {
-               ci->ci_splx_savedipl = newipl;
-               __insn_barrier();
-               ci->ci_splx_restart = &&restart;
-               __insn_barrier();
-checkhwpl:
-               if (ci->ci_hwpl <= newipl) {
-                       ci->ci_cpl = newipl;
-                       __insn_barrier();
-                       ci->ci_splx_restart = NULL;
-                       goto dosoft;
-               } else {
-                       ci->ci_splx_restart = NULL;
-                       goto dohard;
-               }
-restart:
-               ci = curcpu();
-               newipl = ci->ci_splx_savedipl;
-               goto checkhwpl;
-       }
-
-dohard:
        psw = DISABLE_INTERRUPT_SAVE();
        ci->ci_intr_depth++;
        pic_do_pending_ints(psw, newipl, NULL);
@@ -121,7 +86,6 @@
                ENABLE_INTERRUPT();
        }
 
-dosoft:
        cpu_dosoftints();
 }
 
diff -r bfcd9bbfe5ad -r ff077db7d647 sys/arch/arm/cortex/gic_splfuncs_armv8.S
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/arch/arm/cortex/gic_splfuncs_armv8.S  Sat Oct 30 18:44:24 2021 +0000
@@ -0,0 +1,118 @@
+/* $NetBSD: gic_splfuncs_armv8.S,v 1.1 2021/10/30 18:44:24 jmcneill Exp $ */
+
+/*-
+ * Copyright (c) 2021 Jared McNeill <jmcneill%invisible.ca@localhost>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+#include "assym.h"
+
+RCSID("$NetBSD: gic_splfuncs_armv8.S,v 1.1 2021/10/30 18:44:24 jmcneill Exp $")
+
+/*
+ * int
+ * gic_splraise(int newipl)
+ *
+ *     w0 = newipl
+ */
+       .align 7        /* cacheline-aligned */
+ENTRY_NP(gic_splraise)
+       /* Save cpu_info pointer in x1 */
+       mrs     x1, tpidr_el1                   /* get curlwp */
+       ldr     x1, [x1, #L_CPU]                /* get curcpu */
+
+       /* If newipl > cpl, update cpl */
+       ldr     w2, [x1, #CI_CPL]
+       cmp     w0, w2
+       b.le    .Lnoraise
+       str     w0, [x1, #CI_CPL]
+
+.Lnoraise:
+       mov     w0, w2                          /* return oldipl */
+       ret
+END(gic_splraise)
+
+
+/*
+ * void
+ * gic_splx(int newipl)
+ *
+ *     w0 = newipl
+ */
+       .align 7        /* cacheline-aligned */
+ENTRY_NP(gic_splx)
+       /* Save cpu_info pointer in x1 */
+       mrs     x1, tpidr_el1                   /* get curlwp */
+       ldr     x1, [x1, #L_CPU]                /* get curcpu */
+
+       /* If newipl >= cpl, just return */
+       ldr     w2, [x1, #CI_CPL]
+       cmp     w0, w2
+       b.hs    .Ldone
+
+.Lagain:
+       /* Slow path if ci_intr_depth != 0 */
+       ldr     w2, [x1, #CI_INTR_DEPTH]
+       cbnz    w2, .Lslow
+
+       /* Save newipl and restart address in cpu info */
+       str     w0, [x1, #CI_SPLX_SAVEDIPL]
+       adr     x2, .Lrestart
+       str     x2, [x1, #CI_SPLX_RESTART]
+
+       /* Slow path if hwpl > newipl */
+       ldr     w2, [x1, #CI_HWPL]
+       cmp     w2, w0
+       b.hi    .Lrestore
+
+       /* Update cpl */
+       str     w0, [x1, #CI_CPL]
+
+       /* Clear saved restart address from cpu info */
+       str     xzr, [x1, #CI_SPLX_RESTART]
+
+       /* Check for pending softints */
+       ldr     w2, [x1, #CI_SOFTINTS]
+       lsr     w2, w2, w0
+       cbnz    w2, _C_LABEL(dosoftints)
+
+.Ldone:
+       ret
+
+.Lrestart:
+       /* Reload w0 and x1 */
+       mrs     x1, tpidr_el1                   /* get curlwp */
+       ldr     x1, [x1, #L_CPU]                /* get curcpu */
+       ldr     w0, [x1, #CI_SPLX_SAVEDIPL]     /* get newipl */
+       b       .Lagain
+       
+.Lrestore:
+       /* Clear saved restart address from cpu info */
+       str     xzr, [x1, #CI_SPLX_RESTART]
+
+.Lslow:
+       /* Jump to slow path */
+       b       _C_LABEL(Xgic_splx)
+END(gic_splx)



Home | Main Index | Thread Index | Old Index