Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/arch Implement gic_splraise and the gic_splx fast path i...
details: https://anonhg.NetBSD.org/src/rev/6b6da8049cc0
branches: trunk
changeset: 990587:6b6da8049cc0
user: jmcneill <jmcneill%NetBSD.org@localhost>
date: Sat Oct 30 18:44:24 2021 +0000
description:
Implement gic_splraise and the gic_splx fast path in asm (armv8).
diffstat:
sys/arch/aarch64/conf/files.aarch64 | 5 +-
sys/arch/arm/cortex/gic_splfuncs.c | 56 ++------------
sys/arch/arm/cortex/gic_splfuncs_armv8.S | 118 +++++++++++++++++++++++++++++++
3 files changed, 132 insertions(+), 47 deletions(-)
diffs (239 lines):
diff -r f68bee29a053 -r 6b6da8049cc0 sys/arch/aarch64/conf/files.aarch64
--- a/sys/arch/aarch64/conf/files.aarch64 Sat Oct 30 18:23:17 2021 +0000
+++ b/sys/arch/aarch64/conf/files.aarch64 Sat Oct 30 18:44:24 2021 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: files.aarch64,v 1.34 2021/10/10 07:15:25 skrll Exp $
+# $NetBSD: files.aarch64,v 1.35 2021/10/30 18:44:24 jmcneill Exp $
defflag opt_cpuoptions.h AARCH64_ALIGNMENT_CHECK
defflag opt_cpuoptions.h AARCH64_EL0_STACK_ALIGNMENT_CHECK
@@ -119,6 +119,9 @@
file uvm/pmap/pmap_tlb.c
file uvm/pmap/pmap_pvt.c
+# GIC
+file arch/arm/cortex/gic_splfuncs_armv8.S gic_splfuncs
+
# EFI runtime (machdep)
file arch/aarch64/aarch64/efi_machdep.c efi_runtime
diff -r f68bee29a053 -r 6b6da8049cc0 sys/arch/arm/cortex/gic_splfuncs.c
--- a/sys/arch/arm/cortex/gic_splfuncs.c Sat Oct 30 18:23:17 2021 +0000
+++ b/sys/arch/arm/cortex/gic_splfuncs.c Sat Oct 30 18:44:24 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: gic_splfuncs.c,v 1.4 2021/09/26 20:55:15 jmcneill Exp $ */
+/* $NetBSD: gic_splfuncs.c,v 1.5 2021/10/30 18:44:24 jmcneill Exp $ */
/*-
* Copyright (c) 2021 Jared McNeill <jmcneill%invisible.ca@localhost>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.4 2021/09/26 20:55:15 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.5 2021/10/30 18:44:24 jmcneill Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -41,16 +41,12 @@
#include <arm/cortex/gic_splfuncs.h>
-static int
-gic_splraise(int newipl)
-{
- struct cpu_info * const ci = curcpu();
- const int oldipl = ci->ci_cpl;
- if (__predict_true(newipl > oldipl)) {
- ci->ci_cpl = newipl;
- }
- return oldipl;
-}
+/* Prototypes for functions in gic_splfuncs_<arch>.S */
+int gic_splraise(int);
+void gic_splx(int);
+
+/* Local functions */
+void Xgic_splx(int);
static int
gic_spllower(int newipl)
@@ -72,8 +68,8 @@
return oldipl;
}
-static void
-gic_splx(int newipl)
+void
+Xgic_splx(int newipl)
{
struct cpu_info *ci = curcpu();
register_t psw;
@@ -82,37 +78,6 @@
return;
}
- /*
- * Try to avoid touching any hardware registers (DAIF, PMR) as an
- * optimization for the common case of splraise followed by splx
- * with no interrupts in between.
- *
- * If an interrupt fires in this critical section, the vector
- * handler is responsible for returning to the address pointed
- * to by ci_splx_restart to restart the sequence.
- */
- if (__predict_true(ci->ci_intr_depth == 0)) {
- ci->ci_splx_savedipl = newipl;
- __insn_barrier();
- ci->ci_splx_restart = &&restart;
- __insn_barrier();
-checkhwpl:
- if (ci->ci_hwpl <= newipl) {
- ci->ci_cpl = newipl;
- __insn_barrier();
- ci->ci_splx_restart = NULL;
- goto dosoft;
- } else {
- ci->ci_splx_restart = NULL;
- goto dohard;
- }
-restart:
- ci = curcpu();
- newipl = ci->ci_splx_savedipl;
- goto checkhwpl;
- }
-
-dohard:
psw = DISABLE_INTERRUPT_SAVE();
ci->ci_intr_depth++;
pic_do_pending_ints(psw, newipl, NULL);
@@ -121,7 +86,6 @@
ENABLE_INTERRUPT();
}
-dosoft:
cpu_dosoftints();
}
diff -r f68bee29a053 -r 6b6da8049cc0 sys/arch/arm/cortex/gic_splfuncs_armv8.S
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/arch/arm/cortex/gic_splfuncs_armv8.S Sat Oct 30 18:44:24 2021 +0000
@@ -0,0 +1,118 @@
+/* $NetBSD: gic_splfuncs_armv8.S,v 1.1 2021/10/30 18:44:24 jmcneill Exp $ */
+
+/*-
+ * Copyright (c) 2021 Jared McNeill <jmcneill%invisible.ca@localhost>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+#include "assym.h"
+
+RCSID("$NetBSD: gic_splfuncs_armv8.S,v 1.1 2021/10/30 18:44:24 jmcneill Exp $")
+
+/*
+ * int
+ * gic_splraise(int newipl)
+ *
+ * w0 = newipl
+ */
+ .align 7 /* cacheline-aligned */
+ENTRY_NP(gic_splraise)
+ /* Save cpu_info pointer in x1 */
+ mrs x1, tpidr_el1 /* get curlwp */
+ ldr x1, [x1, #L_CPU] /* get curcpu */
+
+ /* If newipl > cpl, update cpl */
+ ldr w2, [x1, #CI_CPL]
+ cmp w0, w2
+ b.le .Lnoraise
+ str w0, [x1, #CI_CPL]
+
+.Lnoraise:
+ mov w0, w2 /* return oldipl */
+ ret
+END(gic_splraise)
+
+
+/*
+ * void
+ * gic_splx(int newipl)
+ *
+ * w0 = newipl
+ */
+ .align 7 /* cacheline-aligned */
+ENTRY_NP(gic_splx)
+ /* Save cpu_info pointer in x1 */
+ mrs x1, tpidr_el1 /* get curlwp */
+ ldr x1, [x1, #L_CPU] /* get curcpu */
+
+ /* If newipl >= cpl, just return */
+ ldr w2, [x1, #CI_CPL]
+ cmp w0, w2
+ b.hs .Ldone
+
+.Lagain:
+ /* Slow path if ci_intr_depth != 0 */
+ ldr w2, [x1, #CI_INTR_DEPTH]
+ cbnz w2, .Lslow
+
+ /* Save newipl and restart address in cpu info */
+ str w0, [x1, #CI_SPLX_SAVEDIPL]
+ adr x2, .Lrestart
+ str x2, [x1, #CI_SPLX_RESTART]
+
+ /* Slow path if hwpl > newipl */
+ ldr w2, [x1, #CI_HWPL]
+ cmp w2, w0
+ b.hi .Lrestore
+
+ /* Update cpl */
+ str w0, [x1, #CI_CPL]
+
+ /* Clear saved restart address from cpu info */
+ str xzr, [x1, #CI_SPLX_RESTART]
+
+ /* Check for pending softints */
+ ldr w2, [x1, #CI_SOFTINTS]
+ lsr w2, w2, w0
+ cbnz w2, _C_LABEL(dosoftints)
+
+.Ldone:
+ ret
+
+.Lrestart:
+ /* Reload w0 and x1 */
+ mrs x1, tpidr_el1 /* get curlwp */
+ ldr x1, [x1, #L_CPU] /* get curcpu */
+ ldr w0, [x1, #CI_SPLX_SAVEDIPL] /* get newipl */
+ b .Lagain
+
+.Lrestore:
+ /* Clear saved restart address from cpu info */
+ str xzr, [x1, #CI_SPLX_RESTART]
+
+.Lslow:
+ /* Jump to slow path */
+ b _C_LABEL(Xgic_splx)
+END(gic_splx)
Home |
Main Index |
Thread Index |
Old Index