[src/trunk]: src/sys/arch/aarch64/aarch64 aarch64: Add missing barriers in cp...

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/3610dbaaa764
branches:  trunk
changeset: 373665:3610dbaaa764
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Thu Feb 23 14:54:57 2023 +0000

description:
aarch64: Add missing barriers in cpu_switchto.

Details in comments.

Note: This is a conservative change that inserts a barrier where
there was a comment saying none is needed, which is probably correct.
The goal of this change is to systematically add barriers to be
confident in correctness; subsequent changes may remove some bariers,
as an optimization, with an explanation of why each barrier is not
needed.

PR kern/57240

XXX pullup-9
XXX pullup-10

diffstat:

 sys/arch/aarch64/aarch64/cpuswitch.S |  33 ++++++++++++++++++++++++++++-----
 sys/arch/aarch64/aarch64/locore.S    |   9 +++++++--
 2 files changed, 35 insertions(+), 7 deletions(-)

diffs (101 lines):

diff -r 1cad32a8fb7f -r 3610dbaaa764 sys/arch/aarch64/aarch64/cpuswitch.S
--- a/sys/arch/aarch64/aarch64/cpuswitch.S      Thu Feb 23 05:20:45 2023 +0000
+++ b/sys/arch/aarch64/aarch64/cpuswitch.S      Thu Feb 23 14:54:57 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpuswitch.S,v 1.39 2022/09/19 17:23:14 ryo Exp $ */
+/* $NetBSD: cpuswitch.S,v 1.40 2023/02/23 14:54:57 riastradh Exp $ */
 
 /*-
  * Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
 #include "opt_ddb.h"
 #include "opt_kasan.h"
 
-RCSID("$NetBSD: cpuswitch.S,v 1.39 2022/09/19 17:23:14 ryo Exp $")
+RCSID("$NetBSD: cpuswitch.S,v 1.40 2023/02/23 14:54:57 riastradh Exp $")
 
        ARMV8_DEFINE_OPTIONS
 
@@ -125,8 +125,29 @@
 
        msr     tpidr_el1, x1           /* switch curlwp to new lwp */
        ldr     x3, [x1, #L_CPU]
+
+       /*
+        * Issue barriers to coordinate mutex_exit on this CPU with
+        * mutex_vector_enter on another CPU.
+        *
+        * 1. Any prior mutex_exit by oldlwp must be visible to other
+        *    CPUs before we set ci_curlwp := newlwp on this one,
+        *    requiring a store-before-store barrier.
+        *
+        * 2. ci_curlwp := newlwp must be visible on all other CPUs
+        *    before any subsequent mutex_exit by newlwp can even test
+        *    whether there might be waiters, requiring a
+        *    store-before-load barrier.
+        *
+        * See kern_mutex.c for details -- this is necessary for
+        * adaptive mutexes to detect whether the lwp is on the CPU in
+        * order to safely block without requiring atomic r/m/w in
+        * mutex_exit.
+        */
+       dmb     ishst                   /* store-before-store */
        str     x1, [x3, #CI_CURLWP]    /* switch curlwp to new lwp */
-       dmb     ishst                   /* see comments in kern_mutex.c */
+       dmb     ish                     /* store-before-load */
+
        ENABLE_INTERRUPT
 
        /*
@@ -201,8 +222,9 @@
        /* onto new stack */
        sub     sp, x4, #TF_SIZE        /* new sp := softlwp->l_md_utf - 1 */
        msr     tpidr_el1, x0           /* curlwp = softlwp; */
+       dmb     ishst                   /* for mutex_enter; see cpu_switchto */
        str     x0, [x20, #CI_CURLWP]   /* curcpu()->ci_curlwp = softlwp; */
-                                       /* no need for memory barrier here */
+       dmb     ish                     /* for mutex_enter; see cpu_switchto */
 
        mov     x5, #CPACR_FPEN_NONE
        msr     cpacr_el1, x5           /* cpacr_el1 = CPACR_FPEN_NONE */
@@ -244,8 +266,9 @@
        DISABLE_INTERRUPT
        msr     tpidr_el1, x19          /* curlwp = pinned_lwp */
        ldr     x3, [x19, #L_CPU]       /* x3 = curlwp->l_cpu */
+       dmb     ishst                   /* for mutex_enter; see cpu_switchto */
        str     x19, [x3, #CI_CURLWP]   /* curlwp->l_cpu->ci_curlwp := x19 */
-       dmb     ishst                   /* see comments in kern_mutex.c */
+       dmb     ish                     /* for mutex_enter; see cpu_switchto */
 
        mov     sp, x4                  /* restore pinned_lwp sp */
        msr     cpacr_el1, x5           /* restore pinned_lwp cpacr */
diff -r 1cad32a8fb7f -r 3610dbaaa764 sys/arch/aarch64/aarch64/locore.S
--- a/sys/arch/aarch64/aarch64/locore.S Thu Feb 23 05:20:45 2023 +0000
+++ b/sys/arch/aarch64/aarch64/locore.S Thu Feb 23 14:54:57 2023 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: locore.S,v 1.90 2023/02/17 06:24:26 skrll Exp $        */
+/*     $NetBSD: locore.S,v 1.91 2023/02/23 14:54:57 riastradh Exp $    */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -38,7 +38,7 @@
 #include <aarch64/hypervisor.h>
 #include "assym.h"
 
-RCSID("$NetBSD: locore.S,v 1.90 2023/02/17 06:24:26 skrll Exp $")
+RCSID("$NetBSD: locore.S,v 1.91 2023/02/23 14:54:57 riastradh Exp $")
 
 #ifdef AARCH64_DEVICE_MEM_NONPOSTED
 #define        MAIR_DEVICE_MEM         MAIR_DEVICE_nGnRnE
@@ -529,6 +529,11 @@
         */
        ldr     x1, [x0, #CI_IDLELWP]   /* x0 = curcpu()->ci_idlelwp */
        msr     tpidr_el1, x1           /* tpidr_el1 = curlwp = x1 */
+       /*
+        * No membar needed because we're not switching from a
+        * previous lwp, and the idle lwp we're switching to can't be
+        * holding locks already; see cpu_switchto.
+        */
        str     x1, [x0, #CI_CURLWP]    /* curlwp is idlelwp */
 
        /* get my stack from lwp */