[src/trunk]: src/sys Use __insn_barrier to enforce ordering in l_ncsw loops.

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/dff0f84fd722
branches:  trunk
changeset: 846890:dff0f84fd722
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Tue Dec 03 15:20:59 2019 +0000

description:
Use __insn_barrier to enforce ordering in l_ncsw loops.

(Only need ordering observable by interruption, not by other CPUs.)

diffstat:

 sys/arch/arm/arm/arm_machdep.c     |  6 ++++--
 sys/arch/mips/mips/cpu_subr.c      |  6 ++++--
 sys/arch/sparc/sparc/intr.c        |  6 ++++--
 sys/arch/sparc64/sparc64/machdep.c |  6 ++++--
 sys/arch/usermode/dev/cpu.c        |  6 ++++--
 sys/arch/x86/x86/pmap.c            |  6 ++++--
 sys/arch/x86/x86/x86_machdep.c     |  8 +++++---
 sys/kern/kern_lock.c               |  6 ++++--
 sys/kern/subr_pool.c               |  9 +++++++--
 9 files changed, 40 insertions(+), 19 deletions(-)

diffs (300 lines):

diff -r 941c6692c28d -r dff0f84fd722 sys/arch/arm/arm/arm_machdep.c
--- a/sys/arch/arm/arm/arm_machdep.c    Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/arch/arm/arm/arm_machdep.c    Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: arm_machdep.c,v 1.57 2019/12/01 14:52:13 ad Exp $      */
+/*     $NetBSD: arm_machdep.c,v 1.58 2019/12/03 15:20:59 riastradh Exp $       */
 
 /*
  * Copyright (c) 2001 Wasabi Systems, Inc.
@@ -80,7 +80,7 @@
 
 #include <sys/param.h>
 
-__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.57 2019/12/01 14:52:13 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.58 2019/12/03 15:20:59 riastradh Exp $");
 
 #include <sys/exec.h>
 #include <sys/proc.h>
@@ -270,10 +270,12 @@
        l = curlwp;
        do {
                ncsw = l->l_ncsw;
+               __insn_barrier();
                idepth = l->l_cpu->ci_intr_depth;
 #ifdef __HAVE_PIC_FAST_SOFTINTS
                cpl = ci->ci_cpl;
 #endif
+               __insn_barrier();
        } while (__predict_false(ncsw != l->l_ncsw));
 
 #ifdef __HAVE_PIC_FAST_SOFTINTS
diff -r 941c6692c28d -r dff0f84fd722 sys/arch/mips/mips/cpu_subr.c
--- a/sys/arch/mips/mips/cpu_subr.c     Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/arch/mips/mips/cpu_subr.c     Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $ */
+/*     $NetBSD: cpu_subr.c,v 1.40 2019/12/03 15:20:59 riastradh Exp $  */
 
 /*-
  * Copyright (c) 2010, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.40 2019/12/03 15:20:59 riastradh Exp $");
 
 #include "opt_cputype.h"
 #include "opt_ddb.h"
@@ -610,7 +610,9 @@
        l = curlwp;
        do {
                ncsw = l->l_ncsw;
+               __insn_barrier();
                idepth = l->l_cpu->ci_idepth;
+               __insn_barrier();
        } while (__predict_false(ncsw != l->l_ncsw));
 
        return idepth != 0;
diff -r 941c6692c28d -r dff0f84fd722 sys/arch/sparc/sparc/intr.c
--- a/sys/arch/sparc/sparc/intr.c       Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/arch/sparc/sparc/intr.c       Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: intr.c,v 1.122 2019/12/01 14:52:14 ad Exp $ */
+/*     $NetBSD: intr.c,v 1.123 2019/12/03 15:20:59 riastradh Exp $ */
 
 /*
  * Copyright (c) 1992, 1993
@@ -41,7 +41,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.122 2019/12/01 14:52:14 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.123 2019/12/03 15:20:59 riastradh Exp $");
 
 #include "opt_multiprocessor.h"
 #include "opt_sparc_arch.h"
@@ -896,7 +896,9 @@
        l = curlwp;
        do {
                ncsw = l->l_ncsw;
+               __insn_barrier();
                idepth = l->l_cpu->ci_idepth;
+               __insn_barrier();
        } while (__predict_false(ncsw != l->l_ncsw));
 
        return idepth != 0;
diff -r 941c6692c28d -r dff0f84fd722 sys/arch/sparc64/sparc64/machdep.c
--- a/sys/arch/sparc64/sparc64/machdep.c        Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/arch/sparc64/sparc64/machdep.c        Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: machdep.c,v 1.294 2019/12/01 14:52:14 ad Exp $ */
+/*     $NetBSD: machdep.c,v 1.295 2019/12/03 15:20:59 riastradh Exp $ */
 
 /*-
  * Copyright (c) 1996, 1997, 1998, 2019 The NetBSD Foundation, Inc.
@@ -71,7 +71,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.294 2019/12/01 14:52:14 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.295 2019/12/03 15:20:59 riastradh Exp $");
 
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
@@ -2661,7 +2661,9 @@
        l = curlwp;
        do {
                ncsw = l->l_ncsw;
+               __insn_barrier();
                idepth = l->l_cpu->ci_idepth;
+               __insn_barrier();
        } while (__predict_false(ncsw != l->l_ncsw));
 
        return idepth >= 0;
diff -r 941c6692c28d -r dff0f84fd722 sys/arch/usermode/dev/cpu.c
--- a/sys/arch/usermode/dev/cpu.c       Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/arch/usermode/dev/cpu.c       Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.82 2019/12/01 14:52:14 ad Exp $ */
+/* $NetBSD: cpu.c,v 1.83 2019/12/03 15:20:59 riastradh Exp $ */
 
 /*-
  * Copyright (c) 2007 Jared D. McNeill <jmcneill%invisible.ca@localhost>
@@ -30,7 +30,7 @@
 #include "opt_hz.h"
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.82 2019/12/01 14:52:14 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.83 2019/12/03 15:20:59 riastradh Exp $");
 
 #include <sys/param.h>
 #include <sys/conf.h>
@@ -535,7 +535,9 @@
        l = curlwp;
        do {
                ncsw = l->l_ncsw;
+               __insn_barrier();
                idepth = l->l_cpu->ci_idepth;
+               __insn_barrier();
        } while (__predict_false(ncsw != l->l_ncsw));
 
        return idepth >= 0;
diff -r 941c6692c28d -r dff0f84fd722 sys/arch/x86/x86/pmap.c
--- a/sys/arch/x86/x86/pmap.c   Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/arch/x86/x86/pmap.c   Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.c,v 1.341 2019/11/16 10:19:29 maxv Exp $  */
+/*     $NetBSD: pmap.c,v 1.342 2019/12/03 15:20:59 riastradh Exp $     */
 
 /*
  * Copyright (c) 2008, 2010, 2016, 2017 The NetBSD Foundation, Inc.
@@ -130,7 +130,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.341 2019/11/16 10:19:29 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.342 2019/12/03 15:20:59 riastradh Exp $");
 
 #include "opt_user_ldt.h"
 #include "opt_lockdebug.h"
@@ -2893,6 +2893,7 @@
        }
        l = ci->ci_curlwp;
        ncsw = l->l_ncsw;
+       __insn_barrier();
 
        /* should be able to take ipis. */
        KASSERT(ci->ci_ilevel < IPL_HIGH);
@@ -2974,6 +2975,7 @@
         */
 
        pmap_destroy(oldpmap);
+       __insn_barrier();
        if (l->l_ncsw != ncsw) {
                goto retry;
        }
diff -r 941c6692c28d -r dff0f84fd722 sys/arch/x86/x86/x86_machdep.c
--- a/sys/arch/x86/x86/x86_machdep.c    Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/arch/x86/x86/x86_machdep.c    Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: x86_machdep.c,v 1.132 2019/12/03 11:50:45 hannken Exp $        */
+/*     $NetBSD: x86_machdep.c,v 1.133 2019/12/03 15:20:59 riastradh Exp $      */
 
 /*-
  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.132 2019/12/03 11:50:45 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.133 2019/12/03 15:20:59 riastradh Exp $");
 
 #include "opt_modular.h"
 #include "opt_physmem.h"
@@ -350,7 +350,7 @@
 cpu_intr_p(void)
 {
        uint64_t ncsw;
-       volatile int idepth;
+       int idepth;
        lwp_t *l;
 
        l = curlwp;
@@ -360,7 +360,9 @@
        }
        do {
                ncsw = l->l_ncsw;
+               __insn_barrier();
                idepth = l->l_cpu->ci_idepth;
+               __insn_barrier();
        } while (__predict_false(ncsw != l->l_ncsw));
 
        return idepth >= 0;
diff -r 941c6692c28d -r dff0f84fd722 sys/kern/kern_lock.c
--- a/sys/kern/kern_lock.c      Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/kern/kern_lock.c      Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: kern_lock.c,v 1.163 2019/05/09 05:00:31 ozaki-r Exp $  */
+/*     $NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $        */
 
 /*-
  * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.163 2019/05/09 05:00:31 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $");
 
 #include <sys/param.h>
 #include <sys/proc.h>
@@ -75,7 +75,9 @@
         */
        do {
                pctr = lwp_pctr();
+               __insn_barrier();
                idle = CURCPU_IDLE_P();
+               __insn_barrier();
        } while (pctr != lwp_pctr());
 
        reason = NULL;
diff -r 941c6692c28d -r dff0f84fd722 sys/kern/subr_pool.c
--- a/sys/kern/subr_pool.c      Tue Dec 03 15:03:14 2019 +0000
+++ b/sys/kern/subr_pool.c      Tue Dec 03 15:20:59 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_pool.c,v 1.262 2019/11/14 16:23:53 maxv Exp $     */
+/*     $NetBSD: subr_pool.c,v 1.263 2019/12/03 15:20:59 riastradh Exp $        */
 
 /*
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018
@@ -33,7 +33,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.262 2019/11/14 16:23:53 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.263 2019/12/03 15:20:59 riastradh Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ddb.h"
@@ -2460,6 +2460,7 @@
         */
        if (__predict_false(!mutex_tryenter(&pc->pc_lock))) {
                ncsw = curlwp->l_ncsw;
+               __insn_barrier();
                mutex_enter(&pc->pc_lock);
                pc->pc_contended++;
 
@@ -2468,6 +2469,7 @@
                 * our view of the per-CPU data is invalid:
                 * retry.
                 */
+               __insn_barrier();
                if (curlwp->l_ncsw != ncsw) {
                        mutex_exit(&pc->pc_lock);
                        return true;
@@ -2625,6 +2627,7 @@
        pcg = NULL;
        cc->cc_misses++;
        ncsw = l->l_ncsw;
+       __insn_barrier();
 
        /*
         * If there are no empty groups in the cache then allocate one
@@ -2638,6 +2641,7 @@
                 * If pool_get() blocked, then our view of
                 * the per-CPU data is invalid: retry.
                 */
+               __insn_barrier();
                if (__predict_false(l->l_ncsw != ncsw)) {
                        if (pcg != NULL) {
                                pool_put(pc->pc_pcgpool, pcg);
@@ -2659,6 +2663,7 @@
                 * If we context switched while locking, then our view of
                 * the per-CPU data is invalid: retry.
                 */
+               __insn_barrier();
                if (__predict_false(l->l_ncsw != ncsw)) {
                        mutex_exit(&pc->pc_lock);
                        if (pcg != NULL) {