Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/sys New atomic load/store operations for the kernel.



details:   https://anonhg.NetBSD.org/src/rev/f3fbfa4084ea
branches:  trunk
changeset: 967094:f3fbfa4084ea
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Fri Nov 29 22:17:23 2019 +0000

description:
New atomic load/store operations for the kernel.

Guarantee no fusing and no tearing, and can optionally impose
ordering relative to other memory operations.

Unordered:
- atomic_load_relaxed
- atomic_store_relaxed

Ordered:
- atomic_load_acquire
- atomic_load_consume
- atomic_store_release

These are intended to match C11 semantics, and can be defined in
terms of the C11 atomic API when ready.

diffstat:

 distrib/sets/lists/comp/mi        |   20 +-
 share/man/man9/Makefile           |    9 +-
 share/man/man9/atomic_loadstore.9 |  744 ++++++++++++++++++++++++++++++++++++++
 sys/sys/atomic.h                  |  107 +++++-
 4 files changed, 877 insertions(+), 3 deletions(-)

diffs (truncated from 938 to 300 lines):

diff -r 45effdfda038 -r f3fbfa4084ea distrib/sets/lists/comp/mi
--- a/distrib/sets/lists/comp/mi        Fri Nov 29 20:54:17 2019 +0000
+++ b/distrib/sets/lists/comp/mi        Fri Nov 29 22:17:23 2019 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: mi,v 1.2293 2019/11/29 20:31:35 riastradh Exp $
+#      $NetBSD: mi,v 1.2294 2019/11/29 22:17:23 riastradh Exp $
 #
 # Note: don't delete entries from here - mark them as "obsolete" instead.
 ./etc/mtree/set.comp                           comp-sys-root
@@ -10704,6 +10704,12 @@
 ./usr/share/man/cat9/arp_ifinit.0              comp-sys-catman         .cat
 ./usr/share/man/cat9/arpintr.0                 comp-sys-catman         .cat
 ./usr/share/man/cat9/arpresolve.0              comp-sys-catman         .cat
+./usr/share/man/cat9/atomic_load_acquire.0     comp-sys-catman         .cat
+./usr/share/man/cat9/atomic_load_consume.0     comp-sys-catman         .cat
+./usr/share/man/cat9/atomic_load_relaxed.0     comp-sys-catman         .cat
+./usr/share/man/cat9/atomic_loadstore.0                comp-sys-catman         .cat
+./usr/share/man/cat9/atomic_store_relaxed.0    comp-sys-catman         .cat
+./usr/share/man/cat9/atomic_store_release.0    comp-sys-catman         .cat
 ./usr/share/man/cat9/atop.0                    comp-sys-catman         .cat
 ./usr/share/man/cat9/audio.0                   comp-sys-catman         .cat
 ./usr/share/man/cat9/audio_system.0            comp-sys-catman         obsolete
@@ -18652,6 +18658,12 @@
 ./usr/share/man/html9/arp_ifinit.html          comp-sys-htmlman        html
 ./usr/share/man/html9/arpintr.html             comp-sys-htmlman        html
 ./usr/share/man/html9/arpresolve.html          comp-sys-htmlman        html
+./usr/share/man/html9/atomic_load_acquire.html comp-sys-htmlman        html
+./usr/share/man/html9/atomic_load_consume.html comp-sys-htmlman        html
+./usr/share/man/html9/atomic_load_relaxed.html comp-sys-htmlman        html
+./usr/share/man/html9/atomic_loadstore.html    comp-sys-htmlman        html
+./usr/share/man/html9/atomic_store_relaxed.html        comp-sys-htmlman        html
+./usr/share/man/html9/atomic_store_release.html        comp-sys-htmlman        html
 ./usr/share/man/html9/atop.html                        comp-sys-htmlman        html
 ./usr/share/man/html9/audio.html               comp-sys-htmlman        html
 ./usr/share/man/html9/audio_system.html                comp-sys-htmlman        obsolete
@@ -26703,6 +26715,12 @@
 ./usr/share/man/man9/arp_ifinit.9              comp-sys-man            .man
 ./usr/share/man/man9/arpintr.9                 comp-sys-man            .man
 ./usr/share/man/man9/arpresolve.9              comp-sys-man            .man
+./usr/share/man/man9/atomic_load_acquire.9     comp-sys-man            .man
+./usr/share/man/man9/atomic_load_consume.9     comp-sys-man            .man
+./usr/share/man/man9/atomic_load_relaxed.9     comp-sys-man            .man
+./usr/share/man/man9/atomic_loadstore.9                comp-sys-man            .man
+./usr/share/man/man9/atomic_store_relaxed.9    comp-sys-man            .man
+./usr/share/man/man9/atomic_store_release.9    comp-sys-man            .man
 ./usr/share/man/man9/atop.9                    comp-sys-man            .man
 ./usr/share/man/man9/audio.9                   comp-sys-man            .man
 ./usr/share/man/man9/audio_system.9            comp-sys-man            obsolete
diff -r 45effdfda038 -r f3fbfa4084ea share/man/man9/Makefile
--- a/share/man/man9/Makefile   Fri Nov 29 20:54:17 2019 +0000
+++ b/share/man/man9/Makefile   Fri Nov 29 22:17:23 2019 +0000
@@ -1,4 +1,4 @@
-#       $NetBSD: Makefile,v 1.442 2019/11/29 20:31:35 riastradh Exp $
+#       $NetBSD: Makefile,v 1.443 2019/11/29 22:17:23 riastradh Exp $
 
 #      Makefile for section 9 (kernel function and variable) manual pages.
 
@@ -67,6 +67,13 @@
        wsbell.9 wscons.9 wsdisplay.9 wsfont.9 wskbd.9 wsmouse.9 \
        xcall.9
 
+MAN+=  atomic_loadstore.9
+MLINKS+=atomic_loadstore.9 atomic_load_acquire.9 \
+       atomic_loadstore.9 atomic_load_consume.9 \
+       atomic_loadstore.9 atomic_load_relaxed.9 \
+       atomic_loadstore.9 atomic_store_relaxed.9 \
+       atomic_loadstore.9 atomic_store_release.9
+
 MAN+=  boothowto.9
 MLINKS+=boothowto.9 BOOT_FLAG.9
 
diff -r 45effdfda038 -r f3fbfa4084ea share/man/man9/atomic_loadstore.9
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/share/man/man9/atomic_loadstore.9 Fri Nov 29 22:17:23 2019 +0000
@@ -0,0 +1,744 @@
+.\"    $NetBSD: atomic_loadstore.9,v 1.1 2019/11/29 22:17:23 riastradh Exp $
+.\"
+.\" Copyright (c) 2019 The NetBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This code is derived from software contributed to The NetBSD Foundation
+.\" by Taylor R. Campbell.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+.\" PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.Dd November 25, 2019
+.Dt ATOMIC_LOADSTORE 9
+.Os
+.Sh NAME
+.Nm atomic_load_relaxed ,
+.Nm atomic_load_acquire ,
+.Nm atomic_load_consume ,
+.Nm atomic_store_relaxed ,
+.Nm atomic_store_release
+.Nd atomic and ordered memory operations
+.Sh SYNOPSIS
+.In sys/atomic.h
+.Ft T
+.Fn atomic_load_relaxed "const volatile T *p"
+.Ft T
+.Fn atomic_load_acquire "const volatile T *p"
+.Ft T
+.Fn atomic_load_consume "const volatile T *p"
+.Ft void
+.Fn atomic_store_relaxed "volatile T *p" "T v"
+.Ft void
+.Fn atomic_store_release "volatile T *p" "T v"
+.Sh DESCRIPTION
+These type-generic macros implement memory operations that are
+.Em atomic
+and that have
+.Em memory ordering constraints .
+Aside from atomicity and ordering, the load operations are equivalent
+to
+.Li * Ns Fa p
+and the store operations are equivalent to
+.Li * Ns Fa p Li "=" Fa v .
+The pointer
+.Fa p
+must be aligned, even on architectures like x86 which generally lack
+strict alignment requirements; see
+.Sx SIZE AND ALIGNMENT
+for details.
+.Pp
+.Em Atomic
+means that the memory operations cannot be
+.Em fused
+or
+.Em torn :
+.Bl -bullet
+.It
+.Em Fusing
+is combining multiple memory operations on a single object into one
+memory operation, such as replacing
+.Bd -literal -compact
+        *p = v;
+       x = *p;
+.Ed
+by
+.Bd -literal -compact
+       *p = v;
+       x = v;
+.Ed
+since the compiler can prove that
+.Li * Ns Fa p
+will yield
+.Fa v
+after
+.Li * Ns Fa p Li = Fa v .
+For
+.Em atomic
+memory operations, the implementation
+.Em will not
+assume that
+.Bl -dash -compact
+.It
+consecutive loads of the same object will return the same value, or
+.It
+a store followed by a load of the same object will return the value
+stored, or
+.It
+consecutive stores of the same object are redundant.
+.El
+Thus, the implementation will not replace two consecutive atomic loads
+by one, will not elide an atomic load following a store, and will not
+combine two consecutive atomic stores into one.
+.Pp
+For example,
+.Bd -literal
+       atomic_store_relaxed(&flag, 1);
+       while (atomic_load_relaxed(&flag))
+               continue;
+.Ed
+.Pp
+may be used to set a flag and then busy-wait until another thread
+clears it, whereas
+.Bd -literal
+       flag = 1;
+       while (flag)
+               continue;
+.Ed
+.Pp
+may be transformed into the infinite loop
+.Bd -literal
+       flag = 1;
+       while (1)
+               continue;
+.Ed
+.It
+.Em Tearing
+is implementing a memory operation on a large data unit such as a
+32-bit word by issuing multiple memory operations on smaller data units
+such as 8-bit bytes.
+The implementation will not tear
+.Em atomic
+loads or stores into smaller ones.
+Thus, as far as any interrupt, other thread, or other CPU can tell, an
+atomic memory operation is issued either all at once or not at all.
+.Pp
+For example, if a 32-bit word
+.Fa w
+is written with
+.Li atomic_store_relaxed(& Ns Fa w Ns Li "," 0x00010002) ,
+then an interrupt, other thread, or other CPU reading it with
+.Li atomic_load_relaxed(& Ns Fa w Ns Li ")"
+will never witness it partially written, whereas
+.Fa w Li = 0x00010002
+might be compiled into a pair of separate 16-bit store instructions
+instead of one single word-sized store instruction, in which case other
+threads may see the intermediate state with only one of the halves
+written.
+.El
+.Pp
+Atomic operations on any single object occur in a total order shared by
+all interrupts, threads, and CPUs, which is consistent with the program
+order in every interrupt, thread, and CPU.
+A single program without interruption or other threads or CPUs will
+always observe its own loads and stores in program order, but another
+program in an interrupt handler, in another thread, or on another CPU
+may issue loads that return values as if the first program's stores
+occurred out of program order, and vice versa.
+Two different threads might each observe a third thread's memory
+operations in different orders.
+.Pp
+The
+.Em memory ordering constraints
+make limited guarantees of ordering relative to memory operations on
+.Em other
+objects as witnessed by interrupts, other threads, or other CPUs, and
+have the following meanings:
+.Bl -tag -width relaxed
+.It relaxed
+No ordering relative to memory operations on any other objects is
+guaranteed.
+Relaxed ordering is the default for ordinary non-atomic memory
+operations like
+.Li * Ns Fa p
+and
+.Li * Ns Fa p Li = Fa v .
+.Pp
+Atomic operations with relaxed ordering are cheap: they are not
+read/modify/write atomic operations, and they do not involve any kind
+of inter-CPU ordering barriers.
+.It acquire
+This memory operation happens before all subsequent memory operations
+in program order.
+However, prior memory operations in program order may be reordered to
+happen after this one.
+For example, assuming no aliasing between the pointers, the
+implementation is allowed to treat
+.Bd -literal
+       int x = *p;
+       if (atomic_load_acquire(q)) {
+               int y = *r;
+               *s = x + y;
+               return 1;
+       }
+.Ed
+.Pp
+as if it were
+.Bd -literal
+       if (atomic_load_acquire(q)) {
+               int x = *p;
+               int y = *r;
+               *s = x + y;
+               return 1;
+       }
+.Ed
+.Pp
+but
+.Em not
+as if it were
+.Bd -literal
+       int x = *p;
+       int y = *r;
+       *s = x + y;
+       if (atomic_load_acquire(q)) {
+               return 1;
+       }
+.Ed
+.It consume



Home | Main Index | Thread Index | Old Index