[src/trunk]: src/common/lib/libc/arch/aarch64/atomic As we're providing the l...

To: source-changes-hg%NetBSD.org@localhost
Subject: [src/trunk]: src/common/lib/libc/arch/aarch64/atomic As we're providing the l...
From: skrll <skrll%NetBSD.org@localhost>
Date: Thu, 29 Jul 2021 12:50:31 +0000
details:   https://anonhg.NetBSD.org/src/rev/6f602f7d3243
branches:  trunk
changeset: 984899:6f602f7d3243
user:      skrll <skrll%NetBSD.org@localhost>
date:      Thu Jul 29 10:29:05 2021 +0000

description:
As we're providing the legacy gcc __sync built-in functions for atomic
memory access we might as well get the memory barriers right...
>From the gcc documentation:

In most cases, these built-in functions are considered a full barrier.
That is, no memory operand is moved across the operation, either forward
or backward. Further, instructions are issued as necessary to prevent the
processor from speculating loads across the operation and from queuing
stores after the operation.

type __sync_lock_test_and_set (type *ptr, type value, ...)

   This built-in function is not a full barrier, but rather an acquire
   barrier. This means that references after the operation cannot move to
   (or be speculated to) before the operation, but previous memory stores
   may not be globally visible yet, and previous memory loads may not yet
   be satisfied.

void __sync_lock_release (type *ptr, ...)

   This built-in function is not a full barrier, but rather a release
   barrier. This means that all previous memory stores are globally
   visible, and all previous memory loads have been satisfied, but
   following memory reads are not prevented from being speculated to
   before the barrier.

diffstat:

 common/lib/libc/arch/aarch64/atomic/atomic_add_16.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_add_32.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_add_64.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_add_8.S   |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_and_16.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_and_32.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_and_64.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_and_8.S   |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S  |  15 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S  |  16 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S  |  16 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S   |  16 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S |  32 +++++-
 common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S |  40 ++++++-
 common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S |  32 +++++-
 common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S  |  33 +++++-
 common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h  |  98 +++++++++++++++++++-
 common/lib/libc/arch/aarch64/atomic/atomic_or_16.S   |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_or_32.S   |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_or_64.S   |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_or_8.S    |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S   |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S |  12 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S |  12 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S |  12 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S  |  12 ++-
 common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S  |   8 +-
 common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S   |   8 +-
 33 files changed, 410 insertions(+), 96 deletions(-)

diffs (truncated from 1220 to 300 lines):

diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_16.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
 
 ATOMIC_OP16(add, add)
 
+SYNC_FETCH_OP16(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_16,_atomic_add_16)
 ATOMIC_OP_ALIAS(atomic_add_short,_atomic_add_16)
-STRONG_ALIAS(__sync_fetch_and_add_2,_atomic_add_16)
 STRONG_ALIAS(_atomic_add_short,_atomic_add_16)
 
 ATOMIC_OP16_NV(add, add)
 
+SYNC_OP16_FETCH(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_16_nv,_atomic_add_16_nv)
 ATOMIC_OP_ALIAS(atomic_add_short_nv,_atomic_add_16_nv)
-STRONG_ALIAS(__sync_add_and_fetch_2,_atomic_add_16_nv)
 STRONG_ALIAS(_atomic_add_short_nv,_atomic_add_16_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_32.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
 
 ATOMIC_OP32(add, add)
 
+SYNC_FETCH_OP32(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_32,_atomic_add_32)
 ATOMIC_OP_ALIAS(atomic_add_int,_atomic_add_32)
-STRONG_ALIAS(__sync_fetch_and_add_4,_atomic_add_32)
 STRONG_ALIAS(_atomic_add_int,_atomic_add_32)
 
 ATOMIC_OP32_NV(add, add)
 
+SYNC_OP32_FETCH(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_32_nv,_atomic_add_32_nv)
 ATOMIC_OP_ALIAS(atomic_add_int_nv,_atomic_add_32_nv)
-STRONG_ALIAS(__sync_add_and_fetch_4,_atomic_add_32_nv)
 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_64.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,18 +33,20 @@
 
 ATOMIC_OP64(add, add)
 
+SYNC_FETCH_OP64(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_64_nv,_atomic_add_64_nv)
 ATOMIC_OP_ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv)
 ATOMIC_OP_ALIAS(atomic_add_long_nv,_atomic_add_64_nv)
 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv)
 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv)
-STRONG_ALIAS(__sync_add_and_fetch_8,_atomic_add_64_nv)
 
 ATOMIC_OP64_NV(add, add)
 
+SYNC_OP64_FETCH(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_64,_atomic_add_64)
 ATOMIC_OP_ALIAS(atomic_add_ptr,_atomic_add_64)
 ATOMIC_OP_ALIAS(atomic_add_long,_atomic_add_64)
 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64)
 STRONG_ALIAS(_atomic_add_long,_atomic_add_64)
-STRONG_ALIAS(__sync_fetch_and_add_8,_atomic_add_64)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_8.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S        Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S        Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_8.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
 
 ATOMIC_OP8(add, add)
 
+SYNC_FETCH_OP8(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_8,_atomic_add_8)
 ATOMIC_OP_ALIAS(atomic_add_char,_atomic_add_8)
-STRONG_ALIAS(__sync_fetch_and_add_1,_atomic_add_8)
 STRONG_ALIAS(_atomic_add_char,_atomic_add_8)
 
 ATOMIC_OP8_NV(add, add)
 
+SYNC_OP8_FETCH(add, add)
+
 ATOMIC_OP_ALIAS(atomic_add_8_nv,_atomic_add_8_nv)
 ATOMIC_OP_ALIAS(atomic_add_char_nv,_atomic_add_8_nv)
-STRONG_ALIAS(__sync_add_and_fetch_1,_atomic_add_8_nv)
 STRONG_ALIAS(_atomic_add_char_nv,_atomic_add_8_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_16.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
 
 ATOMIC_OP16(and, and)
 
+SYNC_FETCH_OP16(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_16,_atomic_and_16)
 ATOMIC_OP_ALIAS(atomic_and_ushort,_atomic_and_16)
-STRONG_ALIAS(__sync_fetch_and_and_2,_atomic_and_16)
 STRONG_ALIAS(_atomic_and_ushort,_atomic_and_16)
 
 ATOMIC_OP16_NV(and, and)
 
+SYNC_OP16_FETCH(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_16_nv,_atomic_and_16_nv)
 ATOMIC_OP_ALIAS(atomic_and_ushort_nv,_atomic_and_16_nv)
-STRONG_ALIAS(__sync_and_and_fetch_2,_atomic_and_16_nv)
 STRONG_ALIAS(_atomic_and_ushort_nv,_atomic_and_16_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_32.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
 
 ATOMIC_OP32(and, and)
 
+SYNC_FETCH_OP32(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_32,_atomic_and_32)
 ATOMIC_OP_ALIAS(atomic_and_uint,_atomic_and_32)
-STRONG_ALIAS(__sync_fetch_and_and_4,_atomic_and_32)
 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32)
 
 ATOMIC_OP32_NV(and, and)
 
+SYNC_OP32_FETCH(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_32_nv,_atomic_and_32_nv)
 ATOMIC_OP_ALIAS(atomic_and_uint_nv,_atomic_and_32_nv)
-STRONG_ALIAS(__sync_and_and_fetch_4,_atomic_and_32_nv)
 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_64.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
 
 ATOMIC_OP64(and, and)
 
+SYNC_FETCH_OP64(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_64,_atomic_and_64)
 ATOMIC_OP_ALIAS(atomic_and_ulong,_atomic_and_64)
 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64)
-STRONG_ALIAS(__sync_fetch_and_and_8,_atomic_and_64)
 
 ATOMIC_OP64_NV(and, and)
 
+SYNC_OP64_FETCH(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_64_nv,_atomic_and_64_nv)
 ATOMIC_OP_ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv)
 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv)
-STRONG_ALIAS(__sync_and_and_fetch_8,_atomic_and_64_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_8.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S        Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S        Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_8.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
 
 ATOMIC_OP8(and, and)
 
+SYNC_FETCH_OP8(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_8,_atomic_and_8)
 ATOMIC_OP_ALIAS(atomic_and_uchar,_atomic_and_8)
-STRONG_ALIAS(__sync_fetch_and_and_1,_atomic_and_8)
 STRONG_ALIAS(_atomic_and_uchar,_atomic_and_8)
 
 ATOMIC_OP8_NV(and, and)
 
+SYNC_OP8_FETCH(and, and)
+
 ATOMIC_OP_ALIAS(atomic_and_8_nv,_atomic_and_8_nv)
 ATOMIC_OP_ALIAS(atomic_and_uchar_nv,_atomic_and_8_nv)
-STRONG_ALIAS(__sync_and_and_fetch_1,_atomic_and_8_nv)
 STRONG_ALIAS(_atomic_and_uchar_nv,_atomic_and_8_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_16.S,v 1.3 2020/10/07 07:31:47 skrll Exp $ */
+/* $NetBSD: atomic_cas_16.S,v 1.4 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -46,3 +46,16 @@
 STRONG_ALIAS(_atomic_cas_short,_atomic_cas_16)
 STRONG_ALIAS(_atomic_cas_ushort,_atomic_cas_16)
 STRONG_ALIAS(__sync_val_compare_and_swap_2,_atomic_cas_16)
+
+ENTRY_NP(__sync_val_compare_and_swap_2)
+       mov     x4, x0                  /* we need x0 for return value */
+       dmb     ish
+1:     ldxrh   w0, [x4]                /* load old value */
+       cmp     w0, w1                  /*   compare? */
+       b.ne    2f
+       stxrh   w3, w2, [x4]            /* store new value */
+       cbnz    w3, 3f                  /*   succeed? nope, try again. */
+       dmb     ish
+2:     ret                             /* return. */
+3:     b       1b
+END(__sync_val_compare_and_swap_2)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_32.S,v 1.3 2020/10/07 07:31:47 skrll Exp $ */
+/* $NetBSD: atomic_cas_32.S,v 1.4 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -46,7 +46,19 @@
 ATOMIC_OP_ALIAS(atomic_cas_uint,_atomic_cas_32)
 ATOMIC_OP_ALIAS(atomic_cas_32_ni,_atomic_cas_32)
 ATOMIC_OP_ALIAS(atomic_cas_uint_ni,_atomic_cas_32)
-STRONG_ALIAS(__sync_val_compare_and_swap_4,_atomic_cas_32)
 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32)
 STRONG_ALIAS(_atomic_cas_32_ni,_atomic_cas_32)
 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32)
+
+ENTRY_NP(__sync_val_compare_and_swap_4)
+       mov     x4, x0                  /* we need x0 for return value */
+       dmb     ish
+1:     ldxr    w0, [x4]                /* load old value */
+       cmp     w0, w1                  /*   compare? */
+       b.ne    2f                      /*     return if different */
+       stxr    w3, w2, [x4]            /* store new value */
+       cbnz    w3, 3f                  /*   succeed? nope, try again. */
+       dmb     ish
+2:     ret                             /* return. */
+3:     b       1b
+END(__sync_val_compare_and_swap_4)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S       Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S       Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_64.S,v 1.5 2020/10/07 07:31:47 skrll Exp $ */
+/* $NetBSD: atomic_cas_64.S,v 1.6 2021/07/29 10:29:05 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -48,9 +48,21 @@
 ATOMIC_OP_ALIAS(atomic_cas_64_ni,_atomic_cas_64)
 ATOMIC_OP_ALIAS(atomic_cas_ulong_ni,_atomic_cas_64)
 ATOMIC_OP_ALIAS(atomic_cas_ptr_ni,_atomic_cas_64)
-STRONG_ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64)
 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64)
 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64)
Prev by Date: [src/trunk]: src/usr.bin/make make: in TryParseNumber, reset errno as late as...
Next by Date: [src/trunk]: src/doc gcc 11.2 out
Previous by Thread: [src/trunk]: src/usr.bin/make make: in TryParseNumber, reset errno as late as...
Next by Thread: [src/trunk]: src/doc gcc 11.2 out
Indexes:
Home | Main Index | Thread Index | Old Index