Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/common/lib/libc/arch/aarch64/atomic aarch64: Implement __aar...
details: https://anonhg.NetBSD.org/src/rev/60d1f32c283e
branches: trunk
changeset: 368812:60d1f32c283e
user: riastradh <riastradh%NetBSD.org@localhost>
date: Sat Aug 06 21:31:33 2022 +0000
description:
aarch64: Implement __aarch64_casN_sync.
gcc generates calls to this symbol in programs that use
__sync_*_compare_and_swap, which require full sequential consistency
barriers, including store-before-load ordering on both sides of the
atomic; none of the release/acquire operations guarantee that, so we
have to insert explicit DMB instructions.
Note: gcc's own definition omits some of the DMB instructions, but I
can't prove that it's correct that way -- stores preceding the CAS
must complete before the load part of the CAS, and the store part of
the CAS must complete before loads following the CAS. Maybe there's
some way to prove that one of these orderings is guaranteed some
other way than a DMB but I'm not seeing it, and store-before-load
ordering is hard to understand.
Patch by skrll@ based on a patch by mrg@, soliloquy in commit message
by me.
diffstat:
common/lib/libc/arch/aarch64/atomic/Makefile.inc | 6 ++--
common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S | 28 ++++++++++++++++++++-
2 files changed, 30 insertions(+), 4 deletions(-)
diffs (152 lines):
diff -r f4f2964fd2e0 -r 60d1f32c283e common/lib/libc/arch/aarch64/atomic/Makefile.inc
--- a/common/lib/libc/arch/aarch64/atomic/Makefile.inc Sat Aug 06 21:26:05 2022 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/Makefile.inc Sat Aug 06 21:31:33 2022 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile.inc,v 1.4 2021/04/27 09:14:24 skrll Exp $
+# $NetBSD: Makefile.inc,v 1.5 2022/08/06 21:31:33 riastradh Exp $
.if defined(LIB) && (${LIB} == "kern" || ${LIB} == "c" || ${LIB} == "pthread" \
|| ${LIB} == "rump")
@@ -14,7 +14,7 @@
#and cas nand or sub swap xor
.for op in swp cas clr set eor add
.for sz in 1 2 4 8
-.for ar in _relax _acq _rel _acq_rel
+.for ar in _relax _acq _rel _acq_rel _sync
__aarch64_${op}${sz}${ar}.S: __aarch64_lse.S
${_MKTARGET_CREATE}
printf '#define OP ${op}\n#define OP_${op}\n#define SZ ${sz}\n#define AR ${ar}\n#define AR${ar}\n#include "__aarch64_lse.S"\n' > ${.TARGET}
@@ -23,7 +23,7 @@
.endfor
.endfor
.for op in casp
-.for ar in _relax _acq _rel _acq_rel
+.for ar in _relax _acq _rel _acq_rel _sync
__aarch64_${op}${ar}.S: __aarch64_lse.S
${_MKTARGET_CREATE}
printf '#define OP ${op}\n#define OP_${op}\n#define AR ${ar}\n#define AR${ar}\n#include "__aarch64_lse.S"\n' > ${.TARGET}
diff -r f4f2964fd2e0 -r 60d1f32c283e common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S
--- a/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S Sat Aug 06 21:26:05 2022 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S Sat Aug 06 21:31:33 2022 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: __aarch64_lse.S,v 1.6 2022/07/23 13:30:15 skrll Exp $ */
+/* $NetBSD: __aarch64_lse.S,v 1.7 2022/08/06 21:31:33 riastradh Exp $ */
/*-
* Copyright (c) 2021 The NetBSD Foundation, Inc.
@@ -64,21 +64,31 @@
#if defined(AR_relax)
#define ACQ
#define REL
+#define DMB
#endif
#if defined(AR_acq)
#define ACQ a
#define REL
+#define DMB
#endif
#if defined(AR_rel)
#define ACQ
#define REL l
+#define DMB
#endif
#if defined(AR_acq_rel)
#define ACQ a
#define REL l
+#define DMB
+#endif
+
+#if defined(AR_sync)
+#define ACQ
+#define REL
+#define DMB dmb ish
#endif
#if defined(OP_clr)
@@ -134,14 +144,18 @@
ENTRY_NP(SWP_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
SWP R0, R0, [x1]
+ DMB
ret
99:
#endif
mov x4, x0 /* need x0 for return value */
+ DMB /* potential barrier */
1: LDXR R0, [x1] /* load old value */
STXR w3, R4, [x1] /* store new value */
cbnz w3, 2f /* succeed?? no, try again */
+ DMB /* potential barrier */
ret /* return old value */
2: b 1b
END(SWP_FUNC)
@@ -151,16 +165,20 @@
ENTRY_NP(CAS_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
CAS R0, R1, [x2]
+ DMB
ret
99:
#endif
mov x4, x0 /* need x0 for return value */
+ DMB /* potential barrier */
1: LDXR R0, [x2] /* load old value */
cmp R0, R4 /* compare */
b.ne 2f /* not equal? return */
STXR w3, R1, [x2] /* store new value */
cbnz w3, 3f /* succeed? nope, try again. */
+ DMB /* potential barrier */
2: ret /* return. */
3: b 1b
END(CAS_FUNC)
@@ -170,12 +188,15 @@
ENTRY_NP(CASP_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
CASP x0, x1, x2, x3, [x4]
+ DMB
ret
99:
#endif
mov x5, x0 /* need x0 for return value */
mov x6, x1 /* need x1 for return value */
+ DMB /* potential barrier */
1: LDXP x0, x1, [x4] /* load old value */
cmp x0, x5 /* compare */
b.ne 2f /* not equal? return */
@@ -183,6 +204,7 @@
b.ne 2f /* not equal? return */
STXP w7, x2, x3, [x4] /* store new value */
cbnz w7, 3f /* succeed? nope, try again. */
+ DMB /* potential barrier */
2: ret /* return. */
3: b 1b
END(CASP_FUNC)
@@ -192,15 +214,19 @@
ENTRY_NP(INSN_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
INSN R0, R0, [x1]
+ DMB
ret
99:
#endif
mov x4, x0 /* need x0 for return value */
+ DMB /* potential barrier */
1: LDXR R0, [x1] /* load old value */
INSNOP R4, R0, R4
STXR w3, R4, [x1] /* store new value */
cbnz w3, 2f /* succeed?? no, try again */
+ DMB /* potential barrier */
ret /* return old value */
2: b 1b
END(INSN_FUNC)
Home |
Main Index |
Thread Index |
Old Index