Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/common/lib/libc/arch/aarch64/atomic Part I of ad@'s performa...
details: https://anonhg.NetBSD.org/src/rev/4f77423b78b4
branches: trunk
changeset: 937227:4f77423b78b4
user: skrll <skrll%NetBSD.org@localhost>
date: Wed Aug 12 12:59:57 2020 +0000
description:
Part I of ad@'s performance improvements for aarch64
- Remove memory barriers from the atomic ops. I don't understand why those
are there. Is it some architectural thing, or for a CPU bug, or just
over-caution maybe? They're not needed for correctness.
- Have unlikely conditional branches go forwards to help the static branch
predictor.
diffstat:
common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S | 6 +++---
common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S | 6 +++---
common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S | 6 +++---
common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S | 6 +++---
common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S | 10 +++++-----
common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S | 6 +++---
common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S | 6 +++---
common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S | 6 +++---
common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S | 6 +++---
16 files changed, 64 insertions(+), 64 deletions(-)
diffs (truncated from 432 to 300 lines):
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_16.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
cmp w0, w1 /* compare? */
b.ne 2f
stxrh w3, w2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st /* data memory barrier */
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_16)
ATOMIC_OP_ALIAS(atomic_cas_16,_atomic_cas_16)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
cmp w0, w1 /* compare? */
b.ne 2f /* return if different */
stxr w3, w2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_64.S,v 1.3 2019/02/19 12:51:44 rin Exp $ */
+/* $NetBSD: atomic_cas_64.S,v 1.4 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
cmp x0, x1 /* compare? */
b.ne 2f /* return if different */
stxr w3, x2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_64)
ATOMIC_OP_ALIAS(atomic_cas_64,_atomic_cas_64)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_8.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
cmp w0, w1 /* compare? */
b.ne 2f
stxrb w3, w2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st /* data memory barrier */
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_8)
ATOMIC_OP_ALIAS(atomic_cas_8,_atomic_cas_8)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_dec_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_dec_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
1: ldxr w3, [x0] /* load old value (return value) */
sub w3, w3, #1 /* calculate new value */
stxr w2, w3, [x0] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_32)
ATOMIC_OP_ALIAS(atomic_dec_32,_atomic_dec_32)
@@ -49,9 +49,9 @@
1: ldxr w0, [x4] /* load old value */
sub w0, w0, #1 /* calculate new value (return value) */
stxr w3, w0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_32_nv)
ATOMIC_OP_ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_dec_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_dec_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
1: ldxr x2, [x0] /* load old value (return value) */
sub x2, x2, #1 /* calculate new value */
stxr w3, x2, [x0] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_64)
ATOMIC_OP_ALIAS(atomic_dec_64,_atomic_dec_64)
@@ -51,9 +51,9 @@
1: ldxr x0, [x4] /* load old value */
sub x0, x0, #1 /* calculate new value (return value) */
stxr w3, x0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_64_nv)
ATOMIC_OP_ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_inc_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_inc_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
1: ldxr w3, [x0] /* load old value (return value) */
add w3, w3, #1 /* calculate new value */
stxr w2, w3, [x0] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_32)
ATOMIC_OP_ALIAS(atomic_inc_32,_atomic_inc_32)
@@ -49,9 +49,9 @@
1: ldxr w0, [x4] /* load old value */
add w0, w0, #1 /* calculate new value (return value) */
stxr w3, w0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_32_nv)
ATOMIC_OP_ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_inc_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_inc_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
1: ldxr x2, [x0] /* load old value (return value) */
add x2, x2, #1 /* calculate new value */
stxr w3, x2, [x0] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_64)
ATOMIC_OP_ALIAS(atomic_inc_64,_atomic_inc_64)
@@ -51,9 +51,9 @@
1: ldxr x0, [x4] /* load old value */
add x0, x0, #1 /* calculate new value (return value) */
stxr w3, x0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_64_nv)
ATOMIC_OP_ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_nand_16.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
mvn w3, w0 /* complement source */
and w3, w3, w1 /* calculate new value */
stxrh w2, w3, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
+2: b 1b
END(_atomic_nand_16)
ATOMIC_OP_ALIAS(atomic_nand_16,_atomic_nand_16)
@@ -53,9 +53,9 @@
mvn w0, w0 /* complement source */
and w0, w0, w1 /* calculate new value (return value) */
stxrh w2, w0, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_nand_16_nv)
ATOMIC_OP_ALIAS(atomic_nand_16_nv,_atomic_nand_16_nv)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_nand_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
mvn w3, w0 /* complement source */
and w3, w3, w1 /* calculate new value */
stxr w2, w3, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
+2: b 1b
END(_atomic_nand_32)
ATOMIC_OP_ALIAS(atomic_nand_32,_atomic_nand_32)
@@ -53,9 +53,9 @@
mvn w0, w0 /* complement source */
and w0, w0, w1 /* calculate new value (return value) */
stxr w2, w0, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_nand_32_nv)
ATOMIC_OP_ALIAS(atomic_nand_32_nv,_atomic_nand_32_nv)
diff -r 081d50c05f4c -r 4f77423b78b4 common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_nand_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
mvn x2, x0 /* complement source */
and x2, x2, x1 /* calculate new value */
stxr w3, x2, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again */
ret /* return old value */
+2: b 1b
Home |
Main Index |
Thread Index |
Old Index