Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/common/lib/libc/arch/aarch64/atomic Part I of ad@'s performa...



details:   https://anonhg.NetBSD.org/src/rev/83ce722f4182
branches:  trunk
changeset: 1012850:83ce722f4182
user:      skrll <skrll%NetBSD.org@localhost>
date:      Wed Aug 12 12:59:57 2020 +0000

description:
Part I of ad@'s performance improvements for aarch64

- Remove memory barriers from the atomic ops.  I don't understand why those
  are there.  Is it some architectural thing, or for a CPU bug, or just
  over-caution maybe?  They're not needed for correctness.

- Have unlikely conditional branches go forwards to help the static branch
  predictor.

diffstat:

 common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S  |   6 +++---
 common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S  |   6 +++---
 common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S  |   6 +++---
 common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S   |   6 +++---
 common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S  |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S  |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S  |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S  |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S  |  10 +++++-----
 common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S |   6 +++---
 common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S |   6 +++---
 common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S |   6 +++---
 common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S  |   6 +++---
 16 files changed, 64 insertions(+), 64 deletions(-)

diffs (truncated from 432 to 300 lines):

diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S       Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S       Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_16.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
        cmp     w0, w1                  /*   compare? */
        b.ne    2f
        stxrh   w3, w2, [x4]            /* store new value */
-       cbnz    w3, 1b                  /*   succeed? nope, try again. */
-       dmb     st                      /* data memory barrier */
+       cbnz    w3, 3f                  /*   succeed? nope, try again. */
 2:     ret                             /* return. */
+3:     b       1b
 END(_atomic_cas_16)
 
 ATOMIC_OP_ALIAS(atomic_cas_16,_atomic_cas_16)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S       Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S       Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
        cmp     w0, w1                  /*   compare? */
        b.ne    2f                      /*     return if different */
        stxr    w3, w2, [x4]            /* store new value */
-       cbnz    w3, 1b                  /*   succeed? nope, try again. */
-       dmb     st
+       cbnz    w3, 3f                  /*   succeed? nope, try again. */
 2:     ret                             /* return. */
+3:     b       1b
 END(_atomic_cas_32)
 
 ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S       Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S       Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_64.S,v 1.3 2019/02/19 12:51:44 rin Exp $ */
+/* $NetBSD: atomic_cas_64.S,v 1.4 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
        cmp     x0, x1                  /*   compare? */
        b.ne    2f                      /*     return if different */
        stxr    w3, x2, [x4]            /* store new value */
-       cbnz    w3, 1b                  /*   succeed? nope, try again. */
-       dmb     st
+       cbnz    w3, 3f                  /*   succeed? nope, try again. */
 2:     ret                             /* return. */
+3:     b       1b
 END(_atomic_cas_64)
 
 ATOMIC_OP_ALIAS(atomic_cas_64,_atomic_cas_64)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S        Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S        Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_8.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
        cmp     w0, w1                  /*   compare? */
        b.ne    2f
        stxrb   w3, w2, [x4]            /* store new value */
-       cbnz    w3, 1b                  /*   succeed? nope, try again. */
-       dmb     st                      /* data memory barrier */
+       cbnz    w3, 3f                  /*   succeed? nope, try again. */
 2:     ret                             /* return. */
+3:     b       1b
 END(_atomic_cas_8)
 
 ATOMIC_OP_ALIAS(atomic_cas_8,_atomic_cas_8)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S       Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S       Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_dec_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_dec_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
 1:     ldxr    w3, [x0]                /* load old value (return value) */
        sub     w3, w3, #1              /* calculate new value */
        stxr    w2, w3, [x0]            /* try to store */
-       cbnz    w2, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w2, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_dec_32)
 
 ATOMIC_OP_ALIAS(atomic_dec_32,_atomic_dec_32)
@@ -49,9 +49,9 @@
 1:     ldxr    w0, [x4]                /* load old value */
        sub     w0, w0, #1              /* calculate new value (return value) */
        stxr    w3, w0, [x4]            /* try to store */
-       cbnz    w3, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w3, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_dec_32_nv)
 
 ATOMIC_OP_ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S       Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S       Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_dec_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_dec_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
 1:     ldxr    x2, [x0]                /* load old value (return value) */
        sub     x2, x2, #1              /* calculate new value */
        stxr    w3, x2, [x0]            /* try to store */
-       cbnz    w3, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w3, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_dec_64)
 
 ATOMIC_OP_ALIAS(atomic_dec_64,_atomic_dec_64)
@@ -51,9 +51,9 @@
 1:     ldxr    x0, [x4]                /* load old value */
        sub     x0, x0, #1              /* calculate new value (return value) */
        stxr    w3, x0, [x4]            /* try to store */
-       cbnz    w3, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w3, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_dec_64_nv)
 
 ATOMIC_OP_ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S       Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S       Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_inc_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_inc_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
 1:     ldxr    w3, [x0]                /* load old value (return value) */
        add     w3, w3, #1              /* calculate new value */
        stxr    w2, w3, [x0]            /* try to store */
-       cbnz    w2, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w2, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_inc_32)
 
 ATOMIC_OP_ALIAS(atomic_inc_32,_atomic_inc_32)
@@ -49,9 +49,9 @@
 1:     ldxr    w0, [x4]                /* load old value */
        add     w0, w0, #1              /* calculate new value (return value) */
        stxr    w3, w0, [x4]            /* try to store */
-       cbnz    w3, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w3, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_inc_32_nv)
 
 ATOMIC_OP_ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S       Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S       Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_inc_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_inc_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@
 1:     ldxr    x2, [x0]                /* load old value (return value) */
        add     x2, x2, #1              /* calculate new value */
        stxr    w3, x2, [x0]            /* try to store */
-       cbnz    w3, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w3, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_inc_64)
 
 ATOMIC_OP_ALIAS(atomic_inc_64,_atomic_inc_64)
@@ -51,9 +51,9 @@
 1:     ldxr    x0, [x4]                /* load old value */
        add     x0, x0, #1              /* calculate new value (return value) */
        stxr    w3, x0, [x4]            /* try to store */
-       cbnz    w3, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w3, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_inc_64_nv)
 
 ATOMIC_OP_ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S      Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S      Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_nand_16.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
        mvn     w3, w0                  /* complement source */
        and     w3, w3, w1              /* calculate new value */
        stxrh   w2, w3, [x4]            /* try to store */
-       cbnz    w2, 1b                  /*   succeed? no, try again */
-       dmb     st
+       cbnz    w2, 2f                  /*   succeed? no, try again */
        ret                             /* return old value */
+2:     b       1b
 END(_atomic_nand_16)
 
 ATOMIC_OP_ALIAS(atomic_nand_16,_atomic_nand_16)
@@ -53,9 +53,9 @@
        mvn     w0, w0                  /* complement source */
        and     w0, w0, w1              /* calculate new value (return value) */
        stxrh   w2, w0, [x4]            /* try to store */
-       cbnz    w2, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w2, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_nand_16_nv)
 
 ATOMIC_OP_ALIAS(atomic_nand_16_nv,_atomic_nand_16_nv)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S      Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S      Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_nand_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
        mvn     w3, w0                  /* complement source */
        and     w3, w3, w1              /* calculate new value */
        stxr    w2, w3, [x4]            /* try to store */
-       cbnz    w2, 1b                  /*   succeed? no, try again */
-       dmb     st
+       cbnz    w2, 2f                  /*   succeed? no, try again */
        ret                             /* return old value */
+2:     b       1b
 END(_atomic_nand_32)
 
 ATOMIC_OP_ALIAS(atomic_nand_32,_atomic_nand_32)
@@ -53,9 +53,9 @@
        mvn     w0, w0                  /* complement source */
        and     w0, w0, w1              /* calculate new value (return value) */
        stxr    w2, w0, [x4]            /* try to store */
-       cbnz    w2, 1b                  /*   succeed? no, try again? */
-       dmb     st
+       cbnz    w2, 2f                  /*   succeed? no, try again? */
        ret                             /* return new value */
+2:     b       1b
 END(_atomic_nand_32_nv)
 
 ATOMIC_OP_ALIAS(atomic_nand_32_nv,_atomic_nand_32_nv)
diff -r 66cf2a95d0ad -r 83ce722f4182 common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S      Wed Aug 12 10:21:00 2020 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S      Wed Aug 12 12:59:57 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_nand_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@
        mvn     x2, x0                  /* complement source */
        and     x2, x2, x1              /* calculate new value */
        stxr    w3, x2, [x4]            /* try to store */
-       cbnz    w3, 1b                  /*   succeed? no, try again */
-       dmb     st
+       cbnz    w3, 2f                  /*   succeed? no, try again */
        ret                             /* return old value */
+2:     b       1b



Home | Main Index | Thread Index | Old Index