[src/trunk]: src/sys Add KCSAN instrumentation for atomic_{load, store}_*.

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/b6e9509f5fce
branches:  trunk
changeset: 846759:b6e9509f5fce
user:      maxv <maxv%NetBSD.org@localhost>
date:      Sun Dec 01 08:15:58 2019 +0000

description:
Add KCSAN instrumentation for atomic_{load,store}_*.

diffstat:

 sys/kern/subr_csan.c |  26 ++++++++++++++++++++++++--
 sys/sys/atomic.h     |  31 +++++++++++++++++++++++++------
 2 files changed, 49 insertions(+), 8 deletions(-)

diffs (125 lines):

diff -r a7fd82bca0bd -r b6e9509f5fce sys/kern/subr_csan.c
--- a/sys/kern/subr_csan.c      Sun Dec 01 06:53:31 2019 +0000
+++ b/sys/kern/subr_csan.c      Sun Dec 01 08:15:58 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $       */
+/*     $NetBSD: subr_csan.c,v 1.6 2019/12/01 08:15:58 maxv Exp $       */
 
 /*
  * Copyright (c) 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.6 2019/12/01 08:15:58 maxv Exp $");
 
 #include <sys/param.h>
 #include <sys/device.h>
@@ -606,6 +606,28 @@
 CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
 CSAN_ATOMIC_FUNC_INC(ptr, void *, void);
 
+void
+kcsan_atomic_load(const volatile void *p, void *v, int size)
+{
+       switch (size) {
+       case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break;
+       case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break;
+       case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break;
+       case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break;
+       }
+}
+
+void
+kcsan_atomic_store(volatile void *p, const void *v, int size)
+{
+       switch (size) {
+       case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break;
+       case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break;
+       case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break;
+       case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break;
+       }
+}
+
 /* -------------------------------------------------------------------------- */
 
 #include <sys/bus.h>
diff -r a7fd82bca0bd -r b6e9509f5fce sys/sys/atomic.h
--- a/sys/sys/atomic.h  Sun Dec 01 06:53:31 2019 +0000
+++ b/sys/sys/atomic.h  Sun Dec 01 08:15:58 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: atomic.h,v 1.18 2019/11/29 22:17:23 riastradh Exp $    */
+/*     $NetBSD: atomic.h,v 1.19 2019/12/01 08:15:58 maxv Exp $ */
 
 /*-
  * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
@@ -411,18 +411,35 @@
        KASSERT(((uintptr_t)(p) & ilog2(sizeof(*(p)))) == 0);                 \
 } while (0)
 
+#ifdef KCSAN
+void kcsan_atomic_load(const volatile void *, void *, int);
+void kcsan_atomic_store(volatile void *, const void *, int);
+#define __DO_ATOMIC_LOAD(p, v) \
+       kcsan_atomic_load(p, __UNVOLATILE(&v), sizeof(v))
+#define __DO_ATOMIC_STORE(p, v) \
+       kcsan_atomic_store(p, __UNVOLATILE(&v), sizeof(v))
+#else
+#define __DO_ATOMIC_LOAD(p, v) \
+       v = *p
+#define __DO_ATOMIC_STORE(p, v) \
+       *p = v
+#endif
+
 #define        atomic_load_relaxed(p)                                                \
 ({                                                                           \
        const volatile __typeof__(*(p)) *__al_ptr = (p);                      \
+       __typeof__(*(p)) __al_val;                                            \
        __ATOMIC_PTR_CHECK(__al_ptr);                                         \
-       *__al_ptr;                                                            \
+       __DO_ATOMIC_LOAD(__al_ptr, __al_val);                                 \
+       __al_val;                                                             \
 })
 
 #define        atomic_load_consume(p)                                                \
 ({                                                                           \
        const volatile __typeof__(*(p)) *__al_ptr = (p);                      \
+       __typeof__(*(p)) __al_val;                                            \
        __ATOMIC_PTR_CHECK(__al_ptr);                                         \
-       __typeof__(*(p)) __al_val = *__al_ptr;                                \
+       __DO_ATOMIC_LOAD(__al_ptr, __al_val);                                 \
        membar_datadep_consumer();                                            \
        __al_val;                                                             \
 })
@@ -436,8 +453,9 @@
 #define        atomic_load_acquire(p)                                                \
 ({                                                                           \
        const volatile __typeof__(*(p)) *__al_ptr = (p);                      \
+       __typeof__(*(p)) __al_val;                                            \
        __ATOMIC_PTR_CHECK(__al_ptr);                                         \
-       __typeof__(*(p)) __al_val = *__al_ptr;                                \
+       __DO_ATOMIC_LOAD(__al_ptr, __al_val);                                 \
        membar_sync();                                                        \
        __al_val;                                                             \
 })
@@ -445,8 +463,9 @@
 #define        atomic_store_relaxed(p,v)                                             \
 ({                                                                           \
        volatile __typeof__(*(p)) *__as_ptr = (p);                            \
+       __typeof__(*(p)) __as_val = (v);                                      \
        __ATOMIC_PTR_CHECK(__as_ptr);                                         \
-       *__as_ptr = (v);                                                      \
+       __DO_ATOMIC_STORE(__as_ptr, __as_val);                                \
 })
 
 #define        atomic_store_release(p,v)                                             \
@@ -455,7 +474,7 @@
        __typeof__(*(p)) __as_val = (v);                                      \
        __ATOMIC_PTR_CHECK(__as_ptr);                                         \
        membar_exit();                                                        \
-       *__as_ptr = __as_val;                                                 \
+       __DO_ATOMIC_STORE(__as_ptr, __as_val);                                \
 })
 
 #else  /* __STDC_VERSION__ >= 201112L */