[src/trunk]: src/sys/kern mutex_vector_enter(): avoid some unneeded reads of ...

[ad <ad%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/9df50ee26367
branches:  trunk
changeset: 466911:9df50ee26367
user:      ad <ad%NetBSD.org@localhost>
date:      Mon Jan 06 11:12:55 2020 +0000

description:
mutex_vector_enter(): avoid some unneeded reads of mtx_owner.

diffstat:

 sys/kern/kern_mutex.c |  59 +++++++++++++++++++++++++++++---------------------
 1 files changed, 34 insertions(+), 25 deletions(-)

diffs (185 lines):

diff -r 90af1c38e094 -r 9df50ee26367 sys/kern/kern_mutex.c
--- a/sys/kern/kern_mutex.c     Mon Jan 06 10:21:21 2020 +0000
+++ b/sys/kern/kern_mutex.c     Mon Jan 06 11:12:55 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: kern_mutex.c,v 1.86 2019/12/11 20:46:06 ad Exp $       */
+/*     $NetBSD: kern_mutex.c,v 1.87 2020/01/06 11:12:55 ad Exp $       */
 
 /*-
  * Copyright (c) 2002, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -40,7 +40,7 @@
 #define        __MUTEX_PRIVATE
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.86 2019/12/11 20:46:06 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.87 2020/01/06 11:12:55 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -186,8 +186,8 @@
 
 #define        MUTEX_OWNER(owner)                                              \
        (owner & MUTEX_THREAD)
-#define        MUTEX_HAS_WAITERS(mtx)                                          \
-       (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0)
+#define        MUTEX_HAS_WAITERS(owner)                                        \
+       ((owner & MUTEX_BIT_WAITERS) != 0)
 
 #define        MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug)                         \
 do {                                                                   \
@@ -209,10 +209,10 @@
        (mtx)->mtx_owner = MUTEX_THREAD;                                \
 } while (/* CONSTCOND */ 0)
 
-#define        MUTEX_SPIN_P(mtx)               \
-    (((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0)
-#define        MUTEX_ADAPTIVE_P(mtx)           \
-    (((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0)
+#define        MUTEX_SPIN_P(owner)             \
+    (((owner) & MUTEX_BIT_SPIN) != 0)
+#define        MUTEX_ADAPTIVE_P(owner)         \
+    (((owner) & MUTEX_BIT_SPIN) == 0)
 
 #define        MUTEX_DEBUG_P(mtx)      (((mtx)->mtx_owner & MUTEX_BIT_NODEBUG) == 0)
 #if defined(LOCKDEBUG)
@@ -310,10 +310,11 @@
 mutex_dump(const volatile void *cookie, lockop_printer_t pr)
 {
        const volatile kmutex_t *mtx = cookie;
+       uintptr_t owner = mtx->mtx_owner;
 
        pr("owner field  : %#018lx wait/spin: %16d/%d\n",
-           (long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx),
-           MUTEX_SPIN_P(mtx));
+           (long)MUTEX_OWNER(owner), MUTEX_HAS_WAITERS(owner),
+           MUTEX_SPIN_P(owner));
 }
 
 /*
@@ -327,7 +328,7 @@
 mutex_abort(const char *func, size_t line, const kmutex_t *mtx, const char *msg)
 {
 
-       LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx) ?
+       LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx->mtx_owner) ?
            &mutex_spin_lockops : &mutex_adaptive_lockops), msg);
 }
 
@@ -380,10 +381,11 @@
 void
 mutex_destroy(kmutex_t *mtx)
 {
+       uintptr_t owner = mtx->mtx_owner;
 
-       if (MUTEX_ADAPTIVE_P(mtx)) {
-               MUTEX_ASSERT(mtx, !MUTEX_OWNED(mtx->mtx_owner) &&
-                   !MUTEX_HAS_WAITERS(mtx));
+       if (MUTEX_ADAPTIVE_P(owner)) {
+               MUTEX_ASSERT(mtx, !MUTEX_OWNED(owner) &&
+                   !MUTEX_HAS_WAITERS(owner));
        } else {
                MUTEX_ASSERT(mtx, !MUTEX_SPINBIT_LOCKED_P(mtx));
        }
@@ -454,7 +456,8 @@
        /*
         * Handle spin mutexes.
         */
-       if (MUTEX_SPIN_P(mtx)) {
+       owner = mtx->mtx_owner;
+       if (MUTEX_SPIN_P(owner)) {
 #if defined(LOCKDEBUG) && defined(MULTIPROCESSOR)
                u_int spins = 0;
 #endif
@@ -501,7 +504,7 @@
 
        curthread = (uintptr_t)curlwp;
 
-       MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
+       MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(owner));
        MUTEX_ASSERT(mtx, curthread != 0);
        MUTEX_ASSERT(mtx, !cpu_intr_p());
        MUTEX_WANTLOCK(mtx);
@@ -519,7 +522,7 @@
         * then we stop spinning, and sleep instead.
         */
        KPREEMPT_DISABLE(curlwp);
-       for (owner = mtx->mtx_owner;;) {
+       for (;;) {
                if (!MUTEX_OWNED(owner)) {
                        /*
                         * Mutex owner clear could mean two things:
@@ -667,12 +670,18 @@
                 * If the waiters bit is not set it's unsafe to go asleep,
                 * as we might never be awoken.
                 */
-               if ((membar_consumer(), mutex_oncpu(owner)) ||
-                   (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) {
+               membar_consumer();
+               if (mutex_oncpu(owner)) {
                        turnstile_exit(mtx);
                        owner = mtx->mtx_owner;
                        continue;
                }
+               membar_consumer();
+               owner = mtx->mtx_owner;
+               if (!MUTEX_HAS_WAITERS(owner)) {
+                       turnstile_exit(mtx);
+                       continue;
+               }
 #endif /* MULTIPROCESSOR */
 
                LOCKSTAT_START_TIMER(lsflag, slptime);
@@ -707,7 +716,7 @@
        turnstile_t *ts;
        uintptr_t curthread;
 
-       if (MUTEX_SPIN_P(mtx)) {
+       if (MUTEX_SPIN_P(mtx->mtx_owner)) {
 #ifdef FULL
                if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) {
                        MUTEX_ABORT(mtx, "exiting unheld spin mutex");
@@ -748,7 +757,7 @@
         */
        {
                int s = splhigh();
-               if (!MUTEX_HAS_WAITERS(mtx)) {
+               if (!MUTEX_HAS_WAITERS(mtx->mtx_owner)) {
                        MUTEX_RELEASE(mtx);
                        splx(s);
                        return;
@@ -810,7 +819,7 @@
 
        if (mtx == NULL)
                return 0;
-       if (MUTEX_ADAPTIVE_P(mtx))
+       if (MUTEX_ADAPTIVE_P(mtx->mtx_owner))
                return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp;
 #ifdef FULL
        return MUTEX_SPINBIT_LOCKED_P(mtx);
@@ -829,7 +838,7 @@
 mutex_owner(const kmutex_t *mtx)
 {
 
-       MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
+       MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx->mtx_owner));
        return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner);
 }
 
@@ -846,7 +855,7 @@
        uintptr_t owner;
        bool rv;
 
-       MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
+       MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx->mtx_owner));
        kpreempt_disable();
        owner = mtx->mtx_owner;
        rv = !MUTEX_OWNED(owner) || mutex_oncpu(MUTEX_OWNER(owner));
@@ -887,7 +896,7 @@
        /*
         * Handle spin mutexes.
         */
-       if (MUTEX_SPIN_P(mtx)) {
+       if (MUTEX_SPIN_P(mtx->mtx_owner)) {
                MUTEX_SPIN_SPLRAISE(mtx);
 #ifdef FULL
                if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {