[src/trunk]: src/sys/kern ASSERT_SLEEPABLE(9): Micro-optimize this a little bit.

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/15bf8d78cbfa
branches:  trunk
changeset: 374180:15bf8d78cbfa
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Sun Apr 09 08:17:36 2023 +0000

description:
ASSERT_SLEEPABLE(9): Micro-optimize this a little bit.

This convinces gcc to do less -- make a smaller stack frame, compute
fewer conditional moves in favour of predicted-not-taken branches --
in the fast path where we are sleepable as the caller expects.

Wasn't able to convince it to do the ncsw loop with a
predicted-not-taken branch, but let's leave the __predict_false in
there anyway because it's still a good prediction.

diffstat:

 sys/kern/kern_lock.c |  33 ++++++++++++++++++---------------
 1 files changed, 18 insertions(+), 15 deletions(-)

diffs (74 lines):

diff -r dd3fde0a837e -r 15bf8d78cbfa sys/kern/kern_lock.c
--- a/sys/kern/kern_lock.c      Sun Apr 09 06:10:03 2023 +0000
+++ b/sys/kern/kern_lock.c      Sun Apr 09 08:17:36 2023 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: kern_lock.c,v 1.183 2023/02/23 14:57:29 riastradh Exp $        */
+/*     $NetBSD: kern_lock.c,v 1.184 2023/04/09 08:17:36 riastradh Exp $        */
 
 /*-
  * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.183 2023/02/23 14:57:29 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.184 2023/04/09 08:17:36 riastradh Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_lockdebug.h"
@@ -67,8 +67,9 @@ bool  kernel_lock_dodebug;
 void
 assert_sleepable(void)
 {
+       struct lwp *l = curlwp;
        const char *reason;
-       uint64_t pctr;
+       uint64_t ncsw;
        bool idle;
 
        if (__predict_false(panicstr != NULL)) {
@@ -82,30 +83,32 @@ assert_sleepable(void)
         * routine may be called in delicate situations.
         */
        do {
-               pctr = lwp_pctr();
+               ncsw = l->l_ncsw;
                __insn_barrier();
                idle = CURCPU_IDLE_P();
                __insn_barrier();
-       } while (pctr != lwp_pctr());
+       } while (__predict_false(ncsw != l->l_ncsw));
 
        reason = NULL;
-       if (idle && !cold) {
+       if (__predict_false(idle) && !cold) {
                reason = "idle";
+               goto panic;
        }
-       if (cpu_intr_p()) {
+       if (__predict_false(cpu_intr_p())) {
                reason = "interrupt";
-       }
-       if (cpu_softintr_p()) {
-               reason = "softint";
+               goto panic;
        }
-       if (!pserialize_not_in_read_section()) {
+       if (__predict_false(cpu_softintr_p())) {
+               reason = "softint";
+               goto panic;
+       }
+       if (__predict_false(!pserialize_not_in_read_section())) {
                reason = "pserialize";
+               goto panic;
        }
+       return;
 
-       if (reason) {
-               panic("%s: %s caller=%p", __func__, reason,
-                   (void *)RETURN_ADDRESS);
-       }
+panic: panic("%s: %s caller=%p", __func__, reason, (void *)RETURN_ADDRESS);
 }
 
 /*