[src/trunk]: src/sys/rump/librump/rumpkern Defer softint thread creation to f...

[pooka <pooka%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/1868046ea50e
branches:  trunk
changeset: 791284:1868046ea50e
user:      pooka <pooka%NetBSD.org@localhost>
date:      Mon Nov 11 23:06:40 2013 +0000

description:
Defer softint thread creation to first softint_establish() for that level.
Speeds up rump kernel bootstrap and saves memory -- very rarely are all
softint levels in a rump kernel used.

diffstat:

 sys/rump/librump/rumpkern/intr.c |  80 +++++++++++++++++++++++++++++++--------
 1 files changed, 63 insertions(+), 17 deletions(-)

diffs (143 lines):

diff -r 4cab2812f27e -r 1868046ea50e sys/rump/librump/rumpkern/intr.c
--- a/sys/rump/librump/rumpkern/intr.c  Mon Nov 11 17:04:06 2013 +0000
+++ b/sys/rump/librump/rumpkern/intr.c  Mon Nov 11 23:06:40 2013 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: intr.c,v 1.40 2013/05/02 21:45:28 pooka Exp $  */
+/*     $NetBSD: intr.c,v 1.41 2013/11/11 23:06:40 pooka Exp $  */
 
 /*
  * Copyright (c) 2008-2010 Antti Kantee.  All Rights Reserved.
@@ -26,7 +26,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.40 2013/05/02 21:45:28 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.41 2013/11/11 23:06:40 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -188,11 +188,52 @@
        panic("sithread unreachable");
 }
 
+static kmutex_t sithr_emtx;
+static unsigned int sithr_est;
+static int sithr_canest;
+
+/*
+ * Create softint handler threads when the softint for each respective
+ * level is established for the first time.  Most rump kernels don't
+ * need at least half of the softint levels, so on-demand saves bootstrap
+ * time and memory resources.  Note, though, that this routine may be
+ * called before it's possible to call kthread_create().  Creation of
+ * those softints (SOFTINT_CLOCK, as of writing this) will be deferred
+ * to until softint_init() is called for the main CPU.
+ */
+static void
+sithread_establish(int level)
+{
+       int docreate, rv;
+       int lvlbit = 1<<level;
+       int i;
+
+       KASSERT((level & ~SOFTINT_LVLMASK) == 0);
+       if (__predict_true(sithr_est & lvlbit))
+               return;
+
+       mutex_enter(&sithr_emtx);
+       docreate = (sithr_est & lvlbit) == 0 && sithr_canest;
+       sithr_est |= lvlbit;
+       mutex_exit(&sithr_emtx);
+
+       if (docreate) {
+               for (i = 0; i < ncpu_final; i++) {
+                       if ((rv = kthread_create(PRI_NONE,
+                           KTHREAD_MPSAFE | KTHREAD_INTR,
+                           cpu_lookup(i), sithread, (void *)(uintptr_t)level,
+                           NULL, "rsi%d/%d", i, level)) != 0)
+                               panic("softint thread create failed: %d", rv);
+               }
+       }
+}
+
 void
 rump_intr_init(int numcpu)
 {
 
        cv_init(&lbolt, "oh kath ra");
+       mutex_init(&sithr_emtx, MUTEX_DEFAULT, IPL_NONE);
        ncpu_final = numcpu;
 }
 
@@ -206,10 +247,23 @@
        if (!rump_threads)
                return;
 
-       /* XXX */
+       /* overloaded global init ... */
        if (ci->ci_index == 0) {
+               int sithr_swap;
+
                rumptc.tc_frequency = hz;
                tc_init(&rumptc);
+
+               /* create deferred softint threads */
+               mutex_enter(&sithr_emtx);
+               sithr_swap = sithr_est;
+               sithr_est = 0;
+               sithr_canest = 1;
+               mutex_exit(&sithr_emtx);
+               for (i = 0; i < SOFTINT_COUNT; i++) {
+                       if (sithr_swap & (1<<i))
+                               sithread_establish(i);
+               }
        }
 
        slev = kmem_alloc(sizeof(struct softint_lev) * SOFTINT_COUNT, KM_SLEEP);
@@ -219,19 +273,9 @@
        }
        cd->cpu_softcpu = slev;
 
-       /* softint might run on different physical CPU */
-       membar_sync();
-
-       for (i = 0; i < SOFTINT_COUNT; i++) {
-               rv = kthread_create(PRI_NONE,
-                   KTHREAD_MPSAFE | KTHREAD_INTR, ci,
-                   sithread, (void *)(uintptr_t)i,
-                   NULL, "rsi%d/%d", ci->ci_index, i);
-       }
-
-       rv = kthread_create(PRI_NONE, KTHREAD_MPSAFE,
-           ci, doclock, NULL, NULL, "rumpclk%d", ci->ci_index);
-       if (rv)
+       /* well, not really a "soft" interrupt ... */
+       if ((rv = kthread_create(PRI_NONE, KTHREAD_MPSAFE,
+           ci, doclock, NULL, NULL, "rumpclk%d", ci->ci_index)) != 0)
                panic("clock thread creation failed: %d", rv);
 }
 
@@ -240,13 +284,14 @@
 {
        struct softint *si;
        struct softint_percpu *sip;
+       int level = flags & SOFTINT_LVLMASK;
        int i;
 
        si = malloc(sizeof(*si), M_TEMP, M_WAITOK);
        si->si_func = func;
        si->si_arg = arg;
        si->si_flags = flags & SOFTINT_MPSAFE ? SI_MPSAFE : 0;
-       si->si_level = flags & SOFTINT_LVLMASK;
+       si->si_level = level;
        KASSERT(si->si_level < SOFTINT_COUNT);
        si->si_entry = malloc(sizeof(*si->si_entry) * ncpu_final,
            M_TEMP, M_WAITOK | M_ZERO);
@@ -254,6 +299,7 @@
                sip = &si->si_entry[i];
                sip->sip_parent = si;
        }
+       sithread_establish(level);
 
        return si;
 }