[src/trunk]: src/sys/rump/librump Add initial work on a rump virtual cpu sche...

[pooka <pooka%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/1afb7e1d52ef
branches:  trunk
changeset: 748157:1afb7e1d52ef
user:      pooka <pooka%NetBSD.org@localhost>
date:      Thu Oct 15 00:28:46 2009 +0000

description:
Add initial work on a rump virtual cpu scheduler.  This is necessary
for kernel code which has been written to avoid MP contention by
using cpu-local storage (most prominently, select and pool_cache).

Instead of always assuming rump_cpu, the scheduler must now be run
(and unrun) on all entry points into rump.  Likewise, rumpuser
unruns and re-runs the scheduler around each potentially blocking
operation.  As an optimization, I modified some locking primitives
to try to get the lock without blocking before releasing the cpu.

Also, ltsleep was modified to assume that it is never called without
the biglock held and made to use the biglock as the sleep interlock.
Otherwise there is just too much drama with deadlocks.  If some
kernel code wants to call ltsleep without the biglock, then, *snif*,
it's no longer supported and rump and should be modified to support
newstyle locks anyway.

diffstat:

 sys/rump/librump/rumpkern/Makefile.rumpkern   |   11 +-
 sys/rump/librump/rumpkern/arch/i386/rumpcpu.c |   15 ++-
 sys/rump/librump/rumpkern/emul.c              |   12 +-
 sys/rump/librump/rumpkern/locks.c             |   65 +++++++++++++-
 sys/rump/librump/rumpkern/ltsleep.c           |   60 ++++--------
 sys/rump/librump/rumpkern/misc_stub.c         |   12 +--
 sys/rump/librump/rumpkern/percpu.c            |    6 +-
 sys/rump/librump/rumpkern/rump.c              |   46 +++++++--
 sys/rump/librump/rumpkern/rump_private.h      |   16 +++-
 sys/rump/librump/rumpkern/scheduler.c         |  116 ++++++++++++++++++++++++++
 sys/rump/librump/rumpuser/rumpuser_int.h      |    8 +-
 sys/rump/librump/rumpuser/rumpuser_pth.c      |   35 ++++++-
 12 files changed, 311 insertions(+), 91 deletions(-)

diffs (truncated from 838 to 300 lines):

diff -r 4286a4fd0677 -r 1afb7e1d52ef sys/rump/librump/rumpkern/Makefile.rumpkern
--- a/sys/rump/librump/rumpkern/Makefile.rumpkern       Thu Oct 15 00:16:28 2009 +0000
+++ b/sys/rump/librump/rumpkern/Makefile.rumpkern       Thu Oct 15 00:28:46 2009 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: Makefile.rumpkern,v 1.51 2009/10/14 17:29:19 pooka Exp $
+#      $NetBSD: Makefile.rumpkern,v 1.52 2009/10/15 00:28:46 pooka Exp $
 #
 
 .include "${RUMPTOP}/Makefile.rump"
@@ -15,7 +15,7 @@
 # Source modules, first the ones specifically implemented for librump.
 # 
 SRCS=  rump.c emul.c intr.c locks.c ltsleep.c percpu.c pool.c  \
-       sleepq.c sysproxy_socket.c vm.c
+       scheduler.c sleepq.c sysproxy_socket.c vm.c
 
 SRCS+= rumpkern_if_wrappers.c
 
@@ -71,8 +71,9 @@
 #
 # If archdir exists, it is required to provide:
 # 1) kobj_reloc() and kobj_machdep()
-# 2) ...?
-# 3) PROFIT!
+# 2) rump_cpu_bootstrap()
+# 3) ...?
+# 4) PROFIT!
 #
 
 #
@@ -90,7 +91,7 @@
 .include "${ARCHDIR}/Makefile.inc"
 .PATH: ${ARCHDIR}
 .else
-SRCS+= kobj_stubs.c
+SRCS+= kobj_stubs.c rumpcpu_generic.c
 .endif
 
 # include libkern source files
diff -r 4286a4fd0677 -r 1afb7e1d52ef sys/rump/librump/rumpkern/arch/i386/rumpcpu.c
--- a/sys/rump/librump/rumpkern/arch/i386/rumpcpu.c     Thu Oct 15 00:16:28 2009 +0000
+++ b/sys/rump/librump/rumpkern/arch/i386/rumpcpu.c     Thu Oct 15 00:28:46 2009 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: rumpcpu.c,v 1.5 2009/10/14 18:18:53 pooka Exp $        */
+/*     $NetBSD: rumpcpu.c,v 1.6 2009/10/15 00:28:47 pooka Exp $        */
 
 /*
  * Copyright (c) 2008 Antti Kantee.  All Rights Reserved.
@@ -29,7 +29,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rumpcpu.c,v 1.5 2009/10/14 18:18:53 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rumpcpu.c,v 1.6 2009/10/15 00:28:47 pooka Exp $");
 
 #include <sys/param.h>
 
@@ -37,13 +37,20 @@
 
 #include "rump_private.h"
 
-struct cpu_info *cpu_info_list = &rump_cpu;
+struct cpu_info *cpu_info_list;
+
+void
+rump_cpu_bootstrap(struct cpu_info *ci)
+{
+
+       cpu_info_list = ci;
+}
 
 struct cpu_info *
 x86_curcpu()
 {
 
-       return &rump_cpu;
+       return curlwp->l_cpu;
 }
 
 struct lwp *
diff -r 4286a4fd0677 -r 1afb7e1d52ef sys/rump/librump/rumpkern/emul.c
--- a/sys/rump/librump/rumpkern/emul.c  Thu Oct 15 00:16:28 2009 +0000
+++ b/sys/rump/librump/rumpkern/emul.c  Thu Oct 15 00:28:46 2009 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: emul.c,v 1.100 2009/10/14 18:18:53 pooka Exp $ */
+/*     $NetBSD: emul.c,v 1.101 2009/10/15 00:28:46 pooka Exp $ */
 
 /*
  * Copyright (c) 2007 Antti Kantee.  All Rights Reserved.
@@ -28,7 +28,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: emul.c,v 1.100 2009/10/14 18:18:53 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: emul.c,v 1.101 2009/10/15 00:28:46 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/malloc.h>
@@ -73,7 +73,6 @@
 dev_t rootdev;
 int physmem = 256*256; /* 256 * 1024*1024 / 4k, PAGE_SIZE not always set */
 int doing_shutdown;
-int ncpu = 1;
 const int schedppq = 1;
 int hardclock_ticks;
 bool mp_online = false;
@@ -370,14 +369,18 @@
        void (*f)(void *);
        void *thrarg;
 
+       /* schedule ourselves first */
        f = k->f;
        thrarg = k->arg;
        rumpuser_set_curlwp(k->mylwp);
+       rump_schedule();
+
        kmem_free(k, sizeof(struct kthdesc));
-
        if ((curlwp->l_pflag & LP_MPSAFE) == 0)
                KERNEL_LOCK(1, NULL);
+
        f(thrarg);
+
        panic("unreachable, should kthread_exit()");
 }
 
@@ -458,6 +461,7 @@
        if ((curlwp->l_pflag & LP_MPSAFE) == 0)
                KERNEL_UNLOCK_ONE(NULL);
        rump_clear_curlwp();
+       rump_unschedule();
        rumpuser_thread_exit();
 }
 
diff -r 4286a4fd0677 -r 1afb7e1d52ef sys/rump/librump/rumpkern/locks.c
--- a/sys/rump/librump/rumpkern/locks.c Thu Oct 15 00:16:28 2009 +0000
+++ b/sys/rump/librump/rumpkern/locks.c Thu Oct 15 00:28:46 2009 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: locks.c,v 1.28 2009/10/02 09:56:08 pooka Exp $ */
+/*     $NetBSD: locks.c,v 1.29 2009/10/15 00:28:46 pooka Exp $ */
 
 /*-
  * Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -55,7 +55,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: locks.c,v 1.28 2009/10/02 09:56:08 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: locks.c,v 1.29 2009/10/15 00:28:46 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -300,12 +300,53 @@
  */
 
 static volatile int lockcnt;
+
+bool
+kernel_biglocked()
+{
+
+       return rumpuser_mutex_held(rump_giantlock) && lockcnt > 0;
+}
+
+void
+kernel_unlock_allbutone(int *countp)
+{
+       int minusone = lockcnt-1;
+
+       KASSERT(kernel_biglocked());
+       if (minusone) {
+               _kernel_unlock(minusone, countp);
+       }
+       KASSERT(lockcnt == 1);
+       *countp = minusone;
+
+       /*
+        * We drop lockcnt to 0 since rumpuser doesn't know that the
+        * kernel biglock is being used as the interlock for cv in
+        * tsleep.
+        */
+       lockcnt = 0;
+}
+
+void
+kernel_ununlock_allbutone(int nlocks)
+{
+
+       KASSERT(rumpuser_mutex_held(rump_giantlock) && lockcnt == 0);
+       lockcnt = 1;
+       _kernel_lock(nlocks);
+}
+
 void
 _kernel_lock(int nlocks)
 {
 
        while (nlocks--) {
-               rumpuser_mutex_enter(rump_giantlock);
+               if (!rumpuser_mutex_tryenter(rump_giantlock)) {
+                       rump_unschedule();
+                       rumpuser_mutex_enter_nowrap(rump_giantlock);
+                       rump_schedule();
+               }
                lockcnt++;
        }
 }
@@ -336,6 +377,24 @@
        }
 }
 
+void
+rump_user_unschedule(int nlocks, int *countp)
+{
+
+       _kernel_unlock(nlocks, countp);
+       rump_unschedule();
+}
+
+void
+rump_user_schedule(int nlocks)
+{
+
+       rump_schedule();
+
+       if (nlocks)
+               _kernel_lock(nlocks);
+}
+
 struct kmutexobj {
        kmutex_t        mo_lock;
        u_int           mo_refcnt;
diff -r 4286a4fd0677 -r 1afb7e1d52ef sys/rump/librump/rumpkern/ltsleep.c
--- a/sys/rump/librump/rumpkern/ltsleep.c       Thu Oct 15 00:16:28 2009 +0000
+++ b/sys/rump/librump/rumpkern/ltsleep.c       Thu Oct 15 00:28:46 2009 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ltsleep.c,v 1.18 2009/10/04 17:40:34 pooka Exp $       */
+/*     $NetBSD: ltsleep.c,v 1.19 2009/10/15 00:28:46 pooka Exp $       */
 
 /*
  * Copyright (c) 2007 Antti Kantee.  All Rights Reserved.
@@ -29,7 +29,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ltsleep.c,v 1.18 2009/10/04 17:40:34 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ltsleep.c,v 1.19 2009/10/15 00:28:46 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/proc.h>
@@ -42,12 +42,11 @@
 
 struct ltsleeper {
        wchan_t id;
-       kcondvar_t cv;
+       struct rumpuser_cv *cv;
        LIST_ENTRY(ltsleeper) entries;
 };
 
 static LIST_HEAD(, ltsleeper) sleepers = LIST_HEAD_INITIALIZER(sleepers);
-static kmutex_t sleepermtx;
 
 kcondvar_t lbolt; /* Oh Kath Ra */
 
@@ -59,24 +58,19 @@
        int nlocks;
 
        lts.id = ident;
-       cv_init(&lts.cv, NULL);
+       rumpuser_cv_init(&lts.cv);
 
-       while (!mutex_tryenter(&sleepermtx))
-               continue;
-       KERNEL_UNLOCK_ALL(curlwp, &nlocks);
        if (slock)
                simple_unlock(slock);
        LIST_INSERT_HEAD(&sleepers, &lts, entries);
+       kernel_unlock_allbutone(&nlocks);
 
-       /* protected by sleepermtx */
-       cv_wait(&lts.cv, &sleepermtx);
+       /* protected by biglock */
+       rumpuser_cv_wait(lts.cv, rump_giantlock);
 
        LIST_REMOVE(&lts, entries);
-       mutex_exit(&sleepermtx);
-
-       cv_destroy(&lts.cv);
-
-       KERNEL_LOCK(nlocks, curlwp);
+       rumpuser_cv_destroy(lts.cv);
+       kernel_ununlock_allbutone(nlocks);
 
        if (slock && (prio & PNORELOCK) == 0)
                simple_lock(slock);
@@ -92,23 +86,18 @@
        int nlocks;
 
        lts.id = ident;
-       cv_init(&lts.cv, NULL);