[src/trunk]: src/sys/rump/librump/rumpkern implement lwp_create() and a few r...

[pooka <pooka%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/517b5e7fa0e0
branches:  trunk
changeset: 328613:517b5e7fa0e0
user:      pooka <pooka%NetBSD.org@localhost>
date:      Wed Apr 09 23:53:36 2014 +0000

description:
implement lwp_create() and a few related interfaces.
main raison d'impalerment: the aio driver

diffstat:

 sys/rump/librump/rumpkern/lwproc.c       |    5 +-
 sys/rump/librump/rumpkern/rump.c         |    6 +-
 sys/rump/librump/rumpkern/rump_private.h |    4 +-
 sys/rump/librump/rumpkern/scheduler.c    |   20 +++-
 sys/rump/librump/rumpkern/threads.c      |  161 +++++++++++++++++++++++++++---
 sys/rump/librump/rumpkern/vm.c           |   19 +++-
 6 files changed, 185 insertions(+), 30 deletions(-)

diffs (truncated from 400 to 300 lines):

diff -r dcacf4fe2687 -r 517b5e7fa0e0 sys/rump/librump/rumpkern/lwproc.c
--- a/sys/rump/librump/rumpkern/lwproc.c        Wed Apr 09 23:50:45 2014 +0000
+++ b/sys/rump/librump/rumpkern/lwproc.c        Wed Apr 09 23:53:36 2014 +0000
@@ -1,4 +1,4 @@
-/*      $NetBSD: lwproc.c,v 1.28 2014/03/16 15:30:05 pooka Exp $       */
+/*      $NetBSD: lwproc.c,v 1.29 2014/04/09 23:53:36 pooka Exp $       */
 
 /*
  * Copyright (c) 2010, 2011 Antti Kantee.  All Rights Reserved.
@@ -28,7 +28,7 @@
 #define RUMP__CURLWP_PRIVATE
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lwproc.c,v 1.28 2014/03/16 15:30:05 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lwproc.c,v 1.29 2014/04/09 23:53:36 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -109,6 +109,7 @@
        if (rump_proc_vfs_release)
                rump_proc_vfs_release(p);
 
+       doexithooks(p);
        lim_free(p->p_limit);
        pstatsfree(p->p_stats);
        kauth_cred_free(p->p_cred);
diff -r dcacf4fe2687 -r 517b5e7fa0e0 sys/rump/librump/rumpkern/rump.c
--- a/sys/rump/librump/rumpkern/rump.c  Wed Apr 09 23:50:45 2014 +0000
+++ b/sys/rump/librump/rumpkern/rump.c  Wed Apr 09 23:53:36 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: rump.c,v 1.293 2014/04/09 23:49:27 pooka Exp $ */
+/*     $NetBSD: rump.c,v 1.294 2014/04/09 23:53:36 pooka Exp $ */
 
 /*
  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
@@ -26,7 +26,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.293 2014/04/09 23:49:27 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.294 2014/04/09 23:53:36 pooka Exp $");
 
 #include <sys/systm.h>
 #define ELFSIZE ARCH_ELFSIZE
@@ -382,7 +382,7 @@
        }
 
        /* CPUs are up.  allow kernel threads to run */
-       rump_thread_allow();
+       rump_thread_allow(NULL);
 
        rnd_init_softint();
 
diff -r dcacf4fe2687 -r 517b5e7fa0e0 sys/rump/librump/rumpkern/rump_private.h
--- a/sys/rump/librump/rumpkern/rump_private.h  Wed Apr 09 23:50:45 2014 +0000
+++ b/sys/rump/librump/rumpkern/rump_private.h  Wed Apr 09 23:53:36 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: rump_private.h,v 1.80 2014/03/15 15:15:27 pooka Exp $  */
+/*     $NetBSD: rump_private.h,v 1.81 2014/04/09 23:53:36 pooka Exp $  */
 
 /*
  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
@@ -146,7 +146,7 @@
 void   rump_xc_highpri(struct cpu_info *);
 
 void   rump_thread_init(void);
-void   rump_thread_allow(void);
+void   rump_thread_allow(struct lwp *);
 
 void   rump_consdev_init(void);
 
diff -r dcacf4fe2687 -r 517b5e7fa0e0 sys/rump/librump/rumpkern/scheduler.c
--- a/sys/rump/librump/rumpkern/scheduler.c     Wed Apr 09 23:50:45 2014 +0000
+++ b/sys/rump/librump/rumpkern/scheduler.c     Wed Apr 09 23:53:36 2014 +0000
@@ -1,4 +1,4 @@
-/*      $NetBSD: scheduler.c,v 1.36 2014/03/15 15:15:27 pooka Exp $    */
+/*      $NetBSD: scheduler.c,v 1.37 2014/04/09 23:53:36 pooka Exp $    */
 
 /*
  * Copyright (c) 2010, 2011 Antti Kantee.  All Rights Reserved.
@@ -26,7 +26,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: scheduler.c,v 1.36 2014/03/15 15:15:27 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: scheduler.c,v 1.37 2014/04/09 23:53:36 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -539,3 +539,19 @@
 
        /* nothing to do for now */
 }
+
+void
+sched_enqueue(struct lwp *l, bool swtch)
+{
+
+       if (swtch)
+               panic("sched_enqueue with switcheroo");
+       rump_thread_allow(l);
+}
+
+void
+sched_dequeue(struct lwp *l)
+{
+
+       panic("sched_dequeue not implemented");
+}
diff -r dcacf4fe2687 -r 517b5e7fa0e0 sys/rump/librump/rumpkern/threads.c
--- a/sys/rump/librump/rumpkern/threads.c       Wed Apr 09 23:50:45 2014 +0000
+++ b/sys/rump/librump/rumpkern/threads.c       Wed Apr 09 23:53:36 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: threads.c,v 1.22 2014/03/15 15:15:27 pooka Exp $       */
+/*     $NetBSD: threads.c,v 1.23 2014/04/09 23:53:36 pooka Exp $       */
 
 /*
  * Copyright (c) 2007-2009 Antti Kantee.  All Rights Reserved.
@@ -29,7 +29,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: threads.c,v 1.22 2014/03/15 15:15:27 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: threads.c,v 1.23 2014/04/09 23:53:36 pooka Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -37,31 +37,36 @@
 #include <sys/kthread.h>
 #include <sys/malloc.h>
 #include <sys/systm.h>
+#include <sys/queue.h>
 
 #include <rump/rumpuser.h>
 
 #include "rump_private.h"
 
-struct kthdesc {
+struct thrdesc {
        void (*f)(void *);
        void *arg;
-       struct lwp *mylwp;
+       struct lwp *newlwp;
+       int runnable;
+
+       TAILQ_ENTRY(thrdesc) entries;
 };
 
 static bool threads_are_go;
 static struct rumpuser_mtx *thrmtx;
 static struct rumpuser_cv *thrcv;
+static TAILQ_HEAD(, thrdesc) newthr;
 
 static void *
 threadbouncer(void *arg)
 {
-       struct kthdesc *k = arg;
-       struct lwp *l = k->mylwp;
+       struct thrdesc *td = arg;
+       struct lwp *l = td->newlwp;
        void (*f)(void *);
        void *thrarg;
 
-       f = k->f;
-       thrarg = k->arg;
+       f = td->f;
+       thrarg = td->arg;
 
        /* don't allow threads to run before all CPUs have fully attached */
        if (!threads_are_go) {
@@ -77,7 +82,7 @@
        rump_schedule();
 
        /* free dance struct */
-       free(k, M_TEMP);
+       kmem_intr_free(td, sizeof(*td));
 
        if ((curlwp->l_pflag & LP_MPSAFE) == 0)
                KERNEL_LOCK(1, NULL);
@@ -93,17 +98,27 @@
 
        rumpuser_mutex_init(&thrmtx, RUMPUSER_MTX_SPIN);
        rumpuser_cv_init(&thrcv);
+       TAILQ_INIT(&newthr);
 }
 
 void
-rump_thread_allow(void)
+rump_thread_allow(struct lwp *l)
 {
+       struct thrdesc *td;
 
        rumpuser_mutex_enter(thrmtx);
-       threads_are_go = true;
+       if (l == NULL) {
+               threads_are_go = true;
+       } else {
+               TAILQ_FOREACH(td, &newthr, entries) {
+                       if (td->newlwp == l) {
+                               td->runnable = 1;
+                               break;
+                       }
+               }
+       }
        rumpuser_cv_broadcast(thrcv);
        rumpuser_mutex_exit(thrmtx);
-
 }
 
 static struct {
@@ -126,7 +141,7 @@
        char thrstore[MAXCOMLEN];
        const char *thrname = NULL;
        va_list ap;
-       struct kthdesc *k;
+       struct thrdesc *td;
        struct lwp *l;
        int rv;
 
@@ -171,10 +186,14 @@
        }
        KASSERT(fmt != NULL);
 
-       k = malloc(sizeof(*k), M_TEMP, M_WAITOK);
-       k->f = func;
-       k->arg = arg;
-       k->mylwp = l = rump__lwproc_alloclwp(&proc0);
+       /*
+        * Allocate with intr-safe allocator, give that we may be
+        * creating interrupt threads.
+        */
+       td = kmem_intr_alloc(sizeof(*td), KM_SLEEP);
+       td->f = func;
+       td->arg = arg;
+       td->newlwp = l = rump__lwproc_alloclwp(&proc0);
        l->l_flag |= LW_SYSTEM;
        if (flags & KTHREAD_MPSAFE)
                l->l_pflag |= LP_MPSAFE;
@@ -189,11 +208,11 @@
                strlcpy(l->l_name, thrname, MAXCOMLEN);
        }
                
-       rv = rumpuser_thread_create(threadbouncer, k, thrname,
+       rv = rumpuser_thread_create(threadbouncer, td, thrname,
            (flags & KTHREAD_MUSTJOIN) == KTHREAD_MUSTJOIN,
            pri, ci ? ci->ci_index : -1, &l->l_ctxlink);
        if (rv)
-               return rv;
+               return rv; /* XXX */
 
        if (newlp) {
                *newlp = l;
@@ -227,3 +246,107 @@
 
        return rv;
 }
+
+/*
+ * Create a non-kernel thread that is scheduled by a rump kernel hypercall.
+ *
+ * Sounds strange and out-of-place?  yup yup yup.  the original motivation
+ * for this was aio.  This is a very infrequent code path in rump kernels.
+ * XXX: threads created with lwp_create() are eternal for local clients.
+ * however, they are correctly reaped for remote clients with process exit.
+ */
+static void *
+lwpbouncer(void *arg)
+{
+       struct thrdesc *td = arg;
+       struct lwp *l = td->newlwp;
+       void (*f)(void *);
+       void *thrarg;
+       int run;
+
+       f = td->f;
+       thrarg = td->arg;
+
+       /* do not run until we've been enqueued */
+       rumpuser_mutex_enter_nowrap(thrmtx);
+       while ((run = td->runnable) == 0) {
+               rumpuser_cv_wait_nowrap(thrcv, thrmtx);
+       }
+       rumpuser_mutex_exit(thrmtx);
+
+       /* schedule ourselves */
+       rump_lwproc_curlwp_set(l);
+       rump_schedule();
+       kmem_free(td, sizeof(*td));
+
+       /* should we just die instead? */
+       if (run == -1) {
+               rump_lwproc_releaselwp();
+               lwp_userret(l);
+               panic("lwpbouncer reached unreachable");
+       }
+
+       /* run, and don't come back! */
+       f(thrarg);
+       panic("lwp return from worker not supported");
+}