next vforkup chapter: lwpctl

[Antti Kantee <pooka%cs.hut.fi@localhost>]

Hi,

Alexander pointed me at a problem where under suitable conditions a
process with rump syscall hijacking would crash when after vfork() and
sent the attached test program.  Under further examination, it turned
out that the problem is due to libpthread and lwpctl.

Having pthread linked causes malloc to use pthread routines instead
of the libc stubs.  Now, the vfork() child will use a pointer to the
parent's lwpctl area and thinks it is running on LWPCTL_CPU_NONE (-1).
When malloc uses this to index the arena map, it unsurprisingly gets
total garbage back.

The following patch makes a vfork child update the parent's lwpctl area
while the child is running.  Comments or better ideas?

-- 
älä karot toivorikkauttas, kyl rätei ja lumpui piisaa

#include <stddef.h>
#include <stdlib.h>
#include <unistd.h>

int main()
{
    malloc(1);
    switch(vfork())
    {
    case -1:
        return EXIT_FAILURE;
    case 0:
        malloc(1);
        _exit(EXIT_FAILURE);
    default:
        ;
    }

    return EXIT_SUCCESS;
}

Index: kern/kern_exec.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_exec.c,v
retrieving revision 1.305
diff -p -u -r1.305 kern_exec.c
--- kern/kern_exec.c    18 Jan 2011 08:21:03 -0000      1.305
+++ kern/kern_exec.c    14 Feb 2011 12:25:28 -0000
@@ -979,6 +979,7 @@ execve1(struct lwp *l, const char *path,
                mutex_enter(proc_lock);
                p->p_lflag &= ~PL_PPWAIT;
                cv_broadcast(&p->p_pptr->p_waitcv);
+               l->l_lwpctl = NULL; /* borrowed from parent */
                mutex_exit(proc_lock);
        }
 
Index: kern/kern_exit.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_exit.c,v
retrieving revision 1.231
diff -p -u -r1.231 kern_exit.c
--- kern/kern_exit.c    18 Dec 2010 01:36:19 -0000      1.231
+++ kern/kern_exit.c    14 Feb 2011 12:25:28 -0000
@@ -343,6 +343,7 @@ exit1(struct lwp *l, int rv)
        if (p->p_lflag & PL_PPWAIT) {
                p->p_lflag &= ~PL_PPWAIT;
                cv_broadcast(&p->p_pptr->p_waitcv);
+               l->l_lwpctl = NULL; /* borrowed from parent */
        }
 
        if (SESS_LEADER(p)) {
Index: kern/kern_lwp.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_lwp.c,v
retrieving revision 1.154
diff -p -u -r1.154 kern_lwp.c
--- kern/kern_lwp.c     17 Jan 2011 08:26:58 -0000      1.154
+++ kern/kern_lwp.c     14 Feb 2011 12:25:29 -0000
@@ -696,6 +696,12 @@ lwp_create(lwp_t *l1, proc_t *p2, vaddr_
        l2->l_pflag = LP_MPSAFE;
        TAILQ_INIT(&l2->l_ld_locks);
 
+       /* For vfork, borrow parent's lwpctl context */
+       if (flags & LWP_VFORK && l1->l_lwpctl) {
+               l2->l_lwpctl = l1->l_lwpctl;
+               l2->l_flag |= LW_LWPCTL;
+       }
+
        /*
         * If not the first LWP in the process, grab a reference to the
         * descriptor table.
@@ -1376,6 +1382,17 @@ lwp_userret(struct lwp *l)
                        KASSERT(0);
                        /* NOTREACHED */
                }
+
+               /* update lwpctl process (for vfork child_return) */
+               if (l->l_flag & LW_LWPCTL) {
+                       lwp_lock(l);
+                       l->l_flag &= ~LW_LWPCTL;
+                       lwp_unlock(l);
+                       KPREEMPT_DISABLE(l);
+                       l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
+                       l->l_lwpctl->lc_pctr++;
+                       KPREEMPT_ENABLE(l);
+               }
        }
 
 #ifdef KERN_SA
@@ -1529,6 +1546,10 @@ lwp_ctl_alloc(vaddr_t *uaddr)
        l = curlwp;
        p = l->l_proc;
 
+       /* don't allow a vforked process to create lwp ctls */
+       if (p->p_lflag & PL_PPWAIT)
+               return EBUSY;
+
        if (l->l_lcpage != NULL) {
                lcp = l->l_lcpage;
                *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
@@ -1653,11 +1674,16 @@ lwp_ctl_alloc(vaddr_t *uaddr)
 void
 lwp_ctl_free(lwp_t *l)
 {
+       struct proc *p = l->l_proc;
        lcproc_t *lp;
        lcpage_t *lcp;
        u_int map, offset;
 
-       lp = l->l_proc->p_lwpctl;
+       /* don't free a lwp context we borrowed for vfork */
+       if (p->p_lflag & PL_PPWAIT)
+               return;
+
+       lp = p->p_lwpctl;
        KASSERT(lp != NULL);
 
        lcp = l->l_lcpage;
Index: sys/lwp.h
===================================================================
RCS file: /cvsroot/src/sys/sys/lwp.h,v
retrieving revision 1.142
diff -p -u -r1.142 lwp.h
--- sys/lwp.h   28 Jan 2011 16:58:27 -0000      1.142
+++ sys/lwp.h   14 Feb 2011 12:25:29 -0000
@@ -214,6 +214,7 @@ extern lwp_t                lwp0;           /* LWP for 
proc0. *
 
 /* These flags are kept in l_flag. */
 #define        LW_IDLE         0x00000001 /* Idle lwp. */
+#define        LW_LWPCTL       0x00000002 /* Adjust lwpctl in userret */
 #define        LW_SINTR        0x00000080 /* Sleep is interruptible. */
 #define        LW_SA_SWITCHING 0x00000100 /* SA LWP in context switch */
 #define        LW_SYSTEM       0x00000200 /* Kernel thread */
@@ -258,7 +259,7 @@ extern lwp_t                lwp0;           /* LWP for 
proc0. *
  * user.
  */
 #define        LW_USERRET 
(LW_WEXIT|LW_PENDSIG|LW_WREBOOT|LW_WSUSPEND|LW_WCORE|\
-                   LW_SA_BLOCKING|LW_SA_UPCALL)
+                   LW_SA_BLOCKING|LW_SA_UPCALL|LW_LWPCTL)
 
 /*
  * Status values.