Subject: Update: rough SA patch (works on SMP)
To: None <tech-kern@netbsd.org>
From: Stephan Uphoff <ups@stups.com>
List: tech-kern
Date: 06/26/2003 16:29:37
Hi,

I updated the patch to Nathan's SA kernel code to eliminate
signal handling problems.
( ^C problems as reported by Frank van der Linden and Matthew Green 
and ^Z and related job control handling )

Phoenix seems to now run stable with the new patch to the extent where
I am running out of interesting web-sites to visit ;-)
(libpthread as of yesterday)

The major difference between the original code and the patched version
is the handling of sa_vp_repossess.
The original sa_vp_repossess assumes that at the time it is running:

(1) The lwp holding the virtual processor is not LSONPROC.
    True for UP - but not for SMP
	
(2) The lwp holding the virtual processor does not hold any kernel
    resources (locks,...) and can be stopped and recycled.
    Unfortunately this is not true if the SA code runs out of stacks
    or the lwp blocks in userret (example: pagefault on copyout).

The patch relies on the lwp holding the virtual processor to donate it.

There are better ways to handle this problem.
Again: The patch is extremely ugly, inefficient, is only meant as a
temporary band-aid and will be trashed as soon as possible.


	Stephan
 

Index: sys/kern/kern_exit.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_exit.c,v
retrieving revision 1.115
diff -c -r1.115 kern_exit.c
*** sys/kern/kern_exit.c	2003/03/19 11:36:33	1.115
--- sys/kern/kern_exit.c	2003/06/26 19:53:50
***************
*** 179,186 ****
--- 179,189 ----
  	 */
  	sa = 0;
  	if (p->p_sa != NULL) {
+ 
  		l->l_flag &= ~L_SA;
+ #if 0
  		p->p_flag &= ~P_SA;
+ #endif
  		sa = 1;
  	}
  
***************
*** 461,469 ****
  	 * them) and then wait for everyone else to finish.  
  	 */
  	LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
  		l2->l_flag &= ~(L_DETACHED|L_SA);
  		if ((l2->l_stat == LSSLEEP && (l2->l_flag & L_SINTR)) ||
! 		    l2->l_stat == LSSUSPENDED) {
  			SCHED_LOCK(s);
  			setrunnable(l2);
  			SCHED_UNLOCK(s);
--- 464,481 ----
  	 * them) and then wait for everyone else to finish.  
  	 */
  	LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
+ #if 0
  		l2->l_flag &= ~(L_DETACHED|L_SA);
+ #endif	
+ 		l2->l_flag &= ~(L_DETACHED);
+ 	
+ 		if(l2->l_flag & L_SA_WANTS_VP)
+ 		{
+ 			wakeup(l2);
+ 		}
+ 
  		if ((l2->l_stat == LSSLEEP && (l2->l_flag & L_SINTR)) ||
! 		    l2->l_stat == LSSUSPENDED || l2->l_stat == LSSTOP) {
  			SCHED_LOCK(s);
  			setrunnable(l2);
  			SCHED_UNLOCK(s);
Index: sys/kern/kern_sa.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_sa.c,v
retrieving revision 1.16
diff -c -r1.16 kern_sa.c
*** sys/kern/kern_sa.c	2003/05/28 22:17:20	1.16
--- sys/kern/kern_sa.c	2003/06/26 19:53:51
***************
*** 53,58 ****
--- 53,59 ----
  
  #include <uvm/uvm_extern.h>
  
+ static void sa_vp_donate(struct lwp *);
  static int sa_newcachelwp(struct lwp *);
  static struct lwp *sa_vp_repossess(struct lwp *l);
  
***************
*** 72,77 ****
--- 73,95 ----
  #endif
  
  
+ 
+ #define SA_LWP_STATE_LOCK(l,s)      \
+     do {                            \
+            s =  (l)->l_flag ;       \
+            (l)->l_flag &= ~L_SA;    \
+     } while (0)                     
+            
+ 
+ 
+ #define SA_LWP_STATE_UNLOCK(l,s)      \
+     do {                               \
+ 	(l)->l_flag |=  ( s & L_SA);   \
+     } while (0)                     
+    
+ 
+ 
+ 
  /*
   * sadata_upcall_alloc:
   *
***************
*** 139,144 ****
--- 157,166 ----
  		simple_lock_init(&sa->sa_lock);
  		sa->sa_flag = SCARG(uap, flags) & SA_FLAG_ALL;
  		sa->sa_vp = NULL;
+ 		
+ 		sa->sa_old_lwp = NULL;
+ 		sa->sa_vp_wait_count = 0;
+ 
  		sa->sa_idle = NULL;
  		sa->sa_woken = NULL;
  		sa->sa_concurrency = 1;
***************
*** 275,281 ****
--- 297,305 ----
  void
  sa_yield(struct lwp *l)
  {
+ #if 0
  	struct lwp *l2;
+ #endif
  	struct proc *p = l->l_proc;
  	struct sadata *sa = p->p_sa;
  	int s, ret;
***************
*** 284,296 ****
  	 * If we're the last running LWP, stick around to recieve
  	 * signals.
  	 */
  	if (p->p_nrlwps == 1) {
  		DPRINTFN(1,("sa_yield(%d.%d) going dormant\n",
  		    p->p_pid, l->l_lid));
  		/*
  		 * A signal will probably wake us up. Worst case, the upcall
  		 * happens and just causes the process to yield again.
! 		 */
  		s = splsched();	/* Protect from timer expirations */
  		KDASSERT(sa->sa_vp == l);
  		/*
--- 308,328 ----
  	 * If we're the last running LWP, stick around to recieve
  	 * signals.
  	 */
+ #if 0
  	if (p->p_nrlwps == 1) {
+ #endif
  		DPRINTFN(1,("sa_yield(%d.%d) going dormant\n",
  		    p->p_pid, l->l_lid));
  		/*
  		 * A signal will probably wake us up. Worst case, the upcall
  		 * happens and just causes the process to yield again.
! 		 */	
! 	SCHED_ASSERT_UNLOCKED();
! 
! 		sa_vp_donate(l);
! 
! 	SCHED_ASSERT_UNLOCKED();
! 
  		s = splsched();	/* Protect from timer expirations */
  		KDASSERT(sa->sa_vp == l);
  		/*
***************
*** 299,316 ****
  		 * going to sleep. It might make more sense for this to
  		 * be handled inside of tsleep....
  		 */
! 		ret = 0;
  		while  ((ret == 0) && (p->p_userret == NULL)) {
  			sa->sa_idle = l;
  			l->l_flag &= ~L_SA;
  			ret = tsleep((caddr_t) l, PUSER | PCATCH, "sawait", 0);
  			l->l_flag |= L_SA;
  			sa->sa_idle = NULL;
! 			sa->sa_vp = l;
  		}
  		splx(s);
  		DPRINTFN(1,("sa_yield(%d.%d) returned\n",
  		    p->p_pid, l->l_lid));
  	} else {
  		DPRINTFN(1,("sa_yield(%d.%d) stepping aside\n", p->p_pid, l->l_lid));
  	
--- 331,365 ----
  		 * going to sleep. It might make more sense for this to
  		 * be handled inside of tsleep....
  		 */
! 		ret = 0;	
! 
  		while  ((ret == 0) && (p->p_userret == NULL)) {
  			sa->sa_idle = l;
  			l->l_flag &= ~L_SA;
+ 			SCHED_ASSERT_UNLOCKED();
+ 		
+ 
  			ret = tsleep((caddr_t) l, PUSER | PCATCH, "sawait", 0);
+ 
+ 			SCHED_ASSERT_UNLOCKED();
+ 
  			l->l_flag |= L_SA;
  			sa->sa_idle = NULL;
! 			splx(s);
! 			sa_vp_donate(l);
! 			KDASSERT(sa->sa_vp == l);
! 
! 		
! 			s = splsched();	/* Protect from timer expirations */
! 
  		}
+ 
+ 
+ 		l->l_flag |= L_SA_UPCALL; /* Must get the virtual CPU */
  		splx(s);
  		DPRINTFN(1,("sa_yield(%d.%d) returned\n",
  		    p->p_pid, l->l_lid));
+ #if 0
  	} else {
  		DPRINTFN(1,("sa_yield(%d.%d) stepping aside\n", p->p_pid, l->l_lid));
  	
***************
*** 334,339 ****
--- 383,389 ----
  		KDASSERT(p->p_flag & P_WEXIT);
  		/* mostly NOTREACHED */
  	}
+ #endif
  }
  
  
***************
*** 371,382 ****
  	struct sadata_upcall *sau;
  	struct sadata *sa = l->l_proc->p_sa;
  	stack_t st;
  
! 	l->l_flag &= ~L_SA; /* XXX prevent recursive upcalls if we sleep for
! 			      memory */
  	sau = sadata_upcall_alloc(1);
- 	l->l_flag |= L_SA;
  
  	if (sa->sa_nstacks == 0) {
  		/* assign to assure that it gets freed */
  		sau->sau_type = type & ~SA_UPCALL_DEFER;
--- 421,436 ----
  	struct sadata_upcall *sau;
  	struct sadata *sa = l->l_proc->p_sa;
  	stack_t st;
+ 	int s;
  
! 	/* XXX prevent recursive upcalls if we sleep formemory */
! 	SA_LWP_STATE_LOCK(l,s);
! 
  	sau = sadata_upcall_alloc(1);
  
+ 	SA_LWP_STATE_UNLOCK(l,s);
+ 
+ 
  	if (sa->sa_nstacks == 0) {
  		/* assign to assure that it gets freed */
  		sau->sau_type = type & ~SA_UPCALL_DEFER;
***************
*** 477,482 ****
--- 531,542 ----
  	    type, sa->sa_vp ? sa->sa_vp->l_lid : 0));
  	SCHED_ASSERT_LOCKED();
  
+ 	if (p->p_flag & P_WEXIT)
+ 	{
+ 		mi_switch(l,0);
+ 		return;
+ 	}
+ 
  	if (sa->sa_vp == l) {
  		/*
  		 * Case 1: we're blocking for the first time; generate
***************
*** 497,502 ****
--- 557,564 ----
  			 * XXX the recovery from this situation deserves
  			 * XXX more thought.
  			 */
+ 
+ 			/* XXXUPSXXX Should only happen with concurrency > 1 */
  #ifdef DIAGNOSTIC
  			printf("sa_switch(%d.%d): no cached LWP for upcall.\n",
  			    p->p_pid, l->l_lid);
***************
*** 547,552 ****
--- 609,615 ----
  
  		l->l_flag |= L_SA_BLOCKING;
  		l2->l_priority = l2->l_usrpri;
+ 		sa->sa_vp = l2;
  		setrunnable(l2);
  		PRELE(l2); /* Remove the artificial hold-count */
  
***************
*** 564,572 ****
  		 */
  		if (sa->sa_idle)
  			l2 = NULL;
! 		else
! 			l2 = sa->sa_vp;
  	} else {
  		/*
  		 * Case 3: The VP is empty. As in case 2, we were
  		 * woken up and called tsleep again, but additionally,
--- 627,640 ----
  		 */
  		if (sa->sa_idle)
  			l2 = NULL;
! 		else {
! 			l2 = sa->sa_vp; /* XXXUPSXXX Unfair advantage for l2 ? */
! 			if((l2->l_stat != LSRUN) || ((l2->l_flag & L_INMEM) == 0))
! 				l2 = NULL;
! 		}
  	} else {
+ 
+ #if 0
  		/*
  		 * Case 3: The VP is empty. As in case 2, we were
  		 * woken up and called tsleep again, but additionally,
***************
*** 585,596 ****
--- 653,681 ----
  			mi_switch(l, NULL);
  			return;
  		}
+ #else
+ 		mi_switch(l, NULL);
+ 		return;
+ #endif
+ 		
  	sa_upcall_failed:
+ #if 0
  		cpu_setfunc(l2, sa_yieldcall, l2);
  
  		l2->l_priority = l2->l_usrpri;
  		setrunnable(l2);
  		PRELE(l2); /* Remove the artificial hold-count */
+ #else
+ 		
+ 		/* sa_putcachelwp does not block because we have a hold count on l2 */
+ 		sa_putcachelwp(p, l2);
+ 		PRELE(l2); /* Remove the artificial hold-count */
+ 
+ 		mi_switch(l, NULL);
+ 		return;
+ 
+ 
+ #endif
  	}
  
  
***************
*** 621,628 ****
--- 706,715 ----
  	 */
  	if (l->l_flag & L_SA_BLOCKING)
  		l->l_flag |= L_SA_UPCALL;
+ #if 0
  	else
  		sa_vp_repossess(l);
+ #endif
  }
  
  void
***************
*** 631,636 ****
--- 718,724 ----
  	struct lwp *l;
  	struct proc *p;
  	struct sadata *sa;
+ 	int s;
  
  	l = arg;
  	p = l->l_proc;
***************
*** 643,653 ****
--- 731,745 ----
  		/* Allocate the next cache LWP */
  		DPRINTFN(6,("sa_switchcall(%d.%d) allocating LWP\n",
  		    p->p_pid, l->l_lid));
+ 		SA_LWP_STATE_LOCK(l,s);
  		sa_newcachelwp(l);
+ 		SA_LWP_STATE_UNLOCK(l,s);
+ 
  	}
  	upcallret(l);
  }
  
+ #if 0
  void
  sa_yieldcall(void *arg)
  {
***************
*** 672,677 ****
--- 764,770 ----
  	sa_yield(l);
  	upcallret(l);
  }
+ #endif
  
  static int
  sa_newcachelwp(struct lwp *l)
***************
*** 768,779 ****
  	void *stack, *ap;
  	ucontext_t u, *up;
  	int i, nsas, nint, nevents, type;
  
  	p = l->l_proc;
  	sa = p->p_sa;
  
  	KERNEL_PROC_LOCK(l);
! 	l->l_flag &= ~L_SA;
  
  	DPRINTFN(7,("sa_upcall_userret(%d.%d %x) \n", p->p_pid, l->l_lid,
  	    l->l_flag));
--- 861,875 ----
  	void *stack, *ap;
  	ucontext_t u, *up;
  	int i, nsas, nint, nevents, type;
+ 	int s;
  
  	p = l->l_proc;
  	sa = p->p_sa;
+ 	
+ 	SCHED_ASSERT_UNLOCKED();
  
  	KERNEL_PROC_LOCK(l);
! 	SA_LWP_STATE_LOCK(l,s);
  
  	DPRINTFN(7,("sa_upcall_userret(%d.%d %x) \n", p->p_pid, l->l_lid,
  	    l->l_flag));
***************
*** 784,791 ****
  		DPRINTFN(8,("sa_upcall_userret(%d.%d) unblocking\n",
  		    p->p_pid, l->l_lid));
  
  		sau = sadata_upcall_alloc(1);
! 		
  		while (sa->sa_nstacks == 0) {
  			/*
  			 * This should be a transient condition, so we'll just
--- 880,888 ----
  		DPRINTFN(8,("sa_upcall_userret(%d.%d) unblocking\n",
  		    p->p_pid, l->l_lid));
  
+ 
  		sau = sadata_upcall_alloc(1);
! 
  		while (sa->sa_nstacks == 0) {
  			/*
  			 * This should be a transient condition, so we'll just
***************
*** 805,820 ****
  			 * Ideally, tsleep() would have a variant that took
  			 * a LWP to switch to.
  			 */
! 			l->l_flag &= ~L_SA;
  			DPRINTFN(7, ("sa_upcall_userret(%d.%d) sleeping"
  			    " for stacks\n", l->l_proc->p_pid, l->l_lid));
  			tsleep((caddr_t) &sa->sa_nstacks, PWAIT|PCATCH, 
  			    "sastacks", 0);
! 			if (p->p_flag & P_WEXIT)
! 				lwp_exit(l);
! 			l->l_flag |= L_SA;
  		}
  		l2 = sa_vp_repossess(l);
  
  		KDASSERT(sa->sa_nstacks > 0);
  
--- 902,939 ----
  			 * Ideally, tsleep() would have a variant that took
  			 * a LWP to switch to.
  			 */
! 
! 			if (p->p_flag & P_WEXIT)
! 			{
! 				sadata_upcall_free(sau);
! 				lwp_exit(l);
! 			}
! 
  			DPRINTFN(7, ("sa_upcall_userret(%d.%d) sleeping"
  			    " for stacks\n", l->l_proc->p_pid, l->l_lid));
  			tsleep((caddr_t) &sa->sa_nstacks, PWAIT|PCATCH, 
  			    "sastacks", 0);
! 
! 			/* XXXUPSXXX NEED TO STOP THE LWP HERE ON REQUEST */
! 
! 		
  		}
+ 
+ 		if (p->p_flag & P_WEXIT) {
+ 			sadata_upcall_free(sau);
+ 			lwp_exit(l);
+ 		}
+ 
+ 		SCHED_ASSERT_UNLOCKED();
+ 
  		l2 = sa_vp_repossess(l);
+ 		
+ 		SCHED_ASSERT_UNLOCKED();
+ 			
+ 		if(l2 == NULL) {
+ 			sadata_upcall_free(sau);
+ 			lwp_exit(l);
+ 		}
  
  		KDASSERT(sa->sa_nstacks > 0);
  
***************
*** 836,843 ****
  		}
  		l->l_flag &= ~L_SA_BLOCKING;
  	}
  
! 	KDASSERT(SIMPLEQ_EMPTY(&sa->sa_upcalls) == 0);
  
  	sau = SIMPLEQ_FIRST(&sa->sa_upcalls);
  	SIMPLEQ_REMOVE_HEAD(&sa->sa_upcalls, sau_next);
--- 955,972 ----
  		}
  		l->l_flag &= ~L_SA_BLOCKING;
  	}
+ 
+ 	if (SIMPLEQ_EMPTY(&sa->sa_upcalls))
+ 	{		
+ 
+ 		l->l_flag &= ~L_SA_UPCALL;
  
! 		sa_vp_donate(l);	
! 		
! 		SA_LWP_STATE_UNLOCK(l,s);
! 		KERNEL_PROC_UNLOCK(l);
! 		return;
! 	}	
  
  	sau = SIMPLEQ_FIRST(&sa->sa_upcalls);
  	SIMPLEQ_REMOVE_HEAD(&sa->sa_upcalls, sau_next);
***************
*** 961,970 ****
  	    l->l_lid, type));
  
  	cpu_upcall(l, type, nevents, nint, sapp, ap, stack, sa->sa_upcall);
! 	l->l_flag |= L_SA;
  	KERNEL_PROC_UNLOCK(l);
  }
  
  static struct lwp *
  sa_vp_repossess(struct lwp *l)
  {
--- 1090,1109 ----
  	    l->l_lid, type));
  
  	cpu_upcall(l, type, nevents, nint, sapp, ap, stack, sa->sa_upcall);
! 
! 	SCHED_ASSERT_UNLOCKED();
! 
! 	sa_vp_donate(l);
! 	SCHED_ASSERT_UNLOCKED();
! 
! 	/* May not be reached  */
! 	
! 	SA_LWP_STATE_UNLOCK(l,s);
!        
  	KERNEL_PROC_UNLOCK(l);
  }
  
+ #if 0
  static struct lwp *
  sa_vp_repossess(struct lwp *l)
  {
***************
*** 977,982 ****
--- 1116,1125 ----
  	 * Put ourselves on the virtual processor and note that the
  	 * previous occupant of that position was interrupted.
  	 */
+ 
+ 
+ 
+ 
  	l2 = sa->sa_vp;
  	sa->sa_vp = l;
  	if (sa->sa_idle == l2)
***************
*** 1011,1016 ****
--- 1154,1274 ----
  	}
  	return l2;
  }
+ #endif
+ 
+ static struct lwp *
+ sa_vp_repossess(struct lwp *l)
+ {
+ 	struct lwp *l2;
+ 	struct proc *p = l->l_proc;
+ 	struct sadata *sa = p->p_sa;
+ 	int s;
+ 		
+ 	SCHED_ASSERT_UNLOCKED();
+ 
+ 	l->l_flag |= L_SA_WANTS_VP;
+ 	sa->sa_vp_wait_count++;
+ 
+ 	if(sa->sa_idle != NULL)
+ 	{
+ 		/* XXXUPSXXX Simple but slow */
+ 		wakeup(sa->sa_idle);
+ 	}
+ 	else
+ 	{
+ 		SCHED_LOCK(s);
+ 		sa->sa_vp->l_flag |= L_SA_UPCALL;
+ 		/* kick the process */
+ 		signotify(p);
+ 		SCHED_UNLOCK(s);
+ 	}
+ 
+ 	
+ 	SCHED_ASSERT_UNLOCKED();
+ 
+ 	while(sa->sa_vp != l)
+ 	{
+ 	
+ 
+ 		tsleep((caddr_t) l, PWAIT, 
+ 		       "sa processor", 0);
+ 		
+ 		/* XXXUPSXXX NEED TO STOP THE LWP HERE ON REQUEST ??? */
+ 	       	if (p->p_flag & P_WEXIT) {
+ 			l->l_flag &= ~L_SA_WANTS_VP;
+ 			sa->sa_vp_wait_count--;
+ 			return 0;
+ 		}
+ 	}
+ 
+ 	l2 = sa->sa_old_lwp;
+ 
+ 	return l2;
+ }
+ 
+ static void
+ sa_vp_donate(struct lwp *l)
+ {
+ 	
+ 	struct proc *p = l->l_proc;
+ 	struct sadata *sa = p->p_sa;	
+ 	struct lwp *l2;
+ 	int s;
+ 
+ 	SCHED_ASSERT_UNLOCKED();
+ 
+ 	if (sa->sa_vp_wait_count == 0)
+ 	{
+ 		return;
+ 	}
+ 
+ 	LIST_FOREACH(l2, &p->p_lwps, l_sibling)
+ 	{
+ 		if(l2->l_flag &  L_SA_WANTS_VP)
+ 		{
+ 		
+ 			SCHED_LOCK(s);
+ 			
+ 			sa_putcachelwp(p, l);
+ 			sa->sa_vp = l2;
+ 			sa->sa_vp_wait_count--;
+ 			l2->l_flag &= ~L_SA_WANTS_VP;
+ 			sa->sa_old_lwp = l;
+ 			
+ 			sched_wakeup((caddr_t) l2);	
+ 
+ 			KERNEL_PROC_UNLOCK(l);
+ 
+ 
+ 			if((l2->l_stat == LSRUN) && ((l2->l_flag & L_INMEM) != 0))
+ 				mi_switch(l,l2);
+ 			else
+ 				mi_switch(l,NULL);
+ 	
+ 			/*
+ 			 * This isn't quite a NOTREACHED; we may get here if
+ 			 * the process exits before this LWP is reused. In
+ 			 * that case, we want to call lwp_exit(), which will
+ 			 * be done by the userret() hooks.
+ 			 */
+ 			SCHED_ASSERT_UNLOCKED();
+ 			splx(s);
+ 
+ 			KERNEL_PROC_LOCK(l);
+ 
+ 
+ 			KDASSERT(p->p_flag & P_WEXIT);
+ 			/* mostly NOTREACHED */
+ 
+ 			lwp_exit(l);
+ 		}
+ 	}
+ 	
+ #ifdef DIAGNOSTIC
+ 	printf("sa_vp_donate couldn't find someone to donate the CPU to \n");
+ #endif	
+ }
+ 
  
  
  #ifdef DEBUG
Index: sys/kern/kern_sig.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_sig.c,v
retrieving revision 1.141
diff -c -r1.141 kern_sig.c
*** sys/kern/kern_sig.c	2003/05/20 17:42:51	1.141
--- sys/kern/kern_sig.c	2003/06/26 19:53:52
***************
*** 528,535 ****
  		(void) spl0();		/* XXXSMP */
  	}
  
  	while (tsleep((caddr_t) ps, PPAUSE|PCATCH, "pause", 0) == 0)
! 		/* void */;
  	/* always return EINTR rather than ERESTART... */
  	return (EINTR);
  }
--- 528,538 ----
  		(void) spl0();		/* XXXSMP */
  	}
  
+ 
+ 
  	while (tsleep((caddr_t) ps, PPAUSE|PCATCH, "pause", 0) == 0)
! 		;	/* void  */
! 	
  	/* always return EINTR rather than ERESTART... */
  	return (EINTR);
  }
***************
*** 867,873 ****
  	 */
  	if ((prop & SA_CANTMASK) == 0
  	    && p->p_sigctx.ps_sigwaited < 0
! 	    && sigismember(&p->p_sigctx.ps_sigwait, signum)) {
  		sigdelset(&p->p_sigctx.ps_siglist, signum);
  		p->p_sigctx.ps_sigwaited = signum;
  		sigemptyset(&p->p_sigctx.ps_sigwait);
--- 870,877 ----
  	 */
  	if ((prop & SA_CANTMASK) == 0
  	    && p->p_sigctx.ps_sigwaited < 0
! 	    && sigismember(&p->p_sigctx.ps_sigwait, signum)
! 	    &&  p->p_stat != SSTOP) {
  		sigdelset(&p->p_sigctx.ps_siglist, signum);
  		p->p_sigctx.ps_sigwaited = signum;
  		sigemptyset(&p->p_sigctx.ps_sigwait);
***************
*** 889,895 ****
  	if (dolock)
  		SCHED_LOCK(s);
  
! 	if (p->p_nrlwps > 0) {
  		/*
  		 * At least one LWP is running or on a run queue. 
  		 * The signal will be noticed when one of them returns 
--- 893,900 ----
  	if (dolock)
  		SCHED_LOCK(s);
  
! 	/* XXXUPSXXX LWPs might go to sleep without passing signal handling */ 
! 	if (p->p_nrlwps > 0 && (p->p_stat != SSTOP)) {
  		/*
  		 * At least one LWP is running or on a run queue. 
  		 * The signal will be noticed when one of them returns 
***************
*** 903,909 ****
  	} else {
  		/* Process is sleeping or stopped */
  		if (p->p_flag & P_SA) {
! 			l = p->p_sa->sa_idle;
  		} else {
  			/*
  			 * Find out if any of the sleeps are interruptable,
--- 908,924 ----
  	} else {
  		/* Process is sleeping or stopped */
  		if (p->p_flag & P_SA) {
! 			struct lwp *l2 = p->p_sa->sa_vp;
! 			l = NULL;		
! 			allsusp = 1;
! 
! 			if ((l2->l_stat == LSSLEEP) &&  (l2->l_flag & L_SINTR))
! 				l = l2; 
! 			else if (l2->l_stat == LSSUSPENDED)
! 				suspended = l2;
! 			else if ((l2->l_stat != LSZOMB) && 
! 				 (l2->l_stat != LSDEAD))
! 				allsusp = 0;
  		} else {
  			/*
  			 * Find out if any of the sleeps are interruptable,
***************
*** 938,943 ****
--- 953,959 ----
  				goto out;
  			}
  
+ 
  			/*
  			 * When a sleeping process receives a stop
  			 * signal, process immediately if possible.
***************
*** 962,967 ****
--- 978,984 ----
  				goto out;
  			}
  
+ 
  			if (l == NULL) {
  				/*
  				 * Special case: SIGKILL of a process
***************
*** 1059,1064 ****
--- 1076,1082 ----
  	if (l->l_priority > PUSER)
  		l->l_priority = PUSER;
   run:
+ 	
  	setrunnable(l);		/* XXXSMP: recurse? */
   out:
  	/* XXXSMP: works, but icky */
***************
*** 1074,1079 ****
--- 1092,1102 ----
  	siginfo_t *si;	
  
  	if (p->p_flag & P_SA) {
+ 
+ 		/* XXXUPSXXX What if not on sa_vp ? */
+ 
+ 		int s = l->l_flag & L_SA;
+ 		l->l_flag &= ~L_SA; 
  		si = pool_get(&siginfo_pool, PR_WAITOK);
  		si->si_signo = sig;
  		si->si_errno = 0;
***************
*** 1086,1091 ****
--- 1109,1117 ----
  
  		sa_upcall(l, SA_UPCALL_SIGNAL | SA_UPCALL_DEFER, le, li, 
  			    sizeof(siginfo_t), si);
+ 
+ 		
+ 		l->l_flag |= s;
  		return;
  	}
  
***************
*** 1140,1145 ****
--- 1166,1181 ----
  	int		dolock = (l->l_flag & L_SINTR) == 0, locked = !dolock;
  	sigset_t	ss;
  
+ 	
+ 	if (l->l_flag & L_SA) {
+ 		struct sadata *sa = p->p_sa;	
+ 
+ 		/* Bail out if we do not own the virtual processor */
+ 		if (sa->sa_vp != l)
+ 			return 0;
+ 	}
+ 
+ 
  	if (p->p_stat == SSTOP) {
  		/*
  		 * The process is stopped/stopping. Stop ourselves now that
***************
*** 1322,1327 ****
--- 1358,1365 ----
  
  	SCHED_ASSERT_LOCKED();
  
+ 
+ 
  	/* XXX lock process LWP state */
  	p->p_stat = SSTOP;
  	p->p_flag &= ~P_WAITED;
***************
*** 1333,1339 ****
  	 */
  	   
  	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
! 		if (l->l_stat == LSONPROC) {
  			/* XXX SMP this assumes that a LWP that is LSONPROC
  			 * is curlwp and hence is about to be mi_switched 
  			 * away; the only callers of proc_stop() are:
--- 1371,1377 ----
  	 */
  	   
  	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
! 		if ((l->l_stat == LSONPROC) && (l == curlwp)) {
  			/* XXX SMP this assumes that a LWP that is LSONPROC
  			 * is curlwp and hence is about to be mi_switched 
  			 * away; the only callers of proc_stop() are:
***************
*** 1346,1352 ****
  			 */
  			l->l_stat = LSSTOP;
  			p->p_nrlwps--;
! 		} else if (l->l_stat == LSRUN) {
  			/* Remove LWP from the run queue */
  			remrunqueue(l);
  			l->l_stat = LSSTOP;
--- 1384,1397 ----
  			 */
  			l->l_stat = LSSTOP;
  			p->p_nrlwps--;
! 		}
! 		 else if ( (l->l_stat == LSSLEEP) && (l->l_flag & L_SINTR)) {
! 			setrunnable(l);
! 		}
! 
! /* !!!UPS!!! FIX ME */
! #if 0
! else if (l->l_stat == LSRUN) {
  			/* Remove LWP from the run queue */
  			remrunqueue(l);
  			l->l_stat = LSSTOP;
***************
*** 1374,1382 ****
  			    p->p_pid, l->l_lid, l->l_stat);
  		}
  #endif
  	}
  	/* XXX unlock process LWP state */
! 		    
  	sched_wakeup((caddr_t)p->p_pptr);
  }
  
--- 1419,1429 ----
  			    p->p_pid, l->l_lid, l->l_stat);
  		}
  #endif
+ #endif
  	}
  	/* XXX unlock process LWP state */
! 
! 	    
  	sched_wakeup((caddr_t)p->p_pptr);
  }
  
***************
*** 1584,1591 ****
--- 1631,1641 ----
  void
  sigexit(struct lwp *l, int signum)
  {
+ 
  	struct proc	*p;
+ #if 0
  	struct lwp	*l2;
+ #endif
  	int		error, exitsig;
  
  	p = l->l_proc;
***************
*** 1601,1611 ****
--- 1651,1663 ----
  	p->p_flag |= P_WEXIT;
  	/* We don't want to switch away from exiting. */
  	/* XXX multiprocessor: stop LWPs on other processors. */
+ #if 0
  	if (p->p_flag & P_SA) {
  		LIST_FOREACH(l2, &p->p_lwps, l_sibling)
  		    l2->l_flag &= ~L_SA;
  		p->p_flag &= ~P_SA;
  	}
+ #endif
  
  	/* Make other LWPs stick around long enough to be dumped */
  	p->p_userret = lwp_coredump_hook;
Index: sys/kern/kern_synch.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_synch.c,v
retrieving revision 1.130
diff -c -r1.130 kern_synch.c
*** sys/kern/kern_synch.c	2003/06/26 02:09:27	1.130
--- sys/kern/kern_synch.c	2003/06/26 19:53:53
***************
*** 471,477 ****
  	 */
  	if (catch) {
  		l->l_flag |= L_SINTR;
! 		if ((sig = CURSIG(l)) != 0) {
  			if (l->l_wchan != NULL)
  				unsleep(l);
  			l->l_stat = LSONPROC;
--- 471,477 ----
  	 */
  	if (catch) {
  		l->l_flag |= L_SINTR;
! 		if (((sig = CURSIG(l)) != 0) || (p->p_flag & P_WEXIT)) {
  			if (l->l_wchan != NULL)
  				unsleep(l);
  			l->l_stat = LSONPROC;
***************
*** 792,803 ****
  {
  	struct lwp *l = curlwp;
  	int r, s;
! 
  	/* XXX Until the preempt() bug is fixed. */
  	if (more && (l->l_proc->p_flag & P_SA)) {
  		l->l_cpu->ci_schedstate.spc_flags &= ~SPCF_SWITCHCLEAR;
  		return;
  	}
  
  	SCHED_LOCK(s);
  	l->l_priority = l->l_usrpri;
--- 792,805 ----
  {
  	struct lwp *l = curlwp;
  	int r, s;
! /* XXXUPSXXX Not needed for SMP patch */
! #if 0   
  	/* XXX Until the preempt() bug is fixed. */
  	if (more && (l->l_proc->p_flag & P_SA)) {
  		l->l_cpu->ci_schedstate.spc_flags &= ~SPCF_SWITCHCLEAR;
  		return;
  	}
+ #endif
  
  	SCHED_LOCK(s);
  	l->l_priority = l->l_usrpri;
Index: sys/kern/kern_time.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_time.c,v
retrieving revision 1.70
diff -c -r1.70 kern_time.c
*** sys/kern/kern_time.c	2003/05/28 22:27:57	1.70
--- sys/kern/kern_time.c	2003/06/26 19:53:53
***************
*** 869,874 ****
--- 869,885 ----
  	struct ptimers *pt = (struct ptimers *)arg;
  	unsigned int i, fired, done;
  	KERNEL_PROC_LOCK(l);
+ 
+ 	{
+ 		struct proc	*p = l->l_proc;
+ 		struct sadata *sa = p->p_sa;	
+ 
+ 		/* Bail out if we do not own the virtual processor */
+ 		if (sa->sa_vp != l) {
+ 			KERNEL_PROC_UNLOCK(l);
+ 			return ;
+ 		}
+ 	}
  	
  	fired = pt->pts_fired;
  	done = 0;
***************
*** 1188,1195 ****
  itimerfire(struct ptimer *pt)
  {
  	struct proc *p = pt->pt_proc;
  	int s;
! 
  	if (pt->pt_ev.sigev_notify == SIGEV_SIGNAL) {
  		/*
  		 * No RT signal infrastructure exists at this time;
--- 1199,1207 ----
  itimerfire(struct ptimer *pt)
  {
  	struct proc *p = pt->pt_proc;
+ #if 0
  	int s;
! #endif
  	if (pt->pt_ev.sigev_notify == SIGEV_SIGNAL) {
  		/*
  		 * No RT signal infrastructure exists at this time;
***************
*** 1215,1231 ****
--- 1227,1250 ----
  			 * makes testing for sa_idle alone insuffucent to
  			 * determine if we really should call setrunnable.
  			 */
+ #if 0
+ 
  		        if ((sa->sa_idle) && (p->p_stat != SSTOP)) {
  				SCHED_LOCK(s);
  				setrunnable(sa->sa_idle);
  				SCHED_UNLOCK(s);
  			}
+ #endif
  			pt->pt_poverruns = pt->pt_overruns;
  			pt->pt_overruns = 0;
  			i = 1 << pt->pt_entry;
  			p->p_timers->pts_fired = i;
  			p->p_userret = timerupcall;
  			p->p_userret_arg = p->p_timers;
+ 			
+ 			if (sa->sa_idle)
+ 				wakeup(sa->sa_idle);
+ 
  		} else if (p->p_userret == timerupcall) {
  			i = 1 << pt->pt_entry;
  			if ((p->p_timers->pts_fired & i) == 0) {
Index: sys/sys/lwp.h
===================================================================
RCS file: /cvsroot/src/sys/sys/lwp.h,v
retrieving revision 1.6
diff -c -r1.6 lwp.h
*** sys/sys/lwp.h	2003/02/04 13:41:48	1.6
--- sys/sys/lwp.h	2003/06/26 19:53:55
***************
*** 117,122 ****
--- 117,123 ----
  #define	L_SA_UPCALL	0x200000 /* SA upcall is pending */
  #define	L_SA_BLOCKING	0x400000 /* Blocking in tsleep() */
  #define	L_DETACHED	0x800000 /* Won't be waited for. */
+ #define L_SA_WANTS_VP   0x1000000 /* SA LWP wants a virtual processor */
  
  /*
   * Status values.
Index: sys/sys/savar.h
===================================================================
RCS file: /cvsroot/src/sys/sys/savar.h,v
retrieving revision 1.4
diff -c -r1.4 savar.h
*** sys/sys/savar.h	2003/02/02 02:22:14	1.4
--- sys/sys/savar.h	2003/06/26 19:53:55
***************
*** 75,80 ****
--- 75,83 ----
  	int	sa_flag;		/* SA_* flags */
  	sa_upcall_t	sa_upcall;	/* upcall entry point */
  	struct lwp	*sa_vp;		/* "virtual processor" allocation */
+ 	struct lwp	*sa_old_lwp;	/*  XXXUPSXXX hack: lwp that used to be on  sa_vp */
+ 	int    sa_vp_wait_count;        /*  XXXUPSXXX hack: number of LWPs waiting 
on VP */
+ 
  	struct lwp	*sa_woken;	/* list of woken lwps */
  	struct lwp	*sa_idle;      	/* lwp in sawait */
  	int	sa_concurrency;		/* desired concurrency */