Subject: sa/pthread smp concurrency (includes patch)
To: None <tech-kern@netbsd.org>
From: Christian Limpach <chris@pin.lu>
List: tech-kern
Date: 07/10/2003 02:04:22
--3138531-1859-1057795464=:2844
Content-Type: TEXT/PLAIN; CHARSET=US-ASCII
Content-Disposition: INLINE
Hello!
I have made the necessary changes to allow a threaded application to use
multiple CPUs if available. So far this was not possible because an
SA-application could only have one active thread. Now top shows that my
simple test program which creates several cpu-hungry threads uses close to
200% on my otherwise idle two-cpu machine. transcode used to encode at
3fps, it's now encoding at almost 5fps (before it crashes :-().
The patch below adds the concurrency support. It also adds support for
cpu-affinity-masks (options MPCPUMASK in your kernel config file) which
is used to address the SMP-specific SA-problems (LWPs fighting over the
VP). Finally it includes large parts from Stephan Uphoff's patch to
address the signal/memory problems. It's missing locking of the SA data
structures within the kernel and this will occasionally terminate an
application or even panic.
The patch includes modifications to libpthread, you'll have to rebuild it.
The cpu-affinity-masks are i386 only at the moment.
The concurrency support can be enable per-application by setting the
environment variable PTHREAD_CONCURRENCY to the number of available CPUs
(or less).
Feedback is appreciated.
--
Christian Limpach <chris@pin.lu>
--3138531-1859-1057795464=:2844
Content-Type: TEXT/PLAIN; CHARSET=US-ASCII; NAME="sa-smp-concurrency.patch"
Content-Disposition: INLINE; FILENAME="sa-smp-concurrency.patch"
Index: sys/arch/i386/conf/GENERIC.MP
===================================================================
RCS file: /cvs/netbsd/src/sys/arch/i386/conf/GENERIC.MP,v
retrieving revision 1.4
diff -u -r1.4 GENERIC.MP
--- sys/arch/i386/conf/GENERIC.MP 5 Mar 2003 10:28:00 -0000 1.4
+++ sys/arch/i386/conf/GENERIC.MP 8 Jul 2003 18:44:09 -0000
@@ -17,5 +17,6 @@
options MPBIOS # configure CPUs and APICs using MPBIOS
#options MPACPI # configure CPUs and APICs using ACPI
# (acpi at mainbus must also be enabled)
+options MPCPUMASK
ioapic* at mainbus? apid ?
Index: sys/arch/i386/i386/genassym.cf
===================================================================
RCS file: /cvs/netbsd/src/sys/arch/i386/i386/genassym.cf,v
retrieving revision 1.38
diff -u -r1.38 genassym.cf
--- sys/arch/i386/i386/genassym.cf 3 Jul 2003 21:25:45 -0000 1.38
+++ sys/arch/i386/i386/genassym.cf 8 Jul 2003 14:11:02 -0000
@@ -127,6 +127,10 @@
include <machine/cpu.h>
endif
+ifdef MPCPUMASK
+include <sys/savar.h>
+endif
+
if NIOAPIC > 0
include <machine/i82093reg.h>
include <machine/i82093var.h>
@@ -166,10 +170,19 @@
define L_MD_TSS_SEL offsetof(struct lwp, l_md.md_tss_sel)
define L_MD_REGS offsetof(struct lwp, l_md.md_regs)
define L_CPU offsetof(struct lwp, l_cpu)
+if defined(MPCPUMASK)
+define L_CPUMASK offsetof(struct lwp, l_cpumask)
+endif
+define L_SAVP offsetof(struct lwp, l_savp)
define P_FLAG offsetof(struct proc, p_flag)
define P_NRAS offsetof(struct proc, p_nras)
define P_MD_SYSCALL offsetof(struct proc, p_md.md_syscall)
define P_MD_ASTPENDING offsetof(struct proc, p_md.md_astpending)
+define P_SA offsetof(struct proc, p_sa)
+if defined(MPCPUMASK)
+define SAVP_CPUID offsetof(struct sadata_vp, savp_cpuid)
+define SAVP_WHICHQS offsetof(struct sadata_vp, savp_whichqs)
+endif
define P_SYSTEM P_SYSTEM
@@ -280,6 +293,10 @@
define CPU_INFO_ILEVEL offsetof(struct cpu_info, ci_ilevel)
define CPU_INFO_IDEPTH offsetof(struct cpu_info, ci_idepth)
define CPU_INFO_ISOURCES offsetof(struct cpu_info, ci_isources)
+define CPU_INFO_CPUID offsetof(struct cpu_info, ci_cpuid)
+if defined(MPCPUMASK)
+define CPU_INFO_SCHED_WHICHQS offsetof(struct cpu_info, ci_sched_whichqs)
+endif
if NIOAPIC > 0
define IOAPIC_SC_REG offsetof(struct ioapic_softc, sc_reg)
Index: sys/arch/i386/i386/locore.S
===================================================================
RCS file: /cvs/netbsd/src/sys/arch/i386/i386/locore.S,v
retrieving revision 1.11
diff -u -r1.11 locore.S
--- sys/arch/i386/i386/locore.S 26 Jun 2003 16:47:15 -0000 1.11
+++ sys/arch/i386/i386/locore.S 6 Jul 2003 18:47:34 -0000
@@ -127,6 +127,12 @@
#endif
+#if defined(MPCPUMASK)
+#define SCHED_WHICHQS CPUVAR(SCHED_WHICHQS)
+#else
+#define SCHED_WHICHQS _C_LABEL(sched_whichqs)
+#endif
+
#define GET_CURPCB(reg) movl CPUVAR(CURPCB),reg
#define SET_CURPCB(reg) movl reg,CPUVAR(CURPCB)
@@ -1645,7 +1651,10 @@
/*****************************************************************************/
- .globl _C_LABEL(sched_whichqs),_C_LABEL(sched_qs)
+#if !defined(MPCPUMASK)
+ .globl _C_LABEL(sched_whichqs)
+#endif
+ .globl _C_LABEL(sched_qs)
.globl _C_LABEL(uvmexp),_C_LABEL(panic)
#ifdef DIAGNOSTIC
@@ -1684,14 +1693,6 @@
movl 16(%esp),%esi # current
/*
- * Clear curlwp so that we don't accumulate system time while idle.
- * This also insures that schedcpu() will move the old lwp to
- * the correct queue if it happens to get called from the spllower()
- * below and changes the priority. (See corresponding comment in
- * userret()).
- */
- movl $0,CPUVAR(CURLWP)
- /*
* First phase: find new lwp.
*
* Registers:
@@ -1705,11 +1706,21 @@
/* Look for new lwp. */
cli # splhigh doesn't do a cli
- movl _C_LABEL(sched_whichqs),%ecx
+clear_retry:
+ movl SCHED_WHICHQS,%ecx
bsfl %ecx,%ebx # find a full q
jnz switch_dequeue
/*
+ * Clear curlwp so that we don't accumulate system time while idle.
+ * This also insures that schedcpu() will move the old lwp to
+ * the correct queue if it happens to get called from the spllower()
+ * below and changes the priority. (See corresponding comment in
+ * userret()).
+ */
+ movl $0,CPUVAR(CURLWP)
+
+ /*
* idling: save old context.
*
* Registers:
@@ -1784,7 +1795,7 @@
sti
call _C_LABEL(uvm_pageidlezero)
cli
- cmpl $0,_C_LABEL(sched_whichqs)
+ cmpl $0,SCHED_WHICHQS
jnz idle_exit
idle_loop:
/* Try to zero some pages. */
@@ -1796,39 +1807,78 @@
NENTRY(mpidle)
idle_start:
cli
- cmpl $0,_C_LABEL(sched_whichqs)
+ cmpl $0,SCHED_WHICHQS
jz idle_loop
idle_exit:
movl $IPL_HIGH,CPUVAR(ILEVEL) # splhigh
#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
call _C_LABEL(sched_lock_idle)
#endif
- movl _C_LABEL(sched_whichqs),%ecx
+idle_retry:
+ movl SCHED_WHICHQS,%ecx
bsfl %ecx,%ebx
jz idle_unlock
switch_dequeue:
- /*
- * we're running at splhigh(), but it's otherwise okay to take
- * interrupts here.
- */
- sti
leal _C_LABEL(sched_qs)(,%ebx,8),%eax # select q
movl L_FORW(%eax),%edi # unlink from front of process q
+#if !defined(MPCPUMASK)
#ifdef DIAGNOSTIC
cmpl %edi,%eax # linked to self (i.e. nothing queued)?
je _C_LABEL(switch_error) # not possible
#endif /* DIAGNOSTIC */
+#else
+2:
+ cmpl %edi,%eax # queue empty?
+ jne 1f
+ btrl %ebx,SCHED_WHICHQS # yes, clear to indicate empty
+ cmpl $0,CPUVAR(CURLWP) # did we idle?
+ je idle_retry
+ jmp clear_retry
+1:
+ movl CPUVAR(CPUID),%edx
+ btl %edx,L_CPUMASK(%edi)
+ jnc 1f
+3:
+ movl L_FORW(%edi),%edi
+ jmp 2b
+1:
+ movl L_SAVP(%edi),%ecx
+ cmpl $0,%ecx
+ je 1f
+ cmpl $-1,SAVP_CPUID(%ecx)
+ je 2f
+ cmpl %edx,SAVP_CPUID(%ecx)
+ je 1f
+ btsl %ebx,SAVP_WHICHQS(%ecx) # mark q skipped by savp_cpuid
+ jmp 3b
+2:
+ movl %edx,SAVP_CPUID(%ecx)
+1:
+ movl L_BACK(%edi),%eax
+#endif
+ /*
+ * we're running at splhigh(), but it's otherwise okay to take
+ * interrupts here.
+ */
+ sti
movl L_FORW(%edi),%edx
movl %edx,L_FORW(%eax)
movl %eax,L_BACK(%edx)
+#if defined(MPCPUMASK)
+ leal _C_LABEL(sched_qs)(,%ebx,8),%eax # select q
+#endif
cmpl %edx,%eax # q empty?
jne 3f
+#if defined(MPCPUMASK)
+ btrl %ebx,SCHED_WHICHQS
+#else
btrl %ebx,%ecx # yes, clear to indicate empty
- movl %ecx,_C_LABEL(sched_whichqs) # update q status
+ movl %ecx,SCHED_WHICHQS # update q status
+#endif
3: /* We just did it. */
xorl %eax,%eax
Index: sys/arch/i386/i386/pmap.c
===================================================================
RCS file: /cvs/netbsd/src/sys/arch/i386/i386/pmap.c,v
retrieving revision 1.155
diff -u -r1.155 pmap.c
--- sys/arch/i386/i386/pmap.c 23 Jun 2003 11:01:20 -0000 1.155
+++ sys/arch/i386/i386/pmap.c 2 Jul 2003 21:39:31 -0000
@@ -2070,7 +2070,7 @@
*zpte = (pa & PG_FRAME) | PG_V | PG_RW; /* map in */
pmap_update_pg((vaddr_t)zerova); /* flush TLB */
for (i = 0, ptr = (int *) zerova; i < PAGE_SIZE / sizeof(int); i++) {
- if (sched_whichqs != 0) {
+ if (RUNQS_NOTEMPTY()) {
/*
* A process has become ready. Abort now,
Index: sys/arch/i386/include/cpu.h
===================================================================
RCS file: /cvs/netbsd/src/sys/arch/i386/include/cpu.h,v
retrieving revision 1.102
diff -u -r1.102 cpu.h
--- sys/arch/i386/include/cpu.h 26 Jun 2003 16:41:32 -0000 1.102
+++ sys/arch/i386/include/cpu.h 5 Jul 2003 15:10:38 -0000
@@ -81,6 +81,9 @@
u_int ci_apicid; /* our APIC ID */
u_long ci_spin_locks; /* # of spin locks held */
u_long ci_simple_locks; /* # of simple locks held */
+#if defined(MPCPUMASK)
+ volatile u_int32_t ci_sched_whichqs; /* runqueues with work for us */
+#endif
/*
* Private members.
@@ -244,6 +247,12 @@
#define aston(p) ((p)->p_md.md_astpending = 1)
#endif /* MULTIPROCESSOR */
+
+#if defined(MPCPUMASK)
+
+#define cpu_sched_whichqs() (curcpu()->ci_sched_whichqs)
+
+#endif /* MPCPUMASK */
extern u_int32_t cpus_attached;
Index: sys/conf/files
===================================================================
RCS file: /cvs/netbsd/src/sys/conf/files,v
retrieving revision 1.617
diff -u -r1.617 files
--- sys/conf/files 6 Jul 2003 22:56:08 -0000 1.617
+++ sys/conf/files 8 Jul 2003 12:56:55 -0000
@@ -29,7 +29,7 @@
defflag opt_sock_counters.h SOSEND_COUNTERS
defflag opt_sosend_loan.h SOSEND_NO_LOAN
-defflag MULTIPROCESSOR
+defflag opt_multiprocessor.h MULTIPROCESSOR MPCPUMASK
defflag opt_config.h INCLUDE_CONFIG_FILE INCLUDE_JUST_CONFIG
Index: sys/kern/kern_exec.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_exec.c,v
retrieving revision 1.169
diff -u -r1.169 kern_exec.c
--- sys/kern/kern_exec.c 29 Jun 2003 22:31:19 -0000 1.169
+++ sys/kern/kern_exec.c 6 Jul 2003 23:26:28 -0000
@@ -362,6 +362,7 @@
int szsigcode;
struct exec_vmcmd *base_vcp;
int oldlwpflags;
+ struct sadata_vp *vp;
/* Disable scheduler activation upcalls. */
oldlwpflags = l->l_flag & (L_SA | L_SA_UPCALL);
@@ -526,6 +527,10 @@
if (p->p_sa) {
p->p_flag &= ~P_SA;
free(p->p_sa->sa_stacks, M_SA);
+ while ((vp = SIMPLEQ_FIRST(&p->p_sa->sa_vps)) != NULL) {
+ SIMPLEQ_REMOVE_HEAD(&p->p_sa->sa_vps, savp_next);
+ pool_put(&savp_pool, vp);
+ }
pool_put(&sadata_pool, p->p_sa);
p->p_sa = NULL;
}
Index: sys/kern/kern_exit.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_exit.c,v
retrieving revision 1.117
diff -u -r1.117 kern_exit.c
--- sys/kern/kern_exit.c 29 Jun 2003 22:31:20 -0000 1.117
+++ sys/kern/kern_exit.c 8 Jul 2003 13:21:37 -0000
@@ -180,7 +180,9 @@
sa = 0;
if (p->p_sa != NULL) {
l->l_flag &= ~L_SA;
+#if 0
p->p_flag &= ~P_SA;
+#endif
sa = 1;
}
@@ -426,6 +428,7 @@
{
struct proc *p;
struct lwp *l2;
+ struct sadata_vp *vp;
int s, error;
lwpid_t waited;
@@ -442,17 +445,21 @@
* Make SA-cached LWPs normal process runnable LWPs so that
* they'll also self-destruct.
*/
- if (p->p_sa && p->p_sa->sa_ncached > 0) {
- DPRINTF(("exit_lwps: Making cached LWPs of %d runnable: ",
- p->p_pid));
- SCHED_LOCK(s);
- while ((l2 = sa_getcachelwp(p)) != 0) {
- l2->l_priority = l2->l_usrpri;
- setrunnable(l2);
- DPRINTF(("%d ", l2->l_lid));
+ if (p->p_sa) {
+ SIMPLEQ_FOREACH(vp, &p->p_sa->sa_vps, savp_next) {
+ if (vp->savp_ncached > 0) {
+ DPRINTF(("exit_lwps: Making cached LWPs of %d runnable: ",
+ p->p_pid));
+ SCHED_LOCK(s);
+ while ((l2 = sa_getcachelwp(vp)) != 0) {
+ l2->l_priority = l2->l_usrpri;
+ setrunnable(l2);
+ DPRINTF(("%d ", l2->l_lid));
+ }
+ SCHED_UNLOCK(s);
+ DPRINTF(("\n"));
+ }
}
- DPRINTF(("\n"));
- SCHED_UNLOCK(s);
}
/*
@@ -461,9 +468,13 @@
* them) and then wait for everyone else to finish.
*/
LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
+#if 0
l2->l_flag &= ~(L_DETACHED|L_SA);
+#else
+ l2->l_flag &= ~(L_DETACHED);
+#endif
if ((l2->l_stat == LSSLEEP && (l2->l_flag & L_SINTR)) ||
- l2->l_stat == LSSUSPENDED) {
+ l2->l_stat == LSSUSPENDED || l2->l_stat == LSSTOP) {
SCHED_LOCK(s);
setrunnable(l2);
SCHED_UNLOCK(s);
@@ -728,6 +739,7 @@
proc_free(struct proc *p)
{
struct proc *parent = p->p_pptr;
+ struct sadata_vp *vp;
int s;
/*
@@ -802,6 +814,10 @@
*/
if (p->p_sa) {
free(p->p_sa->sa_stacks, M_SA);
+ while ((vp = SIMPLEQ_FIRST(&p->p_sa->sa_vps)) != NULL) {
+ SIMPLEQ_REMOVE_HEAD(&p->p_sa->sa_vps, savp_next);
+ pool_put(&savp_pool, vp);
+ }
pool_put(&sadata_pool, p->p_sa);
}
Index: sys/kern/kern_lwp.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_lwp.c,v
retrieving revision 1.8
diff -u -r1.8 kern_lwp.c
--- sys/kern/kern_lwp.c 23 Jun 2003 11:02:05 -0000 1.8
+++ sys/kern/kern_lwp.c 6 Jul 2003 21:49:18 -0000
@@ -295,26 +295,38 @@
lwpid_t target_lid;
struct lwp *t;
struct proc *p;
+ int err, s;
p = l->l_proc;
target_lid = SCARG(uap, target);
+ err = 0;
+
+ SCHED_LOCK(s);
LIST_FOREACH(t, &p->p_lwps, l_sibling)
if (t->l_lid == target_lid)
break;
- if (t == NULL)
- return (ESRCH);
+ if (t == NULL) {
+ err = ESRCH;
+ goto out;
+ }
- if (t->l_stat != LSSLEEP)
- return (ENODEV);
+ if (t->l_stat != LSSLEEP) {
+ err = ENODEV;
+ goto out;
+ }
- if ((t->l_flag & L_SINTR) == 0)
- return (EBUSY);
+ if ((t->l_flag & L_SINTR) == 0) {
+ err = EBUSY;
+ goto out;
+ }
setrunnable(t);
+ out:
+ SCHED_UNLOCK(s);
- return 0;
+ return err;
}
int
@@ -469,6 +481,12 @@
*/
l2->l_cpu = NULL;
#endif /* ! MULTIPROCESSOR */
+
+#if defined(MPCPUMASK)
+ l2->l_cpumask = 0;
+#endif
+
+ l2->l_savp = NULL;
l2->l_flag = inmem ? L_INMEM : 0;
l2->l_flag |= (flags & LWP_DETACHED) ? L_DETACHED : 0;
Index: sys/kern/kern_proc.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_proc.c,v
retrieving revision 1.64
diff -u -r1.64 kern_proc.c
--- sys/kern/kern_proc.c 19 Mar 2003 20:35:04 -0000 1.64
+++ sys/kern/kern_proc.c 6 Jul 2003 18:03:39 -0000
@@ -192,6 +192,7 @@
struct pool pgrp_pool;
struct pool rusage_pool;
struct pool ras_pool;
+struct pool savp_pool;
struct pool sadata_pool;
struct pool saupcall_pool;
struct pool ptimer_pool;
@@ -275,6 +276,8 @@
pool_init(&rusage_pool, sizeof(struct rusage), 0, 0, 0, "rusgepl",
&pool_allocator_nointr);
pool_init(&ras_pool, sizeof(struct ras), 0, 0, 0, "raspl",
+ &pool_allocator_nointr);
+ pool_init(&savp_pool, sizeof(struct sadata_vp), 0, 0, 0, "savppl",
&pool_allocator_nointr);
pool_init(&sadata_pool, sizeof(struct sadata), 0, 0, 0, "sadatapl",
&pool_allocator_nointr);
Index: sys/kern/kern_sa.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_sa.c,v
retrieving revision 1.16
diff -u -r1.16 kern_sa.c
--- sys/kern/kern_sa.c 28 May 2003 22:17:20 -0000 1.16
+++ sys/kern/kern_sa.c 8 Jul 2003 13:31:40 -0000
@@ -39,6 +39,8 @@
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_sa.c,v 1.16 2003/05/28 22:17:20 nathanw Exp $");
+#include "opt_multiprocessor.h"
+
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/pool.h>
@@ -53,6 +55,9 @@
#include <uvm/uvm_extern.h>
+#include <sys/kernel.h> /* For lbolt hack */
+
+static struct sadata_vp *sa_newsavp(struct sadata *);
static int sa_newcachelwp(struct lwp *);
static struct lwp *sa_vp_repossess(struct lwp *l);
@@ -72,6 +77,19 @@
#endif
+
+#define SA_LWP_STATE_LOCK(l,f) \
+do { \
+ f = (l)->l_flag; \
+ (l)->l_flag &= ~L_SA; \
+} while (0)
+
+#define SA_LWP_STATE_UNLOCK(l,f) \
+do { \
+ (l)->l_flag |= (f & L_SA); \
+} while (0)
+
+
/*
* sadata_upcall_alloc:
*
@@ -118,6 +136,32 @@
pool_put(&saupcall_pool, sau);
}
+static struct sadata_vp *
+sa_newsavp(struct sadata *sa)
+{
+ struct sadata_vp *vp;
+
+ /* Allocate scheduler activations data structure */
+ vp = pool_get(&savp_pool, PR_WAITOK);
+ /* Initialize. */
+ memset(vp, 0, sizeof(*vp));
+ simple_lock_init(&vp->savp_lock);
+#if defined(MPCPUMASK)
+ vp->savp_cpuid = -1;
+ vp->savp_whichqs = 0;
+#endif
+ vp->savp_lwp = NULL;
+ vp->savp_idle = NULL;
+ vp->savp_woken = NULL;
+ LIST_INIT(&vp->savp_lwpcache);
+ vp->savp_ncached = 0;
+ SIMPLEQ_INIT(&vp->savp_upcalls);
+ vp->savp_sa = sa;
+ SIMPLEQ_INSERT_TAIL(&sa->sa_vps, vp, savp_next);
+
+ return (vp);
+}
+
int
sys_sa_register(struct lwp *l, void *v, register_t *retval)
{
@@ -138,16 +182,15 @@
memset(sa, 0, sizeof(*sa));
simple_lock_init(&sa->sa_lock);
sa->sa_flag = SCARG(uap, flags) & SA_FLAG_ALL;
- sa->sa_vp = NULL;
- sa->sa_idle = NULL;
- sa->sa_woken = NULL;
sa->sa_concurrency = 1;
sa->sa_stacks = malloc(sizeof(stack_t) * SA_NUMSTACKS,
M_SA, M_WAITOK);
sa->sa_nstacks = 0;
- LIST_INIT(&sa->sa_lwpcache);
- SIMPLEQ_INIT(&sa->sa_upcalls);
+ SIMPLEQ_INIT(&sa->sa_vps);
p->p_sa = sa;
+ }
+ if (l->l_savp == NULL) {
+ l->l_savp = sa_newsavp(p->p_sa);
sa_newcachelwp(l);
}
@@ -202,13 +245,14 @@
{
struct proc *p = l->l_proc;
struct sadata *sa = p->p_sa;
+ struct sadata_vp *vp = l->l_savp;
int error;
DPRINTF(("sys_sa_enable(%d.%d)\n", l->l_proc->p_pid,
l->l_lid));
/* We have to be using scheduler activations */
- if (sa == NULL)
+ if ((sa == NULL) || (vp == NULL))
return (EINVAL);
if (p->p_flag & P_SA) /* Already running! */
@@ -221,13 +265,50 @@
p->p_flag |= P_SA;
l->l_flag |= L_SA; /* We are now an activation LWP */
+#if defined(MPCPUMASK)
+ vp->savp_cpuid = cpu_number();
+#endif
+
/* Assign this LWP to the virtual processor */
- sa->sa_vp = l;
+ vp->savp_lwp = l;
/* This will not return to the place in user space it came from. */
return (0);
}
+static int
+sa_increaseconcurrency(struct lwp *l)
+{
+ struct proc *p;
+ struct lwp *l2;
+ struct sadata *sa;
+ vaddr_t uaddr;
+ boolean_t inmem;
+ int error, s;
+
+ p = l->l_proc;
+ sa = p->p_sa;
+
+ inmem = uvm_uarea_alloc(&uaddr);
+ if (__predict_false(uaddr == 0)) {
+ return (ENOMEM);
+ } else {
+ newlwp(l, p, uaddr, inmem, 0, NULL, 0, child_return, 0, &l2);
+ l2->l_flag |= L_SA;
+ l2->l_savp = sa_newsavp(sa);
+ l2->l_savp->savp_lwp = l2;
+ cpu_setfunc(l2, sa_switchcall, l2);
+ error = sa_upcall(l2, SA_UPCALL_NEWPROC, NULL, NULL, 0, NULL);
+ if (error)
+ return (error);
+ SCHED_LOCK(s);
+ setrunnable(l2);
+ sa->sa_concurrency++;
+ SCHED_UNLOCK(s);
+ }
+
+ return (0);
+}
int
sys_sa_setconcurrency(struct lwp *l, void *v, register_t *retval)
@@ -236,6 +317,13 @@
syscallarg(int) concurrency;
} */ *uap = v;
struct sadata *sa = l->l_proc->p_sa;
+#ifdef MULTIPROCESSOR
+ int ncpus;
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+#else
+#define ncpus 1
+#endif
DPRINTF(("sys_sa_concurrency(%d.%d)\n", l->l_proc->p_pid,
l->l_lid));
@@ -254,10 +342,21 @@
* XXX Should we ever support hot-plug CPUs, this will need
* adjustment.
*/
- sa->sa_concurrency = min(SCARG(uap, concurrency), 1 /* XXX ncpus */);
+#ifdef MULTIPROCESSOR
+ ncpus = 0;
+ for (CPU_INFO_FOREACH(cii, ci))
+ ncpus++;
+#endif
+ while (sa->sa_concurrency < min(SCARG(uap, concurrency), ncpus)) {
+ if (sa_increaseconcurrency(l) != 0)
+ break;
+ }
return (0);
}
+#ifndef MULTIPROCESSOR
+#undef ncpus
+#endif
int
sys_sa_yield(struct lwp *l, void *v, register_t *retval)
@@ -278,13 +377,14 @@
struct lwp *l2;
struct proc *p = l->l_proc;
struct sadata *sa = p->p_sa;
+ struct sadata_vp *vp = l->l_savp;
int s, ret;
/*
* If we're the last running LWP, stick around to recieve
* signals.
*/
- if (p->p_nrlwps == 1) {
+ if (p->p_nrlwps <= sa->sa_concurrency) {
DPRINTFN(1,("sa_yield(%d.%d) going dormant\n",
p->p_pid, l->l_lid));
/*
@@ -292,7 +392,7 @@
* happens and just causes the process to yield again.
*/
s = splsched(); /* Protect from timer expirations */
- KDASSERT(sa->sa_vp == l);
+ KDASSERT(vp->savp_lwp == l);
/*
* If we were told to make an upcall or exit before
* the splsched(), make sure we process it instead of
@@ -301,12 +401,12 @@
*/
ret = 0;
while ((ret == 0) && (p->p_userret == NULL)) {
- sa->sa_idle = l;
+ vp->savp_idle = l;
l->l_flag &= ~L_SA;
ret = tsleep((caddr_t) l, PUSER | PCATCH, "sawait", 0);
l->l_flag |= L_SA;
- sa->sa_idle = NULL;
- sa->sa_vp = l;
+ vp->savp_idle = NULL;
+ vp->savp_lwp = l;
}
splx(s);
DPRINTFN(1,("sa_yield(%d.%d) returned\n",
@@ -315,9 +415,9 @@
DPRINTFN(1,("sa_yield(%d.%d) stepping aside\n", p->p_pid, l->l_lid));
SCHED_LOCK(s);
- l2 = sa->sa_woken;
- sa->sa_woken = NULL;
- sa->sa_vp = NULL;
+ l2 = vp->savp_woken;
+ vp->savp_woken = NULL;
+ vp->savp_lwp = NULL;
p->p_nrlwps--;
sa_putcachelwp(p, l);
KDASSERT((l2 == NULL) || (l2->l_proc == l->l_proc));
@@ -371,11 +471,12 @@
struct sadata_upcall *sau;
struct sadata *sa = l->l_proc->p_sa;
stack_t st;
+ int f;
- l->l_flag &= ~L_SA; /* XXX prevent recursive upcalls if we sleep for
- memory */
+ SA_LWP_STATE_LOCK(l,f); /* XXX prevent recursive upcalls if we
+ sleep for memory */
sau = sadata_upcall_alloc(1);
- l->l_flag |= L_SA;
+ SA_LWP_STATE_UNLOCK(l,f);
if (sa->sa_nstacks == 0) {
/* assign to assure that it gets freed */
@@ -395,8 +496,7 @@
sa_upcall0(struct lwp *l, int type, struct lwp *event, struct lwp *interrupted,
size_t argsize, void *arg, struct sadata_upcall *sau, stack_t *st)
{
- struct proc *p = l->l_proc;
- struct sadata *sa = p->p_sa;
+ struct sadata_vp *vp = l->l_savp;
KDASSERT((event == NULL) || (event != interrupted));
@@ -413,7 +513,7 @@
} else
sa_upcall_getstate(sau, event, interrupted);
- SIMPLEQ_INSERT_TAIL(&sa->sa_upcalls, sau, sau_next);
+ SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
l->l_flag |= L_SA_UPCALL;
return (0);
@@ -468,16 +568,22 @@
{
struct proc *p = l->l_proc;
struct sadata *sa = p->p_sa;
+ struct sadata_vp *vp = l->l_savp;
struct sadata_upcall *sau;
struct lwp *l2;
stack_t st;
int error;
DPRINTFN(4,("sa_switch(%d.%d type %d VP %d)\n", p->p_pid, l->l_lid,
- type, sa->sa_vp ? sa->sa_vp->l_lid : 0));
+ type, vp->savp_lwp ? vp->savp_lwp->l_lid : 0));
SCHED_ASSERT_LOCKED();
- if (sa->sa_vp == l) {
+ if (p->p_flag & P_WEXIT) {
+ mi_switch(l, NULL);
+ return;
+ }
+
+ if (vp->savp_lwp == l) {
/*
* Case 1: we're blocking for the first time; generate
* a SA_BLOCKED upcall and allocate resources for the
@@ -489,7 +595,7 @@
* would be Bad. Therefore, we must use a cached new
* LWP. The first thing that this new LWP must do is
* allocate another LWP for the cache. */
- l2 = sa_getcachelwp(p);
+ l2 = sa_getcachelwp(vp);
if (l2 == NULL) {
/* XXXSMP */
/* No upcall for you! */
@@ -551,7 +657,7 @@
PRELE(l2); /* Remove the artificial hold-count */
KDASSERT(l2 != l);
- } else if (sa->sa_vp != NULL) {
+ } else if (vp->savp_lwp != NULL) {
/*
* Case 2: We've been woken up while another LWP was
* on the VP, but we're going back to sleep without
@@ -562,10 +668,11 @@
* go. If the LWP on the VP was idling, don't make it
* run again, though.
*/
- if (sa->sa_idle)
+ if (vp->savp_idle || ((vp->savp_lwp->l_flag & L_INMEM) == 0))
l2 = NULL;
else
- l2 = sa->sa_vp;
+ l2 = vp->savp_lwp;
+ KDASSERT((l2 == NULL) || (l2->l_stat == LSRUN));
} else {
/*
* Case 3: The VP is empty. As in case 2, we were
@@ -576,7 +683,7 @@
* The right thing is to pull a LWP off the cache and have
* it jump straight back into sa_yield.
*/
- l2 = sa_getcachelwp(p);
+ l2 = sa_getcachelwp(vp);
if (l2 == NULL) {
#ifdef DIAGNOSTIC
printf("sa_switch(%d.%d): no cached LWP for reidling.\n",
@@ -602,8 +709,8 @@
KDASSERT(l->l_wchan == 0);
SCHED_ASSERT_UNLOCKED();
- if (sa->sa_woken == l)
- sa->sa_woken = NULL;
+ if (vp->savp_woken == l)
+ vp->savp_woken = NULL;
/*
@@ -630,20 +737,23 @@
{
struct lwp *l;
struct proc *p;
- struct sadata *sa;
+ struct sadata_vp *vp;
+ int f;
l = arg;
p = l->l_proc;
- sa = p->p_sa;
- sa->sa_vp = l;
+ vp = l->l_savp;
+ vp->savp_lwp = l;
DPRINTFN(6,("sa_switchcall(%d.%d)\n", p->p_pid, l->l_lid));
- if (LIST_EMPTY(&sa->sa_lwpcache)) {
+ if (LIST_EMPTY(&vp->savp_lwpcache)) {
/* Allocate the next cache LWP */
DPRINTFN(6,("sa_switchcall(%d.%d) allocating LWP\n",
p->p_pid, l->l_lid));
+ SA_LWP_STATE_LOCK(l,f);
sa_newcachelwp(l);
+ SA_LWP_STATE_UNLOCK(l,f);
}
upcallret(l);
}
@@ -653,20 +763,23 @@
{
struct lwp *l;
struct proc *p;
- struct sadata *sa;
+ struct sadata_vp *vp;
+ int f;
l = arg;
p = l->l_proc;
- sa = p->p_sa;
- sa->sa_vp = l;
+ vp = l->l_savp;
+ vp->savp_lwp = l;
DPRINTFN(6,("sa_yieldcall(%d.%d)\n", p->p_pid, l->l_lid));
- if (LIST_EMPTY(&sa->sa_lwpcache)) {
+ if (LIST_EMPTY(&vp->savp_lwpcache)) {
/* Allocate the next cache LWP */
DPRINTFN(6,("sa_yieldcall(%d.%d) allocating LWP\n",
p->p_pid, l->l_lid));
+ SA_LWP_STATE_LOCK(l,f);
sa_newcachelwp(l);
+ SA_LWP_STATE_UNLOCK(l,f);
}
sa_yield(l);
@@ -693,6 +806,7 @@
* newlwp helpfully puts it there. Unclear if newlwp should
* be tweaked.
*/
+ l2->l_savp = l->l_savp;
SCHED_LOCK(s);
sa_putcachelwp(p, l2);
SCHED_UNLOCK(s);
@@ -708,11 +822,11 @@
void
sa_putcachelwp(struct proc *p, struct lwp *l)
{
- struct sadata *sa;
+ struct sadata_vp *vp;
SCHED_ASSERT_LOCKED();
- sa = p->p_sa;
+ vp = l->l_savp;
LIST_REMOVE(l, l_sibling);
p->p_nlwps--;
@@ -722,8 +836,8 @@
/* XXX lock sadata */
DPRINTFN(5,("sa_putcachelwp(%d.%d) Adding LWP %d to cache\n",
p->p_pid, curlwp->l_lid, l->l_lid));
- LIST_INSERT_HEAD(&sa->sa_lwpcache, l, l_sibling);
- sa->sa_ncached++;
+ LIST_INSERT_HEAD(&vp->savp_lwpcache, l, l_sibling);
+ vp->savp_ncached++;
/* XXX unlock */
}
@@ -731,20 +845,20 @@
* Fetch a LWP from the cache.
*/
struct lwp *
-sa_getcachelwp(struct proc *p)
+sa_getcachelwp(struct sadata_vp *vp)
{
- struct sadata *sa;
struct lwp *l;
+ struct proc *p;
SCHED_ASSERT_LOCKED();
l = NULL;
- sa = p->p_sa;
/* XXX lock sadata */
- if (sa->sa_ncached > 0) {
- sa->sa_ncached--;
- l = LIST_FIRST(&sa->sa_lwpcache);
+ if (vp->savp_ncached > 0) {
+ vp->savp_ncached--;
+ l = LIST_FIRST(&vp->savp_lwpcache);
LIST_REMOVE(l, l_sibling);
+ p = l->l_proc;
LIST_INSERT_HEAD(&p->p_lwps, l, l_sibling);
p->p_nlwps++;
DPRINTFN(5,("sa_getcachelwp(%d.%d) Got LWP %d from cache.\n",
@@ -760,6 +874,7 @@
{
struct proc *p;
struct sadata *sa;
+ struct sadata_vp *vp;
struct sa_t **sapp, *sap;
struct sadata_upcall *sau;
struct sa_t self_sa;
@@ -768,12 +883,14 @@
void *stack, *ap;
ucontext_t u, *up;
int i, nsas, nint, nevents, type;
+ int ret, f;
p = l->l_proc;
sa = p->p_sa;
+ vp = l->l_savp;
KERNEL_PROC_LOCK(l);
- l->l_flag &= ~L_SA;
+ SA_LWP_STATE_LOCK(l,f);
DPRINTFN(7,("sa_upcall_userret(%d.%d %x) \n", p->p_pid, l->l_lid,
l->l_flag));
@@ -785,8 +902,11 @@
p->p_pid, l->l_lid));
sau = sadata_upcall_alloc(1);
-
- while (sa->sa_nstacks == 0) {
+ sau->sau_arg = NULL;
+
+ ret = EAGAIN;
+ while ((sa->sa_nstacks == 0) && (ret != 0) &&
+ ((p->p_flag & P_WEXIT) == 0)) {
/*
* This should be a transient condition, so we'll just
* sleep until some stacks come in; presumably, some
@@ -805,14 +925,26 @@
* Ideally, tsleep() would have a variant that took
* a LWP to switch to.
*/
- l->l_flag &= ~L_SA;
DPRINTFN(7, ("sa_upcall_userret(%d.%d) sleeping"
" for stacks\n", l->l_proc->p_pid, l->l_lid));
- tsleep((caddr_t) &sa->sa_nstacks, PWAIT|PCATCH,
+ ret = tsleep((caddr_t) &sa->sa_nstacks, PWAIT|PCATCH,
"sastacks", 0);
- if (p->p_flag & P_WEXIT)
- lwp_exit(l);
- l->l_flag |= L_SA;
+ if (ret && ((p->p_flag & P_WEXIT) == 0)) {
+ /* XXXCL needed? */
+#ifdef DIAGNOSTIC
+ printf ("sa_upcall_userret(%d.%d) sleep"
+ " for stacks failed: %d\n", l->l_proc->p_pid,
+ l->l_lid, ret);
+#endif
+ /* Signal pending - can't sleep */
+ /* Wait a while .. things might get better */
+ tsleep((caddr_t) &lbolt, PWAIT,
+ "lbolt: sastacks", 0);
+ }
+ }
+ if (p->p_flag & P_WEXIT) {
+ sadata_upcall_free(sau);
+ lwp_exit(l);
}
l2 = sa_vp_repossess(l);
@@ -837,11 +969,11 @@
l->l_flag &= ~L_SA_BLOCKING;
}
- KDASSERT(SIMPLEQ_EMPTY(&sa->sa_upcalls) == 0);
+ KDASSERT(SIMPLEQ_EMPTY(&vp->savp_upcalls) == 0);
- sau = SIMPLEQ_FIRST(&sa->sa_upcalls);
- SIMPLEQ_REMOVE_HEAD(&sa->sa_upcalls, sau_next);
- if (SIMPLEQ_EMPTY(&sa->sa_upcalls))
+ sau = SIMPLEQ_FIRST(&vp->savp_upcalls);
+ SIMPLEQ_REMOVE_HEAD(&vp->savp_upcalls, sau_next);
+ if (SIMPLEQ_EMPTY(&vp->savp_upcalls))
l->l_flag &= ~L_SA_UPCALL;
if (sau->sau_flags & SAU_FLAG_DEFERRED) {
@@ -961,7 +1093,7 @@
l->l_lid, type));
cpu_upcall(l, type, nevents, nint, sapp, ap, stack, sa->sa_upcall);
- l->l_flag |= L_SA;
+ SA_LWP_STATE_UNLOCK(l,f);
KERNEL_PROC_UNLOCK(l);
}
@@ -970,17 +1102,17 @@
{
struct lwp *l2;
struct proc *p = l->l_proc;
- struct sadata *sa = p->p_sa;
+ struct sadata_vp *vp = l->l_savp;
int s;
/*
* Put ourselves on the virtual processor and note that the
* previous occupant of that position was interrupted.
*/
- l2 = sa->sa_vp;
- sa->sa_vp = l;
- if (sa->sa_idle == l2)
- sa->sa_idle = NULL;
+ l2 = vp->savp_lwp;
+ vp->savp_lwp = l;
+ if (vp->savp_idle == l2)
+ vp->savp_idle = NULL;
KDASSERT(l2 != l);
if (l2) {
@@ -1037,6 +1169,7 @@
{
struct lwp *l;
struct sadata *sa;
+ struct sadata_vp *vp;
printf("Process %d (%s), state %d, address %p, flags %x\n",
p->p_pid, p->p_comm, p->p_stat, p, p->p_flag);
@@ -1046,14 +1179,16 @@
debug_print_lwp(l);
sa = p->p_sa;
if (sa) {
- if (sa->sa_vp)
- printf("SA VP: %d\n", sa->sa_vp->l_lid);
- if (sa->sa_idle)
- printf("SA idle: %d\n", sa->sa_idle->l_lid);
- printf("SAs: %d cached LWPs\n", sa->sa_ncached);
+ SIMPLEQ_FOREACH(vp, &sa->sa_vps, savp_next) {
+ if (vp->savp_lwp)
+ printf("SA VP: %d\n", vp->savp_lwp->l_lid);
+ if (vp->savp_idle)
+ printf("SA idle: %d\n", vp->savp_idle->l_lid);
+ printf("SAs: %d cached LWPs\n", vp->savp_ncached);
+ LIST_FOREACH(l, &vp->savp_lwpcache, l_sibling)
+ debug_print_lwp(l);
+ }
printf("%d upcall stacks\n", sa->sa_nstacks);
- LIST_FOREACH(l, &sa->sa_lwpcache, l_sibling)
- debug_print_lwp(l);
}
return 0;
Index: sys/kern/kern_sig.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_sig.c,v
retrieving revision 1.143
diff -u -r1.143 kern_sig.c
--- sys/kern/kern_sig.c 29 Jun 2003 22:31:22 -0000 1.143
+++ sys/kern/kern_sig.c 6 Jul 2003 19:46:54 -0000
@@ -793,6 +793,8 @@
struct lwp *l, *suspended;
int s = 0, prop, allsusp;
sig_t action;
+ struct sadata_vp *vp;
+ struct lwp *l2;
#ifdef DIAGNOSTIC
if (signum <= 0 || signum >= NSIG)
@@ -867,7 +869,8 @@
*/
if ((prop & SA_CANTMASK) == 0
&& p->p_sigctx.ps_sigwaited < 0
- && sigismember(&p->p_sigctx.ps_sigwait, signum)) {
+ && sigismember(&p->p_sigctx.ps_sigwait, signum)
+ && p->p_stat != SSTOP) {
sigdelset(&p->p_sigctx.ps_siglist, signum);
p->p_sigctx.ps_sigwaited = signum;
sigemptyset(&p->p_sigctx.ps_sigwait);
@@ -889,7 +892,7 @@
if (dolock)
SCHED_LOCK(s);
- if (p->p_nrlwps > 0) {
+ if (p->p_nrlwps > 0 && (p->p_stat != SSTOP)) {
/*
* At least one LWP is running or on a run queue.
* The signal will be noticed when one of them returns
@@ -903,7 +906,21 @@
} else {
/* Process is sleeping or stopped */
if (p->p_flag & P_SA) {
- l = p->p_sa->sa_idle;
+ l = NULL;
+ allsusp = 1;
+ SIMPLEQ_FOREACH(vp, &p->p_sa->sa_vps, savp_next) {
+ l2 = vp->savp_lwp;
+ if ((l2->l_stat == LSSLEEP) &&
+ (l2->l_flag & L_SINTR)) {
+ l = l2;
+ break;
+ }
+ else if (l2->l_stat == LSSUSPENDED)
+ suspended = l2;
+ else if ((l2->l_stat != LSZOMB) &&
+ (l2->l_stat != LSDEAD))
+ allsusp = 0;
+ }
} else {
/*
* Find out if any of the sleeps are interruptable,
@@ -1072,9 +1089,13 @@
struct proc *p = l->l_proc;
struct lwp *le, *li;
siginfo_t *si;
+ int f;
if (p->p_flag & P_SA) {
+ f = l->l_flag & L_SA;
+ l->l_flag &= ~L_SA;
si = pool_get(&siginfo_pool, PR_WAITOK);
+ l->l_flag |= f;
si->si_signo = sig;
si->si_errno = 0;
si->si_code = code;
@@ -1392,6 +1413,7 @@
proc_unstop(struct proc *p)
{
struct lwp *l, *lr = NULL;
+ struct sadata_vp *vp;
int cantake = 0;
SCHED_ASSERT_LOCKED();
@@ -1409,9 +1431,13 @@
* Preferentially select the idle LWP as the interruptable
* LWP to return if it exists.
*/
- lr = p->p_sa->sa_idle;
- if (lr != NULL)
- cantake = 1;
+ SIMPLEQ_FOREACH(vp, &p->p_sa->sa_vps, savp_next) {
+ lr = vp->savp_idle;
+ if (lr != NULL) {
+ cantake = 1;
+ break;
+ }
+ }
}
LIST_FOREACH(l, &p->p_lwps, l_sibling) {
if (l->l_stat == LSRUN) {
@@ -1585,7 +1611,9 @@
sigexit(struct lwp *l, int signum)
{
struct proc *p;
+#if 0
struct lwp *l2;
+#endif
int error, exitsig;
p = l->l_proc;
@@ -1601,11 +1629,13 @@
p->p_flag |= P_WEXIT;
/* We don't want to switch away from exiting. */
/* XXX multiprocessor: stop LWPs on other processors. */
+#if 0
if (p->p_flag & P_SA) {
LIST_FOREACH(l2, &p->p_lwps, l_sibling)
l2->l_flag &= ~L_SA;
p->p_flag &= ~P_SA;
}
+#endif
/* Make other LWPs stick around long enough to be dumped */
p->p_userret = lwp_coredump_hook;
Index: sys/kern/kern_synch.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_synch.c,v
retrieving revision 1.132
diff -u -r1.132 kern_synch.c
--- sys/kern/kern_synch.c 29 Jun 2003 22:31:23 -0000 1.132
+++ sys/kern/kern_synch.c 8 Jul 2003 18:03:19 -0000
@@ -117,7 +117,10 @@
* The global scheduler state.
*/
struct prochd sched_qs[RUNQUE_NQS]; /* run queues */
+#if !defined(MPCPUMASK)
__volatile u_int32_t sched_whichqs; /* bitmap of non-empty queues */
+#endif
+
struct slpque sched_slpque[SLPQUE_TABLESIZE]; /* sleep queues */
struct simplelock sched_lock = SIMPLELOCK_INITIALIZER;
@@ -310,7 +313,11 @@
(l->l_priority / PPQ) != (l->l_usrpri / PPQ)) {
remrunqueue(l);
l->l_priority = l->l_usrpri;
+#if defined(MPCPUMASK)
+ setrunqueue0(l, l->l_cpumask);
+#else
setrunqueue(l);
+#endif
} else
l->l_priority = l->l_usrpri;
}
@@ -471,7 +478,7 @@
*/
if (catch) {
l->l_flag |= L_SINTR;
- if ((sig = CURSIG(l)) != 0) {
+ if (((sig = CURSIG(l)) != 0) || (p->p_flag & P_WEXIT)) {
if (l->l_wchan != NULL)
unsleep(l);
l->l_stat = LSONPROC;
@@ -624,7 +631,7 @@
if (l->l_flag & L_INMEM) {
setrunqueue(l);
if (l->l_flag & L_SA)
- l->l_proc->p_sa->sa_woken = l;
+ l->l_savp->savp_woken = l;
KASSERT(l->l_cpu != NULL);
need_resched(l->l_cpu);
} else
@@ -793,11 +800,13 @@
struct lwp *l = curlwp;
int r, s;
+#if 0
/* XXX Until the preempt() bug is fixed. */
if (more && (l->l_proc->p_flag & P_SA)) {
l->l_cpu->ci_schedstate.spc_flags &= ~SPCF_SWITCHCLEAR;
return;
}
+#endif
SCHED_LOCK(s);
l->l_priority = l->l_usrpri;
@@ -832,6 +841,12 @@
#endif
struct proc *p = l->l_proc;
int retval;
+#if defined(MPCPUMASK)
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+ struct sadata_vp *vp;
+ u_int32_t savp_whichqs;
+#endif
SCHED_ASSERT_LOCKED();
@@ -924,6 +939,16 @@
*/
uvmexp.swtch++;
if (newl == NULL) {
+ if (l->l_savp /* && (l->l_flag & L_SA) */) {
+ vp = l->l_savp;
+ vp->savp_cpuid = -1;
+ if (vp->savp_whichqs) {
+ savp_whichqs = vp->savp_whichqs;
+ for (CPU_INFO_FOREACH(cii, ci))
+ ci->ci_sched_whichqs |= savp_whichqs;
+ vp->savp_whichqs = 0;
+ }
+ }
retval = cpu_switch(l, NULL);
} else {
remrunqueue(newl);
@@ -1183,19 +1208,33 @@
* available queues.
*/
-void
-setrunqueue(struct lwp *l)
+static __inline void
+#if defined(MPCPUMASK)
+setrunqueue1(struct lwp *l, u_int32_t cpumask)
+#else
+setrunqueue1(struct lwp *l)
+#endif
{
struct prochd *rq;
struct lwp *prev;
int whichq;
+#if defined(MPCPUMASK)
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+#endif
#ifdef DIAGNOSTIC
if (l->l_back != NULL || l->l_wchan != NULL || l->l_stat != LSRUN)
panic("setrunqueue");
#endif
whichq = l->l_priority / 4;
+#if defined(MPCPUMASK)
+ for (CPU_INFO_FOREACH(cii, ci))
+ if ((cpumask & (1 << ci->ci_cpuid)) == 0)
+ ci->ci_sched_whichqs |= (1 << whichq);
+#else
sched_whichqs |= (1 << whichq);
+#endif
rq = &sched_qs[whichq];
prev = rq->ph_rlink;
l->l_forw = (struct lwp *)rq;
@@ -1205,13 +1244,51 @@
}
void
+setrunqueue(struct lwp *l)
+{
+
+ SCHED_ASSERT_LOCKED();
+
+#if defined(MPCPUMASK)
+ if ((l->l_flag & L_CPUMASKPERSIST) == 0)
+ l->l_cpumask = 0;
+ setrunqueue1(l, l->l_cpumask);
+#else
+ setrunqueue1(l);
+#endif
+
+}
+
+void
+setrunqueue0(struct lwp *l, u_int32_t cpumask)
+{
+
+ SCHED_ASSERT_LOCKED();
+
+#if defined(MPCPUMASK)
+ if (((l->l_flag & L_CPUMASKPERSIST) == 0) || cpumask != 0)
+ l->l_cpumask = cpumask;
+ setrunqueue1(l, l->l_cpumask);
+#else
+ setrunqueue1(l);
+#endif
+
+}
+
+void
remrunqueue(struct lwp *l)
{
struct lwp *prev, *next;
int whichq;
+#if defined(MPCPUMASK)
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+#endif
+
+ SCHED_ASSERT_LOCKED();
whichq = l->l_priority / 4;
-#ifdef DIAGNOSTIC
+#if defined(DIAGNOSTIC) && !defined(MPCPUMASK)
if (((sched_whichqs & (1 << whichq)) == 0))
panic("remrunqueue");
#endif
@@ -1220,8 +1297,14 @@
next = l->l_forw;
prev->l_forw = next;
next->l_back = prev;
+#if defined(MPCPUMASK)
+ if (prev == next)
+ for (CPU_INFO_FOREACH(cii, ci))
+ ci->ci_sched_whichqs &= ~(1 << whichq);
+#else
if (prev == next)
sched_whichqs &= ~(1 << whichq);
+#endif
}
#endif
Index: sys/kern/kern_time.c
===================================================================
RCS file: /cvs/netbsd/src/sys/kern/kern_time.c,v
retrieving revision 1.70
diff -u -r1.70 kern_time.c
--- sys/kern/kern_time.c 28 May 2003 22:27:57 -0000 1.70
+++ sys/kern/kern_time.c 8 Jul 2003 00:46:26 -0000
@@ -1188,6 +1188,7 @@
itimerfire(struct ptimer *pt)
{
struct proc *p = pt->pt_proc;
+ struct sadata_vp *vp;
int s;
if (pt->pt_ev.sigev_notify == SIGEV_SIGNAL) {
@@ -1215,9 +1216,11 @@
* makes testing for sa_idle alone insuffucent to
* determine if we really should call setrunnable.
*/
- if ((sa->sa_idle) && (p->p_stat != SSTOP)) {
+ if (p->p_stat != SSTOP) {
SCHED_LOCK(s);
- setrunnable(sa->sa_idle);
+ SIMPLEQ_FOREACH(vp, &sa->sa_vps, savp_next)
+ if (vp->savp_idle)
+ setrunnable(vp->savp_idle);
SCHED_UNLOCK(s);
}
pt->pt_poverruns = pt->pt_overruns;
Index: sys/sys/lwp.h
===================================================================
RCS file: /cvs/netbsd/src/sys/sys/lwp.h,v
retrieving revision 1.6
diff -u -r1.6 lwp.h
--- sys/sys/lwp.h 4 Feb 2003 13:41:48 -0000 1.6
+++ sys/sys/lwp.h 6 Jul 2003 18:16:51 -0000
@@ -63,6 +63,10 @@
int l_flag;
int l_stat;
lwpid_t l_lid; /* LWP identifier; local to process. */
+#if defined(MPCPUMASK)
+ int l_cpumask;
+#endif
+ struct sadata_vp *l_savp;
#define l_startzero l_swtime
u_int l_swtime; /* Time swapped in or out. */
@@ -112,6 +116,7 @@
#define L_SELECT 0x00040 /* Selecting; wakeup/waiting danger. */
#define L_SINTR 0x00080 /* Sleep is interruptible. */
#define L_TIMEOUT 0x00400 /* Timing out during sleep. */
+#define L_CPUMASKPERSIST 0x40000 /* cpu scheduling is persistent */
#define L_BIGLOCK 0x80000 /* LWP needs kernel "big lock" to run */
#define L_SA 0x100000 /* Scheduler activations LWP */
#define L_SA_UPCALL 0x200000 /* SA upcall is pending */
@@ -153,6 +158,7 @@
void setrunnable (struct lwp *);
#ifndef setrunqueue
void setrunqueue (struct lwp *);
+void setrunqueue0 (struct lwp *, u_int32_t);
#endif
#ifndef nextrunqueue
struct lwp *nextrunqueue(void);
Index: sys/sys/savar.h
===================================================================
RCS file: /cvs/netbsd/src/sys/sys/savar.h,v
retrieving revision 1.4
diff -u -r1.4 savar.h
--- sys/sys/savar.h 2 Feb 2003 02:22:14 -0000 1.4
+++ sys/sys/savar.h 8 Jul 2003 13:43:45 -0000
@@ -70,23 +70,35 @@
#define SAU_FLAG_DEFERRED 0x1
#define SA_UPCALL_DEFER 0x1000
+struct sadata_vp {
+ SIMPLEQ_ENTRY(sadata_vp) savp_next;
+ struct simplelock savp_lock; /* lock on these fields */
+ struct sadata *savp_sa;
+#if defined(MPCPUMASK)
+ int32_t savp_cpuid;
+ volatile u_int32_t savp_whichqs;
+#endif
+ struct lwp *savp_lwp; /* "virtual processor" allocation */
+ struct lwp *savp_woken; /* list of woken lwps */
+ struct lwp *savp_idle; /* lwp in sawait */
+ LIST_HEAD(, lwp) savp_lwpcache; /* list of avaliable lwps */
+ int savp_ncached; /* list length */
+ SIMPLEQ_HEAD(, sadata_upcall) savp_upcalls; /* pending upcalls */
+};
+
struct sadata {
struct simplelock sa_lock; /* lock on these fields */
int sa_flag; /* SA_* flags */
sa_upcall_t sa_upcall; /* upcall entry point */
- struct lwp *sa_vp; /* "virtual processor" allocation */
- struct lwp *sa_woken; /* list of woken lwps */
- struct lwp *sa_idle; /* lwp in sawait */
int sa_concurrency; /* desired concurrency */
- LIST_HEAD(, lwp) sa_lwpcache; /* list of avaliable lwps */
- int sa_ncached; /* list length */
stack_t *sa_stacks; /* pointer to array of upcall stacks */
int sa_nstacks; /* number of valid stacks */
- SIMPLEQ_HEAD(, sadata_upcall) sa_upcalls; /* pending upcalls */
+ SIMPLEQ_HEAD(, sadata_vp) sa_vps;
};
#define SA_FLAG_ALL SA_FLAG_PREEMPT
+extern struct pool savp_pool; /* memory pool for sadata_vp structures */
extern struct pool sadata_pool; /* memory pool for sadata structures */
extern struct pool saupcall_pool; /* memory pool for pending upcalls */
@@ -111,7 +123,7 @@
size_t, void *, struct sadata_upcall *, stack_t *);
void sa_putcachelwp(struct proc *, struct lwp *);
-struct lwp *sa_getcachelwp(struct proc *);
+struct lwp *sa_getcachelwp(struct sadata_vp *);
void sa_upcall_userret(struct lwp *);
Index: sys/sys/sched.h
===================================================================
RCS file: /cvs/netbsd/src/sys/sys/sched.h,v
retrieving revision 1.18
diff -u -r1.18 sched.h
--- sys/sys/sched.h 8 Jul 2003 06:49:22 -0000 1.18
+++ sys/sys/sched.h 9 Jul 2003 16:48:27 -0000
@@ -199,7 +199,9 @@
*/
extern struct prochd sched_qs[];
extern struct slpque sched_slpque[];
+#if !defined(MPCPUMASK)
extern __volatile u_int32_t sched_whichqs;
+#endif
#define SLPQUE(ident) (&sched_slpque[SLPQUE_LOOKUP(ident)])
@@ -264,6 +266,12 @@
#define SCHED_UNLOCK(s) splx(s)
#endif /* MULTIPROCESSOR || LOCKDEBUG */
+
+#if defined(MPCPUMASK)
+#define RUNQS_NOTEMPTY() (cpu_sched_whichqs() != 0)
+#else
+#define RUNQS_NOTEMPTY() (sched_whichqs != 0)
+#endif
#endif /* _KERNEL */
#endif /* _SYS_SCHED_H_ */
Index: sys/uvm/uvm_page.c
===================================================================
RCS file: /cvs/netbsd/src/sys/uvm/uvm_page.c,v
retrieving revision 1.89
diff -u -r1.89 uvm_page.c
--- sys/uvm/uvm_page.c 1 Jun 2003 09:26:10 -0000 1.89
+++ sys/uvm/uvm_page.c 2 Jul 2003 21:39:31 -0000
@@ -1429,7 +1429,7 @@
s = uvm_lock_fpageq();
firstbucket = nextbucket;
do {
- if (sched_whichqs != 0) {
+ if (RUNQS_NOTEMPTY()) {
uvm_unlock_fpageq(s);
return;
}
@@ -1442,7 +1442,7 @@
pgfl = &uvm.page_free[free_list];
while ((pg = TAILQ_FIRST(&pgfl->pgfl_buckets[
nextbucket].pgfl_queues[PGFL_UNKNOWN])) != NULL) {
- if (sched_whichqs != 0) {
+ if (RUNQS_NOTEMPTY()) {
uvm_unlock_fpageq(s);
return;
}
Index: lib/libpthread/pthread_sa.c
===================================================================
RCS file: /cvs/netbsd/src/lib/libpthread/pthread_sa.c,v
retrieving revision 1.12
diff -u -r1.12 pthread_sa.c
--- lib/libpthread/pthread_sa.c 26 Jun 2003 01:28:14 -0000 1.12
+++ lib/libpthread/pthread_sa.c 8 Jul 2003 13:19:38 -0000
@@ -628,7 +628,7 @@
{
pthread_t self, t;
stack_t upcall_stacks[PT_UPCALLSTACKS];
- int ret, i, errnosave, flags, rr;
+ int ret, i, errnosave, flags, rr, concurrency;
char *value;
flags = 0;
@@ -649,6 +649,11 @@
if (value)
rr = atoi(value);
+ concurrency = 1; /* XXX default to ncpus? */
+ value = getenv("PTHREAD_CONCURRENCY");
+ if (value)
+ concurrency = atoi(value);
+
ret = sa_register(pthread__upcall, NULL, flags);
if (ret) {
if (errno == ENOSYS)
@@ -696,6 +701,9 @@
/* Start the round-robin timer. */
if (rr != 0 && pthread__setrrtimer(rr, 1) != 0)
abort();
+
+ if (concurrency > 1)
+ sa_setconcurrency(concurrency);
}
/*
Index: lib/libpthread/pthread_sig.c
===================================================================
RCS file: /cvs/netbsd/src/lib/libpthread/pthread_sig.c,v
retrieving revision 1.14
diff -u -r1.14 pthread_sig.c
--- lib/libpthread/pthread_sig.c 27 May 2003 15:24:25 -0000 1.14
+++ lib/libpthread/pthread_sig.c 3 Jul 2003 23:44:38 -0000
@@ -419,7 +419,7 @@
error = __sigtimedwait(&wset, info, (timeout) ? &timo : NULL);
pthread_spinlock(self, &pt_sigwaiting_lock);
- if ((error && errno != ECANCELED)
+ if ((error && (errno != ECANCELED || self->pt_cancel))
|| (!error && __sigismember14(set, info->si_signo)) ) {
/*
* Normal function return. Clear pt_sigwmaster,
--3138531-1859-1057795464=:2844
Content-Type: TEXT/PLAIN; CHARSET=US-ASCII
Content-Disposition: INLINE
--3138531-1859-1057795464=:2844--