Subject: port-i386/20251: i386 SMP locking problem after update to 1.6N
To: None <gnats-bugs@gnats.netbsd.org>
From: None <he@netbsd.org>
List: netbsd-bugs
Date: 02/08/2003 07:33:57
>Number:         20251
>Category:       port-i386
>Synopsis:       i386 SMP locking problem after update to 1.6N
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    port-i386-maintainer
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Fri Feb 07 22:35:01 PST 2003
>Closed-Date:
>Last-Modified:
>Originator:     Havard Eidnes
>Release:        NetBSD 1.6N, updated Feb 6 06:54 2003 UTC
>Organization:
	Unorganized, Inc.
>Environment:
System: NetBSD splitter-pine.urc.uninett.no 1.6N NetBSD 1.6N (GENERIC.MPDEBUG) #11: Thu Feb  6 08:46:26 CET 2003     he@splitter-pine.urc.uninett.no:/sys/arch/i386/compile/GENERIC.MPDEBUG i386
Architecture: i386
Machine: i386
>Description:
	Got a panic while doing a largish scp.
	Console log showing some debug info:

timeout delayed -3
timeout delayed -4
timeout delayed -1
LK_SPIN spinout, excl 1, share 0
LK_SPIN spinout, excl 1, share 0
held by CPU 2
held by CPU 2
last locked at ../../../../kern/kern_synch.c:958
last locked at ../../../../kern/kern_synch.c:958
last unlocked at ../../../../kern/kern_synch.c:839
last unlocked at ../../../../kern/kern_synch.c:839
mi_switch(mi_switch(e4a2d300e580e684,,00,,e4a47b5ce64fee10,,c0316f58c0316f58,,e4a2d300e580e684) at ) at mi_switchmi_switch+0x2c4+0x2c4
preempt(preempt(11,,241930,,e4a47cac0,,c035257a1,,e5f33e042000) at ) at preemptpreempt+0x78+0x78
genfs_putpages(uiomove(e4a47d20e6236000,,c030c2142000,,c06e1ec0e64feed0,,c02c10840,,0e64fef80) at ) at genfs_putpagesuiomove+0x5d8+0xa1
ffs_putpages(pipe_read(e4a47d20e5858f70,,0e5858f98,,164e64feed0,,1c2130600,,01) at ) at ffs_putpagespipe_read+0x117+0xae
VOP_PUTPAGES(dofileread(e5f33e04e585a564,,00,,0e5858f70,,0bfbfd37c,,02000) at ) at VOP_PUTPAGESdofileread+0x49+0x9a
ffs_full_fsync(sys_read(e4a47e58e580e684,,c0573f2ce64fef80,,33de64fef78,,0c03bfa6b,,0e580e684) at ) at ffs_full_fsyncsys_read+0xa1+0x6b
ffs_fsync(syscall_plain(e4a47e5880b001f,,10012bfbf001f,,477bfbf001f,,c057db001f,,08082cb0) at ) at ffs_fsyncsyscall_plain+0x3c+0xc0
VStopped in pid 9770.1 (ssh) at  cpu_Debugger+0x4:       leave
db{1}> 
db{1}> show reg
ds                0x10
es                0x10
fs                0x30
gs                0x10
edi                0xd
esi                  0
ebp         0xe64fed60
ebx         0xc0728c20  kernel_lock
edx         0xe64fec8c
ecx         0xc0746878  end+0x34
eax         0xbfbff37c
eip         0xc03acde0  cpu_Debugger+0x4
cs                 0x8
eflags           0x282
esp         0xe64fed60
ss                0x10
cpu_Debugger+0x4:       leave
db{1}> trace
cpu_Debugger(0,59,e585a564,e585a564,0) at cpu_Debugger+0x4
_spinlock_acquire_count(c0728c20,1,c0576ba9,3be,e580e684) at _spinlock_acquire_c
ount+0x15c
mi_switch(e580e684,0,e64fee10,c0316f58,e580e684) at mi_switch+0x2c4
preempt(1,0,0,1,2000) at preempt+0x78
uiomove(e6236000,2000,e64feed0,0,e64fef80) at uiomove+0xa1
pipe_read(e5858f70,e5858f98,e64feed0,c2130600,1) at pipe_read+0xae
dofileread(e585a564,0,e5858f70,bfbfd37c,2000) at dofileread+0x9a
sys_read(e580e684,e64fef80,e64fef78,c03bfa6b,e580e684) at sys_read+0x6b
syscall_plain(80b001f,bfbf001f,bfbf001f,1f,8082cb0) at syscall_plain+0xc0
db{1}> machine cpu 0
using cpu 0
db{1}> show reg
ds                0x10
es                0x10
fs                0x30
gs                0x10
edi                  0
esi               0x34
ebp         0xe4a478bc
ebx              0x3f8
edx              0x3fd
ecx           0x16e2ab  gdt_desc+0x16c17f
eax                  0
eip         0xc021d11f  com_common_putc+0x137
cs                 0x8
eflags           0x246
esp         0xe4a478a4
ss                0x10
com_common_putc+0x137:  testb   $0x20,%al
db{1}> trace
com_common_putc(800,0,3f8,56,c077faf0) at com_common_putc+0x137
comcnputc(800,56,e4a4790c,c02ec0fc,0) at comcnputc+0x1d
cnputc(56,c077faf0,e4a47984,e4a47938,fffffff6) at cnputc+0x23
putchar(56,5,0,c02ebe0a,d) at putchar+0x111
kprintf(c058cd83,5,0,0,e4a479fc) at kprintf+0xa47
printf(c058cd83,c07c4211,e4a47a4c,0,c0728c20) at printf+0x4d
db_stack_trace_print(e4a47adc,1,fff8,c05740c3,c0324e3c) at db_stack_trace_print+
0x1d9
_spinlock_acquire_count(c0728c20,1,c0576ba9,3be,e4a2d300) at _spinlock_acquire_c
ount+0x154
mi_switch(e4a2d300,0,e4a47b5c,c0316f58,e4a2d300) at mi_switch+0x2c4
preempt(1,24193,e4a47cac,c035257a,e5f33e04) at preempt+0x78
genfs_putpages(e4a47d20,c030c214,c06e1ec0,c02c1084,0) at genfs_putpages+0x5d8
ffs_putpages(e4a47d20,0,164,1,0) at ffs_putpages+0x117
VOP_PUTPAGES(e5f33e04,0,0,0,0) at VOP_PUTPAGES+0x49
ffs_full_fsync(e4a47e58,c0573f2c,33d,0,0) at ffs_full_fsync+0xa1
ffs_fsync(e4a47e58,10012,477,c057db00,0) at ffs_fsync+0x3c
VOP_FSYNC(e5f33e04,c1e4df00,0,0,0) at VOP_FSYNC+0x58
ffs_sync(c1fba200,3,c1e4df00,e4a24a80,0) at ffs_sync+0x153
sync_fsync(e4a47f28,0,c1e4ea4c,c03165f7,0) at sync_fsync+0x7a
VOP_FSYNC(e5656544,c1e4df00,8,0,0) at VOP_FSYNC+0x58
sched_sync(e4a2d300,0,0,0,c010030c) at sched_sync+0x121
db{1}> machine cpu 1
using cpu 1
db{1}> show reg
ds                0x10
es                0x10
fs                0x30
gs                0x10
edi                0xd
esi                  0
ebp         0xe64fed60
ebx         0xc0728c20  kernel_lock
edx         0xe64fec8c
ecx         0xc0746878  end+0x34
eax         0xbfbff37c
eip         0xc03acde0  cpu_Debugger+0x4
cs                 0x8
eflags           0x282
esp         0xe64fed60
ss                0x10
cpu_Debugger+0x4:       leave
db{1}> trace
cpu_Debugger(0,59,e585a564,e585a564,0) at cpu_Debugger+0x4
_spinlock_acquire_count(c0728c20,1,c0576ba9,3be,e580e684) at _spinlock_acquire_c
ount+0x15c
mi_switch(e580e684,0,e64fee10,c0316f58,e580e684) at mi_switch+0x2c4
preempt(1,0,0,1,2000) at preempt+0x78
uiomove(e6236000,2000,e64feed0,0,e64fef80) at uiomove+0xa1
pipe_read(e5858f70,e5858f98,e64feed0,c2130600,1) at pipe_read+0xae
dofileread(e585a564,0,e5858f70,bfbfd37c,2000) at dofileread+0x9a
sys_read(e580e684,e64fef80,e64fef78,c03bfa6b,e580e684) at sys_read+0x6b
syscall_plain(80b001f,bfbf001f,bfbf001f,1f,8082cb0) at syscall_plain+0xc0
db{1}> machine cpu 2
using cpu 2
db{1}> show reg
ds                0x10
es                0x10
fs                0x30
gs                0x10
edi              0x33d
esi                  0
ebp         0xe6746d50
ebx                0x2
edx         0xc06e18e0  simplelock_list_slock
ecx                  0
eax                  0
eip         0xc03c1704  __cpu_simple_lock+0x50
cs                 0x8
eflags           0x246
esp         0xe6746d28
ss                0x10
__cpu_simple_lock+0x50: leal             0(,%ebx,4),%eax
db{1}> trace
__cpu_simple_lock(c06e18e0,c2110030,1,1,c2113474) at __cpu_simple_lock+0x50
_simple_unlock(c2113474,c0573f2c,33d,287,c211344c) at _simple_unlock+0x105
_lockmgr(c2113474,2,0,c0579c00,213) at _lockmgr+0x1623
pipe_direct_write(c2113420,e6746ed0,33d,c034ea81,e6746f80) at pipe_direct_write+
0x2fa
pipe_write(e5858c30,e5858c58,e6746ed0,c2130600,1) at pipe_write+0x16f
dofilewrite(e57edc4c,6,e5858c30,8057000,2000) at dofilewrite+0x9b
sys_write(e580e004,e6746f80,e6746f78,c03bfa6b,e580e004) at sys_write+0x6b
syscall_plain(1f,1f,bfbf001f,1f,2000) at syscall_plain+0xc0
db{1}> machine cpu 3
using cpu 3
db{1}> show reg
ds                0x10
es                0x10
fs                0x30
gs                0x10
edi                0xd
esi                  0
ebp         0xe5640e30
ebx         0xc0728c20  kernel_lock
edx         0xc06ec3a0  cpu_info_primary
ecx                  0
eax           0x400400  gdt_desc+0x3fe2d4
eip         0xc030b6c0  _spinlock_acquire_count+0x164
cs                 0x8
eflags           0x202
esp         0xe5640e08
ss                0x10
_spinlock_acquire_count+0x164:  testl   %esi,%esi
db{1}> trace
_spinlock_acquire_count(c0728c20,1,c0576ba9,3be,e4a2d600) at _spinlock_acquire_c
ount+0x164
mi_switch(e4a2d600,0,1a8,c030c214,0) at mi_switch+0x2c4
ltsleep(e5641014,128,c0576538,0,0) at ltsleep+0x3bb
sigsuspend1(e4a608c4,e5640f20,10,c030c61e,0) at sigsuspend1+0x12e
sys___sigsuspend14(e4a2d600,e5640f80,e5640f78,c03bfa6b,e4a2d600) at sys___sigsus
pend14+0x40
syscall_plain(bfbf001f,bfbf001f,1f,bfbf001f,bfbffc35) at syscall_plain+0xc0
db{1}> ps
 PID           PPID     PGRP        UID S   FLAGS LWPS          COMMAND    WAIT
>How-To-Repeat:
	So far this is a singleton, if it occurs again, I'll update
	this PR.

>Fix:
	Sorry, don't know.
>Release-Note:
>Audit-Trail:
>Unformatted:
 >9770          9769     9769          0 2  0x4002    1              ssh
  9769          9735     9769          0 2  0x4002    1              scp
  9752          9747     9752       1000 2  0x4102    1           systat   ttyin
  9747          9746     9747       1000 2  0x4002    1             tcsh   pause
  9746          9743     9743       1000 2  0x4100    1            xterm  select
  9743          9742     9743       1000 2  0x4000    1             tcsh   pause
  9742          9740     9740       1000 2   0x100    1             sshd  select
  9740           220     9740          0 2   0x100    1             sshd   netio
  9735          9712     9735          0 2  0x4002    1              csh   pause
  9712          9711     9712       1000 2  0x4002    1             tcsh   pause
  9711          9708     9708       1000 2  0x4100    1            xterm  select
  9708          9707     9708       1000 2  0x4000    1             tcsh   pause
  9707          9705     9705       1000 2   0x100    1             sshd  select
  9705           220     9705          0 2   0x100    1             sshd   netio
  27774        18420    27774       1001 2  0x4102    1           systat   ttyin
  18440            1    18438       1001 2  0x4003    1         gconfd-1  select
  18438        18420    18438       1001 2  0x4003    1       galeon-bin  select
  18420        18418    18420       1001 2  0x4002    1             bash    wait
  18418        18414    18414       1001 2  0x4100    1            xterm  select
  18414        18413    18414       1001 2  0x4000    1              csh   pause
  18413        18411    18411       1001 2   0x100    1             sshd  select
  18411          220    18411          0 2   0x100    1             sshd   netio
  1697             1     1697       1001 2  0x4000    1             oafd  select
  1691             1     1691       1001 2  0x4002    1       junkbuster  netcon
  261              1      247       1000 2  0x4000    1       setiathome
  260              1      244       1000 2  0x4000    1       setiathome
  259              1      245       1000 2  0x4000    1       setiathome
  258              1      246       1000 2  0x4000    1       setiathome
  252              1      252          0 2  0x4002    1            getty   ttyin
  251              1      251          0 2  0x4002    1            getty   ttyin
  250              1      250          0 2  0x4002    1            getty   ttyin
  249              1        1          0 2  0x4000    1            getty nanosle
  248              1      248          0 2  0x4002    1            getty   ttyin
  238              1      238          0 2       0    1             cron nanosle
  234              1      234          0 2       0    1            inetd   pause
  220              1      220          0 2       0    1             sshd  select
  205              1      205          0 2       0    1             ntpd
  189            188        9       1002 2     0x2    1         postgres  select
  188            184        9       1002 2     0x2    1         postgres  select
  184              1        9       1002 2  0x4002    1         postgres  select
  129              1      129          0 2       0    1        mount_mfs  mfsidl
  107              1      107          0 2       0    1          syslogd
  8                0        0          0 2 0x20200    1         aiodoned aiodone
  7                0        0          0 2 0x20200    1          ioflush
  6                0        0          0 2 0x20200    1           reaper  reaper
  5                0        0          0 2 0x20200    1       pagedaemon pgdaemo
  4                0        0          0 2 0x20200    1         scsibus2  sccomp
  3                0        0          0 2 0x20200    1         scsibus1  sccomp
  2                0        0          0 2 0x20200    1         scsibus0  sccomp
  1                0        1          0 2  0x4000    1             init    wait
  0               -1        0          0 2 0x20200    1          swapper schedul
  9785          1691     1691       1001 5  0x2000    0       junkbuster       *
  9784          1691     1691       1001 5  0x2000    0       junkbuster       *
  9782          1691     1691       1001 5  0x2000    0       junkbuster       *
  9781          1691     1691       1001 5  0x2000    0       junkbuster       *
  9783          1691     1691       1001 5  0x2000    0       junkbuster       *
  9780          1691     1691       1001 5  0x2000    0       junkbuster       *
 db{1}> call cpu_reset
 
 -->> NetBSD/i386 BIOS Boot, Revision 2.7
 -->> (he@hamster.urc.uninett.no, Mon Dec 10 05:46:17 CET 2001)
 -->> Memory: 639/1047544 k
 Use hd1a:netbsd to boot sd0 when wd0 is also installed
 Press return to boot now, any other key for boot menu
 booting wd0a:netbsd - starting in 0
 6098372+134632+342620+[300160+272593]=0x6d2814
 [ using 573392 bytes of netbsd ELF symbol table ]
 BIOS CFG: Model-SubM-Rev: fc-01-00, 0x74<EBDA,KBDINT,RTC,IC2>
 Copyright (c) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003
     The NetBSD Foundation, Inc.  All rights reserved.
 Copyright (c) 1982, 1986, 1989, 1991, 1993
     The Regents of the University of California.  All rights reserved.
 
 NetBSD 1.6N (GENERIC.MPDEBUG) #11: Thu Feb  6 08:46:26 CET 2003
     he@splitter-pine.urc.uninett.no:/sys/arch/i386/compile/GENERIC.MPDEBUG
 total memory = 1023 MB
 avail memory = 931 MB
 using 6144 buffers containing 52508 KB of memory
 BIOS32 rev. 0 found at 0xffe90
 mainbus0 (root)
 mainbus0: scanning 0x9fc00 to 0x9fff0 for MP signature
 mainbus0: scanning 0x9f800 to 0x9fbf0 for MP signature
 mainbus0: scanning 0xf0000 to 0xffff0 for MP signature
 mainbus0: MP floating pointer found in bios at 0xfe710
 mainbus0: MP config table at 0xf0000, 468 bytes long
 mainbus0: Intel MP Specification (Version 1.4) (DELL     PowerEdge 83)
 cpu0 at mainbus0: apid 3 (boot processor)
 cpu0: Intel Pentium II (686-class), 450.01 MHz, id 0x653
 cpu0: features 183fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR>
 cpu0: features 183fbff<PGE,MCA,CMOV,PAT,PSE36,MMX>
 cpu0: features 183fbff<FXSR>
 cpu0: I-cache 16 KB 32b/line 4-way, D-cache 16 KB 32b/line 4-way
 cpu0: L2 cache 512 KB 32b/line 4-way
 cpu0: ITLB 32 4 KB entries 4-way, 2 4 MB entries fully associative
 cpu0: DTLB 64 4 KB entries 4-way, 8 4 MB entries 4-way
 cpu0: calibrating local timer
 cpu0: apic clock running at 99 MHz
 cpu0: 32 page colors
 cpu0: kstack at 0xe418d000 for 16384 bytes
 cpu0: idle pcb at 0xe418d000, idle sp at 0xe4190f98
 cpu1 at mainbus0: apid 0 (application processor)
 cpu1: starting
 cpu1: Intel Pentium II (686-class), 449.98 MHz, id 0x653
 cpu1: features 183fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR>
 cpu1: features 183fbff<PGE,MCA,CMOV,PAT,PSE36,MMX>
 cpu1: features 183fbff<FXSR>
 cpu1: I-cache 16 KB 32b/line 4-way, D-cache 16 KB 32b/line 4-way
 cpu1: L2 cache 512 KB 32b/line 4-way
 cpu1: ITLB 32 4 KB entries 4-way, 2 4 MB entries fully associative
 cpu1: DTLB 64 4 KB entries 4-way, 8 4 MB entries 4-way
 cpu1: kstack at 0xe4199000 for 16384 bytes
 cpu1: idle pcb at 0xe4199000, idle sp at 0xe419cf98
 cpu2 at mainbus0: apid 1 (application processor)
 cpu2: starting
 cpu2: Intel Pentium II (686-class), 449.98 MHz, id 0x653
 cpu2: features 183fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR>
 cpu2: features 183fbff<PGE,MCA,CMOV,PAT,PSE36,MMX>
 cpu2: features 183fbff<FXSR>
 cpu2: I-cache 16 KB 32b/line 4-way, D-cache 16 KB 32b/line 4-way
 cpu2: L2 cache 512 KB 32b/line 4-way
 cpu2: ITLB 32 4 KB entries 4-way, 2 4 MB entries fully associative
 cpu2: DTLB 64 4 KB entries 4-way, 8 4 MB entries 4-way
 cpu2: kstack at 0xe41b5000 for 16384 bytes
 cpu2: idle pcb at 0xe41b5000, idle sp at 0xe41b8f98
 cpu3 at mainbus0: apid 2 (application processor)
 cpu3: starting
 cpu3: Intel Pentium II (686-class), 449.98 MHz, id 0x653
 cpu3: features 183fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR>
 cpu3: features 183fbff<PGE,MCA,CMOV,PAT,PSE36,MMX>
 cpu3: features 183fbff<FXSR>
 cpu3: I-cache 16 KB 32b/line 4-way, D-cache 16 KB 32b/line 4-way
 cpu3: L2 cache 512 KB 32b/line 4-way
 cpu3: ITLB 32 4 KB entries 4-way, 2 4 MB entries fully associative
 cpu3: DTLB 64 4 KB entries 4-way, 8 4 MB entries 4-way
 cpu3: kstack at 0xe41d1000 for 16384 bytes
 cpu3: idle pcb at 0xe41d1000, idle sp at 0xe41d4f98
 mpbios: bus 0 is type PCI   
 mpbios: bus 1 is type PCI   
 mpbios: bus 2 is type PCI   
 mpbios: bus 3 is type ISA   
 ioapic0 at mainbus0 apid 4 (I/O APIC)
 ioapic0: pa 0xfec00000, virtual wire mode, version 11, 24 pins
 ioapic0: misconfigured as apic 0
 ioapic0: remapped to apic 4
 ioapic0: int0 attached to ExtINT (type 3<type=3=ExtINT> flags 5<pol=1=Act Hi,trig=1=Edge>)
 ioapic0: int1 attached to isa0 irq 1 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int2 attached to isa0 irq 0 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int3 attached to isa0 irq 3 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int4 attached to isa0 irq 4 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int6 attached to isa0 irq 6 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int7 attached to isa0 irq 7 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int8 attached to isa0 irq 8 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int9 attached to isa0 irq 9 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int12 attached to isa0 irq 12 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int15 attached to isa0 irq 15 (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int16 attached to pci2 device 1 INT_B (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int16 attached to pci2 device 3 INT_C (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int16 attached to pci2 device 5 INT_D (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int16 attached to pci2 device 7 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int17 attached to pci2 device 1 INT_C (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int17 attached to pci2 device 3 INT_D (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int17 attached to pci2 device 5 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int17 attached to pci2 device 7 INT_B (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int18 attached to pci2 device 1 INT_D (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int18 attached to pci2 device 3 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int18 attached to pci2 device 5 INT_B (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int18 attached to pci2 device 7 INT_C (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int19 attached to pci2 device 1 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int19 attached to pci2 device 3 INT_B (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int19 attached to pci2 device 5 INT_C (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int19 attached to pci2 device 7 INT_D (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int20 attached to pci0 device 6 INT_B (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int20 attached to pci0 device 8 INT_C (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int20 attached to pci0 device 10 INT_D (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int20 attached to pci1 device 8 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int21 attached to pci1 device 4 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int22 attached to pci1 device 6 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int14 attached to pci0 device 6 INT_C (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int14 attached to pci0 device 8 INT_D (type 0<type=0> flags 0<pol=0,trig=0>)
 ioapic0: int14 attached to pci0 device 10 INT_A (type 0<type=0> flags 0<pol=0,trig=0>)
 local apic: int0 attached to ExtINT (type 3<type=3=ExtINT> flags 5<pol=1=Act Hi,trig=1=Edge>)
 local apic: int1 attached to NMI (type 1<type=1=NMI> flags 5<pol=1=Act Hi,trig=1=Edge>)
 mainbus0: MP WARNING: 128 bytes of extended entries not examined
 pci0 at mainbus0 bus 0: configuration mode 1
 pci0: i/o space, memory space enabled, rd/line, rd/mult, wr/inv ok
 pcib0 at pci0 dev 2 function 0
 pcib0: Intel 82371AB PCI-to-ISA Bridge (PIIX4) (rev. 0x02)
 pciide0 at pci0 dev 2 function 1: Intel 82371AB IDE controller (PIIX4) (rev. 0x01)
 pciide0: device disabled (at device)
 uhci0 at pci0 dev 2 function 2: Intel 82371AB USB Host Controller (PIIX4) (rev. 0x01)
 uhci0: can't map i/o space
 Intel 82371AB Power Management Controller (PIIX4) (miscellaneous bridge, revision 0x02) at pci0 dev 2 function 3 not configured
 vga1 at pci0 dev 4 function 0: ATI Technologies 3D Rage Pro (rev. 0x5c)
 wsdisplay0 at vga1 kbdmux 1
 wsmux1: connecting to wsdisplay0
 fxp0 at pci0 dev 10 function 0: i82558 Ethernet, rev 5
 fxp0: interrupting at apic 4 int 14 (irq 14)
 fxp0: Ethernet address 00:90:27:45:e3:2a
 inphy0 at fxp0 phy 1: i82555 10/100 media interface, rev. 0
 inphy0: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
 pchb0 at pci0 dev 16 function 0
 pchb0: Intel 82451NX Memory & I/O Controller (MIOC) (rev. 0x03)
 pchb1 at pci0 dev 18 function 0
 pchb1: Intel 82451NX PCI Expander Bridge (PXB) (rev. 0x02)
 pchb2 at pci0 dev 19 function 0
 pchb2: Intel 82451NX PCI Expander Bridge (PXB) (rev. 0x02)
 pci1 at pchb2 bus 1
 pci1: i/o space, memory space enabled, rd/line, rd/mult, wr/inv ok
 ahc1 at pci1 dev 4 function 0
 ahc1: interrupting at apic 4 int 21 (irq 11)
 ahc1: aic7890/91 Wide Channel A, SCSI Id=7, 16/255 SCBs
 scsibus0 at ahc1: 16 targets, 8 luns per target
 ahc2 at pci1 dev 6 function 0
 ahc2: interrupting at apic 4 int 22 (irq 10)
 ahc2: aic7890/91 Wide Channel A, SCSI Id=7, 16/255 SCBs
 scsibus1 at ahc2: 16 targets, 8 luns per target
 ahc3 at pci1 dev 8 function 0
 ahc3: interrupting at apic 4 int 20 (irq 5)
 ahc3: aic7860 Single Channel A, SCSI Id=7, 3/255 SCBs
 scsibus2 at ahc3: 8 targets, 8 luns per target
 pchb3 at pci0 dev 20 function 0
 pchb3: Intel 82451NX PCI Expander Bridge (PXB) (rev. 0x02)
 pci2 at pchb3 bus 2
 pci2: i/o space, memory space enabled, rd/line, rd/mult, wr/inv ok
 isa0 at pcib0
 com0 at isa0 port 0x3f8-0x3ff irq 4: ns16550a, working fifo
 com0: console
 com1 at isa0 port 0x2f8-0x2ff irq 3: ns16550a, working fifo
 pckbc0 at isa0 port 0x60-0x64
 pckbdprobe: reset error 5
 pmsprobe: reset error 5
 lpt0 at isa0 port 0x378-0x37b irq 7
 lptprobe: mask ff data 55 failed
 lptprobe: mask ff data 55 failed
 seaprobe: board type unknown at address 0xc0804000
 pcppi0 at isa0 port 0x61
 midi0 at pcppi0: PC speaker
 sysbeep0 at pcppi0
 isapnp0 at isa0 port 0x279: ISA Plug 'n Play device support
 npx0 at isa0 port 0xf0-0xff: using exception 16
 fdc0 at isa0 port 0x3f0-0x3f7 irq 6 drq 2
 fd0 at fdc0 drive 0: 1.44MB, 80 cyl, 2 head, 18 sec
 isapnp0: no ISA Plug 'n Play devices found
 cpu0: prelint0 700<vector=0,delmode=7,dest=0> 0<target=0>
 cpu0: prelint1 400<vector=0,delmode=4,dest=0> 0<target=0>
 cpu0: timer0 300c0<vector=c0,delmode=0,masked,dest=0> 0<target=0>
 cpu0: pcint0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu0: lint0 10700<vector=0,delmode=7,masked,dest=0> 0<target=0>
 cpu0: lint1 400<vector=0,delmode=4,dest=0> 0<target=0>
 cpu0: err0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 ioapic0: enabling
 ioapic0: int3 1d1<vector=d1,delmode=1,dest=0> 3000000<target=3>
 ioapic0: int4 1d0<vector=d0,delmode=1,dest=0> 3000000<target=3>
 ioapic0: int6 163<vector=63,delmode=1,dest=0> 3000000<target=3>
 ioapic0: int7 190<vector=90,delmode=1,dest=0> 3000000<target=3>
 ioapic0: int14 a170<vector=70,delmode=1,actlo,level,dest=0> 3000000<target=3>
 ioapic0: int20 a162<vector=62,delmode=1,actlo,level,dest=0> 3000000<target=3>
 ioapic0: int21 a160<vector=60,delmode=1,actlo,level,dest=0> 3000000<target=3>
 ioapic0: int22 a161<vector=61,delmode=1,actlo,level,dest=0> 3000000<target=3>
 raidattach: Asked for 8 units
 Kernelized RAIDframe activated
 scsibus0: waiting 2 seconds for devices to settle...
 sd0 at scsibus0 target 0 lun 0: <IBM, DNES-318350Y, SA60> disk fixed
 sd0: 17366 MB, 11474 cyl, 10 head, 309 sec, 512 bytes/sect x 35566501 sectors
 sd0: sync (25.0ns offset 31), 16-bit (80.000MB/s) transfers, tagged queueing
 timeout delayed -5
 sd1 at scsibus0 target 1 lun 0: <QUANTUM, QM34500TD-SCA, N1K0> disk fixed
 sd1: 4341 MB, 8057 cyl, 5 head, 220 sec, 512 bytes/sect x 8891624 sectors
 sd1: sync (25.0ns offset 31), 16-bit (80.000MB/s) transfers, tagged queueing
 timeout delayed -5
 sd2 at scsibus0 target 2 lun 0: <QUANTUM, QM34500TD-SCA, N1K0> disk fixed
 sd2: 4341 MB, 8057 cyl, 5 head, 220 sec, 512 bytes/sect x 8891624 sectors
 sd2: sync (25.0ns offset 31), 16-bit (80.000MB/s) transfers, tagged queueing
 timeout delayed -4
 timeout delayed -2
 ses0 at scsibus0 target 6 lun 0: <DELL, 1x3 U2W SCSI BP, 5.39> processor fixed
 ses0: SAF-TE Compliant Device
 ses0: async, 8-bit transfers
 timeout delayed -2
 timeout delayed -3
 scsibus1: waiting 2 seconds for devices to settle...
 scsibus2: waiting 2 seconds for devices to settle...
 cd0 at scsibus2 target 5 lun 0: <NEC, CD-ROM DRIVE:465, 1.03> cdrom removable
 cd0: sync (50.0ns offset 15), 8-bit (20.000MB/s) transfers
 timeout delayed -2
 Searching for RAID components...
 boot device: sd0
 root on sd0a dumps on sd0b
 mountroot: trying coda...
 mountroot: trying msdos...
 mountroot: trying cd9660...
 isofs: session offset (part 0) 0
 mountroot: trying ntfs...
 mountroot: trying nfs...
 mountroot: trying lfs...
 mountroot: trying ext2fs...
 mountroot: trying ffs...
 root file system type: ffs
 cpu1: prelint0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu1: prelint1 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu1: timer0 200c0<vector=c0,delmode=0,dest=0> 0<target=0>
 cpu1: pcint0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu1: lint0 10700<vector=0,delmode=7,masked,dest=0> 0<target=0>
 cpu1: lint1 400<vector=0,delmode=4,dest=0> 0<target=0>
 cpu1: err0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu1: CPU 0 running
 cpu2: prelint0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu2: prelint1 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu2: timer0 200c0<vector=c0,delmode=0,dest=0> 0<target=0>
 cpu2: pcint0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu2: lint0 10700<vector=0,delmode=7,masked,dest=0> 0<target=0>
 cpu2: lint1 400<vector=0,delmode=4,dest=0> 0<target=0>
 cpu2: err0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu2: CPU 1 running
 cpu3: prelint0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu3: prelint1 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu3: timer0 200c0<vector=c0,delmode=0,dest=0> 0<target=0>
 cpu3: pcint0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu3: lint0 10700<vector=0,delmode=7,masked,dest=0> 0<target=0>
 cpu3: lint1 400<vector=0,delmode=4,dest=0> 0<target=0>
 cpu3: err0 10000<vector=0,delmode=0,masked,dest=0> 0<target=0>
 cpu3: CPU 2 running
 init: copying out path `/sbin/init' 11
 Sat Feb  8 06:05:34 GMT 2003