GENERIC HEAD from Mar 11 18:34 UTC + palm4.diff, vmcore.210
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2008 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 8.0-CURRENT #0: Thu Mar 13 17:50:20 CET 2008
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1037557760 (989 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
lock order reversal:
1st 0xc4090c94 ufs (ufs) @ kern/vfs_subr.c:2063
2nd 0xc41b5000 vfslock (vfslock) @ kern/vfs_subr.c:364
KDB: stack backtrace:
db_trace_self_wrapper(c0afb191,e291598c,c07aad0e,c0afd68a,c41b5000,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0afd68a,c41b5000,c0b03475,c0b03475,c0b03a1b,...) at kdb_backtrace+0x29
witness_checkorder(c41b5000,1,c0b03a12,16c,e29159b0,...) at witness_checkorder+0x6de
_lockmgr_args(c41b5000,20001,c41b507c,0,ffffffff,...) at _lockmgr_args+0x1e5
vfs_busy(c41b5000,0,0,c3e18d20,e2915b58,...) at vfs_busy+0x1c4
lookup(e2915b44,c0b0312d,c6,bf,c3dee22c,...) at lookup+0x7d2
namei(e2915b44,c41b507c,1c1,c0b0337b,e2915b54,...) at namei+0x34e
kern_unlink(c3e18d20,c0b037b4,1,62f,0,...) at kern_unlink+0x40
vfs_mountroot_try(c0b0396e,c0af1e6d,c0aea88d,1,c07e8750,...) at vfs_mountroot_try+0x499
vfs_mountroot(c0c19990,4,c0af3124,262,0,...) at vfs_mountroot+0x420
start_init(0,e2915d38,c0af4aa9,30c,c3e16ab0,...) at start_init+0x65
fork_exit(c0737b60,0,e2915d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2915d70, ebp = 0 ---
lock order reversal:
1st 0xc3e1c044 user map (user map) @ vm/vm_map.c:3111
2nd 0xc40909dc ufs (ufs) @ kern/vfs_subr.c:2063
KDB: stack backtrace:
db_trace_self_wrapper(c0afb191,e2915978,c07aad0e,c0afd68a,c40909dc,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0afd68a,c40909dc,c0af2704,c0af2704,c0b03a1b,...) at kdb_backtrace+0x29
witness_checkorder(c40909dc,1,c0b03a12,80f,c3e18d20,...) at witness_checkorder+0x6de
_lockmgr_args(c40909dc,30041,c4090a58,0,ffffffff,...) at _lockmgr_args+0x1e5
ffs_lock(e2915a78,c076305d,c0c24c94,30041,c4090984,...) at ffs_lock+0xa3
VOP_LOCK1_APV(c0bd0580,e2915a78,c0af1e6b,3,c4090a58,...) at VOP_LOCK1_APV+0xa5
_vn_lock(c4090984,30041,c0b03a12,80f,0,...) at _vn_lock+0xf7
vget(c4090984,30041,c3e18d20,4a9,c144d100,...) at vget+0x10b
vnode_pager_lock(c1457e80,0,c0b19ee0,127,e2915be8,...) at vnode_pager_lock+0x1b0
vm_fault(c3e1c000,80d3000,2,8,80d3a20,...) at vm_fault+0x1df
trap_pfault(5,0,c0b27fc4,2c8,c3e16ab0,...) at trap_pfault+0x118
trap(e2915d38) at trap+0x267
calltrap() at calltrap+0x6
--- trap 0xc, eip = 0x80480e5, esp = 0xbfbfeef0, ebp = 0xbfbfef10 ---
fxp0: link state changed to UP
lock order reversal:
1st 0xc42ff058 pseudofs (pseudofs) @ kern/vfs_subr.c:2063
2nd 0xc41b58b8 vfslock (vfslock) @ kern/vfs_subr.c:364
KDB: stack backtrace:
db_trace_self_wrapper(c0afb191,e67df9cc,c07aad0e,c0afd68a,c41b58b8,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0afd68a,c41b58b8,c0b03475,c0b03475,c0b03a1b,...) at kdb_backtrace+0x29
witness_checkorder(c41b58b8,1,c0b03a12,16c,c0c6b2e4,...) at witness_checkorder+0x6de
_lockmgr_args(c41b58b8,20001,c41b5934,0,ffffffff,...) at _lockmgr_args+0x1e5
vfs_busy(c41b58b8,10,0,c41898c0,8,...) at vfs_busy+0x1c4
vfs_donmount(810e080,c,e67dfc70,c424b000,810bc90,...) at vfs_donmount+0xdb8
nmount(c41898c0,e67dfcfc,c,c0afe2e4,c0bb1550,...) at nmount+0xb2
syscall(e67dfd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (378, FreeBSD ELF32, nmount), eip = 0x280e015b, esp = 0xbfbfe96c, ebp = 0xbfbfedb8 ---
lock order reversal:
1st 0xc461bdf0 ufs (ufs) @ kern/vfs_lookup.c:412
2nd 0xd7ed9a50 bufwait (bufwait) @ sys/buf.h:279
3rd 0xc58755c8 ufs (ufs) @ kern/vfs_subr.c:2063
KDB: stack backtrace:
db_trace_self_wrapper(c0afb191,e6a5e47c,c07aad0e,c0afd6a3,c58755c8,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0afd6a3,c58755c8,c0af2704,c0af2704,c0b03a1b,...) at kdb_backtrace+0x29
witness_checkorder(c58755c8,9,c0b03a12,80f,c0af9ded,...) at witness_checkorder+0x6de
_lockmgr_args(c58755c8,20002,c5875644,0,ffffffff,...) at _lockmgr_args+0x54a
ffs_lock(e6a5e57c,c076305d,c0c24c94,20002,c5875570,...) at ffs_lock+0xa3
VOP_LOCK1_APV(c0bd0580,e6a5e57c,c0af1e6b,3,c5875644,...) at VOP_LOCK1_APV+0xa5
_vn_lock(c5875570,20002,c0b03a12,80f,0,...) at _vn_lock+0xf7
vget(c5875570,20002,c5dd1460,50,0,...) at vget+0x10b
vfs_hash_get(c42f52e8,33d32,2,c5dd1460,e6a5e6e4,...) at vfs_hash_get+0xed
ffs_vget(c42f52e8,33d32,2,e6a5e6e4,0,...) at ffs_vget+0x49
softdep_sync_metadata(c461bd98,0,c0b183d2,12f,ffffffff,...) at softdep_sync_metadata+0x5b2
ffs_syncvnode(c461bd98,1,f4,c0c6b284,e6a5e7b4,...) at ffs_syncvnode+0x3e2
ffs_truncate(c461bd98,1a00,0,880,c482e300,...) at ffs_truncate+0x659
ufs_direnter(c461bd98,c515c570,e6a5ea20,e6a5ebcc,0,...) at ufs_direnter+0x973
ufs_makeinode(e6a5ebcc,c07f0b86,c0bd0580,e6a5eb4c,c0bd0aa0,...) at ufs_makeinode+0x571
ufs_mknod(e6a5eb90,c0b2aa9e,0,e6a5eb90,e6a5ebf0,...) at ufs_mknod+0x35
VOP_MKNOD_APV(c0bd0580,e6a5eb90,544,542,0,...) at VOP_MKNOD_APV+0xc5
kern_mkfifo(c5dd1460,804ba80,0,180,e6a5ed2c,...) at kern_mkfifo+0x2b3
mkfifo(c5dd1460,e6a5ecfc,8,c0afdd49,c0bafe40,...) at mkfifo+0x29
syscall(e6a5ed38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (132, FreeBSD ELF32, mkfifo), eip = 0x280dc0fb, esp = 0xbfbfe7bc, ebp = 0xbfbfe7e8 ---
lock order reversal:
1st 0xc482e92c filedesc structure (filedesc structure) @ kern/kern_descrip.c:1031
2nd 0xc435f9dc ufs (ufs) @ kern/vfs_subr.c:3874
KDB: stack backtrace:
db_trace_self_wrapper(c0afb191,e687ca9c,c07aad0e,c0afd68a,c435f9dc,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0afd68a,c435f9dc,c0af2704,c0af2704,c0b03a1b,...) at kdb_backtrace+0x29
witness_checkorder(c435f9dc,9,c0b03a12,f22,c0b03a12,...) at witness_checkorder+0x6de
_lockmgr_args(c435f9dc,30002,c435fa58,0,ffffffff,...) at _lockmgr_args+0x54a
ffs_lock(e687cb9c,c435f9dc,10002,10002,c435f984,...) at ffs_lock+0xa3
VOP_LOCK1_APV(c0bd0580,e687cb9c,c0b04918,363,c435fa58,...) at VOP_LOCK1_APV+0xa5
_vn_lock(c435f984,10002,c0b03a12,f22,e687cbf8,...) at _vn_lock+0xf7
vfs_knllock(c435f984,0,c0af455a,64b,c4548c7c,...) at vfs_knllock+0x29
knlist_remove_kq(0,e687cc18,c07ed499,c491a020,c4548c7c,...) at knlist_remove_kq+0xad
knlist_remove(c491a020,c4548c7c,0,e687cc44,c074a645,...) at knlist_remove+0x1b
filt_vfsdetach(c4548c7c,0,c0af455a,71b,3,...) at filt_vfsdetach+0x39
knote_fdclose(c3f98af0,3,c0af4084,407,c41ec478,...) at knote_fdclose+0xf5
kern_close(c3f98af0,3,e687cd2c,c0a64443,c3f98af0,...) at kern_close+0xd5
close(c3f98af0,e687ccfc,4,e687cd38,c0baf270,...) at close+0x1a
syscall(e687cd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (6, FreeBSD ELF32, close), eip = 0x28164b33, esp = 0xbfbfeaec, ebp = 0xbfbfeaf8 ---
Kernel page fault with the following non-sleepable locks held:
exclusive sleep mutex cdev r = 0 (0xc0c1868c) locked @ kern/kern_conf.c:73
KDB: stack backtrace:
db_trace_self_wrapper(c0afb191,e74b976c,c07a9e87,c0afb550,e74b9780,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0afb550,e74b9780,4,1,0,...) at kdb_backtrace+0x29
witness_warn(5,0,c0b27fc4,c6116804,c6116804,...) at witness_warn+0x1d7
trap(e74b980c) at trap+0x122
calltrap() at calltrap+0x6
--- trap 0xc, eip = 0xc070040b, esp = 0xe74b984c, ebp = 0xe74b9880 ---
devfs_populate_loop(c41b9c94,4,c0aeea64,1e0,0,...) at devfs_populate_loop+0x8b
devfs_populate(c41b9c80,e74b9950,c0aeed67,2b3,c0b03a1b,...) at devfs_populate+0x44
devfs_lookup(e74b99d4,c0b03c44,c0be83a0,2,c41bb2b8,...) at devfs_lookup+0x236
VOP_LOOKUP_APV(c0babd60,e74b99d4,c0b0312d,2ae,e74b9b94,...) at VOP_LOOKUP_APV+0xe5
lookup(e74b9b80,c0b0312d,c6,bf,c7819e2c,...) at lookup+0x57e
namei(e74b9b80,c0c24c94,c4cc4690,e74b9a7c,246,...) at namei+0x34e
vn_open_cred(e74b9b80,e74b9c78,1a4,c3dee100,c4950dd0,...) at vn_open_cred+0x62
vn_open(e74b9b80,e74b9c78,1a4,c4950dd0,1,...) at vn_open+0x33
kern_open(c4cc4690,8107330,0,602,1b6,...) at kern_open+0xf2
open(c4cc4690,e74b9cfc,c,e74b9d38,c0baf258,...) at open+0x30
syscall(e74b9d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (5, FreeBSD ELF32, open), eip = 0x281bfa8b, esp = 0xbfbfeb4c, ebp = 0xbfbfebe8 ---
Fatal trap 12: page fault while in kernel mode
cpuid = 2; apic id = 06
fault virtual address = 0xdeadc0de
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc070040b
stack pointer = 0x28:0xe74b984c
frame pointer = 0x28:0xe74b9880
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 68680 (sh)
[thread pid 68680 tid 101057 ]
Stopped at devfs_populate_loop+0x8b: movl 0(%eax),%ebx
db> set $lines 0
db> bt
Tracing pid 68680 tid 101057 td 0xc4cc4690
devfs_populate_loop(c41b9c94,4,c0aeea64,1e0,0,...) at devfs_populate_loop+0x8b
devfs_populate(c41b9c80,e74b9950,c0aeed67,2b3,c0b03a1b,...) at devfs_populate+0x44
devfs_lookup(e74b99d4,c0b03c44,c0be83a0,2,c41bb2b8,...) at devfs_lookup+0x236
VOP_LOOKUP_APV(c0babd60,e74b99d4,c0b0312d,2ae,e74b9b94,...) at VOP_LOOKUP_APV+0xe5
lookup(e74b9b80,c0b0312d,c6,bf,c7819e2c,...) at lookup+0x57e
namei(e74b9b80,c0c24c94,c4cc4690,e74b9a7c,246,...) at namei+0x34e
vn_open_cred(e74b9b80,e74b9c78,1a4,c3dee100,c4950dd0,...) at vn_open_cred+0x62
vn_open(e74b9b80,e74b9c78,1a4,c4950dd0,1,...) at vn_open+0x33
kern_open(c4cc4690,8107330,0,602,1b6,...) at kern_open+0xf2
open(c4cc4690,e74b9cfc,c,e74b9d38,c0baf258,...) at open+0x30
syscall(e74b9d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (5, FreeBSD ELF32, open), eip = 0x281bfa8b, esp = 0xbfbfeb4c, ebp = 0xbfbfebe8 ---
db> show allpcpu
Current CPU: 2
cpuid = 0
curthread = 0xc3e18000: pid 12 "swi4: clock sio"
curpcb = 0xe432bd90
fpcurthread = none
idlethread = 0xc3e18460: pid 11 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc3f98690: pid 586 "syslogd"
curpcb = 0xe6882d90
fpcurthread = none
idlethread = 0xc3e18690: pid 11 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc4cc4690: pid 68680 "sh"
curpcb = 0xe74b9d90
fpcurthread = none
idlethread = 0xc3e188c0: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc4235000: pid 1072 "top"
curpcb = 0xe6930d90
fpcurthread = none
idlethread = 0xc3e18af0: pid 11 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> show alllocks
Process 68680 (sh) thread 0xc4cc4690 (101057)
exclusive sleep mutex cdev r = 0 (0xc0c1868c) locked @ kern/kern_conf.c:73
exclusive sx devfsmount r = 0 (0xc41b9c94) locked @ fs/devfs/devfs_devs.c:267
exclusive lockmgr devfs r = 0 (0xc41bb310) locked @ kern/vfs_subr.c:2063
Process 66692 (sshd) thread 0xc618f8c0 (100294)
exclusive sx so_rcv_sx r = 0 (0xc627420c) locked @ kern/uipc_sockbuf.c:148
Process 1062 (sshd) thread 0xc460cd20 (100115)
exclusive sx so_rcv_sx r = 0 (0xc436bb24) locked @ kern/uipc_sockbuf.c:148
Process 1061 (sshd) thread 0xc460d460 (100112)
exclusive sx so_rcv_sx r = 0 (0xc436b088) locked @ kern/uipc_sockbuf.c:148
Process 1060 (sshd) thread 0xc4189690 (100044)
exclusive sx so_rcv_sx r = 0 (0xc431a390) locked @ kern/uipc_sockbuf.c:148
Process 586 (syslogd) thread 0xc3f98690 (100068)
shared sx filedesc structure r = 0 (0xc430e22c) locked @ kern/sys_generic.c:910
db> show lockedvnods
Locked vnodes
0xc41bb2b8: tag devfs, type VDIR
usecount 3, writecount 0, refcount 3 mountedhere 0
flags (VV_ROOT)
lock type devfs: EXCL (count 1) by thread 0xc4cc4690 (pid 68680)#0 0xc075ec6b at _lockmgr_args+0x6cb
#1 0xc07e2c81 at vop_stdlock+0x51
#2 0xc0a71e05 at VOP_LOCK1_APV+0xa5
#3 0xc07fcb97 at _vn_lock+0xf7
#4 0xc07f1ceb at vget+0x10b
#5 0xc0702a9f at devfs_allocv+0x11f
#6 0xc0701591 at devfs_root+0x51
#7 0xc07e6840 at lookup+0x8d0
#8 0xc07e703e at namei+0x34e
#9 0xc07fc272 at vn_open_cred+0x62
#10 0xc07fc773 at vn_open+0x33
#11 0xc07fb482 at kern_open+0xf2
#12 0xc07fb980 at open+0x30
#13 0xc0a64443 at syscall+0x2b3
#14 0xc0a4a6c0 at Xint0x80_syscall+0x20
db> ps
pid ppid pgrp uid state wmesg wchan cmd
68680 68679 809 0 R CPU 2 sh
68679 809 809 0 S wait 0xc5dbf804 sh
68678 1077 1070 1001 S nanslp 0xc0c1a104 sleep
67452 67451 67452 0 S+ ttyin 0xc41abc10 csh
67451 66696 67451 1001 S+ wait 0xc491f558 su
66696 66694 66696 1001 Ss+ wait 0xc5d882ac bash
66694 66692 66692 1001 S select 0xc7b9a928 sshd
66692 750 66692 0 Ss sbwait 0xc627423c sshd
1079 1070 1070 1001 S piperd 0xc484f794 awk
1077 1070 1070 1001 S wait 0xc41e6ab0 sh
1072 1067 1072 1001 Rs+ CPU 3 top
1070 1068 1070 1001 Ss wait 0xc45f7804 sh
1069 1066 1069 1001 Ss kqread 0xc411eb00 tail
1068 1060 1060 1001 S select 0xc41ff9a8 sshd
1067 1062 1062 1001 S select 0xc41ffba8 sshd
1066 1061 1061 1001 S select 0xc41b87e8 sshd
1062 750 1062 0 Ss sbwait 0xc436bb54 sshd
1061 750 1061 0 Ss sbwait 0xc436b0b8 sshd
1060 750 1060 0 Ss sbwait 0xc431a3c0 sshd
871 1 871 0 Ss+ ttyin 0xc3fb7810 getty
870 1 870 0 Ss+ ttyin 0xc3fa9010 getty
869 1 869 0 Ss+ ttyin 0xc3fb2410 getty
868 1 868 0 Ss+ ttyin 0xc3fa9810 getty
867 1 867 0 Ss+ ttyin 0xc3fa9c10 getty
866 1 866 0 Ss+ ttyin 0xc3fb3810 getty
865 1 865 0 Ss+ ttyin 0xc3f9f410 getty
864 1 864 0 Ss+ ttyin 0xc3fb3c10 getty
849 1 849 0 Ss select 0xc41ffde8 inetd
822 1 822 0 Ss select 0xc42013a8 moused
809 1 809 0 Ss wait 0xc4346804 watchdogd
766 1 766 0 Ss nanslp 0xc0c1a104 cron
759 1 759 25 Ss pause 0xc4233060 sendmail
755 1 755 0 Ss select 0xc45b6a28 sendmail
750 1 750 0 Ss select 0xc3fc69a8 sshd
731 1 731 0 Ss select 0xc416ece8 ntpd
674 658 658 0 S - 0xc4178400 nfsd
673 658 658 0 S - 0xc420f000 nfsd
672 658 658 0 S - 0xc3f37600 nfsd
671 658 658 0 S - 0xc41e2e00 nfsd
670 658 658 0 S - 0xc4178200 nfsd
669 658 658 0 S - 0xc420e400 nfsd
668 658 658 0 S - 0xc420f400 nfsd
667 658 658 0 S - 0xc420e800 nfsd
666 658 658 0 S - 0xc3f37a00 nfsd
665 658 658 0 S - 0xc420ea00 nfsd
663 658 658 0 S - 0xc420ec00 nfsd
662 658 658 0 S - 0xc41e2400 nfsd
661 658 658 0 S - 0xc41ae000 nfsd
660 658 658 0 S - 0xc41e2600 nfsd
659 658 658 0 S - 0xc3f37c00 nfsd
658 1 658 0 Ss select 0xc41b9868 nfsd
656 1 656 0 Ss select 0xc4200368 mountd
603 1 603 0 Ss select 0xc41ffe68 rpcbind
586 1 586 0 Rs CPU 1 syslogd
527 1 527 0 Ss select 0xc3fc68e8 devd
24 0 0 0 SL sdflush 0xc0c77210 [softdepflush]
23 0 0 0 SL syncer 0xc0c19f2c [syncer]
22 0 0 0 SL vlruwt 0xc4188ab0 [vnlru]
21 0 0 0 SL psleep 0xc0c6b8a4 [bufdaemon]
20 0 0 0 SL pgzero 0xc0c77e00 [pagezero]
19 0 0 0 SL psleep 0xc0c77a1c [vmdaemon]
18 0 0 0 SL psleep 0xc0c779e4 [pagedaemon]
17 0 0 0 SL waiting_ 0xc0c6d64c [sctp_iterator]
9 0 0 0 SL - 0xc3f3723c [fdc0]
16 0 0 0 SL usbtsk 0xc0c177b4 [usbtask-dr]
15 0 0 0 SL usbtsk 0xc0c177a0 [usbtask-hc]
14 0 0 0 SL usbevt 0xc3f1f210 [usb0]
8 0 0 0 SL - 0xc3f10500 [em0 taskq]
7 0 0 0 SL - 0xc3e5ed80 [kqueue taskq]
6 0 0 0 SL - 0xc3e78100 [thread taskq]
5 0 0 0 SL ccb_scan 0xc0be8e94 [xpt_thrd]
13 0 0 0 SL - 0xc0c19f34 [yarrow]
4 0 0 0 SL - 0xc0c17f2c [g_down]
3 0 0 0 SL - 0xc0c17f28 [g_up]
2 0 0 0 SL - 0xc0c17f20 [g_event]
12 0 0 0 RL (threaded) intr
100033 I [swi0: sio]
100032 I [irq7: ppbus0 ppc0]
100030 I [irq12: psm0]
100029 I [irq1: atkbd0]
100028 I [irq15: ata1]
100027 I [irq14: ata0]
100026 I [irq17: fxp0]
100022 I [irq16: uhci0]
100019 I [swi6: task queue]
100018 I [swi6: Giant taskq]
100016 I [swi5: +]
100015 I [swi2: cambio]
100009 I [swi3: vm]
100008 Run CPU 0 [swi4: clock sio]
100007 I [swi1: net]
11 0 0 0 RL (threaded) idle
100006 CanRun [idle: cpu0]
100005 CanRun [idle: cpu1]
100004 CanRun [idle: cpu2]
100003 CanRun [idle: cpu3]
1 0 1 0 SLs wait 0xc3e16ab0 [init]
10 0 0 0 SL audit_wo 0xc0c76c68 [audit]
0 0 0 0 WLs [swapper]
db> allt
Tracing command sh pid 68680 tid 101057 td 0xc4cc4690
devfs_populate_loop(c41b9c94,4,c0aeea64,1e0,0,...) at devfs_populate_loop+0x8b
devfs_populate(c41b9c80,e74b9950,c0aeed67,2b3,c0b03a1b,...) at devfs_populate+0x44
devfs_lookup(e74b99d4,c0b03c44,c0be83a0,2,c41bb2b8,...) at devfs_lookup+0x236
VOP_LOOKUP_APV(c0babd60,e74b99d4,c0b0312d,2ae,e74b9b94,...) at VOP_LOOKUP_APV+0xe5
lookup(e74b9b80,c0b0312d,c6,bf,c7819e2c,...) at lookup+0x57e
namei(e74b9b80,c0c24c94,c4cc4690,e74b9a7c,246,...) at namei+0x34e
vn_open_cred(e74b9b80,e74b9c78,1a4,c3dee100,c4950dd0,...) at vn_open_cred+0x62
vn_open(e74b9b80,e74b9c78,1a4,c4950dd0,1,...) at vn_open+0x33
kern_open(c4cc4690,8107330,0,602,1b6,...) at kern_open+0xf2
open(c4cc4690,e74b9cfc,c,e74b9d38,c0baf258,...) at open+0x30
syscall(e74b9d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (5, FreeBSD ELF32, open), eip = 0x281bfa8b, esp = 0xbfbfeb4c, ebp = 0xbfbfebe8 ---
Tracing command sh pid 68679 tid 100236 td 0xc5e20af0
sched_switch(c5e20af0,0,1,189,76f4f846,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c5e20af0,0,c0afbd7a,19b,c5dbf804,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1bb80,4,c0af6fd0,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c5dbf804,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c5dbf804,c5dbf894,15c,c0afdeeb,0,...) at _sleep+0x359
kern_wait(c5e20af0,ffffffff,e6adfc2c,2,0,...) at kern_wait+0xa66
wait4(c5e20af0,e6adfcfc,10,e6adfd38,c0baf288,...) at wait4+0x3b
syscall(e6adfd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815013b, esp = 0xbfbfeb9c, ebp = 0xbfbfebb8 ---
Tracing command sleep pid 68678 tid 100189 td 0xc5dd3000
sched_switch(c5dd3000,0,1,189,56eec80a,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c5dd3000,0,c0afbd7a,19b,c0c1a104,...) at sleepq_switch+0x170
sleepq_catch_signals(e6a47bc4,c076305d,c0c1c880,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c1a104,0,c0af8d66,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c1a104,0,15c,c0af9576,2711,...) at _sleep+0x331
kern_nanosleep(c5dd3000,e6a47c64,e6a47c6c,a,0,...) at kern_nanosleep+0xc1
nanosleep(c5dd3000,e6a47cfc,8,e6a47d38,c0bb0860,...) at nanosleep+0x6f
syscall(e6a47d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2814361f, esp = 0xbfbfec8c, ebp = 0xbfbfecc8 ---
Tracing command csh pid 67452 tid 100267 td 0xc5f7a230
sched_switch(c5f7a230,0,1,189,c9de268c,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c5f7a230,0,c0afbd7a,19b,c41abc10,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1c880,4,c0af6fd0,59,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c41abc10,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c41abc10,0,159,c0aff79d,0,...) at _sleep+0x359
ttysleep(c41abc00,c41abc10,159,c0aff79d,0,...) at ttysleep+0x39
ttread(c41abc00,e6b6bc60,0,c430ee00,6,...) at ttread+0x4e7
ptsread(c430ee00,e6b6bc60,0,1ac,0,...) at ptsread+0x3e
giant_read(c430ee00,e6b6bc60,0,0,1,...) at giant_read+0x5a
devfs_read_f(c434d374,e6b6bc60,c82b2300,0,c5f7a230,...) at devfs_read_f+0x72
dofileread(e6b6bc60,ffffffff,ffffffff,0,c434d374,...) at dofileread+0x96
kern_readv(c5f7a230,10,e6b6bc60,bfbfe796,1,...) at kern_readv+0x58
read(c5f7a230,e6b6bcfc,c,c0ae7be6,c0baf228,...) at read+0x4f
syscall(e6b6bd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (3, FreeBSD ELF32, read), eip = 0x28202b73, esp = 0xbfbfe74c, ebp = 0xbfbfe768 ---
Tracing command su pid 67451 tid 100134 td 0xc493b690
sched_switch(c493b690,0,1,189,db7b8b5a,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c493b690,0,c0afbd7a,19b,c491f558,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1bb80,4,c0af6fd0,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c491f558,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c491f558,c491f5e8,15c,c0afdeeb,0,...) at _sleep+0x359
kern_wait(c493b690,1077c,e6981c2c,2,0,...) at kern_wait+0xa66
wait4(c493b690,e6981cfc,10,e6981d38,c0baf288,...) at wait4+0x3b
syscall(e6981d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2810d13b, esp = 0xbfbfe6ec, ebp = 0xbfbfe708 ---
Tracing command bash pid 66696 tid 100191 td 0xc5dd1d20
sched_switch(c5dd1d20,0,1,189,a6436014,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c5dd1d20,0,c0afbd7a,19b,c5d882ac,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1bb80,4,c0af6fd0,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c5d882ac,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c5d882ac,c5d8833c,15c,c0afdeeb,0,...) at _sleep+0x359
kern_wait(c5dd1d20,ffffffff,e6a4ec2c,6,0,...) at kern_wait+0xa66
wait4(c5dd1d20,e6a4ecfc,10,c0afddc5,c0baf288,...) at wait4+0x3b
syscall(e6a4ed38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x282a413b, esp = 0xbfbfe8ec, ebp = 0xbfbfe908 ---
Tracing command sshd pid 66694 tid 100214 td 0xc5e08460
sched_switch(c5e08460,0,1,189,c9e55d5e,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c5e08460,0,c0afbd7a,19b,c7b9a928,...) at sleepq_switch+0x170
sleepq_catch_signals(c076340a,c7b9a910,0,c0af6fd0,c5e08460,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c7b9a928,c7b9a910,c0afd972,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c7b9a928,c7b9a910,c0afd935,57e,c6f394ac,...) at _cv_wait_sig+0x1fc
seltdwait(c5c0c02c,c0afd935,3d7,c5e08460,e6a97b04,...) at seltdwait+0xa2
kern_select(c5e08460,9,81050e4,81050e8,0,0,0,280e95e8) at kern_select+0x531
select(c5e08460,e6a97cfc,14,c0ae7be6,c0bafa98,...) at select+0x5e
syscall(e6a97d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x283f3af3, esp = 0xbfbfe5fc, ebp = 0xbfbfe648 ---
Tracing command sshd pid 66692 tid 100294 td 0xc618f8c0
sched_switch(c618f8c0,0,1,189,b54910ea,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c618f8c0,0,c0afbd7a,19b,c627423c,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1bb80,4,c0af6fd0,58,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c627423c,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c627423c,c62741f4,158,c0b00ca1,0) at _sleep+0x359
sbwait(c62741d4,4,c0b00db6,5ae,c62741f4,...) at sbwait+0x76
soreceive_generic(c6274184,0,e6bb3c60,0,0,...) at soreceive_generic+0x41f
soreceive(c6274184,0,e6bb3c60,0,0,...) at soreceive+0x4d
soo_read(c4d45a90,e6bb3c60,c8413400,0,c618f8c0,...) at soo_read+0x3a
dofileread(e6bb3c60,ffffffff,ffffffff,0,c4d45a90,...) at dofileread+0x96
kern_readv(c618f8c0,5,e6bb3c60,bfbfe64c,4,...) at kern_readv+0x58
read(c618f8c0,e6bb3cfc,c,c0b13426,c0baf228,...) at read+0x4f
syscall(e6bb3d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (3, FreeBSD ELF32, read), eip = 0x283f3b73, esp = 0xbfbfe60c, ebp = 0xbfbfe628 ---
Tracing command awk pid 1079 tid 100120 td 0xc45b8690
sched_switch(c45b8690,0,1,189,56586ade,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c45b8690,0,c0afbd7a,19b,c484f794,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1c880,4,c0af6fd0,4c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c484f794,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c484f794,c484f8fc,14c,c0afdaed,0,...) at _sleep+0x359
pipe_read(c425be04,e6957c60,c4327100,0,c45b8690,...) at pipe_read+0x3f7
dofileread(e6957c60,ffffffff,ffffffff,0,c425be04,...) at dofileread+0x96
kern_readv(c45b8690,0,e6957c60,8137000,1000,...) at kern_readv+0x58
read(c45b8690,e6957cfc,c,16,c0baf228,...) at read+0x4f
syscall(e6957d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (3, FreeBSD ELF32, read), eip = 0x28195b73, esp = 0xbfbfe94c, ebp = 0xbfbfe968 ---
Tracing command sh pid 1077 tid 100042 td 0xc41e7d20
sched_switch(c41e7d20,0,1,189,56772a46,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c41e7d20,0,c0afbd7a,19b,c41e6ab0,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1bb80,4,c0af6fd0,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c41e6ab0,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c41e6ab0,c41e6b40,15c,c0afdeeb,0,...) at _sleep+0x359
kern_wait(c41e7d20,ffffffff,e67dbc2c,2,0,...) at kern_wait+0xa66
wait4(c41e7d20,e67dbcfc,10,c0afdeca,c0baf288,...) at wait4+0x3b
syscall(e67dbd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815013b, esp = 0xbfbfe91c, ebp = 0xbfbfe938 ---
Tracing command top pid 1072 tid 100107 td 0xc4235000
cpustop_handler(8,e6930adc,c0a6474d,6de,c0c6a8d0,...) at cpustop_handler+0x32
ipi_nmi_handler(6de,c0c6a8d0,e6930a90,c45f9558,c45f9558,...) at ipi_nmi_handler+0x2f
trap(e6930ae8) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0763642, esp = 0xe6930b28, ebp = 0xe6930b44 ---
_mtx_lock_sleep(c0c1868c,c4235000,0,c0af3686,49,...) at _mtx_lock_sleep+0xa2
_mtx_lock_flags(c0c1868c,0,c0af3686,49,e6930ba0,...) at _mtx_lock_flags+0xef
dev_lock(c0c1868c,0,c0af3686,cf,e6930bd8,...) at dev_lock+0x2a
devvn_refthread(c49102b8,e6930bd8,0,c4203af8,e6930c60,...) at devvn_refthread+0x30
devfs_fp_check(c0af408d,c4332b2c,7e9,c0af4084,e6930bf8,...) at devfs_fp_check+0x24
devfs_write_f(c4203af8,e6930c60,c481dd00,0,c4235000,...) at devfs_write_f+0x20
dofilewrite(e6930c60,ffffffff,ffffffff,0,c4203af8,...) at dofilewrite+0x95
kern_writev(c4235000,1,e6930c60,8055fc0,28e,...) at kern_writev+0x58
write(c4235000,e6930cfc,c,c0afe02f,c0baf240,...) at write+0x4f
syscall(e6930d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (4, FreeBSD ELF32, write), eip = 0x281c9b53, esp = 0xbfbfeb3c, ebp = 0xbfbfeb58 ---
Tracing command sh pid 1070 tid 100116 td 0xc460caf0
Tracing command tail pid 1069 tid 100066 td 0xc3f98af0
sched_switch(c3f98af0,0,1,189,f8cfba82,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c3f98af0,0,c0afbd7a,19b,c411eb00,...) at sleepq_switch+0x170
sleepq_catch_signals(e687cab0,c076305d,c0c1bb80,4,58,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c411eb00,0,c0af8d66,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c411eb00,c411eb00,158,c0af479e,3e9,...) at _sleep+0x331
kern_kevent(c3f98af0,4,0,1,e687cc60,...) at kern_kevent+0x3bd
kevent(c3f98af0,e687ccfc,18,c0aaefa9,c0bb13e8,...) at kevent+0x19b
syscall(e687cd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (363, FreeBSD ELF32, kevent), eip = 0x2814783b, esp = 0xbfbfeb4c, ebp = 0xbfbfebf8 ---
Tracing command sshd pid 1068 tid 100072 td 0xc4347d20
Tracing command sshd pid 1067 tid 100093 td 0xc42998c0
sched_switch(c42998c0,0,1,189,1f0dfb3c,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c42998c0,0,c0afbd7a,19b,c41ffba8,...) at sleepq_switch+0x170
sleepq_catch_signals(c076340a,c41ffb90,0,c0af6fd0,c42998c0,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c41ffba8,c41ffb90,c0afd972,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c41ffba8,c41ffb90,c0afd935,57e,c41ec2d8,...) at _cv_wait_sig+0x1fc
seltdwait(c4333c2c,c0afd935,3d7,c42998c0,e6901b04,...) at seltdwait+0xa2
kern_select(c42998c0,9,81050e4,81050e8,0,0,0,280e95e8) at kern_select+0x531
select(c42998c0,e6901cfc,14,c0ae7be6,c0bafa98,...) at select+0x5e
syscall(e6901d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x283f3af3, esp = 0xbfbfe5fc, ebp = 0xbfbfe648 ---
Tracing command sshd pid 1066 tid 100043 td 0xc41898c0
sched_switch(c41898c0,0,1,189,fd6858f2,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c41898c0,0,c0afbd7a,19b,c41b87e8,...) at sleepq_switch+0x170
sleepq_catch_signals(c076340a,c41b87d0,0,c0af6fd0,c41898c0,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c41b87e8,c41b87d0,c0afd972,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c41b87e8,c41b87d0,c0afd935,57e,c425b64c,...) at _cv_wait_sig+0x1fc
seltdwait(c41feb2c,c0afd935,3d7,c41898c0,e67dfb04,...) at seltdwait+0xa2
kern_select(c41898c0,b,81050e4,81050e8,0,0,0,c0bb78c0) at kern_select+0x531
select(c41898c0,e67dfcfc,14,0,c0bafa98,...) at select+0x5e
syscall(e67dfd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x283f3af3, esp = 0xbfbfe5fc, ebp = 0xbfbfe648 ---
Tracing command sshd pid 1062 tid 100115 td 0xc460cd20
Tracing command sshd pid 1061 tid 100112 td 0xc460d460
Tracing command sshd pid 1060 tid 100044 td 0xc4189690
Tracing command getty pid 871 tid 100050 td 0xc4238af0
Tracing command getty pid 870 tid 100053 td 0xc4238460
Tracing command getty pid 869 tid 100099 td 0xc45b9460
Tracing command getty pid 868 tid 100097 td 0xc45b98c0
Tracing command getty pid 867 tid 100096 td 0xc4299230
Tracing command getty pid 866 tid 100095 td 0xc4299460
Tracing command getty pid 865 tid 100094 td 0xc4299690
Tracing command getty pid 864 tid 100046 td 0xc4189230
Tracing command inetd pid 849 tid 100108 td 0xc460dd20
Tracing command moused pid 822 tid 100118 td 0xc460c690
Tracing command watchdogd pid 809 tid 100102 td 0xc45b8d20
sched_switch(c45b8d20,0,1,189,7610fea8,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c45b8d20,0,c0afbd7a,19b,c4346804,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1c880,4,c0af6fd0,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c4346804,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c4346804,c4346894,15c,c0afdeeb,0,...) at _sleep+0x359
kern_wait(c45b8d20,10c47,e6921c2c,0,0,...) at kern_wait+0xa66
wait4(c45b8d20,e6921cfc,10,e6921d38,c0baf288,...) at wait4+0x3b
syscall(e6921d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2810813b, esp = 0xbfbfeccc, ebp = 0xbfbfed78 ---
Tracing command cron pid 766 tid 100101 td 0xc45b9000
sched_switch(c45b9000,0,1,189,86652918,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c45b9000,0,c0afbd7a,19b,c0c1a104,...) at sleepq_switch+0x170
sleepq_catch_signals(e691ebc4,c076305d,c0c1c880,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c1a104,0,c0af8d66,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c1a104,0,15c,c0af9576,ea61,...) at _sleep+0x331
kern_nanosleep(c45b9000,e691ec64,e691ec6c,3c,0,...) at kern_nanosleep+0xc1
nanosleep(c45b9000,e691ecfc,8,c0afe02f,c0bb0860,...) at nanosleep+0x6f
syscall(e691ed38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815e61f, esp = 0xbfbfed0c, ebp = 0xbfbfed38 ---
Tracing command sendmail pid 759 tid 100054 td 0xc4238230
Tracing command sendmail pid 755 tid 100078 td 0xc4347000
sched_switch(c4347000,0,1,189,84c8b22,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c4347000,0,c0afbd7a,19b,c45b6a28,...) at sleepq_switch+0x170
sleepq_catch_signals(e68a0a4c,c076340a,c45b6a10,0,c4347000,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c45b6a28,1389,c0afd972,101,0,...) at sleepq_timedwait_sig+0x17
_cv_timedwait_sig(c45b6a28,c45b6a10,1389,57e,c7edb478,...) at _cv_timedwait_sig+0x20b
seltdwait(e68a0b14,e68a0b1c,3d7,c4347000,e68a0b04,...) at seltdwait+0x8a
kern_select(c4347000,7,bfbfd94c,0,0,e68a0c70,5,0) at kern_select+0x531
select(c4347000,e68a0cfc,14,c0afe2e4,c0bafa98,...) at select+0x5e
syscall(e68a0d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x283adaf3, esp = 0xbfbfd0bc, ebp = 0xbfbfdb58 ---
Tracing command sshd pid 750 tid 100047 td 0xc4189000
sched_switch(c4189000,0,1,189,9daed090,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c4189000,0,c0afbd7a,19b,c3fc69a8,...) at sleepq_switch+0x170
sleepq_catch_signals(c076340a,c3fc6990,0,c0af6fd0,c4189000,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c3fc69a8,c3fc6990,c0afd972,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c3fc69a8,c3fc6990,c0afd935,57e,c42032a4,...) at _cv_wait_sig+0x1fc
seltdwait(c41fed2c,c0afd935,3d7,c4189000,e67efae0,...) at seltdwait+0xa2
kern_select(c4189000,9,810d0bc,0,0,0,c,e67efc98) at kern_select+0x531
select(c4189000,e67efcfc,14,e67efd38,c0bafa98,...) at select+0x5e
syscall(e67efd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x283f3af3, esp = 0xbfbfe6cc, ebp = 0xbfbfee98 ---
Tracing command ntpd pid 731 tid 100092 td 0xc4299af0
sched_switch(c4299af0,0,1,189,4b8cca9a,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c4299af0,0,c0afbd7a,19b,c416ece8,...) at sleepq_switch+0x170
sleepq_catch_signals(c076340a,c416ecd0,0,c0af6fd0,c4299af0,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c416ece8,c416ecd0,c0afd972,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c416ece8,c416ecd0,c0afd935,57e,c41ed09c,...) at _cv_wait_sig+0x1fc
seltdwait(c430e12c,c0afd935,3d7,c4299af0,0,...) at seltdwait+0xa2
kern_select(c4299af0,a,bfbfed7c,0,0,0,246,0) at kern_select+0x531
select(c4299af0,e68fecfc,14,16,c0bafa98,...) at select+0x5e
syscall(e68fed38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x28320af3, esp = 0xbfbfed5c, ebp = 0xbfbfee18 ---
Tracing command nfsd pid 674 tid 100058 td 0xc42358c0
Tracing command nfsd pid 673 tid 100064 td 0xc41e7af0
Tracing command nfsd pid 672 tid 100065 td 0xc41e78c0
Tracing command nfsd pid 671 tid 100049 td 0xc4238d20
Tracing command nfsd pid 670 tid 100090 td 0xc418a8c0
Tracing command nfsd pid 669 tid 100089 td 0xc418aaf0
Tracing command nfsd pid 668 tid 100088 td 0xc418ad20
Tracing command nfsd pid 667 tid 100087 td 0xc41e7000
Tracing command nfsd pid 666 tid 100086 td 0xc41e7230
Tracing command nfsd pid 665 tid 100071 td 0xc4348000
Tracing command nfsd pid 663 tid 100067 td 0xc3f988c0
Tracing command nfsd pid 662 tid 100074 td 0xc43478c0
Tracing command nfsd pid 661 tid 100052 td 0xc4238690
Tracing command nfsd pid 660 tid 100075 td 0xc4347690
Tracing command nfsd pid 659 tid 100069 td 0xc4348460
Tracing command nfsd pid 658 tid 100085 td 0xc41e7460
Tracing command mountd pid 656 tid 100084 td 0xc41e7690
Tracing command rpcbind pid 603 tid 100077 td 0xc4347230
sched_switch(c4347230,0,1,189,4696f5ea,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c4347230,0,c0afbd7a,19b,c41ffe68,...) at sleepq_switch+0x170
sleepq_catch_signals(e689daa8,c076340a,c41ffe50,0,c4347230,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c41ffe68,7531,c0afd972,101,0,...) at sleepq_timedwait_sig+0x17
_cv_timedwait_sig(c41ffe68,c41ffe50,7531,57e,e689dba4,...) at _cv_timedwait_sig+0x20b
seltdwait(e689db5c,e689db64,48e,c4347230,e689db74,...) at seltdwait+0x8a
poll(c4347230,e689dcfc,c,c0afe02f,c0bb0578,...) at poll+0x3ea
syscall(e689dd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (209, FreeBSD ELF32, poll), eip = 0x28128a3f, esp = 0xbfbfcc7c, ebp = 0xbfbfee48 ---
Tracing command syslogd pid 586 tid 100068 td 0xc3f98690
cpustop_handler(2,e68829b4,c0a6474d,c078d185,c0c1ccbc,...) at cpustop_handler+0x32
ipi_nmi_handler(c078d185,c0c1ccbc,2c,c4345804,c4345804,...) at ipi_nmi_handler+0x2f
trap(e68829c0) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0763650, esp = 0xe6882a00, ebp = 0xe6882a1c ---
_mtx_lock_sleep(c0c1868c,c3f98690,0,c0af3686,49,...) at _mtx_lock_sleep+0xb0
_mtx_lock_flags(c0c1868c,0,c0af3686,49,e6882a78,...) at _mtx_lock_flags+0xef
dev_lock(c0c1868c,0,c0af3686,cf,e6882aac,...) at dev_lock+0x2a
devvn_refthread(c435e828,e6882aac,1,c42f4d9c,0,...) at devvn_refthread+0x30
devfs_fp_check(c1451780,c7aa694c,0,4fa,0,...) at devfs_fp_check+0x24
devfs_poll_f(c42f4d9c,40,c3dee100,c3f98690,8,...) at devfs_poll_f+0x15
kern_select(c3f98690,9,81220bc,0,0,0,df54e,28187b98) at kern_select+0x642
select(c3f98690,e6882cfc,14,c0afdeda,c0bafa98,...) at select+0x5e
syscall(e6882d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x28175af3, esp = 0xbfbfdf3c, ebp = 0xbfbfee88 ---
Tracing command devd pid 527 tid 100051 td 0xc42388c0
Tracing command softdepflush pid 24 tid 100041 td 0xc4189af0
sched_switch(c4189af0,0,1,189,fff147da,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,44,...) at mi_switch+0x223
sleepq_switch(c4189af0,0,c0afbd7a,253,44,...) at sleepq_switch+0x170
sleepq_timedwait(c0c77210,3e8,c0b17777,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c77210,c0c771e8,44,c0b17777,3e8,...) at _sleep+0x345
softdep_flush(0,e45ecd38,c0af4aa9,30c,c4188558,...) at softdep_flush+0x2d2
fork_exit(c0956330,0,e45ecd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45ecd70, ebp = 0 ---
Tracing command syncer pid 23 tid 100040 td 0xc4189d20
sched_switch(c4189d20,0,1,189,5fe8e366,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c4189d20,...) at mi_switch+0x223
sleepq_switch(c4189d20,0,c0afbd7a,230,68,...) at sleepq_switch+0x170
sleepq_wait(c0c19f2c,0,c0b04468,0,0,...) at sleepq_wait+0x60
_sleep(c0c19f2c,c0c6bb84,68,c0b04468,0,...) at _sleep+0x369
sched_sync(0,e45e9d38,c0af4aa9,30c,c4188804,...) at sched_sync+0x723
fork_exit(c07f22f0,0,e45e9d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45e9d70, ebp = 0 ---
Tracing command vnlru pid 22 tid 100039 td 0xc418a000
sched_switch(c418a000,0,1,189,79ae4cb2,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,50,...) at mi_switch+0x223
sleepq_switch(c418a000,0,c0afbd7a,253,50,...) at sleepq_switch+0x170
sleepq_timedwait(c4188ab0,3e8,c0b04512,0,0,...) at sleepq_timedwait+0x68
_sleep(c4188ab0,c0c6bb58,250,c0b04512,3e8,...) at _sleep+0x345
vnlru_proc(0,e45e6d38,c0af4aa9,30c,c4188ab0,...) at vnlru_proc+0x115
fork_exit(c07f3260,0,e45e6d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45e6d70, ebp = 0 ---
Tracing command bufdaemon pid 21 tid 100038 td 0xc3f0dd20
sched_switch(c3f0dd20,0,1,189,79acc14a,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,44,...) at mi_switch+0x223
sleepq_switch(c3f0dd20,0,c0afbd7a,253,44,...) at sleepq_switch+0x170
sleepq_timedwait(c0c6b8a4,3e8,c0b02124,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c6b8a4,c0c6b8a8,44,c0b02124,3e8,...) at _sleep+0x345
buf_daemon(0,e45e3d38,c0af4aa9,30c,c3e60804,...) at buf_daemon+0x21e
fork_exit(c07dd570,0,e45e3d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45e3d70, ebp = 0 ---
Tracing command pagezero pid 20 tid 100037 td 0xc3f97000
sched_switch(c3f97000,0,1,189,d1b0d8ec,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c3f97000,0,c0afbd7a,253,0,...) at sleepq_switch+0x170
sleepq_timedwait(c0c77e00,493e0,c0b1c413,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c77e00,c0c779a8,0,c0b1c413,493e0,...) at _sleep+0x345
vm_pagezero(0,e45e0d38,c0af4aa9,30c,c3e60ab0,...) at vm_pagezero+0xdc
fork_exit(c0990380,0,e45e0d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45e0d70, ebp = 0 ---
Tracing command vmdaemon pid 19 tid 100036 td 0xc3f97230
sched_switch(c3f97230,0,1,189,5346817c,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3f97230,...) at mi_switch+0x223
sleepq_switch(c3f97230,0,c0afbd7a,230,68,...) at sleepq_switch+0x170
sleepq_wait(c0c77a1c,c0c77a20,c0b02124,0,0,...) at sleepq_wait+0x60
_sleep(c0c77a1c,c0c77a20,68,c0b02124,0,...) at _sleep+0x369
vm_daemon(0,e45ddd38,c0af4aa9,30c,c3f33000,...) at vm_daemon+0x59
fork_exit(c098b7f0,0,e45ddd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45ddd70, ebp = 0 ---
Tracing command pagedaemon pid 18 tid 100035 td 0xc3f97460
sched_switch(c3f97460,0,1,189,4893d3ee,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,44,...) at mi_switch+0x223
sleepq_switch(c3f97460,0,c0afbd7a,253,44,...) at sleepq_switch+0x170
sleepq_timedwait(c0c779e4,1388,c0b02124,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c779e4,c0c779a8,44,c0b02124,1388,...) at _sleep+0x345
vm_pageout(0,e45dad38,c0af4aa9,30c,c3f332ac,...) at vm_pageout+0x2bb
fork_exit(c098c3a0,0,e45dad38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45dad70, ebp = 0 ---
Tracing command sctp_iterator pid 17 tid 100034 td 0xc3f97690
sched_switch(c3f97690,0,1,189,3808ad52,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3f97690,...) at mi_switch+0x223
sleepq_switch(c3f97690,0,c0afbd7a,230,0,...) at sleepq_switch+0x170
sleepq_wait(c0c6d64c,c0c6d554,c0b0b26f,0,0,...) at sleepq_wait+0x60
_sleep(c0c6d64c,c0c6d554,0,c0b0b26f,0,...) at _sleep+0x369
sctp_iterator_thread(0,e45d7d38,c0af4aa9,30c,c3f33558,...) at sctp_iterator_thread+0x60
fork_exit(c0859990,0,e45d7d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45d7d70, ebp = 0 ---
Tracing command fdc0 pid 9 tid 100031 td 0xc3f97d20
sched_switch(c3f97d20,0,1,189,5fe9ffd6,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,4c,...) at mi_switch+0x223
sleepq_switch(c3f97d20,0,c0afbd7a,253,4c,...) at sleepq_switch+0x170
sleepq_timedwait(c3f3723c,3e8,c0aef969,0,0,...) at sleepq_timedwait+0x68
_sleep(c3f3723c,c3f372f0,4c,c0aef969,3e8,...) at _sleep+0x345
fdc_thread(c3f37200,e45cbd38,c0af4aa9,30c,c3f33804,...) at fdc_thread+0x2b8
fork_exit(c0a25140,c3f37200,e45cbd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45cbd70, ebp = 0 ---
Tracing command usbtask-dr pid 16 tid 100025 td 0xc3e5faf0
sched_switch(c3e5faf0,0,1,189,deb4cd48,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3e5faf0,...) at mi_switch+0x223
sleepq_switch(c3e5faf0,0,c0afbd7a,230,5c,...) at sleepq_switch+0x170
sleepq_wait(c0c177b4,0,c0af8d66,da,0,...) at sleepq_wait+0x60
_sleep(c0c177b4,0,5c,c0aec2e8,0,...) at _sleep+0x369
usb_task_thread(c0c177b4,e437fd38,c0af4aa9,30c,c3f33ab0,...) at usb_task_thread+0x62
fork_exit(c06e29c0,c0c177b4,e437fd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe437fd70, ebp = 0 ---
Tracing command usbtask-hc pid 15 tid 100024 td 0xc3e5fd20
sched_switch(c3e5fd20,0,1,189,deb49d1c,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3e5fd20,...) at mi_switch+0x223
sleepq_switch(c3e5fd20,0,c0afbd7a,230,5c,...) at sleepq_switch+0x170
sleepq_wait(c0c177a0,0,c0af8d66,da,0,...) at sleepq_wait+0x60
_sleep(c0c177a0,0,5c,c0aec2e8,0,...) at _sleep+0x369
usb_task_thread(c0c177a0,e437cd38,c0af4aa9,30c,c3f34000,...) at usb_task_thread+0x62
fork_exit(c06e29c0,c0c177a0,e437cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe437cd70, ebp = 0 ---
Tracing command usb0 pid 14 tid 100023 td 0xc3f0d000
sched_switch(c3f0d000,0,1,189,fdb20850,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,5c,...) at mi_switch+0x223
sleepq_switch(c3f0d000,0,c0afbd7a,253,5c,...) at sleepq_switch+0x170
sleepq_timedwait(c3f1f210,ea60,c0aec2ab,0,0,...) at sleepq_timedwait+0x68
_sleep(c3f1f210,0,5c,c0aec2ab,ea60,...) at _sleep+0x345
usb_event_thread(c3f0ad40,e4379d38,c0af4aa9,30c,c3e172ac,...) at usb_event_thread+0xca
fork_exit(c06e2240,c3f0ad40,e4379d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4379d70, ebp = 0 ---
Tracing command em0 taskq pid 8 tid 100021 td 0xc3f0d460
sched_switch(c3f0d460,0,1,189,deac0e58,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3f0d460,...) at mi_switch+0x223
sleepq_switch(c3f0d460,0,c0afbd7a,230,c3f0d460,...) at sleepq_switch+0x170
sleepq_wait(c3f10500,0,c0af8e7a,c0aef969,0,...) at sleepq_wait+0x60
msleep_spin(c3f10500,c3f1051c,c0aef969,0,c0af6fd0,...) at msleep_spin+0x20d
taskqueue_thread_loop(c3f18350,e4372d38,c0af4aa9,30c,c3e17558,...) at taskqueue_thread_loop+0x8a
fork_exit(c07a4150,c3f18350,e4372d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4372d70, ebp = 0 ---
Tracing command kqueue taskq pid 7 tid 100020 td 0xc3f0d690
sched_switch(c3f0d690,0,1,189,deeae5a4,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3f0d690,...) at mi_switch+0x223
sleepq_switch(c3f0d690,0,c0afbd7a,230,0,...) at sleepq_switch+0x170
sleepq_wait(c3e5ed80,c3e5ed9c,c0aef969,0,0,...) at sleepq_wait+0x60
_sleep(c3e5ed80,c3e5ed9c,0,c0aef969,0,...) at _sleep+0x369
taskqueue_thread_loop(c0c18760,e434fd38,c0af4aa9,30c,c3e17804,...) at taskqueue_thread_loop+0xb4
fork_exit(c07a4150,c0c18760,e434fd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe434fd70, ebp = 0 ---
Tracing command thread taskq pid 6 tid 100017 td 0xc3e1a230
sched_switch(c3e1a230,0,1,189,deeaaa68,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3e1a230,...) at mi_switch+0x223
sleepq_switch(c3e1a230,0,c0afbd7a,230,0,...) at sleepq_switch+0x170
sleepq_wait(c3e78100,c3e7811c,c0aef969,0,0,...) at sleepq_wait+0x60
_sleep(c3e78100,c3e7811c,0,c0aef969,0,...) at _sleep+0x369
taskqueue_thread_loop(c0c23e08,e4346d38,c0af4aa9,30c,c3e17ab0,...) at taskqueue_thread_loop+0xb4
fork_exit(c07a4150,c0c23e08,e4346d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4346d70, ebp = 0 ---
Tracing command xpt_thrd pid 5 tid 100014 td 0xc3e1a8c0
sched_switch(c3e1a8c0,0,1,189,deb42588,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3e1a8c0,...) at mi_switch+0x223
sleepq_switch(c3e1a8c0,0,c0afbd7a,230,4c,...) at sleepq_switch+0x170
sleepq_wait(c0be8e94,0,c0af8d66,da,0,...) at sleepq_wait+0x60
_sleep(c0be8e94,c0be8eac,4c,c0aac270,0,...) at _sleep+0x369
xpt_scanner_thread(0,e433dd38,c0af4aa9,30c,c3e60000,...) at xpt_scanner_thread+0x41
fork_exit(c046adc0,0,e433dd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe433dd70, ebp = 0 ---
Tracing command yarrow pid 13 tid 100013 td 0xc3e1aaf0
sched_switch(c3e1aaf0,0,1,189,79acde5e,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c3e1aaf0,0,c0afbd7a,253,0,...) at sleepq_switch+0x170
sleepq_timedwait(c0c19f34,64,c0aef969,2,0,...) at sleepq_timedwait+0x68
_sleep(c0c19f34,0,0,c0aef969,64,...) at _sleep+0x345
pause(c0aef969,64,c0ae2651,112,0,...) at pause+0x47
random_kthread(0,e433ad38,c0af4aa9,30c,c3e602ac,...) at random_kthread+0x1d9
fork_exit(c065d3a0,0,e433ad38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe433ad70, ebp = 0 ---
Tracing command g_down pid 4 tid 100012 td 0xc3e1ad20
sched_switch(c3e1ad20,0,1,189,79ae933a,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,4c,...) at mi_switch+0x223
sleepq_switch(c3e1ad20,0,c0afbd7a,253,4c,...) at sleepq_switch+0x170
sleepq_timedwait(c0c17f2c,64,c0aef969,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c17f2c,c0c17e48,24c,c0aef969,64,...) at _sleep+0x345
g_io_schedule_down(c3e1ad20,0,c0af1036,74,0,...) at g_io_schedule_down+0x6b
g_down_procbody(0,e4337d38,c0af4aa9,30c,c3e60558,...) at g_down_procbody+0x8d
fork_exit(c071f070,0,e4337d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4337d70, ebp = 0 ---
Tracing command g_up pid 3 tid 100011 td 0xc3e5f000
sched_switch(c3e5f000,0,1,189,79ad964a,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,4c,...) at mi_switch+0x223
sleepq_switch(c3e5f000,0,c0afbd7a,253,4c,...) at sleepq_switch+0x170
sleepq_timedwait(c0c17f28,64,c0aef969,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c17f28,c0c17e88,24c,c0aef969,64,...) at _sleep+0x345
g_io_schedule_up(c3e5f000,0,c0af1036,5d,0,...) at g_io_schedule_up+0x133
g_up_procbody(0,e4334d38,c0af4aa9,30c,c3e16000,...) at g_up_procbody+0x8d
fork_exit(c071f100,0,e4334d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4334d70, ebp = 0 ---
Tracing command g_event pid 2 tid 100010 td 0xc3e5f230
sched_switch(c3e5f230,0,1,189,79ad4d82,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,4c,...) at mi_switch+0x223
sleepq_switch(c3e5f230,0,c0afbd7a,253,4c,...) at sleepq_switch+0x170
sleepq_timedwait(c0c17f20,64,c0aef969,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c17f20,0,4c,c0aef969,64,...) at _sleep+0x345
g_event_procbody(0,e4331d38,c0af4aa9,30c,c3e162ac,...) at g_event_procbody+0xcb
fork_exit(c071f190,0,e4331d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4331d70, ebp = 0 ---
Tracing command intr pid 12 tid 100033 td 0xc3f978c0
sched_switch(c3f978c0,0,1,189,dead3890,...) at sched_switch+0x576
mi_switch(1,0,c0af4d2b,469,c3fc4364,...) at mi_switch+0x223
ithread_loop(c3fa86a0,e45d1d38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x34c
fork_exit(c0752580,c3fa86a0,e45d1d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45d1d70, ebp = 0 ---
Tracing command intr pid 12 tid 100032 td 0xc3f97af0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100030 td 0xc3f98000
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100029 td 0xc3f98230
sched_switch(c3f98230,0,1,189,deacc1c4,...) at sched_switch+0x576
mi_switch(1,0,c0af4d2b,469,c3e13de4,...) at mi_switch+0x223
ithread_loop(c3fa51f0,e45bbd38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x34c
fork_exit(c0752580,c3fa51f0,e45bbd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45bbd70, ebp = 0 ---
Tracing command intr pid 12 tid 100028 td 0xc3f98460
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100027 td 0xc3e5f690
sched_switch(c3e5f690,0,1,189,36017da8,...) at sched_switch+0x576
mi_switch(1,0,c0af4d2b,469,c3e5d0e4,...) at mi_switch+0x223
ithread_loop(c3f917e0,e45a2d38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x34c
fork_exit(c0752580,c3f917e0,e45a2d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45a2d70, ebp = 0 ---
Tracing command intr pid 12 tid 100026 td 0xc3e5f8c0
sched_switch(c3e5f8c0,0,1,189,29b46be2,...) at sched_switch+0x576
mi_switch(1,0,c0af4d2b,469,c3e5ce64,...) at mi_switch+0x223
ithread_loop(c3f91000,e458cd38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x34c
fork_exit(c0752580,c3f91000,e458cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe458cd70, ebp = 0 ---
Tracing command intr pid 12 tid 100022 td 0xc3f0d230
sched_switch(c3f0d230,0,1,189,deac6724,...) at sched_switch+0x576
mi_switch(1,0,c0af4d2b,469,c3e5cee4,...) at mi_switch+0x223
ithread_loop(c3f08980,e4375d38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x34c
fork_exit(c0752580,c3f08980,e4375d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4375d70, ebp = 0 ---
Tracing command intr pid 12 tid 100019 td 0xc3f0d8c0
sched_switch(c3f0d8c0,0,1,189,3703fb80,...) at sched_switch+0x576
mi_switch(1,0,c0af4d2b,469,c3e5ee64,...) at mi_switch+0x223
ithread_loop(c3f080d0,e434cd38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x34c
fork_exit(c0752580,c3f080d0,e434cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe434cd70, ebp = 0 ---
Tracing command intr pid 12 tid 100018 td 0xc3f0daf0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100016 td 0xc3e1a460
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100015 td 0xc3e1a690
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100009 td 0xc3e5f460
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100008 td 0xc3e18000
cpustop_handler(1,e432bb50,c0a6474d,e432bade,0,...) at cpustop_handler+0x32
ipi_nmi_handler(e432bade,0,7080000,78c72353,c3e16558,...) at ipi_nmi_handler+0x2f
trap(e432bb5c) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0a3c05c, esp = 0xe432bb9c, ebp = 0xe432bbc0 ---
siointr1(c0c848ec,0,c0b21fac,56f,e432bc00,...) at siointr1+0x3cc
siointr(c3fc0c00,f4,c0c69638,c3e18000,c3e13c00,...) at siointr+0x32
intr_execute_handlers(c3dfccd0,e432bc20,e432bc64,c0a4aa24,39,...) at intr_execute_handlers+0xec
lapic_handle_intr(39,e432bc20) at lapic_handle_intr+0x3f
Xapic_isr1() at Xapic_isr1+0x34
--- interrupt, eip = 0xc0a56aeb, esp = 0xe432bc60, ebp = 0xe432bc64 ---
spinlock_exit(c0c1a124,4,c0af6fd0,f4) at spinlock_exit+0x2b
_mtx_unlock_spin_flags(c0c1a124,0,c0af9660,f4,6,...) at _mtx_unlock_spin_flags+0xfd
softclock(0,0,c0af4d2b,46f,c3e5dae4,...) at softclock+0x1d9
ithread_loop(c3e151e0,e432bd38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x1b5
fork_exit(c0752580,c3e151e0,e432bd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe432bd70, ebp = 0 ---
Tracing command intr pid 12 tid 100007 td 0xc3e18230
sched_switch(c3e18230,0,1,189,cc78cb28,...) at sched_switch+0x576
mi_switch(1,0,c0af4d2b,469,c3e5db64,...) at mi_switch+0x223
ithread_loop(c3e151f0,e4328d38,c0af4aa9,30c,c3e16558,...) at ithread_loop+0x34c
fork_exit(c0752580,c3e151f0,e4328d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4328d70, ebp = 0 ---
Tracing command idle pid 11 tid 100006 td 0xc3e18460
sched_switch(c3e18460,0,6,184,83a0ee0e,...) at sched_switch+0x576
mi_switch(6,0,c0afa213,b6,c0c85760,...) at mi_switch+0x223
critical_exit(0,c0a53a05,0) at critical_exit+0x99
lapic_handle_timer(e2922c70) at lapic_handle_timer+0x155
Xtimerint() at Xtimerint+0x1f
--- interrupt, eip = 0xc0a53a05, esp = 0xe2922cb0, ebp = 0xe2922cb0 ---
cpu_idle_default(e2922cf8,c078f5da,c0c23d00,e2922cd8,c0af9ded,...) at cpu_idle_default+0x5
cpu_idle(c0c23d00,e2922cd8,c0af9ded,394,c0c1bb80,...) at cpu_idle+0x28
sched_idletd(0,e2922d38,c0af4aa9,30c,c3e16804,...) at sched_idletd+0x1aa
fork_exit(c078f430,0,e2922d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2922d70, ebp = 0 ---
Tracing command idle pid 11 tid 100005 td 0xc3e18690
sched_switch(c3e18690,0,6,184,895642b6,...) at sched_switch+0x576
mi_switch(6,0,c0af9ded,7f6,1,...) at mi_switch+0x223
sched_preempt(c3e18690,1,e291fcb0,c0a4ad4e,c0af0008,...) at sched_preempt+0x99
ipi_bitmap_handler(c0af0008,28,4f20028,2,0,...) at ipi_bitmap_handler+0x30
Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e
--- interrupt, eip = 0x20, esp = 0xc0a55ad8, ebp = 0xe291fc9c ---
uart_z8530_class(c0a53a05,20,246,e291fcb8,c0a55ad8,...) at 0x20
Tracing command idle pid 11 tid 100004 td 0xc3e188c0
sched_switch(c3e188c0,0,6,184,76e8d9dc,...) at sched_switch+0x576
mi_switch(6,0,c0af9ded,7f6,2,...) at mi_switch+0x223
sched_preempt(c3e188c0,2,e291ccb0,c0a4ad4e,c0a40008,...) at sched_preempt+0x99
ipi_bitmap_handler(c0a40008,28,4f20028,2,0,...) at ipi_bitmap_handler+0x30
Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e
--- interrupt, eip = 0x20, esp = 0xc0a55ad8, ebp = 0xe291cc9c ---
uart_z8530_class(c0a53a05,20,246,e291ccb8,c0a55ad8,...) at 0x20
Tracing command idle pid 11 tid 100003 td 0xc3e18af0
sched_switch(c3e18af0,0,6,184,a1719e78,...) at sched_switch+0x576
mi_switch(6,0,c0af9ded,7f6,3,...) at mi_switch+0x223
sched_preempt(c3e18af0,3,e2919cb0,c0a4ad4e,c0a40008,...) at sched_preempt+0x99
ipi_bitmap_handler(c0a40008,e2910028,4f20028,2,0,...) at ipi_bitmap_handler+0x30
Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e
--- interrupt, eip = 0x20, esp = 0xc0a55ad8, ebp = 0xe2919c9c ---
uart_z8530_class(c0a53a05,20,246,e2919cb8,c0a55ad8,...) at 0x20
Tracing command init pid 1 tid 100002 td 0xc3e18d20
sched_switch(c3e18d20,0,1,189,9533e8dc,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,0,...) at mi_switch+0x223
sleepq_switch(c3e18d20,0,c0afbd7a,19b,c3e16ab0,...) at sleepq_switch+0x170
sleepq_catch_signals(c076305d,c0c1bb80,4,c0af6fd0,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c3e16ab0,0,c0af8d66,da,0,...) at sleepq_wait_sig+0x14
_sleep(c3e16ab0,c3e16b40,15c,c0afdeeb,0,...) at _sleep+0x359
kern_wait(c3e18d20,ffffffff,e2915c2c,0,0,...) at kern_wait+0xa66
wait4(c3e18d20,e2915cfc,10,c0afdd06,c0baf288,...) at wait4+0x3b
syscall(e2915d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x805454f, esp = 0xbfbfe97c, ebp = 0xbfbfe998 ---
Tracing command audit pid 10 tid 100001 td 0xc3e1a000
sched_switch(c3e1a000,0,1,189,dead7e60,...) at sched_switch+0x576
mi_switch(1,0,c0afbd7a,1c0,c3e1a000,...) at mi_switch+0x223
sleepq_switch(c3e1a000,0,c0afbd7a,230,c3e1a000,...) at sleepq_switch+0x170
sleepq_wait(c0c76c68,c0c76c48,c0b14ec2,1,0,...) at sleepq_wait+0x60
_cv_wait(c0c76c68,c0c76c48,c0b15885,18a,0,...) at _cv_wait+0x1fc
audit_worker(0,e2912d38,c0af4aa9,30c,c3e17000,...) at audit_worker+0x84
fork_exit(c093aba0,0,e2912d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2912d70, ebp = 0 ---
Tracing command kernel pid 0 tid 100000 td 0xc0c18290
sched_switch(c0c18290,0,1,189,de6baacc,...) at sched_switch+0x576
mi_switch(1,0,c0b1a058,2e4,c423381c,...) at mi_switch+0x223
scheduler(0,101ec00,101ec00,101e000,1025000,...) at scheduler+0x2f2
mi_startup() at mi_startup+0x96
begin() at begin+0x2c
db> call doadump
Physical memory: 1011 MB
Dumping 134 MB: 119 103 87 71 55 39 23 7
Dump complete
= 0xf
db> c
lock order reversal:
1st 0xc0c1868c cdev (cdev) @ kern/kern_conf.c:73
2nd 0xc147105c system map (system map) @ vm/vm_map.c:3111
KDB: stack backtrace:
db_trace_self_wrapper(c0afb191,e74b9568,c07aad0e,c0afd68a,c147105c,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0afd68a,c147105c,c0b19f6d,c0b19f6d,c0b1a4ef,...) at kdb_backtrace+0x29
witness_checkorder(c147105c,9,c0b1a4e6,c27,c04911d2,...) at witness_checkorder+0x6de
_mtx_lock_flags(c147105c,0,c0b1a4e6,c27,e74b95d0,...) at _mtx_lock_flags+0xbc
_vm_map_lock_read(c1471000,c0b1a4e6,c27,0,1,...) at _vm_map_lock_read+0x33
vm_map_lookup(e74b96c8,deadc000,1,e74b96cc,e74b96bc,...) at vm_map_lookup+0x38
vm_fault(c1471000,deadc000,1,0,deadc0de,...) at vm_fault+0x83
trap_pfault(5,0,c0b27fc4,c6116804,c6116804,...) at trap_pfault+0x186
trap(e74b980c) at trap+0x428
calltrap() at calltrap+0x6
--- trap 0xc, eip = 0xc070040b, esp = 0xe74b984c, ebp = 0xe74b9880 ---
devfs_populate_loop(c41b9c94,4,c0aeea64,1e0,0,...) at devfs_populate_loop+0x8b
devfs_populate(c41b9c80,e74b9950,c0aeed67,2b3,c0b03a1b,...) at devfs_populate+0x44
devfs_lookup(e74b99d4,c0b03c44,c0be83a0,2,c41bb2b8,...) at devfs_lookup+0x236
VOP_LOOKUP_APV(c0babd60,e74b99d4,c0b0312d,2ae,e74b9b94,...) at VOP_LOOKUP_APV+0xe5
lookup(e74b9b80,c0b0312d,c6,bf,c7819e2c,...) at lookup+0x57e
namei(e74b9b80,c0c24c94,c4cc4690,e74b9a7c,246,...) at namei+0x34e
vn_open_cred(e74b9b80,e74b9c78,1a4,c3dee100,c4950dd0,...) at vn_open_cred+0x62
vn_open(e74b9b80,e74b9c78,1a4,c4950dd0,1,...) at vn_open+0x33
kern_open(c4cc4690,8107330,0,602,1b6,...) at kern_open+0xf2
open(c4cc4690,e74b9cfc,c,e74b9d38,c0baf258,...) at open+0x30
syscall(e74b9d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (5, FreeBSD ELF32, open), eip = 0x281bfa8b, esp = 0xbfbfeb4c, ebp = 0xbfbfebe8 ---
panic: vm_fault: fault on nofault entry, addr: deadc000
cpuid = 2
KDB: enter: panic
[thread pid 68680 tid 101057 ]
Stopped at kdb_enter+0x3a: movl $0,kdb_why
db> c
Uptime: 23h4m2s
Physical memory: 1011 MB
Dumping 134 MB: 119 103 87 71 55 39 23 7
Dump complete
Automatic reboot in 15 seconds - press a key on the console to abort
Rebooting...
cpu_reset: Stopping other CPUs
$ addr2line -e kernel.debug 0xc070040b
../../../fs/devfs/devfs_devs.c:368
--
Test scenario: tclone.ko