GENERIC HEAD from Aug 28 15:12 UTC, vmcore.74

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Mon Aug 28 17:53:12 CEST 2006
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1032491008 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 535 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 2118
     lock type devfs: EXCL (count 1) by thread 0xc41a5510 (pid 13552)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07df1cf at ffs_sync+0x2ff
#5 0xc070b849 at vfs_write_suspend+0xa5
#6 0xc07cd279 at ffs_snapshot+0xb15
#7 0xc07dd33f at ffs_mount+0x98b
#8 0xc06fa654 at vfs_domount+0x580
#9 0xc06f9ebc at vfs_donmount+0x410
#10 0xc06fc781 at kernel_mount+0x6d
#11 0xc07dd539 at ffs_cmount+0x5d
#12 0xc06fa0ae at mount+0x15e
#13 0xc089d512 at syscall+0x256
#14 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 246 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 970
     lock type devfs: EXCL (count 1) by thread 0xc460a870 (pid 14124)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07df1cf at ffs_sync+0x2ff
#5 0xc070b849 at vfs_write_suspend+0xa5
#6 0xc07cd279 at ffs_snapshot+0xb15
#7 0xc07dd33f at ffs_mount+0x98b
#8 0xc06fa654 at vfs_domount+0x580
#9 0xc06f9ebc at vfs_donmount+0x410
#10 0xc06fc781 at kernel_mount+0x6d
#11 0xc07dd539 at ffs_cmount+0x5d
#12 0xc06fa0ae at mount+0x15e
#13 0xc089d512 at syscall+0x256
#14 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 183 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 718
     lock type devfs: EXCL (count 1) by thread 0xc57afbd0 (pid 16178)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 92 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 530
     lock type devfs: EXCL (count 1) by thread 0xc40781b0 (pid 19055)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 126 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 490
     lock type devfs: EXCL (count 1) by thread 0xc3fe4870 (pid 23659)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 235 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 926
     lock type devfs: EXCL (count 1) by thread 0xc578abd0 (pid 24950)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07df1cf at ffs_sync+0x2ff
#5 0xc070b849 at vfs_write_suspend+0xa5
#6 0xc07cd279 at ffs_snapshot+0xb15
#7 0xc07dd33f at ffs_mount+0x98b
#8 0xc06fa654 at vfs_domount+0x580
#9 0xc06f9ebc at vfs_donmount+0x410
#10 0xc06fc781 at kernel_mount+0x6d
#11 0xc07dd539 at ffs_cmount+0x5d
#12 0xc06fa0ae at mount+0x15e
#13 0xc089d512 at syscall+0x256
#14 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 39 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 754
     lock type devfs: EXCL (count 1) by thread 0xc5fea870 (pid 42477)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 56 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 822
     lock type devfs: EXCL (count 1) by thread 0xc48ffd80 (pid 43292)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 170 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 2370
     lock type devfs: EXCL (count 1) by thread 0xc4100bd0 (pid 79144)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 280 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 1106
     lock type devfs: EXCL (count 1) by thread 0xc57a4bd0 (pid 79394)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 30 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 458
     lock type devfs: EXCL (count 1) by thread 0xc40ffa20 (pid 99676)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 67 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 254
     lock type devfs: EXCL (count 1) by thread 0xc41b2870 (pid 797)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 340 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 1338
     lock type devfs: EXCL (count 1) by thread 0xc3f1fd80 (pid 5327)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07df1cf at ffs_sync+0x2ff
#5 0xc070b849 at vfs_write_suspend+0xa5
#6 0xc07cd279 at ffs_snapshot+0xb15
#7 0xc07dd33f at ffs_mount+0x98b
#8 0xc06fa654 at vfs_domount+0x580
#9 0xc06f9ebc at vfs_donmount+0x410
#10 0xc06fc781 at kernel_mount+0x6d
#11 0xc07dd539 at ffs_cmount+0x5d
#12 0xc06fa0ae at mount+0x15e
#13 0xc089d512 at syscall+0x256
#14 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc4045bd0: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 511 mountedhere 0xc3f9e600
    flags ()
    v_object 0xc40429d8 ref 0 pages 2030
     lock type devfs: EXCL (count 1) by thread 0xc45df870 (pid 25366)#0 0xc06933c5 at lockmgr+0x589
#1 0xc06f63d1 at vop_stdlock+0x21
#2 0xc08af067 at VOP_LOCK_APV+0x87
#3 0xc070b17c at vn_lock+0xac
#4 0xc07d3375 at softdep_flushworklist+0x39
#5 0xc07df147 at ffs_sync+0x277
#6 0xc070b849 at vfs_write_suspend+0xa5
#7 0xc07cd279 at ffs_snapshot+0xb15
#8 0xc07dd33f at ffs_mount+0x98b
#9 0xc06fa654 at vfs_domount+0x580
#10 0xc06f9ebc at vfs_donmount+0x410
#11 0xc06fc781 at kernel_mount+0x6d
#12 0xc07dd539 at ffs_cmount+0x5d
#13 0xc06fa0ae at mount+0x15e
#14 0xc089d512 at syscall+0x256
#15 0xc0887f1f at Xint0x80_syscall+0x1f

        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
panic: detach with active requests
cpuid = 0
KDB: enter: panic
[thread pid 2 tid 100016 ]
Stopped at      kdb_enter+0x2b: nop
db> where
Tracing pid 2 tid 100016 td 0xc3c2f870
kdb_enter(c09233b3) at kdb_enter+0x2b
panic(c091db9c,c47f0480,e26e3ce4,c066a3c6,c47f0480,...) at panic+0x14b
g_detach(c47f0480) at g_detach+0x82
g_wither_washer(e26e3d04,c0668ca9,258,190,c0668c3c,...) at g_wither_washer+0xde
g_run_events(258,190,c0668c3c,c0668c3c,e26e3d24,...) at g_run_events+0x49
g_event_procbody(0,e26e3d38) at g_event_procbody+0x6d
fork_exit(c0668c3c,0,e26e3d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e3d6c, ebp = 0 ---
db> show allpcpu
Current CPU: 0

cpuid        = 0
curthread    = 0xc3c2f870: pid 2 "g_event"
curpcb       = 0xe26e3d90
fpcurthread  = none
idlethread   = 0xc3c2ea20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc57b8360: pid 34938 "umount"
curpcb       = 0xe6a7cd90
fpcurthread  = none
idlethread   = 0xc3c2e870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc3c2e6c0: pid 11 "idle: cpu2"
curpcb       = 0xe26c5d90
fpcurthread  = none
idlethread   = 0xc3c2e6c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc3c2e510: pid 10 "idle: cpu3"
curpcb       = 0xe26c2d90
fpcurthread  = none
idlethread   = 0xc3c2e510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> show alllocks
Process 34938 (umount) thread 0xc57b8360 (100248)
exclusive sleep mutex Giant r = 0 (0xc0a1f0a8) locked @ kern/vfs_mount.c:1069
Process 2 (g_event) thread 0xc3c2f870 (100016)
exclusive sx GEOM topology r = 0 (0xc0a10540) locked @ geom/geom_event.c:228
db> where 34938
Tracing pid 34938 tid 100248 td 0xc57b8360
cpustop_handler(e6a7cc24,c089c792,c184a600,c184a5a0,e6a7cbd8,...) at cpustop_handler+0x31
ipi_nmi_handler(c184a600,c184a5a0,e6a7cbd8,0,0,...) at ipi_nmi_handler+0x28
trap(c0690008,c0a10028,28,c094b7e9,c57b8360,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc069fbb0, esp = 0xe6a7cc6c, ebp = 0xe6a7cc80 ---
panic(c094b7e9,c0928912,2,c4071234,c,...) at panic+0x54
syscall(3b,3b,3b,804a465,bfbfe609,...) at syscall+0x3b3
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (0, FreeBSD ELF32, nosys), eip = 0x280cb693, esp = 0xbfbfe3bc, ebp = 0xbfbfe468 ---
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
34938  1034  1034     0  R+      CPU 1               umount
34924 34816 34700     0  R                           find
34912     0     0     0  SL      mdwait   0xc6f49800 [md0]
34885     0     0     0  SL      mdwait   0xc6d6f800 [md4]
34878     0     0     0  SL      mdwait   0xc6a59800 [md3]
34865     0     0     0  SL      mdwait   0xc6b28000 [md1]
34823 34820 34700     0  S       piperd   0xc4448ad4 cat
34820 34810 34700     0  SW      wait     0xc5f77000 sh
34819 34810 34700     0  S       piperd   0xc4141630 sort
34818 34810 34700     0  S       piperd   0xc41d318c sed
34817 34810 34700     0  S       piperd   0xc3ff918c xargs
34816 34810 34700     0  SW      wait     0xc45de8d0 sh
34810 34807 34700     0  SW      wait     0xc46018d0 sh
34809 34808 34700     0  S       piperd   0xc4028ad4 mail
34808 34801 34700     0  SW      wait     0xc3f6e69c sh
34807 34801 34700     0  SW      wait     0xc5f7169c sh
34801 34800 34700     0  SW      wait     0xc45e869c sh
34800 34708 34700     0  SW      wait     0xc41a369c sh
34710 34709 34700     0  S       piperd   0xc3ff718c mail
34709 34702 34700     0  SW      wait     0xc4601000 sh
34708 34702 34700     0  SW      wait     0xc40fd468 sh
34702 34700 34700     0  SW      wait     0xc46008d0 sh
34700 34698 34700     0  SWs     wait     0xc4602d38 sh
34698   858   858     0  S       piperd   0xc4028dec cron
34673 34669 16544  1001  RL+                         swap
34672 34669 16544  1001  RL+                         swap
34670 34669 16544  1001  RL+                         swap
34669 34663 16544  1001  S+      wait     0xc53fdd38 swap
34663 16548 16544  1001  SW+     wait     0xc4602234 swap
99403     0     0     0  SL      mdwait   0xc5813800 [md2]
16548 16547 16544  1001  SW+     wait     0xc4139000 run
16547 16546 16544  1001  SW+     wait     0xc407169c run
16546 16544 16544  1001  S+      nanslp   0xc0a1f8a4 run
16544   985 16544  1001  SW+     wait     0xc41a68d0 sh
16381 16380 16381  1001  SWs+    ttyin    0xc5a80010 bash
16380 16373 16373  1001  SW      select   0xc0a6c37c sshd
16373   841 16373     0  SWs     sbwait   0xc6013d94 sshd
 1034  1029  1034     0  S+      wait     0xc41a3234 sh
 1029  1026  1029     0  SW+     wait     0xc401d468 bash
 1026  1025  1026     0  SW+     pause    0xc40fe6d0 csh
 1025  1015  1025  1001  SW+     wait     0xc40feb04 su
 1015  1014  1015  1001  SWs+    wait     0xc3f70234 bash
 1014  1012  1012  1001  SW      select   0xc0a6c37c sshd
 1012   841  1012     0  SWs     sbwait   0xc410f5cc sshd
 1000   994  1000  1001  S+      select   0xc0a6c37c top
  994   993   994  1001  SWs+    wait     0xc4139468 bash
  993   991   991  1001  S       select   0xc0a6c37c sshd
  991   841   991     0  SWs     sbwait   0xc41b85cc sshd
  985   984   985  1001  SWs+    wait     0xc401d69c bash
  984   982   982  1001  SW      select   0xc0a6c37c sshd
  982   841   982     0  SWs     sbwait   0xc4447334 sshd
  978     1   978     0  SWs+    ttyin    0xc3df6810 getty
  977     1   977     0  SWs+    ttyin    0xc3dfa810 getty
  976     1   976     0  SWs+    ttyin    0xc3df9810 getty
  975     1   975     0  SWs+    ttyin    0xc3df9010 getty
  974     1   974     0  SWs+    ttyin    0xc3df8c10 getty
  973     1   973     0  SWs+    ttyin    0xc3dfac10 getty
  972     1   972     0  SWs+    ttyin    0xc3dfb010 getty
  971     1   971     0  SWs+    ttyin    0xc3df9c10 getty
  954     1   954     0  SWs     select   0xc0a6c37c inetd
  925     1   925     0  SWs     select   0xc0a6c37c moused
  858     1   858     0  Ss      nanslp   0xc0a1f8a4 cron
  852     1   852    25  SWs     pause    0xc4021d6c sendmail
  848     1   848     0  Ss      select   0xc0a6c37c sendmail
  841     1   841     0  SWs     select   0xc0a6c37c sshd
  821     1   821     0  Ss      select   0xc0a6c37c ntpd
  761   757   757     0  SW      -        0xc3f94a00 nfsd
  760   757   757     0  SW      -        0xc3f95000 nfsd
  759   757   757     0  SW      -        0xc3f94c00 nfsd
  758   757   757     0  SW      -        0xc3f94e00 nfsd
  757     1   757     0  SWs     select   0xc0a6c37c nfsd
  755     1   755     0  SWs     select   0xc0a6c37c mountd
  696     1   696     0  Ss      select   0xc0a6c37c rpcbind
  673     1   673     0  Ss      select   0xc0a6c37c syslogd
  563     1   563     0  SWs     select   0xc0a6c37c devd
   42     0     0     0  SL      -        0xe43eccfc [schedcpu]
   41     0     0     0  SL      sdflush  0xc0a79800 [softdepflush]
   40     0     0     0  SL      vlruwt   0xc3f6ed38 [vnlru]
   39     0     0     0  SL      syncer   0xc0a1f670 [syncer]
   38     0     0     0  SL      psleep   0xc0a6c7f4 [bufdaemon]
   37     0     0     0  SL      pgzero   0xc0a82130 [pagezero]
   36     0     0     0  SL      psleep   0xc0a7a060 [vmdaemon]
   35     0     0     0  SL      psleep   0xc0a7a020 [pagedaemon]
   34     0     0     0  WL                          [irq7: ppc0]
   33     0     0     0  SL      -        0xc3d7603c [fdc0]
   32     0     0     0  WL                          [swi0: sio]
   31     0     0     0  WL                          [irq12: psm0]
   30     0     0     0  WL                          [irq1: atkbd0]
   29     0     0     0  WL                          [irq15: ata1]
   28     0     0     0  WL                          [irq14: ata0]
   27     0     0     0  WL                          [irq17: fxp0]
   26     0     0     0  SL      usbtsk   0xc0a0fde4 [usbtask]
   25     0     0     0  SL      usbevt   0xc3d65210 [usb0]
   24     0     0     0  WL                          [irq16: uhci0]
   23     0     0     0  SL      -        0xc3d52280 [em0 taskq]
   22     0     0     0  WL                          [irq9: acpi0]
    9     0     0     0  SL      -        0xc3c7d600 [thread taskq]
   21     0     0     0  WL                          [swi6: Giant taskq]
   20     0     0     0  WL                          [swi6: task queue]
    8     0     0     0  SL      -        0xc3c7d880 [acpi_task_2]
    7     0     0     0  SL      -        0xc3c7d880 [acpi_task_1]
    6     0     0     0  SL      -        0xc3c7d880 [acpi_task_0]
   19     0     0     0  WL                          [swi2: cambio]
    5     0     0     0  SL      -        0xc3c7da80 [kqueue taskq]
   18     0     0     0  WL                          [swi5: +]
   17     0     0     0  SL      -        0xc0a0da20 [yarrow]
    4     0     0     0  SL      -        0xc0a1059c [g_down]
    3     0     0     0  SL      -        0xc0a10598 [g_up]
    2     0     0     0  RL      CPU 0               [g_event]
   16     0     0     0  WL                          [swi3: vm]
   15     0     0     0  WL                          [swi4: clock sio]
   14     0     0     0  WL                          [swi1: net]
   13     0     0     0  RL                          [idle: cpu0]
   12     0     0     0  RL                          [idle: cpu1]
   11     0     0     0  RL      CPU 2               [idle: cpu2]
   10     0     0     0  RL      CPU 3               [idle: cpu3]
    1     0     1     0  SLs     wait     0xc3c32000 [init]
    0     0     0     0  WLs                         [swapper]
34671 34669 16544  1001  Z+                          swap
34664 16548 16544  1001  Z+                          creat
34665 16548 16544  1001  Z+                          mkdir
34667 16548 16544  1001  Z+                          udp
34668 16548 16544  1001  Z+                          tcp
34666 16548 16544  1001  Z+                          thr1
db> show lock 0xc0a10540
 class: sx
 name: GEOM topology
 state: XLOCK: 0xc3c2f870 (tid 100016, pid 2, "g_event")
 waiters: 0 shared, 0 exclusive
db> call doadump
Physical memory: 1007 MB
Dumping 148 MB: 133 117 101 85 69 53 37 21 5
Dump complete
= 0xf
db> reset
GDB: no debug ports present

(kgdb) l *syscall+0x3b3
0xc089d66f is in syscall (../../../i386/i386/trap.c:1060).
1055            WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
1056                (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
1057            KASSERT(td->td_critnest == 0,
1058                ("System call %s returning in a critical section",
1059                (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???"));
1060            KASSERT(td->td_locks == 0,
1061                ("System call %s returning with %d locks held",
1062                (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???",
1063                td->td_locks));
1064
(kgdb) p *(struct thread *)0xc57b8360
$1 = {td_proc = 0xc4071234, td_ksegrp = 0xc40734e0, td_plist = {tqe_next = 0x0, tqe_prev = 0xc4071244}, td_kglist = {tqe_next = 0x0,
    tqe_prev = 0xc40734ec}, td_slpq = {tqe_next = 0x0, tqe_prev = 0xc48e33a0}, td_lockq = {tqe_next = 0xc5fea360, tqe_prev = 0xe6b69b18},
  td_runq = {tqe_next = 0x0, tqe_prev = 0xc40734f4}, td_selq = {tqh_first = 0x0, tqh_last = 0x0}, td_sleepqueue = 0xc48e33a0,
  td_turnstile = 0xc50aa400, td_umtxq = 0xc4943040, td_tid = 0x18798, td_sigqueue = {sq_signals = {__bits = {0x0, 0x0, 0x0, 0x0}},
    sq_kill = {__bits = {0x0, 0x0, 0x0, 0x0}}, sq_list = {tqh_first = 0x0, tqh_last = 0xc57b83c8}, sq_proc = 0xc4071234, sq_flags = 0x1},
  td_flags = 0x1000000, td_inhibitors = 0x0, td_pflags = 0x0, td_dupfd = 0x0, td_wchan = 0x0, td_wmesg = 0x0, td_lastcpu = 0x1,
  td_oncpu = 0x1, td_owepreempt = 0x0, td_locks = 0x2, td_tsqueue = 0x0, td_blocked = 0x0, td_lockname = 0x0, td_contested = {
    lh_first = 0x0}, td_sleeplocks = 0xc0a6bb10, td_intr_nesting_level = 0x0, td_pinned = 0x0, td_mailbox = 0x0, td_ucred = 0xc442b980,
  td_standin = 0x0, td_upcall = 0x0, td_pticks = 0xf, td_sticks = 0xf, td_iticks = 0x0, td_uticks = 0x0, td_uuticks = 0x0,
  td_usticks = 0x0, td_intrval = 0x0, td_oldsigmask = {__bits = {0x0, 0x0, 0x0, 0x0}}, td_sigmask = {__bits = {0x0, 0x0, 0x0, 0x0}},
  td_generation = 0xb, td_sigstk = {ss_sp = 0x0, ss_size = 0x0, ss_flags = 0x4}, td_kflags = 0x0, td_xsig = 0x0, td_profil_addr = 0x0,
  td_profil_ticks = 0x0, td_name = '\0' <repeats 19 times>, td_base_pri = 0xb4, td_priority = 0xb4, td_pcb = 0xe6a7cd90,
  td_state = TDS_RUNNING, td_retval = {0x0, 0x10}, td_slpcallout = {c_links = {sle = {sle_next = 0x0}, tqe = {tqe_next = 0x0,
        tqe_prev = 0xd7c4b078}}, c_time = 0x1c91d05, c_arg = 0xc57b8360, c_func = 0xc06c4b50 <sleepq_timeout>, c_mtx = 0x0,
    c_flags = 0x10}, td_frame = 0xe6a7cd38, td_kstack_obj = 0xc4b471e0, td_kstack = 0xe6a7b000, td_kstack_pages = 0x2,
  td_altkstack_obj = 0x0, td_altkstack = 0x0, td_altkstack_pages = 0x0, td_critnest = 0x0, td_md = {md_spinlock_count = 0x0,
    md_saved_flags = 0x46}, td_sched = 0xc57b84f0, td_ar = 0x0}