GENERIC HEAD from Sep 15 13:27 UTC, vmcore.81
Stress test with snapshots and a full disk.

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #0: Fri Sep 15 15:44:26 CEST 2006
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1032458240 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
panic: softdep_deallocate_dependencies: dangling deps
cpuid = 1
KDB: enter: panic
[thread pid 40 tid 100048 ]
Stopped at      kdb_enter+0x2b: nop
db> where
Tracing pid 40 tid 100048 td 0xc3f80510
kdb_enter(c091a65b) at kdb_enter+0x2b
panic(c09343a4,e43eebe8,c06e3ea1,d7d22d70,c091837f,...) at panic+0x14b
softdep_deallocate_dependencies(d7d22d70) at softdep_deallocate_dependencies+0x16
brelse(d7d22d70,d7d22d70) at brelse+0xa1
flushbuflist(c42967a4,0,c4296790,0,0) at flushbuflist+0x206
bufobj_invalbuf(c4296790,0,c3f80510,0,0,...) at bufobj_invalbuf+0x12b
vinvalbuf(c4296690,0,c3f80510,0,0) at vinvalbuf+0x2a
vgonel(c4296690,7,0,c09edd00,c4296690,...) at vgonel+0xcd
vlrureclaim(c3fc3b50,c3f80510,c3f7fd38,c06f29b0,c06f29b0,...) at vlrureclaim+0x2a2
vnlru_proc(0,e43eed38) at vnlru_proc+0x193
fork_exit(c06f29b0,0,e43eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43eed6c, ebp = 0 ---
db> show allpcpu
Current CPU: 1

cpuid        = 0
curthread    = 0xc3f806c0: pid 41 "softdepflush"
curpcb       = 0xe43f1d90
fpcurthread  = none
idlethread   = 0xc3c36a20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc3f80510: pid 40 "vnlru"
curpcb       = 0xe43eed90
fpcurthread  = none
idlethread   = 0xc3c36870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc3c366c0: pid 11 "idle: cpu2"
curpcb       = 0xe26cdd90
fpcurthread  = none
idlethread   = 0xc3c366c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc3c36510: pid 10 "idle: cpu3"
curpcb       = 0xe26cad90
fpcurthread  = none
idlethread   = 0xc3c36510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> wher 41
Tracing pid 41 tid 100047 td 0xc3f806c0
cpustop_handler(e43f1900,c0895562,c980258,0,6b970000,...) at cpustop_handler+0x31
ipi_nmi_handler(c980258,0,6b970000,0,0,...) at ipi_nmi_handler+0x28
trap(c0a00008,c3f80028,e43f0028,60,c3df3c00,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc086ca37, esp = 0xe43f1948, ebp = 0xe43f1958 ---
siointr1(c3df3c00,c0a78b6c,0,c093ceba,56e,...) at siointr1+0x87
siointr(c3df3c00) at siointr+0x21
intr_execute_handlers(c3c2c0c4,e43f19b4,4,e43f1a00,c0880cd3,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc068d02c, esp = 0xe43f19f4, ebp = 0xe43f1a00 ---
_mtx_lock_flags(c0a63758,0,c09330e0,1663,c3fb00a4,0,c09330e0,1662) at _mtx_lock_flags+0x98
softdep_request_cleanup(c3fb7800,c4596540) at softdep_request_cleanup+0x102
ffs_alloc(c45c08c4,5065c1,0,2832e08,0,4000,c3c31400,e43f1b10) at ffs_alloc+0x27b
ffs_balloc_ufs2(c4596540,19704000,14,4000,c3c31400,...) at ffs_balloc_ufs2+0x1619
ffs_copyonwrite(c4040d20,d7de4f24) at ffs_copyonwrite+0x3d0
ffs_geom_strategy(c4040e20,d7de4f24,d7de4f84,e43f1c50,c09d8a40,...) at ffs_geom_strategy+0xbd
ufs_strategy(e43f1c60) at ufs_strategy+0xb5
VOP_STRATEGY_APV(c09d8500,e43f1c60) at VOP_STRATEGY_APV+0xaf
bufstrategy(c40f94f0,d7de4f24) at bufstrategy+0x55
bufwrite(d7de4f24) at bufwrite+0x15a
bawrite(d7de4f24) at bawrite+0x51
ffs_syncvnode(c40f93f0,2) at ffs_syncvnode+0x21c
clear_remove(c3f806c0,63,c07cba88,c07cba88,0,...) at clear_remove+0xf2
softdep_flush(0,e43f1d38) at softdep_flush+0x73
fork_exit(c07cba88,0,e43f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43f1d6c, ebp = 0 ---
db> show alllocks
Process 41 (softdepflush) thread 0xc3f806c0 (100047)
exclusive sleep mutex buf queue lock r = 0 (0xc0a56d38) locked @ kern/vfs_bio.c:1445
db> show lockedvnods
Locked vnodes

0xc4040d20: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 595 mountedhere 0xc3fac100
    flags ()
    v_object 0xc40d5708 ref 0 pages 2484
     lock type devfs: EXCL (count 1) by thread 0xc3c3bd80 (pid 38)#0 0xc0689b68 at lockmgr+0x510
#1 0xc06eba35 at vop_stdlock+0x21
#2 0xc08a8a17 at VOP_LOCK_APV+0x87
#3 0xc0700ccc at vn_lock+0xac
#4 0xc06e5dd3 at flushbufqueues+0x443
#5 0xc06e584d at buf_daemon+0x95
#6 0xc06804f8 at fork_exit+0xac
#7 0xc088097c at fork_trampoline+0x8

        dev ad0s1d

0xc640ba80: tag ufs, type VDIR
    usecount 3, writecount 0, refcount 6 mountedhere 0
    flags ()
    v_object 0xc6a545a0 ref 0 pages 1
     lock type ufs: EXCL (count 1) by thread 0xc4710000 (pid 6128)#0 0xc0689b68 at lockmgr+0x510
#1 0xc07d9546 at ffs_lock+0x76
#2 0xc08a8a17 at VOP_LOCK_APV+0x87
#3 0xc0700ccc at vn_lock+0xac
#4 0xc06ed6ca at lookup+0xde
#5 0xc06ed366 at namei+0x382
#6 0xc06fb3c3 at kern_lstat+0x47
#7 0xc06fb35f at lstat+0x1b
#8 0xc08962e2 at syscall+0x256
#9 0xc088096f at Xint0x80_syscall+0x1f

        ino 741719, on dev ad0s1g

0xc4596540: tag ufs, type VREG
    usecount 1, writecount 0, refcount 26 mountedhere 0
    flags (VV_SYSTEM)
    v_object 0xc458fa50 ref 0 pages 472
     lock type snaplk: EXCL (count 1) by thread 0xc3f806c0 (pid 41) with 3 pending#0 0xc0689b68 at lockmgr+0x510
#1 0xc07cb08a at ffs_copyonwrite+0x246
#2 0xc07d9059 at ffs_geom_strategy+0xbd
#3 0xc07e4941 at ufs_strategy+0xb5
#4 0xc08a8ce3 at VOP_STRATEGY_APV+0xaf
#5 0xc06e8625 at bufstrategy+0x55
#6 0xc06e36be at bufwrite+0x15a
#7 0xc06e3cf1 at bawrite+0x51
#8 0xc07d934c at ffs_syncvnode+0x21c
#9 0xc07d4a4e at clear_remove+0xf2
#10 0xc07cbafb at softdep_flush+0x73
#11 0xc06804f8 at fork_exit+0xac
#12 0xc088097c at fork_trampoline+0x8

        ino 7, on dev ad0s1d

0xc47663f0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 3 mountedhere 0
    flags ()
    v_object 0xc45b53c0 ref 0 pages 0
     lock type ufs: EXCL (count 1) by thread 0xc3f80360 (pid 39)#0 0xc0689b68 at lockmgr+0x510
#1 0xc07d9546 at ffs_lock+0x76
#2 0xc08a8a17 at VOP_LOCK_APV+0x87
#3 0xc0700ccc at vn_lock+0xac
#4 0xc06f45a7 at sync_vnode+0xe3
#5 0xc06f484d at sched_sync+0x1f5
#6 0xc06804f8 at fork_exit+0xac
#7 0xc088097c at fork_trampoline+0x8

        ino 376838, on dev ad0s1d

0xc40db3f0: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 4 mountedhere 0
    flags (VV_ROOT)
    v_object 0xc45e1ac8 ref 0 pages 1
     lock type ufs: EXCL (count 1) by thread 0xc3de8bd0 (pid 1117)#0 0xc0689b68 at lockmgr+0x510
#1 0xc07d9546 at ffs_lock+0x76
#2 0xc08a8a17 at VOP_LOCK_APV+0x87
#3 0xc0700ccc at vn_lock+0xac
#4 0xc06f4e7e at vget+0xc2
#5 0xc06ec789 at vfs_hash_get+0x8d
#6 0xc07d82ef at ffs_vget+0x27
#7 0xc07e1a70 at ufs_root+0x1c
#8 0xc06edd83 at lookup+0x797
#9 0xc06ed366 at namei+0x382
#10 0xc06fb295 at kern_stat+0x35
#11 0xc06fb243 at stat+0x1b
#12 0xc08962e2 at syscall+0x256
#13 0xc088096f at Xint0x80_syscall+0x1f

        ino 2, on dev ad0s1d

0xc40f93f0: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 5 mountedhere 0
    flags ()
    v_object 0xc4068078 ref 0 pages 1
     lock type ufs: EXCL (count 1) by thread 0xc3f806c0 (pid 41) with 1 pending#0 0xc0689b68 at lockmgr+0x510
#1 0xc07d9546 at ffs_lock+0x76
#2 0xc08a8a17 at VOP_LOCK_APV+0x87
#3 0xc0700ccc at vn_lock+0xac
#4 0xc06f4e7e at vget+0xc2
#5 0xc06ec789 at vfs_hash_get+0x8d
#6 0xc07d82ef at ffs_vget+0x27
#7 0xc07d4a0f at clear_remove+0xb3
#8 0xc07cbafb at softdep_flush+0x73
#9 0xc06804f8 at fork_exit+0xac
#10 0xc088097c at fork_trampoline+0x8

        ino 10551296, on dev ad0s1d

0xc4183a80: tag ufs, type VREG
    usecount 2, writecount 1, refcount 4 mountedhere 0
    flags ()
    v_object 0xc40d4ca8 ref 0 pages 9
     lock type ufs: EXCL (count 1) by thread 0xc3fe3870 (pid 673)#0 0xc0689b68 at lockmgr+0x510
#1 0xc07d9546 at ffs_lock+0x76
#2 0xc08a8a17 at VOP_LOCK_APV+0x87
#3 0xc0700ccc at vn_lock+0xac
#4 0xc0700634 at vn_write+0x138
#5 0xc06c13e7 at dofilewrite+0x7b
#6 0xc06c128a at kern_writev+0x36
#7 0xc06c1238 at writev+0x30
#8 0xc08962e2 at syscall+0x256
#9 0xc088096f at Xint0x80_syscall+0x1f

        ino 10551310, on dev ad0s1d

0xc4296690: tag ufs, type VREG
    usecount 0, writecount 0, refcount 3 mountedhere 0
    flags (VI_DOOMED)
    v_object 0xc40e8708 ref 0 pages 1
     lock type ufs: EXCL (count 1) by thread 0xc3f80510 (pid 40)#0 0xc0689b68 at lockmgr+0x510
#1 0xc07d9546 at ffs_lock+0x76
#2 0xc08a8a17 at VOP_LOCK_APV+0x87
#3 0xc06f262d at vlrureclaim+0x20d
#4 0xc06f2b43 at vnlru_proc+0x193
#5 0xc06804f8 at fork_exit+0xac
#6 0xc088097c at fork_trampoline+0x8

        ino 10174465, on dev ad0s1d
db> show lockedbufs
buf at 0xd7d22d70
b_flags = 0xa04222a0<remfree,vmio,relbuf,clusterok,inval,done,delwri,cache>
b_error = 28, b_bufsize = 2048, b_bcount = 2048, b_resid = 0
b_bufobj = (0xc4296790), b_data = 0xd9398000, b_blkno = 192336892
b_npages = 1, pages(OBJ, IDX, PA): (0xc40e8708, 0x0, 0x7e27000)
 lock type flushbuf: EXCL (count 1) by thread 0xc3f80510 (pid 40)#0 0xc0689b68 at lockmgr+0x510
#1 0xc06f3708 at flushbuflist+0xdc
#2 0xc06f3433 at bufobj_invalbuf+0x12b
#3 0xc06f3626 at vinvalbuf+0x2a
#4 0xc06f5be1 at vgonel+0xcd
#5 0xc06f26c2 at vlrureclaim+0x2a2
#6 0xc06f2b43 at vnlru_proc+0x193
#7 0xc06804f8 at fork_exit+0xac
#8 0xc088097c at fork_trampoline+0x8

buf at 0xd7d7d7c0
b_flags = 0x20000020<vmio,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc4596640), b_data = 0xda168000, b_blkno = -4196365
b_npages = 4, pages(OBJ, IDX, PA): (0xc458fa50, 0xfeffdfcc, 0x6185000),(0xc458fa50, 0xfeffdfcd, 0x12806000),(0xc458fa50, 0xfeffdfce, 0x2b187000),(0xc458fa50, 0xfeffdfcf, 0x29a08000)
 lock type getblk: EXCL (count 1) by thread 0xc3f806c0 (pid 41)#0 0xc0689b68 at lockmgr+0x510
#1 0xc06e64db at getblk+0x12f
#2 0xc06e3467 at breadn+0x2f
#3 0xc06e32c0 at bread+0x20
#4 0xc07c3248 at ffs_balloc_ufs2+0x1290
#5 0xc07cb14f at ffs_copyonwrite+0x30b
#6 0xc07d9059 at ffs_geom_strategy+0xbd
#7 0xc07e4941 at ufs_strategy+0xb5
#8 0xc08a8ce3 at VOP_STRATEGY_APV+0xaf
#9 0xc06e8625 at bufstrategy+0x55
#10 0xc06e36be at bufwrite+0x15a
#11 0xc06e3cf1 at bawrite+0x51
#12 0xc07d934c at ffs_syncvnode+0x21c
#13 0xc07d4a4e at clear_remove+0xf2
#14 0xc07cbafb at softdep_flush+0x73
#15 0xc06804f8 at fork_exit+0xac
#16 0xc088097c at fork_trampoline+0x8

buf at 0xd7de4f24
b_flags = 0xa0000024<remfree,vmio,cache,async>
b_error = 28, b_bufsize = 2048, b_bcount = 2048, b_resid = 0
b_bufobj = (0xc40f94f0), b_data = 0xdb12c000, b_blkno = 168605728
b_npages = 1, pages(OBJ, IDX, PA): (0xc4068078, 0x0, 0x33b6f000)
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
 6227  1401  1397     0  R+                          rw
 6128  6099  5886     0  R                           find
 6118  6104  5886     0  S       piperd   0xc3ff3ad4 cat
 6104  6093  5886     0  S       wait     0xc498d234 sh
 6103  6093  5886     0  S       piperd   0xc4129c60 sort
 6101  6093  5886     0  S       piperd   0xc4447000 sed
 6100  6093  5886     0  S       piperd   0xc44474a4 xargs
 6099  6093  5886     0  S       wait     0xc45a9d38 sh
 6093  6090  5886     0  S       wait     0xc4550468 sh
 6092  6091  5886     0  S       piperd   0xc3ff2630 mail
 6091  6084  5886     0  S       wait     0xc473dd38 sh
 6090  6084  5886     0  S       wait     0xc4553000 sh
 6084  6083  5886     0  S       wait     0xc4740d38 sh
 6083  5899  5886     0  S       wait     0xc411cd38 sh
 5901  5900  5886     0  S       piperd   0xc412a630 mail
 5900  5889  5886     0  S       wait     0xc470db04 sh
 5899  5889  5886     0  S       wait     0xc455069c sh
 5889  5886  5886     0  S       wait     0xc45a9234 sh
 5886  5885  5886     0  Ss      wait     0xc411c234 sh
 5885   858   858     0  S       piperd   0xc3ff2ad4 cron
 1421  1418  1421     0  S+      ttyin    0xc3f78810 bash
 1418  1417  1418     0  S+      pause    0xc45536d0 csh
 1417  1095  1417  1001  S+      wait     0xc4553468 su
 1401  1400  1397     0  S+      wait     0xc4550000 run
 1400  1399  1397     0  S+      wait     0xc41e169c run
 1399  1397  1397     0  R+                          run
 1397  1131  1397     0  S+      wait     0xc45aa8d0 sh
 1131  1128  1131     0  S+      wait     0xc41e1000 bash
 1128  1127  1128     0  SW+     pause    0xc45aa268 csh
 1127  1121  1127  1001  SW+     wait     0xc404f234 su
 1121  1120  1121  1001  SWs+    wait     0xc411b468 bash
 1120  1118  1118  1001  S       select   0xc0a5683c sshd
 1118   841  1118     0  SWs     sbwait   0xc41c0334 sshd
 1117  1018  1117  1001  S+      ufs      0xc40f9448 tail
 1116  1110  1116  1001  S+      select   0xc0a5683c top
 1110  1109  1110  1001  SWs+    wait     0xc411c69c bash
 1109  1107  1107  1001  S       select   0xc0a5683c sshd
 1107   841  1107     0  SWs     sbwait   0xc414d9b0 sshd
 1095  1094  1095  1001  Ss+     wait     0xc411cb04 bash
 1094  1092  1092  1001  S       select   0xc0a5683c sshd
 1092   841  1092     0  SWs     sbwait   0xc414d09c sshd
 1018  1017  1018  1001  SWs+    wait     0xc41ded38 bash
 1017  1015  1015  1001  S       select   0xc0a5683c sshd
 1015   841  1015     0  SWs     sbwait   0xc4132ee0 sshd
  979     1   979     0  SWs+    ttyin    0xc3dfe810 getty
  978     1   978     0  SWs+    ttyin    0xc3e02810 getty
  977     1   977     0  SWs+    ttyin    0xc3e01810 getty
  976     1   976     0  SWs+    ttyin    0xc3e01010 getty
  975     1   975     0  SWs+    ttyin    0xc3e00c10 getty
  974     1   974     0  SWs+    ttyin    0xc3e02c10 getty
  973     1   973     0  SWs+    ttyin    0xc3e03010 getty
  972     1   972     0  SWs+    ttyin    0xc3e01c10 getty
  955     1   955     0  SWs     select   0xc0a5683c inetd
  926     1   926     0  SWs     select   0xc0a5683c moused
  909     1   909     0  Rs                          watchdogd
  858     1   858     0  Ss      nanslp   0xc0a09d64 cron
  852     1   852    25  Ss      pause    0xc404f034 sendmail
  848     1   848     0  Ss      select   0xc0a5683c sendmail
  841     1   841     0  SWs     select   0xc0a5683c sshd
  821     1   821     0  Ss      select   0xc0a5683c ntpd
  761   757   757     0  SW      -        0xc416fc00 nfsd
  760   757   757     0  SW      -        0xc3d7ec00 nfsd
  759   757   757     0  SW      -        0xc416fe00 nfsd
  758   757   757     0  SW      -        0xc414b200 nfsd
  757     1   757     0  SWs     select   0xc0a5683c nfsd
  755     1   755     0  SWs     select   0xc0a5683c mountd
  696     1   696     0  Ss      select   0xc0a5683c rpcbind
  673     1   673     0  Ss      snaplk   0xc4422214 syslogd
  563     1   563     0  SWs     select   0xc0a5683c devd
   42     0     0     0  SL      -        0xe43f4cfc [schedcpu]
   41     0     0     0  RL      CPU 0               [softdepflush]
   40     0     0     0  RL      CPU 1               [vnlru]
   39     0     0     0  SL      snaplk   0xc4422214 [syncer]
   38     0     0     0  SL      snaplk   0xc4422214 [bufdaemon]
   37     0     0     0  SL      pgzero   0xc0a6c0b0 [pagezero]
   36     0     0     0  SL      psleep   0xc0a63fe0 [vmdaemon]
   35     0     0     0  SL      psleep   0xc0a63fa0 [pagedaemon]
   34     0     0     0  WL                          [irq7: ppc0]
   33     0     0     0  SL      -        0xc3d7e03c [fdc0]
   32     0     0     0  WL                          [swi0: sio]
   31     0     0     0  WL                          [irq12: psm0]
   30     0     0     0  WL                          [irq1: atkbd0]
   29     0     0     0  WL                          [irq15: ata1]
   28     0     0     0  WL                          [irq14: ata0]
   27     0     0     0  WL                          [irq17: fxp0]
   26     0     0     0  SL      usbtsk   0xc0a072e4 [usbtask]
   25     0     0     0  SL      usbevt   0xc3d6d210 [usb0]
   24     0     0     0  WL                          [irq16: uhci0]
   23     0     0     0  SL      -        0xc3d5a280 [em0 taskq]
   22     0     0     0  WL                          [irq9: acpi0]
   21     0     0     0  WL                          [swi5: +]
    9     0     0     0  SL      -        0xc3c85700 [thread taskq]
   20     0     0     0  WL                          [swi6: Giant taskq]
    8     0     0     0  SL      -        0xc3c85880 [acpi_task_2]
    7     0     0     0  SL      -        0xc3c85880 [acpi_task_1]
    6     0     0     0  SL      -        0xc3c85880 [acpi_task_0]
   19     0     0     0  WL                          [swi6: task queue]
    5     0     0     0  SL      -        0xc3c85a00 [kqueue taskq]
   18     0     0     0  WL                          [swi2: cambio]
   17     0     0     0  SL      -        0xc0a04f20 [yarrow]
    4     0     0     0  SL      -        0xc0a07a9c [g_down]
    3     0     0     0  SL      -        0xc0a07a98 [g_up]
    2     0     0     0  SL      -        0xc0a07a90 [g_event]
   16     0     0     0  WL                          [swi3: vm]
   15     0     0     0  WL                          [swi4: clock sio]
   14     0     0     0  WL                          [swi1: net]
   13     0     0     0  RL                          [idle: cpu0]
   12     0     0     0  RL                          [idle: cpu1]
   11     0     0     0  RL      CPU 2               [idle: cpu2]
   10     0     0     0  RL      CPU 3               [idle: cpu3]
    1     0     1     0  SLs     wait     0xc3c3a000 [init]
    0     0     0     0  WLs                         [swapper]
 6228  6227  1397     0  Z+                          rw
db> allt

Tracing command rw pid 6227 tid 100044 td 0xc3d46bd0
sched_switch(c3d46bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a09d64) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a09d64) at sleepq_timedwait_sig+0x1e
msleep(c0a09d64,0,15c,c091b792,3e9,...) at msleep+0x244
kern_nanosleep(c3d46bd0,e414dc74,e414dc6c) at kern_nanosleep+0xab
nanosleep(c3d46bd0,e414dd04) at nanosleep+0x4f
syscall(3b,3b,3b,2805288c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28138b2b, esp = 0xbfbfe8bc, ebp = 0xbfbfe8e8 ---

Tracing command find pid 6128 tid 100179 td 0xc4710000
sched_switch(c4710000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a56fd8,c0a09550,0,c091da47,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a56fd8) at sleepq_timedwait+0x4a
msleep(c0a56fd8,c0a56f78,50,c0924cad,3e8) at msleep+0x259
getnewvnode(c0922945,c3fc9000,c09d8500,e68a4910) at getnewvnode+0xb8
ffs_vget(c3fc9000,b51bc,2,e68a4974) at ffs_vget+0xc2
ufs_lookup(e68a4a1c) at ufs_lookup+0xac4
VOP_CACHEDLOOKUP_APV(c09d8500,e68a4a1c) at VOP_CACHEDLOOKUP_APV+0x9b
vfs_cache_lookup(e68a4abc) at vfs_cache_lookup+0xb5
VOP_LOOKUP_APV(c09d8500,e68a4abc) at VOP_LOOKUP_APV+0xa1
lookup(e68a4b48) at lookup+0x49c
namei(e68a4b48) at namei+0x382
kern_lstat(c4710000,82d3da8,0,e68a4c1c) at kern_lstat+0x47
lstat(c4710000,e68a4d04) at lstat+0x1b
syscall(825003b,82c003b,bfbf003b,82d3d48,82d3d00,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (190, FreeBSD ELF32, lstat), eip = 0x28149c37, esp = 0xbfbfebbc, ebp = 0xbfbfec58 ---

Tracing command cat pid 6118 tid 100070 td 0xc3ffe360
sched_switch(c3ffe360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3ff3ad4) at sleepq_switch+0xcd
sleepq_wait_sig(c3ff3ad4) at sleepq_wait_sig+0x1d
msleep(c3ff3ad4,c3ff3c44,14c,c091f7ed,0) at msleep+0x26e
pipe_read(c40253a8,e6637c64,c472a180,0,c3ffe360) at pipe_read+0x377
dofileread(c3ffe360,0,c40253a8,e6637c64,ffffffff,...) at dofileread+0x89
kern_readv(c3ffe360,0,e6637c64,8201000,1000,...) at kern_readv+0x36
read(c3ffe360,e6637d04) at read+0x45
syscall(3b,3b,3b,1,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28151117, esp = 0xbfbfecec, ebp = 0xbfbfee08 ---

Tracing command sh pid 6104 tid 100148 td 0xc4052000
sched_switch(c4052000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c498d234) at sleepq_switch+0xcd
sleepq_wait_sig(c498d234) at sleepq_wait_sig+0x1d
msleep(c498d234,c498d29c,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c4052000,ffffffff,e66acc2c,2,0) at kern_wait+0x7cf
wait4(c4052000,e66acd04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe2b8,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe26c, ebp = 0xbfbfe288 ---

Tracing command sort pid 6103 tid 100200 td 0xc473ea20
sched_switch(c473ea20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4129c60) at sleepq_switch+0xcd
sleepq_wait_sig(c4129c60) at sleepq_wait_sig+0x1d
msleep(c4129c60,c4129dd0,14c,c091f7ed,0) at msleep+0x26e
pipe_read(c3ff5708,e68d1c64,c472a180,0,c473ea20) at pipe_read+0x377
dofileread(c473ea20,0,c3ff5708,e68d1c64,ffffffff,...) at dofileread+0x89
kern_readv(c473ea20,0,e68d1c64,8201000,1000,...) at kern_readv+0x36
read(c473ea20,e68d1d04) at read+0x45
syscall(3b,3b,3b,100001,2816b7c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x2815b117, esp = 0xbfbfe9ec, ebp = 0xbfbfea08 ---

Tracing command sed pid 6101 tid 100181 td 0xc470fbd0
sched_switch(c470fbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4447000) at sleepq_switch+0xcd
sleepq_wait_sig(c4447000) at sleepq_wait_sig+0x1d
msleep(c4447000,c4447170,14c,c091f7ed,0) at msleep+0x26e
pipe_read(c40257e0,e689ec64,c472a180,0,c470fbd0) at pipe_read+0x377
dofileread(c470fbd0,0,c40257e0,e689ec64,ffffffff,...) at dofileread+0x89
kern_readv(c470fbd0,0,e689ec64,8230000,1000,...) at kern_readv+0x36
read(c470fbd0,e689ed04) at read+0x45
syscall(3b,3b,3b,0,281657c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28155117, esp = 0xbfbfeb4c, ebp = 0xbfbfeb68 ---

Tracing command xargs pid 6100 tid 100078 td 0xc4047bd0
sched_switch(c4047bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c44474a4) at sleepq_switch+0xcd
sleepq_wait_sig(c44474a4) at sleepq_wait_sig+0x1d
msleep(c44474a4,c4447614,14c,c091f7ed,0) at msleep+0x26e
pipe_read(c4420828,e668bc64,c472a180,0,c4047bd0) at pipe_read+0x377
dofileread(c4047bd0,0,c4420828,e668bc64,ffffffff,...) at dofileread+0x89
kern_readv(c4047bd0,0,e668bc64,8201000,1000,...) at kern_readv+0x36
read(c4047bd0,e668bd04) at read+0x45
syscall(3b,3b,3b,8240040,281627c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28152117, esp = 0xbfbfed0c, ebp = 0xbfbfed28 ---

Tracing command sh pid 6099 tid 100132 td 0xc411d510
sched_switch(c411d510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c45a9d38) at sleepq_switch+0xcd
sleepq_wait_sig(c45a9d38) at sleepq_wait_sig+0x1d
msleep(c45a9d38,c45a9da0,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c411d510,ffffffff,e66eec2c,2,0) at kern_wait+0x7cf
wait4(c411d510,e66eed04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe708,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe6bc, ebp = 0xbfbfe6d8 ---

Tracing command sh pid 6093 tid 100163 td 0xc3de8360
sched_switch(c3de8360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4550468) at sleepq_switch+0xcd
sleepq_wait_sig(c4550468) at sleepq_wait_sig+0x1d
msleep(c4550468,c45504d0,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c3de8360,ffffffff,e43b3c2c,2,0) at kern_wait+0x7cf
wait4(c3de8360,e43b3d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe9a8,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe95c, ebp = 0xbfbfe978 ---

Tracing command mail pid 6092 tid 100113 td 0xc3fe3510
sched_switch(c3fe3510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3ff2630) at sleepq_switch+0xcd
sleepq_wait_sig(c3ff2630) at sleepq_wait_sig+0x1d
msleep(c3ff2630,c3ff27a0,14c,c091f7ed,0) at msleep+0x26e
pipe_read(c4025948,e6610c64,c472a180,0,c3fe3510) at pipe_read+0x377
dofileread(c3fe3510,0,c4025948,e6610c64,ffffffff,...) at dofileread+0x89
kern_readv(c3fe3510,0,e6610c64,8201000,1000,...) at kern_readv+0x36
read(c3fe3510,e6610d04) at read+0x45
syscall(3b,3b,3b,0,281707c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28160117, esp = 0xbfbfdfbc, ebp = 0xbfbfdfd8 ---

Tracing command sh pid 6091 tid 100195 td 0xc473f360
sched_switch(c473f360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c473dd38) at sleepq_switch+0xcd
sleepq_wait_sig(c473dd38) at sleepq_wait_sig+0x1d
msleep(c473dd38,c473dda0,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c473f360,ffffffff,e68e0c2c,2,0) at kern_wait+0x7cf
wait4(c473f360,e68e0d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe808,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe7bc, ebp = 0xbfbfe7d8 ---

Tracing command sh pid 6090 tid 100168 td 0xc45526c0
sched_switch(c45526c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4553000) at sleepq_switch+0xcd
sleepq_wait_sig(c4553000) at sleepq_wait_sig+0x1d
msleep(c4553000,c4553068,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c45526c0,ffffffff,e6850c2c,2,0) at kern_wait+0x7cf
wait4(c45526c0,e6850d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe4c8,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe47c, ebp = 0xbfbfe498 ---

Tracing command sh pid 6084 tid 100202 td 0xc473e6c0
sched_switch(c473e6c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4740d38) at sleepq_switch+0xcd
sleepq_wait_sig(c4740d38) at sleepq_wait_sig+0x1d
msleep(c4740d38,c4740da0,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c473e6c0,ffffffff,e68cbc2c,2,0) at kern_wait+0x7cf
wait4(c473e6c0,e68cbd04) at wait4+0x2a
syscall(3b,3b,3b,bfbfeb58,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfeb0c, ebp = 0xbfbfeb28 ---

Tracing command sh pid 6083 tid 100111 td 0xc411e1b0
sched_switch(c411e1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c411cd38) at sleepq_switch+0xcd
sleepq_wait_sig(c411cd38) at sleepq_wait_sig+0x1d
msleep(c411cd38,c411cda0,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c411e1b0,ffffffff,e6703c2c,2,0) at kern_wait+0x7cf
wait4(c411e1b0,e6703d04) at wait4+0x2a
syscall(bfbf003b,bfbf003b,bfbf003b,bfbfe9f8,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe9ac, ebp = 0xbfbfe9c8 ---

Tracing command mail pid 5901 tid 100124 td 0xc41df510
sched_switch(c41df510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c412a630) at sleepq_switch+0xcd
sleepq_wait_sig(c412a630) at sleepq_wait_sig+0x1d
msleep(c412a630,c412a7a0,14c,c091f7ed,0) at msleep+0x26e
pipe_read(c3fe8360,e6760c64,c472a180,0,c41df510) at pipe_read+0x377
dofileread(c41df510,0,c3fe8360,e6760c64,ffffffff,...) at dofileread+0x89
kern_readv(c41df510,0,e6760c64,8201000,1000,...) at kern_readv+0x36
read(c41df510,e6760d04) at read+0x45
syscall(3b,3b,3b,0,281707c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28160117, esp = 0xbfbfdfdc, ebp = 0xbfbfdff8 ---

Tracing command sh pid 5900 tid 100180 td 0xc470fd80
sched_switch(c470fd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c470db04) at sleepq_switch+0xcd
sleepq_wait_sig(c470db04) at sleepq_wait_sig+0x1d
msleep(c470db04,c470db6c,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c470fd80,ffffffff,e68a1c2c,2,0) at kern_wait+0x7cf
wait4(c470fd80,e68a1d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe848,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe7fc, ebp = 0xbfbfe818 ---

Tracing command sh pid 5899 tid 100164 td 0xc3de81b0
sched_switch(c3de81b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c455069c) at sleepq_switch+0xcd
sleepq_wait_sig(c455069c) at sleepq_wait_sig+0x1d
msleep(c455069c,c4550704,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c3de81b0,ffffffff,e43b0c2c,2,0) at kern_wait+0x7cf
wait4(c3de81b0,e43b0d04) at wait4+0x2a
syscall(3b,828003b,bfbf003b,bfbfe508,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe4bc, ebp = 0xbfbfe4d8 ---

Tracing command sh pid 5889 tid 100127 td 0xc411dd80
sched_switch(c411dd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c45a9234) at sleepq_switch+0xcd
sleepq_wait_sig(c45a9234) at sleepq_wait_sig+0x1d
msleep(c45a9234,c45a929c,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c411dd80,ffffffff,e66fdc2c,2,0) at kern_wait+0x7cf
wait4(c411dd80,e66fdd04) at wait4+0x2a
syscall(3b,3b,3b,bfbfeb98,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfeb4c, ebp = 0xbfbfeb68 ---

Tracing command sh pid 5886 tid 100106 td 0xc411ea20
sched_switch(c411ea20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c411c234) at sleepq_switch+0xcd
sleepq_wait_sig(c411c234) at sleepq_wait_sig+0x1d
msleep(c411c234,c411c29c,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c411ea20,ffffffff,e6712c2c,2,0) at kern_wait+0x7cf
wait4(c411ea20,e6712d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfebe8,16fe,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfeb9c, ebp = 0xbfbfebb8 ---

Tracing command cron pid 5885 tid 100157 td 0xc4051000
sched_switch(c4051000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3ff2ad4) at sleepq_switch+0xcd
sleepq_wait_sig(c3ff2ad4) at sleepq_wait_sig+0x1d
msleep(c3ff2ad4,c3ff2c44,14c,c091f7ed,0) at msleep+0x26e
pipe_read(c47b42d0,e6691c64,c40f8b80,0,c4051000) at pipe_read+0x377
dofileread(c4051000,5,c47b42d0,e6691c64,ffffffff,...) at dofileread+0x89
kern_readv(c4051000,5,e6691c64,8221000,1000,...) at kern_readv+0x36
read(c4051000,e6691d04) at read+0x45
syscall(bfbf003b,ffff003b,bfbf003b,28188580,28188580,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28162117, esp = 0xbfbfe66c, ebp = 0xbfbfe688 ---

Tracing command bash pid 1421 tid 100169 td 0xc4552510
sched_switch(c4552510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3f78810) at sleepq_switch+0xcd
sleepq_wait_sig(c3f78810) at sleepq_wait_sig+0x1d
msleep(c3f78810,0,159,c0921127,0) at msleep+0x26e
ttysleep(c3f78800,c3f78810,159,c0921127,0,...) at ttysleep+0x21
ttread(c3f78800,e684dc64,0) at ttread+0x48f
ptsread(c3fa2000,e684dc64,0,c0a09568,0,...) at ptsread+0x2d
giant_read(c3fa2000,e684dc64,0,1,c3fa2000,...) at giant_read+0x2d
devfs_read_f(c418a318,e684dc64,c45bba80,0,c4552510) at devfs_read_f+0x62
dofileread(c4552510,0,c418a318,e684dc64,ffffffff,...) at dofileread+0x89
kern_readv(c4552510,0,e684dc64,bfbfe66f,1,...) at kern_readv+0x36
read(c4552510,e684dd04) at read+0x45
syscall(3b,3b,3b,0,bfbfe66f,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x2830e117, esp = 0xbfbfe64c, ebp = 0xbfbfe678 ---

Tracing command csh pid 1418 tid 100171 td 0xc45521b0
sched_switch(c45521b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c45536d0) at sleepq_switch+0xcd
sleepq_wait_sig(c45536d0) at sleepq_wait_sig+0x1d
msleep(c45536d0,c4553704,168,c08e4117,0) at msleep+0x26e
kern_sigsuspend(c45521b0,2,0,0,0,...) at kern_sigsuspend+0xa7
sigsuspend(c45521b0,e6847d04) at sigsuspend+0x33
syscall(bfbf003b,bfbf003b,bfbf003b,8140b08,bfbf1a70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x281626c3, esp = 0xbfbf1a5c, ebp = 0xbfbf1a88 ---

Tracing command su pid 1417 tid 100170 td 0xc4552360
sched_switch(c4552360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4553468) at sleepq_switch+0xcd
sleepq_wait_sig(c4553468) at sleepq_wait_sig+0x1d
msleep(c4553468,c45534d0,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c4552360,58a,e684ac2c,2,0) at kern_wait+0x7cf
wait4(c4552360,e684ad04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe740,58a,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f2047, esp = 0xbfbfe69c, ebp = 0xbfbfe6b8 ---

Tracing command run pid 1401 tid 100161 td 0xc3de86c0
sched_switch(c3de86c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4550000) at sleepq_switch+0xcd
sleepq_wait_sig(c4550000) at sleepq_wait_sig+0x1d
msleep(c4550000,c4550068,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c3de86c0,1853,e43b9c2c,0,0) at kern_wait+0x7cf
wait4(c3de86c0,e43b9d04) at wait4+0x2a
syscall(c088003b,3b,3b,2805188c,bfbfe9d8,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280df047, esp = 0xbfbfe3cc, ebp = 0xbfbfe3e8 ---

Tracing command run pid 1400 tid 100122 td 0xc41df870
sched_switch(c41df870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c41e169c) at sleepq_switch+0xcd
sleepq_wait_sig(c41e169c) at sleepq_wait_sig+0x1d
msleep(c41e169c,c41e1704,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c41df870,579,e6766c2c,0,0) at kern_wait+0x7cf
wait4(c41df870,e6766d04) at wait4+0x2a
syscall(3b,3b,3b,2805188c,bfbfe9d8,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280df047, esp = 0xbfbfe88c, ebp = 0xbfbfe8a8 ---

Tracing command run pid 1399 tid 100162 td 0xc3de8510
sched_switch(c3de8510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a09d64) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a09d64) at sleepq_timedwait_sig+0x1e
msleep(c0a09d64,0,15c,c091b792,3e9,...) at msleep+0x244
kern_nanosleep(c3de8510,e43b6c74,e43b6c6c) at kern_nanosleep+0xab
nanosleep(c3de8510,e43b6d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28137b2b, esp = 0xbfbfe88c, ebp = 0xbfbfe8b8 ---

Tracing command sh pid 1397 tid 100137 td 0xc45ac1b0
sched_switch(c45ac1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c45aa8d0) at sleepq_switch+0xcd
sleepq_wait_sig(c45aa8d0) at sleepq_wait_sig+0x1d
msleep(c45aa8d0,c45aa938,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c45ac1b0,ffffffff,e67d8c2c,2,0) at kern_wait+0x7cf
wait4(c45ac1b0,e67d8d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe898,575,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe84c, ebp = 0xbfbfe868 ---

Tracing command bash pid 1131 tid 100119 td 0xc41dfd80
sched_switch(c41dfd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c41e1000) at sleepq_switch+0xcd
sleepq_wait_sig(c41e1000) at sleepq_wait_sig+0x1d
msleep(c41e1000,c41e1068,15c,c091fb1d,0) at msleep+0x26e
kern_wait(c41dfd80,ffffffff,e676fc2c,6,0) at kern_wait+0x7cf
wait4(c41dfd80,e676fd04) at wait4+0x2a
syscall(839003b,3b,bfbf003b,575,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2829b047, esp = 0xbfbfe70c, ebp = 0xbfbfe728 ---

Tracing command csh pid 1128 tid 100134 td 0xc411d1b0
db> call doadump
Physical memory: 1007 MB
Dumping 184 MB: 169 153 137 121 105 89 73 57 41 25 9
Dump complete
= 0xf
db> reset
cpu_reset: Restarting BSP
cpu_reset_proxy: Stopped CPU 1

(kgdb) bt
#0  doadump () at pcpu.h:166
#1  0xc04762af in db_fncall (dummy1=0xe43ee9d8, dummy2=0x0, dummy3=0xc0a89d40, dummy4=0xe43ee9b4 "\035")
    at ../../../ddb/db_command.c:481
#2  0xc04760bb in db_command (last_cmdp=0xc09ef104, cmd_table=0x0) at ../../../ddb/db_command.c:396
#3  0xc0476176 in db_command_loop () at ../../../ddb/db_command.c:448
#4  0xc0477d75 in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5  0xc06b4044 in kdb_trap (type=0x3, code=0x0, tf=0xe43eeb44) at ../../../kern/subr_kdb.c:502
#6  0xc0895a94 in trap (frame=
      {tf_fs = 0xe43e0008, tf_es = 0xc06b0028, tf_ds = 0xc0910028, tf_edi = 0xc09343a4, tf_esi = 0x1, tf_ebp = 0xe43eeb84, tf_isp = 0xe43eeb70, tf_ebx = 0xe43eebb0, tf_edx = 0x0, tf_ecx = 0xc1833000, tf_eax = 0x12, tf_trapno = 0x3, tf_err = 0x0, tf_eip = 0xc06b3d6b, tf_cs = 0x20, tf_eflags = 0x296, tf_esp = 0xe43eeba4, tf_ss = 0xc0695a7f}) at ../../../i386/i386/trap.c:620
#7  0xc088091a in calltrap () at ../../../i386/i386/exception.s:138
#8  0xc06b3d6b in kdb_enter (msg=0x12 <Address 0x12 out of bounds>) at cpufunc.h:60
#9  0xc0695a7f in panic (fmt=0xc09343a4 "softdep_deallocate_dependencies: dangling deps") at ../../../kern/kern_shutdown.c:559
#10 0xc07d53be in softdep_deallocate_dependencies (bp=0x12) at ../../../ufs/ffs/ffs_softdep.c:6247
#11 0xc06e3ea1 in brelse (bp=0xd7d22d70) at buf.h:440
#12 0xc06f3832 in flushbuflist (bufv=0x12, flags=0x0, bo=0xc4296790, slpflag=0x0, slptimeo=0x0) at ../../../kern/vfs_subr.c:1152
#13 0xc06f3433 in bufobj_invalbuf (bo=0xc4296790, flags=0x0, td=0xc3f80510, slpflag=0x0, slptimeo=0x0) at ../../../kern/vfs_subr.c:1032
#14 0xc06f3626 in vinvalbuf (vp=0xc4296790, flags=0x0, td=0xc3f80510, slpflag=0x0, slptimeo=0x0) at ../../../kern/vfs_subr.c:1088
#15 0xc06f5be1 in vgonel (vp=0xc4296690) at ../../../kern/vfs_subr.c:2436
#16 0xc06f26c2 in vlrureclaim (mp=0xc3fc3b50) at ../../../kern/vfs_subr.c:617
#17 0xc06f2b43 in vnlru_proc () at ../../../kern/vfs_subr.c:733
#18 0xc06804f8 in fork_exit (callout=0xc06f29b0 <vnlru_proc>, arg=0x0, frame=0xe43eed38) at ../../../kern/kern_fork.c:818
#19 0xc088097c in fork_trampoline () at ../../../i386/i386/exception.s:199
(kgdb) p *(struct bufv *)0xc42967a4
$12 = {bv_hd = {tqh_first = 0xd7d22d70, tqh_last = 0xd7d22da8}, bv_root = 0xd7d22d70, bv_cnt = 0x1}
(kgdb) p *(struct buf *)0xd7d22d70
$13 = {b_bufobj = 0xc4296790, b_bcount = 0x800, b_caller1 = 0x0, b_data = 0xd9398000 "1 0\n", b_error = 0x1c, b_iocmd = 0x2,
  b_ioflags = 0x0, b_iooffset = 0x16eda7f800, b_resid = 0x0, b_iodone = 0, b_blkno = 0xb76d3fc, b_offset = 0x0, b_bobufs = {
    tqe_next = 0x0, tqe_prev = 0xc42967a4}, b_left = 0x0, b_right = 0x0, b_vflags = 0x1, b_freelist = {tqe_next = 0x0,
    tqe_prev = 0xd7f00e54}, b_qindex = 0x2, b_flags = 0xa04222a0, b_xflags = 0x1, b_lock = {lk_interlock = 0xc0a08a10,
    lk_flags = 0x40000, lk_sharecount = 0x0, lk_waitcount = 0x0, lk_exclusivecount = 0x1, lk_prio = 0x50,
    lk_wmesg = 0xc0924d49 "flushbuf", lk_timo = 0x0, lk_lockholder = 0xc3f80510, lk_newlock = 0x0, lk_stack = {depth = 0x9, pcs = {
        0xc0689b68, 0xc06f3708, 0xc06f3433, 0xc06f3626, 0xc06f5be1, 0xc06f26c2, 0xc06f2b43, 0xc06804f8, 0xc088097c, 0x0, 0x0, 0x0, 0x0,
        0x0, 0x0, 0x0, 0x0, 0x0}}}, b_bufsize = 0x800, b_runningbufspace = 0x0, b_kvabase = 0xd9398000 "1 0\n", b_kvasize = 0x4000,
  b_lblkno = 0x0, b_vp = 0xc4296690, b_dirtyoff = 0x0, b_dirtyend = 0x0, b_rcred = 0x0, b_wcred = 0x0, b_saveaddr = 0xd9398000,
  b_pager = {pg_reqpage = 0x0}, b_cluster = {cluster_head = {tqh_first = 0xd7d7320c, tqh_last = 0xd7eef680}, cluster_entry = {
      tqe_next = 0xd7d7320c, tqe_prev = 0xd7eef680}}, b_pages = {0xc1ab5ab0, 0x0 <repeats 31 times>}, b_npages = 0x1, b_dep = {
    lh_first = 0xc4649580}, b_fsprivate1 = 0x0, b_fsprivate2 = 0x0, b_fsprivate3 = 0x0, b_pin_count = 0x0}
(kgdb) p *(struct bufobj *)0xc4296790
$14 = {bo_mtx = 0xc4296758, bo_clean = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc4296794}, bv_root = 0x0, bv_cnt = 0x0}, bo_dirty = {
    bv_hd = {tqh_first = 0xd7d22d70, tqh_last = 0xd7d22da8}, bv_root = 0xd7d22d70, bv_cnt = 0x1}, bo_numoutput = 0x0, bo_flag = 0x1,
  bo_ops = 0xc09cb684, bo_bsize = 0x4000, bo_object = 0xc40e8708, bo_synclist = {le_next = 0xc41d0a30, le_prev = 0xc3e119d4},
  bo_private = 0xc4296690, __bo_vnode = 0xc4296690}