GENERIC HEAD from May 14 06:55 UTC, vmcore.62
watchdogd -t 3600 fired

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Sun May 14 09:48:46 CEST 2006
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/QUOTA
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I  OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1041444864 (993 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
pid 8346: corrected slot count (8->1)
free inode /tmp/27825 had -2068624 blocks
free inode /tmp/27826 had -2068624 blocks
handle_workitem_freeblocks: block count
handle_workitem_freeblocks: block count
free inode /tmp/27825 had -2068624 blocks
free inode /tmp/27826 had -2068624 blocks
handle_workitem_freeblocks: block count
handle_workitem_freeblocks: block count
free inode /tmp/27825 had -2068624 blocks
free inode /tmp/27826 had -2068624 blocks
handle_workitem_freeblocks: block count
~KDB: enter: Line break on console
[thread pid 13 tid 100003 ]
Stopped at      kdb_enter+0x2b: nop
db> show allpcpu
Current CPU: 0

cpuid        = 0
curthread    = 0xc342fa20: pid 13 "idle: cpu0"
curpcb       = 0xe1f1ed90
fpcurthread  = none
idlethread   = 0xc342fa20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc342f870: pid 12 "idle: cpu1"
curpcb       = 0xe1f1bd90
fpcurthread  = none
idlethread   = 0xc342f870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc3872bd0: pid 20689 "swap"
curpcb       = 0xe5f09d90
fpcurthread  = none
idlethread   = 0xc342f6c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc342f510: pid 10 "idle: cpu3"
curpcb       = 0xe1f15d90
fpcurthread  = none
idlethread   = 0xc342f510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> kill 9 20689 # Fix for the usual swap livelock problem
db> c
calcru: runtime went backwards from 2880493 usec to 1539972 usec for pid 20711 (creat)
calcru: runtime went backwards from 4924801 usec to 2363599 usec for pid 20710 (creat)
calcru: runtime went backwards from 3395541 usec to 1648194 usec for pid 20709 (creat)
calcru: runtime went backwards from 3876932 usec to 1922581 usec for pid 20708 (creat)
calcru: runtime went backwards from 2929864 usec to 1414604 usec for pid 20707 (creat)
calcru: runtime went backwards from 3012048 usec to 1532250 usec for pid 20706 (creat)
calcru: runtime went backwards from 2901498 usec to 1426960 usec for pid 20705 (creat)
calcru: runtime went backwards from 2940035 usec to 1416771 usec for pid 20704 (creat)
calcru: runtime went backwards from 2702771 usec to 1313267 usec for pid 20703 (creat)
calcru: runtime went backwards from 2828918 usec to 1430087 usec for pid 20702 (creat)
calcru: runtime went backwards from 3754862 usec to 1881567 usec for pid 20701 (creat)
calcru: runtime went backwards from 23654 usec to 10588 usec for pid 20700 (creat)
calcru: runtime went backwards from 20691 usec to 9261 usec for pid 20666 (creat)
calcru: runtime went backwards from 2655 usec to 1188 usec for pid 17624 (md4)
calcru: runtime went backwards from 3161 usec to 1414 usec for pid 17613 (md3)
calcru: runtime went backwards from 5691 usec to 2547 usec for pid 17604 (md2)
calcru: runtime went backwards from 1696 usec to 759 usec for pid 17556 (md1)
calcru: runtime went backwards from 48594457 usec to 21751650 usec for pid 10497 (tail)
calcru: runtime went backwards from 2085630 usec to 937285 usec for pid 10400 (run)
calcru: runtime went backwards from 35257172754 usec to 15781420174 usec for pid 10400 (run)
calcru: runtime went backwards from 385149 usec to 172438 usec for pid 10398 (run)
calcru: runtime went backwards from 424127673 usec to 190907328 usec for pid 8054 (top)
calcru: runtime went backwards from 10207229 usec to 4568840 usec for pid 4468 (sshd)
calcru: runtime went backwards from 41028759 usec to 18366763 usec for pid 1079 (sshd)
calcru: runtime went backwards from 72929068 usec to 32644288 usec for pid 823 (watchdogd)
calcru: runtime went backwards from 14138289 usec to 6328415 usec for pid 772 (cron)
calcru: runtime went backwards from 68477248 usec to 30650982 usec for pid 772 (cron)
calcru: runtime went backwards from 834621 usec to 373583 usec for pid 766 (sendmail)
calcru: runtime went backwards from 1973942 usec to 883553 usec for pid 766 (sendmail)
calcru: runtime went backwards from 82662896 usec to 37000595 usec for pid 762 (sendmail)
calcru: runtime went backwards from 1353328 usec to 605761 usec for pid 762 (sendmail)
calcru: runtime went backwards from 11560451 usec to 5174614 usec for pid 734 (ntpd)
calcru: runtime went backwards from 21754823 usec to 9737638 usec for pid 614 (rpcbind)
calcru: runtime went backwards from 48978472 usec to 21923168 usec for pid 590 (syslogd)
calcru: runtime went backwards from 14117658 usec to 6319376 usec for pid 46 (schedcpu)
calcru: runtime went backwards from 31 usec to 13 usec for pid 45 (nfsiod 3)
calcru: runtime went backwards from 29 usec to 13 usec for pid 44 (nfsiod 2)
calcru: runtime went backwards from 42 usec to 19 usec for pid 43 (nfsiod 1)
calcru: runtime went backwards from 48 usec to 21 usec for pid 42 (nfsiod 0)
calcru: runtime went backwards from 1162225666 usec to 520224362 usec for pid 41 (softdepflush)
calcru: runtime went backwards from 109329213 usec to 48937010 usec for pid 40 (syncer)
calcru: runtime went backwards from 82110757 usec to 36753792 usec for pid 39 (vnlru)
calcru: runtime went backwards from 421201780 usec to 188533414 usec for pid 38 (bufdaemon)
calcru: runtime went backwards from 261122871 usec to 116880755 usec for pid 37 (pagezero)
calcru: runtime went backwards from 82250084 usec to 36815817 usec for pid 36 (vmdaemon)
calcru: runtime went backwards from 4434098462 usec to 1984739120 usec for pid 35 (pagedaemon)
calcru: runtime went backwards from 555233 usec to 248539 usec for pid 33 (fdc0)
calcru: runtime went backwards from 193354955 usec to 86547661 usec for pid 28 (irq14: ata0)
calcru: runtime went backwards from 3422700 usec to 1532068 usec for pid 27 (irq17: fxp0)
calcru: runtime went backwards from 677547 usec to 303275 usec for pid 25 (usb0)
calcru: runtime went backwards from 37510 usec to 16790 usec for pid 21 (swi6: task queue)
calcru: runtime went backwards from 15985794 usec to 7155452 usec for pid 17 (yarrow)
calcru: runtime went backwards from 443622258 usec to 198569962 usec for pid 4 (g_down)
calcru: runtime went backwards from 1729218278 usec to 774013746 usec for pid 3 (g_up)
calcru: runtime went backwards from 10259830 usec to 4592523 usec for pid 2 (g_event)
calcru: runtime went backwards from 2077607297 usec to 929969075 usec for pid 15 (swi4: clock sio)
calcru: runtime went backwards from 243344089 usec to 108922885 usec for pid 14 (swi1: net)
calcru: runtime went backwards from 9752484771 usec to 4365293875 usec for pid 13 (idle: cpu0)
calcru: runtime went backwards from 9436623259 usec to 4223911703 usec for pid 12 (idle: cpu1)
calcru: runtime went backwards from 6565015310 usec to 2938556308 usec for pid 11 (idle: cpu2)
calcru: runtime went backwards from 9730016688 usec to 4355236831 usec for pid 10 (idle: cpu3)
calcru: runtime went backwards from 1499190 usec to 671049 usec for pid 1 (init)
calcru: runtime went backwards from 50936260858 usec to 22799487656 usec for pid 1 (init)
calcru: runtime went backwards from 18941386 usec to 8478321 usec for pid 0 (swapper)
calcru: runtime went backwards from 23180100 usec to 10608667 usec for pid 20667 (mkdir)
calcru: runtime went backwards from 23146704 usec to 10584861 usec for pid 20667 (mkdir)
calcru: runtime went backwards from 21257602 usec to 9515078 usec for pid 20681 (mmap)
calcru: runtime went backwards from 21198741 usec to 9488731 usec for pid 20681 (mmap)
calcru: runtime went backwards from 104623758 usec to 46830458 usec for pid 20668 (thr1)
calcru: runtime went backwards from 104566822 usec to 46804974 usec for pid 20668 (thr1)
calcru: runtime went backwards from 51458 usec to 23033 usec for pid 20679 (thr2)
calcru: runtime went backwards from 25755 usec to 11528 usec for pid 20674 (pty)
calcru: runtime went backwards from 27370 usec to 12251 usec for pid 20675 (mkfifo)
calcru: runtime went backwards from 26996 usec to 12083 usec for pid 20676 (shm)
calcru: runtime went backwards from 24929 usec to 11158 usec for pid 20672 (tcp)
calcru: runtime went backwards from 24734 usec to 11071 usec for pid 20680 (badcode)
calcru: runtime went backwards from 26659 usec to 11932 usec for pid 20669 (rw)
calcru: runtime went backwards from 26838 usec to 12013 usec for pid 20678 (symlink)
calcru: runtime went backwards from 23085 usec to 10333 usec for pid 20682 (socket)
calcru: runtime went backwards from 28893 usec to 12932 usec for pid 20677 (lockf)
calcru: runtime went backwards from 26785 usec to 11989 usec for pid 20673 (udp)
calcru: runtime went backwards from 26708 usec to 11954 usec for pid 20670 (syscall)
calcru: runtime went backwards from 29009 usec to 12985 usec for pid 20671 (sysctl)
calcru: runtime went backwards from 6740289 usec to 3021227 usec for pid 1070 (sshd)
free inode /tmp/27826 had -2068624 blocks
calcru: runtime went backwards from 404985 usec to 186731 usec for pid 673 (mountd)
calcru: runtime went backwards from 980781 usec to 439276 usec for pid 1071 (bash)
calcru: runtime went backwards from 3865639857 usec to 1730411931 usec for pid 1071 (bash)
handle_workitem_freeblocks: block count
handle_workitem_freeblocks: block count
free inode /tmp/27825 had -2068624 blocks
free inode /tmp/27826 had -2068624 blocks
handle_workitem_freeblocks: block count
handle_workitem_freeblocks: block count
free inode /tmp/27825 had -2068624 blocks
handle_workitem_freeblocks: block count
free inode /tmp/27825 had -2068624 blocks
free inode /tmp/27826 had -2068624 blocks
interrupt                   total
irq4: sio0                          8999
irq6: fdc0                             3
irq14: ata0                      5946638
irq16: uhci0                           1
irq17: fxp0                        42114
cpu0: timer                     91122296
cpu1: timer                     91127689
cpu2: timer                     91136484
cpu3: timer                     91120903
Total                   370505471
KDB: stack backtrace:
kdb_backtrace(c0903a9b,161576ff,0,c09d3849,c09d2b60) at kdb_backtrace+0x29
watchdog_fire(c0a744e0,c34f5618,e1f1ec90,c0875a29,0) at watchdog_fire+0x9d
hardclock(0,c0bab245) at hardclock+0xd9
lapic_handle_timer(c3420008,c0670028,28,c34f5600,c34f5618) at lapic_handle_timer+0x71
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc0bab245, esp = 0xe1f1ecd8, ebp = 0xe1f1ecd8 ---
acpi_cpu_c1(c09f3650,1,e1f1ecf8,1,c067d320) at acpi_cpu_c1+0x5
acpi_cpu_idle(e1f1ed0c,c067d37d,c342e69c,e1f1ed24,c067d198) at acpi_cpu_idle+0x15a
cpu_idle(c342e69c,e1f1ed24,c067d198,0,e1f1ed38) at cpu_idle+0x28
idle_proc(0,e1f1ed38) at idle_proc+0x5d
fork_exit(c067d320,0,e1f1ed38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe1f1ed6c, ebp = 0 ---
KDB: enter: watchdog timeout
[thread pid 13 tid 100003 ]
Stopped at      kdb_enter+0x2b: nop
db> show allpcpu
Current CPU: 0

cpuid        = 0
curthread    = 0xc342fa20: pid 13 "idle: cpu0"
curpcb       = 0xe1f1ed90
fpcurthread  = none
idlethread   = 0xc342fa20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc342f870: pid 12 "idle: cpu1"
curpcb       = 0xe1f1bd90
fpcurthread  = none
idlethread   = 0xc342f870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc68ba1b0: pid 32798 "thr2"
curpcb       = 0xe76a3d90
fpcurthread  = none
idlethread   = 0xc342f6c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc342f510: pid 10 "idle: cpu3"
curpcb       = 0xe1f15d90
fpcurthread  = none
idlethread   = 0xc342f510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> show alllocks
Process 32802 (thr2) thread 0xc3cfa510 (100111)
exclusive sleep mutex vm object (standard object) r = 0 (0xc4924bb8) locked @ vm/vm_fault.c:297
exclusive sx user map r = 0 (0xc492d884) locked @ vm/vm_map.c:2993
Process 32799 (thr2) thread 0xc62e21b0 (100212)
exclusive sleep mutex vm object (standard object) r = 0 (0xc519e000) locked @ kern/vfs_bio.c:3393
exclusive sleep mutex Giant r = 0 (0xc09f3668) locked @ kern/kern_lockf.c:110
Process 32791 (thr1) thread 0xc35db1b0 (100241)
exclusive sleep mutex vm object (standard object) r = 0 (0xc5434960) locked @ vm/vm_fault.c:297
exclusive sx user map r = 0 (0xc383fdac) locked @ vm/vm_map.c:2993
Process 32780 (thr1) thread 0xc493d000 (100661)
exclusive sx user map r = 0 (0xc3d24ca4) locked @ vm/vm_map.c:1456
Process 32754 (syscall) thread 0xc62e3000 (100371)
exclusive sleep mutex vm page queue mutex r = 0 (0xc0a4da18) locked @ i386/i386/pmap.c:1966
exclusive sx user map r = 0 (0xc393b674) locked @ vm/vm_unix.c:88
Process 3 (g_up) thread 0xc3430a20 (100015)
exclusive sleep mutex vm object (standard object) r = 0 (0xc38b0c30) locked @ kern/vfs_bio.c:3182
db> where 32802
Tracing pid 32802 tid 100106 td 0xc6296a20
sched_switch(c6296a20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c35dd394) at sleepq_switch+0xcd
sleepq_timedwait_sig(c35dd394,100,c09055b2,1a6,8) at sleepq_timedwait_sig+0x1e
msleep(c35dd394,c3c1329c,168,c09055dc,ea61) at msleep+0x244
kse_release(c6296a20,e6227d04,c0680d93,c3bc8900,c46113d8) at kse_release+0x1eb
syscall(bfbf003b,bfbf003b,bfbf003b,28365f48,0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (383, FreeBSD ELF32, kse_release), eip = 0x280a871f, esp = 0x2836af1c, ebp = 0x2836af58 ---
db> where 32799
Tracing pid 32799 tid 100255 td 0xc3d63360
sched_switch(c3d63360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3815874) at sleepq_switch+0xcd
sleepq_timedwait_sig(c3815874,100,c09055b2,1a6,8) at sleepq_timedwait_sig+0x1e
msleep(c3815874,c3966da0,168,c09055dc,ea61) at msleep+0x244
kse_release(c3d63360,e614fd04,c0680d93,c3bc8900,c4727738) at kse_release+0x1eb
syscall(3b,2837003b,bfbf003b,28365f48,0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (383, FreeBSD ELF32, kse_release), eip = 0x280a871f, esp = 0x2836af1c, ebp = 0x2836af58 ---
db> where 32791
Tracing pid 32791 tid 100098 td 0xc54b7a20
sched_switch(c54b7a20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c383fdd0,c09f3650,0,c090ad65,211) at sleepq_switch+0xcd
sleepq_wait(c383fdd0,c0908579,e7175b8c,c09f2e40,c383fdd0) at sleepq_wait+0x46
cv_wait_unlock(c383fdd0,c09f2e40,bb1,e7175bb0,c383fdac) at cv_wait_unlock+0x17e
cv_wait(c383fdd0,c09f2e40,e7175c48,2,c383fd68) at cv_wait+0x36
_sx_xlock(c383fdac,c0923244,bb1) at _sx_xlock+0x68
_vm_map_lock_read(c383fd68,c0923244,bb1,2689759,e7175c48) at _vm_map_lock_read+0x33
vm_map_lookup(e7175c10,28299f34,2,e7175c14,e7175c4c,e7175c18,e7175c23,e7175c24) at vm_map_lookup+0x28
umtx_key_get(c54b7a20,28299f34,e7175c48) at umtx_key_get+0x3d
kern_umtx_wake(c54b7a20,28299f34,7fffffff,28299f34,1) at kern_umtx_wake+0x16
thr_exit(c54b7a20,e7175d04,c090976f,26e,c0a7450c) at thr_exit+0x29
syscall(3b,2831003b,bfbf003b,0,28299f34) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (431, FreeBSD ELF32, thr_exit), eip = 0x280ddbcf, esp = 0xb3f44edc, ebp = 0xb3f44f98 ---
db> where 32780
Tracing pid 32780 tid 100661 td 0xc493d000
sched_switch(c493d000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a4da18,c62e3000,0,c0a4da18,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c0a4da18,c493d000,0,c092e353,7ae) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c0a4da18,0,c092e353,7ae) at _mtx_lock_flags+0x95
pmap_remove(c3d24d08,28314000,28316000,e607ac38,c07e12d7) at pmap_remove+0x36
pmap_protect(c3d24d08,28314000,28316000,0) at pmap_protect+0x1e
vm_map_protect(c3d24c60,28314000,28316000,0,0) at vm_map_protect+0x145
mprotect(c493d000,e607ad04,c3d7e69c,c,c493d000) at mprotect+0x58
syscall(3b,2831003b,bfbf003b,198,2831485c) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (74, FreeBSD ELF32, mprotect), eip = 0x280df1cf, esp = 0xbfbfe1ec, ebp = 0xbfbfe208 ---
db> where 32754
Tracing pid 32754 tid 100371 td 0xc62e3000
sched_switch(c62e3000,c35da870,6) at sched_switch+0x183
mi_switch(6,c35da870,c35daa00,c35da870,e62eda84) at mi_switch+0x280
maybe_preempt(c35da870) at maybe_preempt+0x169
sched_add(c35da870,4,c3482700,c35da870,c35d70a0) at sched_add+0x15a
setrunqueue(c35da870,4) at setrunqueue+0xab
intr_event_schedule_thread(c3482700) at intr_event_schedule_thread+0xbb
intr_execute_handlers(c342422c,e62edb08,e,e62edb70,c086eed3) at intr_execute_handlers+0x118
lapic_handle_intr(34) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc06b75a5, esp = 0xe62edb48, ebp = 0xe62edb70 ---
witness_checkorder(c09f9874,9,c090bf0e,39a) at witness_checkorder+0x2d
_mtx_lock_spin_flags(c09f9874,0,c090bf0e,39a,c0a4da18) at _mtx_lock_spin_flags+0x69
witness_checkorder(c393b6d8,9,c092e35c,7b0) at witness_checkorder+0x262
_mtx_lock_flags(c393b6d8,0,c092e353,7b0,0) at _mtx_lock_flags+0x6b
pmap_remove(c393b6d8,138f4000,1fc64000) at pmap_remove+0x59
vm_map_delete(c393b630,138f4000,1fc64000) at vm_map_delete+0x159
obreak(c62e3000,e62edd04,c1cf6800,7ffbc02c,369e99) at obreak+0x1a9
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (17, FreeBSD ELF32, obreak), eip = 0x280b4419, esp = 0xbfbfe770, ebp = 0xbfbfe7b8 ---
db> where 3
Tracing pid 3 tid 100015 td 0xc3430a20
sched_switch(c3430a20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a4da18,c62e3000,0,c0a4da18,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c0a4da18,c3430a20,0,c0910137,c81) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c0a4da18,0,c0910137,c81,d764bb70) at _mtx_lock_flags+0x95
bufdone_finish(d764bb70) at bufdone_finish+0x10e
bufdone(d764bb70) at bufdone+0xaa
g_vfs_done(c78ef528) at g_vfs_done+0x8a
biodone(c78ef528) at biodone+0x58
g_io_schedule_up(c3430a20) at g_io_schedule_up+0xe6
g_up_procbody(0,e1f39d38) at g_up_procbody+0x5a
fork_exit(c065ba84,0,e1f39d38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe1f39d6c, ebp = 0 ---
db> shpw lockedvnods
No such command
db> show lockedvnods
Locked vnodes

0xc572e514: tag ufs, type VREG
    usecount 1, writecount 1, refcount 22 mountedhere 0
    flags ()
    v_object 0xc519e000 ref 0 pages 80
     lock type ufs: EXCL (count 1) by thread 0xc62e21b0 (pid 32799)
        ino 141445, on dev ad0s1e

0xc46dd208: tag ufs, type VLNK
    usecount 0, writecount 0, refcount 1 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc37a66c0 (pid 41)
        ino 210354, on dev ad0s1e
db> where 32799
Tracing pid 32799 tid 100255 td 0xc3d63360
sched_switch(c3d63360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3815874) at sleepq_switch+0xcd
sleepq_timedwait_sig(c3815874,100,c09055b2,1a6,8) at sleepq_timedwait_sig+0x1e
msleep(c3815874,c3966da0,168,c09055dc,ea61) at msleep+0x244
kse_release(c3d63360,e614fd04,c0680d93,c3bc8900,c4727738) at kse_release+0x1eb
syscall(3b,2837003b,bfbf003b,28365f48,0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (383, FreeBSD ELF32, kse_release), eip = 0x280a871f, esp = 0x2836af1c, ebp = 0x2836af58 ---
db> where 41
Tracing pid 41 tid 100047 td 0xc37a66c0
sched_switch(c37a66c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d764bb70,c09f3650,0,c090ad65,211) at sleepq_switch+0xcd
sleepq_wait(d764bb70,0,c0910140,eb5,c0911d69) at sleepq_wait+0x46
msleep(d764bb70,c0a40488,4c,c09108ba,0) at msleep+0x27d
bwait(d764bb70,4c,c09108ba) at bwait+0x47
bufwait(d764bb70,d764bb70,0,e3c399ac,c07c83b2) at bufwait+0x2d
bufwrite(d764bb70,0,0,e3c399fc,c07b3521) at bufwrite+0x168
ffs_bufwrite(d764bb70) at ffs_bufwrite+0x282
ffs_update(c46dd208,1,c06a599f,c09f3650,2) at ffs_update+0x335
ffs_truncate(c46dd208,0,0,c00,0) at ffs_truncate+0x3f2
ufs_inactive(e3c39c14) at ufs_inactive+0x1a0
VOP_INACTIVE_APV(c09c23a0,e3c39c14) at VOP_INACTIVE_APV+0x7e
vinactive(c46dd208,c37a66c0) at vinactive+0x72
vput(c46dd208,c0a4d1f8,0,c091fa9a,df7) at vput+0x1b3
handle_workitem_remove(c42b6820,0) at handle_workitem_remove+0x10f
process_worklist_item(c3902ca8,0) at process_worklist_item+0x183
softdep_process_worklist(c3902ca8,0,0,c37a5b04,c07bb0b4) at softdep_process_worklist+0x7c
softdep_flush(0,e3c39d38) at softdep_flush+0x132
fork_exit(c07bb0b4,0,e3c39d38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe3c39d6c, ebp = 0 ---
db> ps
 pid   uid  ppid  pgrp  state   wmesg      wchan    cmd
32806 1001 32796 25354  S+      nanslp   0xc09f3f24 thr2
32805 1001 32796 25354  S+      nanslp   0xc09f3f24 thr2
32804 1001 32796 25354  S+      nanslp   0xc09f3f24 thr2
32803 1001 32796 25354  S+      nanslp   0xc09f3f24 thr2
32802 1001 32796 25354  LL+     (threaded)          thr2
    100106              S       kserel   0xc35dd394 thr2
    100105              S       kserel   0xc35dd394 thr2
    100093              S       kserel   0xc35dd394 thr2
    100067              S       ksesigwa 0xc3c132b8 thr2
    100111              L      *vm page  0xc556ccc0 thr2
32801 1001 32796 25354  S+      nanslp   0xc09f3f24 thr2
32800 1001 32796 25354  S+      nanslp   0xc09f3f24 thr2
32799 1001 32796 25354  T+      (threaded)          thr2
    100255              Ss      kserel   0xc3815874 thr2
    100249              Ss      kserel   0xc3815874 thr2
    100244              Ss      kserel   0xc3815874 thr2
    100240              Ss      ksesigwa 0xc3966dbc thr2
    100212              L      *vm page  0xc556ccc0 thr2
32798 1001 32796 25354  R+      (threaded)          thr2
    100088              Run     CPU 2               thr2
    100079              S       ksesigwa 0xc57ba720 thr2
32797 1001 32796 25354  S+      nanslp   0xc09f3f24 thr2
32796 1001 32712 25354  S+      wait     0xc3c14000 thr2
32795 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32794 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32793 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32792 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32791 1001 32752 25354  LL+     (threaded)          thr1
    100098              D       user map 0xc383fdd0 thr1
    100363              D       user map 0xc383fdd0 thr1
    100241              L      *vm page  0xc556ccc0 thr1
32790 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32789 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32788 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32787 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32786 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32785 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32784 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32783 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32782 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32781 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32780 1001 32752 25354  L+      (threaded)          thr1
    100661              L      *vm page  0xc556ccc0 thr1
32779 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32778 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32777 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32776 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32775 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32774 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32773 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32772 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32771 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32770 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32769 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32768 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32767 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32766 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32765 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32764 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32763 1001 32753 25354  S+      nanslp   0xc09f3f24 symlink
32762 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32761 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32760 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32759 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32758 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32757 1001 32752 25354  S+      nanslp   0xc09f3f24 thr1
32756 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32755 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32754 1001 32726 25354  R+                          syscall
32753 1001 32711 25354  S+      wait     0xc3870468 symlink
32752 1001 32701 25354  S+      wait     0xc392e468 thr1
32751 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32750 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32749 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32748 1001 32726 25354  L+     *Giant    0xc6945bc0 syscall
32747 1001 32718 25354  S+      nanslp   0xc09f3f24 creat
32746 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32745 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32744 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32743 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32742 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32741 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32740 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32739 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32738 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32737 1001 32726 25354  S+      nanslp   0xc09f3f24 syscall
32736 1001 32726 25354  L+     *Giant    0xc6945bc0 syscall
32735 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32734 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32733 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32732 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32731 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32730 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32729 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32728 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32727 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32726 1001 32703 25354  S+      wait     0xc49a5d38 syscall
32725 1001 32716 25354  S+      nanslp   0xc09f3f24 mkdir
32724 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32723 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32722 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32721 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32720 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32719 1001 32717 25354  S+      nanslp   0xc09f3f24 rw
32718 1001 32699 25354  S+      wait     0xc392ed38 creat
32717 1001 32702 25354  S+      wait     0xc7a47b04 rw
32716 1001 32700 25354  S+      wait     0xc37a78d0 mkdir
32712 1001 25361 25354  S+      nanslp   0xc09f3f24 thr2
32711 1001 25361 25354  S+      nanslp   0xc09f3f24 symlink
32703 1001 25361 25354  S+      nanslp   0xc09f3f24 syscall
32702 1001 25361 25354  S+      nanslp   0xc09f3f24 rw
32701 1001 25361 25354  S+      nanslp   0xc09f3f24 initial thread
32700 1001 25361 25354  S+      nanslp   0xc09f3f24 mkdir
32699 1001 25361 25354  S+      nanslp   0xc09f3f24 creat
29486    0     0     0  SL      mdwait   0xc4567800 [md4]
29479    0     0     0  SL      mdwait   0xc6420000 [md3]
29472    0     0     0  SL      mdwait   0xc46af000 [md2]
29465    0     0     0  SL      mdwait   0xc6e97000 [md1]
25361 1001 25360 25354  S+      wait     0xc3d61000 run
25360 1001 25359 25354  SW+     wait     0xc3d6f468 run
25359 1001 25354 25354  S+      nanslp   0xc09f3f24 run
25354 1001  1071 25354  SW+     wait     0xc7a4869c sh
10497 1001  4469 10497  S+      kqread   0xc5491480 tail
 8054 1001  1080  8054  S+      select   0xc0a3ffbc top
 4469 1001  4468  4469  SWs+    wait     0xc493bb04 bash
 4468 1001  4466  4466  S       select   0xc0a3ffbc sshd
 4466    0   755  4466  SWs     sbwait   0xc437f334 sshd
 1080 1001  1079  1080  SWs+    wait     0xc382f8d0 bash
 1079 1001  1077  1077  S       select   0xc0a3ffbc sshd
 1077    0   755  1077  SWs     sbwait   0xc3956afc sshd
 1071 1001  1070  1071  SWs+    wait     0xc381269c bash
 1070 1001  1068  1068  S       select   0xc0a3ffbc sshd
 1068    0   755  1068  SWs     sbwait   0xc39405cc sshd
  897    0     1   897  SWs+    ttyin    0xc3606810 getty
  896    0     1   896  SWs+    ttyin    0xc3604c10 getty
  895    0     1   895  SWs+    ttyin    0xc3604810 getty
  894    0     1   894  SWs+    ttyin    0xc3605810 getty
  893    0     1   893  SWs+    ttyin    0xc3605410 getty
  892    0     1   892  SWs+    ttyin    0xc3601c10 getty
  891    0     1   891  SWs+    ttyin    0xc3604010 getty
  890    0     1   890  SWs+    ttyin    0xc3604410 getty
  871    0     1   871  SWs     select   0xc0a3ffbc inetd
  842    0     1   842  SWs     select   0xc0a3ffbc moused
  823    0     1   823  Ss      nanslp   0xc09f3f24 watchdogd
  772    0     1   772  Ss      nanslp   0xc09f3f24 cron
  766   25     1   766  SWs     pause    0xc386cd6c sendmail
  762    0     1   762  Ss      select   0xc0a3ffbc sendmail
  755    0     1   755  SWs     select   0xc0a3ffbc sshd
  734    0     1   734  Ss      select   0xc0a3ffbc ntpd
  679    0   675   675  SW      -        0xc37d0a00 nfsd
  678    0   675   675  SW      -        0xc38fcc00 nfsd
  677    0   675   675  SW      -        0xc37d0c00 nfsd
  676    0   675   675  SW      -        0xc38fce00 nfsd
  675    0     1   675  SWs     select   0xc0a3ffbc nfsd
  673    0     1   673  SWs     select   0xc0a3ffbc mountd
  614    0     1   614  Ss      select   0xc0a3ffbc rpcbind
  590    0     1   590  Ss      select   0xc0a3ffbc syslogd
  528    0     1   528  SWs     select   0xc0a3ffbc devd
   46    0     0     0  SL      -        0xe3bdbd04 [schedcpu]
   45    0     0     0  SL      -        0xc0a47ccc [nfsiod 3]
   44    0     0     0  SL      -        0xc0a47cc8 [nfsiod 2]
   43    0     0     0  SL      -        0xc0a47cc4 [nfsiod 1]
   42    0     0     0  SL      -        0xc0a47cc0 [nfsiod 0]
   41    0     0     0  SL      biowr    0xd764bb70 [softdepflush]
   40    0     0     0  SL      syncer   0xc09f3c30 [syncer]
   39    0     0     0  SL      vlruwt   0xc37a7000 [vnlru]
   38    0     0     0  SL      psleep   0xc0a40434 [bufdaemon]
   37    0     0     0  SL      pgzero   0xc0a55b50 [pagezero]
   36    0     0     0  SL      psleep   0xc0a4da7c [vmdaemon]
   35    0     0     0  SL      psleep   0xc0a4da3c [pagedaemon]
   34    0     0     0  WL                          [irq7: ppc0]
   33    0     0     0  SL      -        0xc35de23c [fdc0]
   32    0     0     0  WL                          [swi0: sio]
   31    0     0     0  WL                          [irq12: psm0]
   30    0     0     0  WL                          [irq1: atkbd0]
   29    0     0     0  WL                          [irq15: ata1]
   28    0     0     0  WL                          [irq14: ata0]
   27    0     0     0  WL                          [irq17: fxp0]
   26    0     0     0  SL      usbtsk   0xc09f1424 [usbtask]
   25    0     0     0  SL      usbevt   0xc3562210 [usb0]
   24    0     0     0  WL                          [irq16: uhci0]
   23    0     0     0  SL      -        0xc354ec80 [em0 taskq]
   22    0     0     0  WL                          [irq9: acpi0]
   21    0     0     0  WL                          [swi6: task queue]
    9    0     0     0  SL      -        0xc3484200 [kqueue taskq]
    8    0     0     0  SL      -        0xc3484280 [acpi_task_2]
    7    0     0     0  SL      -        0xc3484280 [acpi_task_1]
    6    0     0     0  SL      -        0xc3484280 [acpi_task_0]
   20    0     0     0  WL                          [swi2: cambio]
   19    0     0     0  WL                          [swi5: +]
    5    0     0     0  SL      -        0xc3484580 [thread taskq]
   18    0     0     0  WL                          [swi6: Giant taskq]
   17    0     0     0  SL      -        0xc09ef060 [yarrow]
    4    0     0     0  SL      -        0xc09f1b9c [g_down]
    3    0     0     0  LL     *vm page  0xc556ccc0 [g_up]
    2    0     0     0  SL      -        0xc09f1b90 [g_event]
   16    0     0     0  WL                          [swi3: vm]
   15    0     0     0  LL     *Giant    0xc6945bc0 [swi4: clock sio]
   14    0     0     0  WL                          [swi1: net]
   13    0     0     0  RL      CPU 0               [idle: cpu0]
   12    0     0     0  RL      CPU 1               [idle: cpu1]
   11    0     0     0  RL                          [idle: cpu2]
   10    0     0     0  RL      CPU 3               [idle: cpu3]
    1    0     0     1  SLs     wait     0xc3433000 [init]
    0    0     0     0  WLs                         [swapper]
32708 1001 25361 25354  Z+                          mkfifo
32707 1001 25361 25354  Z+                          pty
32710 1001 25361 25354  Z+                          lockf
32714 1001 25361 25354  Z+                          mmap
32709 1001 25361 25354  Z+                          shm
32704 1001 25361 25354  Z+                          sysctl
32715 1001 25361 25354  Z+                          socket
32706 1001 25361 25354  Z+                          udp
32713 1001 25361 25354  Z+                          badcode
32705 1001 25361 25354  Z+                          tcp
db> call doadump
Physical memory: 1015 MB
Dumping 165 MB: 150 134 118 102 86 70 54 38 22 6
Dump complete
= 0xf
db> reset

(kgdb) p *(struct buf *)0xd764bb70
$5 = {b_bufobj = 0xc3860eec, b_bcount = 0x4000, b_caller1 = 0x0, b_data = 0xdc7a4000 "\201\001", b_error = 0x0, b_iocmd = 0x2,
  b_ioflags = 0x2, b_iooffset = 0x5c394000, b_resid = 0x0, b_iodone = 0, b_blkno = 0x2e1ca0, b_offset = 0x5c394000, b_bobufs = {
    tqe_next = 0xd75efc08, tqe_prev = 0xd755da20}, b_left = 0xd755d9e8, b_right = 0xd75efc08, b_vflags = 0x0, b_freelist = {
    tqe_next = 0x0, tqe_prev = 0xd76cca64}, b_qindex = 0x3, b_flags = 0xa2020020, b_xflags = 0x2, b_lock = {lk_interlock = 0xc09f2e70,
    lk_flags = 0x40000, lk_sharecount = 0x0, lk_waitcount = 0x0, lk_exclusivecount = 0x1, lk_prio = 0x50, lk_wmesg = 0xc0910763 "getblk",
    lk_timo = 0x0, lk_lockholder = 0xc37a66c0, lk_newlock = 0x0}, b_bufsize = 0x4000, b_runningbufspace = 0x0,
  b_kvabase = 0xdc7a4000 "\201\001", b_kvasize = 0x4000, b_lblkno = 0x2e1ca0, b_vp = 0xc3860e38, b_dirtyoff = 0x0, b_dirtyend = 0x0,
  b_rcred = 0x0, b_wcred = 0x0, b_saveaddr = 0xdc7a4000, b_pager = {pg_reqpage = 0x0}, b_cluster = {cluster_head = {
      tqh_first = 0xd75e4140, tqh_last = 0xd753e410}, cluster_entry = {tqe_next = 0xd75e4140, tqe_prev = 0xd753e410}}, b_pages = {
    0xc13192d0, 0xc1c59998, 0xc15ea0e0, 0xc163eba8, 0x0 <repeats 28 times>}, b_npages = 0x4, b_dep = {lh_first = 0x0}, b_fsprivate1 = 0x0,
  b_fsprivate2 = 0x0, b_fsprivate3 = 0x0, b_pin_count = 0x0}
p/d (*(struct thread *)0xc37a66c0)->td_proc.p_pid
$7 = 41