GENERIC HEAD from May 26 06:09 UTC, vmcore.333

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 6.0-CURRENT #0: Thu May 26 12:15:40 CEST 2005
    pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I  OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf13  Stepping = 3
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory  = 267583488 (255 MB)
avail memory = 252354560 (240 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
rl0: link state changed to DOWN
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
panic: wrong b_bufobj 0xc1db0e20 should be 0xc2241a84
cpuid = 0
KDB: enter: panic
[thread pid 18548 tid 100400 ]
Stopped at      kdb_enter+0x2b: nop
db> where
Tracing pid 18548 tid 100400 td 0xc2179a80
kdb_enter(c084bdcd) at kdb_enter+0x2b
panic(c0853ba6,c1db0e20,c2241a84,23,c6642788) at panic+0x14b
flushbuflist(c2241a88,4,c2241a84,0,0) at flushbuflist+0x111
bufobj_invalbuf(c2241a84,4,c2179a80,0,0) at bufobj_invalbuf+0x10d
vinvalbuf(c22419a0,4,c2179a80,0,0,c1f47b58,0,0,800) at vinvalbuf+0x1d
ffs_truncate(c22419a0,0,0,c00,0) at ffs_truncate+0x575
ufs_inactive(cf5adbd8) at ufs_inactive+0x9d
VOP_INACTIVE_APV(c08e9080,cf5adbd8) at VOP_INACTIVE_APV+0x7e
vinactive(c22419a0,c2179a80) at vinactive+0x6d
vput(c22419a0) at vput+0x162
kern_unlink(c2179a80,bfbfd930,0,cf5add30,c07e1267) at kern_unlink+0x19e
unlink(c2179a80,cf5add04,1,8e,296) at unlink+0x12
syscall(2805003b,bfbf003b,bfbf003b,28050300,bfbfeb00) at syscall+0x227
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (10, FreeBSD ELF32, unlink), eip = 0x280c15f7, esp = 0xbfbfd91c, ebp = 0xbfbfe9c8 ---
db> show lockedvnods
Locked vnodes
KDB: stack backtrace:
kdb_backtrace(c091f8a0,0,c084c64a,13b,7b1df6e5) at kdb_backtrace+0x29
mi_switch(1,0) at mi_switch+0x1af
turnstile_wait(c091ef48,c2262900) at turnstile_wait+0x32b
_mtx_lock_sleep(c091ef48,c2179a80,0,c0849e49,243) at _mtx_lock_sleep+0x1ce
_mtx_lock_flags(c091ef48,0,c0849e49,243) at _mtx_lock_flags+0x8a
lockstatus(c1f388c4,0,cf5ad73c,c07f41ca,cf5ad74c) at lockstatus+0x24
vop_stdislocked(cf5ad74c) at vop_stdislocked+0x17
VOP_ISLOCKED_APV(c08e9080,cf5ad74c) at VOP_ISLOCKED_APV+0x7e
lockedvnodes(c064080f,0,ffffffff,cf5ad78c,0) at lockedvnodes+0x45
db_command(c08f88a4,c087f7a0,c0876a44,c0876a60,c08180dd) at db_command+0x264
db_command_loop(0,0,cf5ad844,cf5ad830,cf5ad878) at db_command_loop+0x5c
db_trap(3,0,c2179a80,3,c2179a80) at db_trap+0xdd
kdb_trap(3,0,cf5ad8d0) at kdb_trap+0x88
trap(cf5a0008,c0640028,c0840028,c0853ba6,1) at trap+0x4b0
calltrap() at calltrap+0x5
--- trap 0x3, eip = 0xc064080f, esp = 0xcf5ad910, ebp = 0xcf5ad910 ---
kdb_enter(c084bdcd) at kdb_enter+0x2b
panic(c0853ba6,c1db0e20,c2241a84,23,c6642788) at panic+0x14b
flushbuflist(c2241a88,4,c2241a84,0,0) at flushbuflist+0x111
bufobj_invalbuf(c2241a84,4,c2179a80,0,0) at bufobj_invalbuf+0x10d
vinvalbuf(c22419a0,4,c2179a80,0,0,c1f47b58,0,0,800) at vinvalbuf+0x1d
ffs_truncate(c22419a0,0,0,c00,0) at ffs_truncate+0x575
ufs_inactive(cf5adbd8) at ufs_inactive+0x9d
VOP_INACTIVE_APV(c08e9080,cf5adbd8) at VOP_INACTIVE_APV+0x7e
vinactive(c22419a0,c2179a80) at vinactive+0x6d
vput(c22419a0) at vput+0x162
kern_unlink(c2179a80,bfbfd930,0,cf5add30,c07e1267) at kern_unlink+0x19e
unlink(c2179a80,cf5add04,1,8e,296) at unlink+0x12
syscall(2805003b,bfbf003b,bfbf003b,28050300,bfbfeb00) at syscall+0x227
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (10, FreeBSD ELF32, unlink), eip = 0x280c15f7, esp = 0xbfbfd91c, ebp = 0xbfbfe9c8 ---
db> show alllocks
Process 18555 (thr1) thread 0xc2262900 (100117)
exclusive sx user map r = 0 (0xc1a7dd28) locked @ vm/vm_map.c:2997
Process 18553 (thr1) thread 0xc2dbf180 (100465)
exclusive sleep mutex pmap r = 0 (0xc2e92c78) locked @ i386/i386/pmap.c:1626
exclusive sleep mutex vm page queue mutex r = 0 (0xc097b200) locked @ i386/i386/pmap.c:1624
exclusive sx user map r = 0 (0xc2e92bfc) locked @ vm/vm_unix.c:88
Process 18550 (creat) thread 0xc214f900 (100290)
exclusive sleep mutex vnode interlock r = 0 (0xc25eef08) locked @ kern/vfs_bio.c:2326
Process 18549 (creat) thread 0xc21c3000 (100181)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1794ad4) locked @ kern/vfs_bio.c:3340
Process 18540 (udp) thread 0xc34d5480 (100102)
exclusive sleep mutex inp (udpinp) r = 0 (0xc17d2b1c) locked @ netinet/udp_usrreq.c:773
Process 18539 (udp) thread 0xc2159480 (100300)
exclusive sleep mutex inp (udpinp) r = 0 (0xc17d2f54) locked @ netinet/udp_usrreq.c:773
Process 18537 (udp) thread 0xc2f24780 (100151)
exclusive sleep mutex inp (udpinp) r = 0 (0xc17d29b4) locked @ netinet/udp_usrreq.c:773
Process 18535 (udp) thread 0xc2f15c00 (100325)
exclusive sleep mutex rl0 (network driver) r = 0 (0xc1611f08) locked @ pci/if_rl.c:1444
exclusive sleep mutex inp (udpinp) r = 0 (0xc17d1f54) locked @ netinet/udp_usrreq.c:773
db> ps
  pid   proc     uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
18593 c2eb6200 1001 18561  8678 0000002 [RUNQ] tcp
18588 c3590800 1001 18562  8678 0000002 [RUNQ] tcp
18580 c262d600 1001 18560  8678 0000002 [RUNQ] tcp
18576 c358f800 1001 18559  8678 0000002 [RUNQ] tcp
18573 c3591000 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc22a7000 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2f1f600 ksegrp 0xc3592360 [RUNQ]
   thread 0xc24cdd80 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2f1f300 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc232c000 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc34cb480 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2f25600 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc34d5d80 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2179c00 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc34d7780 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2262d80 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc1824600 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2eb8c00 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2f1c300 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2f15480 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
   thread 0xc2e68480 ksegrp 0xc2dbd3c0 [SLPQ ksesigwait 0xc3591138][SLP]
   thread 0xc2eb7a80 ksegrp 0xc3592360 [SLPQ kserel 0xc3592394][SLP]
18572 c2e66c00 1001 18558  8678 0000002 [SLPQ nanslp 0xc09201ac][SLP] syscall
18571 c180fc00 1001 18558  8678 0000002 [SLPQ nanslp 0xc09201ac][SLP] syscall
18570 c2dbee00 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc2263780 ksegrp 0xc2dbd000 [RUNQ]
   thread 0xc24e2900 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2f1ed80 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2120780 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc34d4900 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2355d80 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2f14600 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2f13600 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc1812c00 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2eb7900 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2f1e780 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2262480 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2f23180 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2125d80 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2f24c00 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
   thread 0xc2dc0600 ksegrp 0xc2dbd6c0 [SLPQ ksesigwait 0xc2dbef38][SLP]
   thread 0xc2159a80 ksegrp 0xc2dbd000 [SLPQ kserel 0xc2dbd034][SLP]
18569 c2eb4400 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc2dbfd80 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc214f300 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc232f000 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc2f1ec00 ksegrp 0xc2eb5540 [RUNQ]
   thread 0xc24e2480 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc2eb7300 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc232ca80 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc24cd000 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc24cd780 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc24cd480 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc34d7c00 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc2dbf900 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc2dbfa80 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc2e8a480 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc24cdc00 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
   thread 0xc24e2d80 ksegrp 0xc2eb5d20 [SLPQ ksesigwait 0xc2eb4538][SLP]
   thread 0xc2f24600 ksegrp 0xc2eb5540 [SLPQ kserel 0xc2eb5574][SLP]
18567 c2e66400 1001 18558  8678 0000002 [SLPQ nanslp 0xc09201ac][SLP] syscall
18565 c2613c00 1001 18558  8678 0000002 [SLPQ nanslp 0xc09201ac][SLP] syscall
18563 c1804c00 1001 18558  8678 0000002 [SLPQ nanslp 0xc09201ac][SLP] syscall
18562 c2e88400 1001 18552  8678 0000002 [SLPQ sbwait 0xc1a8e480][SLP] tcp
18561 c2dbc600 1001 18552  8678 0000002 [SLPQ sbwait 0xc2f17718][SLP] tcp
18560 c2eb6800 1001 18552  8678 0000002 [SLPQ sbwait 0xc2f18c48][SLP] tcp
18559 c1abac00 1001 18552  8678 0000002 [SLPQ sbwait 0xc2f185cc][SLP] tcp
18558 c1abbc00 1001 18529  8678 0000002 [SLPQ wait 0xc1abbc00][SLP] syscall
18557 c2613200 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc2e68180 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2125900 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2f23000 ksegrp 0xc2dbdea0 [RUNQ]
   thread 0xc34cb180 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2dc0900 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2e69900 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc22a7780 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2262a80 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2f24a80 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2f24d80 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2f22000 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc34d5180 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc232ba80 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc1824180 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc232c480 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
   thread 0xc2263480 ksegrp 0xc2dbd780 [SLPQ ksesigwait 0xc2613338][SLP]
   thread 0xc24cd600 ksegrp 0xc2dbdea0 [SLPQ kserel 0xc2dbded4][SLP]
18556 c1811a00 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc2eb7c00 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc24b4000 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc24cd300 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc2218900 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc2e8ac00 ksegrp 0xc1813120 [RUNQ]
   thread 0xc2262600 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc2e8aa80 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc21c3300 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc2159900 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc1821180 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc164a780 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc22a7c00 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc24e2000 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc1824d80 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc34d4000 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
   thread 0xc2f1cd80 ksegrp 0xc2dbd660 [SLPQ ksesigwait 0xc1811b38][SLP]
   thread 0xc2e69d80 ksegrp 0xc1813120 [SLPQ kserel 0xc1813154][SLP]
18555 c2eb6e00 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc2eb7000 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2262900 ksegrp 0xc2eb5060 [RUNQ]
   thread 0xc2636300 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2f24480 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2f25a80 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2f1ea80 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2f1c600 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc34d4c00 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc232b600 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2159300 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc1821300 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2f26600 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2f21900 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2179600 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc2f1d900 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
   thread 0xc214f600 ksegrp 0xc2dbd8a0 [SLPQ ksesigwait 0xc2eb6f38][SLP]
   thread 0xc21c3d80 ksegrp 0xc2eb5060 [SLPQ kserel 0xc2eb5094][SLP]
18554 c1abaa00 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc2120c00 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2f26180 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2eb7d80 ksegrp 0xc1573720 [RUNQ]
   thread 0xc2f26900 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2f1c180 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc24e2c00 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc21a1600 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2218180 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2e68000 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2f14300 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc21a1480 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc21a1900 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2eb8780 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2e69180 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc2159180 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
   thread 0xc34d4600 ksegrp 0xc2dbd900 [SLPQ ksesigwait 0xc1abab38][SLP]
   thread 0xc1812a80 ksegrp 0xc1573720 [SLPQ kserel 0xc1573754][SLP]
18553 c3590200 1001 18541  8678 0008082 (threaded)  thr1
   thread 0xc2dbf180 ksegrp 0xc2dbd600 [RUNQ]
   thread 0xc232b780 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2263a80 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2186000 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc24fe000 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2f22c00 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc1abc000 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc34d5a80 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc232b900 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2f15300 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2f13d80 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2179780 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2e69300 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2159000 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc2f26300 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
   thread 0xc34d5900 ksegrp 0xc2dbd840 [SLPQ ksesigwait 0xc3590338][SLP]
   thread 0xc2eb8a80 ksegrp 0xc2dbd600 [SLPQ kserel 0xc2dbd634][SLP]
18552 c2613800 1001 18531  8678 0000002 [SLPQ wait 0xc2613800][SLP] tcp
18551 c1804000 1001 18542  8678 0000002 [RUNQ] creat
18550 c2dbe400 1001 18542  8678 0000002 [LOCK lockbuilder mtxpool c18ce9c0] creat
18549 c2613600 1001 18542  8678 0000002 [LOCK vm page queue mutex c16a5d00] creat
18548 c2e89c00 1001 18534  8678 0000002 [LOCK lockbuilder mtxpool c18ce9c0] rw
18547 c2613e00 1001 18534  8678 0000002 [RUNQ] rw
18546 c1abb200 1001 18534  8678 0000002 [RUNQ] rw
18545 c2e66800 1001 18534  8678 0000002 [RUNQ] rw
18544 c1abb600 1001 18534  8678 0000002 [RUNQ] rw
18543 c2e66600 1001 18534  8678 0000002 [RUNQ] rw
18542 c2e67c00 1001 18525  8678 0000002 [SLPQ wait 0xc2e67c00][SLP] creat
18541 c2dbc800 1001 18527  8678 0000002 [SLPQ wait 0xc2dbc800][SLP] thr1
18540 c2e66200 1001 18533  8678 0000002 [RUNQ] udp
18539 c262dc00 1001 18533  8678 0000002 [RUNQ] udp
18538 c2e67e00 1001 18533  8678 0000002 [RUNQ] udp
18537 c2dbc000 1001 18533  8678 0000002 [RUNQ] udp
18536 c2e66a00 1001 18533  8678 0000002 [RUNQ] udp
18535 c2e67400 1001 18533  8678 0000002 [RUNQ] udp
18534 c3590000 1001 18528  8678 0000002 [SLPQ wait 0xc3590000][SLP] rw
18533 c2e89a00 1001 18532  8678 0000002 [SLPQ wait 0xc2e89a00][SLP] udp
18532 c2e67a00 1001  8685  8678 0004002 [SLPQ nanslp 0xc09201ac][SLP] udp
18531 c2eb6c00 1001  8685  8678 0004002 [SLPQ nanslp 0xc09201ac][SLP] tcp
18529 c2613a00 1001  8685  8678 0004002 [SLPQ nanslp 0xc09201ac][SLP] syscall
18528 c262d000 1001  8685  8678 0004002 [SLPQ nanslp 0xc09201ac][SLP] rw
18527 c358f600 1001  8685  8678 0004002 [SLPQ nanslp 0xc09201ac][SLP] thr1
18525 c2e88200 1001  8685  8678 0004002 [SLPQ nanslp 0xc09201ac][SLP] creat
 8685 c1aba200 1001  8684  8678 0000002 [SLPQ wait 0xc1aba200][SLP] run
 8684 c2e66e00 1001  8683  8678 0000002 [SLPQ wait 0xc2e66e00][SLP] run
 8683 c2dbe800 1001  8678  8678 0004002 [SLPQ nanslp 0xc09201ac][SLP] run
 8678 c2dbcc00 1001   586  8678 0004002 [SLPQ wait 0xc2dbcc00][SLP] sh
  586 c1811c00 1001   585   586 0004002 [SLPQ wait 0xc1811c00][SLP] bash
  585 c1811e00 1001   583   583 0000100 [SLPQ select 0xc096d4a4][SLP] sshd
  583 c1820200    0   423   583 0004100 [SLPQ sbwait 0xc1a8ec48][SLP] sshd
  570 c1804800 1001   564   570 0004002 [SLPQ select 0xc096d4a4][SLP] top
  564 c1811800 1001   563   564 0004002 [SLPQ wait 0xc1811800][SLP] bash
  563 c174dc00 1001   561   561 0000100 [SLPQ select 0xc096d4a4][SLP] sshd
  561 c1820600    0   423   561 0004100 [SLPQ sbwait 0xc1839480][SLP] sshd
  553 c1804200    0     1   553 0004002 [SLPQ ttyin 0xc16b3c10][SLP] getty
  552 c1804e00    0     1   552 0004002 [SLPQ ttyin 0xc16b4010][SLP] getty
  551 c1804400    0     1   551 0004002 [SLPQ ttyin 0xc16b4410][SLP] getty
  550 c1804a00    0     1   550 0004002 [SLPQ ttyin 0xc16ab010][SLP] getty
  549 c1804600    0     1   549 0004002 [SLPQ ttyin 0xc16a9410][SLP] getty
  548 c1820000    0     1   548 0004002 [SLPQ ttyin 0xc16ab410][SLP] getty
  547 c1646e00    0     1   547 0004002 [SLPQ ttyin 0xc16a8c10][SLP] getty
  546 c1750c00    0     1   546 0004002 [SLPQ ttyin 0xc16a8010][SLP] getty
  514 c180f600    0     1   514 0000000 [SLPQ select 0xc096d4a4][SLP] moused
  447 c180f400    0     1   447 0000000 [SLPQ nanslp 0xc09201ac][SLP] cron
  435 c1811600   25     1   435 0000100 [SLPQ pause 0xc1811634][SLP] sendmail
  429 c174da00    0     1   429 0000100 [SLPQ select 0xc096d4a4][SLP] sendmail
  423 c180fa00    0     1   423 0000100 [SLPQ select 0xc096d4a4][SLP] sshd
  405 c180fe00    0     1   405 0000000 [SLPQ select 0xc096d4a4][SLP] ntpd
  376 c180f800    0     1   376 0000000 [SLPQ select 0xc096d4a4][SLP] usbd
  356 c1811400    0   351   351 0000000 [SLPQ - 0xc1818600][SLP] nfsd
  355 c180f000    0   351   351 0000000 [SLPQ - 0xc1818400][SLP] nfsd
  354 c174d400    0   351   351 0000000 [SLPQ - 0xc1818800][SLP] nfsd
  353 c1750a00    0   351   351 0000000 [SLPQ - 0xc1818a00][SLP] nfsd
  351 c1750e00    0     1   351 0000000 [SLPQ select 0xc096d4a4][SLP] nfsd
  349 c180f200    0     1   349 0000000 [SLPQ select 0xc096d4a4][SLP] mountd
  280 c174d800    0     1   280 0000000 [SLPQ select 0xc096d4a4][SLP] rpcbind
  266 c174d600    0     1   266 0000000 [SLPQ select 0xc096d4a4][SLP] syslogd
  237 c174d200    0     1   237 0000000 [SLPQ select 0xc096d4a4][SLP] devd
   61 c174de00    0     0     0 0000204 [SLPQ - 0xce9e7d04][SLP] schedcpu
   60 c1750000    0     0     0 0000204 [SLPQ - 0xc097518c][SLP] nfsiod 3
   59 c1750200    0     0     0 0000204 [SLPQ - 0xc0975188][SLP] nfsiod 2
   58 c1750400    0     0     0 0000204 [SLPQ - 0xc0975184][SLP] nfsiod 1
   57 c1750600    0     0     0 0000204 [SLPQ - 0xc0975180][SLP] nfsiod 0
   56 c1750800    0     0     0 0000204 [SLPQ syncer 0xc091ff20][SLP] syncer
   55 c1572400    0     0     0 0000204 [SLPQ vlruwt 0xc1572400][SLP] vnlru
   54 c1572600    0     0     0 0000204 [SLPQ psleep 0xc096d9f0][SLP] bufdaemon
   53 c1572800    0     0     0 000020c [SLPQ pgzero 0xc097b724][SLP] pagezero
   52 c1572a00    0     0     0 0000204 [SLPQ psleep 0xc097b274][SLP] vmdaemon
   51 c1572c00    0     0     0 0000204 [SLPQ psleep 0xc097b230][SLP] pagedaemon
   50 c1572e00    0     0     0 0000204 [SLPQ - 0xc166aa3c][SLP] fdc0
   49 c1646000    0     0     0 0000204 [IWAIT] swi0: sio
   48 c1646200    0     0     0 0000204 [SLPQ usbevt 0xc160d210][SLP] usb4
   47 c1646400    0     0     0 0000204 [SLPQ usbevt 0xc1661210][SLP] usb3
   46 c1646600    0     0     0 0000204 [SLPQ usbevt 0xc164e210][SLP] usb2
   45 c1646800    0     0     0 0000204 [SLPQ usbevt 0xc1650210][SLP] usb1
   44 c1646a00    0     0     0 0000204 [SLPQ usbtsk 0xc090eeb4][SLP] usbtask
   43 c1646c00    0     0     0 0000204 [SLPQ usbevt 0xc1624210][SLP] usb0
   42 c1565c00    0     0     0 0000204 [IWAIT] swi2: cambio
   41 c1565e00    0     0     0 0000204 [IWAIT] swi5:+
    9 c1570000    0     0     0 0000204 [SLPQ - 0xc161c280][SLP] acpi_task2
    8 c1570200    0     0     0 0000204 [SLPQ - 0xc161c280][SLP] acpi_task1
    7 c1570400    0     0     0 0000204 [SLPQ - 0xc161c280][SLP] acpi_task0
    6 c1570600    0     0     0 0000204 [SLPQ - 0xc161c300][SLP] thread taskq
   40 c1570800    0     0     0 0000204 [IWAIT] swi6:+
   39 c1570a00    0     0     0 0000204 [IWAIT] swi6: task queue
    5 c1570c00    0     0     0 0000204 [SLPQ - 0xc1520c00][SLP] kqueue taskq
   38 c1570e00    0     0     0 0000204 [SLPQ - 0xc090cbe0][SLP] yarrow
    4 c1572000    0     0     0 0000204 [SLPQ - 0xc09116a8][SLP] g_down
    3 c1572200    0     0     0 0000204 [SLPQ - 0xc09116a4][SLP] g_up
    2 c1557600    0     0     0 0000204 [SLPQ - 0xc091169c][SLP] g_event
   37 c1557800    0     0     0 0000204 [IWAIT] swi1: net
   36 c1557a00    0     0     0 0000204 [IWAIT] swi3: vm
   35 c1557c00    0     0     0 000020c [RUNQ] swi4: clock sio
   34 c1557e00    0     0     0 0000204 [IWAIT] irq23: ehci0
   33 c1565000    0     0     0 0000204 [IWAIT] irq22: rl0
   32 c1565200    0     0     0 0000204 [IWAIT] irq21:
   31 c1565400    0     0     0 0000204 [IWAIT] irq20:
   30 c1565600    0     0     0 0000204 [IWAIT] irq19: uhci1
   29 c1565800    0     0     0 0000204 [IWAIT] irq18: uhci2
   28 c1565a00    0     0     0 0000204 [IWAIT] irq17: pcm0
   27 c1526200    0     0     0 0000204 [IWAIT] irq16: uhci0 uhci3
   26 c1526400    0     0     0 0000204 [IWAIT] irq15: ata1
   25 c1526600    0     0     0 0000204 [IWAIT] irq14: ata0
   24 c1526800    0     0     0 0000204 [IWAIT] irq13:
   23 c1526a00    0     0     0 0000204 [IWAIT] irq12: psm0
   22 c1526c00    0     0     0 0000204 [IWAIT] irq11:
   21 c1526e00    0     0     0 0000204 [IWAIT] irq10:
   20 c1557000    0     0     0 0000204 [IWAIT] irq9: acpi0
   19 c1557200    0     0     0 0000204 [IWAIT] irq8:
   18 c1557400    0     0     0 0000204 [IWAIT] irq7: ppc0
   17 c1521000    0     0     0 0000204 [IWAIT] irq6: fdc0
   16 c1521200    0     0     0 0000204 [IWAIT] irq5:
   15 c1521400    0     0     0 0000204 [IWAIT] irq4: sio0
   14 c1521600    0     0     0 0000204 [IWAIT] irq3:
   13 c1521800    0     0     0 0000204 [IWAIT] irq0:
   12 c1521a00    0     0     0 0000204 [IWAIT] irq1: atkbd0
   11 c1521c00    0     0     0 000020c [Can run] idle: cpu0
    1 c1521e00    0     0     1 0004200 [SLPQ wait 0xc1521e00][SLP] init
   10 c1526000    0     0     0 0000204 [SLPQ ktrace 0xc091e118][SLP] ktrace
    0 c09117a0    0     0     0 0000200 [IWAIT] swapper
18564 c2dbec00 1001 18558  8678 0002002 zomb[INACTIVE] syscall
18566 c262da00 1001 18558  8678 0002002 zomb[INACTIVE] syscall
18568 c1aba600 1001 18558  8678 0002002 zomb[INACTIVE] syscall
18530 c2613000 1001  8685  8678 0006002 zomb[INACTIVE] sysctl
18526 c2e67800 1001  8685  8678 0006002 zomb[INACTIVE] mkdir
db> call doadump
Dumping 255 MB
 16 32 48 64 80 96 112 128 144 160 176 192 208 224 240
Dump complete
0xf
db> reset

(kgdb) f 11
#11 0xc067aad9 in flushbuflist (bufv=0x0, flags=0x4, bo=0xc2241a84, slpflag=0x0, slptimeo=0x0) at ../../../kern/vfs_subr.c:1056
1056                    KASSERT(bp->b_bufobj == bo,
(kgdb) info loc
bp = (struct buf *) 0xc668e9b8
nbp = (struct buf *) 0xc6642788
retval = 0x23
error = 0x0
(kgdb) p *bp
$1 = {b_bufobj = 0xc1db0e20, b_bcount = 0x4000, b_caller1 = 0x0, b_data = 0xc8688000 "", b_error = 0x0, b_iocmd = 0x2, b_ioflags = 0x0,
  b_iooffset = 0x296ce800, b_resid = 0x0, b_iodone = 0, b_blkno = 0x1a0, b_offset = 0x680000, b_bobufs = {tqe_next = 0xc6642788,
    tqe_prev = 0xc6644658}, b_left = 0xc6644620, b_right = 0x0, b_vflags = 0x0, b_freelist = {tqe_next = 0xc6642788,
    tqe_prev = 0xc664466c}, b_qindex = 0x1, b_flags = 0x20000020, b_xflags = 0x2, b_lock = {lk_interlock = 0xc091f7b8,
    lk_flags = 0x40000, lk_sharecount = 0x0, lk_waitcount = 0x0, lk_exclusivecount = 0x1, lk_prio = 0x50,
    lk_wmesg = 0xc085506f "flushbuf", lk_timo = 0x0, lk_lockholder = 0xc2179a80, lk_newlock = 0x0,
    lk_filename = 0xc0853636 "../../../sys/buf.h", lk_lockername = 0xc083fa05 "lockmgr", lk_lineno = 0x129, lk_slockholder = 0xffffffff,
    lk_sfilename = 0xc0825203 "none", lk_slockername = 0xc084a12b "never share locked", lk_slineno = 0x0}, b_bufsize = 0x4000,
  b_runningbufspace = 0x0, b_kvabase = 0xc8688000 "", b_kvasize = 0x4000, b_lblkno = 0x1a0, b_vp = 0xc1db0d3c, b_dirtyoff = 0x0,
  b_dirtyend = 0x0, b_rcred = 0x0, b_wcred = 0x0, b_saveaddr = 0xc8688000, b_pager = {pg_reqpage = 0x0}, b_cluster = {cluster_head = {
      tqh_first = 0xc6642788, tqh_last = 0xc65db020}, cluster_entry = {tqe_next = 0xc6642788, tqe_prev = 0xc65db020}}, b_pages = {
    0xc11c6830, 0xc1184b78, 0xc12223c0, 0xc11cd508, 0x0 <repeats 28 times>}, b_npages = 0x4, b_dep = {lh_first = 0x0}}
(kgdb) up
#12 0xc067a7c9 in bufobj_invalbuf (bo=0xc2241a84, flags=0x4, td=0xc2179a80, slpflag=0x0, slptimeo=0x0) at ../../../kern/vfs_subr.c:966
966                     error = flushbuflist(&bo->bo_clean,
(kgdb) up
#13 0xc067a9c5 in vinvalbuf (vp=0xc22419a0, flags=0x4, td=0xc2179a80, slpflag=0x0, slptimeo=0x0) at ../../../kern/vfs_subr.c:1023
1023            return (bufobj_invalbuf(&vp->v_bufobj, flags, td, slpflag, slptimeo));
(kgdb) p *vp
$2 = {v_type = VREG, v_tag = 0xc0852e4f "ufs", v_op = 0xc08e9080, v_data = 0xc1f47b58, v_mount = 0xc173a800, v_nmntvnodes = {
    tqe_next = 0xc1e57738, tqe_prev = 0xc1df8014}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0},
  v_hashlist = {le_next = 0x0, le_prev = 0xc176cd5c}, v_hash = 0x172f7, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0x0,
    tqh_last = 0xc22419d0}, v_dd = 0x0, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0, v_clen = 0x0, v_lock = {lk_interlock = 0xc091f434,
    lk_flags = 0x40040, lk_sharecount = 0x0, lk_waitcount = 0x0, lk_exclusivecount = 0x1, lk_prio = 0x50, lk_wmesg = 0xc0852e4f "ufs",
    lk_timo = 0x33, lk_lockholder = 0xc2179a80, lk_newlock = 0x0, lk_filename = 0xc0854ecd "../../../kern/vfs_subr.c",
    lk_lockername = 0xc08545d8 "vop_stdlock", lk_lineno = 0x718, lk_slockholder = 0xffffffff, lk_sfilename = 0xc0825203 "none",
    lk_slockername = 0xc084a12b "never share locked", lk_slineno = 0x0}, v_interlock = {mtx_object = {lo_class = 0xc08af408,
      lo_name = 0xc084f77e "vnode interlock", lo_type = 0xc084f77e "vnode interlock", lo_flags = 0x30000, lo_list = {
        tqe_next = 0xc1e577d0, tqe_prev = 0xc1df80a8}, lo_witness = 0xc09307c8}, mtx_lock = 0x4, mtx_recurse = 0x0},
  v_vnlock = 0xc22419f8, filename = 0xc0854ecd "../../../kern/vfs_subr.c", line = 0x718, v_holdcnt = 0x253, v_usecount = 0x0,
  v_iflag = 0x1800, v_vflag = 0x4, v_writecount = 0x0, v_freelist = {tqe_next = 0x0, tqe_prev = 0x0}, v_bufobj = {bo_mtx = 0xc2241a38,
    bo_clean = {bv_hd = {tqh_first = 0xc65f7a34, tqh_last = 0xc667d3d0}, bv_root = 0xc65f7a34, bv_cnt = 0x252}, bo_dirty = {bv_hd = {
        tqh_first = 0x0, tqh_last = 0xc2241a98}, bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x0, bo_ops = 0xc08b6128,
    bo_bsize = 0x4000, bo_object = 0xc1aa1528, bo_synclist = {le_next = 0x0, le_prev = 0xc17e711c}, bo_private = 0xc22419a0,
    __bo_vnode = 0xc22419a0}, v_pollinfo = 0x0, v_label = 0x0}

index  trace
------ -----
   264 acquire(): lkp == 0xc668ea18, extflags == 0x120, wanted == 0x1050000
   263 acquire(): lkp == 0xc668ea18, extflags == 0x120, wanted == 0x60000
   262 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/kern_lock.c:218
   261 callers vfs_subr.c:966 vfs_subr.c:1024 ffs_inode.c:280 ufs_inode.c:95 vnode_if.c:1529 vnode_if.h:795
   260 lockmgr(): lkp == 0xc668ea18 (lk_wmesg == "flushbuf"), flags == 0x202122, td == 0xc2179a80 ../../../sys/buf.h:297
   259 LOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_subr.c:1081
   258 callers vfs_subr.c:1081 vfs_subr.c:966 vfs_subr.c:1024 ffs_inode.c:280 ufs_inode.c:95 vnode_if.c:1529
   257 lockmgr(): lkp == 0xc6669b14 (lk_wmesg == "bufwait"), flags == 0x6, td == 0xc2179a80 ../../../sys/buf.h:312
   256 UNLOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:314
   255 LOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:309
   254 UNLOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:358
   253 LOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:351
   252 UNLOCK (sleep mutex) buf queue lock r = 0 at ../../../kern/vfs_bio.c:1335
   251 bremfreel(0xc6669ab4) vp 0x0 flags 8040A000
   250 LOCK (sleep mutex) buf queue lock r = 0 at ../../../kern/vfs_bio.c:1296
   249 UNLOCK (spin mutex) sched lock r = 0 at ../../../kern/subr_turnstile.c:641
   248 turnstile_wait: td 100400 free from blocked on [0xc096db00] buf queue lock
   247 turnstile_wait: td 100181 blocked on [0xc097b200] vm page queue mutex
   246 LOCK (spin mutex) sched lock r = 0 at ../../../kern/subr_turnstile.c:596
   245 UNLOCK (spin mutex) turnstile chain r = 0 at ../../../kern/subr_turnstile.c:594
   244 UNLOCK (spin mutex) td_contested r = 0 at ../../../kern/subr_turnstile.c:578
   243 LOCK (spin mutex) td_contested r = 0 at ../../../kern/subr_turnstile.c:575
   242 LOCK (spin mutex) turnstile chain r = 0 at ../../../kern/subr_turnstile.c:459
   241 _mtx_lock_sleep: vm page queue mutex contested (lock=0xc2dbf180) at ../../../kern/vfs_bio.c:3341
   240 LOCK (sleep mutex) vm object r = 0 at ../../../kern/vfs_bio.c:3340