GENERIC 6.0-BETA5 from Oct 4 08:36 UTC + ambrisko.patch, vmcore.44
Livelock

KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 6.0-BETA5 #21: Tue Oct  4 22:14:12 CEST 2005
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/Generic
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Hyperthreading: 2 logical CPUs
real memory  = 1073676288 (1023 MB)
avail memory = 1041612800 (993 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
WARNING: R/W mount of /tmp denied.  Filesystem is not clean - run fsck
/tmp: unmount pending error: blocks 0 files -30
~KDB: enter: Line break on console
[thread pid 14 tid 100002 ]
Stopped at      kdb_enter+0x2b: nop
db> show pcpu
cpuid        = 0
curthread    = 0xc22d7c00: pid 14 "idle: cpu0"
curpcb       = 0xe35f8d90
fpcurthread  = none
idlethread   = 0xc22d7c00: pid 14 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
db> ps
  pid   proc     uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
18437 c465ac48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18436 c465b000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18435 c465b20c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18434 c465b418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18433 c465b624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18432 c465b830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18431 c2709c48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18430 c377da3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18429 c2c45830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18428 c272d000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18427 c2703418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18426 c2c41a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18425 c2449c48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18424 c3780624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18423 c612e418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18422 c26a1418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18421 c2709a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18420 c6134830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18419 c6817000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18418 c2703c48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18417 c2755000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18416 c67c7418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18415 c67c7624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18414 c26a1624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18413 c67cbc48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18412 c269f000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18411 c67cc418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18410 c270320c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18409 c60efa3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18408 c2c41624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18407 c681da3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18406 c67cc624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18405 c68aa418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18404 c2703000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18403 c612d20c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18402 c269f624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18401 c270920c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18400 c68a7000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18399 c275c830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18398 c67cb000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18397 c269fc48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18396 c275520c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18395 c3783418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18394 c2c4520c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18393 c67cc20c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18392 c6818a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18391 c2c4320c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18390 c2449830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18389 c6816000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18388 c68aa624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18387 c272d418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18386 c2449a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18385 c6817a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18384 c272da3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18383 c26a1a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18382 c2c45a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18381 c275b624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18380 c612d830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18379 c60ee624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18378 c272dc48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18377 c60ef418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18376 c68abc48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18375 c67c7830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18374 c60f4624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18373 c67ccc48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18372 c275c418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18371 c68ab830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18370 c269f418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18369 c67cc830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18368 c377d000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18367 c60f4830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18366 c68aa000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18365 c275b830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18364 c68a7a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18363 c612e20c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18362 c2c45418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18361 c2c43c48    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18360 c67cb830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18359 c3780418    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18358 c6818000    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18357 c2c41830    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18356 c67c7a3c    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18355 c272d624    0  1001  1001 0000000 [SLPQ vmpfw 0xc16bcf00][SLP] cron
18354 c2755c48    0  1001  1001 0000000 [SLPQ newbuf 0xc093f664][SLP] cron
18347 c2c43624 1001 18343  1199 0008082 (threaded)  thr1
   thread 0xc67c9180 ksegrp 0xc6819420 [SLPQ kserel 0xc6819454][SLP]
   thread 0xc4bf5d80 ksegrp 0xc6819420 [SLPQ vmpfw 0xc17db8c8][SLP]
   thread 0xc36d9d80 ksegrp 0xc6819420 [SLPQ kserel 0xc6819454][SLP]
   thread 0xc2728c00 ksegrp 0xc6819420 [SLPQ kserel 0xc6819454][SLP]
   thread 0xc24d0900 ksegrp 0xc2b40780 [SLPQ ksesigwait 0xc2c4375c][SLP]
18346 c68ab418 1001 18343  1199 0008082 (threaded)  thr1
   thread 0xc26da180 ksegrp 0xc6898780 [SLPQ vmpfw 0xc17db8c8][SLP]
   thread 0xc37d5780 ksegrp 0xc6898780 [SLPQ kserel 0xc68987b4][SLP]
   thread 0xc293ec00 ksegrp 0xc6898780 [SLPQ kserel 0xc68987b4][SLP]
   thread 0xc2495600 ksegrp 0xc6898780 [SLPQ kserel 0xc68987b4][SLP]
   thread 0xc67c9000 ksegrp 0xc2b3f5a0 [SLPQ ksesigwait 0xc68ab550][SLP]
18345 c67cca3c 1001 18343  1199 0008082 (threaded)  thr1
   thread 0xc2a91d80 ksegrp 0xc272a780 [SLPQ kserel 0xc272a7b4][SLP]
   thread 0xc29eec00 ksegrp 0xc272a780 [SLPQ kserel 0xc272a7b4][SLP]
   thread 0xc67ca180 ksegrp 0xc272a780 [SLPQ kserel 0xc272a7b4][SLP]
   thread 0xc2b19000 ksegrp 0xc272a780 [SLPQ vmpfw 0xc17db8c8][SLP]
   thread 0xc37d3300 ksegrp 0xc3202180 [SLPQ ksesigwait 0xc67ccb74][SLP]
18344 c60eec48 1001 18343  1199 0008082 (threaded)  thr1
   thread 0xc3781300 ksegrp 0xc27074e0 [SLPQ vmpfw 0xc17db8c8][SLP]
   thread 0xc60f3780 ksegrp 0xc27074e0 [SLPQ kserel 0xc2707514][SLP]
   thread 0xc2ae2d80 ksegrp 0xc27074e0 [SLPQ kserel 0xc2707514][SLP]
   thread 0xc2b08d80 ksegrp 0xc27074e0 [SLPQ kserel 0xc2707514][SLP]
   thread 0xc2b1ba80 ksegrp 0xc2b40900 [SLPQ ksesigwait 0xc60eed80][SLP]
18343 c68aba3c 1001 18299  1199 0000002 [SLPQ wait 0xc68aba3c][SLP] thr1
18342 c60f420c 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18341 c67c7000 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18340 c68ab624 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18339 c2c43a3c 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18338 c67c7c48 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18337 c26a1000 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18336 c2708624 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18335 c275b418 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18334 c275cc48 1001 18310  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] swap
18333 c6817c48 1001 18306  1199 0000002 [SLPQ getblk 0xd63c3210][SLP] rw
18332 c275ca3c 1001 18310  1199 0000002 [SLPQ vmpfw 0xc12315a8][SLP] swap
18331 c3780c48 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18330 c60eea3c 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18329 c2726830 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18328 c2755418 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18327 c6818830 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18326 c2755a3c 1001 18306  1199 0000002 [SLPQ getblk 0xd63c3210][SLP] rw
18325 c612d000 1001 18305  1199 0000002 [SLPQ vmpfw 0xc1a24370][SLP] tcp
18324 c68a8a3c 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18323 c26a1c48 1001 18305  1199 0000002 [SLPQ vmpfw 0xc1a24370][SLP] tcp
18322 c67cba3c 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18321 c68aaa3c 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18320 c3780a3c 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18319 c68a8830 1001 18305  1199 0000002 [SLPQ vmpfw 0xc1a24370][SLP] tcp
18318 c275c20c 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18317 c272d830 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18316 c2c41000 1001 18305  1199 0000002 [SLPQ vmpfw 0xc1a24370][SLP] tcp
18315 c68aac48 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18314 c6817830 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18313 c377d624 1001 18305  1199 0000002 [SLPQ vmpfw 0xc1a24370][SLP] tcp
18312 c2726624 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18311 c60f4a3c 1001 18307  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] mkdir
18310 c68a7c48 1001 18296  1199 0000002 [SLPQ wait 0xc68a7c48][SLP] swap
18309 c681d830 1001 18305  1199 0000002 [SLPQ vmpfw 0xc1a24370][SLP] tcp
18308 c681d624 1001 18306  1199 0000002 [SLPQ newbuf 0xc093f664][SLP] rw
18307 c60ef20c 1001 18298  1199 0000002 [SLPQ wait 0xc60ef20c][SLP] mkdir
18306 c2726000 1001 18300  1199 0000002 [SLPQ wait 0xc2726000][SLP] rw
18305 c681d20c 1001 18303  1199 0000002 [SLPQ wait 0xc681d20c][SLP] tcp
18303 c269fa3c 1001  1206  1199 0004002 [SLPQ vmpfw 0xc17db8c8][SLP] tcp
18300 c67cb418 1001  1206  1199 0004002 [SLPQ newbuf 0xc093f664][SLP] rw
18299 c6134a3c 1001  1206  1199 0004002 [SLPQ vmpfw 0xc17db8c8][SLP] thr1
18298 c2c4120c 1001  1206  1199 0004002 [SLPQ vmpfw 0xc17db8c8][SLP] mkdir
18296 c612e000 1001  1206  1199 0004002 [SLPQ vmpfw 0xc17db8c8][SLP] swap
17407 c2709000    0 17146 17044 0004000 [SLPQ newbuf 0xc093f664][SLP] find
17153 c6816c48    0 17151 17044 0004000 [SLPQ piperd 0xc3d96990][SLP] cat
17151 c681d000    0 17140 17044 0000000 [SLPQ wait 0xc681d000][SLP][SWAP] sh
17150 c2703830    0 17140 17044 0004000 [SLPQ piperd 0xc3d83660][SLP] sort
17149 c275b000    0 17140 17044 0004000 [SLPQ piperd 0xc3d837f8][SLP] sed
17147 c3783830    0 17140 17044 0004000 [SLPQ piperd 0xc2754b28][SLP] xargs
17146 c6816418    0 17140 17044 0000000 [SLPQ wait 0xc6816418][SLP][SWAP] sh
17140 c612d624    0 17137 17044 0004000 [SLPQ wait 0xc612d624][SLP][SWAP] sh
17139 c612d418    0 17138 17044 0004000 [SLPQ piperd 0xc3d7e330][SLP] mail
17138 c377d830    0 17131 17044 0000000 [SLPQ wait 0xc377d830][SLP][SWAP] sh
17137 c6818624    0 17131 17044 0000000 [SLPQ wait 0xc6818624][SLP][SWAP] sh
17131 c2726c48    0 17130 17044 0004000 [SLPQ wait 0xc2726c48][SLP][SWAP] sh
17130 c68a8418    0 17052 17044 0004000 [SLPQ wait 0xc68a8418][SLP][SWAP] sh
17056 c67cc000    0 17053 17044 0004000 [SLPQ piperd 0xc3d96000][SLP] mail
17053 c6134624    0 17046 17044 0000000 [SLPQ wait 0xc6134624][SLP][SWAP] sh
17052 c270820c    0 17046 17044 0000000 [SLPQ wait 0xc270820c][SLP][SWAP] sh
17046 c613420c    0 17044 17044 0004000 [SLPQ wait 0xc613420c][SLP][SWAP] sh
17044 c2708830    0 17042 17044 0004000 [SLPQ wait 0xc2708830][SLP][SWAP] sh
17042 c60f4c48    0  1001  1001 0000000 [SLPQ piperd 0xc3d95198][SLP] cron
 2064 c2755830 1001  2058  2064 0004002 [SLPQ newbuf 0xc093f664][SLP] top
 2058 c3780000 1001  2057  2058 0004002 [SLPQ wait 0xc3780000][SLP][SWAP] bash
 2057 c26a1830 1001  1999  1999 0000100 [SLPQ newbuf 0xc093f664][SLP] sshd
 1999 c3783000    0   979  1999 0004100 [SLPQ sbwait 0xc68a4e90][SLP][SWAP] sshd
 1206 c2709624 1001  1205  1199 0000002 [SLPQ wait 0xc2709624][SLP][SWAP] run
 1205 c275ba3c 1001  1204  1199 0000002 [SLPQ wait 0xc275ba3c][SLP][SWAP] run
 1204 c275bc48 1001  1199  1199 0004002 [SLPQ nanslp 0xc093850c][SLP] run
 1199 c60f4000 1001  1175  1199 0004002 [SLPQ wait 0xc60f4000][SLP][SWAP] sh
 1175 c612e830 1001  1174  1175 0004002 [SLPQ wait 0xc612e830][SLP][SWAP] bash
 1174 c2449624 1001  1172  1172 0000100 [SLPQ select 0xc093f0a4][SLP][SWAP] sshd
 1172 c60ee830    0   979  1172 0004100 [SLPQ sbwait 0xc26f20a8][SLP][SWAP] sshd
 1109 c2708a3c    0     1  1109 0004002 [SLPQ ttyin 0xc250f010][SLP][SWAP] getty
 1108 c60ef624    0     1  1108 0004002 [SLPQ ttyin 0xc2505010][SLP][SWAP] getty
 1107 c2708000    0     1  1107 0004002 [SLPQ ttyin 0xc247b410][SLP][SWAP] getty
 1106 c3783a3c    0     1  1106 0004002 [SLPQ ttyin 0xc250e810][SLP][SWAP] getty
 1105 c275c000    0     1  1105 0004002 [SLPQ ttyin 0xc250e410][SLP][SWAP] getty
 1104 c60ee418    0     1  1104 0004002 [SLPQ ttyin 0xc250d410][SLP][SWAP] getty
 1103 c60ee000    0     1  1103 0004002 [SLPQ ttyin 0xc250d010][SLP][SWAP] getty
 1102 c612ea3c    0     1  1102 0004002 [SLPQ ttyin 0xc250e010][SLP][SWAP] getty
 1088 c275c624    0     1  1088 0000000 [SLPQ select 0xc093f0a4][SLP][SWAP] inetd
 1067 c612dc48    0     1  1067 0000000 [SLPQ select 0xc093f0a4][SLP][SWAP] moused
 1001 c60ef830    0     1  1001 0000000 [SLPQ nanslp 0xc093850c][SLP] cron
  989 c612ec48   25     1   989 0000100 [SLPQ newbuf 0xc093f664][SLP] sendmail
  985 c2726a3c    0     1   985 0000100 [SLPQ newbuf 0xc093f664][SLP] sendmail
  979 c272d20c    0     1   979 0000100 [SLPQ select 0xc093f0a4][SLP][SWAP] sshd
  935 c272620c    0     1   935 0000000 [SLPQ select 0xc093f0a4][SLP] usbd
  844 c2709418    0     1   844 0000000 [SLPQ select 0xc093f0a4][SLP][SWAP] devd
  160 c2726418    0     1   160 0000000 [SLPQ select 0xc093f0a4][SLP] syslogd
  108 c239a20c    0     0     0 0000204 [SLPQ - 0xe5101d08][SLP] schedcpu
  107 c239a418    0     0     0 0000204 [SLPQ - 0xc094762c][SLP] nfsiod 3
  106 c239a624    0     0     0 0000204 [SLPQ - 0xc0947628][SLP] nfsiod 2
  105 c239a830    0     0     0 0000204 [SLPQ - 0xc0947624][SLP] nfsiod 1
  104 c239aa3c    0     0     0 0000204 [SLPQ - 0xc0947620][SLP] nfsiod 0
  103 c239ac48    0     0     0 0000204 [SLPQ getblk 0xd63c3210][SLP] syncer
  102 c2447000    0     0     0 0000204 [SLPQ vlruwt 0xc2447000][SLP] vnlru
  101 c244720c    0     0     0 0000204 [SLPQ newbuf 0xc093f664][SLP] bufdaemon
  100 c2447418    0     0     0 000020c [SLPQ pgzero 0xc094dbe4][SLP] pagezero
   99 c2447624    0     0     0 0000204 [SLPQ psleep 0xc094d734][SLP] vmdaemon
   98 c2447830    0     0     0 0000204 [SLPQ psleep 0xc094d6f0][SLP] pagedaemon
   97 c2447a3c    0     0     0 0000204 [SLPQ - 0xc2446e3c][SLP] fdc0
   96 c2447c48    0     0     0 0000204 [IWAIT] swi0: sio
   95 c2449000    0     0     0 0000204 [SLPQ usbtsk 0xc0935544][SLP] usbtask
   94 c244920c    0     0     0 0000204 [SLPQ usbevt 0xc246e210][SLP] usb0
   93 c2449418    0     0     0 0000204 [IWAIT] swi6: task queue
    9 c2394000    0     0     0 0000204 [SLPQ - 0xc2370a80][SLP] acpi_task2
    8 c239420c    0     0     0 0000204 [SLPQ - 0xc2370a80][SLP] acpi_task1
    7 c2394418    0     0     0 0000204 [SLPQ - 0xc2370a80][SLP] acpi_task0
   92 c2394624    0     0     0 0000204 [IWAIT] swi2: cambio
    6 c2394830    0     0     0 0000204 [SLPQ - 0xc2370c00][SLP] kqueue taskq
   91 c2394a3c    0     0     0 0000204 [IWAIT] swi5:+
    5 c2394c48    0     0     0 0000204 [SLPQ - 0xc2370d80][SLP] thread taskq
   90 c2399000    0     0     0 0000204 [IWAIT] swi6:+
   89 c239920c    0     0     0 0000204 [SLPQ - 0xc0933260][SLP] yarrow
    4 c2399418    0     0     0 0000204 [SLPQ - 0xc0935a28][SLP] g_down
    3 c2399624    0     0     0 0000204 [SLPQ - 0xc0935a24][SLP] g_up
    2 c2399830    0     0     0 0000204 [SLPQ - 0xc0935a1c][SLP] g_event
   88 c2399a3c    0     0     0 0000204 [IWAIT] swi1: net
   87 c2399c48    0     0     0 0000204 [IWAIT] swi3: vm
   86 c239a000    0     0     0 000020c [IWAIT] swi4: clock sio
   85 c237f000    0     0     0 0000204 [IWAIT] irq71:
   84 c237f20c    0     0     0 0000204 [IWAIT] irq70:
   83 c237f418    0     0     0 0000204 [IWAIT] irq69:
   82 c237f624    0     0     0 0000204 [IWAIT] irq68:
   81 c237f830    0     0     0 0000204 [IWAIT] irq67:
   80 c237fa3c    0     0     0 0000204 [IWAIT] irq66:
   79 c237fc48    0     0     0 0000204 [IWAIT] irq65:
   78 c2384000    0     0     0 0000204 [IWAIT] irq64:
   77 c238420c    0     0     0 0000204 [IWAIT] irq63:
   76 c2384418    0     0     0 0000204 [IWAIT] irq62:
   75 c2384624    0     0     0 0000204 [IWAIT] irq61:
   74 c2384830    0     0     0 0000204 [IWAIT] irq60:
   73 c2384a3c    0     0     0 0000204 [IWAIT] irq59:
   72 c2384c48    0     0     0 0000204 [IWAIT] irq58:
   71 c235f20c    0     0     0 0000204 [IWAIT] irq57:
   70 c235f418    0     0     0 0000204 [IWAIT] irq56:
   69 c235f624    0     0     0 0000204 [IWAIT] irq55:
   68 c235f830    0     0     0 0000204 [IWAIT] irq54:
   67 c235fa3c    0     0     0 0000204 [IWAIT] irq53:
   66 c235fc48    0     0     0 0000204 [IWAIT] irq52:
   65 c2371000    0     0     0 0000204 [IWAIT] irq51:
   64 c237120c    0     0     0 0000204 [IWAIT] irq50:
   63 c2371418    0     0     0 0000204 [IWAIT] irq49:
   62 c2371624    0     0     0 0000204 [IWAIT] irq48: em0
   61 c2371830    0     0     0 0000204 [IWAIT] irq47:
   60 c2371a3c    0     0     0 0000204 [IWAIT] irq46:
   59 c2371c48    0     0     0 0000204 [IWAIT] irq45:
   58 c2347624    0     0     0 0000204 [IWAIT] irq44:
   57 c2347830    0     0     0 0000204 [IWAIT] irq43:
   56 c2347a3c    0     0     0 0000204 [IWAIT] irq42:
   55 c2347c48    0     0     0 0000204 [IWAIT] irq41:
   54 c235e000    0     0     0 0000204 [IWAIT] irq40:
   53 c235e20c    0     0     0 0000204 [IWAIT] irq39:
   52 c235e418    0     0     0 0000204 [IWAIT] irq38:
   51 c235e624    0     0     0 0000204 [IWAIT] irq37:
   50 c235e830    0     0     0 0000204 [IWAIT] irq36:
   49 c235ea3c    0     0     0 0000204 [IWAIT] irq35:
   48 c235ec48    0     0     0 0000204 [IWAIT] irq34:
   47 c235f000    0     0     0 0000204 [IWAIT] irq33:
   46 c2338c48    0     0     0 0000204 [IWAIT] irq32:
   45 c2346000    0     0     0 0000204 [IWAIT] irq31:
   44 c234620c    0     0     0 0000204 [IWAIT] irq30:
   43 c2346418    0     0     0 0000204 [IWAIT] irq29:
   42 c2346624    0     0     0 0000204 [IWAIT] irq28:
   41 c2346830    0     0     0 0000204 [IWAIT] irq27:
   40 c2346a3c    0     0     0 0000204 [IWAIT] irq26:
   39 c2346c48    0     0     0 0000204 [IWAIT] irq25:
   38 c2347000    0     0     0 0000204 [IWAIT] irq24:
   37 c234720c    0     0     0 0000204 [IWAIT] irq23:
   36 c2347418    0     0     0 0000204 [IWAIT] irq22:
   35 c232e624    0     0     0 0000204 [IWAIT] irq21:
   34 c232e830    0     0     0 0000204 [IWAIT] irq20:
   33 c232ea3c    0     0     0 0000204 [IWAIT] irq19:
   32 c232ec48    0     0     0 0000204 [IWAIT] irq18:
   31 c2338000    0     0     0 0000204 [IWAIT] irq17: fxp0
   30 c233820c    0     0     0 0000204 [IWAIT] irq16: uhci0
   29 c2338418    0     0     0 0000204 [IWAIT] irq15: ata1
   28 c2338624    0     0     0 0000204 [IWAIT] irq14: ata0
   27 c2338830    0     0     0 0000204 [IWAIT] irq13:
   26 c2338a3c    0     0     0 0000204 [IWAIT] irq12: psm0
   25 c22db20c    0     0     0 0000204 [IWAIT] irq11:
   24 c22db418    0     0     0 0000204 [IWAIT] irq10:
   23 c22db624    0     0     0 0000204 [IWAIT] irq9: acpi0
   22 c22db830    0     0     0 0000204 [IWAIT] irq8:
   21 c22dba3c    0     0     0 0000204 [IWAIT] irq7: ppc0
   20 c22dbc48    0     0     0 0000204 [IWAIT] irq6: fdc0
   19 c232e000    0     0     0 0000204 [IWAIT] irq5:
   18 c232e20c    0     0     0 0000204 [IWAIT] irq4: sio0
   17 c232e418    0     0     0 0000204 [IWAIT] irq3: sio1
   16 c22d6000    0     0     0 0000204 [IWAIT] irq0:
   15 c22d620c    0     0     0 0000204 [IWAIT] irq1: atkbd0
   14 c22d6418    0     0     0 000020c [CPU 0] idle: cpu0
   13 c22d6624    0     0     0 000020c [CPU 1] idle: cpu1
   12 c22d6830    0     0     0 000020c [CPU 2] idle: cpu2
   11 c22d6a3c    0     0     0 000020c [CPU 3] idle: cpu3
    1 c22d6c48    0     0     1 0004200 [SLPQ wait 0xc22d6c48][SLP] init
   10 c22db000    0     0     0 0000204 [SLPQ ktrace 0xc0936478][SLP] ktrace
    0 c0935b20    0     0     0 0000200 [IWAIT] swapper
18352 c377d20c 1001 18316  1199 0002002 zomb[INACTIVE] tcp
18351 c6818c48 1001 18323  1199 0002002 zomb[INACTIVE] tcp
18350 c2c43000 1001 18309  1199 0002002 zomb[INACTIVE] tcp
18353 c612da3c 1001 18325  1199 0002002 zomb[INACTIVE] tcp
18349 c2c41418 1001 18319  1199 0002002 zomb[INACTIVE] tcp
18348 c681820c 1001 18313  1199 0002002 zomb[INACTIVE] tcp
18297 c6134c48 1001  1206  1199 0006002 zomb[INACTIVE] creat
18304 c6817624 1001  1206  1199 0006002 zomb[INACTIVE] udp
18302 c6817418 1001  1206  1199 0006002 zomb[INACTIVE] sysctl
18301 c2c43830 1001  1206  1199 0006002 zomb[INACTIVE] syscall
db>  where 18437
Tracing pid 18437 tid 100367 td 0xc3a3cd80
sched_switch(c3a3cd80,0,1) at sched_switch+0x14b
mi_switch(1,0,c3a3cd80,0,c3a3cd80) at mi_switch+0x1ba
sleepq_switch(c16bcf00) at sleepq_switch+0x133
sleepq_wait(c16bcf00,0,c26b0220,0,c2c57b58) at sleepq_wait+0xb
msleep(c16bcf00,c094d6c0,244,c088d9a0,0) at msleep+0x362
vm_page_sleep_if_busy(c16bcf00,1,c088d9a0) at vm_page_sleep_if_busy+0x75
vm_fault(c465ece4,28126000,1,0,c3a3cd80) at vm_fault+0x5df
trap_pfault(e7ce5d38,1,28126cb4,28126cb4,0) at trap_pfault+0x123
trap(bfbf003b,ffff003b,bfbf003b,0,8051040) at trap+0x1ff
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x28126cb4, esp = 0xbfbfec6c, ebp = 0xbfbfed78 ---
db> where 18354
Tracing pid 18354 tid 100167 td 0xc2757c00
sched_switch(c2757c00,0,1) at sched_switch+0x14b
mi_switch(1,0,c2757c00,0,c2757c00) at mi_switch+0x1ba
sleepq_switch(c093f664) at sleepq_switch+0x133
sleepq_wait(c093f664,0,0,0,c087d928) at sleepq_wait+0xb
msleep(c093f664,c093f680,50,c087d928,0) at msleep+0x362
getnewbuf(0,0,4000,4000,fffd0000) at getnewbuf+0x692
getblk(c26b0220,fffffff4,ffffffff,4000,0) at getblk+0x370
ufs_bmaparray(c26b0220,28,0,e76d1b24,0) at ufs_bmaparray+0x5c6
ufs_bmap(e76d1b70) at ufs_bmap+0x4b
VOP_BMAP_APV(c090f360,e76d1b70) at VOP_BMAP_APV+0x41
vnode_pager_haspage(c1067528,a0,0,e76d1bd8,e76d1bd4) at vnode_pager_haspage+0x23a
vm_fault_additional_pages(c1628670,0,f,e76d1c84,e76d1c48) at vm_fault_additional_pages+0x59
vm_fault(c56de384,28126000,1,0,c2757c00) at vm_fault+0xaef
trap_pfault(e76d1d38,1,28126cb4,28126cb4,0) at trap_pfault+0x123
trap(bfbf003b,ffff003b,bfbf003b,0,8051040) at trap+0x1ff
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x28126cb4, esp = 0xbfbfec6c, ebp = 0xbfbfed78 ---
db> x needsbuffer
needsbuffer:    1
db> where 101
Tracing pid 101 tid 100099 td 0xc239e000
sched_switch(c239e000,0,1) at sched_switch+0x14b
mi_switch(1,0,c239e000,1,c239e000) at mi_switch+0x1ba
sleepq_switch(c093f664,c239e000,e514cabc,c0644ce5,c093f664) at sleepq_switch+0x133
sleepq_timedwait(c093f664,0,23,0,c087d928) at sleepq_timedwait+0xc
msleep(c093f664,c093f680,50,c087d928,3e8) at msleep+0x339
getnewbuf(0,3e8,4000,4000,d63c31b0) at getnewbuf+0x692
geteblk(4000,4000,c26a3990,d63c31b0,e514cb74) at geteblk+0x20
ffs_bufwrite(d63c31b0) at ffs_bufwrite+0x18d
vfs_bio_awrite(d63c31b0,c2339600,356,1,c239e000) at vfs_bio_awrite+0x207
flushbufqueues(0) at flushbufqueues+0x443
buf_daemon(0,e514cd38) at buf_daemon+0xf7
fork_exit(c0686ed4,0,e514cd38) at fork_exit+0x71
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe514cd6c, ebp = 0 ---
db> call doadump
Dumping 1023 MB (2 chunks)
  chunk 0: 1MB (159 pages) ... ok
  chunk 1: 1023MB (261872 pages) 1007 991 975 959 943 927 911 895 879 863 847 831 815 799 783 767 751 735 719 703 687 671 655 639 623 607 591 575 559 543 527 511 495 479 463 447 431 415 399 383 367 351 335 319 303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 ... ok

Dump complete
= 0xf
db> reset

(kgdb) l *getnewbuf+0x692
0xc0686d3a is in getnewbuf (../../../kern/vfs_bio.c:1900).
1895             needsbuffer |= flags;
1896             while (needsbuffer & flags) {
1897               /* Don't let the buffer daemon sleep forever */
1898               if (!slptimeo && curthread->td_proc == bufdaemonproc)
1899                    slptimeo = hz;
1900               if (msleep(&needsbuffer, &nblock,
1901                   (PRIBIO + 4) | slpflag, waitmsg, slptimeo)) {
1902                    mtx_unlock(&nblock);
1903                    return (NULL);
1904               }
(kgdb) l *vm_fault+0x5df
0xc07a6bc3 is in vm_fault (../../../vm/vm_fault.c:395).
390                     }
391                     VM_OBJECT_LOCK(fs.object);
392                     if (fs.m == vm_page_lookup(fs.object,
393                         fs.pindex)) {
394                           vm_page_lock_queues();
395                           if (!vm_page_sleep_if_busy(fs.m, TRUE,
396                               "vmpfw"))
397                             vm_page_unlock_queues();
398                     }
399                     vm_object_pip_wakeup(fs.object);