GENERIC HEAD from Apr 10 13:22 UTC, vmcore.281
Livelock
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 6.0-CURRENT #3: Sun Apr 10 15:24:53 CEST 2005
pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.14-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf13 Stepping = 3
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory = 267583488 (255 MB)
avail memory = 252354560 (240 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
lock order reversal
1st 0xc098f080 vm page queue mutex (vm page queue mutex) @ kern/vfs_bio.c:1485
2nd 0xc1753cc8 vnode interlock (vnode interlock) @ kern/vfs_subr.c:1989
KDB: stack backtrace:
kdb_backtrace(0,ffffffff,c0944348,c0942bb0,c089f288) at kdb_backtrace+0x29
witness_checkorder(c1753cc8,9,c0844e21,7c5) at witness_checkorder+0x550
_mtx_lock_flags(c1753cc8,0,c0844e18,7c5,c105c294) at _mtx_lock_flags+0x5b
vdrop(c1753c30) at vdrop+0x20
vm_page_remove(c13e61d8,c13e61d8) at vm_page_remove+0xd4
vm_page_free_toq(c13e61d8,c13e61d8,40,c13e61d8,cbf7d8ec) at vm_page_free_toq+0x90
vm_page_free(c13e61d8,c13e61d8) at vm_page_free+0x15
vfs_vmio_release(c65dd210) at vfs_vmio_release+0x9b
brelse(c65dd210,c151fa80,1,c0613a9c,0) at brelse+0x4ad
ffs_mountfs(c1753c30,c175c400,c1522480,c16b85a0,0) at ffs_mountfs+0x563
ffs_mount(c175c400,c1522480,0,0,c1754000) at ffs_mount+0x9f5
vfs_domount(c1522480,c16b8510,c16b8560,4001,c16b84d0) at vfs_domount+0x5b1
vfs_donmount(c1522480,4001,cbf7dc30,c1732980,6) at vfs_donmount+0xce
kernel_mount(c16b8490,4001,cbf7dc98,cbf7dcc4,c066b5c2) at kernel_mount+0x6d
kernel_vmount(4001,c084489a,c16b8470,c08448a1,c0839c49) at kernel_vmount+0x37
vfs_mountroot_try(c16b8450,c083829e,204,c1521de4,c05f5524) at vfs_mountroot_try+0xd2
vfs_mountroot(c1521de4,c1522480,25d,c1522480,cbf7dd08) at vfs_mountroot+0x18f
start_init(0,cbf7dd48,0,c05f5524,0) at start_init+0x4b
fork_exit(c05f5524,0,cbf7dd48) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xcbf7dd7c, ebp = 0 ---
rl0: link state changed to DOWN
ad0: WARNING - READ_DMA interrupt was seen but timeout fired LBA=615167
~KDB: enter: Line break on console
[thread pid 32 tid 100025 ]
Stopped at kdb_enter+0x2b: nop
db> where
Tracing pid 32 tid 100025 td 0xc1558480
kdb_enter(c085b9e4) at kdb_enter+0x2b
siointr1(c16a1000,c099b8c0,0,c085b7f4,56f) at siointr1+0xce
siointr(c16a1000) at siointr+0x21
intr_execute_handlers(c1510090,cc9a7c94,4,cc9a7cdc,c07c0b23) at intr_execute_handlers+0x9d
lapic_handle_intr(34) at lapic_handle_intr+0x2e
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc07c9727, esp = 0xcc9a7cd8, ebp = 0xcc9a7cdc ---
spinlock_exit(0,cc9a7d1c,c06093ef,c0933540,0) at spinlock_exit+0x27
_mtx_unlock_spin_flags(c0933540,0,c08396b0,256) at _mtx_unlock_spin_flags+0x8d
ithread_loop(c1556700,cc9a7d48,c1556700,c060919c,0) at ithread_loop+0x253
fork_exit(c060919c,c1556700,cc9a7d48) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xcc9a7d7c, ebp = 0 ---
db> x panic_pid
panic_pid: 0 db> show alllocks
Process 2671 (udp) thread 0xc1adc180 (100142)
exclusive sleep mutex inp (udpinp) r = 0 (0xc17f5798) locked @ netinet/udp_usrreq.c:772
Process 570 (top) thread 0xc187a000 (100099)
shared sx allproc r = 0 (0xc0933600) locked @ kern/kern_proc.c:228
exclusive sx sysctl lock r = 0 (0xc0933bc0) locked @ kern/kern_sysctl.c:1335
db> show lockedvnods
Locked vnodes
0xc1aaad68: tag ufs, type VDIR
usecount 32, writecount 0, refcount 34 mountedhere 0
flags ()
v_object 0xc1aa7084 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xc1f41780 (pid 2676) with 2 pending
ino 71075, on dev ad0s1e
0xc1fb4270: tag ufs, type VDIR
usecount 2, writecount 0, refcount 7 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc187a480 (pid 2662)
ino 73979, on dev ad0s1e
0xc20aa750: tag ufs, type VDIR
usecount 2, writecount 0, refcount 7 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc187aa80 (pid 2663)
ino 74015, on dev ad0s1e
0xc1e68888: tag ufs, type VDIR
usecount 2, writecount 0, refcount 7 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1ec4a80 (pid 2664)
ino 96780, on dev ad0s1e
0xc1fcbc30: tag ufs, type VREG
usecount 1, writecount 1, refcount 1625 mountedhere 0
flags ()
v_object 0xc1c1239c ref 0 pages 6492
lock type ufs: EXCL (count 1) by thread 0xc1f40c00 (pid 2666)
ino 97009, on dev ad0s1e
0xc1c0dd68: tag ufs, type VNON
usecount 1, writecount 0, refcount 1 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc187aa80 (pid 2663)
ino 82263, on dev ad0s1e
0xc1f95138: tag ufs, type VDIR
usecount 2, writecount 0, refcount 3 mountedhere 0
flags ()
lock type ufs: EXCL (count 2) by thread 0xc1f41780 (pid 2676)
ino 82288, on dev ad0s1e
0xc1d74618: tag ufs, type VNON
usecount 1, writecount 0, refcount 1 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc187a480 (pid 2662)
ino 82289, on dev ad0s1e
0xc17a9d68: tag syncer, type VNON
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type syncer: EXCL (count 1) by thread 0xc1571480 (pid 55)
0xc18064e0: tag ufs, type VREG
usecount 2, writecount 1, refcount 4 mountedhere 0
flags ()
v_object 0xc18025ac ref 0 pages 2
lock type ufs: EXCL (count 1) by thread 0xc1571480 (pid 55)
ino 47302, on dev ad0s1d
db> ps
pid proc uid ppid pgrp flag stat wmesg wchan cmd
2680 c1afa5f4 1001 2678 587 0008082 (threaded) thr1
thread 0xc1f42c00 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc233ec00 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc1f41d80 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc1a72600 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc20e1480 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc1df0180 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc1e89a80 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc187ad80 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc1f41000 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc250c180 ksegrp 0xc182cd80 [SLPQ pfault 0xc098ee18][SLP]
thread 0xc1abb000 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc1e89d80 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc1e89180 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc250d480 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc250c780 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc250c000 ksegrp 0xc182cd80 [SLPQ kserel 0xc182cdb4][SLP]
thread 0xc250e000 ksegrp 0xc1d5fea0 [SLPQ pfault 0xc098ee18][SLP]
2679 c1c9b5f4 1001 2678 587 0008082 (threaded) thr1
thread 0xc1df0a80 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc2509000 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc250a900 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc1e13d80 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc1eca600 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc250a480 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc250b300 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc250a000 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc250b180 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc2508a80 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc2509180 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc182bc00 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc250b780 ksegrp 0xc1ada660 [SLPQ pfault 0xc098ee18][SLP]
thread 0xc1ca9a80 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc1ec8c00 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc1f40480 ksegrp 0xc1ada660 [SLPQ kserel 0xc1ada694][SLP]
thread 0xc1e0e780 ksegrp 0xc1d5fe40 [SLPQ pfault 0xc098ee18][SLP]
2678 c1aba7f0 1001 2651 587 0000002 [SLPQ wait 0xc1aba7f0][SLP][SWAP] thr1
2677 c1ab97f0 1001 2668 587 0000002 [SLPQ ufs 0xc1aaadc0][SLP] mkdir
2676 c20dc1fc 1001 2668 587 0000002 [SLPQ getblk 0xc6602b30][SLP] mkdir
2675 c1b1e5f4 1001 2668 587 0000002 [SLPQ ufs 0xc1aaadc0][SLP] mkdir
2674 c1ad9be8 1001 2667 587 0000002 [RUNQ] udp
2673 c18751fc 1001 2667 587 0000002 [RUNQ] udp
2672 c1d5e5f4 1001 2667 587 0000002 [SLPQ pfault 0xc098ee18][SLP] udp
2671 c1afabe8 1001 2667 587 0000002 [RUNQ] udp
2670 c20dc9ec 1001 2667 587 0000002 [RUNQ] udp
2669 c1c573f8 1001 2667 587 0000002 [RUNQ] udp
2668 c1ca1be8 1001 2650 587 0000002 [SLPQ wait 0xc1ca1be8][SLP][SWAP] mkdir
2667 c1c9bbe8 1001 2656 587 0000002 [SLPQ wait 0xc1c9bbe8][SLP][SWAP] udp
2666 c1afbde4 1001 2665 587 0000002 [SLPQ wdrain 0xc0981644][SLP] rw
2665 c1adbde4 1001 2652 587 0000002 [SLPQ wait 0xc1adbde4][SLP][SWAP] rw
2664 c1d5ebe8 1001 2658 587 0000002 [SLPQ getblk 0xc66928ac][SLP] creat
2663 c1ad97f0 1001 2658 587 0000002 [SLPQ getblk 0xc6602b30][SLP] creat
2662 c1d5e1fc 1001 2658 587 0000002 [SLPQ getblk 0xc6602b30][SLP] creat
2661 c20dc3f8 1001 2657 587 0000002 [SLPQ pfault 0xc098ee18][SLP] swap
2660 c18769ec 1001 2657 587 0000002 [SLPQ pfault 0xc098ee18][SLP] swap
2659 c20dc7f0 1001 2657 587 0000002 [RUNQ] swap
2658 c1afa9ec 1001 2649 587 0000002 [SLPQ wait 0xc1afa9ec][SLP][SWAP] creat
2657 c1ca15f4 1001 2648 587 0000002 [SLPQ wait 0xc1ca15f4][SLP][SWAP] swap
2656 c1afb1fc 1001 594 587 0004002 [SLPQ pfault 0xc098ee18][SLP] udp
2652 c1c575f4 1001 594 587 0004002 [SLPQ pfault 0xc098ee18][SLP] rw
2651 c1c571fc 1001 594 587 0004002 [SLPQ pfault 0xc098ee18][SLP] thr1
2650 c1adb000 1001 594 587 0004002 [SLPQ pfault 0xc098ee18][SLP] mkdir
2649 c1d5e000 1001 594 587 0004002 [SLPQ pfault 0xc098ee18][SLP] creat
2648 c1ad91fc 1001 594 587 0004002 [SLPQ pfault 0xc098ee18][SLP] swap
594 c182a9ec 1001 593 587 0000002 [SLPQ wait 0xc182a9ec][SLP] run
593 c182abe8 1001 592 587 0000002 [SLPQ wait 0xc182abe8][SLP] run
592 c172ede4 1001 587 587 0004002 [SLPQ nanslp 0xc0933dec][SLP] run
587 c172fde4 1001 581 587 0004002 [SLPQ wait 0xc172fde4][SLP] sh
581 c182a7f0 1001 580 581 0004002 [SLPQ wait 0xc182a7f0][SLP][SWAP] bash
580 c1875000 1001 578 578 0000100 [SLPQ select 0xc09810c4][SLP][SWAP] sshd
578 c18753f8 0 439 578 0004100 [SLPQ sbwait 0xc183ec48][SLP][SWAP] sshd
570 c18765f4 1001 564 570 0004002 [SLPQ vmwait 0xc098ee18][SLP] top
564 c172f9ec 1001 563 564 0004002 [SLPQ wait 0xc172f9ec][SLP][SWAP] bash
563 c18761fc 1001 561 561 0000100 [SLPQ select 0xc09810c4][SLP] sshd
561 c18299ec 0 439 561 0004100 [SLPQ sbwait 0xc183e1e8][SLP][SWAP] sshd
560 c18757f0 0 1 560 0004002 [SLPQ ttyin 0xc16b2c10][SLP][SWAP] getty
559 c18763f8 0 1 559 0004002 [SLPQ ttyin 0xc16b3010][SLP][SWAP] getty
558 c18759ec 0 1 558 0004002 [SLPQ ttyin 0xc16b3410][SLP][SWAP] getty
557 c18293f8 0 1 557 0004002 [SLPQ ttyin 0xc16aa010][SLP][SWAP] getty
556 c1829de4 0 1 556 0004002 [SLPQ ttyin 0xc16a8410][SLP][SWAP] getty
555 c1829be8 0 1 555 0004002 [SLPQ ttyin 0xc16aa410][SLP][SWAP] getty
554 c18755f4 0 1 554 0004002 [SLPQ ttyin 0xc16a7c10][SLP][SWAP] getty
553 c1875be8 0 1 553 0004002 [SWAP] getty
523 c1875de4 0 1 523 0000000 [SLPQ select 0xc09810c4][SLP][SWAP] moused
461 c182a000 0 1 461 0000000 [SWAP] cron
449 c18291fc 25 1 449 0000100 [SWAP] sendmail
445 c182a1fc 0 1 445 0000100 [SLPQ pfault 0xc098ee18][SLP] sendmail
439 c1829000 0 1 439 0000100 [SLPQ select 0xc09810c4][SLP][SWAP] sshd
421 c172e000 0 1 421 0000000 [SLPQ pfault 0xc098ee18][SLP] ntpd
390 c18295f4 0 1 390 0000000 [SWAP] usbd
370 c182a3f8 0 365 365 0000000 [SLPQ - 0xc1828400][SLP][SWAP] nfsd
369 c182a5f4 0 365 365 0000000 [SLPQ - 0xc1828800][SLP][SWAP] nfsd
368 c172e9ec 0 365 365 0000000 [SLPQ - 0xc1828a00][SLP][SWAP] nfsd
367 c172e1fc 0 365 365 0000000 [SLPQ - 0xc1828600][SLP][SWAP] nfsd
365 c172e3f8 0 1 365 0000000 [SLPQ select 0xc09810c4][SLP][SWAP] nfsd
363 c172e5f4 0 1 363 0000000 [SLPQ select 0xc09810c4][SLP][SWAP] mountd
294 c172e7f0 0 1 294 0000000 [SWAP] rpcbind
280 c172ebe8 0 1 280 0000000 [SLPQ pfault 0xc098ee18][SLP] syslogd
251 c1645de4 0 1 251 0000000 [SLPQ select 0xc09810c4][SLP][SWAP] devd
60 c172f000 0 0 0 0000204 [SLPQ - 0xce9e4d14][SLP] schedcpu
59 c172f1fc 0 0 0 0000204 [SLPQ - 0xc0988dac][SLP] nfsiod 3
58 c172f3f8 0 0 0 0000204 [SLPQ - 0xc0988da8][SLP] nfsiod 2
57 c172f5f4 0 0 0 0000204 [SLPQ - 0xc0988da4][SLP] nfsiod 1
56 c172f7f0 0 0 0 0000204 [SLPQ - 0xc0988da0][SLP] nfsiod 0
55 c15723f8 0 0 0 0000204 [SLPQ vmwait 0xc098ee18][SLP] syncer
54 c15725f4 0 0 0 0000204 [SLPQ vlruwt 0xc15725f4][SLP] vnlru
53 c15727f0 0 0 0 0000204 [SLPQ psleep 0xc0981610][SLP] bufdaemon
52 c15729ec 0 0 0 000020c [RUNQ] pagezero
51 c1572be8 0 0 0 0000204 [SLPQ psleep 0xc098f0f4][SLP] vmdaemon
9 c1572de4 0 0 0 0000204 [SLPQ wswbuf0 0xc098e654][SLP] pagedaemon
8 c1645000 0 0 0 0000204 [SLPQ - 0xc168183c][SLP] fdc0
50 c16451fc 0 0 0 0000204 [IWAIT] swi0: sio
49 c16453f8 0 0 0 0000204 [SLPQ usbevt 0xc160f210][SLP] usb4
48 c16455f4 0 0 0 0000204 [SLPQ usbevt 0xc165b210][SLP] usb3
47 c16457f0 0 0 0 0000204 [SLPQ usbevt 0xc1648210][SLP] usb2
46 c16459ec 0 0 0 0000204 [SLPQ usbevt 0xc164c210][SLP] usb1
45 c1645be8 0 0 0 0000204 [SLPQ usbtsk 0xc08feb54][SLP] usbtask
44 c1565be8 0 0 0 0000204 [SLPQ usbevt 0xc161e210][SLP] usb0
7 c1565de4 0 0 0 0000204 [SLPQ actask 0xc0af5f2c][SLP] acpi_task0
43 c1570000 0 0 0 0000204 [IWAIT] swi5:+
42 c15701fc 0 0 0 0000204 [IWAIT] swi6: acpitaskq
6 c15703f8 0 0 0 0000204 [SLPQ - 0xc1618c80][SLP] thread taskq
41 c15705f4 0 0 0 0000204 [IWAIT] swi6:+
40 c15707f0 0 0 0 0000204 [IWAIT] swi6: task queue
39 c15709ec 0 0 0 0000204 [IWAIT] swi2: cambio
5 c1570be8 0 0 0 0000204 [SLPQ - 0xc159ba00][SLP] kqueue taskq
38 c1570de4 0 0 0 0000204 [SLPQ - 0xc08fc880][SLP] yarrow
4 c1572000 0 0 0 0000204 [SLPQ g_down 0xcc9cbcd0][SLP] g_down
3 c15721fc 0 0 0 0000204 [SLPQ - 0xc0901344][SLP] g_up
2 c15575f4 0 0 0 0000204 [SLPQ - 0xc090133c][SLP] g_event
37 c15577f0 0 0 0 0000204 [IWAIT] swi1: net
36 c15579ec 0 0 0 0000204 [IWAIT] swi3: vm
35 c1557be8 0 0 0 000020c [IWAIT] swi4: clock sio
34 c1557de4 0 0 0 0000204 [IWAIT] irq0: clk
33 c1565000 0 0 0 0000204 [IWAIT] irq23: ehci0
32 c15651fc 0 0 0 0000204 [CPU 0] irq22: rl0
31 c15653f8 0 0 0 0000204 [IWAIT] irq21:
30 c15655f4 0 0 0 0000204 [IWAIT] irq20:
29 c15657f0 0 0 0 0000204 [IWAIT] irq19: uhci1
28 c15659ec 0 0 0 0000204 [IWAIT] irq18: uhci2
27 c15261fc 0 0 0 0000204 [IWAIT] irq17: pcm0
26 c15263f8 0 0 0 0000204 [IWAIT] irq16: uhci0 uhci3
25 c15265f4 0 0 0 0000204 [IWAIT] irq15: ata1
24 c15267f0 0 0 0 0000204 [IWAIT] irq14: ata0
23 c15269ec 0 0 0 0000204 [IWAIT] irq13:
22 c1526be8 0 0 0 0000204 [IWAIT] irq12: psm0
21 c1526de4 0 0 0 0000204 [IWAIT] irq11:
20 c1557000 0 0 0 0000204 [IWAIT] irq10:
19 c15571fc 0 0 0 0000204 [IWAIT] irq9: acpi0
18 c15573f8 0 0 0 0000204 [IWAIT] irq8: rtc
17 c1521000 0 0 0 0000204 [IWAIT] irq7: ppc0
16 c15211fc 0 0 0 0000204 [IWAIT] irq6: fdc0
15 c15213f8 0 0 0 0000204 [IWAIT] irq5:
14 c15215f4 0 0 0 0000204 [IWAIT] irq4: sio0
13 c15217f0 0 0 0 0000204 [IWAIT] irq3:
12 c15219ec 0 0 0 0000204 [IWAIT] irq1: atkbd0
11 c1521be8 0 0 0 000020c [Can run] idle: cpu0
1 c1521de4 0 0 1 0004200 [SLPQ wait 0xc1521de4][SLP] init
10 c1526000 0 0 0 0000204 [SLPQ ktrace 0xc0931db8][SLP] ktrace
0 c0901440 0 0 0 0000200 [SLPQ vmwait 0xc098ee18][SLP] swapper
2653 c1ad95f4 1001 594 587 0006002 zomb[INACTIVE] syscall
2655 c1afa1fc 1001 594 587 0006002 zomb[INACTIVE] tcp
2654 c1ad99ec 1001 594 587 0006002 zomb[INACTIVE] sysctl
db> call doadump
Dumping 255 MB
16 32 48 64 80 96 112 128 144 160 176 192 208 224 240
Dump complete
0xf
db> reset
(kgdb) btp 570
frame 0 at 0xcf3c9740: ebp cf3c9790, eip 0xc06215b4 <mi_switch+624>: add $0xc,%esp
frame 1 at 0xcf3c9790: ebp cf3c97a8, eip 0xc0638c34 <sleepq_switch+224>: add $0x8,%esp
frame 2 at 0xcf3c97a8: ebp cf3c97b4, eip 0xc0638e14 <sleepq_wait+48>: push $0x220
frame 3 at 0xcf3c97b4: ebp cf3c97e4, eip 0xc0621225 <msleep+785>: mov $0x0,%edi
frame 4 at 0xcf3c97e4: ebp cf3c9800, eip 0xc0778d5b <vm_wait+147>: add $0x14,%esp
frame 5 at 0xcf3c9800: ebp cf3c9824, eip 0xc077972d <vm_page_grab+305>: push $0x58c
frame 6 at 0xcf3c9824: ebp cf3c98d0, eip 0xc076ee90 <vm_thread_swapin+92>: mov %eax,0xffffff70(%ebp)
frame 7 at 0xcf3c98d0: ebp cf3c98e4, eip 0xc076f282 <faultin+198>: add $0x4,%esp
frame 8 at 0xcf3c98e4: ebp cf3c9b94, eip 0xc06165ea <sysctl_out_proc+190>: add $0x4,%esp
frame 9 at 0xcf3c9b94: ebp cf3c9bc0, eip 0xc0616b60 <sysctl_kern_proc+1084>: mov %eax,%ebx
frame 10 at 0xcf3c9bc0: ebp cf3c9bec, eip 0xc0622bef <sysctl_root+283>: mov %eax,%edx
frame 11 at 0xcf3c9bec: ebp cf3c9c40, eip 0xc0622dc0 <userland_sysctl+244>: mov %eax,%ebx
frame 12 at 0xcf3c9c40: ebp cf3c9cec, eip 0xc0622c77 <__sysctl+119>: mov %eax,%ebx
frame 13 at 0xcf3c9cec: ebp cf3c9d40, eip 0xc07d2f5f <syscall+531>: mov %eax,%ebx
(kgdb) l *vm_wait+147
0xc0778d5b is in vm_wait (../../../vm/vm_page.c:901).
896 } else {
897 if (!vm_pages_needed) {
898 vm_pages_needed = 1;
899 wakeup(&vm_pages_needed);
900 }
901 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
902 "vmwait", 0);
903 }
904 }
905
(kgdb) p cnt
$4 = {v_swtch = 0x1ec397e, v_trap = 0x2a51a5, v_syscall = 0x0, v_intr = 0x10144d4, v_soft = 0x26f57, v_vm_faults = 0x29c305,
v_cow_faults = 0xf8bc, v_cow_optim = 0x147, v_zfod = 0x19423b, v_ozfod = 0xbddc1, v_swapin = 0x16407, v_swapout = 0xfd21,
v_swappgsin = 0x8ba9a, v_swappgsout = 0xadb84, v_vnodein = 0x9d6, v_vnodeout = 0x0, v_vnodepgsin = 0x2c46, v_vnodepgsout = 0x0,
v_intrans = 0x169e2, v_reactivated = 0xe0f39, v_pdwakeups = 0x743, v_pdpages = 0x2d0d515, v_dfree = 0xf1, v_pfree = 0x28da7,
v_tfree = 0x2dbf8a, v_page_size = 0x1000, v_page_count = 0xf1f9, v_free_reserved = 0xa2, v_free_target = 0x7fa, v_free_min = 0x1d6,
v_free_count = 0x0, v_wire_count = 0x3f3a, v_active_count = 0x85ab, v_inactive_target = 0xbf7, v_inactive_count = 0x2ce2,
v_cache_count = 0x0, v_cache_min = 0x7fa, v_cache_max = 0xff4, v_pageout_free_min = 0x22, v_interrupt_free_min = 0x2,
v_free_severe = 0x13c, v_forks = 0x96b, v_vforks = 0xe1, v_rforks = 0x0, v_kthreads = 0x3c, v_forkpages = 0x1bb26, v_vforkpages = 0x21cd,
v_rforkpages = 0x0, v_kthreadpages = 0x0}
(kgdb) lock 0xc6602b30
$5 = {lk_interlock = 0xc09329f0, lk_flags = 0x2040000, lk_sharecount = 0x0, lk_waitcount = 0x3, lk_exclusivecount = 0x1, lk_prio = 0x50,
lk_wmesg = 0xc0843a8a "getblk", lk_timo = 0x0, lk_lockholder = 0xfffffffe, lk_newlock = 0x0,
lk_filename = 0xc084360f "../../../sys/buf.h", lk_lockername = 0xc0830519 "lockmgr", lk_lineno = 0x114, lk_slockholder = 0xffffffff,
lk_sfilename = 0xc08150fa "none", lk_slockername = 0xc083a3e6 "never share locked", lk_slineno = 0x0}
(kgdb) defproc 9
There is no member named p_uarea.
(kgdb) btp 9
frame 0 at 0xcc9b6a30: ebp cc9b6a80, eip 0xc06215b4 <mi_switch+624>: add $0xc,%esp
frame 1 at 0xcc9b6a80: ebp cc9b6a98, eip 0xc0638c34 <sleepq_switch+224>: add $0x8,%esp
frame 2 at 0xcc9b6a98: ebp cc9b6aa4, eip 0xc0638e14 <sleepq_wait+48>: push $0x220
frame 3 at 0xcc9b6aa4: ebp cc9b6ad4, eip 0xc0621225 <msleep+785>: mov $0x0,%edi
frame 4 at 0xcc9b6ad4: ebp cc9b6af8, eip 0xc077c118 <getpbuf+68>: add $0x14,%esp
frame 5 at 0xcc9b6af8: ebp cc9b6b2c, eip 0xc0766b6b <swap_pager_putpages+467>: mov %eax,0xffffffe8(%ebp)
frame 6 at 0xcc9b6b2c: ebp cc9b6b98, eip 0xc077a1ba <vm_pageout_flush+274>: add $0x14,%esp
frame 7 at 0xcc9b6b98: ebp cc9b6c6c, eip 0xc077a09d <vm_pageout_clean+717>: mov 0xffffffec(%ebp),%esp
frame 8 at 0xcc9b6c6c: ebp cc9b6cfc, eip 0xc077ace7 <vm_pageout_scan+1463>: add $0x4,%esp
frame 9 at 0xcc9b6cfc: ebp cc9b6d1c, eip 0xc077b8bb <vm_pageout+707>: add $0x14,%esp
frame 10 at 0xcc9b6d1c: ebp cc9b6d34, eip 0xc06086e8 <fork_exit+160>: push $0x31a
(kgdb) l *getpbuf+68
0xc077c118 is in getpbuf (../../../vm/vm_pager.c:330).
325 mtx_lock(&pbuf_mtx);
326
327 for (;;) {
328 if (pfreecnt) {
329 while (*pfreecnt == 0) {
330 msleep(pfreecnt, &pbuf_mtx, PVM, "wswbuf0", 0);
331 }
332 }
333
334 /* get a bp from the swap buffer header pool */
(kgdb) fr 4
frame 4 at 0xcc9b6ad4: ebp cc9b6af8, eip 0xc077c118 <getpbuf+68>: add $0x14,%esp
Called from c0766b6b, stack frame at cc9b6b2c
last 20 local variables:
0xcc9b6aa8: 0xc0621225 0xc098e654 0x00000000 0x00000000
0xcc9b6ab8: 0xc0857de4 0x00000145 0xc0857c61 0x0000029a
0xcc9b6ac8: 0xc23a6000 0xc098e654 0x00000007 0xcc9b6af8
0xcc9b6ad8: 0xc077c118 0xc098e654 0xc098f560 0x00000044
0xcc9b6ae8: 0xc0857e18 0x00000000 0xc23a6000 0x00000000
call parameters:
0xcc9b6b00: 0xc098e654 0xcc9b6bf0 0xcc9b6b28 0x00002332
0xcc9b6b10: 0x00000000 0x00000001 0xc0857063 0x00000000
(kgdb) p *(int *)0xc098e654
$2 = 0x0
$ ps -o vsz -alx -N /usr/src/sys/i386/compile/PHO/kernel.debug -M /var/crash/vmcore.281 | sort -rn | head -6
106400 1001 2660 2657 194 -84 0 0 pfault ?L+ #C: 0:01,34 [swap]
102432 1001 2661 2657 201 -84 0 0 pfault ?L+ #C: 0:01,30 [swap]
95720 1001 2659 2657 200 -84 0 0 - ?+ #C: 2:09,34 [swap]
37316 1001 2679 2678 0 -84 0 0 kserel ?L+ #C: 0:01,47 [thr1]
37216 1001 2680 2678 0 -84 0 0 kserel ?L+ #C: 0:30,71 [thr1]
6116 0 578 439 4 -84 0 0 sbwait ?Ws ?? 0:00,00 [sshd]
$ ps -o vsz -alx -N /usr/src/sys/i386/compile/PHO/kernel.debug -M /var/crash/vmcore.281 | awk '{sum=sum+$1} END{print sum/1024}'
475,18