GENERIC HEAD from Jan 9 15:52 UTC, vmcore.203
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2008 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 8.0-CURRENT #0: Thu Jan 10 18:16:40 CET 2008
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1037570048 (989 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
negative vmsize for uid = 1001
panic: swap_reserved < decr
cpuid = 2
KDB: enter: panic
[thread pid 8075 tid 100370 ]
Stopped at kdb_enter+0x3a: movl $0,kdb_why
db> where
Tracing pid 8075 tid 100370 td 0xc6352cc0
kdb_enter(c0b0ae52,c0b0ae52,c0b2c10e,e6d46b84,2,...) at kdb_enter+0x3a
panic(c0b2c10e,0,c0b2c06e,101,c5a2271c,...) at panic+0x12c
swap_release_by_uid(6200000,0,c41dc880,2b7,1cf,...) at swap_release_by_uid+0x75
vm_object_terminate(c5a2271c,0,c0b2de3c,1cf,c5a2271c,...) at vm_object_terminate+0x2dd
vm_object_deallocate(c5a2271c,0,c0b2d432,99e,c85a0dd4,...) at vm_object_deallocate+0x5df
vm_map_delete(c85a0dd4,28300000,2e500000,0,28173c58,...) at vm_map_delete+0x3b8
munmap(c6352cc0,e6d46cfc,8,e6d46ccc,c0bbce18,...) at munmap+0xc4
syscall(e6d46d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (73, FreeBSD ELF32, munmap), eip = 0x2814a4ab, esp = 0xbfbfe7dc, ebp = 0xbfbfe848 ---
db> set $lines 0
db> show allpcpu
Current CPU: 2
cpuid = 0
curthread = 0xc5de7880: pid 8089 "swap"
curpcb = 0xe6b46d90
fpcurthread = none
idlethread = 0xc3e1b440: pid 11 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc3e1b660: pid 11 "idle: cpu1"
curpcb = 0xe2953d90
fpcurthread = none
idlethread = 0xc3e1b660: pid 11 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc6352cc0: pid 8075 "swap"
curpcb = 0xe6d46d90
fpcurthread = none
idlethread = 0xc3e1b880: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc3e1baa0: pid 11 "idle: cpu3"
curpcb = 0xe294dd90
fpcurthread = none
idlethread = 0xc3e1baa0: pid 11 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> show alllocks
Process 8089 (swap) thread 0xc5de7880 (100247)
exclusive sleep mutex vm page queue mutex r = 0 (0xc0c859cc) locked @ vm/vm_object.c:663
exclusive sleep mutex vm object (standard object) r = 0 (0xc51ad834) locked @ vm/vm_object.c:463
exclusive sx user map r = 0 (0xc435c3f8) locked @ vm/vm_mmap.c:574
Process 8075 (swap) thread 0xc6352cc0 (100370)
exclusive sleep mutex swapdev r = 0 (0xc0c852dc) locked @ vm/swap_pager.c:257
exclusive sx user map r = 0 (0xc85a0e1c) locked @ vm/vm_mmap.c:574
Process 943 (sshd) thread 0xc46facc0 (100115)
exclusive sx so_rcv_sx r = 0 (0xc458020c) locked @ kern/uipc_sockbuf.c:145
Process 930 (sshd) thread 0xc4398440 (100101)
exclusive sx so_rcv_sx r = 0 (0xc435420c) locked @ kern/uipc_sockbuf.c:145
db> ps
pid ppid pgrp uid state wmesg wchan cmd
8144 937 937 1001 S+ nanslp 0xc0c27e24 sleep
8089 8068 971 1001 R+ CPU 0 swap
8075 8068 971 1001 R+ CPU 2 swap
8068 8064 971 1001 S+ wait 0xc42792ac swap
8064 7841 971 1001 S+ wait 0xc4fac000 swap
7841 7840 971 1001 S+ wait 0xc85c3ab0 run
7840 7839 971 1001 S+ wait 0xc4e03804 run
7839 7835 971 1001 S+ nanslp 0xc0c27e24 run
7835 971 971 1001 S+ wait 0xc6498558 sh
971 946 971 1001 S+ wait 0xc494a804 sh
946 945 946 1001 Ss+ wait 0xc46c2ab0 bash
945 943 943 1001 S select 0xc4337e68 sshd
943 773 943 0 Ss sbwait 0xc458023c sshd
937 933 937 1001 S+ wait 0xc437d558 sh
933 932 933 1001 Ss+ wait 0xc46c1ab0 bash
932 930 930 1001 S select 0xc41dcb28 sshd
930 773 930 0 Ss sbwait 0xc435423c sshd
894 1 894 0 Ss+ ttyin 0xc3fe7410 getty
893 1 893 0 Ss+ ttyin 0xc3feb010 getty
892 1 892 0 Ss+ ttyin 0xc3feac10 getty
891 1 891 0 Ss+ ttyin 0xc3febc10 getty
890 1 890 0 Ss+ ttyin 0xc3feb810 getty
889 1 889 0 Ss+ ttyin 0xc3fe9c10 getty
888 1 888 0 Ss+ ttyin 0xc3fe9810 getty
887 1 887 0 Ss+ ttyin 0xc3fea810 getty
872 1 872 0 Ss select 0xc41db268 inetd
845 1 845 0 Ss select 0xc41ff1e8 moused
832 1 832 0 Ss nanslp 0xc0c27e24 watchdogd
789 1 789 0 Ss nanslp 0xc0c27e24 cron
782 1 782 25 Ss pause 0xc3f25864 sendmail
778 1 778 0 Ss select 0xc41db468 sendmail
773 1 773 0 Ss select 0xc4336c68 sshd
754 1 754 0 Rs ntpd
697 681 681 0 S - 0xc41cd800 nfsd
696 681 681 0 S - 0xc41d1e00 nfsd
695 681 681 0 S - 0xc431fe00 nfsd
694 681 681 0 S - 0xc43ece00 nfsd
693 681 681 0 S - 0xc431f800 nfsd
692 681 681 0 S - 0xc431f600 nfsd
691 681 681 0 S - 0xc43ed000 nfsd
690 681 681 0 S - 0xc43ed200 nfsd
689 681 681 0 S - 0xc41d1c00 nfsd
687 681 681 0 S - 0xc431f400 nfsd
686 681 681 0 S - 0xc4208e00 nfsd
685 681 681 0 S - 0xc43ed400 nfsd
684 681 681 0 S - 0xc43ed600 nfsd
683 681 681 0 S - 0xc431f000 nfsd
682 681 681 0 S - 0xc41cda00 nfsd
681 1 681 0 Ss select 0xc41db6a8 nfsd
679 1 679 0 Ss select 0xc4282ea8 mountd
626 1 626 0 Ss select 0xc43366e8 rpcbind
609 1 609 0 Ss select 0xc3fe13e8 syslogd
550 1 550 0 Ss select 0xc4337068 devd
27 0 0 0 SL sdflush 0xc0c851c4 [softdepflush]
26 0 0 0 SL syncer 0xc0c27c4c [syncer]
25 0 0 0 SL vlruwt 0xc41a82ac [vnlru]
24 0 0 0 SL psleep 0xc0c79884 [bufdaemon]
23 0 0 0 SL pgzero 0xc0c85e20 [pagezero]
22 0 0 0 SL psleep 0xc0c85a28 [vmdaemon]
21 0 0 0 SL psleep 0xc0c859f0 [pagedaemon]
20 0 0 0 SL waiting_ 0xc0c7b60c [sctp_iterator]
19 0 0 0 SL - 0xc3f23a3c [fdc0]
18 0 0 0 SL usbtsk 0xc0c25534 [usbtask-dr]
17 0 0 0 SL usbtsk 0xc0c25520 [usbtask-hc]
16 0 0 0 SL usbevt 0xc3f58210 [usb0]
15 0 0 0 SL - 0xc3f40280 [em0 taskq]
14 0 0 0 SL ccb_scan 0xc0bf5b54 [xpt_thrd]
9 0 0 0 SL - 0xc3ed2a80 [kqueue taskq]
8 0 0 0 SL - 0xc3ed2c80 [acpi_task_2]
7 0 0 0 SL - 0xc3ed2c80 [acpi_task_1]
6 0 0 0 SL - 0xc3ed2c80 [acpi_task_0]
5 0 0 0 SL - 0xc3ed2e00 [thread taskq]
13 0 0 0 SL - 0xc0c27c54 [yarrow]
4 0 0 0 SL - 0xc0c25c8c [g_down]
3 0 0 0 SL - 0xc0c25c88 [g_up]
2 0 0 0 SL - 0xc0c25c80 [g_event]
12 0 0 0 WL (threaded) intr
100037 I [irq7: ppc0]
100035 I [swi0: sio]
100034 I [irq12: psm0]
100033 I [irq1: atkbd0]
100032 I [irq15: ata1]
100031 I [irq14: ata0]
100030 I [irq17: fxp0]
100026 I [irq16: uhci0]
100024 I [irq9: acpi0]
100023 I [swi2: cambio]
100020 I [swi6: task queue]
100016 I [swi6: Giant taskq]
100014 I [swi5: +]
100009 I [swi3: vm]
100008 I [swi4: clock sio]
100007 I [swi1: net]
11 0 0 0 RL (threaded) idle
100006 CanRun [idle: cpu0]
100005 Run CPU 1 [idle: cpu1]
100004 CanRun [idle: cpu2]
100003 Run CPU 3 [idle: cpu3]
1 0 1 0 SLs wait 0xc3e19ab0 [init]
10 0 0 0 SL audit_wo 0xc0c84c30 [audit]
0 0 0 0 WLs [swapper]
8094 8068 971 1001 Z+ swap
8077 8068 971 1001 Z+ swap
8095 8068 971 1001 Z+ swap
8091 8068 971 1001 Z+ swap
8093 8068 971 1001 Z+ swap
8065 7841 971 1001 Z+ syscall
8079 8068 971 1001 Z+ swap
db> allt
Tracing command sleep pid 8144 tid 100299 td 0xc61b5880
sched_switch(c61b5880,0,1,180,14b8d0ac,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c61b5880,0,c0b0e874,19b,c0c27e24,...) at sleepq_switch+0xf0
sleepq_catch_signals(e6c35bc4,c0755aad,c0c2fc80,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c27e24,0,c0b0b7b1,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c27e24,0,15c,c0b0bfc1,2711,...) at _sleep+0x331
kern_nanosleep(c61b5880,e6c35c64,e6c35c6c,a,0,...) at kern_nanosleep+0xc1
nanosleep(c61b5880,e6c35cfc,8,e6c35d38,c0bbddc0,...) at nanosleep+0x6f
syscall(e6c35d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x281417bf, esp = 0xbfbfebdc, ebp = 0xbfbfec18 ---
Tracing command swap pid 8089 tid 100247 td 0xc5de7880
cpustop_handler(1,e6b46a50,c0a7865d,ca260d1,258,...) at cpustop_handler+0x32
ipi_nmi_handler(ca260d1,258,e6b46a48,870d2754,c5ffe804,...) at ipi_nmi_handler+0x2f
trap(e6b46a5c) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0a4ff8d, esp = 0xe6b46a9c, ebp = 0xe6b46ac0 ---
siointr1(c0c92e8c,0,c0b350f9,56f,e6b46b00,...) at siointr1+0x25d
siointr(c3fd8400,c0a5ebef,e6b46aec,c5de7880,c3e16280,...) at siointr+0x32
intr_execute_handlers(c3e00cd0,e6b46b20,e6b46b70,c0a5e9d4,38,...) at intr_execute_handlers+0xec
lapic_handle_intr(38,e6b46b20) at lapic_handle_intr+0x3f
Xapic_isr1() at Xapic_isr1+0x34
--- interrupt, eip = 0xc098884a, esp = 0xe6b46b60, ebp = 0xe6b46b70 ---
vm_page_zero_idle_wakeup(c1ee622c,0,c0b2e3b1,537,297,...) at vm_page_zero_idle_wakeup+0x3a
vm_page_free_toq(c1ee622c,e6b46bc8,c097e2a8,c1ee622c,0,...) at vm_page_free_toq+0x17c
vm_page_free(c1ee622c,0,c0b2de3c,297,1cf,...) at vm_page_free+0x16
vm_object_terminate(c51ad834,0,c0b2de3c,1cf,c51ad834,...) at vm_object_terminate+0x178
vm_object_deallocate(c51ad834,0,c0b2d432,99e,c435c3b0,...) at vm_object_deallocate+0x5df
vm_map_delete(c435c3b0,28300000,2e500000,0,28173c58,...) at vm_map_delete+0x3b8
munmap(c5de7880,e6b46cfc,8,0,c0bbce18,...) at munmap+0xc4
syscall(e6b46d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (73, FreeBSD ELF32, munmap), eip = 0x2814a4ab, esp = 0xbfbfe7dc, ebp = 0xbfbfe848 ---
Tracing command swap pid 8075 tid 100370 td 0xc6352cc0
kdb_enter(c0b0ae52,c0b0ae52,c0b2c10e,e6d46b84,2,...) at kdb_enter+0x3a
panic(c0b2c10e,0,c0b2c06e,101,c5a2271c,...) at panic+0x12c
swap_release_by_uid(6200000,0,c41dc880,2b7,1cf,...) at swap_release_by_uid+0x75
vm_object_terminate(c5a2271c,0,c0b2de3c,1cf,c5a2271c,...) at vm_object_terminate+0x2dd
vm_object_deallocate(c5a2271c,0,c0b2d432,99e,c85a0dd4,...) at vm_object_deallocate+0x5df
vm_map_delete(c85a0dd4,28300000,2e500000,0,28173c58,...) at vm_map_delete+0x3b8
munmap(c6352cc0,e6d46cfc,8,e6d46ccc,c0bbce18,...) at munmap+0xc4
syscall(e6d46d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (73, FreeBSD ELF32, munmap), eip = 0x2814a4ab, esp = 0xbfbfe7dc, ebp = 0xbfbfe848 ---
Tracing command swap pid 8068 tid 100093 td 0xc427f880
sched_switch(c427f880,0,1,180,5116fff4,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c427f880,0,c0b0e874,19b,c42792ac,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0b0e874,155,0,100,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c42792ac,c427933c,c0b109b6,100,0,...) at sleepq_wait_sig+0x14
_sleep(c42792ac,c427933c,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c427f880,1f8b,e6918c2c,0,0,...) at kern_wait+0xa66
wait4(c427f880,e6918cfc,10,c0b107d1,c0bbc7e8,...) at wait4+0x3b
syscall(e6918d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e37fb, esp = 0xbfbfe95c, ebp = 0xbfbfe978 ---
Tracing command swap pid 8064 tid 100229 td 0xc4fd8aa0
Tracing command run pid 7841 tid 100263 td 0xc66eeaa0
sched_switch(c66eeaa0,0,1,180,c0e98678,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c66eeaa0,0,c0b0e874,19b,c85c3ab0,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0b0e874,155,0,100,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c85c3ab0,c85c3b40,c0b109b6,100,0,...) at sleepq_wait_sig+0x14
_sleep(c85c3ab0,c85c3b40,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c66eeaa0,1f80,e6bbbc2c,0,0,...) at kern_wait+0xa66
wait4(c66eeaa0,e6bbbcfc,10,e6bbbd38,c0bbc7e8,...) at wait4+0x3b
syscall(e6bbbd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e27fb, esp = 0xbfbfe45c, ebp = 0xbfbfe478 ---
Tracing command run pid 7840 tid 100385 td 0xc5debaa0
Tracing command run pid 7839 tid 100210 td 0xc4fcd440
sched_switch(c4fcd440,0,1,180,14e6925c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c4fcd440,0,c0b0e874,19b,c0c27e24,...) at sleepq_switch+0xf0
sleepq_catch_signals(e6ab0bc4,c0755aad,c0c2fd00,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c27e24,0,c0b0b7b1,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c27e24,0,15c,c0b0bfc1,3e9,...) at _sleep+0x331
kern_nanosleep(c4fcd440,e6ab0c64,e6ab0c6c,1,0,...) at kern_nanosleep+0xc1
nanosleep(c4fcd440,e6ab0cfc,8,c0b10afa,c0bbddc0,...) at nanosleep+0x6f
syscall(e6ab0d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x281437bf, esp = 0xbfbfe92c, ebp = 0xbfbfe958 ---
Tracing command sh pid 7835 tid 100545 td 0xc5cec880
Tracing command sh pid 971 tid 100126 td 0xc46f9440
Tracing command bash pid 946 tid 100114 td 0xc4383660
Tracing command sshd pid 945 tid 100111 td 0xc4383cc0
Tracing command sshd pid 943 tid 100115 td 0xc46facc0
Tracing command sh pid 937 tid 100107 td 0xc4397660
sched_switch(c4397660,0,1,180,12c8c8ac,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c4397660,0,c0b0e874,19b,c437d558,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755aad,c0c2fd00,4,c0b09a44,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c437d558,0,c0b0b7b1,da,0,...) at sleepq_wait_sig+0x14
_sleep(c437d558,c437d5e8,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c4397660,ffffffff,e6944c2c,2,0,...) at kern_wait+0xa66
wait4(c4397660,e6944cfc,10,c0b10995,c0bbc7e8,...) at wait4+0x3b
syscall(e6944d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x281527fb, esp = 0xbfbfe8dc, ebp = 0xbfbfe8f8 ---
Tracing command bash pid 933 tid 100119 td 0xc46fa440
Tracing command sshd pid 932 tid 100110 td 0xc4397000
Tracing command sshd pid 930 tid 100101 td 0xc4398440
Tracing command getty pid 894 tid 100116 td 0xc46faaa0
Tracing command getty pid 893 tid 100099 td 0xc4398880
Tracing command getty pid 892 tid 100049 td 0xc3f70aa0
Tracing command getty pid 891 tid 100090 td 0xc4382000
Tracing command getty pid 890 tid 100102 td 0xc4398220
Tracing command getty pid 889 tid 100121 td 0xc46fa000
Tracing command getty pid 888 tid 100087 td 0xc4382660
Tracing command getty pid 887 tid 100123 td 0xc46f9aa0
Tracing command inetd pid 872 tid 100122 td 0xc46f9cc0
Tracing command moused pid 845 tid 100105 td 0xc4397aa0
Tracing command watchdogd pid 832 tid 100097 td 0xc41a6440
sched_switch(c41a6440,0,1,180,f7a12d5c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c41a6440,0,c0b0e874,19b,c0c27e24,...) at sleepq_switch+0xf0
sleepq_catch_signals(e6926bc4,c0755aad,c0c2fd00,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c27e24,0,c0b0b7b1,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c27e24,0,15c,c0b0bfc1,ea61,...) at _sleep+0x331
kern_nanosleep(c41a6440,e6926c64,e6926c6c,3c,0,...) at kern_nanosleep+0xc1
nanosleep(c41a6440,e6926cfc,8,c0b3dc43,c0bbddc0,...) at nanosleep+0x6f
syscall(e6926d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x281657bf, esp = 0xbfbfed4c, ebp = 0xbfbfed78 ---
Tracing command cron pid 789 tid 100080 td 0xc3f70660
Tracing command sendmail pid 782 tid 100047 td 0xc41a5000
Tracing command sendmail pid 778 tid 100079 td 0xc427faa0
sched_switch(c427faa0,0,1,180,7a58c59c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c427faa0,0,c0b0e874,19b,c41db468,...) at sleepq_switch+0xf0
sleepq_catch_signals(e68e9a4c,c0755e5a,c41db450,0,c427faa0,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c41db468,1389,c0b1043d,101,0,...) at sleepq_timedwait_sig+0x17
_cv_timedwait_sig(c41db468,c41db450,1389,57e,c670aaf8,...) at _cv_timedwait_sig+0x20b
seltdwait(e68e9b14,e68e9b1c,3d7,c427faa0,e68e9ae0,...) at seltdwait+0x8a
kern_select(c427faa0,7,bfbfd94c,0,0,e68e9c70,5,0) at kern_select+0x531
select(c427faa0,e68e9cfc,14,e68e9d38,c0bbcff8,...) at select+0x5e
syscall(e68e9d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x283a8c83, esp = 0xbfbfd0bc, ebp = 0xbfbfdb58 ---
Tracing command sshd pid 773 tid 100100 td 0xc4398660
Tracing command ntpd pid 754 tid 100095 td 0xc41a6880
sched_switch(c41a6880,0,1,180,eb2c0d0,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c41a6880,0,c0b0e874,19b,c4282c68,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755e5a,c4282c50,0,c0b09a44,c41a6880,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c4282c68,c4282c50,c0b1043d,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c4282c68,c4282c50,c0b10400,57e,c42964e0,...) at _cv_wait_sig+0x1fc
seltdwait(c436352c,c0b10400,3d7,c41a6880,0,...) at seltdwait+0xa2
kern_select(c41a6880,a,bfbfed7c,0,0,0,246,0) at kern_select+0x531
select(c41a6880,e6920cfc,14,16,c0bbcff8,...) at select+0x5e
syscall(e6920d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x28319c83, esp = 0xbfbfed5c, ebp = 0xbfbfee18 ---
Tracing command nfsd pid 697 tid 100050 td 0xc4230aa0
Tracing command nfsd pid 696 tid 100062 td 0xc4318000
Tracing command nfsd pid 695 tid 100063 td 0xc4317cc0
Tracing command nfsd pid 694 tid 100064 td 0xc4317aa0
Tracing command nfsd pid 693 tid 100065 td 0xc4317880
Tracing command nfsd pid 692 tid 100066 td 0xc4317660
Tracing command nfsd pid 691 tid 100073 td 0xc4314660
Tracing command nfsd pid 690 tid 100072 td 0xc4314880
Tracing command nfsd pid 689 tid 100071 td 0xc4314aa0
Tracing command nfsd pid 687 tid 100070 td 0xc4314cc0
Tracing command nfsd pid 686 tid 100069 td 0xc4317000
Tracing command nfsd pid 685 tid 100068 td 0xc4317220
Tracing command nfsd pid 684 tid 100067 td 0xc4317440
Tracing command nfsd pid 683 tid 100085 td 0xc4382aa0
Tracing command nfsd pid 682 tid 100092 td 0xc4318aa0
Tracing command nfsd pid 681 tid 100108 td 0xc4397440
Tracing command mountd pid 679 tid 100098 td 0xc41a6220
Tracing command rpcbind pid 626 tid 100086 td 0xc4382880
Tracing command syslogd pid 609 tid 100104 td 0xc4397cc0
sched_switch(c4397cc0,0,1,180,689d24f6,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c4397cc0,0,c0b0e874,19b,c3fe13e8,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755e5a,c3fe13d0,0,c0b09a44,c4397cc0,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c3fe13e8,c3fe13d0,c0b1043d,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c3fe13e8,c3fe13d0,c0b10400,57e,c4236208,...) at _cv_wait_sig+0x1fc
seltdwait(c41ddc2c,c0b10400,3d7,c4397cc0,6,...) at seltdwait+0xa2
kern_select(c4397cc0,9,282200bc,0,0,0,c41fdba0,28203000) at kern_select+0x531
select(c4397cc0,e693bcfc,14,c0b2a109,c0bbcff8,...) at select+0x5e
syscall(e693bd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x28172c83, esp = 0xbfbfdf3c, ebp = 0xbfbfee88 ---
Tracing command devd pid 550 tid 100057 td 0xc41a6aa0
Tracing command softdepflush pid 27 tid 100045 td 0xc41a5220
sched_switch(c41a5220,0,1,180,5ed6ba7e,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,44,...) at mi_switch+0x217
sleepq_switch(c41a5220,0,c0b0e874,240,44,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c851c4,3e8,c0b2a586,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c851c4,c0c8519c,44,c0b2a586,3e8,...) at _sleep+0x345
softdep_flush(0,e4632d38,c0b075f1,314,c3f25ab0,...) at softdep_flush+0x2d2
fork_exit(c094d180,0,e4632d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4632d70, ebp = 0 ---
Tracing command syncer pid 26 tid 100044 td 0xc41a5440
sched_switch(c41a5440,0,1,180,48ec6554,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c41a5440,...) at mi_switch+0x217
sleepq_switch(c41a5440,0,c0b0e874,21d,68,...) at sleepq_switch+0xf0
sleepq_wait(c0c27c4c,0,c0b16ef3,0,0,...) at sleepq_wait+0x60
_sleep(c0c27c4c,c0c79b64,68,c0b16ef3,0,...) at _sleep+0x369
sched_sync(0,e462fd38,c0b075f1,314,c41a8000,...) at sched_sync+0x742
fork_exit(c07e34f0,0,e462fd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe462fd70, ebp = 0 ---
Tracing command vnlru pid 25 tid 100043 td 0xc41a5660
sched_switch(c41a5660,0,1,180,5635e2c2,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,50,...) at mi_switch+0x217
sleepq_switch(c41a5660,0,c0b0e874,240,50,...) at sleepq_switch+0xf0
sleepq_timedwait(c41a82ac,3e8,c0b17000,0,0,...) at sleepq_timedwait+0x68
_sleep(c41a82ac,c0c79b38,250,c0b17000,3e8,...) at _sleep+0x345
vnlru_proc(0,e462cd38,c0b075f1,314,c41a82ac,...) at vnlru_proc+0x115
fork_exit(c07e4af0,0,e462cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe462cd70, ebp = 0 ---
Tracing command bufdaemon pid 24 tid 100042 td 0xc41a5880
sched_switch(c41a5880,0,1,180,2e1f3540,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,44,...) at mi_switch+0x217
sleepq_switch(c41a5880,0,c0b0e874,240,44,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c79884,3e8,c0b14bd3,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c79884,c0c79888,44,c0b14bd3,3e8,...) at _sleep+0x345
buf_daemon(0,e4629d38,c0b075f1,314,c41a8558,...) at buf_daemon+0x21e
fork_exit(c07cf0c0,0,e4629d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4629d70, ebp = 0 ---
Tracing command pagezero pid 23 tid 100041 td 0xc41a5aa0
sched_switch(c41a5aa0,0,1,180,3cb9506,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c41a5aa0,0,c0b0e874,240,0,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c85e20,493e0,c0b2f48d,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c85e20,c0c859b4,0,c0b2f48d,493e0,...) at _sleep+0x345
vm_pagezero(0,e4626d38,c0b075f1,314,c41a8804,...) at vm_pagezero+0xdc
fork_exit(c0988720,0,e4626d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4626d70, ebp = 0 ---
Tracing command vmdaemon pid 22 tid 100040 td 0xc41a5cc0
sched_switch(c41a5cc0,0,1,180,f59dda32,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c41a5cc0,...) at mi_switch+0x217
sleepq_switch(c41a5cc0,0,c0b0e874,21d,68,...) at sleepq_switch+0xf0
sleepq_wait(c0c85a28,c0c85a2c,c0b14bd3,0,0,...) at sleepq_wait+0x60
_sleep(c0c85a28,c0c85a2c,68,c0b14bd3,0,...) at _sleep+0x369
vm_daemon(0,e4623d38,c0b075f1,314,c41a8ab0,...) at vm_daemon+0x59
fork_exit(c0983b80,0,e4623d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4623d70, ebp = 0 ---
Tracing command pagedaemon pid 21 tid 100039 td 0xc41a6000
sched_switch(c41a6000,0,1,180,926bef80,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,44,...) at mi_switch+0x217
sleepq_switch(c41a6000,0,c0b0e874,240,44,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c859f0,1388,c0b14bd3,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c859f0,c0c859b4,44,c0b14bd3,1388,...) at _sleep+0x345
vm_pageout(0,e4620d38,c0b075f1,314,c3e63804,...) at vm_pageout+0x2bb
fork_exit(c0984730,0,e4620d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4620d70, ebp = 0 ---
Tracing command sctp_iterator pid 20 tid 100038 td 0xc3f21cc0
sched_switch(c3f21cc0,0,1,180,33f4a84c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21cc0,...) at mi_switch+0x217
sleepq_switch(c3f21cc0,0,c0b0e874,21d,10,...) at sleepq_switch+0xf0
sleepq_wait(c0c7b60c,c0c7b514,c0b1dcda,0,0,...) at sleepq_wait+0x60
_sleep(c0c7b60c,c0c7b514,10,c0b1dcda,0,...) at _sleep+0x369
sctp_iterator_thread(0,e461dd38,c0b075f1,314,c3e63ab0,...) at sctp_iterator_thread+0x60
fork_exit(c084b620,0,e461dd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe461dd70, ebp = 0 ---
Tracing command fdc0 pid 19 tid 100036 td 0xc3f6f220
sched_switch(c3f6f220,0,1,180,149358c0,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3f6f220,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c3f23a3c,3e8,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c3f23a3c,c3f23af0,4c,c0b02548,3e8,...) at _sleep+0x345
fdc_thread(c3f23a00,e4614d38,c0b075f1,314,c3f24000,...) at fdc_thread+0x2b8
fork_exit(c0a35aa0,c3f23a00,e4614d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4614d70, ebp = 0 ---
Tracing command usbtask-dr pid 18 tid 100029 td 0xc3f70220
sched_switch(c3f70220,0,1,180,727b2e40,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f70220,...) at mi_switch+0x217
sleepq_switch(c3f70220,0,c0b0e874,21d,5c,...) at sleepq_switch+0xf0
sleepq_wait(c0c25534,0,c0b0b7b1,da,0,...) at sleepq_wait+0x60
_sleep(c0c25534,0,5c,c0aff443,0,...) at _sleep+0x369
usb_task_thread(c0c25534,e43c5d38,c0b075f1,314,c3f242ac,...) at usb_task_thread+0x62
fork_exit(c06dc980,c0c25534,e43c5d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43c5d70, ebp = 0 ---
Tracing command usbtask-hc pid 17 tid 100028 td 0xc3f70440
sched_switch(c3f70440,0,1,180,727afeb0,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f70440,...) at mi_switch+0x217
sleepq_switch(c3f70440,0,c0b0e874,21d,5c,...) at sleepq_switch+0xf0
sleepq_wait(c0c25520,0,c0b0b7b1,da,0,...) at sleepq_wait+0x60
_sleep(c0c25520,0,5c,c0aff443,0,...) at _sleep+0x369
usb_task_thread(c0c25520,e43c2d38,c0b075f1,314,c3f24558,...) at usb_task_thread+0x62
fork_exit(c06dc980,c0c25520,e43c2d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43c2d70, ebp = 0 ---
Tracing command usb0 pid 16 tid 100027 td 0xc3e62660
sched_switch(c3e62660,0,1,180,bab88b0c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,5c,...) at mi_switch+0x217
sleepq_switch(c3e62660,0,c0b0e874,240,5c,...) at sleepq_switch+0xf0
sleepq_timedwait(c3f58210,ea60,c0aff406,0,0,...) at sleepq_timedwait+0x68
_sleep(c3f58210,0,5c,c0aff406,ea60,...) at _sleep+0x345
usb_event_thread(c3f633c0,e43bfd38,c0b075f1,314,c3f24804,...) at usb_event_thread+0xca
fork_exit(c06dc200,c3f633c0,e43bfd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43bfd70, ebp = 0 ---
Tracing command em0 taskq pid 15 tid 100025 td 0xc3e62aa0
sched_switch(c3e62aa0,0,1,180,726ea4b4,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e62aa0,...) at mi_switch+0x217
sleepq_switch(c3e62aa0,0,c0b0e874,21d,c3e62aa0,...) at sleepq_switch+0xf0
sleepq_wait(c3f40280,0,c0b0b8c5,c0b02548,0,...) at sleepq_wait+0x60
msleep_spin(c3f40280,c3f4029c,c0b02548,0,c0b09a44,...) at msleep_spin+0x20d
taskqueue_thread_loop(c3f1ab14,e43b8d38,c0b075f1,314,c3f24ab0,...) at taskqueue_thread_loop+0x8a
fork_exit(c0795fc0,c3f1ab14,e43b8d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43b8d70, ebp = 0 ---
Tracing command xpt_thrd pid 14 tid 100022 td 0xc3f21220
sched_switch(c3f21220,0,1,180,727a81e8,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21220,...) at mi_switch+0x217
sleepq_switch(c3f21220,0,c0b0e874,21d,4c,...) at sleepq_switch+0xf0
sleepq_wait(c0bf5b54,0,c0b0b7b1,da,0,...) at sleepq_wait+0x60
_sleep(c0bf5b54,c0bf5b6c,4c,c0ac020e,0,...) at _sleep+0x369
xpt_scanner_thread(0,e4389d38,c0b075f1,314,c3f25000,...) at xpt_scanner_thread+0x41
fork_exit(c046a660,0,e4389d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4389d70, ebp = 0 ---
Tracing command kqueue taskq pid 9 tid 100021 td 0xc3f21440
sched_switch(c3f21440,0,1,180,72b27380,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21440,...) at mi_switch+0x217
sleepq_switch(c3f21440,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2a80,c3ed2a9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2a80,c3ed2a9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0c26494,e4386d38,c0b075f1,314,c3e1a2ac,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0c26494,e4386d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4386d70, ebp = 0 ---
Tracing command acpi_task_2 pid 8 tid 100019 td 0xc3f21880
sched_switch(c3f21880,0,1,180,72b244bc,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21880,...) at mi_switch+0x217
sleepq_switch(c3f21880,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2c80,c3ed2c9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2c80,c3ed2c9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0e07ebc,e4380d38,c0b075f1,314,c3e1a558,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0e07ebc,e4380d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4380d70, ebp = 0 ---
Tracing command acpi_task_1 pid 7 tid 100018 td 0xc3f21aa0
sched_switch(c3f21aa0,0,1,180,72b21994,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21aa0,...) at mi_switch+0x217
sleepq_switch(c3f21aa0,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2c80,c3ed2c9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2c80,c3ed2c9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0e07ebc,e437dd38,c0b075f1,314,c3e1a804,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0e07ebc,e437dd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe437dd70, ebp = 0 ---
Tracing command acpi_task_0 pid 6 tid 100017 td 0xc3e1d220
sched_switch(c3e1d220,0,1,180,72b1e61c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e1d220,...) at mi_switch+0x217
sleepq_switch(c3e1d220,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2c80,c3ed2c9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2c80,c3ed2c9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0e07ebc,e437ad38,c0b075f1,314,c3e1aab0,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0e07ebc,e437ad38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe437ad70, ebp = 0 ---
Tracing command thread taskq pid 5 tid 100015 td 0xc3e1d660
sched_switch(c3e1d660,0,1,180,8c5ddff4,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e1d660,...) at mi_switch+0x217
sleepq_switch(c3e1d660,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2e00,c3ed2e1c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2e00,c3ed2e1c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0c31dd8,e4374d38,c0b075f1,314,c3e63000,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0c31dd8,e4374d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4374d70, ebp = 0 ---
Tracing command yarrow pid 13 tid 100013 td 0xc3e1daa0
sched_switch(c3e1daa0,0,1,180,7ce4eb18,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c3e1daa0,0,c0b0e874,240,0,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c27c54,64,c0b02548,2,0,...) at sleepq_timedwait+0x68
_sleep(c0c27c54,0,0,c0b02548,64,...) at _sleep+0x345
pause(c0b02548,64,c0af62d2,112,0,...) at pause+0x47
random_kthread(0,e436ed38,c0b075f1,314,c3e632ac,...) at random_kthread+0x1d9
fork_exit(c065bfb0,0,e436ed38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe436ed70, ebp = 0 ---
Tracing command g_down pid 4 tid 100012 td 0xc3e1dcc0
sched_switch(c3e1dcc0,0,1,180,7fcbc4dc,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3e1dcc0,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c25c8c,64,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c25c8c,c0c25ba8,24c,c0b02548,64,...) at _sleep+0x345
g_io_schedule_down(c3e1dcc0,0,c0b03c15,74,0,...) at g_io_schedule_down+0x6b
g_down_procbody(0,e436bd38,c0b075f1,314,c3e63558,...) at g_down_procbody+0x8d
fork_exit(c07140d0,0,e436bd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe436bd70, ebp = 0 ---
Tracing command g_up pid 3 tid 100011 td 0xc3e62000
sched_switch(c3e62000,0,1,180,7fe6acd8,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3e62000,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c25c88,64,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c25c88,c0c25be8,24c,c0b02548,64,...) at _sleep+0x345
g_io_schedule_up(c3e62000,0,c0b03c15,5d,0,...) at g_io_schedule_up+0x133
g_up_procbody(0,e4368d38,c0b075f1,314,c3e19000,...) at g_up_procbody+0x8d
fork_exit(c0714160,0,e4368d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4368d70, ebp = 0 ---
Tracing command g_event pid 2 tid 100010 td 0xc3e62220
sched_switch(c3e62220,0,1,180,7bb727d8,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3e62220,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c25c80,64,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c25c80,0,4c,c0b02548,64,...) at _sleep+0x345
g_event_procbody(0,e4365d38,c0b075f1,314,c3e192ac,...) at g_event_procbody+0xcb
fork_exit(c07141f0,0,e4365d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4365d70, ebp = 0 ---
Tracing command intr pid 12 tid 100037 td 0xc3f6f000
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100035 td 0xc3f6f440
sched_switch(c3f6f440,0,1,180,7273cf44,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3fdc864,...) at mi_switch+0x217
ithread_loop(c3fd60e0,e4607d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fd60e0,e4607d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4607d70, ebp = 0 ---
Tracing command intr pid 12 tid 100034 td 0xc3f6f660
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100033 td 0xc3f6f880
sched_switch(c3f6f880,0,1,180,727351c4,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e16464,...) at mi_switch+0x217
ithread_loop(c3fd6530,e4601d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fd6530,e4601d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4601d70, ebp = 0 ---
Tracing command intr pid 12 tid 100032 td 0xc3f6faa0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100031 td 0xc3f6fcc0
sched_switch(c3f6fcc0,0,1,180,68771f5c,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e15ce4,...) at mi_switch+0x217
ithread_loop(c3fcdab0,e45e8d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fcdab0,e45e8d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45e8d70, ebp = 0 ---
Tracing command intr pid 12 tid 100030 td 0xc3f70000
sched_switch(c3f70000,0,1,180,59e8a4e0,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e586e4,...) at mi_switch+0x217
ithread_loop(c3fcd120,e45d2d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fcd120,e45d2d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45d2d70, ebp = 0 ---
Tracing command intr pid 12 tid 100026 td 0xc3e62880
sched_switch(c3e62880,0,1,180,7272f8bc,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e58764,...) at mi_switch+0x217
ithread_loop(c3f61140,e43bbd38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3f61140,e43bbd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43bbd70, ebp = 0 ---
Tracing command intr pid 12 tid 100024 td 0xc3e62cc0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100023 td 0xc3f21000
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100020 td 0xc3f21660
sched_switch(c3f21660,0,1,180,32f24d26,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3ed2be4,...) at mi_switch+0x217
ithread_loop(c3f04080,e4383d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3f04080,e4383d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4383d70, ebp = 0 ---
Tracing command intr pid 12 tid 100016 td 0xc3e1d440
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100014 td 0xc3e1d880
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100009 td 0xc3e62440
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100008 td 0xc3e1b000
sched_switch(c3e1b000,0,1,180,84ab8e94,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e59564,...) at mi_switch+0x217
ithread_loop(c3e18260,e435fd38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3e18260,e435fd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe435fd70, ebp = 0 ---
Tracing command intr pid 12 tid 100007 td 0xc3e1b220
sched_switch(c3e1b220,0,1,180,79504c32,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e595e4,...) at mi_switch+0x217
ithread_loop(c3e18270,e435cd38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3e18270,e435cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe435cd70, ebp = 0 ---
Tracing command idle pid 11 tid 100006 td 0xc3e1b440
sched_switch(c3e1b440,0,1,17b,662e6524,...) at sched_switch+0x49b
mi_switch(1,0,c0b0c838,320,c0c29880,...) at mi_switch+0x217
sched_idletd(0,e2956d38,c0b075f1,314,c3e19804,...) at sched_idletd+0x217
fork_exit(c0781810,0,e2956d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2956d70, ebp = 0 ---
Tracing command idle pid 11 tid 100005 td 0xc3e1b660
cpustop_handler(2,e2953c80,c0a7865d,c07812a4,c3e19804,...) at cpustop_handler+0x32
ipi_nmi_handler(c07812a4,c3e19804,c0c2fc80,0,c3e19804,...) at ipi_nmi_handler+0x2f
trap(e2953c8c) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0a6ae00, esp = 0xe2953ccc, ebp = 0xe2953cf8 ---
spinlock_enter(0,e2953d38,c0b075f1,314,c3e19804,...) at spinlock_enter
fork_exit(c0781810,0,e2953d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2953d70, ebp = 0 ---
Tracing command idle pid 11 tid 100004 td 0xc3e1b880
sched_switch(c3e1b880,0,6,17b,72d975e8,...) at sched_switch+0x49b
mi_switch(6,0,c0b396c2,47e,2,...) at mi_switch+0x217
ipi_bitmap_handler(c3e10008,e2950028,c0770028,c3f22618,c3f22600,...) at ipi_bitmap_handler+0x72
Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e
--- interrupt, eip = 0x20, esp = 0xc0df0a0c, ebp = 0xe2950c7c ---
uart_z8530_class(c0df7675,20,246,e2950cc0,c0df0a0c,...) at 0x20
Tracing command idle pid 11 tid 100003 td 0xc3e1baa0
cpustop_handler(8,e294dc44,c0a7865d,c077fd68,e294dbf4,...) at cpustop_handler+0x32
ipi_nmi_handler(c077fd68,e294dbf4,c07812a4,c3e19804,c3e19804,...) at ipi_nmi_handler+0x2f
trap(e294dc50) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0df7675, esp = 0xe294dc90, ebp = 0xe294dc90 ---
acpi_cpu_c1(c076b577,c3e1baa0,e294dcac,3,c0c2fd00,...) at acpi_cpu_c1+0x5
acpi_cpu_idle(e294dcf8,c0781a59,c0c2fd00,0,c0b0c838,...) at acpi_cpu_idle+0x18c
cpu_idle(c0c2fd00,0,c0b0c838,322,c0c2ab40,...) at cpu_idle+0x28
sched_idletd(0,e294dd38,c0b075f1,314,c3e19804,...) at sched_idletd+0x249
fork_exit(c0781810,0,e294dd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe294dd70, ebp = 0 ---
Tracing command init pid 1 tid 100002 td 0xc3e1bcc0
sched_switch(c3e1bcc0,0,1,180,1ceaf564,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c3e1bcc0,0,c0b0e874,19b,c3e19ab0,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755aad,c0c2fc80,4,c0b09a44,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c3e19ab0,0,c0b0b7b1,da,0,...) at sleepq_wait_sig+0x14
_sleep(c3e19ab0,c3e19b40,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c3e1bcc0,ffffffff,e2949c2c,0,0,...) at kern_wait+0xa66
wait4(c3e1bcc0,e2949cfc,10,c0b107d1,c0bbc7e8,...) at wait4+0x3b
syscall(e2949d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x80544bf, esp = 0xbfbfe97c, ebp = 0xbfbfe998 ---
Tracing command audit pid 10 tid 100001 td 0xc3e1d000
sched_switch(c3e1d000,0,1,180,7274c318,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e1d000,...) at mi_switch+0x217
sleepq_switch(c3e1d000,0,c0b0e874,21d,c3e1d000,...) at sleepq_switch+0xf0
sleepq_wait(c0c84c30,c0c84c10,c0b27cf4,1,0,...) at sleepq_wait+0x60
_cv_wait(c0c84c30,c0c84c10,c0b28694,1c2,0,...) at _cv_wait+0x1fc
audit_worker(0,e2946d38,c0b075f1,314,c3e1a000,...) at audit_worker+0x99
fork_exit(c0931880,0,e2946d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2946d70, ebp = 0 ---
Tracing command kernel pid 0 tid 100000 td 0xc0c25ff0
sched_switch(c0c25ff0,0,1,180,59d5d8c6,...) at sched_switch+0x49b
mi_switch(1,0,c0b2cf2c,2e4,c422e2c4,...) at mi_switch+0x217
scheduler(0,101ec00,101ec00,101e000,1025000,...) at scheduler+0x2f2
mi_startup() at mi_startup+0x96
begin() at begin+0x2c
db> call doadump
Physical memory: 1011 MB
Dumping 96 MB: 81 65 49 33 17 1
Dump complete
= 0xf
db> c
Uptime: 1h48m52s
Automatic reboot in 15 seconds - press a key on the console to abort
panic: lockmgr_disown: 0xd7f40020 lockmgr must be exclusively locked
cpuid = 2
KDB: enter: panic
[thread pid 26 tid 100044 ]
Stopped at kdb_enter+0x3a: movl $0,kdb_why
db> c
(kgdb) bt
#0 doadump () at pcpu.h:195
#1 0xc048f929 in db_fncall (dummy1=0x3f9, dummy2=0x0, dummy3=0x3fb, dummy4=0xe6d4695c "ø\003") at ../../../ddb/db_command.c:514
#2 0xc048feac in db_command (last_cmdp=0xc0bf6814, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:411
#3 0xc048ffba in db_command_loop () at ../../../ddb/db_command.c:464
#4 0xc049175d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5 0xc078c246 in kdb_trap (type=0x3, code=0x0, tf=0xe6d46b04) at ../../../kern/subr_kdb.c:510
#6 0xc0a78bdb in trap (frame=0xe6d46b04) at ../../../i386/i386/trap.c:647
#7 0xc0a5e60b in calltrap () at ../../../i386/i386/exception.s:146
#8 0xc078c3ca in kdb_enter (why=0xc0b0ae52 "panic", msg=0xc0b0ae52 "panic") at cpufunc.h:60
#9 0xc076332c in panic (fmt=0xc0b2c10e "swap_reserved < decr") at ../../../kern/kern_shutdown.c:555
#10 0xc0967d05 in swap_release_by_uid (decr=Unhandled dwarf expression opcode 0x93
) at ../../../vm/swap_pager.c:259
#11 0xc097e40d in vm_object_terminate (object=0xc5a2271c) at ../../../vm/vm_object.c:705
#12 0xc098039f in vm_object_deallocate (object=0xc5a2271c) at ../../../vm/vm_object.c:602
#13 0xc0977d68 in vm_map_delete (map=0xc85a0dd4, start=0x28300000, end=0x2e500000) at ../../../vm/vm_map.c:2463
#14 0xc097c414 in munmap (td=0xc6352cc0, uap=0xe6d46cfc) at ../../../vm/vm_mmap.c:603
#15 0xc0a78353 in syscall (frame=0xe6d46d38) at ../../../i386/i386/trap.c:1034
#16 0xc0a5e670 in Xint0x80_syscall () at ../../../i386/i386/exception.s:203
#17 0x00000033 in ?? ()
(kgdb) f 11
#11 0xc097e40d in vm_object_terminate (object=0xc5a2271c) at ../../../vm/vm_object.c:705
705 swap_release_by_uid(object->charge, object->uip);
(kgdb) p *object
$2 = {mtx = {lock_object = {lo_name = 0xc0b2c877 "vm object", lo_type = 0xc0b2e022 "standard object", lo_flags = 0x1430000, lo_witness_data = {lod_list = {
stqe_next = 0xc0c3b710}, lod_witness = 0xc0c3b710}}, mtx_lock = 0x4, mtx_recurse = 0x0}, object_list = {tqe_next = 0xc4f8d118, tqe_prev = 0xc4ff8b94}, shadow_head = {
lh_first = 0x0}, shadow_list = {le_next = 0x0, le_prev = 0xc57c4b10}, memq = {tqh_first = 0x0, tqh_last = 0xc5a22748}, root = 0x0, size = 0x6200, generation = 0xc401,
ref_count = 0x0, shadow_count = 0x0, type = 0x0, flags = 0x2108, pg_color = 0x0, paging_in_progress = 0x0, resident_page_count = 0x0, backing_object = 0x0,
backing_object_offset = 0x0, pager_object_list = {tqe_next = 0x0, tqe_prev = 0x0}, rvq = {lh_first = 0x0}, cache = 0x0, handle = 0x0, un_pager = {vnp = {vnp_size = 0x0},
devp = {devp_pglist = {tqh_first = 0x0, tqh_last = 0x0}}, swp = {swp_bcount = 0x0}}, uip = 0xc41dc880, charge = 0x6200000}
(kgdb)
$ ./swap_reserved.sh
18:55:48 127418368
18:55:58 146042880
18:56:08 152793088
18:56:28 152858624
18:56:39 742522880
18:57:42 1459417088
18:57:56 1461497856
18:58:29 1462820864
19:01:52 1645301760
19:02:05 1651392512
19:02:20 1656172544
19:02:46 1657823232
19:03:07 1666215936