GENERIC HEAD from Jan 9 15:52 UTC, vmcore.203

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2008 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 8.0-CURRENT #0: Thu Jan 10 18:16:40 CET 2008
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1037570048 (989 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
negative vmsize for uid = 1001
panic: swap_reserved < decr
cpuid = 2
KDB: enter: panic
[thread pid 8075 tid 100370 ]
Stopped at      kdb_enter+0x3a: movl    $0,kdb_why
db> where
Tracing pid 8075 tid 100370 td 0xc6352cc0
kdb_enter(c0b0ae52,c0b0ae52,c0b2c10e,e6d46b84,2,...) at kdb_enter+0x3a
panic(c0b2c10e,0,c0b2c06e,101,c5a2271c,...) at panic+0x12c
swap_release_by_uid(6200000,0,c41dc880,2b7,1cf,...) at swap_release_by_uid+0x75
vm_object_terminate(c5a2271c,0,c0b2de3c,1cf,c5a2271c,...) at vm_object_terminate+0x2dd
vm_object_deallocate(c5a2271c,0,c0b2d432,99e,c85a0dd4,...) at vm_object_deallocate+0x5df
vm_map_delete(c85a0dd4,28300000,2e500000,0,28173c58,...) at vm_map_delete+0x3b8
munmap(c6352cc0,e6d46cfc,8,e6d46ccc,c0bbce18,...) at munmap+0xc4
syscall(e6d46d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (73, FreeBSD ELF32, munmap), eip = 0x2814a4ab, esp = 0xbfbfe7dc, ebp = 0xbfbfe848 ---
db> set $lines 0
db> show allpcpu
Current CPU: 2

cpuid        = 0
curthread    = 0xc5de7880: pid 8089 "swap"
curpcb       = 0xe6b46d90
fpcurthread  = none
idlethread   = 0xc3e1b440: pid 11 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc3e1b660: pid 11 "idle: cpu1"
curpcb       = 0xe2953d90
fpcurthread  = none
idlethread   = 0xc3e1b660: pid 11 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc6352cc0: pid 8075 "swap"
curpcb       = 0xe6d46d90
fpcurthread  = none
idlethread   = 0xc3e1b880: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc3e1baa0: pid 11 "idle: cpu3"
curpcb       = 0xe294dd90
fpcurthread  = none
idlethread   = 0xc3e1baa0: pid 11 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> show alllocks
Process 8089 (swap) thread 0xc5de7880 (100247)
exclusive sleep mutex vm page queue mutex r = 0 (0xc0c859cc) locked @ vm/vm_object.c:663
exclusive sleep mutex vm object (standard object) r = 0 (0xc51ad834) locked @ vm/vm_object.c:463
exclusive sx user map r = 0 (0xc435c3f8) locked @ vm/vm_mmap.c:574
Process 8075 (swap) thread 0xc6352cc0 (100370)
exclusive sleep mutex swapdev r = 0 (0xc0c852dc) locked @ vm/swap_pager.c:257
exclusive sx user map r = 0 (0xc85a0e1c) locked @ vm/vm_mmap.c:574
Process 943 (sshd) thread 0xc46facc0 (100115)
exclusive sx so_rcv_sx r = 0 (0xc458020c) locked @ kern/uipc_sockbuf.c:145
Process 930 (sshd) thread 0xc4398440 (100101)
exclusive sx so_rcv_sx r = 0 (0xc435420c) locked @ kern/uipc_sockbuf.c:145
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
 8144   937   937  1001  S+      nanslp   0xc0c27e24 sleep
 8089  8068   971  1001  R+      CPU 0               swap
 8075  8068   971  1001  R+      CPU 2               swap
 8068  8064   971  1001  S+      wait     0xc42792ac swap
 8064  7841   971  1001  S+      wait     0xc4fac000 swap
 7841  7840   971  1001  S+      wait     0xc85c3ab0 run
 7840  7839   971  1001  S+      wait     0xc4e03804 run
 7839  7835   971  1001  S+      nanslp   0xc0c27e24 run
 7835   971   971  1001  S+      wait     0xc6498558 sh
  971   946   971  1001  S+      wait     0xc494a804 sh
  946   945   946  1001  Ss+     wait     0xc46c2ab0 bash
  945   943   943  1001  S       select   0xc4337e68 sshd
  943   773   943     0  Ss      sbwait   0xc458023c sshd
  937   933   937  1001  S+      wait     0xc437d558 sh
  933   932   933  1001  Ss+     wait     0xc46c1ab0 bash
  932   930   930  1001  S       select   0xc41dcb28 sshd
  930   773   930     0  Ss      sbwait   0xc435423c sshd
  894     1   894     0  Ss+     ttyin    0xc3fe7410 getty
  893     1   893     0  Ss+     ttyin    0xc3feb010 getty
  892     1   892     0  Ss+     ttyin    0xc3feac10 getty
  891     1   891     0  Ss+     ttyin    0xc3febc10 getty
  890     1   890     0  Ss+     ttyin    0xc3feb810 getty
  889     1   889     0  Ss+     ttyin    0xc3fe9c10 getty
  888     1   888     0  Ss+     ttyin    0xc3fe9810 getty
  887     1   887     0  Ss+     ttyin    0xc3fea810 getty
  872     1   872     0  Ss      select   0xc41db268 inetd
  845     1   845     0  Ss      select   0xc41ff1e8 moused
  832     1   832     0  Ss      nanslp   0xc0c27e24 watchdogd
  789     1   789     0  Ss      nanslp   0xc0c27e24 cron
  782     1   782    25  Ss      pause    0xc3f25864 sendmail
  778     1   778     0  Ss      select   0xc41db468 sendmail
  773     1   773     0  Ss      select   0xc4336c68 sshd
  754     1   754     0  Rs                          ntpd
  697   681   681     0  S       -        0xc41cd800 nfsd
  696   681   681     0  S       -        0xc41d1e00 nfsd
  695   681   681     0  S       -        0xc431fe00 nfsd
  694   681   681     0  S       -        0xc43ece00 nfsd
  693   681   681     0  S       -        0xc431f800 nfsd
  692   681   681     0  S       -        0xc431f600 nfsd
  691   681   681     0  S       -        0xc43ed000 nfsd
  690   681   681     0  S       -        0xc43ed200 nfsd
  689   681   681     0  S       -        0xc41d1c00 nfsd
  687   681   681     0  S       -        0xc431f400 nfsd
  686   681   681     0  S       -        0xc4208e00 nfsd
  685   681   681     0  S       -        0xc43ed400 nfsd
  684   681   681     0  S       -        0xc43ed600 nfsd
  683   681   681     0  S       -        0xc431f000 nfsd
  682   681   681     0  S       -        0xc41cda00 nfsd
  681     1   681     0  Ss      select   0xc41db6a8 nfsd
  679     1   679     0  Ss      select   0xc4282ea8 mountd
  626     1   626     0  Ss      select   0xc43366e8 rpcbind
  609     1   609     0  Ss      select   0xc3fe13e8 syslogd
  550     1   550     0  Ss      select   0xc4337068 devd
   27     0     0     0  SL      sdflush  0xc0c851c4 [softdepflush]
   26     0     0     0  SL      syncer   0xc0c27c4c [syncer]
   25     0     0     0  SL      vlruwt   0xc41a82ac [vnlru]
   24     0     0     0  SL      psleep   0xc0c79884 [bufdaemon]
   23     0     0     0  SL      pgzero   0xc0c85e20 [pagezero]
   22     0     0     0  SL      psleep   0xc0c85a28 [vmdaemon]
   21     0     0     0  SL      psleep   0xc0c859f0 [pagedaemon]
   20     0     0     0  SL      waiting_ 0xc0c7b60c [sctp_iterator]
   19     0     0     0  SL      -        0xc3f23a3c [fdc0]
   18     0     0     0  SL      usbtsk   0xc0c25534 [usbtask-dr]
   17     0     0     0  SL      usbtsk   0xc0c25520 [usbtask-hc]
   16     0     0     0  SL      usbevt   0xc3f58210 [usb0]
   15     0     0     0  SL      -        0xc3f40280 [em0 taskq]
   14     0     0     0  SL      ccb_scan 0xc0bf5b54 [xpt_thrd]
    9     0     0     0  SL      -        0xc3ed2a80 [kqueue taskq]
    8     0     0     0  SL      -        0xc3ed2c80 [acpi_task_2]
    7     0     0     0  SL      -        0xc3ed2c80 [acpi_task_1]
    6     0     0     0  SL      -        0xc3ed2c80 [acpi_task_0]
    5     0     0     0  SL      -        0xc3ed2e00 [thread taskq]
   13     0     0     0  SL      -        0xc0c27c54 [yarrow]
    4     0     0     0  SL      -        0xc0c25c8c [g_down]
    3     0     0     0  SL      -        0xc0c25c88 [g_up]
    2     0     0     0  SL      -        0xc0c25c80 [g_event]
   12     0     0     0  WL      (threaded)          intr
100037                   I                           [irq7: ppc0]
100035                   I                           [swi0: sio]
100034                   I                           [irq12: psm0]
100033                   I                           [irq1: atkbd0]
100032                   I                           [irq15: ata1]
100031                   I                           [irq14: ata0]
100030                   I                           [irq17: fxp0]
100026                   I                           [irq16: uhci0]
100024                   I                           [irq9: acpi0]
100023                   I                           [swi2: cambio]
100020                   I                           [swi6: task queue]
100016                   I                           [swi6: Giant taskq]
100014                   I                           [swi5: +]
100009                   I                           [swi3: vm]
100008                   I                           [swi4: clock sio]
100007                   I                           [swi1: net]
   11     0     0     0  RL      (threaded)          idle
100006                   CanRun                      [idle: cpu0]
100005                   Run     CPU 1               [idle: cpu1]
100004                   CanRun                      [idle: cpu2]
100003                   Run     CPU 3               [idle: cpu3]
    1     0     1     0  SLs     wait     0xc3e19ab0 [init]
   10     0     0     0  SL      audit_wo 0xc0c84c30 [audit]
    0     0     0     0  WLs                         [swapper]
 8094  8068   971  1001  Z+                          swap
 8077  8068   971  1001  Z+                          swap
 8095  8068   971  1001  Z+                          swap
 8091  8068   971  1001  Z+                          swap
 8093  8068   971  1001  Z+                          swap
 8065  7841   971  1001  Z+                          syscall
 8079  8068   971  1001  Z+                          swap
db> allt

Tracing command sleep pid 8144 tid 100299 td 0xc61b5880
sched_switch(c61b5880,0,1,180,14b8d0ac,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c61b5880,0,c0b0e874,19b,c0c27e24,...) at sleepq_switch+0xf0
sleepq_catch_signals(e6c35bc4,c0755aad,c0c2fc80,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c27e24,0,c0b0b7b1,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c27e24,0,15c,c0b0bfc1,2711,...) at _sleep+0x331
kern_nanosleep(c61b5880,e6c35c64,e6c35c6c,a,0,...) at kern_nanosleep+0xc1
nanosleep(c61b5880,e6c35cfc,8,e6c35d38,c0bbddc0,...) at nanosleep+0x6f
syscall(e6c35d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x281417bf, esp = 0xbfbfebdc, ebp = 0xbfbfec18 ---

Tracing command swap pid 8089 tid 100247 td 0xc5de7880
cpustop_handler(1,e6b46a50,c0a7865d,ca260d1,258,...) at cpustop_handler+0x32
ipi_nmi_handler(ca260d1,258,e6b46a48,870d2754,c5ffe804,...) at ipi_nmi_handler+0x2f
trap(e6b46a5c) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0a4ff8d, esp = 0xe6b46a9c, ebp = 0xe6b46ac0 ---
siointr1(c0c92e8c,0,c0b350f9,56f,e6b46b00,...) at siointr1+0x25d
siointr(c3fd8400,c0a5ebef,e6b46aec,c5de7880,c3e16280,...) at siointr+0x32
intr_execute_handlers(c3e00cd0,e6b46b20,e6b46b70,c0a5e9d4,38,...) at intr_execute_handlers+0xec
lapic_handle_intr(38,e6b46b20) at lapic_handle_intr+0x3f
Xapic_isr1() at Xapic_isr1+0x34
--- interrupt, eip = 0xc098884a, esp = 0xe6b46b60, ebp = 0xe6b46b70 ---
vm_page_zero_idle_wakeup(c1ee622c,0,c0b2e3b1,537,297,...) at vm_page_zero_idle_wakeup+0x3a
vm_page_free_toq(c1ee622c,e6b46bc8,c097e2a8,c1ee622c,0,...) at vm_page_free_toq+0x17c
vm_page_free(c1ee622c,0,c0b2de3c,297,1cf,...) at vm_page_free+0x16
vm_object_terminate(c51ad834,0,c0b2de3c,1cf,c51ad834,...) at vm_object_terminate+0x178
vm_object_deallocate(c51ad834,0,c0b2d432,99e,c435c3b0,...) at vm_object_deallocate+0x5df
vm_map_delete(c435c3b0,28300000,2e500000,0,28173c58,...) at vm_map_delete+0x3b8
munmap(c5de7880,e6b46cfc,8,0,c0bbce18,...) at munmap+0xc4
syscall(e6b46d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (73, FreeBSD ELF32, munmap), eip = 0x2814a4ab, esp = 0xbfbfe7dc, ebp = 0xbfbfe848 ---

Tracing command swap pid 8075 tid 100370 td 0xc6352cc0
kdb_enter(c0b0ae52,c0b0ae52,c0b2c10e,e6d46b84,2,...) at kdb_enter+0x3a
panic(c0b2c10e,0,c0b2c06e,101,c5a2271c,...) at panic+0x12c
swap_release_by_uid(6200000,0,c41dc880,2b7,1cf,...) at swap_release_by_uid+0x75
vm_object_terminate(c5a2271c,0,c0b2de3c,1cf,c5a2271c,...) at vm_object_terminate+0x2dd
vm_object_deallocate(c5a2271c,0,c0b2d432,99e,c85a0dd4,...) at vm_object_deallocate+0x5df
vm_map_delete(c85a0dd4,28300000,2e500000,0,28173c58,...) at vm_map_delete+0x3b8
munmap(c6352cc0,e6d46cfc,8,e6d46ccc,c0bbce18,...) at munmap+0xc4
syscall(e6d46d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (73, FreeBSD ELF32, munmap), eip = 0x2814a4ab, esp = 0xbfbfe7dc, ebp = 0xbfbfe848 ---

Tracing command swap pid 8068 tid 100093 td 0xc427f880
sched_switch(c427f880,0,1,180,5116fff4,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c427f880,0,c0b0e874,19b,c42792ac,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0b0e874,155,0,100,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c42792ac,c427933c,c0b109b6,100,0,...) at sleepq_wait_sig+0x14
_sleep(c42792ac,c427933c,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c427f880,1f8b,e6918c2c,0,0,...) at kern_wait+0xa66
wait4(c427f880,e6918cfc,10,c0b107d1,c0bbc7e8,...) at wait4+0x3b
syscall(e6918d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e37fb, esp = 0xbfbfe95c, ebp = 0xbfbfe978 ---

Tracing command swap pid 8064 tid 100229 td 0xc4fd8aa0

Tracing command run pid 7841 tid 100263 td 0xc66eeaa0
sched_switch(c66eeaa0,0,1,180,c0e98678,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c66eeaa0,0,c0b0e874,19b,c85c3ab0,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0b0e874,155,0,100,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c85c3ab0,c85c3b40,c0b109b6,100,0,...) at sleepq_wait_sig+0x14
_sleep(c85c3ab0,c85c3b40,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c66eeaa0,1f80,e6bbbc2c,0,0,...) at kern_wait+0xa66
wait4(c66eeaa0,e6bbbcfc,10,e6bbbd38,c0bbc7e8,...) at wait4+0x3b
syscall(e6bbbd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e27fb, esp = 0xbfbfe45c, ebp = 0xbfbfe478 ---

Tracing command run pid 7840 tid 100385 td 0xc5debaa0

Tracing command run pid 7839 tid 100210 td 0xc4fcd440
sched_switch(c4fcd440,0,1,180,14e6925c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c4fcd440,0,c0b0e874,19b,c0c27e24,...) at sleepq_switch+0xf0
sleepq_catch_signals(e6ab0bc4,c0755aad,c0c2fd00,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c27e24,0,c0b0b7b1,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c27e24,0,15c,c0b0bfc1,3e9,...) at _sleep+0x331
kern_nanosleep(c4fcd440,e6ab0c64,e6ab0c6c,1,0,...) at kern_nanosleep+0xc1
nanosleep(c4fcd440,e6ab0cfc,8,c0b10afa,c0bbddc0,...) at nanosleep+0x6f
syscall(e6ab0d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x281437bf, esp = 0xbfbfe92c, ebp = 0xbfbfe958 ---

Tracing command sh pid 7835 tid 100545 td 0xc5cec880

Tracing command sh pid 971 tid 100126 td 0xc46f9440

Tracing command bash pid 946 tid 100114 td 0xc4383660

Tracing command sshd pid 945 tid 100111 td 0xc4383cc0

Tracing command sshd pid 943 tid 100115 td 0xc46facc0

Tracing command sh pid 937 tid 100107 td 0xc4397660
sched_switch(c4397660,0,1,180,12c8c8ac,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c4397660,0,c0b0e874,19b,c437d558,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755aad,c0c2fd00,4,c0b09a44,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c437d558,0,c0b0b7b1,da,0,...) at sleepq_wait_sig+0x14
_sleep(c437d558,c437d5e8,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c4397660,ffffffff,e6944c2c,2,0,...) at kern_wait+0xa66
wait4(c4397660,e6944cfc,10,c0b10995,c0bbc7e8,...) at wait4+0x3b
syscall(e6944d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x281527fb, esp = 0xbfbfe8dc, ebp = 0xbfbfe8f8 ---

Tracing command bash pid 933 tid 100119 td 0xc46fa440

Tracing command sshd pid 932 tid 100110 td 0xc4397000

Tracing command sshd pid 930 tid 100101 td 0xc4398440

Tracing command getty pid 894 tid 100116 td 0xc46faaa0

Tracing command getty pid 893 tid 100099 td 0xc4398880

Tracing command getty pid 892 tid 100049 td 0xc3f70aa0

Tracing command getty pid 891 tid 100090 td 0xc4382000

Tracing command getty pid 890 tid 100102 td 0xc4398220

Tracing command getty pid 889 tid 100121 td 0xc46fa000

Tracing command getty pid 888 tid 100087 td 0xc4382660

Tracing command getty pid 887 tid 100123 td 0xc46f9aa0

Tracing command inetd pid 872 tid 100122 td 0xc46f9cc0

Tracing command moused pid 845 tid 100105 td 0xc4397aa0

Tracing command watchdogd pid 832 tid 100097 td 0xc41a6440
sched_switch(c41a6440,0,1,180,f7a12d5c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c41a6440,0,c0b0e874,19b,c0c27e24,...) at sleepq_switch+0xf0
sleepq_catch_signals(e6926bc4,c0755aad,c0c2fd00,4,5c,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c0c27e24,0,c0b0b7b1,da,0,...) at sleepq_timedwait_sig+0x17
_sleep(c0c27e24,0,15c,c0b0bfc1,ea61,...) at _sleep+0x331
kern_nanosleep(c41a6440,e6926c64,e6926c6c,3c,0,...) at kern_nanosleep+0xc1
nanosleep(c41a6440,e6926cfc,8,c0b3dc43,c0bbddc0,...) at nanosleep+0x6f
syscall(e6926d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x281657bf, esp = 0xbfbfed4c, ebp = 0xbfbfed78 ---

Tracing command cron pid 789 tid 100080 td 0xc3f70660

Tracing command sendmail pid 782 tid 100047 td 0xc41a5000

Tracing command sendmail pid 778 tid 100079 td 0xc427faa0
sched_switch(c427faa0,0,1,180,7a58c59c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c427faa0,0,c0b0e874,19b,c41db468,...) at sleepq_switch+0xf0
sleepq_catch_signals(e68e9a4c,c0755e5a,c41db450,0,c427faa0,...) at sleepq_catch_signals+0x21e
sleepq_timedwait_sig(c41db468,1389,c0b1043d,101,0,...) at sleepq_timedwait_sig+0x17
_cv_timedwait_sig(c41db468,c41db450,1389,57e,c670aaf8,...) at _cv_timedwait_sig+0x20b
seltdwait(e68e9b14,e68e9b1c,3d7,c427faa0,e68e9ae0,...) at seltdwait+0x8a
kern_select(c427faa0,7,bfbfd94c,0,0,e68e9c70,5,0) at kern_select+0x531
select(c427faa0,e68e9cfc,14,e68e9d38,c0bbcff8,...) at select+0x5e
syscall(e68e9d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x283a8c83, esp = 0xbfbfd0bc, ebp = 0xbfbfdb58 ---

Tracing command sshd pid 773 tid 100100 td 0xc4398660

Tracing command ntpd pid 754 tid 100095 td 0xc41a6880
sched_switch(c41a6880,0,1,180,eb2c0d0,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c41a6880,0,c0b0e874,19b,c4282c68,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755e5a,c4282c50,0,c0b09a44,c41a6880,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c4282c68,c4282c50,c0b1043d,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c4282c68,c4282c50,c0b10400,57e,c42964e0,...) at _cv_wait_sig+0x1fc
seltdwait(c436352c,c0b10400,3d7,c41a6880,0,...) at seltdwait+0xa2
kern_select(c41a6880,a,bfbfed7c,0,0,0,246,0) at kern_select+0x531
select(c41a6880,e6920cfc,14,16,c0bbcff8,...) at select+0x5e
syscall(e6920d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x28319c83, esp = 0xbfbfed5c, ebp = 0xbfbfee18 ---

Tracing command nfsd pid 697 tid 100050 td 0xc4230aa0

Tracing command nfsd pid 696 tid 100062 td 0xc4318000

Tracing command nfsd pid 695 tid 100063 td 0xc4317cc0

Tracing command nfsd pid 694 tid 100064 td 0xc4317aa0

Tracing command nfsd pid 693 tid 100065 td 0xc4317880

Tracing command nfsd pid 692 tid 100066 td 0xc4317660

Tracing command nfsd pid 691 tid 100073 td 0xc4314660

Tracing command nfsd pid 690 tid 100072 td 0xc4314880

Tracing command nfsd pid 689 tid 100071 td 0xc4314aa0

Tracing command nfsd pid 687 tid 100070 td 0xc4314cc0

Tracing command nfsd pid 686 tid 100069 td 0xc4317000

Tracing command nfsd pid 685 tid 100068 td 0xc4317220

Tracing command nfsd pid 684 tid 100067 td 0xc4317440

Tracing command nfsd pid 683 tid 100085 td 0xc4382aa0

Tracing command nfsd pid 682 tid 100092 td 0xc4318aa0

Tracing command nfsd pid 681 tid 100108 td 0xc4397440

Tracing command mountd pid 679 tid 100098 td 0xc41a6220

Tracing command rpcbind pid 626 tid 100086 td 0xc4382880

Tracing command syslogd pid 609 tid 100104 td 0xc4397cc0
sched_switch(c4397cc0,0,1,180,689d24f6,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c4397cc0,0,c0b0e874,19b,c3fe13e8,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755e5a,c3fe13d0,0,c0b09a44,c4397cc0,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c3fe13e8,c3fe13d0,c0b1043d,101,0,...) at sleepq_wait_sig+0x14
_cv_wait_sig(c3fe13e8,c3fe13d0,c0b10400,57e,c4236208,...) at _cv_wait_sig+0x1fc
seltdwait(c41ddc2c,c0b10400,3d7,c4397cc0,6,...) at seltdwait+0xa2
kern_select(c4397cc0,9,282200bc,0,0,0,c41fdba0,28203000) at kern_select+0x531
select(c4397cc0,e693bcfc,14,c0b2a109,c0bbcff8,...) at select+0x5e
syscall(e693bd38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (93, FreeBSD ELF32, select), eip = 0x28172c83, esp = 0xbfbfdf3c, ebp = 0xbfbfee88 ---

Tracing command devd pid 550 tid 100057 td 0xc41a6aa0

Tracing command softdepflush pid 27 tid 100045 td 0xc41a5220
sched_switch(c41a5220,0,1,180,5ed6ba7e,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,44,...) at mi_switch+0x217
sleepq_switch(c41a5220,0,c0b0e874,240,44,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c851c4,3e8,c0b2a586,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c851c4,c0c8519c,44,c0b2a586,3e8,...) at _sleep+0x345
softdep_flush(0,e4632d38,c0b075f1,314,c3f25ab0,...) at softdep_flush+0x2d2
fork_exit(c094d180,0,e4632d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4632d70, ebp = 0 ---

Tracing command syncer pid 26 tid 100044 td 0xc41a5440
sched_switch(c41a5440,0,1,180,48ec6554,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c41a5440,...) at mi_switch+0x217
sleepq_switch(c41a5440,0,c0b0e874,21d,68,...) at sleepq_switch+0xf0
sleepq_wait(c0c27c4c,0,c0b16ef3,0,0,...) at sleepq_wait+0x60
_sleep(c0c27c4c,c0c79b64,68,c0b16ef3,0,...) at _sleep+0x369
sched_sync(0,e462fd38,c0b075f1,314,c41a8000,...) at sched_sync+0x742
fork_exit(c07e34f0,0,e462fd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe462fd70, ebp = 0 ---

Tracing command vnlru pid 25 tid 100043 td 0xc41a5660
sched_switch(c41a5660,0,1,180,5635e2c2,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,50,...) at mi_switch+0x217
sleepq_switch(c41a5660,0,c0b0e874,240,50,...) at sleepq_switch+0xf0
sleepq_timedwait(c41a82ac,3e8,c0b17000,0,0,...) at sleepq_timedwait+0x68
_sleep(c41a82ac,c0c79b38,250,c0b17000,3e8,...) at _sleep+0x345
vnlru_proc(0,e462cd38,c0b075f1,314,c41a82ac,...) at vnlru_proc+0x115
fork_exit(c07e4af0,0,e462cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe462cd70, ebp = 0 ---

Tracing command bufdaemon pid 24 tid 100042 td 0xc41a5880
sched_switch(c41a5880,0,1,180,2e1f3540,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,44,...) at mi_switch+0x217
sleepq_switch(c41a5880,0,c0b0e874,240,44,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c79884,3e8,c0b14bd3,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c79884,c0c79888,44,c0b14bd3,3e8,...) at _sleep+0x345
buf_daemon(0,e4629d38,c0b075f1,314,c41a8558,...) at buf_daemon+0x21e
fork_exit(c07cf0c0,0,e4629d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4629d70, ebp = 0 ---

Tracing command pagezero pid 23 tid 100041 td 0xc41a5aa0
sched_switch(c41a5aa0,0,1,180,3cb9506,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c41a5aa0,0,c0b0e874,240,0,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c85e20,493e0,c0b2f48d,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c85e20,c0c859b4,0,c0b2f48d,493e0,...) at _sleep+0x345
vm_pagezero(0,e4626d38,c0b075f1,314,c41a8804,...) at vm_pagezero+0xdc
fork_exit(c0988720,0,e4626d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4626d70, ebp = 0 ---

Tracing command vmdaemon pid 22 tid 100040 td 0xc41a5cc0
sched_switch(c41a5cc0,0,1,180,f59dda32,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c41a5cc0,...) at mi_switch+0x217
sleepq_switch(c41a5cc0,0,c0b0e874,21d,68,...) at sleepq_switch+0xf0
sleepq_wait(c0c85a28,c0c85a2c,c0b14bd3,0,0,...) at sleepq_wait+0x60
_sleep(c0c85a28,c0c85a2c,68,c0b14bd3,0,...) at _sleep+0x369
vm_daemon(0,e4623d38,c0b075f1,314,c41a8ab0,...) at vm_daemon+0x59
fork_exit(c0983b80,0,e4623d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4623d70, ebp = 0 ---

Tracing command pagedaemon pid 21 tid 100039 td 0xc41a6000
sched_switch(c41a6000,0,1,180,926bef80,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,44,...) at mi_switch+0x217
sleepq_switch(c41a6000,0,c0b0e874,240,44,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c859f0,1388,c0b14bd3,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c859f0,c0c859b4,44,c0b14bd3,1388,...) at _sleep+0x345
vm_pageout(0,e4620d38,c0b075f1,314,c3e63804,...) at vm_pageout+0x2bb
fork_exit(c0984730,0,e4620d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4620d70, ebp = 0 ---

Tracing command sctp_iterator pid 20 tid 100038 td 0xc3f21cc0
sched_switch(c3f21cc0,0,1,180,33f4a84c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21cc0,...) at mi_switch+0x217
sleepq_switch(c3f21cc0,0,c0b0e874,21d,10,...) at sleepq_switch+0xf0
sleepq_wait(c0c7b60c,c0c7b514,c0b1dcda,0,0,...) at sleepq_wait+0x60
_sleep(c0c7b60c,c0c7b514,10,c0b1dcda,0,...) at _sleep+0x369
sctp_iterator_thread(0,e461dd38,c0b075f1,314,c3e63ab0,...) at sctp_iterator_thread+0x60
fork_exit(c084b620,0,e461dd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe461dd70, ebp = 0 ---

Tracing command fdc0 pid 19 tid 100036 td 0xc3f6f220
sched_switch(c3f6f220,0,1,180,149358c0,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3f6f220,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c3f23a3c,3e8,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c3f23a3c,c3f23af0,4c,c0b02548,3e8,...) at _sleep+0x345
fdc_thread(c3f23a00,e4614d38,c0b075f1,314,c3f24000,...) at fdc_thread+0x2b8
fork_exit(c0a35aa0,c3f23a00,e4614d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4614d70, ebp = 0 ---

Tracing command usbtask-dr pid 18 tid 100029 td 0xc3f70220
sched_switch(c3f70220,0,1,180,727b2e40,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f70220,...) at mi_switch+0x217
sleepq_switch(c3f70220,0,c0b0e874,21d,5c,...) at sleepq_switch+0xf0
sleepq_wait(c0c25534,0,c0b0b7b1,da,0,...) at sleepq_wait+0x60
_sleep(c0c25534,0,5c,c0aff443,0,...) at _sleep+0x369
usb_task_thread(c0c25534,e43c5d38,c0b075f1,314,c3f242ac,...) at usb_task_thread+0x62
fork_exit(c06dc980,c0c25534,e43c5d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43c5d70, ebp = 0 ---

Tracing command usbtask-hc pid 17 tid 100028 td 0xc3f70440
sched_switch(c3f70440,0,1,180,727afeb0,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f70440,...) at mi_switch+0x217
sleepq_switch(c3f70440,0,c0b0e874,21d,5c,...) at sleepq_switch+0xf0
sleepq_wait(c0c25520,0,c0b0b7b1,da,0,...) at sleepq_wait+0x60
_sleep(c0c25520,0,5c,c0aff443,0,...) at _sleep+0x369
usb_task_thread(c0c25520,e43c2d38,c0b075f1,314,c3f24558,...) at usb_task_thread+0x62
fork_exit(c06dc980,c0c25520,e43c2d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43c2d70, ebp = 0 ---

Tracing command usb0 pid 16 tid 100027 td 0xc3e62660
sched_switch(c3e62660,0,1,180,bab88b0c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,5c,...) at mi_switch+0x217
sleepq_switch(c3e62660,0,c0b0e874,240,5c,...) at sleepq_switch+0xf0
sleepq_timedwait(c3f58210,ea60,c0aff406,0,0,...) at sleepq_timedwait+0x68
_sleep(c3f58210,0,5c,c0aff406,ea60,...) at _sleep+0x345
usb_event_thread(c3f633c0,e43bfd38,c0b075f1,314,c3f24804,...) at usb_event_thread+0xca
fork_exit(c06dc200,c3f633c0,e43bfd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43bfd70, ebp = 0 ---

Tracing command em0 taskq pid 15 tid 100025 td 0xc3e62aa0
sched_switch(c3e62aa0,0,1,180,726ea4b4,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e62aa0,...) at mi_switch+0x217
sleepq_switch(c3e62aa0,0,c0b0e874,21d,c3e62aa0,...) at sleepq_switch+0xf0
sleepq_wait(c3f40280,0,c0b0b8c5,c0b02548,0,...) at sleepq_wait+0x60
msleep_spin(c3f40280,c3f4029c,c0b02548,0,c0b09a44,...) at msleep_spin+0x20d
taskqueue_thread_loop(c3f1ab14,e43b8d38,c0b075f1,314,c3f24ab0,...) at taskqueue_thread_loop+0x8a
fork_exit(c0795fc0,c3f1ab14,e43b8d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43b8d70, ebp = 0 ---

Tracing command xpt_thrd pid 14 tid 100022 td 0xc3f21220
sched_switch(c3f21220,0,1,180,727a81e8,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21220,...) at mi_switch+0x217
sleepq_switch(c3f21220,0,c0b0e874,21d,4c,...) at sleepq_switch+0xf0
sleepq_wait(c0bf5b54,0,c0b0b7b1,da,0,...) at sleepq_wait+0x60
_sleep(c0bf5b54,c0bf5b6c,4c,c0ac020e,0,...) at _sleep+0x369
xpt_scanner_thread(0,e4389d38,c0b075f1,314,c3f25000,...) at xpt_scanner_thread+0x41
fork_exit(c046a660,0,e4389d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4389d70, ebp = 0 ---

Tracing command kqueue taskq pid 9 tid 100021 td 0xc3f21440
sched_switch(c3f21440,0,1,180,72b27380,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21440,...) at mi_switch+0x217
sleepq_switch(c3f21440,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2a80,c3ed2a9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2a80,c3ed2a9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0c26494,e4386d38,c0b075f1,314,c3e1a2ac,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0c26494,e4386d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4386d70, ebp = 0 ---

Tracing command acpi_task_2 pid 8 tid 100019 td 0xc3f21880
sched_switch(c3f21880,0,1,180,72b244bc,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21880,...) at mi_switch+0x217
sleepq_switch(c3f21880,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2c80,c3ed2c9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2c80,c3ed2c9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0e07ebc,e4380d38,c0b075f1,314,c3e1a558,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0e07ebc,e4380d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4380d70, ebp = 0 ---

Tracing command acpi_task_1 pid 7 tid 100018 td 0xc3f21aa0
sched_switch(c3f21aa0,0,1,180,72b21994,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3f21aa0,...) at mi_switch+0x217
sleepq_switch(c3f21aa0,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2c80,c3ed2c9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2c80,c3ed2c9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0e07ebc,e437dd38,c0b075f1,314,c3e1a804,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0e07ebc,e437dd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe437dd70, ebp = 0 ---

Tracing command acpi_task_0 pid 6 tid 100017 td 0xc3e1d220
sched_switch(c3e1d220,0,1,180,72b1e61c,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e1d220,...) at mi_switch+0x217
sleepq_switch(c3e1d220,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2c80,c3ed2c9c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2c80,c3ed2c9c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0e07ebc,e437ad38,c0b075f1,314,c3e1aab0,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0e07ebc,e437ad38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe437ad70, ebp = 0 ---

Tracing command thread taskq pid 5 tid 100015 td 0xc3e1d660
sched_switch(c3e1d660,0,1,180,8c5ddff4,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e1d660,...) at mi_switch+0x217
sleepq_switch(c3e1d660,0,c0b0e874,21d,0,...) at sleepq_switch+0xf0
sleepq_wait(c3ed2e00,c3ed2e1c,c0b02548,0,0,...) at sleepq_wait+0x60
_sleep(c3ed2e00,c3ed2e1c,0,c0b02548,0,...) at _sleep+0x369
taskqueue_thread_loop(c0c31dd8,e4374d38,c0b075f1,314,c3e63000,...) at taskqueue_thread_loop+0xb4
fork_exit(c0795fc0,c0c31dd8,e4374d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4374d70, ebp = 0 ---

Tracing command yarrow pid 13 tid 100013 td 0xc3e1daa0
sched_switch(c3e1daa0,0,1,180,7ce4eb18,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c3e1daa0,0,c0b0e874,240,0,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c27c54,64,c0b02548,2,0,...) at sleepq_timedwait+0x68
_sleep(c0c27c54,0,0,c0b02548,64,...) at _sleep+0x345
pause(c0b02548,64,c0af62d2,112,0,...) at pause+0x47
random_kthread(0,e436ed38,c0b075f1,314,c3e632ac,...) at random_kthread+0x1d9
fork_exit(c065bfb0,0,e436ed38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe436ed70, ebp = 0 ---

Tracing command g_down pid 4 tid 100012 td 0xc3e1dcc0
sched_switch(c3e1dcc0,0,1,180,7fcbc4dc,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3e1dcc0,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c25c8c,64,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c25c8c,c0c25ba8,24c,c0b02548,64,...) at _sleep+0x345
g_io_schedule_down(c3e1dcc0,0,c0b03c15,74,0,...) at g_io_schedule_down+0x6b
g_down_procbody(0,e436bd38,c0b075f1,314,c3e63558,...) at g_down_procbody+0x8d
fork_exit(c07140d0,0,e436bd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe436bd70, ebp = 0 ---

Tracing command g_up pid 3 tid 100011 td 0xc3e62000
sched_switch(c3e62000,0,1,180,7fe6acd8,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3e62000,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c25c88,64,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c25c88,c0c25be8,24c,c0b02548,64,...) at _sleep+0x345
g_io_schedule_up(c3e62000,0,c0b03c15,5d,0,...) at g_io_schedule_up+0x133
g_up_procbody(0,e4368d38,c0b075f1,314,c3e19000,...) at g_up_procbody+0x8d
fork_exit(c0714160,0,e4368d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4368d70, ebp = 0 ---

Tracing command g_event pid 2 tid 100010 td 0xc3e62220
sched_switch(c3e62220,0,1,180,7bb727d8,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,4c,...) at mi_switch+0x217
sleepq_switch(c3e62220,0,c0b0e874,240,4c,...) at sleepq_switch+0xf0
sleepq_timedwait(c0c25c80,64,c0b02548,0,0,...) at sleepq_timedwait+0x68
_sleep(c0c25c80,0,4c,c0b02548,64,...) at _sleep+0x345
g_event_procbody(0,e4365d38,c0b075f1,314,c3e192ac,...) at g_event_procbody+0xcb
fork_exit(c07141f0,0,e4365d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4365d70, ebp = 0 ---

Tracing command intr pid 12 tid 100037 td 0xc3f6f000
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100035 td 0xc3f6f440
sched_switch(c3f6f440,0,1,180,7273cf44,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3fdc864,...) at mi_switch+0x217
ithread_loop(c3fd60e0,e4607d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fd60e0,e4607d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4607d70, ebp = 0 ---

Tracing command intr pid 12 tid 100034 td 0xc3f6f660
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100033 td 0xc3f6f880
sched_switch(c3f6f880,0,1,180,727351c4,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e16464,...) at mi_switch+0x217
ithread_loop(c3fd6530,e4601d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fd6530,e4601d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4601d70, ebp = 0 ---

Tracing command intr pid 12 tid 100032 td 0xc3f6faa0
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100031 td 0xc3f6fcc0
sched_switch(c3f6fcc0,0,1,180,68771f5c,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e15ce4,...) at mi_switch+0x217
ithread_loop(c3fcdab0,e45e8d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fcdab0,e45e8d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45e8d70, ebp = 0 ---

Tracing command intr pid 12 tid 100030 td 0xc3f70000
sched_switch(c3f70000,0,1,180,59e8a4e0,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e586e4,...) at mi_switch+0x217
ithread_loop(c3fcd120,e45d2d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3fcd120,e45d2d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe45d2d70, ebp = 0 ---

Tracing command intr pid 12 tid 100026 td 0xc3e62880
sched_switch(c3e62880,0,1,180,7272f8bc,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e58764,...) at mi_switch+0x217
ithread_loop(c3f61140,e43bbd38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3f61140,e43bbd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe43bbd70, ebp = 0 ---

Tracing command intr pid 12 tid 100024 td 0xc3e62cc0
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100023 td 0xc3f21000
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100020 td 0xc3f21660
sched_switch(c3f21660,0,1,180,32f24d26,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3ed2be4,...) at mi_switch+0x217
ithread_loop(c3f04080,e4383d38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3f04080,e4383d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe4383d70, ebp = 0 ---

Tracing command intr pid 12 tid 100016 td 0xc3e1d440
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100014 td 0xc3e1d880
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100009 td 0xc3e62440
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100008 td 0xc3e1b000
sched_switch(c3e1b000,0,1,180,84ab8e94,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e59564,...) at mi_switch+0x217
ithread_loop(c3e18260,e435fd38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3e18260,e435fd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe435fd70, ebp = 0 ---

Tracing command intr pid 12 tid 100007 td 0xc3e1b220
sched_switch(c3e1b220,0,1,180,79504c32,...) at sched_switch+0x49b
mi_switch(1,0,c0b07873,469,c3e595e4,...) at mi_switch+0x217
ithread_loop(c3e18270,e435cd38,c0b075f1,314,c3e19558,...) at ithread_loop+0x34c
fork_exit(c0745650,c3e18270,e435cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe435cd70, ebp = 0 ---

Tracing command idle pid 11 tid 100006 td 0xc3e1b440
sched_switch(c3e1b440,0,1,17b,662e6524,...) at sched_switch+0x49b
mi_switch(1,0,c0b0c838,320,c0c29880,...) at mi_switch+0x217
sched_idletd(0,e2956d38,c0b075f1,314,c3e19804,...) at sched_idletd+0x217
fork_exit(c0781810,0,e2956d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2956d70, ebp = 0 ---

Tracing command idle pid 11 tid 100005 td 0xc3e1b660
cpustop_handler(2,e2953c80,c0a7865d,c07812a4,c3e19804,...) at cpustop_handler+0x32
ipi_nmi_handler(c07812a4,c3e19804,c0c2fc80,0,c3e19804,...) at ipi_nmi_handler+0x2f
trap(e2953c8c) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0a6ae00, esp = 0xe2953ccc, ebp = 0xe2953cf8 ---
spinlock_enter(0,e2953d38,c0b075f1,314,c3e19804,...) at spinlock_enter
fork_exit(c0781810,0,e2953d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2953d70, ebp = 0 ---

Tracing command idle pid 11 tid 100004 td 0xc3e1b880
sched_switch(c3e1b880,0,6,17b,72d975e8,...) at sched_switch+0x49b
mi_switch(6,0,c0b396c2,47e,2,...) at mi_switch+0x217
ipi_bitmap_handler(c3e10008,e2950028,c0770028,c3f22618,c3f22600,...) at ipi_bitmap_handler+0x72
Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e
--- interrupt, eip = 0x20, esp = 0xc0df0a0c, ebp = 0xe2950c7c ---
uart_z8530_class(c0df7675,20,246,e2950cc0,c0df0a0c,...) at 0x20

Tracing command idle pid 11 tid 100003 td 0xc3e1baa0
cpustop_handler(8,e294dc44,c0a7865d,c077fd68,e294dbf4,...) at cpustop_handler+0x32
ipi_nmi_handler(c077fd68,e294dbf4,c07812a4,c3e19804,c3e19804,...) at ipi_nmi_handler+0x2f
trap(e294dc50) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0df7675, esp = 0xe294dc90, ebp = 0xe294dc90 ---
acpi_cpu_c1(c076b577,c3e1baa0,e294dcac,3,c0c2fd00,...) at acpi_cpu_c1+0x5
acpi_cpu_idle(e294dcf8,c0781a59,c0c2fd00,0,c0b0c838,...) at acpi_cpu_idle+0x18c
cpu_idle(c0c2fd00,0,c0b0c838,322,c0c2ab40,...) at cpu_idle+0x28
sched_idletd(0,e294dd38,c0b075f1,314,c3e19804,...) at sched_idletd+0x249
fork_exit(c0781810,0,e294dd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe294dd70, ebp = 0 ---

Tracing command init pid 1 tid 100002 td 0xc3e1bcc0
sched_switch(c3e1bcc0,0,1,180,1ceaf564,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,0,...) at mi_switch+0x217
sleepq_switch(c3e1bcc0,0,c0b0e874,19b,c3e19ab0,...) at sleepq_switch+0xf0
sleepq_catch_signals(c0755aad,c0c2fc80,4,c0b09a44,5c,...) at sleepq_catch_signals+0x21e
sleepq_wait_sig(c3e19ab0,0,c0b0b7b1,da,0,...) at sleepq_wait_sig+0x14
_sleep(c3e19ab0,c3e19b40,15c,c0b109b6,0,...) at _sleep+0x359
kern_wait(c3e1bcc0,ffffffff,e2949c2c,0,0,...) at kern_wait+0xa66
wait4(c3e1bcc0,e2949cfc,10,c0b107d1,c0bbc7e8,...) at wait4+0x3b
syscall(e2949d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x80544bf, esp = 0xbfbfe97c, ebp = 0xbfbfe998 ---

Tracing command audit pid 10 tid 100001 td 0xc3e1d000
sched_switch(c3e1d000,0,1,180,7274c318,...) at sched_switch+0x49b
mi_switch(1,0,c0b0e874,1bf,c3e1d000,...) at mi_switch+0x217
sleepq_switch(c3e1d000,0,c0b0e874,21d,c3e1d000,...) at sleepq_switch+0xf0
sleepq_wait(c0c84c30,c0c84c10,c0b27cf4,1,0,...) at sleepq_wait+0x60
_cv_wait(c0c84c30,c0c84c10,c0b28694,1c2,0,...) at _cv_wait+0x1fc
audit_worker(0,e2946d38,c0b075f1,314,c3e1a000,...) at audit_worker+0x99
fork_exit(c0931880,0,e2946d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe2946d70, ebp = 0 ---

Tracing command kernel pid 0 tid 100000 td 0xc0c25ff0
sched_switch(c0c25ff0,0,1,180,59d5d8c6,...) at sched_switch+0x49b
mi_switch(1,0,c0b2cf2c,2e4,c422e2c4,...) at mi_switch+0x217
scheduler(0,101ec00,101ec00,101e000,1025000,...) at scheduler+0x2f2
mi_startup() at mi_startup+0x96
begin() at begin+0x2c
db> call doadump
Physical memory: 1011 MB
Dumping 96 MB: 81 65 49 33 17 1
Dump complete
= 0xf
db> c
Uptime: 1h48m52s
Automatic reboot in 15 seconds - press a key on the console to abort
panic: lockmgr_disown: 0xd7f40020 lockmgr must be exclusively locked
cpuid = 2
KDB: enter: panic
[thread pid 26 tid 100044 ]
Stopped at      kdb_enter+0x3a: movl    $0,kdb_why
db> c

(kgdb) bt
#0  doadump () at pcpu.h:195
#1  0xc048f929 in db_fncall (dummy1=0x3f9, dummy2=0x0, dummy3=0x3fb, dummy4=0xe6d4695c "\003") at ../../../ddb/db_command.c:514
#2  0xc048feac in db_command (last_cmdp=0xc0bf6814, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:411
#3  0xc048ffba in db_command_loop () at ../../../ddb/db_command.c:464
#4  0xc049175d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5  0xc078c246 in kdb_trap (type=0x3, code=0x0, tf=0xe6d46b04) at ../../../kern/subr_kdb.c:510
#6  0xc0a78bdb in trap (frame=0xe6d46b04) at ../../../i386/i386/trap.c:647
#7  0xc0a5e60b in calltrap () at ../../../i386/i386/exception.s:146
#8  0xc078c3ca in kdb_enter (why=0xc0b0ae52 "panic", msg=0xc0b0ae52 "panic") at cpufunc.h:60
#9  0xc076332c in panic (fmt=0xc0b2c10e "swap_reserved < decr") at ../../../kern/kern_shutdown.c:555
#10 0xc0967d05 in swap_release_by_uid (decr=Unhandled dwarf expression opcode 0x93
) at ../../../vm/swap_pager.c:259
#11 0xc097e40d in vm_object_terminate (object=0xc5a2271c) at ../../../vm/vm_object.c:705
#12 0xc098039f in vm_object_deallocate (object=0xc5a2271c) at ../../../vm/vm_object.c:602
#13 0xc0977d68 in vm_map_delete (map=0xc85a0dd4, start=0x28300000, end=0x2e500000) at ../../../vm/vm_map.c:2463
#14 0xc097c414 in munmap (td=0xc6352cc0, uap=0xe6d46cfc) at ../../../vm/vm_mmap.c:603
#15 0xc0a78353 in syscall (frame=0xe6d46d38) at ../../../i386/i386/trap.c:1034
#16 0xc0a5e670 in Xint0x80_syscall () at ../../../i386/i386/exception.s:203
#17 0x00000033 in ?? ()
(kgdb) f 11
#11 0xc097e40d in vm_object_terminate (object=0xc5a2271c) at ../../../vm/vm_object.c:705
705                     swap_release_by_uid(object->charge, object->uip);
(kgdb) p *object
$2 = {mtx = {lock_object = {lo_name = 0xc0b2c877 "vm object", lo_type = 0xc0b2e022 "standard object", lo_flags = 0x1430000, lo_witness_data = {lod_list = {
          stqe_next = 0xc0c3b710}, lod_witness = 0xc0c3b710}}, mtx_lock = 0x4, mtx_recurse = 0x0}, object_list = {tqe_next = 0xc4f8d118, tqe_prev = 0xc4ff8b94}, shadow_head = {
    lh_first = 0x0}, shadow_list = {le_next = 0x0, le_prev = 0xc57c4b10}, memq = {tqh_first = 0x0, tqh_last = 0xc5a22748}, root = 0x0, size = 0x6200, generation = 0xc401,
  ref_count = 0x0, shadow_count = 0x0, type = 0x0, flags = 0x2108, pg_color = 0x0, paging_in_progress = 0x0, resident_page_count = 0x0, backing_object = 0x0,
  backing_object_offset = 0x0, pager_object_list = {tqe_next = 0x0, tqe_prev = 0x0}, rvq = {lh_first = 0x0}, cache = 0x0, handle = 0x0, un_pager = {vnp = {vnp_size = 0x0},
    devp = {devp_pglist = {tqh_first = 0x0, tqh_last = 0x0}}, swp = {swp_bcount = 0x0}}, uip = 0xc41dc880, charge = 0x6200000}
(kgdb)

$ ./swap_reserved.sh
18:55:48 127418368
18:55:58 146042880
18:56:08 152793088
18:56:28 152858624
18:56:39 742522880
18:57:42 1459417088
18:57:56 1461497856
18:58:29 1462820864
19:01:52 1645301760
19:02:05 1651392512
19:02:20 1656172544
19:02:46 1657823232
19:03:07 1666215936