GENERIC HEAD from Oct 9 15:08 UTC, vmcore.95

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 7.0-CURRENT #1: Fri Oct 13 16:38:45 CEST 2006
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1032458240 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
fxp0: link state changed to UP
Kernel page fault with the following non-sleepable locks held:
exclusive sleep mutex inp (udpinp) r = 0 (0xc44c73d8) locked @ netinet/udp_usrreq.c:804
KDB: stack backtrace:
kdb_backtrace(1,c45f8234,c,c42a0bd0,e67ae9ec,...) at kdb_backtrace+0x29
witness_warn(5,0,c0946b7b) at witness_warn+0x192
trap(c4470008,28,28,0,e67aeaa8,...) at trap+0x108
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc073e0c1, esp = 0xe67aea34, ebp = 0xe67aea38 ---
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---


Fatal trap 12: page fault while in kernel mode
cpuid = 3; apic id = 07
fault virtual address   = 0xc4479cf0
fault code              = supervisor read, page not present
instruction pointer     = 0x20:0xc073e0c1
stack pointer           = 0x28:0xe67aea34
frame pointer           = 0x28:0xe67aea38
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 1606 (udp)
[thread pid 1606 tid 100155 ]
Stopped at      fnv_32_buf+0x39:        movzbl  0(%eax),%edx
db> where
Tracing pid 1606 tid 100155 td 0xc42a0bd0
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---
db> show allpcpu
Current CPU: 3

cpuid        = 0
curthread    = 0xc3c36d80: pid 15 "swi4: clock sio"
curpcb       = 0xe26d9d90
fpcurthread  = none
idlethread   = 0xc3c36a20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc429fd80: pid 1616 "swap"
curpcb       = 0xe6796d90
fpcurthread  = none
idlethread   = 0xc3c36870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc3c3b870: pid 35 "pagedaemon"
curpcb       = 0xe2706d90
fpcurthread  = none
idlethread   = 0xc3c366c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc42a0bd0: pid 1606 "udp"
curpcb       = 0xe67aed90
fpcurthread  = none
idlethread   = 0xc3c36510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> set $lines 0
db> show alllocks
Process 1645 (sendmail) thread 0xc419b000 (100141)
exclusive sx user map r = 0 (0xc445b148) locked @ vm/vm_map.c:3073
Process 1639 (swap) thread 0xc3ff2bd0 (100060)
exclusive sx user map r = 0 (0xc406e968) locked @ vm/vm_fault.c:797
Process 1636 (swap) thread 0xc45f6000 (100144)
exclusive sleep mutex vm object (standard object) r = 0 (0xc4681a50) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc4405e7c) locked @ vm/vm_map.c:3073
Process 1627 (swap) thread 0xc419bd80 (100119)
exclusive sx user map r = 0 (0xc3c3c968) locked @ vm/vm_fault.c:797
Process 1625 (swap) thread 0xc429fa20 (100176)
exclusive sleep mutex vm object (standard object) r = 0 (0xc3db7618) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc411865c) locked @ vm/vm_map.c:3073
Process 1619 (swap) thread 0xc3ff2360 (100100)
exclusive sleep mutex vm object (standard object) r = 0 (0xc444d348) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc4186b70) locked @ vm/vm_map.c:3073
Process 1616 (swap) thread 0xc429fd80 (100139)
exclusive sleep mutex vm object (standard object) r = 0 (0xc4681870) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc3c3dc74) locked @ vm/vm_map.c:3073
Process 1613 (swap) thread 0xc4055bd0 (100078)
exclusive sleep mutex vm object (standard object) r = 0 (0xc4123960) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc445b454) locked @ vm/vm_map.c:3073
Process 1612 (swap) thread 0xc405b870 (100097)
exclusive sleep mutex vm object (standard object) r = 0 (0xc45f3ca8) locked @ vm/vm_fault.c:978
exclusive sx user map r = 0 (0xc3ff5e7c) locked @ vm/vm_fault.c:797
Process 1611 (swap) thread 0xc419b1b0 (100140)
exclusive sx user map r = 0 (0xc45ec148) locked @ vm/vm_fault.c:797
Process 1606 (udp) thread 0xc42a0bd0 (100155)
exclusive sleep mutex inp (udpinp) r = 0 (0xc44c73d8) locked @ netinet/udp_usrreq.c:804
Process 35 (pagedaemon) thread 0xc3c3b870 (100030)
exclusive sleep mutex pmap r = 0 (0xc0a956e0) locked @ i386/i386/pmap.c:2058
exclusive sleep mutex vm page queue mutex r = 0 (0xc0a6839c) locked @ i386/i386/pmap.c:2056
exclusive sleep mutex system map r = 0 (0xc1871120) locked @ vm/vm_map.c:2389
exclusive sleep mutex UMA lock r = 0 (0xc0a6800c) locked @ vm/uma_core.c:1490
exclusive sleep mutex Giant r = 0 (0xc0a0d968) locked @ vm/vm_pageout.c:687
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
 1650   774   774    25  SL      vmpfw    0xc29e7a78 sendmail
 1645   768   768     0  RL                          sendmail
 1641  1607  1034  1001  R+                          swap
 1640  1607  1034  1001  R+                          swap
 1639  1607  1034  1001  RL+                         swap
 1638  1607  1034  1001  R+                          swap
 1637  1607  1034  1001  R+                          swap
 1636  1607  1034  1001  RL+                         swap
 1635  1607  1034  1001  R+                          swap
 1634  1607  1034  1001  R+                          swap
 1633  1607  1034  1001  SL+     swread   0xc297dd58 swap
 1632  1607  1034  1001  R+                          swap
 1631  1607  1034  1001  R+                          swap
 1630  1607  1034  1001  SL+     swread   0xc2923578 swap
 1629  1607  1034  1001  R+                          swap
 1628  1607  1034  1001  R+                          swap
 1627  1607  1034  1001  RL+                         swap
 1626  1607  1034  1001  R+                          swap
 1625  1607  1034  1001  RL+                         swap
 1624  1607  1034  1001  R+                          swap
 1623  1607  1034  1001  R+                          swap
 1622  1607  1034  1001  R+                          swap
 1621  1607  1034  1001  R+                          swap
 1620  1607  1034  1001  R+                          swap
 1619  1607  1034  1001  RL+                         swap
 1618  1607  1034  1001  R+                          swap
 1617  1607  1034  1001  R+                          swap
 1616  1607  1034  1001  RL+     CPU 1               swap
 1615  1607  1034  1001  R+                          swap
 1614  1607  1034  1001  R+                          swap
 1613  1607  1034  1001  RL+                         swap
 1612  1607  1034  1001  RL+                         swap
 1611  1607  1034  1001  RL+                         swap
 1610  1607  1034  1001  R+                          swap
 1609  1607  1034  1001  R+                          swap
 1608  1607  1034  1001  R+                          swap
 1607  1602  1034  1001  SW+     wait     0xc4111000 swap
 1606  1604  1034  1001  R+      CPU 3               udp
 1605  1604  1034  1001  R+                          udp
 1604  1603  1034  1001  SW+     wait     0xc40588d0 udp
 1603  1038  1034  1001  S+      nanslp   0xc0a0e164 udp
 1602  1038  1034  1001  S+      nanslp   0xc0a0e164 swap
 1392  1391  1392  1001  SWs+    ttyin    0xc3d61010 bash
 1391  1386  1386  1001  SW      select   0xc0a5ac5c sshd
 1386   762  1386     0  SWs     sbwait   0xc479b1e8 sshd
 1085  1079  1085  1001  S+      select   0xc0a5ac5c top
 1079  1078  1079  1001  SWs+    wait     0xc45f7234 bash
 1078  1063  1063  1001  S       select   0xc0a5ac5c sshd
 1063   762  1063     0  SWs     sbwait   0xc41479b0 sshd
 1038  1037  1034  1001  SW+     wait     0xc4058000 run
 1037  1036  1034  1001  SW+     wait     0xc3f81b04 run
 1036  1034  1034  1001  S+      nanslp   0xc0a0e164 run
 1034  1027  1034  1001  SW+     wait     0xc4033b04 sh
 1027  1026  1027  1001  SWs+    wait     0xc403bb04 bash
 1026  1024  1024  1001  SW      select   0xc0a5ac5c sshd
 1024   762  1024     0  SWs     sbwait   0xc411bd94 sshd
  891     1   891     0  SWs+    ttyin    0xc3dfe810 getty
  890     1   890     0  SWs+    ttyin    0xc3e02810 getty
  889     1   889     0  SWs+    ttyin    0xc3e01810 getty
  888     1   888     0  SWs+    ttyin    0xc3e01010 getty
  887     1   887     0  SWs+    ttyin    0xc3e00c10 getty
  886     1   886     0  SWs+    ttyin    0xc3e02c10 getty
  885     1   885     0  SWs+    ttyin    0xc3e03010 getty
  884     1   884     0  SWs+    ttyin    0xc3e01c10 getty
  870     1   870     0  SWs     select   0xc0a5ac5c inetd
  843     1   843     0  SWs     select   0xc0a5ac5c moused
  828     1   828     0  Ss      nanslp   0xc0a0e164 watchdogd
  780     1   780     0  Ss      nanslp   0xc0a0e164 cron
  774     1   774    25  SWs     pause    0xc3de96d0 sendmail
  768     1   768     0  Ss      select   0xc0a5ac5c sendmail
  762     1   762     0  SWs     select   0xc0a5ac5c sshd
  743     1   743     0  Ss      select   0xc0a5ac5c ntpd
  689   685   685     0  SW      -        0xc40dae00 nfsd
  688   685   685     0  SW      -        0xc3f72600 nfsd
  687   685   685     0  SW      -        0xc40db000 nfsd
  686   685   685     0  SW      -        0xc3fbf400 nfsd
  685     1   685     0  SWs     select   0xc0a5ac5c nfsd
  683     1   683     0  SWs     select   0xc0a5ac5c mountd
  633     1   633     0  SWs     select   0xc0a5ac5c rpcbind
  614     1   614     0  SWs     select   0xc0a5ac5c syslogd
  547     1   547     0  SWs     select   0xc0a5ac5c devd
   42     0     0     0  SL      -        0xe43f4cfc [schedcpu]
   41     0     0     0  SL      sdflush  0xc0a67ba0 [softdepflush]
   40     0     0     0  SL      vlruwt   0xc3f7fd38 [vnlru]
   39     0     0     0  SL      syncer   0xc0a0df30 [syncer]
   38     0     0     0  SL      psleep   0xc0a5b0d4 [bufdaemon]
   37     0     0     0  RL                          [pagezero]
   36     0     0     0  SL      psleep   0xc0a68400 [vmdaemon]
   35     0     0     0  RL      CPU 2               [pagedaemon]
   34     0     0     0  WL                          [irq7: ppc0]
   33     0     0     0  SL      -        0xc3d7e03c [fdc0]
   32     0     0     0  WL                          [swi0: sio]
   31     0     0     0  WL                          [irq12: psm0]
   30     0     0     0  WL                          [irq1: atkbd0]
   29     0     0     0  WL                          [irq15: ata1]
   28     0     0     0  WL                          [irq14: ata0]
   27     0     0     0  WL                          [irq17: fxp0]
   26     0     0     0  SL      usbtsk   0xc0a0b6e4 [usbtask]
   25     0     0     0  SL      usbevt   0xc3d6d210 [usb0]
   24     0     0     0  WL                          [irq16: uhci0]
   23     0     0     0  SL      -        0xc3d5a280 [em0 taskq]
   22     0     0     0  WL                          [irq9: acpi0]
   21     0     0     0  WL                          [swi5: +]
    9     0     0     0  SL      -        0xc3c85700 [thread taskq]
   20     0     0     0  WL                          [swi6: Giant taskq]
    8     0     0     0  SL      -        0xc3c85880 [acpi_task_2]
    7     0     0     0  SL      -        0xc3c85880 [acpi_task_1]
    6     0     0     0  SL      -        0xc3c85880 [acpi_task_0]
   19     0     0     0  WL                          [swi6: task queue]
    5     0     0     0  SL      -        0xc3c85a00 [kqueue taskq]
   18     0     0     0  WL                          [swi2: cambio]
   17     0     0     0  SL      -        0xc0a09320 [yarrow]
    4     0     0     0  SL      -        0xc0a0be9c [g_down]
    3     0     0     0  RL                          [g_up]
    2     0     0     0  SL      -        0xc0a0be90 [g_event]
   16     0     0     0  WL                          [swi3: vm]
   15     0     0     0  RL      CPU 0               [swi4: clock sio]
   14     0     0     0  WL                          [swi1: net]
   13     0     0     0  RL                          [idle: cpu0]
   12     0     0     0  RL                          [idle: cpu1]
   11     0     0     0  RL                          [idle: cpu2]
   10     0     0     0  RL                          [idle: cpu3]
    1     0     1     0  SLs     wait     0xc3c3a000 [init]
    0     0     0     0  WLs                         [swapper]
db> allt

Tracing command sendmail pid 1650 tid 100085 td 0xc4055000
sched_switch(c4055000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c29e7a78,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c29e7a78,0,c093a45d,185,c093b7e7,...) at sleepq_wait+0x46
msleep(c29e7a78,c4088ca8,44,c093a500,0,...) at msleep+0x27d
vm_page_sleep(c29e7a78,c093a500) at vm_page_sleep+0x80
vm_fault(c404c410,808e000,1,0,c4055000,...) at vm_fault+0x4b7
trap_pfault(e6676d38,1,808edf8,808edf8,808edf8,...) at trap_pfault+0xee
trap(c088003b,3b,3b,0,0,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x808edf8, esp = 0xbfbfcb6c, ebp = 0xbfbfcfb8 ---

Tracing command sendmail pid 1645 tid 100141 td 0xc419b000
sched_switch(c419b000,c3de5870,6) at sched_switch+0x183
mi_switch(6,c3de5870,c3de5a00,c3de5870,e6748838,...) at mi_switch+0x280
maybe_preempt(c3de5870) at maybe_preempt+0x199
sched_add(c3de5870,4,c3c83b00,c3de5870,c3de3460,...) at sched_add+0x15a
setrunqueue(c3de5870,4) at setrunqueue+0xab
intr_event_schedule_thread(c3c83b00) at intr_event_schedule_thread+0xbb
intr_execute_handlers(c3c2c22c,e67488bc,e,e6748984,c0884933,...) at intr_execute_handlers+0x118
lapic_handle_intr(34) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc08a212c, esp = 0xe67488fc, ebp = 0xe6748984 ---
__qdivrem(800,0,800,0,0) at __qdivrem+0xcc
__divdi3(800,0,800,0) at __divdi3+0x5b
ufs_getlbns(c4188150,5,0,e6748a6c,e6748a58) at ufs_getlbns+0x1d7
ufs_bmaparray(c4188150,11,0,e6748acc,0,...) at ufs_bmaparray+0x8b
ufs_bmap(e6748b20) at ufs_bmap+0x4b
VOP_BMAP_APV(c09dca20,e6748b20) at VOP_BMAP_APV+0xa9
vnode_pager_haspage(c4088ca8,46,0,e6748b84,e6748b80) at vnode_pager_haspage+0x174
vm_fault_additional_pages(c29e7a78,7,8,e6748c30,e6748bf4,...) at vm_fault_additional_pages+0x9d
vm_fault(c445b104,808e000,1,0,c419b000,...) at vm_fault+0x883
trap_pfault(e6748d38,1,808edf8,808edf8,808edf8,...) at trap_pfault+0xee
trap(823003b,80d003b,bfbf003b,0,0,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x808edf8, esp = 0xbfbfbf5c, ebp = 0xbfbfc3a8 ---

Tracing command swap pid 1641 tid 100122 td 0xc419b870
sched_switch(c419b870,0,2) at sched_switch+0x183
mi_switch(2,0,c419b870,d4,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6757d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1640 tid 100169 td 0xc3d9e1b0
sched_switch(c3d9e1b0,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9e1b0,d4,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e687dd38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1639 tid 100060 td 0xc3ff2bd0
sched_switch(c3ff2bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c4037000,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c3ff2bd0,0,c0945f66,915) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c0945f66,915,c3c12440,...) at _mtx_lock_flags+0xae
pmap_enter(c406e9cc,9b19000,c2538948,7,0,c3db80f0,0,c093a454,378) at pmap_enter+0x38
vm_fault(c406e924,9b19000,2,8,c3ff2bd0,...) at vm_fault+0x1040
trap_pfault(e6601d38,1,9b19000,9b19000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1638 tid 100129 td 0xc4112a20
sched_switch(c4112a20,0,2) at sched_switch+0x183
mi_switch(2,0,c4112a20,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66e8d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1637 tid 100185 td 0xc419ca20
sched_switch(c419ca20,0,2) at sched_switch+0x183
mi_switch(2,0,c419ca20,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6775d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1636 tid 100144 td 0xc45f6000
sched_switch(c45f6000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c45f6000,0,c093a454,1f2) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093a454,1f2,1ec2,...) at _mtx_lock_flags+0xae
vm_fault(c4405e38,9f13000,2,8,c45f6000,...) at vm_fault+0x77e
trap_pfault(e6829d38,1,9f13000,9f13000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1635 tid 100145 td 0xc45f2d80
sched_switch(c45f2d80,0,2) at sched_switch+0x183
mi_switch(2,0,c45f2d80,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6826d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1634 tid 100131 td 0xc41126c0
sched_switch(c41126c0,0,2) at sched_switch+0x183
mi_switch(2,0,c41126c0,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66e2d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1633 tid 100063 td 0xc4037000
sched_switch(c4037000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c297dd58,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c297dd58) at sleepq_timedwait+0x4a
msleep(c297dd58,c467f078,40,c09397fe,4e20) at msleep+0x259
swap_pager_getpages(c467f078,e664cc30,1,0) at swap_pager_getpages+0x348
vm_fault(c406e000,82e9000,2,8,c4037000,...) at vm_fault+0x928
trap_pfault(e664cd38,1,82e9000,82e9000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1632 tid 100073 td 0xc3ff3d80
sched_switch(c3ff3d80,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff3d80,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e661fd38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1631 tid 100087 td 0xc4037bd0
sched_switch(c4037bd0,0,2) at sched_switch+0x183
mi_switch(2,0,c4037bd0,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6661d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1630 tid 100102 td 0xc3ff2000
sched_switch(c3ff2000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c2923578,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c2923578) at sleepq_timedwait+0x4a
msleep(c2923578,c4613c30,40,c09397fe,4e20) at msleep+0x259
swap_pager_getpages(c4613c30,e65ecc30,2,0) at swap_pager_getpages+0x348
vm_fault(c445b618,8155000,2,8,c3ff2000,...) at vm_fault+0x928
trap_pfault(e65ecd38,1,8155000,8155000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1629 tid 100079 td 0xc4055a20
sched_switch(c4055a20,0,2) at sched_switch+0x183
mi_switch(2,0,c4055a20,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6688d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1628 tid 100174 td 0xc3d9c870
sched_switch(c3d9c870,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9c870,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e686ed38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1627 tid 100119 td 0xc419bd80
sched_switch(c419bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c419bd80,0,c0945f66,915) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c0945f66,915,c3c12440,...) at _mtx_lock_flags+0xae
pmap_enter(c3c3c9cc,9331000,c2065398,7,0,c467f168,0,c093a454,378) at pmap_enter+0x38
vm_fault(c3c3c924,9331000,2,8,c419bd80,...) at vm_fault+0x1040
trap_pfault(e6760d38,1,9331000,9331000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1626 tid 100162 td 0xc3de8510
sched_switch(c3de8510,0,2) at sched_switch+0x183
mi_switch(2,0,c3de8510,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e43b6d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1625 tid 100176 td 0xc429fa20
sched_switch(c429fa20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c4037000,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c429fa20,0,c093a454,34b) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093a454,34b,2,...) at _mtx_lock_flags+0xae
vm_fault(c4118618,9341000,2,8,c429fa20,...) at vm_fault+0xf3f
trap_pfault(e6790d38,1,9341000,9341000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1624 tid 100135 td 0xc4112000
sched_switch(c4112000,0,2) at sched_switch+0x183
mi_switch(2,0,c4112000,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66d6d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1623 tid 100113 td 0xc3ff3510
sched_switch(c3ff3510,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff3510,d0,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6610d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1622 tid 100172 td 0xc3d9cbd0
sched_switch(c3d9cbd0,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9cbd0,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6874d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1621 tid 100179 td 0xc429f510
sched_switch(c429f510,0,2) at sched_switch+0x183
mi_switch(2,0,c429f510,d0,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6787d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1620 tid 100168 td 0xc3d9e360
sched_switch(c3d9e360,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9e360,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6880d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1619 tid 100100 td 0xc3ff2360
sched_switch(c3ff2360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c3ff2360,0,c093b7e7,368) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093b7e7,368,c0a68384,0,c093b7e7,362) at _mtx_lock_flags+0xae
vm_page_alloc(c444d348,1602,0,40,2,...) at vm_page_alloc+0x131
vm_fault(c4186b2c,9652000,2,8,c3ff2360,...) at vm_fault+0x60a
trap_pfault(e65f2d38,1,9652000,9652000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1618 tid 100183 td 0xc419cd80
sched_switch(c419cd80,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0d950,0,c09205db,...) at mi_switch+0x280
critical_exit(1,bfbfe948,c0884b4e,2805003b,3b,...) at critical_exit+0x92
lapic_handle_timer(2805003b,3b,bfbf003b,2805188c,bfbfea7c,...) at lapic_handle_timer+0x149
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0x8049183, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1617 tid 100057 td 0xc3ff31b0
sched_switch(c3ff31b0,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff31b0,d4,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e660ad38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1616 tid 100139 td 0xc429fd80
cpustop_handler(e6796ae4,c0899182,c429fd80,d5,d46abd29,...) at cpustop_handler+0x31
ipi_nmi_handler(c429fd80,d5,d46abd29,0,0,...) at ipi_nmi_handler+0x28
trap(c4290008,e6790028,28,c429fd80,c3c3b870,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068f052, esp = 0xe6796b2c, ebp = 0xe6796b3c ---
_mtx_lock_sleep(c0a6839c,c429fd80,0,c093b7e7,368) at _mtx_lock_sleep+0x112
_mtx_lock_flags(c0a6839c,0,c093b7e7,368,c0a68384,0,c093b7e7,362) at _mtx_lock_flags+0xae
vm_page_alloc(c4681870,12b2,0,40,c06ad59b,...) at vm_page_alloc+0x131
vm_fault(c3c3dc30,9302000,2,8,c429fd80,...) at vm_fault+0x60a
trap_pfault(e6796d38,1,9302000,9302000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1615 tid 100112 td 0xc3ff36c0
sched_switch(c3ff36c0,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff36c0,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6613d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1614 tid 100072 td 0xc4034000
sched_switch(c4034000,0,2) at sched_switch+0x183
mi_switch(2,0,c4034000,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6631d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1613 tid 100078 td 0xc4055bd0
sched_switch(c4055bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c4055bd0,0,c093b7e7,368) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093b7e7,368,c0a68384,0,c093b7e7,362) at _mtx_lock_flags+0xae
vm_page_alloc(c4123960,1703,0,40,c4055bd0,...) at vm_page_alloc+0x131
vm_fault(c445b410,9753000,2,8,c4055bd0,...) at vm_fault+0x60a
trap_pfault(e668bd38,1,9753000,9753000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1612 tid 100097 td 0xc405b870
sched_switch(c405b870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3ff2360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c405b870,0,c093a454,3e8) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093a454,3e8,c45f3ca8,...) at _mtx_lock_flags+0xae
vm_fault_prefault(c3ff5ee0,991d000,c46b750c) at vm_fault_prefault+0x1c0
vm_fault(c3ff5e38,991d000,2,8,c405b870,...) at vm_fault+0x1063
trap_pfault(e66bbd38,1,991d000,991d000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1611 tid 100140 td 0xc419b1b0
sched_switch(c419b1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c419b1b0,0,c0945f66,915) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c0945f66,915,c3c12440,...) at _mtx_lock_flags+0xae
pmap_enter(c45ec1ac,9da1000,c24cbaa8,7,0,c46b8438,0,c093a454,378) at pmap_enter+0x38
vm_fault(c45ec104,9da1000,2,8,c419b1b0,...) at vm_fault+0x1040
trap_pfault(e674bd38,1,9da1000,9da1000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---

Tracing command swap pid 1610 tid 100163 td 0xc3de8360
sched_switch(c3de8360,0,2) at sched_switch+0x183
mi_switch(2,0,c3de8360,ca,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e43b3d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1609 tid 100094 td 0xc405bd80
sched_switch(c405bd80,0,2) at sched_switch+0x183
mi_switch(2,0,c405bd80,d1,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66c4d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1608 tid 100153 td 0xc45f2000
sched_switch(c45f2000,0,2) at sched_switch+0x183
mi_switch(2,0,c45f2000,d2,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e680ed38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1607 tid 100105 td 0xc3f80a20

Tracing command udp pid 1606 tid 100155 td 0xc42a0bd0
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---

Tracing command udp pid 1605 tid 100111 td 0xc41131b0
sched_switch(c41131b0,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0d950,0,c09205db,...) at mi_switch+0x280
critical_exit(c0a0d950,e66f4a64,c068ee4d,c3c36bd0,e66f4a90,...) at critical_exit+0x92
spinlock_exit(c3c36bd0,e66f4a90,c06bf249,c0a0d950,0,...) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c0a0d950,0,c0922599,371,0,...) at _mtx_unlock_spin_flags+0xb1
turnstile_unpend(c3e06100,0,c411e678,39f,e66f4acc,...) at turnstile_unpend+0x371
_mtx_unlock_sleep(c411e678,0,c08c9b31,39f) at _mtx_unlock_sleep+0x6b
_mtx_unlock_flags(c411e678,0,c08c9b31,39f) at _mtx_unlock_flags+0xcc
udp_output(c411e5e8,c465ae00,0,0,c41131b0) at udp_output+0x669
udp_send(c4609530,0,c465ae00,0,0,...) at udp_send+0x33
sosend_dgram(c4609530,0,e66f4c64,c465ae00,0,...) at sosend_dgram+0x30d
sosend(c4609530,0,e66f4c64,0,0,0,c41131b0) at sosend+0x3c
soo_write(c4004c60,e66f4c64,c4527200,0,c41131b0) at soo_write+0x42
dofilewrite(c41131b0,3,c4004c60,e66f4c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c41131b0,3,e66f4c64,8050040,0,...) at kern_writev+0x36
write(c41131b0,e66f4d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---

Tracing command udp pid 1604 tid 100088 td 0xc4037a20

Tracing command udp pid 1603 tid 100052 td 0xc3de8d80
sched_switch(c3de8d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,3e9,...) at msleep+0x244
kern_nanosleep(c3de8d80,e43c5c74,e43c5c6c) at kern_nanosleep+0xab
nanosleep(c3de8d80,e43c5d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe93c, ebp = 0xbfbfe968 ---

Tracing command swap pid 1602 tid 100187 td 0xc4616510
sched_switch(c4616510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,3e9,...) at msleep+0x244
kern_nanosleep(c4616510,e68b9c74,e68b9c6c) at kern_nanosleep+0xab
nanosleep(c4616510,e68b9d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe93c, ebp = 0xbfbfe968 ---

Tracing command bash pid 1392 tid 100156 td 0xc42a0a20

Tracing command sshd pid 1391 tid 100164 td 0xc3de81b0

Tracing command sshd pid 1386 tid 100127 td 0xc4112d80

Tracing command top pid 1085 tid 100067 td 0xc4034870
sched_switch(c4034870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a5ac5c,c0a5ac5c,3e9,e6640ae8,246,...) at sleepq_timedwait_sig+0x1e
cv_timedwait_sig(c0a5ac5c,c0a5ac44,3e9) at cv_timedwait_sig+0x196
kern_select(c4034870,2,bfbfeaf0,0,0,...) at kern_select+0x4a9
select(c4034870,e6640d04) at select+0x44
syscall(3b,bfbf003b,bfbf003b,bfbfeaf0,1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x281b316b, esp = 0xbfbfea6c, ebp = 0xbfbfebe8 ---

Tracing command bash pid 1079 tid 100148 td 0xc45f2870

Tracing command sshd pid 1078 tid 100039 td 0xc3de5510
sched_switch(c3de5510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_wait_sig(c0a5ac5c,246,c0a0d950,c09239cd,307,...) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a5ac5c,c0a5ac44) at cv_wait_sig+0x18d
kern_select(c3de5510,b,8201168,820116c,0,...) at kern_select+0x4bf
select(c3de5510,e439bd04) at select+0x44
syscall(bfbf003b,bfbf003b,bfbf003b,0,bfbfe13c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x2838b16b, esp = 0xbfbfe0cc, ebp = 0xbfbfe118 ---

Tracing command sshd pid 1063 tid 100053 td 0xc3de8bd0

Tracing command run pid 1038 tid 100084 td 0xc40551b0

Tracing command run pid 1037 tid 100054 td 0xc3de8a20

Tracing command run pid 1036 tid 100081 td 0xc40556c0
sched_switch(c40556c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,3e9,...) at msleep+0x244
kern_nanosleep(c40556c0,e6682c74,e6682c6c) at kern_nanosleep+0xab
nanosleep(c40556c0,e6682d04) at nanosleep+0x4f
syscall(2805003b,3b,bfbf003b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe8fc, ebp = 0xbfbfe928 ---

Tracing command sh pid 1034 tid 100068 td 0xc40346c0

Tracing command bash pid 1027 tid 100075 td 0xc3ff3a20

Tracing command sshd pid 1026 tid 100118 td 0xc419c000

Tracing command sshd pid 1024 tid 100056 td 0xc3ff3360

Tracing command getty pid 891 tid 100089 td 0xc4037870

Tracing command getty pid 890 tid 100099 td 0xc3ff2510

Tracing command getty pid 889 tid 100137 td 0xc42a01b0

Tracing command getty pid 888 tid 100108 td 0xc41136c0

Tracing command getty pid 887 tid 100062 td 0xc3ff2870

Tracing command getty pid 886 tid 100124 td 0xc419b510

Tracing command getty pid 885 tid 100125 td 0xc419b360

Tracing command getty pid 884 tid 100126 td 0xc4113000

Tracing command inetd pid 870 tid 100128 td 0xc4112bd0

Tracing command moused pid 843 tid 100132 td 0xc4112510

Tracing command watchdogd pid 828 tid 100136 td 0xc42a0360
sched_switch(c42a0360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,ea61,...) at msleep+0x244
kern_nanosleep(c42a0360,e679fc74,e679fc6c) at kern_nanosleep+0xab
nanosleep(c42a0360,e679fd04) at nanosleep+0x4f
syscall(3b,3b,bfbf003b,1,3c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28157beb, esp = 0xbfbfed3c, ebp = 0xbfbfed68 ---

Tracing command cron pid 780 tid 100042 td 0xc3de5000
sched_switch(c3de5000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,88b9,...) at msleep+0x244
kern_nanosleep(c3de5000,e4392c74,e4392c6c) at kern_nanosleep+0xab
nanosleep(c3de5000,e4392d04) at nanosleep+0x4f
syscall(3b,3b,3b,805008c,23,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28147beb, esp = 0xbfbfecec, ebp = 0xbfbfed18 ---

Tracing command sendmail pid 774 tid 100038 td 0xc3de56c0

Tracing command sendmail pid 768 tid 100051 td 0xc3f80000
sched_switch(c3f80000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a5ac5c,c0a5ac5c,1389,e43e5ae8,246,...) at sleepq_timedwait_sig+0x1e
cv_timedwait_sig(c0a5ac5c,c0a5ac44,1389) at cv_timedwait_sig+0x196
kern_select(c3f80000,5,bfbfc520,0,0,...) at kern_select+0x4a9
select(c3f80000,e43e5d04) at select+0x44
syscall(bfbf003b,3b,bfbf003b,1,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x2834a16b, esp = 0xbfbfc42c, ebp = 0xbfbfd028 ---

Tracing command sshd pid 762 tid 100080 td 0xc4055870

Tracing command ntpd pid 743 tid 100107 td 0xc4113870
sched_switch(c4113870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_wait_sig(c0a5ac5c,246,c0a0d950,c09239cd,307,...) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a5ac5c,c0a5ac44) at cv_wait_sig+0x18d
kern_select(c4113870,9,bfbfed70,0,0,...) at kern_select+0x4bf
select(c4113870,e6700d04) at select+0x44
syscall(bfbf003b,3b,bfbf003b,bfbfed70,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x282cf16b, esp = 0xbfbfed2c, ebp = 0xbfbfee08 ---

Tracing command nfsd pid 689 tid 100059 td 0xc3ff2d80

Tracing command nfsd pid 688 tid 100065 td 0xc4034bd0

Tracing command nfsd pid 687 tid 100106 td 0xc4113a20

Tracing command nfsd pid 686 tid 100074 td 0xc3ff3bd0

Tracing command nfsd pid 685 tid 100083 td 0xc4055360

Tracing command mountd pid 683 tid 100050 td 0xc3f801b0

Tracing command rpcbind pid 633 tid 100055 td 0xc3de8870

Tracing command syslogd pid 614 tid 100066 td 0xc4034a20

Tracing command devd pid 547 tid 100058 td 0xc3ff3000

Tracing command schedcpu pid 42 tid 100046 td 0xc3f80870
sched_switch(c3f80870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(e43f4cfc,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(e43f4cfc) at sleepq_timedwait+0x4a
msleep(e43f4cfc,0,0,c0917342,3e8) at msleep+0x259
schedcpu_thread(0,e43f4d38) at schedcpu_thread+0x26
fork_exit(c06abf90,0,e43f4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43f4d6c, ebp = 0 ---

Tracing command softdepflush pid 41 tid 100047 td 0xc3f806c0
sched_switch(c3f806c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a67ba0,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a67ba0) at sleepq_timedwait+0x4a
msleep(c0a67ba0,c0a67b78,44,c093737f,3e8) at msleep+0x259
softdep_flush(0,e43f1d38) at softdep_flush+0x203
fork_exit(c07cf1f4,0,e43f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43f1d6c, ebp = 0 ---

Tracing command vnlru pid 40 tid 100048 td 0xc3f80510
sched_switch(c3f80510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3f7fd38,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3f7fd38) at sleepq_timedwait+0x4a
msleep(c3f7fd38,c0a5b398,250,c0928f82,3e8,c0a5b3f8) at msleep+0x259
vnlru_proc(0,e43eed38) at vnlru_proc+0xdf
fork_exit(c06f4cbc,0,e43eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43eed6c, ebp = 0 ---

Tracing command syncer pid 39 tid 100049 td 0xc3f80360
sched_switch(c3f80360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0df30,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a0df30,0,c068ec81,c0a5b3c4,c0928ec1,...) at sleepq_wait+0x46
msleep(c0a0df30,0,68,c0929290,0) at msleep+0x27d
sched_sync(0,e43ebd38) at sched_sync+0x361
fork_exit(c06f6964,0,e43ebd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43ebd6c, ebp = 0 ---

Tracing command bufdaemon pid 38 tid 100027 td 0xc3c3bd80
sched_switch(c3c3bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5b0d4,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a5b0d4) at sleepq_timedwait+0x4a
msleep(c0a5b0d4,c0a5b0d8,44,c092783c,3e8) at msleep+0x259
buf_daemon(0,e270fd38) at buf_daemon+0x198
fork_exit(c06e77ac,0,e270fd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe270fd6c, ebp = 0 ---

Tracing command pagezero pid 37 tid 100028 td 0xc3c3bbd0
sched_switch(c3c3bbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a704d0,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a704d0) at sleepq_timedwait+0x4a
msleep(c0a704d0,c0a6839c,200,c093c392,493e0,c0a6839c,0,c093c378,9a) at msleep+0x259
vm_pagezero(0,e270cd38) at vm_pagezero+0x6c
fork_exit(c0801cbc,0,e270cd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe270cd6c, ebp = 0 ---

Tracing command vmdaemon pid 36 tid 100029 td 0xc3c3ba20
sched_switch(c3c3ba20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a68400,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a68400,0,e2709cd0,c069d310,c093bf66,...) at sleepq_wait+0x46
msleep(c0a68400,0,68,c092783c,0) at msleep+0x27d
vm_daemon(0,e2709d38) at vm_daemon+0x36
fork_exit(c0800908,0,e2709d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2709d6c, ebp = 0 ---

Tracing command pagedaemon pid 35 tid 100030 td 0xc3c3b870
cpustop_handler(e2706ac4,c0899182,3,a,2,...) at cpustop_handler+0x31
ipi_nmi_handler(3,a,2,0,0,...) at ipi_nmi_handler+0x28
trap(c0008,28,c0a90028,c0a956e0,c3dbb000,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc0890cdd, esp = 0xe2706b0c, ebp = 0xe2706b10 ---
smp_tlb_shootdown(f4,c3dbb000,0) at smp_tlb_shootdown+0x61
smp_invlpg(c3dbb000) at smp_invlpg+0x1b
pmap_invalidate_page(c0a956e0,c3dbb000) at pmap_invalidate_page+0x6c
pmap_remove_pte(c0a956e0,bff0f6ec,c3dbb000,c1870c38,c3dbb000,...) at pmap_remove_pte+0x5d
pmap_remove_page(c0a956e0,c3dbb000) at pmap_remove_page+0x6d
pmap_remove(c0a956e0,c3dbb000,c3dbc000,c07f4f43,c1871120,...) at pmap_remove+0x7f
vm_map_delete(c18710a8,c3dbb000,c3dbc000,1,c464baf8,...) at vm_map_delete+0x119
vm_map_remove(c18710a8,c3dbb000,c3dbc000,e2706c04,c07ed2ed,...) at vm_map_remove+0x42
kmem_free(c18710a8,c3dbb000,1000,e2706c34,c07eccbb,...) at kmem_free+0x25
page_free(c3dbb000,1000,2) at page_free+0x29
zone_drain(c1864b40) at zone_drain+0x27b
zone_foreach(c07eca40,e2706ce4,c07ff6af,e2706c6c,246,...) at zone_foreach+0x37
uma_reclaim(e2706c6c,246,0,e2706c78,c068ee4d,...) at uma_reclaim+0x12
vm_pageout_scan(0,c0a6839c,0,c093bf5d,5ee) at vm_pageout_scan+0x103
vm_pageout(0,e2706d38) at vm_pageout+0x2c7
fork_exit(c08005c4,0,e2706d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2706d6c, ebp = 0 ---

Tracing command irq7: ppc0 pid 34 tid 100031 td 0xc3c3b6c0
fork_trampoline() at fork_trampoline

Tracing command fdc0 pid 33 tid 100032 td 0xc3c3b510
sched_switch(c3c3b510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d7e03c,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3d7e03c) at sleepq_timedwait+0x4a
msleep(c3d7e03c,c3d7e0f0,4c,c0917342,3e8) at msleep+0x259
fdc_worker(c3d7e000,c3d7e0f0,0,c093f04c,4a1) at fdc_worker+0x1fe
fdc_thread(c3d7e000,e2700d38) at fdc_thread+0x54
fork_exit(c085e2ec,c3d7e000,e2700d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2700d6c, ebp = 0 ---

Tracing command swi0: sio pid 32 tid 100033 td 0xc3de8000
sched_switch(c3de8000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3defc90,e43add38) at ithread_loop+0xda
fork_exit(c06830c8,c3defc90,e43add38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43add6c, ebp = 0 ---

Tracing command irq12: psm0 pid 31 tid 100034 td 0xc3de5d80
fork_trampoline() at fork_trampoline

Tracing command irq1: atkbd0 pid 30 tid 100035 td 0xc3de5bd0
fork_trampoline() at fork_trampoline

Tracing command irq15: ata1 pid 29 tid 100036 td 0xc3de5a20
fork_trampoline() at fork_trampoline

Tracing command irq14: ata0 pid 28 tid 100037 td 0xc3de5870
sched_switch(c3de5870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3de3460,e43a1d38) at ithread_loop+0xda
fork_exit(c06830c8,c3de3460,e43a1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43a1d6c, ebp = 0 ---

Tracing command irq17: fxp0 pid 27 tid 100017 td 0xc3c376c0
sched_switch(c3c376c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3d7b460,e26e8d38) at ithread_loop+0xda
fork_exit(c06830c8,c3d7b460,e26e8d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e8d6c, ebp = 0 ---

Tracing command usbtask pid 26 tid 100018 td 0xc3c37510
sched_switch(c3c37510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0b6e4,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a0b6e4,0,c068ebc8,c0a0d968,c091438a,...) at sleepq_wait+0x46
msleep(c0a0b6e4,0,5c,c09143a6,0,...) at msleep+0x27d
usb_task_thread(0,e26e5d38) at usb_task_thread+0x4b
fork_exit(c0632498,0,e26e5d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e5d6c, ebp = 0 ---

Tracing command usb0 pid 25 tid 100019 td 0xc3c37360
sched_switch(c3c37360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d6d210,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3d6d210) at sleepq_timedwait+0x4a
msleep(c3d6d210,0,5c,c091439f,ea60,c3d798c0) at msleep+0x259
usb_event_thread(c3d798c0,e26e2d38) at usb_event_thread+0x94
fork_exit(c06323e4,c3d798c0,e26e2d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e2d6c, ebp = 0 ---

Tracing command irq16: uhci0 pid 24 tid 100020 td 0xc3c371b0
sched_switch(c3c371b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3d690b0,e26dfd38) at ithread_loop+0xda
fork_exit(c06830c8,c3d690b0,e26dfd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26dfd6c, ebp = 0 ---

Tracing command em0 taskq pid 23 tid 100021 td 0xc3d46870
sched_switch(c3d46870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d5a280,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3d5a280,c3d5a29c,8,c092245d,47,...) at sleepq_wait+0x46
msleep_spin(c3d5a280,c3d5a29c,c0917342,0) at msleep_spin+0x1a1
taskqueue_thread_loop(c3d30208,e4147d38) at taskqueue_thread_loop+0x61
fork_exit(c06bd590,c3d30208,e4147d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe4147d6c, ebp = 0 ---

Tracing command irq9: acpi0 pid 22 tid 100022 td 0xc3d466c0
fork_trampoline() at fork_trampoline

Tracing command swi5: + pid 21 tid 100023 td 0xc3d46510
fork_trampoline() at fork_trampoline

Tracing command thread taskq pid 9 tid 100024 td 0xc3d46360
sched_switch(c3d46360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85700,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85700,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85700,c3c8571c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c0a13670,e413ed38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c0a13670,e413ed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe413ed6c, ebp = 0 ---

Tracing command swi6: Giant taskq pid 20 tid 100025 td 0xc3d461b0
fork_trampoline() at fork_trampoline

Tracing command acpi_task_2 pid 8 tid 100026 td 0xc3d46000
sched_switch(c3d46000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c10a1a34,e4138d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c10a1a34,e4138d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe4138d6c, ebp = 0 ---

Tracing command acpi_task_1 pid 7 tid 100008 td 0xc3c361b0
sched_switch(c3c361b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c10a1a34,e26c4d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c10a1a34,e26c4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26c4d6c, ebp = 0 ---

Tracing command acpi_task_0 pid 6 tid 100009 td 0xc3c36000
sched_switch(c3c36000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0922466,49,c3d52800,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c10a1a34,e26c1d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c10a1a34,e26c1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26c1d6c, ebp = 0 ---

Tracing command swi6: task queue pid 19 tid 100010 td 0xc3c3b360
sched_switch(c3c3b360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3cff590,e26fdd38) at ithread_loop+0xda
fork_exit(c06830c8,c3cff590,e26fdd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26fdd6c, ebp = 0 ---

Tracing command kqueue taskq pid 5 tid 100011 td 0xc3c3b1b0
sched_switch(c3c3b1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85a00,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85a00,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85a00,c3c85a1c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c0a0c69c,e26fad38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c0a0c69c,e26fad38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26fad6c, ebp = 0 ---

Tracing command swi2: cambio pid 18 tid 100012 td 0xc3c3b000
fork_trampoline() at fork_trampoline

Tracing command yarrow pid 17 tid 100013 td 0xc3c37d80
sched_switch(c3c37d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a09320,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a09320) at sleepq_timedwait+0x4a
msleep(c0a09320,0,0,c0917342,64) at msleep+0x259
random_kthread(0,e26f4d38) at random_kthread+0x180
fork_exit(c05d7b78,0,e26f4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f4d6c, ebp = 0 ---

Tracing command g_down pid 4 tid 100014 td 0xc3c37bd0
sched_switch(c3c37bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0be9c,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a0be9c) at sleepq_timedwait+0x4a
msleep(c0a0be9c,c0a0bd88,24c,c0917342,64) at msleep+0x259
g_io_schedule_down(c3c37bd0) at g_io_schedule_down+0x56
g_down_procbody(0,e26f1d38) at g_down_procbody+0x5a
fork_exit(c0660628,0,e26f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f1d6c, ebp = 0 ---

Tracing command g_up pid 3 tid 100015 td 0xc3c37a20
sched_switch(c3c37a20,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0d950,0,c09205db,...) at mi_switch+0x280
critical_exit(c0a5a9d8,e26eec44,c0884b4e,e26e0008,c3c30028,...) at critical_exit+0x92
lapic_handle_timer(e26e0008,c3c30028,c3c30028,c0a5aa10,c6,...) at lapic_handle_timer+0x149
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc0897402, esp = 0xe26eec20, ebp = 0xe26eec44 ---
generic_bzero(c0a1c6c8,c0a0bd34,c6,0,e26eec80,...) at generic_bzero+0x1a
witness_lock(c0a0bd34,8,c0917d01,c6) at witness_lock+0xc1
_mtx_lock_flags(c0a0bd34,0,c0917cf8,c6,c065c000,...) at _mtx_lock_flags+0xc4
g_disk_done(c3d92dec) at g_disk_done+0x1e
biodone(c3d92dec) at biodone+0x58
ad_done(c44e2540) at ad_done+0x2a
ata_completed(c44e2540) at ata_completed+0x4f0
g_io_schedule_up(c3c37a20) at g_io_schedule_up+0x5e
g_up_procbody(0,e26eed38) at g_up_procbody+0x5a
fork_exit(c06605c8,0,e26eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26eed6c, ebp = 0 ---

Tracing command g_event pid 2 tid 100016 td 0xc3c37870
sched_switch(c3c37870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0be90,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a0be90) at sleepq_timedwait+0x4a
msleep(c0a0be90,0,4c,c0917342,64) at msleep+0x259
g_event_procbody(0,e26ebd38) at g_event_procbody+0x9e
fork_exit(c0660688,0,e26ebd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26ebd6c, ebp = 0 ---

Tracing command swi3: vm pid 16 tid 100000 td 0xc3c37000
fork_trampoline() at fork_trampoline

Tracing command swi4: clock sio pid 15 tid 100001 td 0xc3c36d80
cpustop_handler(e26d9b60,c0899182,c06ad407,3,a,...) at cpustop_handler+0x31
ipi_nmi_handler(c06ad407,3,a,0,0,...) at ipi_nmi_handler+0x28
trap(360008,c3c30028,e26d0028,60,c3df3c00,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc08707fc, esp = 0xe26d9ba8, ebp = 0xe26d9bb8 ---
siointr1(c3df3c00,c0a7cf8c,0,c0940fd0,56e,...) at siointr1+0x374
siointr(c3df3c00) at siointr+0x21
intr_execute_handlers(c3c2c0c4,e26d9c14,4,e26d9c64,c0884933,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc068f052, esp = 0xe26d9c54, ebp = 0xe26d9c64 ---
_mtx_lock_sleep(c0a0d968,c3c36d80,0,c091fd5e,f0) at _mtx_lock_sleep+0x112
_mtx_lock_flags(c0a0d968,0,c091fd5e,f0) at _mtx_lock_flags+0xae
softclock(0) at softclock+0x1b7
ithread_execute_handlers(c3c35234,c3c84500) at ithread_execute_handlers+0x122
ithread_loop(c3c118a0,e26d9d38) at ithread_loop+0x67
fork_exit(c06830c8,c3c118a0,e26d9d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d9d6c, ebp = 0 ---

Tracing command swi1: net pid 14 tid 100002 td 0xc3c36bd0
sched_switch(c3c36bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3c118b0,e26d6d38) at ithread_loop+0xda
fork_exit(c06830c8,c3c118b0,e26d6d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d6d6c, ebp = 0 ---

Tracing command idle: cpu0 pid 13 tid 100003 td 0xc3c36a20
sched_switch(c3c36a20,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26d3d38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26d3d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d3d6c, ebp = 0 ---

Tracing command idle: cpu1 pid 12 tid 100004 td 0xc3c36870
sched_switch(c3c36870,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26d0d38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26d0d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d0d6c, ebp = 0 ---

Tracing command idle: cpu2 pid 11 tid 100005 td 0xc3c366c0
sched_switch(c3c366c0,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26cdd38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26cdd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26cdd6c, ebp = 0 ---

Tracing command idle: cpu3 pid 10 tid 100006 td 0xc3c36510
sched_switch(c3c36510,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26cad38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26cad38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26cad6c, ebp = 0 ---

Tracing command init pid 1 tid 100007 td 0xc3c36360
sched_switch(c3c36360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c3a000) at sleepq_switch+0xcd
sleepq_wait_sig(c3c3a000) at sleepq_wait_sig+0x1d
msleep(c3c3a000,c3c3a068,15c,c0923f61,0) at msleep+0x26e
kern_wait(c3c36360,ffffffff,e26c7c2c,0,0) at kern_wait+0x7cf
wait4(c3c36360,e26c7d04) at wait4+0x2a
syscall(3b,3b,3b,0,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x8053047, esp = 0xbfbfed9c, ebp = 0xbfbfedb8 ---

Tracing command swapper pid 0 tid 0 td 0xc0a0c1d8
sched_switch(c0a0c1d8,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
scheduler(0,141ec00,141e000,0,c0451735,...) at scheduler+0x195
mi_startup() at mi_startup+0x96
begin() at begin+0x2c
db> call doadump
Physical memory: 1007 MB
Dumping 70 MB:KDB: stack backtrace:
kdb_backtrace(c0a0d950,0,c091f394,16c,c06bdd0a,...) at kdb_backtrace+0x29
mi_switch(1,0) at mi_switch+0xf4
turnstile_wait(c0a0bd34,c3c37a20,0,c0a0bd34,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a0bd34,c42a0bd0,0,c0917cf8,c6) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a0bd34,0,c0917cf8,c6,c065c000,...) at _mtx_lock_flags+0xae
g_disk_done(c3d92294) at g_disk_done+0x1e
biodone(c3d92294) at biodone+0x58
ad_done(c44e2480) at ad_done+0x2a
ata_completed(c44e2480,0) at ata_completed+0x4f0
ata_finish(c44e2480,c3d3a800,0,e67ae5a8,c04e808c,...) at ata_finish+0x34
ata_interrupt(c3d3a800) at ata_interrupt+0xf0
ata_start(c3d5be00,c3d3a908,0,c08ecdf9,5d) at ata_start+0x284
ata_queue_request(c4589300,c3e0c800,c3e12500,200,0,...) at ata_queue_request+0x21a
ad_strategy(e67ae5fc,e67ae5fc,84,2,0,...) at ad_strategy+0x161
ad_dump(c3e12500,c0a91960,0,9a07da00,0,200,c0945448,46,0,c094542f,3ef,0) at ad_dump+0xa4
minidumpsys(c0a0dc80,c0ac23e4,e67ae754,e67ae784,e67ae760,...) at minidumpsys+0x2ec
dumpsys(c0a0dc80,e67ae810,c047629b,c08797d3,c0956c40,...) at dumpsys+0x1a
doadump(c08797d3,c0956c40,e67ae804,c09f3d80,e67ae814,c0475d8b,c09f3d80,c0956c40,e67ae830,e67ae804) at doadump+0x55
db_fncall(e67ae858,0,c0a8e160,e67ae834,c0a8e160,...) at db_fncall+0x107
db_command(c09f3504,c0956ce0,c08e24db) at db_command+0x233
db_command_loop(0,0,e67ae8e4,e67ae8d0,e67ae918,...) at db_command_loop+0x52
db_trap(c,0,c42a0bd0,28,e67ae9f4,...) at db_trap+0xdd
kdb_trap(c,0,e67ae9f4) at kdb_trap+0x88
trap_fatal(e67ae9f4,c4479cf0) at trap_fatal+0x29d
trap(c4470008,28,28,0,e67aeaa8,...) at trap+0x11b
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc073e0c1, esp = 0xe67aea34, ebp = 0xe67aea38 ---
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---
db>
Bad function
db> call doadump
Physical memory: 1007 MB
Dumping 70 MB: 55 39 23 7
Dump complete
= 0xf
db> reset
cpu_reset: Restarting BSP
cpu_reset_proxy: Stopped CPU 3

(kgdb) bt full
#0  doadump () at pcpu.h:166
No locals.
#1  0xc047629b in db_fncall (dummy1=-428152744, dummy2=0, dummy3=-1062674080, dummy4=0xe67ae834 "") at ../../../ddb/db_command.c:484
        fn_addr = -1066831824
        args = {-1064855597, -1063949248, -428152828, -1063305856, -428152812, -1069064821, -1063305856, -1063949248, -428152784,
  -428152828}
        nargs = 0
        retval = 547938656
        t = 0
#2  0xc04760a7 in db_command (last_cmdp=0xc09f3504, cmd_table=0x0) at ../../../ddb/db_command.c:399
        cmd = (struct command *) 0xc0956c40
        t = 0
        modif = "\000b\000\000\000\000<\025\224\003\000\000xzF\000\000\000dz\003\000\000hz\026\207\003\000\000\003\000\000\r\000\000\000\220z\030\207xz\003\000\000\f\000\017\003x\000\000\000\000>\237F\000\000\000z\200G\221@~GF\000\000\000\000>\237uG\000>\237\2108\237"
        addr = -428152744
        count = -1062674080
        have_addr = 0
        result = 0
#3  0xc0476162 in db_command_loop () at ../../../ddb/db_command.c:451
No locals.
#4  0xc0477dad in db_trap (type=12, code=0) at ../../../ddb/db_main.c:228
        jb = {{_jb = {-428152604, -428152624, -428152552, 1, 70, -1069056698, 1, 7, 1, 70, -428152544, -1066677732}}}
        prev_jb = (void *) 0x0
        bkpt = 0
#5  0xc06b5ea0 in kdb_trap (type=12, code=0, tf=0xe67ae9f4) at ../../../kern/subr_kdb.c:502
        intr = 70
        did_stop_cpus = 1
        handled = -428152332
#6  0xc0899be5 in trap_fatal (frame=0xe67ae9f4, eva=3293027568) at ../../../i386/i386/trap.c:858
        code = 0
        type = 12
        ss = 40
        esp = 0
        softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27, ssd_dpl = 0, ssd_p = 1, ssd_xx = 0, ssd_xx1 = 0, ssd_def32 = 1,
  ssd_gran = 1}
        msg = 0x0
#7  0xc0899263 in trap (frame=
      {tf_fs = -1001979896, tf_es = 40, tf_ds = 40, tf_edi = 0, tf_esi = -428152152, tf_ebp = -428152264, tf_isp = -428152288, tf_ebx = -428152152, tf_edx = 33554467, tf_ecx = 4, tf_eax = -1001939728, tf_trapno = 12, tf_err = -1001939728, tf_eip = -1066147647, tf_cs = 32, tf_eflags = 66050, tf_esp = -1001939728, tf_ss = -428152124}) at ../../../i386/i386/trap.c:277
        td = (struct thread *) 0xc42a0bd0
        p = (struct proc *) 0xc45f8234
        i = 0
        ucode = 0
        type = 12
        code = 0
        addr = 1514
        eva = 3293027568
        ksi = {ksi_link = {tqe_next = 0x0, tqe_prev = 0xc08f95c4}, ksi_info = {si_signo = 1180, si_errno = -1009253376,
    si_code = -1009344492, si_pid = 0, si_uid = 3230635460, si_status = 1178, si_addr = 0xc3d80400, si_value = {sival_int = -1009253376,
      sival_ptr = 0xc3d80400}, _reason = {_fault = {_trapno = -428152376}, _timer = {_timerid = -428152376, _overrun = -1066350925},
      _mesgq = {_mqd = -428152376}, _poll = {_band = -428152376}, __spare__ = {__spare1__ = -428152376, __spare2__ = {-1066350925,
          -1009253376, 0, -428152344, -1066345660, -1009253376, 402785280}}}}, ksi_flags = 0, ksi_sigq = 0xc4479cd6}
#8  0xc088457a in calltrap () at ../../../i386/i386/exception.s:138
No locals.
#9  0xc073e0c1 in fnv_32_buf (buf=0xc4479cf0, len=3, hval=1224750873) at fnv_hash.h:27
        s = (const u_int8_t *) 0xc4479cf0 <Address 0xc4479cf0 out of bounds>
#10 0xc073def3 in ip_output (m=0xc3d97c00, opt=0x0, ro=0xe67aea6c, flags=0, imo=0x0, inp=0xc44c7348) at ../../../netinet/ip_output.c:589
        ip = (struct ip *) 0xc4479ce4
        ifp = (struct ifnet *) 0xc3d80400
        m0 = (struct mbuf *) 0xc4641c00
        hlen = 20
        mtu = 1500
        len = -428152112
        error = 0
        dst = (struct sockaddr_in *) 0xe67aea70
        ia = (struct in_ifaddr *) 0xc3fbac00
        sia = (struct in_ifaddr *) 0xc3fbac00
        isbroadcast = 0
        sw_csum = 1
        iproute = {ro_rt = 0xc4472438, ro_dst = {sa_len = 16 '\020', sa_family = 2 '\002',
    sa_data = "\000\000\001\002\000\000\000\000\000\000\000"}}
        odst = {s_addr = 33663168}
        __func__ = "ip_output"
#11 0xc075099d in udp_output (inp=0xc44c7348, m=0xc4479c00, addr=0x0, control=0x0, td=0xc42a0bd0) at ../../../netinet/udp_usrreq.c:925
        ui = (struct udpiphdr *) 0xc4479ce4
        len = 8192
        faddr = {s_addr = 33663168}
        laddr = {s_addr = 83994816}
        cm = (struct cmsghdr *) 0xc46f6c00
        sin = (struct sockaddr_in *) 0xe67aec64
        src = {sin_len = 88 'X', sin_family = 235 '', sin_port = 59002, sin_addr = {s_addr = 0}, sin_zero = "\000\210Z\000\000\000"}
        error = 0
        ipflags = 0
        fport = 2304
        lport = 1222
        unlock_udbinfo = 0
        __func__ = "udp_output"
#12 0xc0751093 in udp_send (so=0xc4195530, flags=0, m=0xc4495300, addr=0x0, control=0x0, td=0xc42a0bd0)
    at ../../../netinet/udp_usrreq.c:1107
        inp = (struct inpcb *) 0xc44c7348
#13 0xc06dbc71 in sosend_dgram (so=0xc4195530, addr=0x0, uio=0xe67aec64, top=0xc4495300, control=0x0, flags=0, td=0xc42a0bd0)
    at ../../../kern/uipc_socket.c:1064
        space = 1024
        resid = 0
        clen = 0
        error = 0
        dontroute = 0
        atomic = 1
#14 0xc06dc254 in sosend (so=0x2000023, addr=0x0, uio=0xe67aec64, top=0x0, control=0x0, flags=0, td=0xc42a0bd0)
    at ../../../kern/uipc_socket.c:1286
No locals.
#15 0xc06c864e in soo_write (fp=0xc4479cf0, uio=0xe67aec64, active_cred=0xc4527200, flags=0, td=0xc42a0bd0)
    at ../../../kern/sys_socket.c:118
        so = (struct socket *) 0xc4195530
        error = -428151708
#16 0xc06c3243 in dofilewrite (td=0xc42a0bd0, fd=3, fp=0xc479f240, auio=0xe67aec64, offset=Unhandled dwarf expression opcode 0x93
) at file.h:253
        cnt = 8192
        error = -998641088
        ktruio = (struct uio *) 0x0
#17 0xc06c30e6 in kern_writev (td=0xc42a0bd0, fd=3, auio=0xe67aec64) at ../../../kern/sys_generic.c:402
        fp = (struct file *) 0xc479f240
        error = 0
#18 0xc06c3011 in write (td=0xc42a0bd0, uap=0x4) at ../../../kern/sys_generic.c:326
        auio = {uio_iov = 0xe67aec5c, uio_iovcnt = 1, uio_offset = 8191, uio_resid = 0, uio_segflg = UIO_USERSPACE, uio_rw = UIO_WRITE,
  uio_td = 0xc42a0bd0}
        aiov = {iov_base = 0x8052000, iov_len = 0}
#19 0xc0899f02 in syscall (frame=
      {tf_fs = 671416379, tf_es = -1078001605, tf_ds = -1078001605, tf_edi = 671422604, tf_esi = -1077941636, tf_ebp = -1077941944, tf_isp = -428151452, tf_ebx = 1, tf_edx = 0, tf_ecx = 0, tf_eax = 4, tf_trapno = 59, tf_err = 2, tf_eip = 672420655, tf_cs = 51, tf_eflags = 662, tf_esp = -1077942020, tf_ss = 59}) at ../../../i386/i386/trap.c:1006
        params = 0xbfbfe900 <Address 0xbfbfe900 out of bounds>
        callp = (struct sysent *) 0xc09c21a0
        td = (struct thread *) 0xc42a0bd0
        p = (struct proc *) 0xc45f8234
        orig_tf_eflags = 662
        error = 0
        narg = 3
        args = {3, 134545408, 8192, -1063200432, 2, -1064172069, 625, 1}
        code = 4
        ksi = {ksi_link = {tqe_next = 0x2, tqe_prev = 0xc09205db}, ksi_info = {si_signo = 625, si_errno = -1003877424, si_code = 1,
    si_pid = 1, si_uid = 1, si_status = -1003877424, si_addr = 0xe67aed2c, si_value = {sival_int = -1066672929, sival_ptr = 0xc06bdcdf},
    _reason = {_fault = {_trapno = -1063200408}, _timer = {_timerid = -1063200408, _overrun = 2}, _mesgq = {_mqd = -1063200408},
      _poll = {_band = -1063200408}, __spare__ = {__spare1__ = -1063200408, __spare2__ = {2, -1064164088, 276, -1003877424, -428151496,
          -1066865075, -1007021824}}}}, ksi_flags = -1003877424, ksi_sigq = 0xbfbfea7c}