GENERIC HEAD from Oct 9 15:08 UTC, vmcore.95
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 7.0-CURRENT #1: Fri Oct 13 16:38:45 CEST 2006
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1032458240 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
fxp0: link state changed to UP
Kernel page fault with the following non-sleepable locks held:
exclusive sleep mutex inp (udpinp) r = 0 (0xc44c73d8) locked @ netinet/udp_usrreq.c:804
KDB: stack backtrace:
kdb_backtrace(1,c45f8234,c,c42a0bd0,e67ae9ec,...) at kdb_backtrace+0x29
witness_warn(5,0,c0946b7b) at witness_warn+0x192
trap(c4470008,28,28,0,e67aeaa8,...) at trap+0x108
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc073e0c1, esp = 0xe67aea34, ebp = 0xe67aea38 ---
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---
Fatal trap 12: page fault while in kernel mode
cpuid = 3; apic id = 07
fault virtual address = 0xc4479cf0
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc073e0c1
stack pointer = 0x28:0xe67aea34
frame pointer = 0x28:0xe67aea38
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 1606 (udp)
[thread pid 1606 tid 100155 ]
Stopped at fnv_32_buf+0x39: movzbl 0(%eax),%edx
db> where
Tracing pid 1606 tid 100155 td 0xc42a0bd0
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---
db> show allpcpu
Current CPU: 3
cpuid = 0
curthread = 0xc3c36d80: pid 15 "swi4: clock sio"
curpcb = 0xe26d9d90
fpcurthread = none
idlethread = 0xc3c36a20: pid 13 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc429fd80: pid 1616 "swap"
curpcb = 0xe6796d90
fpcurthread = none
idlethread = 0xc3c36870: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc3c3b870: pid 35 "pagedaemon"
curpcb = 0xe2706d90
fpcurthread = none
idlethread = 0xc3c366c0: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc42a0bd0: pid 1606 "udp"
curpcb = 0xe67aed90
fpcurthread = none
idlethread = 0xc3c36510: pid 10 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> set $lines 0
db> show alllocks
Process 1645 (sendmail) thread 0xc419b000 (100141)
exclusive sx user map r = 0 (0xc445b148) locked @ vm/vm_map.c:3073
Process 1639 (swap) thread 0xc3ff2bd0 (100060)
exclusive sx user map r = 0 (0xc406e968) locked @ vm/vm_fault.c:797
Process 1636 (swap) thread 0xc45f6000 (100144)
exclusive sleep mutex vm object (standard object) r = 0 (0xc4681a50) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc4405e7c) locked @ vm/vm_map.c:3073
Process 1627 (swap) thread 0xc419bd80 (100119)
exclusive sx user map r = 0 (0xc3c3c968) locked @ vm/vm_fault.c:797
Process 1625 (swap) thread 0xc429fa20 (100176)
exclusive sleep mutex vm object (standard object) r = 0 (0xc3db7618) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc411865c) locked @ vm/vm_map.c:3073
Process 1619 (swap) thread 0xc3ff2360 (100100)
exclusive sleep mutex vm object (standard object) r = 0 (0xc444d348) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc4186b70) locked @ vm/vm_map.c:3073
Process 1616 (swap) thread 0xc429fd80 (100139)
exclusive sleep mutex vm object (standard object) r = 0 (0xc4681870) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc3c3dc74) locked @ vm/vm_map.c:3073
Process 1613 (swap) thread 0xc4055bd0 (100078)
exclusive sleep mutex vm object (standard object) r = 0 (0xc4123960) locked @ vm/vm_fault.c:293
exclusive sx user map r = 0 (0xc445b454) locked @ vm/vm_map.c:3073
Process 1612 (swap) thread 0xc405b870 (100097)
exclusive sleep mutex vm object (standard object) r = 0 (0xc45f3ca8) locked @ vm/vm_fault.c:978
exclusive sx user map r = 0 (0xc3ff5e7c) locked @ vm/vm_fault.c:797
Process 1611 (swap) thread 0xc419b1b0 (100140)
exclusive sx user map r = 0 (0xc45ec148) locked @ vm/vm_fault.c:797
Process 1606 (udp) thread 0xc42a0bd0 (100155)
exclusive sleep mutex inp (udpinp) r = 0 (0xc44c73d8) locked @ netinet/udp_usrreq.c:804
Process 35 (pagedaemon) thread 0xc3c3b870 (100030)
exclusive sleep mutex pmap r = 0 (0xc0a956e0) locked @ i386/i386/pmap.c:2058
exclusive sleep mutex vm page queue mutex r = 0 (0xc0a6839c) locked @ i386/i386/pmap.c:2056
exclusive sleep mutex system map r = 0 (0xc1871120) locked @ vm/vm_map.c:2389
exclusive sleep mutex UMA lock r = 0 (0xc0a6800c) locked @ vm/uma_core.c:1490
exclusive sleep mutex Giant r = 0 (0xc0a0d968) locked @ vm/vm_pageout.c:687
db> ps
pid ppid pgrp uid state wmesg wchan cmd
1650 774 774 25 SL vmpfw 0xc29e7a78 sendmail
1645 768 768 0 RL sendmail
1641 1607 1034 1001 R+ swap
1640 1607 1034 1001 R+ swap
1639 1607 1034 1001 RL+ swap
1638 1607 1034 1001 R+ swap
1637 1607 1034 1001 R+ swap
1636 1607 1034 1001 RL+ swap
1635 1607 1034 1001 R+ swap
1634 1607 1034 1001 R+ swap
1633 1607 1034 1001 SL+ swread 0xc297dd58 swap
1632 1607 1034 1001 R+ swap
1631 1607 1034 1001 R+ swap
1630 1607 1034 1001 SL+ swread 0xc2923578 swap
1629 1607 1034 1001 R+ swap
1628 1607 1034 1001 R+ swap
1627 1607 1034 1001 RL+ swap
1626 1607 1034 1001 R+ swap
1625 1607 1034 1001 RL+ swap
1624 1607 1034 1001 R+ swap
1623 1607 1034 1001 R+ swap
1622 1607 1034 1001 R+ swap
1621 1607 1034 1001 R+ swap
1620 1607 1034 1001 R+ swap
1619 1607 1034 1001 RL+ swap
1618 1607 1034 1001 R+ swap
1617 1607 1034 1001 R+ swap
1616 1607 1034 1001 RL+ CPU 1 swap
1615 1607 1034 1001 R+ swap
1614 1607 1034 1001 R+ swap
1613 1607 1034 1001 RL+ swap
1612 1607 1034 1001 RL+ swap
1611 1607 1034 1001 RL+ swap
1610 1607 1034 1001 R+ swap
1609 1607 1034 1001 R+ swap
1608 1607 1034 1001 R+ swap
1607 1602 1034 1001 SW+ wait 0xc4111000 swap
1606 1604 1034 1001 R+ CPU 3 udp
1605 1604 1034 1001 R+ udp
1604 1603 1034 1001 SW+ wait 0xc40588d0 udp
1603 1038 1034 1001 S+ nanslp 0xc0a0e164 udp
1602 1038 1034 1001 S+ nanslp 0xc0a0e164 swap
1392 1391 1392 1001 SWs+ ttyin 0xc3d61010 bash
1391 1386 1386 1001 SW select 0xc0a5ac5c sshd
1386 762 1386 0 SWs sbwait 0xc479b1e8 sshd
1085 1079 1085 1001 S+ select 0xc0a5ac5c top
1079 1078 1079 1001 SWs+ wait 0xc45f7234 bash
1078 1063 1063 1001 S select 0xc0a5ac5c sshd
1063 762 1063 0 SWs sbwait 0xc41479b0 sshd
1038 1037 1034 1001 SW+ wait 0xc4058000 run
1037 1036 1034 1001 SW+ wait 0xc3f81b04 run
1036 1034 1034 1001 S+ nanslp 0xc0a0e164 run
1034 1027 1034 1001 SW+ wait 0xc4033b04 sh
1027 1026 1027 1001 SWs+ wait 0xc403bb04 bash
1026 1024 1024 1001 SW select 0xc0a5ac5c sshd
1024 762 1024 0 SWs sbwait 0xc411bd94 sshd
891 1 891 0 SWs+ ttyin 0xc3dfe810 getty
890 1 890 0 SWs+ ttyin 0xc3e02810 getty
889 1 889 0 SWs+ ttyin 0xc3e01810 getty
888 1 888 0 SWs+ ttyin 0xc3e01010 getty
887 1 887 0 SWs+ ttyin 0xc3e00c10 getty
886 1 886 0 SWs+ ttyin 0xc3e02c10 getty
885 1 885 0 SWs+ ttyin 0xc3e03010 getty
884 1 884 0 SWs+ ttyin 0xc3e01c10 getty
870 1 870 0 SWs select 0xc0a5ac5c inetd
843 1 843 0 SWs select 0xc0a5ac5c moused
828 1 828 0 Ss nanslp 0xc0a0e164 watchdogd
780 1 780 0 Ss nanslp 0xc0a0e164 cron
774 1 774 25 SWs pause 0xc3de96d0 sendmail
768 1 768 0 Ss select 0xc0a5ac5c sendmail
762 1 762 0 SWs select 0xc0a5ac5c sshd
743 1 743 0 Ss select 0xc0a5ac5c ntpd
689 685 685 0 SW - 0xc40dae00 nfsd
688 685 685 0 SW - 0xc3f72600 nfsd
687 685 685 0 SW - 0xc40db000 nfsd
686 685 685 0 SW - 0xc3fbf400 nfsd
685 1 685 0 SWs select 0xc0a5ac5c nfsd
683 1 683 0 SWs select 0xc0a5ac5c mountd
633 1 633 0 SWs select 0xc0a5ac5c rpcbind
614 1 614 0 SWs select 0xc0a5ac5c syslogd
547 1 547 0 SWs select 0xc0a5ac5c devd
42 0 0 0 SL - 0xe43f4cfc [schedcpu]
41 0 0 0 SL sdflush 0xc0a67ba0 [softdepflush]
40 0 0 0 SL vlruwt 0xc3f7fd38 [vnlru]
39 0 0 0 SL syncer 0xc0a0df30 [syncer]
38 0 0 0 SL psleep 0xc0a5b0d4 [bufdaemon]
37 0 0 0 RL [pagezero]
36 0 0 0 SL psleep 0xc0a68400 [vmdaemon]
35 0 0 0 RL CPU 2 [pagedaemon]
34 0 0 0 WL [irq7: ppc0]
33 0 0 0 SL - 0xc3d7e03c [fdc0]
32 0 0 0 WL [swi0: sio]
31 0 0 0 WL [irq12: psm0]
30 0 0 0 WL [irq1: atkbd0]
29 0 0 0 WL [irq15: ata1]
28 0 0 0 WL [irq14: ata0]
27 0 0 0 WL [irq17: fxp0]
26 0 0 0 SL usbtsk 0xc0a0b6e4 [usbtask]
25 0 0 0 SL usbevt 0xc3d6d210 [usb0]
24 0 0 0 WL [irq16: uhci0]
23 0 0 0 SL - 0xc3d5a280 [em0 taskq]
22 0 0 0 WL [irq9: acpi0]
21 0 0 0 WL [swi5: +]
9 0 0 0 SL - 0xc3c85700 [thread taskq]
20 0 0 0 WL [swi6: Giant taskq]
8 0 0 0 SL - 0xc3c85880 [acpi_task_2]
7 0 0 0 SL - 0xc3c85880 [acpi_task_1]
6 0 0 0 SL - 0xc3c85880 [acpi_task_0]
19 0 0 0 WL [swi6: task queue]
5 0 0 0 SL - 0xc3c85a00 [kqueue taskq]
18 0 0 0 WL [swi2: cambio]
17 0 0 0 SL - 0xc0a09320 [yarrow]
4 0 0 0 SL - 0xc0a0be9c [g_down]
3 0 0 0 RL [g_up]
2 0 0 0 SL - 0xc0a0be90 [g_event]
16 0 0 0 WL [swi3: vm]
15 0 0 0 RL CPU 0 [swi4: clock sio]
14 0 0 0 WL [swi1: net]
13 0 0 0 RL [idle: cpu0]
12 0 0 0 RL [idle: cpu1]
11 0 0 0 RL [idle: cpu2]
10 0 0 0 RL [idle: cpu3]
1 0 1 0 SLs wait 0xc3c3a000 [init]
0 0 0 0 WLs [swapper]
db> allt
Tracing command sendmail pid 1650 tid 100085 td 0xc4055000
sched_switch(c4055000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c29e7a78,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c29e7a78,0,c093a45d,185,c093b7e7,...) at sleepq_wait+0x46
msleep(c29e7a78,c4088ca8,44,c093a500,0,...) at msleep+0x27d
vm_page_sleep(c29e7a78,c093a500) at vm_page_sleep+0x80
vm_fault(c404c410,808e000,1,0,c4055000,...) at vm_fault+0x4b7
trap_pfault(e6676d38,1,808edf8,808edf8,808edf8,...) at trap_pfault+0xee
trap(c088003b,3b,3b,0,0,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x808edf8, esp = 0xbfbfcb6c, ebp = 0xbfbfcfb8 ---
Tracing command sendmail pid 1645 tid 100141 td 0xc419b000
sched_switch(c419b000,c3de5870,6) at sched_switch+0x183
mi_switch(6,c3de5870,c3de5a00,c3de5870,e6748838,...) at mi_switch+0x280
maybe_preempt(c3de5870) at maybe_preempt+0x199
sched_add(c3de5870,4,c3c83b00,c3de5870,c3de3460,...) at sched_add+0x15a
setrunqueue(c3de5870,4) at setrunqueue+0xab
intr_event_schedule_thread(c3c83b00) at intr_event_schedule_thread+0xbb
intr_execute_handlers(c3c2c22c,e67488bc,e,e6748984,c0884933,...) at intr_execute_handlers+0x118
lapic_handle_intr(34) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc08a212c, esp = 0xe67488fc, ebp = 0xe6748984 ---
__qdivrem(800,0,800,0,0) at __qdivrem+0xcc
__divdi3(800,0,800,0) at __divdi3+0x5b
ufs_getlbns(c4188150,5,0,e6748a6c,e6748a58) at ufs_getlbns+0x1d7
ufs_bmaparray(c4188150,11,0,e6748acc,0,...) at ufs_bmaparray+0x8b
ufs_bmap(e6748b20) at ufs_bmap+0x4b
VOP_BMAP_APV(c09dca20,e6748b20) at VOP_BMAP_APV+0xa9
vnode_pager_haspage(c4088ca8,46,0,e6748b84,e6748b80) at vnode_pager_haspage+0x174
vm_fault_additional_pages(c29e7a78,7,8,e6748c30,e6748bf4,...) at vm_fault_additional_pages+0x9d
vm_fault(c445b104,808e000,1,0,c419b000,...) at vm_fault+0x883
trap_pfault(e6748d38,1,808edf8,808edf8,808edf8,...) at trap_pfault+0xee
trap(823003b,80d003b,bfbf003b,0,0,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x808edf8, esp = 0xbfbfbf5c, ebp = 0xbfbfc3a8 ---
Tracing command swap pid 1641 tid 100122 td 0xc419b870
sched_switch(c419b870,0,2) at sched_switch+0x183
mi_switch(2,0,c419b870,d4,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6757d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1640 tid 100169 td 0xc3d9e1b0
sched_switch(c3d9e1b0,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9e1b0,d4,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e687dd38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1639 tid 100060 td 0xc3ff2bd0
sched_switch(c3ff2bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c4037000,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c3ff2bd0,0,c0945f66,915) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c0945f66,915,c3c12440,...) at _mtx_lock_flags+0xae
pmap_enter(c406e9cc,9b19000,c2538948,7,0,c3db80f0,0,c093a454,378) at pmap_enter+0x38
vm_fault(c406e924,9b19000,2,8,c3ff2bd0,...) at vm_fault+0x1040
trap_pfault(e6601d38,1,9b19000,9b19000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1638 tid 100129 td 0xc4112a20
sched_switch(c4112a20,0,2) at sched_switch+0x183
mi_switch(2,0,c4112a20,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66e8d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1637 tid 100185 td 0xc419ca20
sched_switch(c419ca20,0,2) at sched_switch+0x183
mi_switch(2,0,c419ca20,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6775d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1636 tid 100144 td 0xc45f6000
sched_switch(c45f6000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c45f6000,0,c093a454,1f2) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093a454,1f2,1ec2,...) at _mtx_lock_flags+0xae
vm_fault(c4405e38,9f13000,2,8,c45f6000,...) at vm_fault+0x77e
trap_pfault(e6829d38,1,9f13000,9f13000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1635 tid 100145 td 0xc45f2d80
sched_switch(c45f2d80,0,2) at sched_switch+0x183
mi_switch(2,0,c45f2d80,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6826d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1634 tid 100131 td 0xc41126c0
sched_switch(c41126c0,0,2) at sched_switch+0x183
mi_switch(2,0,c41126c0,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66e2d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1633 tid 100063 td 0xc4037000
sched_switch(c4037000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c297dd58,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c297dd58) at sleepq_timedwait+0x4a
msleep(c297dd58,c467f078,40,c09397fe,4e20) at msleep+0x259
swap_pager_getpages(c467f078,e664cc30,1,0) at swap_pager_getpages+0x348
vm_fault(c406e000,82e9000,2,8,c4037000,...) at vm_fault+0x928
trap_pfault(e664cd38,1,82e9000,82e9000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1632 tid 100073 td 0xc3ff3d80
sched_switch(c3ff3d80,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff3d80,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e661fd38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1631 tid 100087 td 0xc4037bd0
sched_switch(c4037bd0,0,2) at sched_switch+0x183
mi_switch(2,0,c4037bd0,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6661d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1630 tid 100102 td 0xc3ff2000
sched_switch(c3ff2000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c2923578,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c2923578) at sleepq_timedwait+0x4a
msleep(c2923578,c4613c30,40,c09397fe,4e20) at msleep+0x259
swap_pager_getpages(c4613c30,e65ecc30,2,0) at swap_pager_getpages+0x348
vm_fault(c445b618,8155000,2,8,c3ff2000,...) at vm_fault+0x928
trap_pfault(e65ecd38,1,8155000,8155000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1629 tid 100079 td 0xc4055a20
sched_switch(c4055a20,0,2) at sched_switch+0x183
mi_switch(2,0,c4055a20,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6688d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1628 tid 100174 td 0xc3d9c870
sched_switch(c3d9c870,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9c870,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e686ed38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1627 tid 100119 td 0xc419bd80
sched_switch(c419bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c419bd80,0,c0945f66,915) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c0945f66,915,c3c12440,...) at _mtx_lock_flags+0xae
pmap_enter(c3c3c9cc,9331000,c2065398,7,0,c467f168,0,c093a454,378) at pmap_enter+0x38
vm_fault(c3c3c924,9331000,2,8,c419bd80,...) at vm_fault+0x1040
trap_pfault(e6760d38,1,9331000,9331000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1626 tid 100162 td 0xc3de8510
sched_switch(c3de8510,0,2) at sched_switch+0x183
mi_switch(2,0,c3de8510,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e43b6d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1625 tid 100176 td 0xc429fa20
sched_switch(c429fa20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c4037000,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c429fa20,0,c093a454,34b) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093a454,34b,2,...) at _mtx_lock_flags+0xae
vm_fault(c4118618,9341000,2,8,c429fa20,...) at vm_fault+0xf3f
trap_pfault(e6790d38,1,9341000,9341000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1624 tid 100135 td 0xc4112000
sched_switch(c4112000,0,2) at sched_switch+0x183
mi_switch(2,0,c4112000,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66d6d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1623 tid 100113 td 0xc3ff3510
sched_switch(c3ff3510,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff3510,d0,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6610d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1622 tid 100172 td 0xc3d9cbd0
sched_switch(c3d9cbd0,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9cbd0,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6874d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1621 tid 100179 td 0xc429f510
sched_switch(c429f510,0,2) at sched_switch+0x183
mi_switch(2,0,c429f510,d0,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6787d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1620 tid 100168 td 0xc3d9e360
sched_switch(c3d9e360,0,2) at sched_switch+0x183
mi_switch(2,0,c3d9e360,d3,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6880d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1619 tid 100100 td 0xc3ff2360
sched_switch(c3ff2360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c3ff2360,0,c093b7e7,368) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093b7e7,368,c0a68384,0,c093b7e7,362) at _mtx_lock_flags+0xae
vm_page_alloc(c444d348,1602,0,40,2,...) at vm_page_alloc+0x131
vm_fault(c4186b2c,9652000,2,8,c3ff2360,...) at vm_fault+0x60a
trap_pfault(e65f2d38,1,9652000,9652000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1618 tid 100183 td 0xc419cd80
sched_switch(c419cd80,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0d950,0,c09205db,...) at mi_switch+0x280
critical_exit(1,bfbfe948,c0884b4e,2805003b,3b,...) at critical_exit+0x92
lapic_handle_timer(2805003b,3b,bfbf003b,2805188c,bfbfea7c,...) at lapic_handle_timer+0x149
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0x8049183, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1617 tid 100057 td 0xc3ff31b0
sched_switch(c3ff31b0,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff31b0,d4,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e660ad38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1616 tid 100139 td 0xc429fd80
cpustop_handler(e6796ae4,c0899182,c429fd80,d5,d46abd29,...) at cpustop_handler+0x31
ipi_nmi_handler(c429fd80,d5,d46abd29,0,0,...) at ipi_nmi_handler+0x28
trap(c4290008,e6790028,28,c429fd80,c3c3b870,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068f052, esp = 0xe6796b2c, ebp = 0xe6796b3c ---
_mtx_lock_sleep(c0a6839c,c429fd80,0,c093b7e7,368) at _mtx_lock_sleep+0x112
_mtx_lock_flags(c0a6839c,0,c093b7e7,368,c0a68384,0,c093b7e7,362) at _mtx_lock_flags+0xae
vm_page_alloc(c4681870,12b2,0,40,c06ad59b,...) at vm_page_alloc+0x131
vm_fault(c3c3dc30,9302000,2,8,c429fd80,...) at vm_fault+0x60a
trap_pfault(e6796d38,1,9302000,9302000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1615 tid 100112 td 0xc3ff36c0
sched_switch(c3ff36c0,0,2) at sched_switch+0x183
mi_switch(2,0,c3ff36c0,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6613d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1614 tid 100072 td 0xc4034000
sched_switch(c4034000,0,2) at sched_switch+0x183
mi_switch(2,0,c4034000,d8,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e6631d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1613 tid 100078 td 0xc4055bd0
sched_switch(c4055bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c4055bd0,0,c093b7e7,368) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093b7e7,368,c0a68384,0,c093b7e7,362) at _mtx_lock_flags+0xae
vm_page_alloc(c4123960,1703,0,40,c4055bd0,...) at vm_page_alloc+0x131
vm_fault(c445b410,9753000,2,8,c4055bd0,...) at vm_fault+0x60a
trap_pfault(e668bd38,1,9753000,9753000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1612 tid 100097 td 0xc405b870
sched_switch(c405b870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3ff2360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c405b870,0,c093a454,3e8) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c093a454,3e8,c45f3ca8,...) at _mtx_lock_flags+0xae
vm_fault_prefault(c3ff5ee0,991d000,c46b750c) at vm_fault_prefault+0x1c0
vm_fault(c3ff5e38,991d000,2,8,c405b870,...) at vm_fault+0x1063
trap_pfault(e66bbd38,1,991d000,991d000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1611 tid 100140 td 0xc419b1b0
sched_switch(c419b1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a6839c,c3de8360,0,c0a6839c,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a6839c,c419b1b0,0,c0945f66,915) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a6839c,0,c0945f66,915,c3c12440,...) at _mtx_lock_flags+0xae
pmap_enter(c45ec1ac,9da1000,c24cbaa8,7,0,c46b8438,0,c093a454,378) at pmap_enter+0x38
vm_fault(c45ec104,9da1000,2,8,c419b1b0,...) at vm_fault+0x1040
trap_pfault(e674bd38,1,9da1000,9da1000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea7c,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe930, ebp = 0xbfbfe948 ---
Tracing command swap pid 1610 tid 100163 td 0xc3de8360
sched_switch(c3de8360,0,2) at sched_switch+0x183
mi_switch(2,0,c3de8360,ca,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e43b3d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1609 tid 100094 td 0xc405bd80
sched_switch(c405bd80,0,2) at sched_switch+0x183
mi_switch(2,0,c405bd80,d1,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e66c4d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1608 tid 100153 td 0xc45f2000
sched_switch(c45f2000,0,2) at sched_switch+0x183
mi_switch(2,0,c45f2000,d2,c0a0d950,0,c0922508,101) at mi_switch+0x280
ast(e680ed38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 1607 tid 100105 td 0xc3f80a20
Tracing command udp pid 1606 tid 100155 td 0xc42a0bd0
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---
Tracing command udp pid 1605 tid 100111 td 0xc41131b0
sched_switch(c41131b0,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0d950,0,c09205db,...) at mi_switch+0x280
critical_exit(c0a0d950,e66f4a64,c068ee4d,c3c36bd0,e66f4a90,...) at critical_exit+0x92
spinlock_exit(c3c36bd0,e66f4a90,c06bf249,c0a0d950,0,...) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c0a0d950,0,c0922599,371,0,...) at _mtx_unlock_spin_flags+0xb1
turnstile_unpend(c3e06100,0,c411e678,39f,e66f4acc,...) at turnstile_unpend+0x371
_mtx_unlock_sleep(c411e678,0,c08c9b31,39f) at _mtx_unlock_sleep+0x6b
_mtx_unlock_flags(c411e678,0,c08c9b31,39f) at _mtx_unlock_flags+0xcc
udp_output(c411e5e8,c465ae00,0,0,c41131b0) at udp_output+0x669
udp_send(c4609530,0,c465ae00,0,0,...) at udp_send+0x33
sosend_dgram(c4609530,0,e66f4c64,c465ae00,0,...) at sosend_dgram+0x30d
sosend(c4609530,0,e66f4c64,0,0,0,c41131b0) at sosend+0x3c
soo_write(c4004c60,e66f4c64,c4527200,0,c41131b0) at soo_write+0x42
dofilewrite(c41131b0,3,c4004c60,e66f4c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c41131b0,3,e66f4c64,8050040,0,...) at kern_writev+0x36
write(c41131b0,e66f4d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---
Tracing command udp pid 1604 tid 100088 td 0xc4037a20
Tracing command udp pid 1603 tid 100052 td 0xc3de8d80
sched_switch(c3de8d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,3e9,...) at msleep+0x244
kern_nanosleep(c3de8d80,e43c5c74,e43c5c6c) at kern_nanosleep+0xab
nanosleep(c3de8d80,e43c5d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe93c, ebp = 0xbfbfe968 ---
Tracing command swap pid 1602 tid 100187 td 0xc4616510
sched_switch(c4616510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,3e9,...) at msleep+0x244
kern_nanosleep(c4616510,e68b9c74,e68b9c6c) at kern_nanosleep+0xab
nanosleep(c4616510,e68b9d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe93c, ebp = 0xbfbfe968 ---
Tracing command bash pid 1392 tid 100156 td 0xc42a0a20
Tracing command sshd pid 1391 tid 100164 td 0xc3de81b0
Tracing command sshd pid 1386 tid 100127 td 0xc4112d80
Tracing command top pid 1085 tid 100067 td 0xc4034870
sched_switch(c4034870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a5ac5c,c0a5ac5c,3e9,e6640ae8,246,...) at sleepq_timedwait_sig+0x1e
cv_timedwait_sig(c0a5ac5c,c0a5ac44,3e9) at cv_timedwait_sig+0x196
kern_select(c4034870,2,bfbfeaf0,0,0,...) at kern_select+0x4a9
select(c4034870,e6640d04) at select+0x44
syscall(3b,bfbf003b,bfbf003b,bfbfeaf0,1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x281b316b, esp = 0xbfbfea6c, ebp = 0xbfbfebe8 ---
Tracing command bash pid 1079 tid 100148 td 0xc45f2870
Tracing command sshd pid 1078 tid 100039 td 0xc3de5510
sched_switch(c3de5510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_wait_sig(c0a5ac5c,246,c0a0d950,c09239cd,307,...) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a5ac5c,c0a5ac44) at cv_wait_sig+0x18d
kern_select(c3de5510,b,8201168,820116c,0,...) at kern_select+0x4bf
select(c3de5510,e439bd04) at select+0x44
syscall(bfbf003b,bfbf003b,bfbf003b,0,bfbfe13c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x2838b16b, esp = 0xbfbfe0cc, ebp = 0xbfbfe118 ---
Tracing command sshd pid 1063 tid 100053 td 0xc3de8bd0
Tracing command run pid 1038 tid 100084 td 0xc40551b0
Tracing command run pid 1037 tid 100054 td 0xc3de8a20
Tracing command run pid 1036 tid 100081 td 0xc40556c0
sched_switch(c40556c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,3e9,...) at msleep+0x244
kern_nanosleep(c40556c0,e6682c74,e6682c6c) at kern_nanosleep+0xab
nanosleep(c40556c0,e6682d04) at nanosleep+0x4f
syscall(2805003b,3b,bfbf003b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe8fc, ebp = 0xbfbfe928 ---
Tracing command sh pid 1034 tid 100068 td 0xc40346c0
Tracing command bash pid 1027 tid 100075 td 0xc3ff3a20
Tracing command sshd pid 1026 tid 100118 td 0xc419c000
Tracing command sshd pid 1024 tid 100056 td 0xc3ff3360
Tracing command getty pid 891 tid 100089 td 0xc4037870
Tracing command getty pid 890 tid 100099 td 0xc3ff2510
Tracing command getty pid 889 tid 100137 td 0xc42a01b0
Tracing command getty pid 888 tid 100108 td 0xc41136c0
Tracing command getty pid 887 tid 100062 td 0xc3ff2870
Tracing command getty pid 886 tid 100124 td 0xc419b510
Tracing command getty pid 885 tid 100125 td 0xc419b360
Tracing command getty pid 884 tid 100126 td 0xc4113000
Tracing command inetd pid 870 tid 100128 td 0xc4112bd0
Tracing command moused pid 843 tid 100132 td 0xc4112510
Tracing command watchdogd pid 828 tid 100136 td 0xc42a0360
sched_switch(c42a0360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,ea61,...) at msleep+0x244
kern_nanosleep(c42a0360,e679fc74,e679fc6c) at kern_nanosleep+0xab
nanosleep(c42a0360,e679fd04) at nanosleep+0x4f
syscall(3b,3b,bfbf003b,1,3c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28157beb, esp = 0xbfbfed3c, ebp = 0xbfbfed68 ---
Tracing command cron pid 780 tid 100042 td 0xc3de5000
sched_switch(c3de5000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0e164) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0e164) at sleepq_timedwait_sig+0x1e
msleep(c0a0e164,0,15c,c091fbac,88b9,...) at msleep+0x244
kern_nanosleep(c3de5000,e4392c74,e4392c6c) at kern_nanosleep+0xab
nanosleep(c3de5000,e4392d04) at nanosleep+0x4f
syscall(3b,3b,3b,805008c,23,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28147beb, esp = 0xbfbfecec, ebp = 0xbfbfed18 ---
Tracing command sendmail pid 774 tid 100038 td 0xc3de56c0
Tracing command sendmail pid 768 tid 100051 td 0xc3f80000
sched_switch(c3f80000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a5ac5c,c0a5ac5c,1389,e43e5ae8,246,...) at sleepq_timedwait_sig+0x1e
cv_timedwait_sig(c0a5ac5c,c0a5ac44,1389) at cv_timedwait_sig+0x196
kern_select(c3f80000,5,bfbfc520,0,0,...) at kern_select+0x4a9
select(c3f80000,e43e5d04) at select+0x44
syscall(bfbf003b,3b,bfbf003b,1,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x2834a16b, esp = 0xbfbfc42c, ebp = 0xbfbfd028 ---
Tracing command sshd pid 762 tid 100080 td 0xc4055870
Tracing command ntpd pid 743 tid 100107 td 0xc4113870
sched_switch(c4113870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5ac5c) at sleepq_switch+0xcd
sleepq_wait_sig(c0a5ac5c,246,c0a0d950,c09239cd,307,...) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a5ac5c,c0a5ac44) at cv_wait_sig+0x18d
kern_select(c4113870,9,bfbfed70,0,0,...) at kern_select+0x4bf
select(c4113870,e6700d04) at select+0x44
syscall(bfbf003b,3b,bfbf003b,bfbfed70,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x282cf16b, esp = 0xbfbfed2c, ebp = 0xbfbfee08 ---
Tracing command nfsd pid 689 tid 100059 td 0xc3ff2d80
Tracing command nfsd pid 688 tid 100065 td 0xc4034bd0
Tracing command nfsd pid 687 tid 100106 td 0xc4113a20
Tracing command nfsd pid 686 tid 100074 td 0xc3ff3bd0
Tracing command nfsd pid 685 tid 100083 td 0xc4055360
Tracing command mountd pid 683 tid 100050 td 0xc3f801b0
Tracing command rpcbind pid 633 tid 100055 td 0xc3de8870
Tracing command syslogd pid 614 tid 100066 td 0xc4034a20
Tracing command devd pid 547 tid 100058 td 0xc3ff3000
Tracing command schedcpu pid 42 tid 100046 td 0xc3f80870
sched_switch(c3f80870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(e43f4cfc,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(e43f4cfc) at sleepq_timedwait+0x4a
msleep(e43f4cfc,0,0,c0917342,3e8) at msleep+0x259
schedcpu_thread(0,e43f4d38) at schedcpu_thread+0x26
fork_exit(c06abf90,0,e43f4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43f4d6c, ebp = 0 ---
Tracing command softdepflush pid 41 tid 100047 td 0xc3f806c0
sched_switch(c3f806c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a67ba0,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a67ba0) at sleepq_timedwait+0x4a
msleep(c0a67ba0,c0a67b78,44,c093737f,3e8) at msleep+0x259
softdep_flush(0,e43f1d38) at softdep_flush+0x203
fork_exit(c07cf1f4,0,e43f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43f1d6c, ebp = 0 ---
Tracing command vnlru pid 40 tid 100048 td 0xc3f80510
sched_switch(c3f80510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3f7fd38,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3f7fd38) at sleepq_timedwait+0x4a
msleep(c3f7fd38,c0a5b398,250,c0928f82,3e8,c0a5b3f8) at msleep+0x259
vnlru_proc(0,e43eed38) at vnlru_proc+0xdf
fork_exit(c06f4cbc,0,e43eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43eed6c, ebp = 0 ---
Tracing command syncer pid 39 tid 100049 td 0xc3f80360
sched_switch(c3f80360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0df30,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a0df30,0,c068ec81,c0a5b3c4,c0928ec1,...) at sleepq_wait+0x46
msleep(c0a0df30,0,68,c0929290,0) at msleep+0x27d
sched_sync(0,e43ebd38) at sched_sync+0x361
fork_exit(c06f6964,0,e43ebd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43ebd6c, ebp = 0 ---
Tracing command bufdaemon pid 38 tid 100027 td 0xc3c3bd80
sched_switch(c3c3bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a5b0d4,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a5b0d4) at sleepq_timedwait+0x4a
msleep(c0a5b0d4,c0a5b0d8,44,c092783c,3e8) at msleep+0x259
buf_daemon(0,e270fd38) at buf_daemon+0x198
fork_exit(c06e77ac,0,e270fd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe270fd6c, ebp = 0 ---
Tracing command pagezero pid 37 tid 100028 td 0xc3c3bbd0
sched_switch(c3c3bbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a704d0,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a704d0) at sleepq_timedwait+0x4a
msleep(c0a704d0,c0a6839c,200,c093c392,493e0,c0a6839c,0,c093c378,9a) at msleep+0x259
vm_pagezero(0,e270cd38) at vm_pagezero+0x6c
fork_exit(c0801cbc,0,e270cd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe270cd6c, ebp = 0 ---
Tracing command vmdaemon pid 36 tid 100029 td 0xc3c3ba20
sched_switch(c3c3ba20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a68400,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a68400,0,e2709cd0,c069d310,c093bf66,...) at sleepq_wait+0x46
msleep(c0a68400,0,68,c092783c,0) at msleep+0x27d
vm_daemon(0,e2709d38) at vm_daemon+0x36
fork_exit(c0800908,0,e2709d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2709d6c, ebp = 0 ---
Tracing command pagedaemon pid 35 tid 100030 td 0xc3c3b870
cpustop_handler(e2706ac4,c0899182,3,a,2,...) at cpustop_handler+0x31
ipi_nmi_handler(3,a,2,0,0,...) at ipi_nmi_handler+0x28
trap(c0008,28,c0a90028,c0a956e0,c3dbb000,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc0890cdd, esp = 0xe2706b0c, ebp = 0xe2706b10 ---
smp_tlb_shootdown(f4,c3dbb000,0) at smp_tlb_shootdown+0x61
smp_invlpg(c3dbb000) at smp_invlpg+0x1b
pmap_invalidate_page(c0a956e0,c3dbb000) at pmap_invalidate_page+0x6c
pmap_remove_pte(c0a956e0,bff0f6ec,c3dbb000,c1870c38,c3dbb000,...) at pmap_remove_pte+0x5d
pmap_remove_page(c0a956e0,c3dbb000) at pmap_remove_page+0x6d
pmap_remove(c0a956e0,c3dbb000,c3dbc000,c07f4f43,c1871120,...) at pmap_remove+0x7f
vm_map_delete(c18710a8,c3dbb000,c3dbc000,1,c464baf8,...) at vm_map_delete+0x119
vm_map_remove(c18710a8,c3dbb000,c3dbc000,e2706c04,c07ed2ed,...) at vm_map_remove+0x42
kmem_free(c18710a8,c3dbb000,1000,e2706c34,c07eccbb,...) at kmem_free+0x25
page_free(c3dbb000,1000,2) at page_free+0x29
zone_drain(c1864b40) at zone_drain+0x27b
zone_foreach(c07eca40,e2706ce4,c07ff6af,e2706c6c,246,...) at zone_foreach+0x37
uma_reclaim(e2706c6c,246,0,e2706c78,c068ee4d,...) at uma_reclaim+0x12
vm_pageout_scan(0,c0a6839c,0,c093bf5d,5ee) at vm_pageout_scan+0x103
vm_pageout(0,e2706d38) at vm_pageout+0x2c7
fork_exit(c08005c4,0,e2706d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2706d6c, ebp = 0 ---
Tracing command irq7: ppc0 pid 34 tid 100031 td 0xc3c3b6c0
fork_trampoline() at fork_trampoline
Tracing command fdc0 pid 33 tid 100032 td 0xc3c3b510
sched_switch(c3c3b510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d7e03c,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3d7e03c) at sleepq_timedwait+0x4a
msleep(c3d7e03c,c3d7e0f0,4c,c0917342,3e8) at msleep+0x259
fdc_worker(c3d7e000,c3d7e0f0,0,c093f04c,4a1) at fdc_worker+0x1fe
fdc_thread(c3d7e000,e2700d38) at fdc_thread+0x54
fork_exit(c085e2ec,c3d7e000,e2700d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2700d6c, ebp = 0 ---
Tracing command swi0: sio pid 32 tid 100033 td 0xc3de8000
sched_switch(c3de8000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3defc90,e43add38) at ithread_loop+0xda
fork_exit(c06830c8,c3defc90,e43add38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43add6c, ebp = 0 ---
Tracing command irq12: psm0 pid 31 tid 100034 td 0xc3de5d80
fork_trampoline() at fork_trampoline
Tracing command irq1: atkbd0 pid 30 tid 100035 td 0xc3de5bd0
fork_trampoline() at fork_trampoline
Tracing command irq15: ata1 pid 29 tid 100036 td 0xc3de5a20
fork_trampoline() at fork_trampoline
Tracing command irq14: ata0 pid 28 tid 100037 td 0xc3de5870
sched_switch(c3de5870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3de3460,e43a1d38) at ithread_loop+0xda
fork_exit(c06830c8,c3de3460,e43a1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43a1d6c, ebp = 0 ---
Tracing command irq17: fxp0 pid 27 tid 100017 td 0xc3c376c0
sched_switch(c3c376c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3d7b460,e26e8d38) at ithread_loop+0xda
fork_exit(c06830c8,c3d7b460,e26e8d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e8d6c, ebp = 0 ---
Tracing command usbtask pid 26 tid 100018 td 0xc3c37510
sched_switch(c3c37510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0b6e4,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a0b6e4,0,c068ebc8,c0a0d968,c091438a,...) at sleepq_wait+0x46
msleep(c0a0b6e4,0,5c,c09143a6,0,...) at msleep+0x27d
usb_task_thread(0,e26e5d38) at usb_task_thread+0x4b
fork_exit(c0632498,0,e26e5d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e5d6c, ebp = 0 ---
Tracing command usb0 pid 25 tid 100019 td 0xc3c37360
sched_switch(c3c37360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d6d210,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3d6d210) at sleepq_timedwait+0x4a
msleep(c3d6d210,0,5c,c091439f,ea60,c3d798c0) at msleep+0x259
usb_event_thread(c3d798c0,e26e2d38) at usb_event_thread+0x94
fork_exit(c06323e4,c3d798c0,e26e2d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e2d6c, ebp = 0 ---
Tracing command irq16: uhci0 pid 24 tid 100020 td 0xc3c371b0
sched_switch(c3c371b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3d690b0,e26dfd38) at ithread_loop+0xda
fork_exit(c06830c8,c3d690b0,e26dfd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26dfd6c, ebp = 0 ---
Tracing command em0 taskq pid 23 tid 100021 td 0xc3d46870
sched_switch(c3d46870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d5a280,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3d5a280,c3d5a29c,8,c092245d,47,...) at sleepq_wait+0x46
msleep_spin(c3d5a280,c3d5a29c,c0917342,0) at msleep_spin+0x1a1
taskqueue_thread_loop(c3d30208,e4147d38) at taskqueue_thread_loop+0x61
fork_exit(c06bd590,c3d30208,e4147d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe4147d6c, ebp = 0 ---
Tracing command irq9: acpi0 pid 22 tid 100022 td 0xc3d466c0
fork_trampoline() at fork_trampoline
Tracing command swi5: + pid 21 tid 100023 td 0xc3d46510
fork_trampoline() at fork_trampoline
Tracing command thread taskq pid 9 tid 100024 td 0xc3d46360
sched_switch(c3d46360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85700,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85700,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85700,c3c8571c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c0a13670,e413ed38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c0a13670,e413ed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe413ed6c, ebp = 0 ---
Tracing command swi6: Giant taskq pid 20 tid 100025 td 0xc3d461b0
fork_trampoline() at fork_trampoline
Tracing command acpi_task_2 pid 8 tid 100026 td 0xc3d46000
sched_switch(c3d46000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c10a1a34,e4138d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c10a1a34,e4138d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe4138d6c, ebp = 0 ---
Tracing command acpi_task_1 pid 7 tid 100008 td 0xc3c361b0
sched_switch(c3c361b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c10a1a34,e26c4d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c10a1a34,e26c4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26c4d6c, ebp = 0 ---
Tracing command acpi_task_0 pid 6 tid 100009 td 0xc3c36000
sched_switch(c3c36000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0922466,49,c3d52800,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c10a1a34,e26c1d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c10a1a34,e26c1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26c1d6c, ebp = 0 ---
Tracing command swi6: task queue pid 19 tid 100010 td 0xc3c3b360
sched_switch(c3c3b360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3cff590,e26fdd38) at ithread_loop+0xda
fork_exit(c06830c8,c3cff590,e26fdd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26fdd6c, ebp = 0 ---
Tracing command kqueue taskq pid 5 tid 100011 td 0xc3c3b1b0
sched_switch(c3c3b1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85a00,c0a0d950,0,c0921e8b,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85a00,0,c0922466,49,8,...) at sleepq_wait+0x46
msleep(c3c85a00,c3c85a1c,0,c0917342,0) at msleep+0x27d
taskqueue_thread_loop(c0a0c69c,e26fad38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c0a0c69c,e26fad38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26fad6c, ebp = 0 ---
Tracing command swi2: cambio pid 18 tid 100012 td 0xc3c3b000
fork_trampoline() at fork_trampoline
Tracing command yarrow pid 17 tid 100013 td 0xc3c37d80
sched_switch(c3c37d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a09320,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a09320) at sleepq_timedwait+0x4a
msleep(c0a09320,0,0,c0917342,64) at msleep+0x259
random_kthread(0,e26f4d38) at random_kthread+0x180
fork_exit(c05d7b78,0,e26f4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f4d6c, ebp = 0 ---
Tracing command g_down pid 4 tid 100014 td 0xc3c37bd0
sched_switch(c3c37bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0be9c,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a0be9c) at sleepq_timedwait+0x4a
msleep(c0a0be9c,c0a0bd88,24c,c0917342,64) at msleep+0x259
g_io_schedule_down(c3c37bd0) at g_io_schedule_down+0x56
g_down_procbody(0,e26f1d38) at g_down_procbody+0x5a
fork_exit(c0660628,0,e26f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f1d6c, ebp = 0 ---
Tracing command g_up pid 3 tid 100015 td 0xc3c37a20
sched_switch(c3c37a20,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0d950,0,c09205db,...) at mi_switch+0x280
critical_exit(c0a5a9d8,e26eec44,c0884b4e,e26e0008,c3c30028,...) at critical_exit+0x92
lapic_handle_timer(e26e0008,c3c30028,c3c30028,c0a5aa10,c6,...) at lapic_handle_timer+0x149
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc0897402, esp = 0xe26eec20, ebp = 0xe26eec44 ---
generic_bzero(c0a1c6c8,c0a0bd34,c6,0,e26eec80,...) at generic_bzero+0x1a
witness_lock(c0a0bd34,8,c0917d01,c6) at witness_lock+0xc1
_mtx_lock_flags(c0a0bd34,0,c0917cf8,c6,c065c000,...) at _mtx_lock_flags+0xc4
g_disk_done(c3d92dec) at g_disk_done+0x1e
biodone(c3d92dec) at biodone+0x58
ad_done(c44e2540) at ad_done+0x2a
ata_completed(c44e2540) at ata_completed+0x4f0
g_io_schedule_up(c3c37a20) at g_io_schedule_up+0x5e
g_up_procbody(0,e26eed38) at g_up_procbody+0x5a
fork_exit(c06605c8,0,e26eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26eed6c, ebp = 0 ---
Tracing command g_event pid 2 tid 100016 td 0xc3c37870
sched_switch(c3c37870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0be90,c0a0d950,0,c0921e8b,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a0be90) at sleepq_timedwait+0x4a
msleep(c0a0be90,0,4c,c0917342,64) at msleep+0x259
g_event_procbody(0,e26ebd38) at g_event_procbody+0x9e
fork_exit(c0660688,0,e26ebd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26ebd6c, ebp = 0 ---
Tracing command swi3: vm pid 16 tid 100000 td 0xc3c37000
fork_trampoline() at fork_trampoline
Tracing command swi4: clock sio pid 15 tid 100001 td 0xc3c36d80
cpustop_handler(e26d9b60,c0899182,c06ad407,3,a,...) at cpustop_handler+0x31
ipi_nmi_handler(c06ad407,3,a,0,0,...) at ipi_nmi_handler+0x28
trap(360008,c3c30028,e26d0028,60,c3df3c00,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc08707fc, esp = 0xe26d9ba8, ebp = 0xe26d9bb8 ---
siointr1(c3df3c00,c0a7cf8c,0,c0940fd0,56e,...) at siointr1+0x374
siointr(c3df3c00) at siointr+0x21
intr_execute_handlers(c3c2c0c4,e26d9c14,4,e26d9c64,c0884933,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc068f052, esp = 0xe26d9c54, ebp = 0xe26d9c64 ---
_mtx_lock_sleep(c0a0d968,c3c36d80,0,c091fd5e,f0) at _mtx_lock_sleep+0x112
_mtx_lock_flags(c0a0d968,0,c091fd5e,f0) at _mtx_lock_flags+0xae
softclock(0) at softclock+0x1b7
ithread_execute_handlers(c3c35234,c3c84500) at ithread_execute_handlers+0x122
ithread_loop(c3c118a0,e26d9d38) at ithread_loop+0x67
fork_exit(c06830c8,c3c118a0,e26d9d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d9d6c, ebp = 0 ---
Tracing command swi1: net pid 14 tid 100002 td 0xc3c36bd0
sched_switch(c3c36bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3c118b0,e26d6d38) at ithread_loop+0xda
fork_exit(c06830c8,c3c118b0,e26d6d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d6d6c, ebp = 0 ---
Tracing command idle: cpu0 pid 13 tid 100003 td 0xc3c36a20
sched_switch(c3c36a20,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26d3d38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26d3d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d3d6c, ebp = 0 ---
Tracing command idle: cpu1 pid 12 tid 100004 td 0xc3c36870
sched_switch(c3c36870,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26d0d38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26d0d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d0d6c, ebp = 0 ---
Tracing command idle: cpu2 pid 11 tid 100005 td 0xc3c366c0
sched_switch(c3c366c0,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26cdd38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26cdd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26cdd6c, ebp = 0 ---
Tracing command idle: cpu3 pid 10 tid 100006 td 0xc3c36510
sched_switch(c3c36510,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0d950,0,c091bc3b,...) at mi_switch+0x280
idle_proc(0,e26cad38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26cad38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26cad6c, ebp = 0 ---
Tracing command init pid 1 tid 100007 td 0xc3c36360
sched_switch(c3c36360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c3a000) at sleepq_switch+0xcd
sleepq_wait_sig(c3c3a000) at sleepq_wait_sig+0x1d
msleep(c3c3a000,c3c3a068,15c,c0923f61,0) at msleep+0x26e
kern_wait(c3c36360,ffffffff,e26c7c2c,0,0) at kern_wait+0x7cf
wait4(c3c36360,e26c7d04) at wait4+0x2a
syscall(3b,3b,3b,0,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x8053047, esp = 0xbfbfed9c, ebp = 0xbfbfedb8 ---
Tracing command swapper pid 0 tid 0 td 0xc0a0c1d8
sched_switch(c0a0c1d8,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
scheduler(0,141ec00,141e000,0,c0451735,...) at scheduler+0x195
mi_startup() at mi_startup+0x96
begin() at begin+0x2c
db> call doadump
Physical memory: 1007 MB
Dumping 70 MB:KDB: stack backtrace:
kdb_backtrace(c0a0d950,0,c091f394,16c,c06bdd0a,...) at kdb_backtrace+0x29
mi_switch(1,0) at mi_switch+0xf4
turnstile_wait(c0a0bd34,c3c37a20,0,c0a0bd34,2,c091db40,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a0bd34,c42a0bd0,0,c0917cf8,c6) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a0bd34,0,c0917cf8,c6,c065c000,...) at _mtx_lock_flags+0xae
g_disk_done(c3d92294) at g_disk_done+0x1e
biodone(c3d92294) at biodone+0x58
ad_done(c44e2480) at ad_done+0x2a
ata_completed(c44e2480,0) at ata_completed+0x4f0
ata_finish(c44e2480,c3d3a800,0,e67ae5a8,c04e808c,...) at ata_finish+0x34
ata_interrupt(c3d3a800) at ata_interrupt+0xf0
ata_start(c3d5be00,c3d3a908,0,c08ecdf9,5d) at ata_start+0x284
ata_queue_request(c4589300,c3e0c800,c3e12500,200,0,...) at ata_queue_request+0x21a
ad_strategy(e67ae5fc,e67ae5fc,84,2,0,...) at ad_strategy+0x161
ad_dump(c3e12500,c0a91960,0,9a07da00,0,200,c0945448,46,0,c094542f,3ef,0) at ad_dump+0xa4
minidumpsys(c0a0dc80,c0ac23e4,e67ae754,e67ae784,e67ae760,...) at minidumpsys+0x2ec
dumpsys(c0a0dc80,e67ae810,c047629b,c08797d3,c0956c40,...) at dumpsys+0x1a
doadump(c08797d3,c0956c40,e67ae804,c09f3d80,e67ae814,c0475d8b,c09f3d80,c0956c40,e67ae830,e67ae804) at doadump+0x55
db_fncall(e67ae858,0,c0a8e160,e67ae834,c0a8e160,...) at db_fncall+0x107
db_command(c09f3504,c0956ce0,c08e24db) at db_command+0x233
db_command_loop(0,0,e67ae8e4,e67ae8d0,e67ae918,...) at db_command_loop+0x52
db_trap(c,0,c42a0bd0,28,e67ae9f4,...) at db_trap+0xdd
kdb_trap(c,0,e67ae9f4) at kdb_trap+0x88
trap_fatal(e67ae9f4,c4479cf0) at trap_fatal+0x29d
trap(c4470008,28,28,0,e67aeaa8,...) at trap+0x11b
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc073e0c1, esp = 0xe67aea34, ebp = 0xe67aea38 ---
fnv_32_buf(c4479cf0,3,49003719) at fnv_32_buf+0x39
ip_output(c3d97c00,0,e67aea6c,0,0,c44c7348) at ip_output+0xbc7
udp_output(c44c7348,c4479c00,0,0,c42a0bd0) at udp_output+0x649
udp_send(c4195530,0,c4495300,0,0,...) at udp_send+0x33
sosend_dgram(c4195530,0,e67aec64,c4495300,0,...) at sosend_dgram+0x30d
sosend(c4195530,0,e67aec64,0,0,0,c42a0bd0) at sosend+0x3c
soo_write(c479f240,e67aec64,c4527200,0,c42a0bd0) at soo_write+0x42
dofilewrite(c42a0bd0,3,c479f240,e67aec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c42a0bd0,3,e67aec64,8052000,0,...) at kern_writev+0x36
write(c42a0bd0,e67aed04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8fc, ebp = 0xbfbfe948 ---
db>
Bad function
db> call doadump
Physical memory: 1007 MB
Dumping 70 MB: 55 39 23 7
Dump complete
= 0xf
db> reset
cpu_reset: Restarting BSP
cpu_reset_proxy: Stopped CPU 3
(kgdb) bt full
#0 doadump () at pcpu.h:166
No locals.
#1 0xc047629b in db_fncall (dummy1=-428152744, dummy2=0, dummy3=-1062674080, dummy4=0xe67ae834 "") at ../../../ddb/db_command.c:484
fn_addr = -1066831824
args = {-1064855597, -1063949248, -428152828, -1063305856, -428152812, -1069064821, -1063305856, -1063949248, -428152784,
-428152828}
nargs = 0
retval = 547938656
t = 0
#2 0xc04760a7 in db_command (last_cmdp=0xc09f3504, cmd_table=0x0) at ../../../ddb/db_command.c:399
cmd = (struct command *) 0xc0956c40
t = 0
modif = "\000b©À\000\000\000\000<\025\224Àø\003\000\000xèzæF\000\000\000dèzæø\003\000\000hèzæÅ\026\207Àø\003\000\000ø\003\000\000\r\000\000\000\220èzæÆ\030\207Àxèzæø\003\000\000\f\000\017\003x\000\000\000\000>\237ÀF\000\000\000¤èzæü\200GÀÓë\221À@~GÀF\000\000\000\000>\237ÀòuGÀ\000>\237À\2108\237À"
addr = -428152744
count = -1062674080
have_addr = 0
result = 0
#3 0xc0476162 in db_command_loop () at ../../../ddb/db_command.c:451
No locals.
#4 0xc0477dad in db_trap (type=12, code=0) at ../../../ddb/db_main.c:228
jb = {{_jb = {-428152604, -428152624, -428152552, 1, 70, -1069056698, 1, 7, 1, 70, -428152544, -1066677732}}}
prev_jb = (void *) 0x0
bkpt = 0
#5 0xc06b5ea0 in kdb_trap (type=12, code=0, tf=0xe67ae9f4) at ../../../kern/subr_kdb.c:502
intr = 70
did_stop_cpus = 1
handled = -428152332
#6 0xc0899be5 in trap_fatal (frame=0xe67ae9f4, eva=3293027568) at ../../../i386/i386/trap.c:858
code = 0
type = 12
ss = 40
esp = 0
softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27, ssd_dpl = 0, ssd_p = 1, ssd_xx = 0, ssd_xx1 = 0, ssd_def32 = 1,
ssd_gran = 1}
msg = 0x0
#7 0xc0899263 in trap (frame=
{tf_fs = -1001979896, tf_es = 40, tf_ds = 40, tf_edi = 0, tf_esi = -428152152, tf_ebp = -428152264, tf_isp = -428152288, tf_ebx = -428152152, tf_edx = 33554467, tf_ecx = 4, tf_eax = -1001939728, tf_trapno = 12, tf_err = -1001939728, tf_eip = -1066147647, tf_cs = 32, tf_eflags = 66050, tf_esp = -1001939728, tf_ss = -428152124}) at ../../../i386/i386/trap.c:277
td = (struct thread *) 0xc42a0bd0
p = (struct proc *) 0xc45f8234
i = 0
ucode = 0
type = 12
code = 0
addr = 1514
eva = 3293027568
ksi = {ksi_link = {tqe_next = 0x0, tqe_prev = 0xc08f95c4}, ksi_info = {si_signo = 1180, si_errno = -1009253376,
si_code = -1009344492, si_pid = 0, si_uid = 3230635460, si_status = 1178, si_addr = 0xc3d80400, si_value = {sival_int = -1009253376,
sival_ptr = 0xc3d80400}, _reason = {_fault = {_trapno = -428152376}, _timer = {_timerid = -428152376, _overrun = -1066350925},
_mesgq = {_mqd = -428152376}, _poll = {_band = -428152376}, __spare__ = {__spare1__ = -428152376, __spare2__ = {-1066350925,
-1009253376, 0, -428152344, -1066345660, -1009253376, 402785280}}}}, ksi_flags = 0, ksi_sigq = 0xc4479cd6}
#8 0xc088457a in calltrap () at ../../../i386/i386/exception.s:138
No locals.
#9 0xc073e0c1 in fnv_32_buf (buf=0xc4479cf0, len=3, hval=1224750873) at fnv_hash.h:27
s = (const u_int8_t *) 0xc4479cf0 <Address 0xc4479cf0 out of bounds>
#10 0xc073def3 in ip_output (m=0xc3d97c00, opt=0x0, ro=0xe67aea6c, flags=0, imo=0x0, inp=0xc44c7348) at ../../../netinet/ip_output.c:589
ip = (struct ip *) 0xc4479ce4
ifp = (struct ifnet *) 0xc3d80400
m0 = (struct mbuf *) 0xc4641c00
hlen = 20
mtu = 1500
len = -428152112
error = 0
dst = (struct sockaddr_in *) 0xe67aea70
ia = (struct in_ifaddr *) 0xc3fbac00
sia = (struct in_ifaddr *) 0xc3fbac00
isbroadcast = 0
sw_csum = 1
iproute = {ro_rt = 0xc4472438, ro_dst = {sa_len = 16 '\020', sa_family = 2 '\002',
sa_data = "\000\000ˬ\001\002\000\000\000\000\000\000\000"}}
odst = {s_addr = 33663168}
__func__ = "ip_output"
#11 0xc075099d in udp_output (inp=0xc44c7348, m=0xc4479c00, addr=0x0, control=0x0, td=0xc42a0bd0) at ../../../netinet/udp_usrreq.c:925
ui = (struct udpiphdr *) 0xc4479ce4
len = 8192
faddr = {s_addr = 33663168}
laddr = {s_addr = 83994816}
cm = (struct cmsghdr *) 0xc46f6c00
sin = (struct sockaddr_in *) 0xe67aec64
src = {sin_len = 88 'X', sin_family = 235 'ë', sin_port = 59002, sin_addr = {s_addr = 0}, sin_zero = "\000\210ZÄ\000\000\000"}
error = 0
ipflags = 0
fport = 2304
lport = 1222
unlock_udbinfo = 0
__func__ = "udp_output"
#12 0xc0751093 in udp_send (so=0xc4195530, flags=0, m=0xc4495300, addr=0x0, control=0x0, td=0xc42a0bd0)
at ../../../netinet/udp_usrreq.c:1107
inp = (struct inpcb *) 0xc44c7348
#13 0xc06dbc71 in sosend_dgram (so=0xc4195530, addr=0x0, uio=0xe67aec64, top=0xc4495300, control=0x0, flags=0, td=0xc42a0bd0)
at ../../../kern/uipc_socket.c:1064
space = 1024
resid = 0
clen = 0
error = 0
dontroute = 0
atomic = 1
#14 0xc06dc254 in sosend (so=0x2000023, addr=0x0, uio=0xe67aec64, top=0x0, control=0x0, flags=0, td=0xc42a0bd0)
at ../../../kern/uipc_socket.c:1286
No locals.
#15 0xc06c864e in soo_write (fp=0xc4479cf0, uio=0xe67aec64, active_cred=0xc4527200, flags=0, td=0xc42a0bd0)
at ../../../kern/sys_socket.c:118
so = (struct socket *) 0xc4195530
error = -428151708
#16 0xc06c3243 in dofilewrite (td=0xc42a0bd0, fd=3, fp=0xc479f240, auio=0xe67aec64, offset=Unhandled dwarf expression opcode 0x93
) at file.h:253
cnt = 8192
error = -998641088
ktruio = (struct uio *) 0x0
#17 0xc06c30e6 in kern_writev (td=0xc42a0bd0, fd=3, auio=0xe67aec64) at ../../../kern/sys_generic.c:402
fp = (struct file *) 0xc479f240
error = 0
#18 0xc06c3011 in write (td=0xc42a0bd0, uap=0x4) at ../../../kern/sys_generic.c:326
auio = {uio_iov = 0xe67aec5c, uio_iovcnt = 1, uio_offset = 8191, uio_resid = 0, uio_segflg = UIO_USERSPACE, uio_rw = UIO_WRITE,
uio_td = 0xc42a0bd0}
aiov = {iov_base = 0x8052000, iov_len = 0}
#19 0xc0899f02 in syscall (frame=
{tf_fs = 671416379, tf_es = -1078001605, tf_ds = -1078001605, tf_edi = 671422604, tf_esi = -1077941636, tf_ebp = -1077941944, tf_isp = -428151452, tf_ebx = 1, tf_edx = 0, tf_ecx = 0, tf_eax = 4, tf_trapno = 59, tf_err = 2, tf_eip = 672420655, tf_cs = 51, tf_eflags = 662, tf_esp = -1077942020, tf_ss = 59}) at ../../../i386/i386/trap.c:1006
params = 0xbfbfe900 <Address 0xbfbfe900 out of bounds>
callp = (struct sysent *) 0xc09c21a0
td = (struct thread *) 0xc42a0bd0
p = (struct proc *) 0xc45f8234
orig_tf_eflags = 662
error = 0
narg = 3
args = {3, 134545408, 8192, -1063200432, 2, -1064172069, 625, 1}
code = 4
ksi = {ksi_link = {tqe_next = 0x2, tqe_prev = 0xc09205db}, ksi_info = {si_signo = 625, si_errno = -1003877424, si_code = 1,
si_pid = 1, si_uid = 1, si_status = -1003877424, si_addr = 0xe67aed2c, si_value = {sival_int = -1066672929, sival_ptr = 0xc06bdcdf},
_reason = {_fault = {_trapno = -1063200408}, _timer = {_timerid = -1063200408, _overrun = 2}, _mesgq = {_mqd = -1063200408},
_poll = {_band = -1063200408}, __spare__ = {__spare1__ = -1063200408, __spare2__ = {2, -1064164088, 276, -1003877424, -428151496,
-1066865075, -1007021824}}}}, ksi_flags = -1003877424, ksi_sigq = 0xbfbfea7c}