GENERIC HEAD from May 14 06:55 UTC, No vmcore
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Sun May 14 09:48:46 CEST 2006
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/QUOTA
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1041444864 (993 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
WARNING: /home was not properly dismounted
WARNING: /tmp was not properly dismounted
/tmp: mount pending error: blocks 113774320 files 2621
WARNING: /usr was not properly dismounted
WARNING: /var was not properly dismounted
fxp0: link state changed to UP
mode = 040770, inum = 193745, fs = /tmp
panic: ffs_valloc: dup alloc
cpuid = 3
KDB: enter: panic
[thread pid 18041 tid 100219 ]
Stopped at kdb_enter+0x2b: nop
db> where
Tracing pid 18041 tid 100219 td 0xc43dd510
kdb_enter(c0907dc2) at kdb_enter+0x2b
panic(c091ef1c,c091eefb,41f8,2f4d1,c382c0d4) at panic+0x14b
ffs_valloc(c51f1924,81b0,c3e61480,e66ce880,c4dc1618) at ffs_valloc+0x152
ufs_makeinode(81b0,c51f1924,e66ceb90,e66ceba4) at ufs_makeinode+0x4e
ufs_create(e66cea14,0,e66ceb7c,e66cead0,c06f7258) at ufs_create+0x25
VOP_CREATE_APV(c09c23a0,e66cea14) at VOP_CREATE_APV+0x7e
vn_open_cred(e66ceb7c,e66cec7c,1b0,c3e61480,3) at vn_open_cred+0x1c0
vn_open(e66ceb7c,e66cec7c,1b0,3,0) at vn_open+0x1e
kern_open(c43dd510,bfbfe720,0,602,1b0) at kern_open+0xb9
open(c43dd510,e66ced04,e66ceccc,c069896c,c43dd510) at open+0x18
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (5, FreeBSD ELF32, open), eip = 0x28146cf7, esp = 0xbfbfe6ec, ebp = 0xbfbfe708 ---
db> show allpcpu
Current CPU: 3
cpuid = 0
curthread = 0xc3f48510: pid 18093 "mmap"
curpcb = 0xe5fcfd90
fpcurthread = none
idlethread = 0xc342fa20: pid 13 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc39c9000: pid 18100 "mmap"
curpcb = 0xe6329d90
fpcurthread = none
idlethread = 0xc342f870: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc6be4bd0: pid 18124 "sh"
curpcb = 0xe67b2d90
fpcurthread = none
idlethread = 0xc342f6c0: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc43dd510: pid 18041 "creat"
curpcb = 0xe66ced90
fpcurthread = none
idlethread = 0xc342f510: pid 10 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> show alllocks
Process 18105 (mmap) thread 0xc43a1a20 (100245)
exclusive sx user map r = 0 (0xc3af3044) locked @ vm/vm_mmap.c:541
Process 18100 (mmap) thread 0xc39c9000 (100300)
exclusive sx user map r = 0 (0xc3a9077c) locked @ vm/vm_map.c:2993
Process 18095 (mmap) thread 0xc4111d80 (100333)
exclusive sx user map r = 0 (0xc417a254) locked @ vm/vm_map.c:2993
Process 18088 (swap) thread 0xc43dd000 (100275)
exclusive sleep mutex vm object (standard object) r = 0 (0xc3854690) locked @ vm/vm_fault.c:987
exclusive sx user map r = 0 (0xc3436b9c) locked @ vm/vm_map.c:2993
Process 18080 (mmap) thread 0xc39c96c0 (100168)
exclusive sx user map r = 0 (0xc3af3dac) locked @ vm/vm_map.c:2993
Process 18072 (mmap) thread 0xc39af360 (100130)
exclusive sx user map r = 0 (0xc343514c) locked @ vm/vm_map.c:2993
Process 18066 (swap) thread 0xc39c2510 (100126)
exclusive sx user map r = 0 (0xc411d044) locked @ vm/vm_map.c:2993
Process 18041 (creat) thread 0xc43dd510 (100219)
exclusive sleep mutex Giant r = 0 (0xc09f3668) locked @ kern/vfs_subr.c:671
db> ps
pid uid ppid pgrp state wmesg wchan cmd
18124 2 18121 18124 RLs CPU 2 sh
18123 0 18122 18123 Ss ufs 0xc37ff670 sh
18122 0 773 773 S piperd 0xc384d630 cron
18121 0 773 773 S piperd 0xc381b7bc cron
18117 1001 18063 18010 R+ tcp
18113 1001 18064 18010 R+ tcp
18112 1001 18050 18010 R+ swap
18111 1001 18050 18010 R+ swap
18110 1001 18050 18010 R+ swap
18108 1001 18050 18010 R+ swap
18107 1001 18062 18010 R+ syscall
18106 1001 18067 18010 R+ mmap
18105 1001 18067 18010 R+ mmap
18104 1001 18067 18010 R+ mmap
18102 1001 18067 18010 R+ mmap
18101 1001 18062 18010 R+ syscall
18100 1001 18067 18010 RL+ CPU 1 mmap
18099 1001 18067 18010 R+ mmap
18098 1001 18062 18010 R+ syscall
18097 1001 18067 18010 L+ *Giant 0xc39fea40 mmap
18096 1001 18062 18010 R+ syscall
18095 1001 18067 18010 RL+ mmap
18094 1001 18062 18010 R+ syscall
18093 1001 18067 18010 RL+ CPU 0 mmap
18092 1001 18062 18010 R+ syscall
18091 1001 18067 18010 R+ mmap
18090 1001 18050 18010 R+ swap
18089 1001 18067 18010 R+ mmap
18088 1001 18050 18010 RL+ swap
18087 1001 18067 18010 RL+ mmap
18086 1001 18050 18010 R+ swap
18085 1001 18067 18010 R+ mmap
18084 1001 18050 18010 SL+ swread 0xc15571e8 swap
18083 1001 18067 18010 R+ mmap
18081 1001 18062 18010 R+ syscall
18080 1001 18067 18010 RL+ mmap
18078 1001 18050 18010 R+ swap
18077 1001 18050 18010 R+ swap
18076 1001 18050 18010 R+ swap
18075 1001 18050 18010 R+ swap
18074 1001 18067 18010 R+ mmap
18073 1001 18050 18010 R+ swap
18072 1001 18067 18010 RL+ mmap
18071 1001 18050 18010 R+ swap
18070 1001 18062 18010 R+ syscall
18069 1001 18062 18010 SW+ lthr 0xc3884510 syscall
18067 1001 18034 18010 SW+ wait 0xc3e2b8d0 mmap
18066 1001 18050 18010 RL+ swap
18065 1001 18050 18010 R+ swap
18064 1001 18049 18010 S+ sbwait 0xc3b4dd94 tcp
18063 1001 18049 18010 S+ sbwait 0xc3946afc tcp
18062 1001 18023 18010 SW+ wait 0xc3a5fb04 syscall
18061 1001 18037 18010 R+ creat
18060 1001 18037 18010 R+ creat
18059 1001 18037 18010 R+ creat
18058 1001 18037 18010 R+ creat
18057 1001 18037 18010 R+ creat
18056 1001 18037 18010 L+ *Giant 0xc39fea40 creat
18055 1001 18037 18010 R+ creat
18054 1001 18037 18010 R+ creat
18053 1001 18037 18010 R+ creat
18052 1001 18037 18010 R+ creat
18051 1001 18037 18010 R+ creat
18050 1001 18018 18010 SW+ wait 0xc3f47b04 swap
18049 1001 18025 18010 SW+ wait 0xc3fc969c tcp
18048 1001 18037 18010 R+ creat
18047 1001 18037 18010 R+ creat
18046 1001 18037 18010 R+ creat
18045 1001 18037 18010 R+ creat
18044 1001 18037 18010 R+ creat
18043 1001 18037 18010 R+ creat
18042 1001 18037 18010 R+ creat
18041 1001 18037 18010 R+ CPU 3 creat
18040 1001 18037 18010 R+ creat
18039 1001 18036 18010 R+ mkdir
18038 1001 18036 18010 R+ mkdir
18037 1001 18019 18010 SW+ wait 0xc384e8d0 creat
18036 1001 18020 18010 SW+ wait 0xc38a8000 mkdir
18034 1001 18017 18010 S+ nanslp 0xc09f3f24 mmap
18033 1001 18017 18010 S+ nanslp 0xc09f3f24 badcode
18025 1001 18017 18010 S+ nanslp 0xc09f3f24 tcp
18023 1001 18017 18010 S+ nanslp 0xc09f3f24 syscall
18020 1001 18017 18010 S+ nanslp 0xc09f3f24 mkdir
18019 1001 18017 18010 S+ nanslp 0xc09f3f24 creat
18018 1001 18017 18010 S+ nanslp 0xc09f3f24 swap
18017 1001 18016 18010 SW+ wait 0xc41e0b04 run
18016 1001 18015 18010 SW+ wait 0xc3a608d0 run
18015 1001 18010 18010 S+ nanslp 0xc09f3f24 run
18010 1001 935 18010 SW+ wait 0xc6e0cd38 sh
18009 1001 905 18009 S+ select 0xc0a3ffbc top
18002 1001 1 989 R+ (threaded) thr2
100164 RunQ thr2
100156 S ksesigwa 0xc3afd720 thr2
17997 1001 1 989 R+ (threaded) thr2
100111 RunQ thr2
100102 S ksesigwa 0xc41e1954 thr2
17164 0 0 0 SL mdwait 0xc6d5f800 [md4]
17157 0 0 0 SL mdwait 0xc674e000 [md3]
17150 0 0 0 SL mdwait 0xc6b43000 [md2]
17143 0 0 0 SL mdwait 0xc6b62800 [md1]
935 1001 934 935 SWs+ wait 0xc380bd38 bash
934 1001 932 932 SW select 0xc0a3ffbc sshd
932 0 754 932 SWs sbwait 0xc3e2a1e8 sshd
905 1001 904 905 SWs+ wait 0xc3852d38 bash
904 1001 902 902 S select 0xc0a3ffbc sshd
902 0 754 902 SWs sbwait 0xc395e334 sshd
901 0 1 901 SWs+ ttyin 0xc3606810 getty
900 0 1 900 SWs+ ttyin 0xc3604c10 getty
899 0 1 899 SWs+ ttyin 0xc3604810 getty
898 0 1 898 SWs+ ttyin 0xc3605810 getty
897 0 1 897 SWs+ ttyin 0xc3605410 getty
896 0 1 896 SWs+ ttyin 0xc3601c10 getty
895 0 1 895 SWs+ ttyin 0xc3604010 getty
894 0 1 894 SWs+ ttyin 0xc3604410 getty
875 0 1 875 SWs select 0xc0a3ffbc inetd
846 0 1 846 SWs select 0xc0a3ffbc moused
826 0 1 826 Ss nanslp 0xc09f3f24 watchdogd
773 0 1 773 Ss nanslp 0xc09f3f24 cron
767 25 1 767 SWs pause 0xc38a9b38 sendmail
761 0 1 761 Ss select 0xc0a3ffbc sendmail
754 0 1 754 SWs select 0xc0a3ffbc sshd
733 0 1 733 Ss select 0xc0a3ffbc ntpd
680 0 676 676 SW - 0xc37e4200 nfsd
679 0 676 676 SW - 0xc3b37600 nfsd
678 0 676 676 SW - 0xc35de600 nfsd
677 0 676 676 SW - 0xc37e4000 nfsd
676 0 1 676 SWs select 0xc0a3ffbc nfsd
674 0 1 674 SWs select 0xc0a3ffbc mountd
615 0 1 615 SWs select 0xc0a3ffbc rpcbind
591 0 1 591 Ss select 0xc0a3ffbc syslogd
529 0 1 529 SWs select 0xc0a3ffbc devd
46 0 0 0 SL - 0xe3bdbd04 [schedcpu]
45 0 0 0 SL - 0xc0a47ccc [nfsiod 3]
44 0 0 0 SL - 0xc0a47cc8 [nfsiod 2]
43 0 0 0 SL - 0xc0a47cc4 [nfsiod 1]
42 0 0 0 SL - 0xc0a47cc0 [nfsiod 0]
41 0 0 0 SL sdflush 0xc0a4d220 [softdepflush]
40 0 0 0 SL syncer 0xc09f3c30 [syncer]
39 0 0 0 SL vlruwt 0xc37a7000 [vnlru]
38 0 0 0 SL psleep 0xc0a40434 [bufdaemon]
37 0 0 0 RL [pagezero]
36 0 0 0 SL psleep 0xc0a4da7c [vmdaemon]
35 0 0 0 SL psleep 0xc0a4da3c [pagedaemon]
34 0 0 0 WL [irq7: ppc0]
33 0 0 0 SL - 0xc35de23c [fdc0]
32 0 0 0 WL [swi0: sio]
31 0 0 0 WL [irq12: psm0]
30 0 0 0 WL [irq1: atkbd0]
29 0 0 0 WL [irq15: ata1]
28 0 0 0 RL [irq14: ata0]
27 0 0 0 WL [irq17: fxp0]
26 0 0 0 SL usbtsk 0xc09f1424 [usbtask]
25 0 0 0 SL usbevt 0xc3562210 [usb0]
24 0 0 0 WL [irq16: uhci0]
23 0 0 0 SL - 0xc354ec80 [em0 taskq]
22 0 0 0 WL [irq9: acpi0]
21 0 0 0 WL [swi6: task queue]
9 0 0 0 SL - 0xc3484200 [kqueue taskq]
8 0 0 0 SL - 0xc3484280 [acpi_task_2]
7 0 0 0 SL - 0xc3484280 [acpi_task_1]
6 0 0 0 SL - 0xc3484280 [acpi_task_0]
20 0 0 0 WL [swi2: cambio]
19 0 0 0 WL [swi5: +]
5 0 0 0 SL - 0xc3484580 [thread taskq]
18 0 0 0 WL [swi6: Giant taskq]
17 0 0 0 SL - 0xc09ef060 [yarrow]
4 0 0 0 SL - 0xc09f1b9c [g_down]
3 0 0 0 SL - 0xc09f1b98 [g_up]
2 0 0 0 SL - 0xc09f1b90 [g_event]
16 0 0 0 WL [swi3: vm]
15 0 0 0 RL [swi4: clock sio]
14 0 0 0 WL [swi1: net]
13 0 0 0 RL [idle: cpu0]
12 0 0 0 RL [idle: cpu1]
11 0 0 0 RL [idle: cpu2]
10 0 0 0 RL [idle: cpu3]
1 0 0 1 SLs wait 0xc3433000 [init]
0 0 0 0 WLs [swapper]
18068 1001 18033 18010 Z+ badcode
18103 1001 18062 18010 Z+ syscall
18109 1001 18062 18010 Z+ syscall
18032 1001 18017 18010 Z+ thr2
18026 1001 18017 18010 Z+ udp
18027 1001 18017 18010 Z+ pty
18031 1001 18017 18010 Z+ symlink
18035 1001 18017 18010 Z+ socket
18029 1001 18017 18010 Z+ shm
18030 1001 18017 18010 Z+ lockf
18021 1001 18017 18010 Z+ initial thread
18028 1001 18017 18010 Z+ mkfifo
18022 1001 18017 18010 Z+ rw
18024 1001 18017 18010 Z+ sysctl
db> set $lines 0
db> allt
Tracing command sh pid 18124 tid 100109 td 0xc6be4bd0
cpustop_handler(e67b29c4,c088333e,c0689759,c3482700,e67b298c) at cpustop_handler+0x31
ipi_nmi_handler(c0689759,c3482700,e67b298c,0,0) at ipi_nmi_handler+0x28
trap(c6be0008,e67b0028,28,c6be4bd0,c09f3668) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068991e, esp = 0xe67b2a0c, ebp = 0xe67b2a18 ---
_mtx_lock_sleep(c09f3668,c6be4bd0,0,c09086ae,da) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c09f3668,0,c09086ae,da) at _mtx_lock_flags+0x95
msleep(d7492200,c0a40488,44,c0924df7,0) at msleep+0x2de
bwait(d7492200,44,c0924df7) at bwait+0x47
vnode_pager_generic_getpages(c37ff618,e67b2c30,6000,5,c1044438) at vnode_pager_generic_getpages+0x767
ffs_getpages(e67b2b6c) at ffs_getpages+0x100
VOP_GETPAGES_APV(c09c23a0,e67b2b6c) at VOP_GETPAGES_APV+0x8c
vnode_pager_getpages(c1044438,e67b2c30,6,5) at vnode_pager_getpages+0xa1
vm_fault(c3862948,281ca000,2,8,c6be4bd0) at vm_fault+0x984
trap_pfault(e67b2d38,1,281cae50,281cae50,28076117) at trap_pfault+0xee
trap(3b,3b,3b,281cae50,1b0) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x28076117, esp = 0xbfbfe930, ebp = 0xbfbfe9d8 ---
Tracing command sh pid 18123 tid 100208 td 0xc38aad80
sched_switch(c38aad80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c37ff670,c09f3650,0,c090ad65,211) at sleepq_switch+0xcd
sleepq_wait(c37ff670,0,c37ff694,b2,c0923a63) at sleepq_wait+0x46
msleep(c37ff670,c09f2db0,50,c090fbaf,0) at msleep+0x27d
acquire(e5ef1a48,40,1050000,c38aad80,0) at acquire+0x76
lockmgr(c37ff670,2002,c37ff694,c38aad80) at lockmgr+0x471
ffs_lock(e5ef1aa0,2002,c37ff618,e5ef1abc,c06f8274) at ffs_lock+0x6e
VOP_LOCK_APV(c09c23a0,e5ef1aa0) at VOP_LOCK_APV+0x87
vn_lock(c37ff618,2002,c38aad80,c37ff618,2) at vn_lock+0xa8
vget(c37ff618,2,c38aad80,1,c09f9874) at vget+0xba
vm_mmap_vnode(c38aad80,6000,3,e5ef1beb,e5ef1c18,c37ff618,d5000,0,e5ef1bec) at vm_mmap_vnode+0x66
vm_mmap(c3a5eb58,e5ef1c70,6000,3,7) at vm_mmap+0x1a0
mmap(c38aad80,e5ef1d04,c3f6e234,c,c38aad80) at mmap+0x31b
syscall(3b,3b,3b,2808d9e0,6) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (198, FreeBSD ELF32, nosys), eip = 0x2807600b, esp = 0xbfbfe8fc, ebp = 0xbfbfe938 ---
Tracing command cron pid 18122 tid 100091 td 0xc43a2360
sched_switch(c43a2360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c384d630) at sleepq_switch+0xcd
sleepq_wait_sig(c384d630,100,c090ca6b,24f,8) at sleepq_wait_sig+0x1d
msleep(c384d630,c384d7a0,14c,c090cb40,0) at msleep+0x26e
pipe_read(c3816750,e65f3c64,c3b47600,0,c43a2360) at pipe_read+0x377
dofileread(c43a2360,5,c3816750,e65f3c64,ffffffff) at dofileread+0x89
kern_readv(c43a2360,5,e65f3c64,8221000,1000) at kern_readv+0x36
read(c43a2360,e65f3d04,c38a8d38,c,c43a2360) at read+0x45
syscall(805003b,3b,bfbf003b,28186500,28186500) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28160143, esp = 0xbfbfe64c, ebp = 0xbfbfe668 ---
Tracing command cron pid 18121 tid 100139 td 0xc43fa1b0
sched_switch(c43fa1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c381b7bc) at sleepq_switch+0xcd
sleepq_wait_sig(c381b7bc,100,c090ca6b,24f,8) at sleepq_wait_sig+0x1d
msleep(c381b7bc,c381b92c,14c,c090cb40,0) at msleep+0x26e
pipe_read(c3816bd0,e6380c64,c3b47600,0,c43fa1b0) at pipe_read+0x377
dofileread(c43fa1b0,5,c3816bd0,e6380c64,ffffffff) at dofileread+0x89
kern_readv(c43fa1b0,5,e6380c64,8221000,1000) at kern_readv+0x36
read(c43fa1b0,e6380d04,c40fcb04,c,c43fa1b0) at read+0x45
syscall(805003b,3b,bfbf003b,28186500,28186500) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28160143, esp = 0xbfbfe64c, ebp = 0xbfbfe668 ---
Tracing command tcp pid 18117 tid 100107 td 0xc6bdd360
sched_switch(c6bdd360,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c09f3650,e64a9ad8,c0689759,c342fbd0,e64a9b04) at critical_exit+0x92
spinlock_exit(c342fbd0,e64a9b04,c06b584d,c09f3650,0) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c09f3650,0,c090b474,36f,0) at _mtx_unlock_spin_flags+0x9d
turnstile_unpend(c39fee80,0,c3aef678,39e,e64a9b40) at turnstile_unpend+0x33d
_mtx_unlock_sleep(c3aef678,0,c0918b54,39e) at _mtx_unlock_sleep+0x6b
_mtx_unlock_flags(c3aef678,0,c0918b54,39e) at _mtx_unlock_flags+0xab
tcp_usr_send(c4005298,0,c3b31300,0,0) at tcp_usr_send+0x418
sosend(c4005298,0,e64a9c64,c3b31300,0) at sosend+0x3e5
soo_write(c386f678,e64a9c64,c3e61480,0,c6bdd360) at soo_write+0x46
dofilewrite(c6bdd360,3,c386f678,e64a9c64,ffffffff) at dofilewrite+0x7b
kern_writev(c6bdd360,3,e64a9c64,2818e000,0) at kern_writev+0x36
write(c6bdd360,e64a9d04,0,e64a9d00,c069af11) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x28154123, esp = 0xbfbfe75c, ebp = 0xbfbfe7a8 ---
Tracing command tcp pid 18113 tid 100133 td 0xc39296c0
sched_switch(c39296c0,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c09f3650,e5f9cad8,c0689759,c342fbd0,e5f9cb04) at critical_exit+0x92
spinlock_exit(c342fbd0,e5f9cb04,c06b584d,c09f3650,0) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c09f3650,0,c090b474,36f,0) at _mtx_unlock_spin_flags+0x9d
turnstile_unpend(c39fee80,0,c3af09c0,39e,e5f9cb40) at turnstile_unpend+0x33d
_mtx_unlock_sleep(c3af09c0,0,c0918b54,39e) at _mtx_unlock_sleep+0x6b
_mtx_unlock_flags(c3af09c0,0,c0918b54,39e) at _mtx_unlock_flags+0xab
tcp_usr_send(c40057c8,0,c3d77900,0,0) at tcp_usr_send+0x418
sosend(c40057c8,0,e5f9cc64,c3d77900,0) at sosend+0x3e5
soo_write(c3815630,e5f9cc64,c3e61480,0,c39296c0) at soo_write+0x46
dofilewrite(c39296c0,3,c3815630,e5f9cc64,ffffffff) at dofilewrite+0x7b
kern_writev(c39296c0,3,e5f9cc64,2818e000,0) at kern_writev+0x36
write(c39296c0,e5f9cd04,0,e5f9cd00,c069af11) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x28154123, esp = 0xbfbfe75c, ebp = 0xbfbfe7a8 ---
Tracing command swap pid 18112 tid 100119 td 0xc380cd80
sched_switch(c380cd80,0,2) at sched_switch+0x183
mi_switch(2,0,c380cd80,c8,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e5e4cd38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 18111 tid 100038 td 0xc442ebd0
sched_switch(c442ebd0,0,2) at sched_switch+0x183
mi_switch(2,0,c442ebd0,c4,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e6248d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 18110 tid 100211 td 0xc3891360
sched_switch(c3891360,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(1,bfbfe7b8,c086f0ee,c086003b,3b) at critical_exit+0x92
lapic_handle_timer(c086003b,3b,3b,2805188c,bfbfe8f4) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0x8049183, esp = 0xbfbfe7a0, ebp = 0xbfbfe7b8 ---
Tracing command swap pid 18108 tid 100060 td 0xc39cda20
sched_switch(c39cda20,0,2) at sched_switch+0x183
mi_switch(2,0,c39cda20,ce,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e6416d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command syscall pid 18107 tid 100270 td 0xc43a3510
sched_switch(c43a3510,0,2) at sched_switch+0x183
mi_switch(2,0,c43a3510,cb,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e662cd38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18106 tid 100064 td 0xc43a2d80
sched_switch(c43a2d80,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c5952870,e6605b80,c086f0ee,c43a0008,e6600028) at critical_exit+0x92
lapic_handle_timer(c43a0008,e6600028,28,c43a2d80,c09f3668) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc068991e, esp = 0xe6605b74, ebp = 0xe6605b80 ---
_mtx_lock_sleep(c09f3668,c43a2d80,0,c091267a,6d5) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c09f3668,0,c091267a,6d5) at _mtx_lock_flags+0x95
lseek(c43a2d80,e6605d04,c38a9234,c,c43a2d80) at lseek+0x95
syscall(bfbf003b,bfbf003b,e660003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (198, FreeBSD ELF32, nosys), eip = 0x28154197, esp = 0xbfbfe66c, ebp = 0xbfbfe698 ---
Tracing command mmap pid 18105 tid 100245 td 0xc43a1a20
sched_switch(c43a1a20,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c3a03510,e6482ba4,c086f0ee,c43a0008,e6480028) at critical_exit+0x92
lapic_handle_timer(c43a0008,e6480028,28,c43a1a20,c09f3668) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc068991e, esp = 0xe6482b98, ebp = 0xe6482ba4 ---
_mtx_lock_sleep(c09f3668,c43a1a20,0,c0923b3c,1ca) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c09f3668,0,c0923b3c,1ca,c6589e10,0,c0923b3c,1c9) at _mtx_lock_flags+0x95
vm_object_deallocate(c6589e10,c6589e10,0,c0923244,89a) at vm_object_deallocate+0x9f
vm_map_entry_delete(c3af3000,c39cab28,28280000,c3af3000,c39cab28) at vm_map_entry_delete+0x130
vm_map_delete(c3af3000,28280000,28284000,0,c43a1a20) at vm_map_delete+0x18f
munmap(c43a1a20,e6482d04,c38a9468,c,c43a1a20) at munmap+0xa3
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (73, FreeBSD ELF32, munmap), eip = 0x2813f1b7, esp = 0xbfbfe69c, ebp = 0xbfbfe7b8 ---
Tracing command mmap pid 18104 tid 100184 td 0xc40b21b0
sched_switch(c40b21b0,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c5952870,e68aebb8,c086f0ee,c40b0008,e68a0028) at critical_exit+0x92
lapic_handle_timer(c40b0008,e68a0028,28,c40b21b0,c09f3668) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc068991e, esp = 0xe68aebac, ebp = 0xe68aebb8 ---
_mtx_lock_sleep(c09f3668,c40b21b0,0,c09045a8,77c) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c09f3668,0,c09045a8,77c) at _mtx_lock_flags+0x95
closef(c3e29048,c40b21b0,0,0,3) at closef+0x50
close(c40b21b0,e68aed04,c3f68234,c,c40b21b0) at close+0x1be
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x28154103, esp = 0xbfbfe69c, ebp = 0xbfbfe7b8 ---
Tracing command mmap pid 18102 tid 100095 td 0xc442d510
sched_switch(c442d510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c09f2324,c09f3650,0,c090ad65,211) at sleepq_switch+0xcd
sleepq_wait(c09f2324,c0908579,e61d0b84,c09f3578,c09f2324) at sleepq_wait+0x46
cv_wait_unlock(c09f2324,c09f3578,87e,e61d0ba8,c09f2300) at cv_wait_unlock+0x17e
cv_wait(c09f2324,c09f3578,c38155a0,0,c37f671c) at cv_wait+0x36
_sx_xlock(c09f2300,c09045a8,87e) at _sx_xlock+0x68
fdrop_locked(c38155a0,c442d510,c34117d8,0,c09045a8) at fdrop_locked+0xa1
fdrop(c38155a0,c442d510,6b5,c09f9874,0) at fdrop+0x24
closef(c38155a0,c442d510,0,0,3) at closef+0x367
close(c442d510,e61d0d04,c4312b04,c,c442d510) at close+0x1be
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x28154103, esp = 0xbfbfe69c, ebp = 0xbfbfe7b8 ---
Tracing command syscall pid 18101 tid 100113 td 0xc35da1b0
sched_switch(c35da1b0,0,2) at sched_switch+0x183
mi_switch(2,0,c35da1b0,c8,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e3bded38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18100 tid 100300 td 0xc39c9000
cpustop_handler(e6329acc,c088333e,c39c9000,c1,c06a4325) at cpustop_handler+0x31
ipi_nmi_handler(c39c9000,c1,c06a4325,0,0) at ipi_nmi_handler+0x28
trap(c39c0008,e6320028,28,c39c9000,c09f3668) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068991e, esp = 0xe6329b14, ebp = 0xe6329b20 ---
_mtx_lock_sleep(c09f3668,c39c9000,0,c0923b3c,1ca) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c09f3668,0,c0923b3c,1ca,c40499d8,0,c0923b3c,1c9) at _mtx_lock_flags+0x95
vm_object_deallocate(c40499d8) at vm_object_deallocate+0x9f
unlock_and_deallocate(e6329c00,c0a4da18,0,c0922c99,39d,c1de8f08) at unlock_and_deallocate+0xb3
vm_fault(c3a90738,28281000,2,8,c39c9000) at vm_fault+0x1169
trap_pfault(e6329d38,1,28281000,28281000,2815398a) at trap_pfault+0xee
trap(2805003b,bfbf003b,bfbf003b,28281000,2827d000) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x2815398a, esp = 0xbfbfe694, ebp = 0xbfbfe7b8 ---
Tracing command mmap pid 18099 tid 100192 td 0xc43fabd0
sched_switch(c43fabd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c09f3668,c39d36c0,0,c09f3668,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c09f3668,c43fabd0,0,c09045a8,77c) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c09f3668,0,c09045a8,77c) at _mtx_lock_flags+0x95
closef(c395d000,c43fabd0,0,0,4) at closef+0x50
close(c43fabd0,e6392d04,c43128d0,c,c43fabd0) at close+0x1be
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x28154103, esp = 0xbfbfe69c, ebp = 0xbfbfe7b8 ---
Tracing command syscall pid 18098 tid 100225 td 0xc4072360
sched_switch(c4072360,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(1,bfbfe7b8,c086f0ee,2805003b,bfbf003b) at critical_exit+0x92
lapic_handle_timer(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f0) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0x280b4419, esp = 0xbfbfe770, ebp = 0xbfbfe7b8 ---
Tracing command mmap pid 18097 tid 100103 td 0xc442dbd0
sched_switch(c442dbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c09f3668,c43dd510,0,c09f3668,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c09f3668,c442dbd0,0,c09127bd,34e) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c09f3668,0,c09127bd,34e) at _mtx_lock_flags+0x95
vn_closefile(c38160d8,c442dbd0) at vn_closefile+0x3b
fdrop_locked(c38160d8,c442dbd0,c34117a8,0,c09045a8) at fdrop_locked+0x88
fdrop(c38160d8,c442dbd0,6b5,c09f9874,0) at fdrop+0x24
closef(c38160d8,c442dbd0,0,0,4) at closef+0x367
close(c442dbd0,e61dcd04,c3f47234,c,c442dbd0) at close+0x1be
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x28154103, esp = 0xbfbfe69c, ebp = 0xbfbfe7b8 ---
Tracing command syscall pid 18096 tid 100090 td 0xc402a1b0
sched_switch(c402a1b0,0,2) at sched_switch+0x183
mi_switch(2,0,c402a1b0,c9,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e691ad38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18095 tid 100333 td 0xc4111d80
sched_switch(c4111d80,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c6bdd870,e6965b70,c086f0ee,c4110008,e6960028) at critical_exit+0x92
lapic_handle_timer(c4110008,e6960028,28,c4111d80,c09f3668) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc068991e, esp = 0xe6965b64, ebp = 0xe6965b70 ---
_mtx_lock_sleep(c09f3668,c4111d80,0,c0922c99,128) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c09f3668,0,c0922c99,128) at _mtx_lock_flags+0x95
vm_fault(c417a210,28280000,2,8,c4111d80) at vm_fault+0x200
trap_pfault(e6965d38,1,28280000,28280000,2815398a) at trap_pfault+0xee
trap(2805003b,bfbf003b,bfbf003b,28280000,2827c000) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x2815398a, esp = 0xbfbfe694, ebp = 0xbfbfe7b8 ---
Tracing command syscall pid 18094 tid 100201 td 0xc43a2a20
sched_switch(c43a2a20,c35da870,6) at sched_switch+0x183
mi_switch(6,c35da870,c35daa00,c35da870,e65ffcb4) at mi_switch+0x280
maybe_preempt(c35da870) at maybe_preempt+0x169
sched_add(c35da870,4,c3482700,c35da870,c35d70a0) at sched_add+0x15a
setrunqueue(c35da870,4) at setrunqueue+0xab
intr_event_schedule_thread(c3482700) at intr_event_schedule_thread+0xbb
intr_execute_handlers(c342422c,e65ffd38,e,bfbfe788,c086eed3) at intr_execute_handlers+0x118
lapic_handle_intr(34) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0x2814512a, esp = 0xbfbfe760, ebp = 0xbfbfe788 ---
Tracing command mmap pid 18093 tid 100240 td 0xc3f48510
cpustop_handler(e5fcfa44,c088333e,c3f485b4,3,a) at cpustop_handler+0x31
ipi_nmi_handler(c3f485b4,3,a,0,0) at ipi_nmi_handler+0x28
trap(360008,28,c3f40028,60,c35e8000) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc085abec, esp = 0xe5fcfa8c, ebp = 0xe5fcfa9c ---
siointr1(c35e8000,c0a6260c,0,c09296d4,56e) at siointr1+0x230
siointr(c35e8000) at siointr+0x21
intr_execute_handlers(c34240c4,e5fcfaf8,4,e5fcfb44,c086eed3) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc068991e, esp = 0xe5fcfb38, ebp = 0xe5fcfb44 ---
_mtx_lock_sleep(c09f3668,c3f48510,0,c0922c99,ae) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c09f3668,0,c0922c99,ae) at _mtx_lock_flags+0x95
unlock_and_deallocate(e5fcfc00,c0a4da18,0,c0922c99,39d,c218eb08) at unlock_and_deallocate+0x109
vm_fault(c380fe70,28283000,2,8,c3f48510) at vm_fault+0x1169
trap_pfault(e5fcfd38,1,28283000,28283000,2815398a) at trap_pfault+0xee
trap(2805003b,bfbf003b,bfbf003b,28283000,2827f000) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x2815398a, esp = 0xbfbfe694, ebp = 0xbfbfe7b8 ---
Tracing command syscall pid 18092 tid 100140 td 0xc40b2d80
sched_switch(c40b2d80,0,2) at sched_switch+0x183
mi_switch(2,0,c40b2d80,cc,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e68c3d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18091 tid 100155 td 0xc402b6c0
sched_switch(c402b6c0,0,2) at sched_switch+0x183
mi_switch(2,0,c402b6c0,c0,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e693ed38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 18090 tid 100326 td 0xc39c9510
sched_switch(c39c9510,0,2) at sched_switch+0x183
mi_switch(2,0,c39c9510,ca,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e6332d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18089 tid 100180 td 0xc35dbd80
sched_switch(c35dbd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c09f3668,c39d36c0,0,c09f3668,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c09f3668,c35dbd80,0,c09127bd,34e) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c09f3668,0,c09127bd,34e) at _mtx_lock_flags+0x95
vn_closefile(c3e29870,c35dbd80) at vn_closefile+0x3b
fdrop_locked(c3e29870,c35dbd80,c3411400,0,c09045a8) at fdrop_locked+0x88
fdrop(c3e29870,c35dbd80,6b5,c09f9874,0) at fdrop+0x24
closef(c3e29870,c35dbd80,0,0,4) at closef+0x367
close(c35dbd80,e3c0ed04,c3f4ed38,c,c35dbd80) at close+0x1be
syscall(3b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x28154103, esp = 0xbfbfe69c, ebp = 0xbfbfe7b8 ---
Tracing command swap pid 18088 tid 100275 td 0xc43dd000
sched_switch(c43dd000,c35da870,6) at sched_switch+0x183
mi_switch(6,c35da870,c35daa00,c35da870,e66c5a5c) at mi_switch+0x280
maybe_preempt(c35da870) at maybe_preempt+0x169
sched_add(c35da870,4,c3482700,c35da870,c35d70a0) at sched_add+0x15a
setrunqueue(c35da870,4) at setrunqueue+0xab
intr_event_schedule_thread(c3482700) at intr_event_schedule_thread+0xbb
intr_execute_handlers(c342422c,e66c5ae0,e,e66c5b34,c086eed3) at intr_execute_handlers+0x118
lapic_handle_intr(34) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc06b4bcb, esp = 0xe66c5b20, ebp = 0xe66c5b34 ---
turnstile_release(c0a4da18,c43dd000,0,c0922c99,3f1) at turnstile_release+0x2f
_mtx_lock_flags(c0a4da18,0,c0922c99,3f1,c3854690) at _mtx_lock_flags+0x95
vm_fault_prefault(c3436c00,29336000,c4342c38) at vm_fault_prefault+0x1c5
vm_fault(c3436b58,29336000,2,8,c43dd000) at vm_fault+0x10cf
trap_pfault(e66c5d38,1,29336f4c,29336f4c,8049180) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfe8f4) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe7a0, ebp = 0xbfbfe7b8 ---
Tracing command mmap pid 18087 tid 100283 td 0xc40e2a20
sched_switch(c40e2a20,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c3f49bd0,e6101b44,c086f0ee,c40e0008,e6100028) at critical_exit+0x92
lapic_handle_timer(c40e0008,e6100028,28,c40e2a20,c09f3668) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc0689929, esp = 0xe6101b38, ebp = 0xe6101b44 ---
_mtx_lock_sleep(c09f3668,c40e2a20,0,c0922c99,ae) at _mtx_lock_sleep+0xfd
_mtx_lock_flags(c09f3668,0,c0922c99,ae) at _mtx_lock_flags+0x95
unlock_and_deallocate(e6101c00,c0a4da18,0,c0922c99,39d,c10d3be8) at unlock_and_deallocate+0x109
vm_fault(c3968b58,28283000,2,8,c40e2a20) at vm_fault+0x1169
trap_pfault(e6101d38,1,28283000,28283000,2815398a) at trap_pfault+0xee
trap(2805003b,bfbf003b,bfbf003b,28283000,2827f000) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x2815398a, esp = 0xbfbfe694, ebp = 0xbfbfe7b8 ---
Tracing command swap pid 18086 tid 100120 td 0xc3a03870
sched_switch(c3a03870,0,2) at sched_switch+0x183
mi_switch(2,0,c3a03870,c4,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e65c6d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18085 tid 100174 td 0xc4032d80
sched_switch(c4032d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c09f2324,c09f3650,0,c090ad65,211) at sleepq_switch+0xcd
sleepq_wait(c09f2324,c0908579,e69e0b84,c09f3578,c09f2324) at sleepq_wait+0x46
cv_wait_unlock(c09f2324,c09f3578,87e,e69e0ba8,c09f2300) at cv_wait_unlock+0x17e
cv_wait(c09f2324,c09f3578,c3e293f0,0,c37f671c) at cv_wait+0x36
_sx_xlock(c09f2300,c09045a8,87e) at _sx_xlock+0x68
fdrop_locked(c3e293f0,c4032d80,c3411400,0,c09045a8) at fdrop_locked+0xa1
fdrop(c3e293f0,c4032d80,6b5,c09f9874,0) at fdrop+0x24
closef(c3e293f0,c4032d80,0,0,3) at closef+0x367
close(c4032d80,e69e0d04,c4312234,c,c4032d80) at close+0x1be
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f4) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x28154103, esp = 0xbfbfe69c, ebp = 0xbfbfe7b8 ---
Tracing command swap pid 18084 tid 100148 td 0xc39c7000
sched_switch(c39c7000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c15571e8,c09f3650,0,c090ad65,236) at sleepq_switch+0xcd
sleepq_timedwait(c15571e8,0,c0921ef3,44c,82) at sleepq_timedwait+0x4a
msleep(c15571e8,c0a4da18,40,c0921fcd,4e20,c15571e8,82) at msleep+0x259
swap_pager_getpages(c3854348,e626cc30,2,0) at swap_pager_getpages+0x343
vm_fault(c380f318,2a24a000,2,8,c39c7000) at vm_fault+0x984
trap_pfault(e626cd38,1,2a24af4c,2a24af4c,8049180) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfe8f4) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe7a0, ebp = 0xbfbfe7b8 ---
Tracing command mmap pid 18083 tid 100331 td 0xc4072d80
sched_switch(c4072d80,0,2) at sched_switch+0x183
mi_switch(2,0,c4072d80,c0,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e60b6d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command syscall pid 18081 tid 100296 td 0xc4402360
sched_switch(c4402360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c09f3668,c40121b0,0,c09f3668,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c09f3668,c4402360,0,c092efe0,3ed) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c09f3668,0,c092efe0,3ed) at _mtx_lock_flags+0x95
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f0) at syscall+0x1b6
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (226, FreeBSD ELF32, msgsnd), eip = 0x280b4419, esp = 0xbfbfe770, ebp = 0xbfbfe7b8 ---
Tracing command mmap pid 18080 tid 100168 td 0xc39c96c0
sched_switch(c39c96c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c09f3668,c40121b0,0,c09f3668,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c09f3668,c39c96c0,0,c0923b3c,1ca) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c09f3668,0,c0923b3c,1ca,c3a9e2d0,0,c0923b3c,1c9) at _mtx_lock_flags+0x95
vm_object_deallocate(c3a9e2d0) at vm_object_deallocate+0x9f
unlock_and_deallocate(e6335c00,c0a4da18,0,c0922c99,39d,c1900e08) at unlock_and_deallocate+0xb3
vm_fault(c3af3d68,28283000,2,8,c39c96c0) at vm_fault+0x1169
trap_pfault(e6335d38,1,28283000,28283000,2815398a) at trap_pfault+0xee
trap(2805003b,bfbf003b,bfbf003b,28283000,2827f000) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x2815398a, esp = 0xbfbfe694, ebp = 0xbfbfe7b8 ---
Tracing command swap pid 18078 tid 100195 td 0xc40e5a20
sched_switch(c40e5a20,0,2) at sched_switch+0x183
mi_switch(2,0,c40e5a20,c9,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e653cd38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 18077 tid 100146 td 0xc40b2a20
sched_switch(c40b2a20,0,2) at sched_switch+0x183
mi_switch(2,0,c40b2a20,cd,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e68bdd38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 18076 tid 100159 td 0xc3930360
sched_switch(c3930360,0,2) at sched_switch+0x183
mi_switch(2,0,c3930360,cb,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e5fb1d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 18075 tid 100277 td 0xc43a31b0
sched_switch(c43a31b0,0,2) at sched_switch+0x183
mi_switch(2,0,c43a31b0,c7,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e6626d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18074 tid 100077 td 0xc442d6c0
sched_switch(c442d6c0,0,2) at sched_switch+0x183
mi_switch(2,0,c442d6c0,c0,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e61d3d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command swap pid 18073 tid 100273 td 0xc39cd360
sched_switch(c39cd360,0,2) at sched_switch+0x183
mi_switch(2,0,c39cd360,cb,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e640ad38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command mmap pid 18072 tid 100130 td 0xc39af360
sched_switch(c39af360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c09f3668,c3f48360,0,c09f3668,2,c0906f4c,225) at turnstile_wait+0x394
_mtx_lock_sleep(c09f3668,c39af360,0,c0922c99,128) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c09f3668,0,c0922c99,128) at _mtx_lock_flags+0x95
vm_fault(c3435108,28282000,2,8,c39af360) at vm_fault+0x200
trap_pfault(e621ed38,1,28282000,28282000,2815398a) at trap_pfault+0xee
trap(2805003b,bfbf003b,bfbf003b,28282000,2827e000) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x2815398a, esp = 0xbfbfe694, ebp = 0xbfbfe7b8 ---
Tracing command swap pid 18071 tid 100203 td 0xc39c76c0
sched_switch(c39c76c0,0,2) at sched_switch+0x183
mi_switch(2,0,c39c76c0,ca,c09f3650,0,c090b3e3,101) at mi_switch+0x280
ast(e6278d38) at ast+0x355
doreti_ast() at doreti_ast+0x17
Tracing command syscall pid 18070 tid 100226 td 0xc38a6a20
sched_switch(c38a6a20,0,2) at sched_switch+0x183
mi_switch(2,0,c09f3650,0,c090976f) at mi_switch+0x280
critical_exit(c428cd2c,e5ed0be0,c086f0ee,e5ed0008,c06a0028) at critical_exit+0x92
lapic_handle_timer(e5ed0008,c06a0028,c09f0028,6a0,c09f9874) at lapic_handle_timer+0x131
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc0689623, esp = 0xe5ed0bd4, ebp = 0xe5ed0be0 ---
_mtx_lock_spin_flags(c09f9874,0,c090bf0e,6a0,c428cd2c) at _mtx_lock_spin_flags+0x33
witness_lock_list_get(c0a03800,c428cd2c,f5b,c091267a,e5ed0c38) at witness_lock_list_get+0x2c
witness_lock(c428cd2c,8,c0912683,f5b,9) at witness_lock+0xc1
_mtx_lock_flags(c428cd2c,0,c091267a,f5b) at _mtx_lock_flags+0xa7
getvnode(c428cd00,d4b4c995,e5ed0c74,c38a6a20,0) at getvnode+0x109
fchflags(c38a6a20,e5ed0d04,e5ed0ccc,c069896c,c38a6a20) at fchflags+0x1f
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfe8f0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (35, FreeBSD ELF32, fchflags), eip = 0x280b4419, esp = 0xbfbfe770, ebp = 0xbfbfe7b8 ---
Tracing command syscall pid 18069 tid 100279 td 0xc3884510
db> where 18066
Tracing pid 18066 tid 100126 td 0xc39c2510
sched_switch(c39c2510,c35da870,6) at sched_switch+0x183
mi_switch(6,c35da870,c35daa00,c35da870,e636ba88) at mi_switch+0x280
maybe_preempt(c35da870) at maybe_preempt+0x169
sched_add(c35da870,4,c3482700,c35da870,c35d70a0) at sched_add+0x15a
setrunqueue(c35da870,4) at setrunqueue+0xab
intr_event_schedule_thread(c3482700) at intr_event_schedule_thread+0xbb
intr_execute_handlers(c342422c,e636bb0c,e,e636bb74,c086eed3) at intr_execute_handlers+0x118
lapic_handle_intr(34) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc06b779d, esp = 0xe636bb4c, ebp = 0xe636bb74 ---
witness_checkorder(c40298e8,9,c0922ca2,38c) at witness_checkorder+0x225
_mtx_lock_flags(c40298e8,0,c0922c99,38c,e636bbcc) at _mtx_lock_flags+0x6b
vm_fault(c411d000,2a109000,2,8,c39c2510) at vm_fault+0x10e6
trap_pfault(e636bd38,1,2a109f4c,2a109f4c,8049180) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfe8f4) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe7a0, ebp = 0xbfbfe7b8 ---
db> show lockedvnods
Locked vnodes
0xc37ff618: tag ufs, type VREG
usecount 115, writecount 0, refcount 119 mountedhere 0
flags ()
v_object 0xc1044438 ref 112 pages 146
lock type ufs: SHARED (count 1) with 1 pending
ino 308, on dev ad0s1a
0xc51f1924: tag ufs, type VDIR
usecount 2, writecount 0, refcount 9 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc43dd510 (pid 18041)
ino 164864, on dev ad0s1e
0xc5f5e924: tag ufs, type VREG
usecount 4, writecount 1, refcount 6 mountedhere 0
flags ()
v_object 0xc40499d8 ref 2 pages 4
lock type ufs: SHARED (count 1)
ino 248247, on dev ad0s1e
0xc4c22820: tag ufs, type VREG
usecount 3, writecount 1, refcount 5 mountedhere 0
flags ()
v_object 0xc5884000 ref 1 pages 4
lock type ufs: SHARED (count 1)
ino 248237, on dev ad0s1e
0xc4eb3410: tag ufs, type VREG
usecount 4, writecount 1, refcount 6 mountedhere 0
flags ()
v_object 0xc3a9e2d0 ref 2 pages 4
lock type ufs: SHARED (count 1)
ino 193614, on dev ad0s1e
0xc62d2a28: tag ufs, type VREG
usecount 3, writecount 1, refcount 5 mountedhere 0
flags ()
v_object 0xc3f70618 ref 1 pages 4
lock type ufs: SHARED (count 1)
ino 164016, on dev ad0s1e
0xc4dc1618: tag ufs, type VDIR
usecount 1, writecount 0, refcount 1 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc43dd510 (pid 18041)
ino 193745, on dev ad0s1e
db> where 41
Tracing pid 41 tid 100047 td 0xc37a66c0
sched_switch(c37a66c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a4d220,c09f3650,0,c090ad65,236) at sleepq_switch+0xcd
sleepq_timedwait(c0a4d220,0,c091faa3,2ff,8) at sleepq_timedwait+0x4a
msleep(c0a4d220,c0a4d1f8,44,c091fc23,3e8) at msleep+0x259
softdep_flush(0,e3c39d38) at softdep_flush+0x1f3
fork_exit(c07bb0b4,0,e3c39d38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe3c39d6c, ebp = 0 ---
db> call doadump
Physical memory: 1015 MB
Dumping 163 MB: 148 132 116 100 84 68 52 36 20 4
Dump complete
= 0xf
db> reset
cpu_reset: Restarting BSP
cpu_reset_proxy: Stopped CPU 3