GENERIC HEAD from May 7 03:55 UTC, vmcore.59
Snapshot stress test: Processes hang on /tmp usage.
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #2: Sun May 7 05:58:12 CEST 2006
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1041444864 (993 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
Mount point /mnt had 1 dangling refs
Mount point /mnt had 1 dangling refs
Mount point /mnt had 1 dangling refs
~KDB: enter: Line break on console
[thread pid 13 tid 100003 ]
Stopped at kdb_enter+0x2b: nop
db> show allpcpu
Current CPU: 0
cpuid = 0
curthread = 0xc342fa20: pid 13 "idle: cpu0"
curpcb = 0xe1f1ed90
fpcurthread = none
idlethread = 0xc342fa20: pid 13 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc342f870: pid 12 "idle: cpu1"
curpcb = 0xe1f1bd90
fpcurthread = none
idlethread = 0xc342f870: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc342f6c0: pid 11 "idle: cpu2"
curpcb = 0xe1f18d90
fpcurthread = none
idlethread = 0xc342f6c0: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc342f510: pid 10 "idle: cpu3"
curpcb = 0xe1f15d90
fpcurthread = none
idlethread = 0xc342f510: pid 10 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> show locks
db> show lockedvnods
Locked vnodes
db> ps
pid uid ppid pgrp state wmesg wchan cmd
1962 0 0 0 SL mdwait 0xc3d04000 [md4]
1955 0 0 0 SL mdwait 0xc4905000 [md3]
1948 0 0 0 SL mdwait 0xc5074000 [md2]
1941 0 0 0 SL mdwait 0xc4f85800 [md1]
1864 1001 1858 1864 T+ suspfs 0xc37be06c ls
1858 1001 1857 1858 Ss+ wait 0xc584bd38 bash
1857 1001 1855 1855 S select 0xc0a3f69c sshd
1855 0 755 1855 Ss sbwait 0xc5b00718 sshd
1253 1001 1250 1074 S+ suspfs 0xc37be06c chmod
1252 1001 1248 1074 S+ suspfs 0xc37be06c chmod
1251 1001 1249 1074 S+ suspfs 0xc37be06c chmod
1250 1001 1247 1074 S+ wait 0xc5a97b04 find
1249 1001 1246 1074 S+ wait 0xc5a97d38 find
1248 1001 1245 1074 S+ wait 0xc5ab8234 find
1247 1001 1 1074 S+ wait 0xc5ab8468 sh
1246 1001 1 1074 S+ wait 0xc5485468 sh
1245 1001 1 1074 S+ wait 0xc5ab869c sh
1241 1001 1194 1074 SW+ suspfs 0xc37be06c lockf
1240 1001 1189 1074 SW+ lockf 0xc5bbe740 lockf
1239 1001 1186 1074 SW+ lockf 0xc59d7a00 lockf
1238 0 965 965 SW+ suspwt 0xc37be064 mksnap_ffs
1235 1001 1181 1074 SW+ suspfs 0xc37be06c lockf
1234 1001 1188 1074 SW+ lockf 0xc3784440 lockf
1233 1001 1184 1074 SW+ suspfs 0xc37be06c lockf
1232 1001 1182 1074 SW+ suspfs 0xc37be06c lockf
1228 1001 1227 1074 T+ (threaded) thr1
100219 s thr1
100081 s thr1
100066 S suspfs 0xc37be06c thr1
1227 1001 1154 1074 S+ wait 0xc5a6cb04 thr1
1203 1001 1179 1074 S+ suspfs 0xc37be06c creat
1202 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1201 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1200 1001 1179 1074 S+ suspfs 0xc37be06c creat
1199 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1198 1001 1179 1074 S+ suspfs 0xc37be06c creat
1197 1001 1179 1074 S+ suspfs 0xc37be06c creat
1196 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1195 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1194 1001 1180 1074 S+ lockf 0xc5bd3180 lockf
1193 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1192 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1189 1001 1180 1074 S+ suspfs 0xc37be06c lockf
1188 1001 1180 1074 S+ suspfs 0xc37be06c lockf
1187 1001 1179 1074 S+ suspfs 0xc37be06c creat
1186 1001 1180 1074 S+ suspfs 0xc37be06c lockf
1185 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1184 1001 1180 1074 S+ lockf 0xc5bd31c0 lockf
1183 1001 1179 1074 SW+ suspfs 0xc37be06c creat
1182 1001 1180 1074 S+ lockf 0xc5bd3200 lockf
1181 1001 1180 1074 S+ lockf 0xc5bd3240 lockf
1180 1001 1163 1074 S+ wait 0xc37ca69c lockf
1179 1001 1152 1074 S+ wait 0xc548269c creat
1170 1001 1169 1074 S+ suspfs 0xc37be06c mkdir
1169 1001 1153 1074 S+ wait 0xc5a678d0 mkdir
1163 1001 1150 1074 S+ wait 0xc5a6b69c lockf
1154 1001 1150 1074 S+ wait 0xc5485234 thr1
1153 1001 1150 1074 S+ wait 0xc5482234 mkdir
1152 1001 1150 1074 S+ wait 0xc5485000 creat
1150 1001 1149 1074 S+ wait 0xc38e18d0 run
1149 1001 1148 1074 SW+ wait 0xc5482d38 run
1148 1001 1074 1074 S+ nanslp 0xc09f3804 run
1074 1001 1030 1074 SW+ wait 0xc398969c sh
1030 1001 1029 1030 SWs+ wait 0xc548569c bash
1029 1001 1027 1027 S select 0xc0a3f69c sshd
1027 0 755 1027 SWs sbwait 0xc38ee1e8 sshd
1008 1001 992 1008 S+ select 0xc0a3f69c top
992 1001 991 992 SWs+ wait 0xc37ca8d0 bash
991 1001 989 989 S select 0xc0a3f69c sshd
989 0 755 989 SWs sbwait 0xc38f4864 sshd
965 0 913 965 SW+ wait 0xc3989468 sh
913 0 910 913 SW+ wait 0xc3830000 bash
910 0 909 910 SW+ pause 0xc3833b38 csh
909 1001 901 909 SW+ wait 0xc38e1000 su
901 1001 900 901 SWs+ wait 0xc3830b04 bash
900 1001 898 898 SW select 0xc0a3f69c sshd
898 0 755 898 SWs sbwait 0xc38ef1e8 sshd
897 0 1 897 SWs+ ttyin 0xc3606810 getty
896 0 1 896 SWs+ ttyin 0xc3604c10 getty
895 0 1 895 SWs+ ttyin 0xc3604810 getty
894 0 1 894 SWs+ ttyin 0xc3605810 getty
893 0 1 893 SWs+ ttyin 0xc3605410 getty
892 0 1 892 SWs+ ttyin 0xc3601c10 getty
891 0 1 891 SWs+ ttyin 0xc3604010 getty
890 0 1 890 SWs+ ttyin 0xc3604410 getty
871 0 1 871 SWs select 0xc0a3f69c inetd
842 0 1 842 SWs select 0xc0a3f69c moused
822 0 1 822 Ss nanslp 0xc09f3804 watchdogd
772 0 1 772 Ss nanslp 0xc09f3804 cron
766 25 1 766 Ss pause 0xc380e49c sendmail
762 0 1 762 Ss select 0xc0a3f69c sendmail
755 0 1 755 Ss select 0xc0a3f69c sshd
734 0 1 734 Ss select 0xc0a3f69c ntpd
679 0 675 675 SW - 0xc3789400 nfsd
678 0 675 675 SW - 0xc378de00 nfsd
677 0 675 675 SW - 0xc3789600 nfsd
676 0 675 675 SW - 0xc37a8000 nfsd
675 0 1 675 SWs select 0xc0a3f69c nfsd
673 0 1 673 Ss select 0xc0a3f69c mountd
614 0 1 614 Ss select 0xc0a3f69c rpcbind
590 0 1 590 Ss select 0xc0a3f69c syslogd
528 0 1 528 SWs select 0xc0a3f69c devd
46 0 0 0 SL - 0xe3bdbd04 [schedcpu]
45 0 0 0 SL - 0xc0a473cc [nfsiod 3]
44 0 0 0 SL - 0xc0a473c8 [nfsiod 2]
43 0 0 0 SL - 0xc0a473c4 [nfsiod 1]
42 0 0 0 SL - 0xc0a473c0 [nfsiod 0]
41 0 0 0 SL sdflush 0xc0a4c920 [softdepflush]
40 0 0 0 SL vlruwt 0xc3765d38 [vnlru]
39 0 0 0 SL syncer 0xc09f3510 [syncer]
38 0 0 0 SL psleep 0xc0a3fb14 [bufdaemon]
37 0 0 0 SL pgzero 0xc0a55250 [pagezero]
36 0 0 0 SL psleep 0xc0a4d17c [vmdaemon]
35 0 0 0 SL psleep 0xc0a4d13c [pagedaemon]
34 0 0 0 WL [irq7: ppc0]
33 0 0 0 SL - 0xc35de23c [fdc0]
32 0 0 0 WL [swi0: sio]
31 0 0 0 WL [irq12: psm0]
30 0 0 0 WL [irq1: atkbd0]
29 0 0 0 WL [irq15: ata1]
28 0 0 0 WL [irq14: ata0]
27 0 0 0 WL [irq17: fxp0]
26 0 0 0 SL usbtsk 0xc09f0d04 [usbtask]
25 0 0 0 SL usbevt 0xc3565210 [usb0]
24 0 0 0 WL [irq16: uhci0]
23 0 0 0 SL - 0xc354eb00 [em0 taskq]
22 0 0 0 WL [irq9: acpi0]
21 0 0 0 WL [swi6: Giant taskq]
20 0 0 0 WL [swi6: task queue]
9 0 0 0 SL - 0xc3484300 [acpi_task_2]
8 0 0 0 SL - 0xc3484300 [acpi_task_1]
7 0 0 0 SL - 0xc3484300 [acpi_task_0]
6 0 0 0 SL - 0xc3484380 [kqueue taskq]
19 0 0 0 WL [swi2: cambio]
18 0 0 0 WL [swi5: +]
5 0 0 0 SL - 0xc3484680 [thread taskq]
17 0 0 0 SL - 0xc09ee940 [yarrow]
4 0 0 0 SL - 0xc09f147c [g_down]
3 0 0 0 SL - 0xc09f1478 [g_up]
2 0 0 0 SL - 0xc09f1470 [g_event]
16 0 0 0 WL [swi3: vm]
15 0 0 0 WL [swi4: clock sio]
14 0 0 0 WL [swi1: net]
13 0 0 0 RL CPU 0 [idle: cpu0]
12 0 0 0 RL CPU 1 [idle: cpu1]
11 0 0 0 RL CPU 2 [idle: cpu2]
10 0 0 0 RL CPU 3 [idle: cpu3]
1 0 0 1 SLs wait 0xc3433000 [init]
0 0 0 0 WLs [swapper]
1172 1001 1169 1074 Z+ mkdir
1178 1001 1169 1074 Z+ mkdir
1175 1001 1169 1074 Z+ mkdir
1171 1001 1169 1074 Z+ mkdir
1177 1001 1169 1074 Z+ mkdir
1174 1001 1169 1074 Z+ mkdir
1173 1001 1169 1074 Z+ mkdir
1176 1001 1169 1074 Z+ mkdir
1156 1001 1150 1074 Z+ syscall
1158 1001 1150 1074 Z+ tcp
1229 1001 1227 1074 Z+ thr1
1190 1001 1180 1074 Z+ lockf
1191 1001 1180 1074 Z+ lockf
1165 1001 1150 1074 Z+ thr2
1168 1001 1150 1074 Z+ socket
1161 1001 1150 1074 Z+ mkfifo
1164 1001 1150 1074 Z+ symlink
1159 1001 1150 1074 Z+ udp
1157 1001 1150 1074 Z+ sysctl
1160 1001 1150 1074 Z+ pty
1155 1001 1150 1074 Z+ rw
1167 1001 1150 1074 Z+ mmap
1162 1001 1150 1074 Z+ shm
1166 1001 1150 1074 Z+ badcode
db> show sleepq 0xc37be06c
Wait channel: 0xc37be06c
Queue type: 0
Associated Interlock: 0xc37be024 - (sleep mutex) struct mount mtx
Blocked threads:
0xc5484a20 (tid 100128, pid 1199, "creat")
0xc37ccbd0 (tid 100074, pid 1192, "creat")
0xc3836d80 (tid 100094, pid 1196, "creat")
0xc3831360 (tid 100083, pid 1193, "creat")
0xc38e31b0 (tid 100197, pid 1233, "lockf")
0xc38e3360 (tid 100196, pid 1235, "lockf")
0xc35db510 (tid 100176, pid 1241, "lockf")
0xc38e3000 (tid 100198, pid 1232, "lockf")
0xc380fa20 (tid 100066, pid 1228, "thr1")
0xc35db870 (tid 100055, pid 1202, "creat")
0xc35dba20 (tid 100054, pid 1183, "creat")
0xc38e4510 (tid 100109, pid 1195, "creat")
0xc37cb510 (tid 100099, pid 1185, "creat")
0xc3835d80 (tid 100117, pid 1201, "creat")
0xc5a96360 (tid 100171, pid 1188, "lockf")
0xc5a96510 (tid 100170, pid 1189, "lockf")
0xc5483510 (tid 100140, pid 1186, "lockf")
0xc380f000 (tid 100072, pid 1200, "creat")
0xc5483870 (tid 100138, pid 1170, "mkdir")
0xc380fd80 (tid 100064, pid 1197, "creat")
0xc37cb870 (tid 100062, pid 1198, "creat")
0xc380f510 (tid 100069, pid 1203, "creat")
0xc3835870 (tid 100120, pid 1187, "creat")
0xc38e4000 (tid 100189, pid 1251, "chmod")
0xc5a68d80 (tid 100158, pid 1252, "chmod")
0xc3831000 (tid 100085, pid 1253, "chmod")
0xc5857d80 (tid 100249, pid 1864, "ls")
db> allt
Tracing command md4 pid 1962 tid 100230 td 0xc5863000
sched_switch(c5863000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d04000,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd
sleepq_wait(c3d04000,0,c08e860e,28f,8) at sleepq_wait+0x46
msleep(c3d04000,c3d04020,24c,c08e864f,0) at msleep+0x27d
md_kthread(c3d04000,e620fd38) at md_kthread+0xb1
fork_exit(c058250c,c3d04000,e620fd38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe620fd6c, ebp = 0 ---
Tracing command md3 pid 1955 tid 100211 td 0xc5864a20
sched_switch(c5864a20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4905000,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd
sleepq_wait(c4905000,0,c08e860e,28f,8) at sleepq_wait+0x46
msleep(c4905000,c4905020,24c,c08e864f,0) at msleep+0x27d
md_kthread(c4905000,e623cd38) at md_kthread+0xb1
fork_exit(c058250c,c4905000,e623cd38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe623cd6c, ebp = 0 ---
Tracing command md2 pid 1948 tid 100237 td 0xc5861360
sched_switch(c5861360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5074000,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd
sleepq_wait(c5074000,0,c08e860e,28f,8) at sleepq_wait+0x46
msleep(c5074000,c5074020,24c,c08e864f,0) at msleep+0x27d
md_kthread(c5074000,e61fad38) at md_kthread+0xb1
fork_exit(c058250c,c5074000,e61fad38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe61fad6c, ebp = 0 ---
Tracing command md1 pid 1941 tid 100222 td 0xc5863d80
sched_switch(c5863d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4f85800,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd
sleepq_wait(c4f85800,0,c08e860e,28f,8) at sleepq_wait+0x46
msleep(c4f85800,c4f85820,24c,c08e864f,0) at msleep+0x27d
md_kthread(c4f85800,e6227d38) at md_kthread+0xb1
fork_exit(c058250c,c4f85800,e6227d38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe6227d6c, ebp = 0 ---
Tracing command ls pid 1864 tid 100249 td 0xc5857d80
sched_switch(c5857d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c37be06c,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd
sleepq_wait(c37be06c,0,c0911f49,37c,8) at sleepq_wait+0x46
msleep(c37be06c,c37be024,9f,c0911f83,0) at msleep+0x27d
vn_start_write(c38aa514,e61d6b18,1,c37be000,e61d6b28) at vn_start_write+0xbf
vn_close(c38aa514,5,c580fa80,c5857d80,e61d6b80) at vn_close+0x4f
vn_closefile(c37ffb88,c5857d80) at vn_closefile+0xca
fdrop_locked(c37ffb88,c5857d80,c3411640,0,c0903d2b) at fdrop_locked+0x88
fdrop(c37ffb88,c5857d80,6b5,c09f8f54,0) at fdrop+0x24
closef(c37ffb88,c5857d80,0,0,4) at closef+0x367
close(c5857d80,e61d6d04,c5936000,c,c5857d80) at close+0x1be
syscall(3b,3b,3b,82401c0,804e510) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x2819e103, esp = 0xbfbfe48c, ebp = 0xbfbfe4b8 ---
Tracing command bash pid 1858 tid 100216 td 0xc58641b0
sched_switch(c58641b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c584bd38) at sleepq_switch+0xcd
sleepq_wait_sig(c584bd38,100,c0904595,385,c5936068) at sleepq_wait_sig+0x1d
msleep(c584bd38,c584bda0,15c,c090c5fb,0) at msleep+0x26e
kern_wait(c58641b0,ffffffff,e622dc2c,6,0) at kern_wait+0x7cf
wait4(c58641b0,e622dd04,c584bd38,c,c58641b0) at wait4+0x2a
syscall(839003b,3b,bfbf003b,1,0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x281c2b4b, esp = 0xbfbfe89c, ebp = 0xbfbfe8b8 ---
Tracing command sshd pid 1857 tid 100188 td 0xc5ab36c0
sched_switch(c5ab36c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a3f69c) at sleepq_switch+0xcd
sleepq_wait_sig(c0a3f69c,246,c09f2f30,c090c05f,2fc) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a3f69c,c0a3f684,4,e6077b4c,0) at cv_wait_sig+0x18d
kern_select(c5ab36c0,b,8201168,820116c,0) at kern_select+0x4bf
select(c5ab36c0,e6077d04,0,e6077d00,c069a325) at select+0x44
syscall(3b,3b,bfbf003b,0,bfbfde5c) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x283560c3, esp = 0xbfbfddec, ebp = 0xbfbfde38 ---
Tracing command sshd pid 1855 tid 100248 td 0xc5860000
sched_switch(c5860000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5b00718) at sleepq_switch+0xcd
sleepq_wait_sig(c5b00718,100,c090e928,4e8,c06b6f99) at sleepq_wait_sig+0x1d
msleep(c5b00718,c5b006f0,158,c090ec76,0) at msleep+0x26e
sbwait(c5b006cc,c0688b69,c0a3f5a0,e61d9bb4,c06b7d6c) at sbwait+0x4b
soreceive(c5b0067c,0,e61d9c64,0,0) at soreceive+0x2da
soo_read(c5704e10,e61d9c64,c594cd00,0,c5860000) at soo_read+0x41
dofileread(c5860000,5,c5704e10,e61d9c64,ffffffff) at dofileread+0x89
kern_readv(c5860000,5,e61d9c64,bfbfde48,4) at kern_readv+0x36
read(c5860000,e61d9d04,e61d9ccc,c0697d80,c5860000) at read+0x45
syscall(3b,3b,bfbf003b,4,0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28356143, esp = 0xbfbfddec, ebp = 0xbfbfde18 ---
Tracing command chmod pid 1253 tid 100085 td 0xc3831000
sched_switch(c3831000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c37be06c) at sleepq_switch+0xcd
sleepq_wait_sig(c37be06c,100,c0911f49,37c,8) at sleepq_wait_sig+0x1d
msleep(c37be06c,c37be024,19f,c0911f83,0) at msleep+0x26e
vn_start_write(c5bcfb2c,e5ebeb24,101,c37be000,c0a3f56c) at vn_start_write+0xbf
setfmode(c3831000,c5bcfb2c,41ff,e5ebec10,ffffffdf) at setfmode+0x24
kern_chmod(c3831000,2817fb00,0,41ff,e5ebed30) at kern_chmod+0x5b
chmod(c3831000,e5ebed04,c3833234,c,c3831000) at chmod+0x15
syscall(3b,3b,3b,28183f54,41ff) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (15, FreeBSD ELF32, chmod), eip = 0x280cd44f, esp = 0xbfbfe80c, ebp = 0xbfbfe898 ---
Tracing command chmod pid 1252 tid 100158 td 0xc5a68d80
sched_switch(c5a68d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c37be06c) at sleepq_switch+0xcd
sleepq_wait_sig(c37be06c,100,c0911f49,37c,8) at sleepq_wait_sig+0x1d
msleep(c37be06c,c37be024,19f,c0911f83,0) at msleep+0x26e
vn_start_write(c49da820,e5ffcb24,101,c37be000,c0a3f25c) at vn_start_write+0xbf
setfmode(c5a68d80,c49da820,41ff,e5ffcc10,ffffffdf) at setfmode+0x24
kern_chmod(c5a68d80,2817fb00,0,41ff,e5ffcd30) at kern_chmod+0x5b
chmod(c5a68d80,e5ffcd04,c5a6c8d0,c,c5a68d80) at chmod+0x15
syscall(3b,3b,3b,28183f54,41ff) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (15, FreeBSD ELF32, chmod), eip = 0x280cd44f, esp = 0xbfbfe80c, ebp = 0xbfbfe898 ---
Tracing command chmod pid 1251 tid 100189 td 0xc38e4000
sched_switch(c38e4000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c37be06c) at sleepq_switch+0xcd
sleepq_wait_sig(c37be06c,100,c0911f49,37c,8) at sleepq_wait_sig+0x1d
msleep(c37be06c,c37be024,19f,c0911f83,0) at msleep+0x26e
vn_start_write(c49da104,e5f39b24,101,c37be000,c0a3f224) at vn_start_write+0xbf
setfmode(c38e4000,c49da104,41ff,e5f39c10,ffffffdf) at setfmode+0x24
kern_chmod(c38e4000,2817fb00,0,41ff,e5f39d30) at kern_chmod+0x5b
chmod(c38e4000,e5f39d04,c5ab8000,c,c38e4000) at chmod+0x15
syscall(3b,3b,3b,28183f54,41ff) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (15, FreeBSD ELF32, chmod), eip = 0x280cd44f, esp = 0xbfbfe80c, ebp = 0xbfbfe898 ---
Tracing command find pid 1250 tid 100173 td 0xc5a96000
sched_switch(c5a96000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5a97b04) at sleepq_switch+0xcd
sleepq_wait_sig(c5a97b04,100,c0904595,385,c383329c) at sleepq_wait_sig+0x1d
msleep(c5a97b04,c5a97b6c,15c,c090c5fb,0) at msleep+0x26e
kern_wait(c5a96000,4e5,e6050c2c,0,0) at kern_wait+0x7cf
wait4(c5a96000,e6050d04,c5a97b04,c,c5a96000) at wait4+0x2a
syscall(3b,bfbf003b,bfbf003b,0,28194fd0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e8b4b, esp = 0xbfbfe79c, ebp = 0xbfbfe7b8 ---
Tracing command find pid 1249 tid 100174 td 0xc5a95d80
sched_switch(c5a95d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5a97d38) at sleepq_switch+0xcd
sleepq_wait_sig(c5a97d38,100,c0904595,385,c5ab8068) at sleepq_wait_sig+0x1d
msleep(c5a97d38,c5a97da0,15c,c090c5fb,0) at msleep+0x26e
kern_wait(c5a95d80,4e3,e604dc2c,0,0) at kern_wait+0x7cf
wait4(c5a95d80,e604dd04,c5a97d38,c,c5a95d80) at wait4+0x2a
syscall(3b,3b,3b,0,28194fd0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e8b4b, esp = 0xbfbfe79c, ebp = 0xbfbfe7b8 ---
Tracing command find pid 1248 tid 100190 td 0xc38e3d80
sched_switch(c38e3d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5ab8234) at sleepq_switch+0xcd
sleepq_wait_sig(c5ab8234,100,c0904595,385,c5a6c938) at sleepq_wait_sig+0x1d
msleep(c5ab8234,c5ab829c,15c,c090c5fb,0) at msleep+0x26e
kern_wait(c38e3d80,4e4,e5f36c2c,0,0) at kern_wait+0x7cf
wait4(c38e3d80,e5f36d04,c5ab8234,c,c38e3d80) at wait4+0x2a
syscall(3b,3b,3b,0,28194fd0) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e8b4b, esp = 0xbfbfe79c, ebp = 0xbfbfe7b8 ---
Tracing command sh pid 1247 tid 100191 td 0xc38e3bd0
sched_switch(c38e3bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5ab8468) at sleepq_switch+0xcd
sleepq_wait_sig(c5ab8468,100,c0904595,385,c5a97b6c) at sleepq_wait_sig+0x1d
msleep(c5ab8468,c5ab84d0,15c,c090c5fb,0) at msleep+0x26e
kern_wait(c38e3bd0,ffffffff,e5f33c2c,2,0) at kern_wait+0x7cf
wait4(c38e3bd0,e5f33d04,c5ab8468,c,c38e3bd0) at wait4+0x2a
syscall(2809003b,3b,bfbf003b,bfbfe628,432) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2814cb4b, esp = 0xbfbfe5dc, ebp = 0xbfbfe5f8 ---
Tracing command sh pid 1246 tid 100135 td 0xc5483d80
sched_switch(c5483d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5485468) at sleepq_switch+0xcd
sleepq_wait_sig(c5485468,100,c0904595,385,c5a97da0) at sleepq_wait_sig+0x1d
msleep(c5485468,c54854d0,15c,c090c5fb,0) at msleep+0x26e
kern_wait(c5483d80,ffffffff,e5fa8c2c,2,0) at kern_wait+0x7cf
wait4(c5483d80,e5fa8d04,c5485468,c,c5483d80) at wait4+0x2a
syscall(3b,3b,3b,bfbfe628,432) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2814cb4b, esp = 0xbfbfe5dc, ebp = 0xbfbfe5f8 ---
Tracing command sh pid 1245 tid 100192 td 0xc38e3a20
sched_switch(c38e3a20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5ab869c) at sleepq_switch+0xcd
sleepq_wait_sig(c5ab869c,100,c0904595,385,c5ab829c) at sleepq_wait_sig+0x1d
msleep(c5ab869c,c5ab8704,15c,c090c5fb,0) at msleep+0x26e
kern_wait(c38e3a20,ffffffff,e5f30c2c,2,0) at kern_wait+0x7cf
wait4(c38e3a20,e5f30d04,c5ab869c,c,c38e3a20) at wait4+0x2a
syscall(3b,2821003b,bfbf003b,bfbfe628,432) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x2814cb4b, esp = 0xbfbfe5dc, ebp = 0xbfbfe5f8 ---
Tracing command lockf pid 1241 tid 100176 td 0xc35db510
db> call doadump
Physical memory: 1015 MB
Dumping 113 MB: 98 82 66 50 34 18 2
Dump complete
= 0xf
db> reset
(kgdb) p *(struct vnode *)0xc38aa514
$3 = {v_type = VDIR, v_tag = 0xc090f332 "ufs", v_op = 0xc09c1bc0, v_data = 0xc37fb294,
v_mount = 0xc37be000, v_nmntvnodes = {tqe_next = 0xc38aa410, tqe_prev = 0xc37be058},
v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0},
v_hashlist = {le_next = 0x0, le_prev = 0xc3699e88}, v_hash = 0x2, v_cache_src = {
lh_first = 0xc3d0d7f8}, v_cache_dst = {tqh_first = 0x0, tqh_last = 0xc38aa544},
v_dd = 0x0, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0, v_clen = 0x0, v_lock = {
lk_interlock = 0xc09f2c78, lk_flags = 0x40, lk_sharecount = 0x0,
lk_waitcount = 0x0, lk_exclusivecount = 0x0, lk_prio = 0x50,
lk_wmesg = 0xc090f332 "ufs", lk_timo = 0x33, lk_lockholder = 0xffffffff,
lk_newlock = 0x0}, v_interlock = {mtx_object = {
lo_name = 0xc090b5c5 "vnode interlock", lo_type = 0xc090b5c5 "vnode interlock",
lo_flags = 0x1030000, lo_witness_data = {lod_list = {stqe_next = 0xc0a02990},
lod_witness = 0xc0a02990}}, mtx_lock = 0x4, mtx_recurse = 0x0},
v_vnlock = 0xc38aa56c, v_holdcnt = 0x6, v_usecount = 0x3, v_iflag = 0x20,
v_vflag = 0x1, v_writecount = 0x0, v_freelist = {tqe_next = 0x0,
tqe_prev = 0xc37f9bd8}, v_bufobj = {bo_mtx = 0xc38aa590, bo_clean = {bv_hd = {
tqh_first = 0xd751fa38, tqh_last = 0xd751fa70}, bv_root = 0xd751fa38,
bv_cnt = 0x1}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc38aa5dc},
bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x0,
bo_ops = 0xc09b52a4, bo_bsize = 0x4000, bo_object = 0xc382ce10, bo_synclist = {
le_next = 0x0, le_prev = 0xc360fb04}, bo_private = 0xc38aa514,
__bo_vnode = 0xc38aa514}, v_pollinfo = 0x0, v_label = 0x0}
(kgdb) p *(*(struct vnode *)0xc38aa514)->v_mount
$4 = {mnt_lock = {lk_interlock = 0xc09f2420, lk_flags = 0x1000000, lk_sharecount = 0x1,
lk_waitcount = 0x0, lk_exclusivecount = 0x0, lk_prio = 0x50,
lk_wmesg = 0xc0910d99 "vfslock", lk_timo = 0x0, lk_lockholder = 0xffffffff,
lk_newlock = 0x0}, mnt_mtx = {mtx_object = {
lo_name = 0xc0910d88 "struct mount mtx", lo_type = 0xc0910d88 "struct mount mtx",
lo_flags = 0x1030000, lo_witness_data = {lod_list = {stqe_next = 0xc0a01220},
lod_witness = 0xc0a01220}}, mtx_lock = 0x4, mtx_recurse = 0x0}, mnt_list = {
tqe_next = 0xc37b7ca8, tqe_prev = 0xc37be2c4}, mnt_op = 0xc09c1860,
mnt_vfc = 0xc09c18a0, mnt_vnodecovered = 0xc3807d34, mnt_syncer = 0xc38aa410,
mnt_ref = 0x297, mnt_nvnodelist = {tqh_first = 0xc38aa514, tqh_last = 0xc5921938},
mnt_nvnodelistsize = 0x27b, mnt_writeopcount = 0x1, mnt_kern_flag = 0x28000000,
mnt_flag = 0x1211300, mnt_opt = 0xc4dc0170, mnt_optnew = 0xc3849a70,
mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x7,
f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0xf20a7,
f_bfree = 0x635de, f_bavail = 0x5000f, f_files = 0x3f3fe, f_ffree = 0x389e4,
f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0,
f_spare = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff,
f_owner = 0x0, f_fsid = {val = {0x434f6181, 0xa1e353a3}},
f_charspare = '\0' <repeats 79 times>,
f_fstypename = "ufs", '\0' <repeats 12 times>,
f_mntfromname = "/dev/ad0s1e", '\0' <repeats 76 times>,
f_mntonname = "/tmp", '\0' <repeats 83 times>}, mnt_cred = 0xc371e900,
mnt_data = 0xc3816700, mnt_time = 0x0, mnt_iosize_max = 0x20000,
mnt_export = 0xc5a3a200, mnt_mntlabel = 0x0, mnt_fslabel = 0x0,
mnt_hashseed = 0xa30717a0, mnt_markercnt = 0x0, mnt_holdcnt = 0x0,
mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0,
mnt_secondary_accwrites = 0xfbb6}