GENERIC HEAD from May 7 03:55 UTC, vmcore.59 Snapshot stress test: Processes hang on /tmp usage. GDB: no debug ports present KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2006 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD 7.0-CURRENT #2: Sun May 7 05:58:12 CEST 2006 pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. ACPI APIC Table: <A M I OEMAPIC > Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0xf24 Stepping = 4 Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM> Logical CPUs per core: 2 real memory = 1073676288 (1023 MB) avail memory = 1041444864 (993 MB) : Trying to mount root from ufs:/dev/ad0s1a fxp0: link state changed to UP Mount point /mnt had 1 dangling refs Mount point /mnt had 1 dangling refs Mount point /mnt had 1 dangling refs ~KDB: enter: Line break on console [thread pid 13 tid 100003 ] Stopped at kdb_enter+0x2b: nop db> show allpcpu Current CPU: 0 cpuid = 0 curthread = 0xc342fa20: pid 13 "idle: cpu0" curpcb = 0xe1f1ed90 fpcurthread = none idlethread = 0xc342fa20: pid 13 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 curthread = 0xc342f870: pid 12 "idle: cpu1" curpcb = 0xe1f1bd90 fpcurthread = none idlethread = 0xc342f870: pid 12 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 curthread = 0xc342f6c0: pid 11 "idle: cpu2" curpcb = 0xe1f18d90 fpcurthread = none idlethread = 0xc342f6c0: pid 11 "idle: cpu2" APIC ID = 6 currentldt = 0x50 spin locks held: cpuid = 3 curthread = 0xc342f510: pid 10 "idle: cpu3" curpcb = 0xe1f15d90 fpcurthread = none idlethread = 0xc342f510: pid 10 "idle: cpu3" APIC ID = 7 currentldt = 0x50 spin locks held: db> show locks db> show lockedvnods Locked vnodes db> ps pid uid ppid pgrp state wmesg wchan cmd 1962 0 0 0 SL mdwait 0xc3d04000 [md4] 1955 0 0 0 SL mdwait 0xc4905000 [md3] 1948 0 0 0 SL mdwait 0xc5074000 [md2] 1941 0 0 0 SL mdwait 0xc4f85800 [md1] 1864 1001 1858 1864 T+ suspfs 0xc37be06c ls 1858 1001 1857 1858 Ss+ wait 0xc584bd38 bash 1857 1001 1855 1855 S select 0xc0a3f69c sshd 1855 0 755 1855 Ss sbwait 0xc5b00718 sshd 1253 1001 1250 1074 S+ suspfs 0xc37be06c chmod 1252 1001 1248 1074 S+ suspfs 0xc37be06c chmod 1251 1001 1249 1074 S+ suspfs 0xc37be06c chmod 1250 1001 1247 1074 S+ wait 0xc5a97b04 find 1249 1001 1246 1074 S+ wait 0xc5a97d38 find 1248 1001 1245 1074 S+ wait 0xc5ab8234 find 1247 1001 1 1074 S+ wait 0xc5ab8468 sh 1246 1001 1 1074 S+ wait 0xc5485468 sh 1245 1001 1 1074 S+ wait 0xc5ab869c sh 1241 1001 1194 1074 SW+ suspfs 0xc37be06c lockf 1240 1001 1189 1074 SW+ lockf 0xc5bbe740 lockf 1239 1001 1186 1074 SW+ lockf 0xc59d7a00 lockf 1238 0 965 965 SW+ suspwt 0xc37be064 mksnap_ffs 1235 1001 1181 1074 SW+ suspfs 0xc37be06c lockf 1234 1001 1188 1074 SW+ lockf 0xc3784440 lockf 1233 1001 1184 1074 SW+ suspfs 0xc37be06c lockf 1232 1001 1182 1074 SW+ suspfs 0xc37be06c lockf 1228 1001 1227 1074 T+ (threaded) thr1 100219 s thr1 100081 s thr1 100066 S suspfs 0xc37be06c thr1 1227 1001 1154 1074 S+ wait 0xc5a6cb04 thr1 1203 1001 1179 1074 S+ suspfs 0xc37be06c creat 1202 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1201 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1200 1001 1179 1074 S+ suspfs 0xc37be06c creat 1199 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1198 1001 1179 1074 S+ suspfs 0xc37be06c creat 1197 1001 1179 1074 S+ suspfs 0xc37be06c creat 1196 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1195 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1194 1001 1180 1074 S+ lockf 0xc5bd3180 lockf 1193 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1192 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1189 1001 1180 1074 S+ suspfs 0xc37be06c lockf 1188 1001 1180 1074 S+ suspfs 0xc37be06c lockf 1187 1001 1179 1074 S+ suspfs 0xc37be06c creat 1186 1001 1180 1074 S+ suspfs 0xc37be06c lockf 1185 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1184 1001 1180 1074 S+ lockf 0xc5bd31c0 lockf 1183 1001 1179 1074 SW+ suspfs 0xc37be06c creat 1182 1001 1180 1074 S+ lockf 0xc5bd3200 lockf 1181 1001 1180 1074 S+ lockf 0xc5bd3240 lockf 1180 1001 1163 1074 S+ wait 0xc37ca69c lockf 1179 1001 1152 1074 S+ wait 0xc548269c creat 1170 1001 1169 1074 S+ suspfs 0xc37be06c mkdir 1169 1001 1153 1074 S+ wait 0xc5a678d0 mkdir 1163 1001 1150 1074 S+ wait 0xc5a6b69c lockf 1154 1001 1150 1074 S+ wait 0xc5485234 thr1 1153 1001 1150 1074 S+ wait 0xc5482234 mkdir 1152 1001 1150 1074 S+ wait 0xc5485000 creat 1150 1001 1149 1074 S+ wait 0xc38e18d0 run 1149 1001 1148 1074 SW+ wait 0xc5482d38 run 1148 1001 1074 1074 S+ nanslp 0xc09f3804 run 1074 1001 1030 1074 SW+ wait 0xc398969c sh 1030 1001 1029 1030 SWs+ wait 0xc548569c bash 1029 1001 1027 1027 S select 0xc0a3f69c sshd 1027 0 755 1027 SWs sbwait 0xc38ee1e8 sshd 1008 1001 992 1008 S+ select 0xc0a3f69c top 992 1001 991 992 SWs+ wait 0xc37ca8d0 bash 991 1001 989 989 S select 0xc0a3f69c sshd 989 0 755 989 SWs sbwait 0xc38f4864 sshd 965 0 913 965 SW+ wait 0xc3989468 sh 913 0 910 913 SW+ wait 0xc3830000 bash 910 0 909 910 SW+ pause 0xc3833b38 csh 909 1001 901 909 SW+ wait 0xc38e1000 su 901 1001 900 901 SWs+ wait 0xc3830b04 bash 900 1001 898 898 SW select 0xc0a3f69c sshd 898 0 755 898 SWs sbwait 0xc38ef1e8 sshd 897 0 1 897 SWs+ ttyin 0xc3606810 getty 896 0 1 896 SWs+ ttyin 0xc3604c10 getty 895 0 1 895 SWs+ ttyin 0xc3604810 getty 894 0 1 894 SWs+ ttyin 0xc3605810 getty 893 0 1 893 SWs+ ttyin 0xc3605410 getty 892 0 1 892 SWs+ ttyin 0xc3601c10 getty 891 0 1 891 SWs+ ttyin 0xc3604010 getty 890 0 1 890 SWs+ ttyin 0xc3604410 getty 871 0 1 871 SWs select 0xc0a3f69c inetd 842 0 1 842 SWs select 0xc0a3f69c moused 822 0 1 822 Ss nanslp 0xc09f3804 watchdogd 772 0 1 772 Ss nanslp 0xc09f3804 cron 766 25 1 766 Ss pause 0xc380e49c sendmail 762 0 1 762 Ss select 0xc0a3f69c sendmail 755 0 1 755 Ss select 0xc0a3f69c sshd 734 0 1 734 Ss select 0xc0a3f69c ntpd 679 0 675 675 SW - 0xc3789400 nfsd 678 0 675 675 SW - 0xc378de00 nfsd 677 0 675 675 SW - 0xc3789600 nfsd 676 0 675 675 SW - 0xc37a8000 nfsd 675 0 1 675 SWs select 0xc0a3f69c nfsd 673 0 1 673 Ss select 0xc0a3f69c mountd 614 0 1 614 Ss select 0xc0a3f69c rpcbind 590 0 1 590 Ss select 0xc0a3f69c syslogd 528 0 1 528 SWs select 0xc0a3f69c devd 46 0 0 0 SL - 0xe3bdbd04 [schedcpu] 45 0 0 0 SL - 0xc0a473cc [nfsiod 3] 44 0 0 0 SL - 0xc0a473c8 [nfsiod 2] 43 0 0 0 SL - 0xc0a473c4 [nfsiod 1] 42 0 0 0 SL - 0xc0a473c0 [nfsiod 0] 41 0 0 0 SL sdflush 0xc0a4c920 [softdepflush] 40 0 0 0 SL vlruwt 0xc3765d38 [vnlru] 39 0 0 0 SL syncer 0xc09f3510 [syncer] 38 0 0 0 SL psleep 0xc0a3fb14 [bufdaemon] 37 0 0 0 SL pgzero 0xc0a55250 [pagezero] 36 0 0 0 SL psleep 0xc0a4d17c [vmdaemon] 35 0 0 0 SL psleep 0xc0a4d13c [pagedaemon] 34 0 0 0 WL [irq7: ppc0] 33 0 0 0 SL - 0xc35de23c [fdc0] 32 0 0 0 WL [swi0: sio] 31 0 0 0 WL [irq12: psm0] 30 0 0 0 WL [irq1: atkbd0] 29 0 0 0 WL [irq15: ata1] 28 0 0 0 WL [irq14: ata0] 27 0 0 0 WL [irq17: fxp0] 26 0 0 0 SL usbtsk 0xc09f0d04 [usbtask] 25 0 0 0 SL usbevt 0xc3565210 [usb0] 24 0 0 0 WL [irq16: uhci0] 23 0 0 0 SL - 0xc354eb00 [em0 taskq] 22 0 0 0 WL [irq9: acpi0] 21 0 0 0 WL [swi6: Giant taskq] 20 0 0 0 WL [swi6: task queue] 9 0 0 0 SL - 0xc3484300 [acpi_task_2] 8 0 0 0 SL - 0xc3484300 [acpi_task_1] 7 0 0 0 SL - 0xc3484300 [acpi_task_0] 6 0 0 0 SL - 0xc3484380 [kqueue taskq] 19 0 0 0 WL [swi2: cambio] 18 0 0 0 WL [swi5: +] 5 0 0 0 SL - 0xc3484680 [thread taskq] 17 0 0 0 SL - 0xc09ee940 [yarrow] 4 0 0 0 SL - 0xc09f147c [g_down] 3 0 0 0 SL - 0xc09f1478 [g_up] 2 0 0 0 SL - 0xc09f1470 [g_event] 16 0 0 0 WL [swi3: vm] 15 0 0 0 WL [swi4: clock sio] 14 0 0 0 WL [swi1: net] 13 0 0 0 RL CPU 0 [idle: cpu0] 12 0 0 0 RL CPU 1 [idle: cpu1] 11 0 0 0 RL CPU 2 [idle: cpu2] 10 0 0 0 RL CPU 3 [idle: cpu3] 1 0 0 1 SLs wait 0xc3433000 [init] 0 0 0 0 WLs [swapper] 1172 1001 1169 1074 Z+ mkdir 1178 1001 1169 1074 Z+ mkdir 1175 1001 1169 1074 Z+ mkdir 1171 1001 1169 1074 Z+ mkdir 1177 1001 1169 1074 Z+ mkdir 1174 1001 1169 1074 Z+ mkdir 1173 1001 1169 1074 Z+ mkdir 1176 1001 1169 1074 Z+ mkdir 1156 1001 1150 1074 Z+ syscall 1158 1001 1150 1074 Z+ tcp 1229 1001 1227 1074 Z+ thr1 1190 1001 1180 1074 Z+ lockf 1191 1001 1180 1074 Z+ lockf 1165 1001 1150 1074 Z+ thr2 1168 1001 1150 1074 Z+ socket 1161 1001 1150 1074 Z+ mkfifo 1164 1001 1150 1074 Z+ symlink 1159 1001 1150 1074 Z+ udp 1157 1001 1150 1074 Z+ sysctl 1160 1001 1150 1074 Z+ pty 1155 1001 1150 1074 Z+ rw 1167 1001 1150 1074 Z+ mmap 1162 1001 1150 1074 Z+ shm 1166 1001 1150 1074 Z+ badcode db> show sleepq 0xc37be06c Wait channel: 0xc37be06c Queue type: 0 Associated Interlock: 0xc37be024 - (sleep mutex) struct mount mtx Blocked threads: 0xc5484a20 (tid 100128, pid 1199, "creat") 0xc37ccbd0 (tid 100074, pid 1192, "creat") 0xc3836d80 (tid 100094, pid 1196, "creat") 0xc3831360 (tid 100083, pid 1193, "creat") 0xc38e31b0 (tid 100197, pid 1233, "lockf") 0xc38e3360 (tid 100196, pid 1235, "lockf") 0xc35db510 (tid 100176, pid 1241, "lockf") 0xc38e3000 (tid 100198, pid 1232, "lockf") 0xc380fa20 (tid 100066, pid 1228, "thr1") 0xc35db870 (tid 100055, pid 1202, "creat") 0xc35dba20 (tid 100054, pid 1183, "creat") 0xc38e4510 (tid 100109, pid 1195, "creat") 0xc37cb510 (tid 100099, pid 1185, "creat") 0xc3835d80 (tid 100117, pid 1201, "creat") 0xc5a96360 (tid 100171, pid 1188, "lockf") 0xc5a96510 (tid 100170, pid 1189, "lockf") 0xc5483510 (tid 100140, pid 1186, "lockf") 0xc380f000 (tid 100072, pid 1200, "creat") 0xc5483870 (tid 100138, pid 1170, "mkdir") 0xc380fd80 (tid 100064, pid 1197, "creat") 0xc37cb870 (tid 100062, pid 1198, "creat") 0xc380f510 (tid 100069, pid 1203, "creat") 0xc3835870 (tid 100120, pid 1187, "creat") 0xc38e4000 (tid 100189, pid 1251, "chmod") 0xc5a68d80 (tid 100158, pid 1252, "chmod") 0xc3831000 (tid 100085, pid 1253, "chmod") 0xc5857d80 (tid 100249, pid 1864, "ls") db> allt Tracing command md4 pid 1962 tid 100230 td 0xc5863000 sched_switch(c5863000,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c3d04000,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd sleepq_wait(c3d04000,0,c08e860e,28f,8) at sleepq_wait+0x46 msleep(c3d04000,c3d04020,24c,c08e864f,0) at msleep+0x27d md_kthread(c3d04000,e620fd38) at md_kthread+0xb1 fork_exit(c058250c,c3d04000,e620fd38) at fork_exit+0xa4 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xe620fd6c, ebp = 0 --- Tracing command md3 pid 1955 tid 100211 td 0xc5864a20 sched_switch(c5864a20,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c4905000,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd sleepq_wait(c4905000,0,c08e860e,28f,8) at sleepq_wait+0x46 msleep(c4905000,c4905020,24c,c08e864f,0) at msleep+0x27d md_kthread(c4905000,e623cd38) at md_kthread+0xb1 fork_exit(c058250c,c4905000,e623cd38) at fork_exit+0xa4 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xe623cd6c, ebp = 0 --- Tracing command md2 pid 1948 tid 100237 td 0xc5861360 sched_switch(c5861360,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5074000,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd sleepq_wait(c5074000,0,c08e860e,28f,8) at sleepq_wait+0x46 msleep(c5074000,c5074020,24c,c08e864f,0) at msleep+0x27d md_kthread(c5074000,e61fad38) at md_kthread+0xb1 fork_exit(c058250c,c5074000,e61fad38) at fork_exit+0xa4 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xe61fad6c, ebp = 0 --- Tracing command md1 pid 1941 tid 100222 td 0xc5863d80 sched_switch(c5863d80,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c4f85800,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd sleepq_wait(c4f85800,0,c08e860e,28f,8) at sleepq_wait+0x46 msleep(c4f85800,c4f85820,24c,c08e864f,0) at msleep+0x27d md_kthread(c4f85800,e6227d38) at md_kthread+0xb1 fork_exit(c058250c,c4f85800,e6227d38) at fork_exit+0xa4 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xe6227d6c, ebp = 0 --- Tracing command ls pid 1864 tid 100249 td 0xc5857d80 sched_switch(c5857d80,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c37be06c,c09f2f30,0,c090a4e8,211) at sleepq_switch+0xcd sleepq_wait(c37be06c,0,c0911f49,37c,8) at sleepq_wait+0x46 msleep(c37be06c,c37be024,9f,c0911f83,0) at msleep+0x27d vn_start_write(c38aa514,e61d6b18,1,c37be000,e61d6b28) at vn_start_write+0xbf vn_close(c38aa514,5,c580fa80,c5857d80,e61d6b80) at vn_close+0x4f vn_closefile(c37ffb88,c5857d80) at vn_closefile+0xca fdrop_locked(c37ffb88,c5857d80,c3411640,0,c0903d2b) at fdrop_locked+0x88 fdrop(c37ffb88,c5857d80,6b5,c09f8f54,0) at fdrop+0x24 closef(c37ffb88,c5857d80,0,0,4) at closef+0x367 close(c5857d80,e61d6d04,c5936000,c,c5857d80) at close+0x1be syscall(3b,3b,3b,82401c0,804e510) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (6, FreeBSD ELF32, close), eip = 0x2819e103, esp = 0xbfbfe48c, ebp = 0xbfbfe4b8 --- Tracing command bash pid 1858 tid 100216 td 0xc58641b0 sched_switch(c58641b0,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c584bd38) at sleepq_switch+0xcd sleepq_wait_sig(c584bd38,100,c0904595,385,c5936068) at sleepq_wait_sig+0x1d msleep(c584bd38,c584bda0,15c,c090c5fb,0) at msleep+0x26e kern_wait(c58641b0,ffffffff,e622dc2c,6,0) at kern_wait+0x7cf wait4(c58641b0,e622dd04,c584bd38,c,c58641b0) at wait4+0x2a syscall(839003b,3b,bfbf003b,1,0) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (7, FreeBSD ELF32, wait4), eip = 0x281c2b4b, esp = 0xbfbfe89c, ebp = 0xbfbfe8b8 --- Tracing command sshd pid 1857 tid 100188 td 0xc5ab36c0 sched_switch(c5ab36c0,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c0a3f69c) at sleepq_switch+0xcd sleepq_wait_sig(c0a3f69c,246,c09f2f30,c090c05f,2fc) at sleepq_wait_sig+0x1d cv_wait_sig(c0a3f69c,c0a3f684,4,e6077b4c,0) at cv_wait_sig+0x18d kern_select(c5ab36c0,b,8201168,820116c,0) at kern_select+0x4bf select(c5ab36c0,e6077d04,0,e6077d00,c069a325) at select+0x44 syscall(3b,3b,bfbf003b,0,bfbfde5c) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (93, FreeBSD ELF32, select), eip = 0x283560c3, esp = 0xbfbfddec, ebp = 0xbfbfde38 --- Tracing command sshd pid 1855 tid 100248 td 0xc5860000 sched_switch(c5860000,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5b00718) at sleepq_switch+0xcd sleepq_wait_sig(c5b00718,100,c090e928,4e8,c06b6f99) at sleepq_wait_sig+0x1d msleep(c5b00718,c5b006f0,158,c090ec76,0) at msleep+0x26e sbwait(c5b006cc,c0688b69,c0a3f5a0,e61d9bb4,c06b7d6c) at sbwait+0x4b soreceive(c5b0067c,0,e61d9c64,0,0) at soreceive+0x2da soo_read(c5704e10,e61d9c64,c594cd00,0,c5860000) at soo_read+0x41 dofileread(c5860000,5,c5704e10,e61d9c64,ffffffff) at dofileread+0x89 kern_readv(c5860000,5,e61d9c64,bfbfde48,4) at kern_readv+0x36 read(c5860000,e61d9d04,e61d9ccc,c0697d80,c5860000) at read+0x45 syscall(3b,3b,bfbf003b,4,0) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (3, FreeBSD ELF32, read), eip = 0x28356143, esp = 0xbfbfddec, ebp = 0xbfbfde18 --- Tracing command chmod pid 1253 tid 100085 td 0xc3831000 sched_switch(c3831000,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c37be06c) at sleepq_switch+0xcd sleepq_wait_sig(c37be06c,100,c0911f49,37c,8) at sleepq_wait_sig+0x1d msleep(c37be06c,c37be024,19f,c0911f83,0) at msleep+0x26e vn_start_write(c5bcfb2c,e5ebeb24,101,c37be000,c0a3f56c) at vn_start_write+0xbf setfmode(c3831000,c5bcfb2c,41ff,e5ebec10,ffffffdf) at setfmode+0x24 kern_chmod(c3831000,2817fb00,0,41ff,e5ebed30) at kern_chmod+0x5b chmod(c3831000,e5ebed04,c3833234,c,c3831000) at chmod+0x15 syscall(3b,3b,3b,28183f54,41ff) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (15, FreeBSD ELF32, chmod), eip = 0x280cd44f, esp = 0xbfbfe80c, ebp = 0xbfbfe898 --- Tracing command chmod pid 1252 tid 100158 td 0xc5a68d80 sched_switch(c5a68d80,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c37be06c) at sleepq_switch+0xcd sleepq_wait_sig(c37be06c,100,c0911f49,37c,8) at sleepq_wait_sig+0x1d msleep(c37be06c,c37be024,19f,c0911f83,0) at msleep+0x26e vn_start_write(c49da820,e5ffcb24,101,c37be000,c0a3f25c) at vn_start_write+0xbf setfmode(c5a68d80,c49da820,41ff,e5ffcc10,ffffffdf) at setfmode+0x24 kern_chmod(c5a68d80,2817fb00,0,41ff,e5ffcd30) at kern_chmod+0x5b chmod(c5a68d80,e5ffcd04,c5a6c8d0,c,c5a68d80) at chmod+0x15 syscall(3b,3b,3b,28183f54,41ff) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (15, FreeBSD ELF32, chmod), eip = 0x280cd44f, esp = 0xbfbfe80c, ebp = 0xbfbfe898 --- Tracing command chmod pid 1251 tid 100189 td 0xc38e4000 sched_switch(c38e4000,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c37be06c) at sleepq_switch+0xcd sleepq_wait_sig(c37be06c,100,c0911f49,37c,8) at sleepq_wait_sig+0x1d msleep(c37be06c,c37be024,19f,c0911f83,0) at msleep+0x26e vn_start_write(c49da104,e5f39b24,101,c37be000,c0a3f224) at vn_start_write+0xbf setfmode(c38e4000,c49da104,41ff,e5f39c10,ffffffdf) at setfmode+0x24 kern_chmod(c38e4000,2817fb00,0,41ff,e5f39d30) at kern_chmod+0x5b chmod(c38e4000,e5f39d04,c5ab8000,c,c38e4000) at chmod+0x15 syscall(3b,3b,3b,28183f54,41ff) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (15, FreeBSD ELF32, chmod), eip = 0x280cd44f, esp = 0xbfbfe80c, ebp = 0xbfbfe898 --- Tracing command find pid 1250 tid 100173 td 0xc5a96000 sched_switch(c5a96000,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5a97b04) at sleepq_switch+0xcd sleepq_wait_sig(c5a97b04,100,c0904595,385,c383329c) at sleepq_wait_sig+0x1d msleep(c5a97b04,c5a97b6c,15c,c090c5fb,0) at msleep+0x26e kern_wait(c5a96000,4e5,e6050c2c,0,0) at kern_wait+0x7cf wait4(c5a96000,e6050d04,c5a97b04,c,c5a96000) at wait4+0x2a syscall(3b,bfbf003b,bfbf003b,0,28194fd0) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e8b4b, esp = 0xbfbfe79c, ebp = 0xbfbfe7b8 --- Tracing command find pid 1249 tid 100174 td 0xc5a95d80 sched_switch(c5a95d80,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5a97d38) at sleepq_switch+0xcd sleepq_wait_sig(c5a97d38,100,c0904595,385,c5ab8068) at sleepq_wait_sig+0x1d msleep(c5a97d38,c5a97da0,15c,c090c5fb,0) at msleep+0x26e kern_wait(c5a95d80,4e3,e604dc2c,0,0) at kern_wait+0x7cf wait4(c5a95d80,e604dd04,c5a97d38,c,c5a95d80) at wait4+0x2a syscall(3b,3b,3b,0,28194fd0) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e8b4b, esp = 0xbfbfe79c, ebp = 0xbfbfe7b8 --- Tracing command find pid 1248 tid 100190 td 0xc38e3d80 sched_switch(c38e3d80,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5ab8234) at sleepq_switch+0xcd sleepq_wait_sig(c5ab8234,100,c0904595,385,c5a6c938) at sleepq_wait_sig+0x1d msleep(c5ab8234,c5ab829c,15c,c090c5fb,0) at msleep+0x26e kern_wait(c38e3d80,4e4,e5f36c2c,0,0) at kern_wait+0x7cf wait4(c38e3d80,e5f36d04,c5ab8234,c,c38e3d80) at wait4+0x2a syscall(3b,3b,3b,0,28194fd0) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280e8b4b, esp = 0xbfbfe79c, ebp = 0xbfbfe7b8 --- Tracing command sh pid 1247 tid 100191 td 0xc38e3bd0 sched_switch(c38e3bd0,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5ab8468) at sleepq_switch+0xcd sleepq_wait_sig(c5ab8468,100,c0904595,385,c5a97b6c) at sleepq_wait_sig+0x1d msleep(c5ab8468,c5ab84d0,15c,c090c5fb,0) at msleep+0x26e kern_wait(c38e3bd0,ffffffff,e5f33c2c,2,0) at kern_wait+0x7cf wait4(c38e3bd0,e5f33d04,c5ab8468,c,c38e3bd0) at wait4+0x2a syscall(2809003b,3b,bfbf003b,bfbfe628,432) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2814cb4b, esp = 0xbfbfe5dc, ebp = 0xbfbfe5f8 --- Tracing command sh pid 1246 tid 100135 td 0xc5483d80 sched_switch(c5483d80,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5485468) at sleepq_switch+0xcd sleepq_wait_sig(c5485468,100,c0904595,385,c5a97da0) at sleepq_wait_sig+0x1d msleep(c5485468,c54854d0,15c,c090c5fb,0) at msleep+0x26e kern_wait(c5483d80,ffffffff,e5fa8c2c,2,0) at kern_wait+0x7cf wait4(c5483d80,e5fa8d04,c5485468,c,c5483d80) at wait4+0x2a syscall(3b,3b,3b,bfbfe628,432) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2814cb4b, esp = 0xbfbfe5dc, ebp = 0xbfbfe5f8 --- Tracing command sh pid 1245 tid 100192 td 0xc38e3a20 sched_switch(c38e3a20,0,1) at sched_switch+0x183 mi_switch(1,0) at mi_switch+0x280 sleepq_switch(c5ab869c) at sleepq_switch+0xcd sleepq_wait_sig(c5ab869c,100,c0904595,385,c5ab829c) at sleepq_wait_sig+0x1d msleep(c5ab869c,c5ab8704,15c,c090c5fb,0) at msleep+0x26e kern_wait(c38e3a20,ffffffff,e5f30c2c,2,0) at kern_wait+0x7cf wait4(c38e3a20,e5f30d04,c5ab869c,c,c38e3a20) at wait4+0x2a syscall(3b,2821003b,bfbf003b,bfbfe628,432) at syscall+0x27e Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2814cb4b, esp = 0xbfbfe5dc, ebp = 0xbfbfe5f8 --- Tracing command lockf pid 1241 tid 100176 td 0xc35db510 db> call doadump Physical memory: 1015 MB Dumping 113 MB: 98 82 66 50 34 18 2 Dump complete = 0xf db> reset (kgdb) p *(struct vnode *)0xc38aa514 $3 = {v_type = VDIR, v_tag = 0xc090f332 "ufs", v_op = 0xc09c1bc0, v_data = 0xc37fb294, v_mount = 0xc37be000, v_nmntvnodes = {tqe_next = 0xc38aa410, tqe_prev = 0xc37be058}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0}, v_hashlist = {le_next = 0x0, le_prev = 0xc3699e88}, v_hash = 0x2, v_cache_src = { lh_first = 0xc3d0d7f8}, v_cache_dst = {tqh_first = 0x0, tqh_last = 0xc38aa544}, v_dd = 0x0, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0, v_clen = 0x0, v_lock = { lk_interlock = 0xc09f2c78, lk_flags = 0x40, lk_sharecount = 0x0, lk_waitcount = 0x0, lk_exclusivecount = 0x0, lk_prio = 0x50, lk_wmesg = 0xc090f332 "ufs", lk_timo = 0x33, lk_lockholder = 0xffffffff, lk_newlock = 0x0}, v_interlock = {mtx_object = { lo_name = 0xc090b5c5 "vnode interlock", lo_type = 0xc090b5c5 "vnode interlock", lo_flags = 0x1030000, lo_witness_data = {lod_list = {stqe_next = 0xc0a02990}, lod_witness = 0xc0a02990}}, mtx_lock = 0x4, mtx_recurse = 0x0}, v_vnlock = 0xc38aa56c, v_holdcnt = 0x6, v_usecount = 0x3, v_iflag = 0x20, v_vflag = 0x1, v_writecount = 0x0, v_freelist = {tqe_next = 0x0, tqe_prev = 0xc37f9bd8}, v_bufobj = {bo_mtx = 0xc38aa590, bo_clean = {bv_hd = { tqh_first = 0xd751fa38, tqh_last = 0xd751fa70}, bv_root = 0xd751fa38, bv_cnt = 0x1}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc38aa5dc}, bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x0, bo_ops = 0xc09b52a4, bo_bsize = 0x4000, bo_object = 0xc382ce10, bo_synclist = { le_next = 0x0, le_prev = 0xc360fb04}, bo_private = 0xc38aa514, __bo_vnode = 0xc38aa514}, v_pollinfo = 0x0, v_label = 0x0} (kgdb) p *(*(struct vnode *)0xc38aa514)->v_mount $4 = {mnt_lock = {lk_interlock = 0xc09f2420, lk_flags = 0x1000000, lk_sharecount = 0x1, lk_waitcount = 0x0, lk_exclusivecount = 0x0, lk_prio = 0x50, lk_wmesg = 0xc0910d99 "vfslock", lk_timo = 0x0, lk_lockholder = 0xffffffff, lk_newlock = 0x0}, mnt_mtx = {mtx_object = { lo_name = 0xc0910d88 "struct mount mtx", lo_type = 0xc0910d88 "struct mount mtx", lo_flags = 0x1030000, lo_witness_data = {lod_list = {stqe_next = 0xc0a01220}, lod_witness = 0xc0a01220}}, mtx_lock = 0x4, mtx_recurse = 0x0}, mnt_list = { tqe_next = 0xc37b7ca8, tqe_prev = 0xc37be2c4}, mnt_op = 0xc09c1860, mnt_vfc = 0xc09c18a0, mnt_vnodecovered = 0xc3807d34, mnt_syncer = 0xc38aa410, mnt_ref = 0x297, mnt_nvnodelist = {tqh_first = 0xc38aa514, tqh_last = 0xc5921938}, mnt_nvnodelistsize = 0x27b, mnt_writeopcount = 0x1, mnt_kern_flag = 0x28000000, mnt_flag = 0x1211300, mnt_opt = 0xc4dc0170, mnt_optnew = 0xc3849a70, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x7, f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0xf20a7, f_bfree = 0x635de, f_bavail = 0x5000f, f_files = 0x3f3fe, f_ffree = 0x389e4, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x434f6181, 0xa1e353a3}}, f_charspare = '\0' <repeats 79 times>, f_fstypename = "ufs", '\0' <repeats 12 times>, f_mntfromname = "/dev/ad0s1e", '\0' <repeats 76 times>, f_mntonname = "/tmp", '\0' <repeats 83 times>}, mnt_cred = 0xc371e900, mnt_data = 0xc3816700, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc5a3a200, mnt_mntlabel = 0x0, mnt_fslabel = 0x0, mnt_hashseed = 0xa30717a0, mnt_markercnt = 0x0, mnt_holdcnt = 0x0, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0, mnt_secondary_accwrites = 0xfbb6}