GENERIC HEAD from Oct 2 08:57 UTC, vmcore.459
$ INCARNATIONS=100 ./run.sh ./mkdir.cfg
run: run time 3+00:00:00, incarnations 1, load 100, verbose 1
13:07:32 Loop #1
mkdir: run time 0+00:02:00, incarnations 4, load 100, verbose 1
13:09:41 Loop #2
mkdir: run time 0+00:02:00, incarnations 83, load 100, verbose 1
13:12:10 Loop #3
mkdir: run time 0+00:02:00, incarnations 64, load 100, verbose 1
13:14:21 Loop #4
mkdir: run time 0+00:02:00, incarnations 92, load 100, verbose 1
13:17:38 Loop #5
mkdir: run time 0+00:02:00, incarnations 41, load 100, verbose 1
13:20:46 Loop #6
mkdir: run time 0+00:02:00, incarnations 40, load 100, verbose 1
13:23:05 Loop #7
mkdir: run time 0+00:02:00, incarnations 33, load 100, verbose 1
13:26:22 Loop #8
mkdir: run time 0+00:02:00, incarnations 46, load 100, verbose 1
13:29:53 Loop #9
mkdir: run time 0+00:02:00, incarnations 14, load 100, verbose 1
13:32:39 Loop #10
mkdir: run time 0+00:02:00, incarnations 26, load 100, verbose 1
13:35:55 Loop #11
mkdir: run time 0+00:02:00, incarnations 81, load 100, verbose 1
13:39:17 Loop #12
mkdir: run time 0+00:02:00, incarnations 10, load 100, verbose 1
13:41:23 Loop #13
mkdir: run time 0+00:02:00, incarnations 29, load 100, verbose 1
13:44:42 Loop #14
mkdir: run time 0+00:02:00, incarnations 92, load 100, verbose 1
13:52:36 Loop #15
mkdir: run time 0+00:02:00, incarnations 42, load 100, verbose 1
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Sun Oct 2 11:44:35 CEST 2005
pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf13 Stepping = 3
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory = 267583488 (255 MB)
avail memory = 251998208 (240 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
rl0: link state changed to DOWN
freebsd4_sigreturn: eflags = 0x0
freebsd4_sigreturn: eflags = 0x0
freebsd4_sigreturn: eflags = 0x0
ad0: FAILURE - out of memory in start
/tmp: mount pending error: blocks -20 files -54
/tmp: mount pending error: blocks -20 files -54
/tmp: mount pending error: blocks -20 files -54
:
/tmp: mount pending error: blocks -20 files -54
/tmp: mount pending error: blocks -20 files -54
lock order reversal:
1st 0xc17f4e60 vnode interlock (vnode interlock) @ ufs/ffs/ffs_snapshot.c:2036
2nd 0xc1060084 system map (system map) @ vm/vm_map.c:2997
KDB: stack backtrace:
kdb_backtrace(0,ffffffff,c0972e50,c0972ea0,c08cace4) at kdb_backtrace+0x29
witness_checkorder(c1060084,9,c08803fa,bb5) at witness_checkorder+0x578
_mtx_lock_flags(c1060084,0,c08803f1,bb5) at _mtx_lock_flags+0x5b
_vm_map_lock_read(c1060000,c08803f1,bb5,1647742,c) at _vm_map_lock_read+0x24
vm_map_lookup(cf51297c,deadc000,1,cf512980,cf512970) at vm_map_lookup+0x28
vm_fault(c1060000,deadc000,1,0,c2885600) at vm_fault+0x66
trap_pfault(cf512a44,0,deadc0e6) at trap_pfault+0x137
trap(c0870008,cf510028,c0620028,deadc0de,5) at trap+0x341
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc076d200, esp = 0xcf512a84, ebp = 0xcf512acc ---
ffs_copyonwrite(c17f4d98,c668fc38) at ffs_copyonwrite+0x1ac
ffs_geom_strategy(c17f4ea4,c668fc38) at ffs_geom_strategy+0x89
bufwrite(c668fc38,0,0,cf512b64,c07663d9) at bufwrite+0x172
ffs_bufwrite(c668fc38) at ffs_bufwrite+0x2ae
ffs_update(c1ff06cc,0,c1ff06cc,c176f400,1) at ffs_update+0x3b9
ufs_inactive(cf512bb8,c1ff0794,c1ff06cc,cf512bd0,c068b2e1) at ufs_inactive+0x184
VOP_INACTIVE_APV(c09051a0,cf512bb8) at VOP_INACTIVE_APV+0x9b
vinactive(c1ff06cc,c2885600) at vinactive+0x8d
vput(c1ff06cc) at vput+0x167
kern_rmdir(c2885600,bfbe3e40,0,cf512d30,c07f978b) at kern_rmdir+0x200
rmdir(c2885600,cf512d04,1,c,292) at rmdir+0x12
syscall(3b,3b,3b,28050288,bfbfea7c) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280b6c0b, esp = 0xbfbe39fc, ebp = 0xbfbe3e28 ---
Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address = 0xdeadc0e6
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc076d200
stack pointer = 0x28:0xcf512a84
frame pointer = 0x28:0xcf512acc
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 9942 (mkdir)
[thread pid 9942 tid 100219 ]
Stopped at ffs_copyonwrite+0x1ac: movl 0x8(%edi),%edx
db> where
Tracing pid 9942 tid 100219 td 0xc2885600
ffs_copyonwrite(c17f4d98,c668fc38) at ffs_copyonwrite+0x1ac
ffs_geom_strategy(c17f4ea4,c668fc38) at ffs_geom_strategy+0x89
bufwrite(c668fc38,0,0,cf512b64,c07663d9) at bufwrite+0x172
ffs_bufwrite(c668fc38) at ffs_bufwrite+0x2ae
ffs_update(c1ff06cc,0,c1ff06cc,c176f400,1) at ffs_update+0x3b9
ufs_inactive(cf512bb8,c1ff0794,c1ff06cc,cf512bd0,c068b2e1) at ufs_inactive+0x184
VOP_INACTIVE_APV(c09051a0,cf512bb8) at VOP_INACTIVE_APV+0x9b
vinactive(c1ff06cc,c2885600) at vinactive+0x8d
vput(c1ff06cc) at vput+0x167
kern_rmdir(c2885600,bfbe3e40,0,cf512d30,c07f978b) at kern_rmdir+0x200
rmdir(c2885600,cf512d04,1,c,292) at rmdir+0x12
syscall(3b,3b,3b,28050288,bfbfea7c) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280b6c0b, esp = 0xbfbe39fc, ebp = 0xbfbe3e28 ---
db> show lockedvnods
Locked vnodes
0xc2435c3c: tag ufs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b2e480 (pid 10028)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc0680512 at cache_lookup+0x3c2
#8 0xc0680a9a at vfs_cache_lookup+0x92
#9 0xc08095f3 at VOP_LOOKUP_APV+0x87
#10 0xc0684a5a at lookup+0x3e2
#11 0xc0684416 at namei+0x37e
#12 0xc06905aa at kern_unlink+0x3a
#13 0xc069056e at unlink+0x12
#14 0xc07f978b at syscall+0x22f
#15 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 3, on dev ad0s1e
0xc2927c3c: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2885900 (pid 9950)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 48834, on dev ad0s1e
0xc19382b8: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2885780 (pid 9970)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 118152, on dev ad0s1e
0xc2a772b8: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2073000 (pid 9938)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 118392, on dev ad0s1e
0xc23bb6cc: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc186a300 (pid 9967)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 25494, on dev ad0s1e
0xc1c63ae0: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e9f000 (pid 9971)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 25517, on dev ad0s1e
0xc2a336cc: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc20d0300 (pid 9934)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 118721, on dev ad0s1e
0xc1ba46cc: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b99900 (pid 9952)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 118954, on dev ad0s1e
0xc2a35ae0: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b9b600 (pid 9936)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119030, on dev ad0s1e
0xc1d726cc: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e65c00 (pid 9947)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119126, on dev ad0s1e
0xc2b05414: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e9fa80 (pid 9953)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119146, on dev ad0s1e
0xc1763ae0: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc20d0480 (pid 9941)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119403, on dev ad0s1e
0xc1dcb414: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e9f900 (pid 9964)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119421, on dev ad0s1e
0xc24542b8: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e9f480 (pid 9948)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119522, on dev ad0s1e
0xc1be0570: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc20d0000 (pid 9946)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119589, on dev ad0s1e
0xc2724000: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b55480 (pid 9958)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119682, on dev ad0s1e
0xc270f000: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc17ca300 (pid 9951)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119705, on dev ad0s1e
0xc2329414: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2065d80 (pid 9930)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119713, on dev ad0s1e
0xc24bfae0: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2065000 (pid 9944)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119731, on dev ad0s1e
0xc218f15c: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b2ec00 (pid 9963)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119883, on dev ad0s1e
0xc2944414: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e9f180 (pid 9929)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 119918, on dev ad0s1e
0xc22cb2b8: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e64d80 (pid 9959)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 49150, on dev ad0s1e
0xc2843ae0: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2065480 (pid 9949)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 49448, on dev ad0s1e
0xc22ca414: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc186b300 (pid 9932)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 95461, on dev ad0s1e
0xc25de000: tag ufs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc165fa80 (pid 56)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc0771b1e at handle_workitem_remove+0x2e
#10 0xc076dbc1 at process_worklist_item+0x181
#11 0xc076d928 at softdep_process_worklist+0xf4
#12 0xc068a76e at sched_sync+0x266
#13 0xc0622c2c at fork_exit+0xa0
#14 0xc07e6a1c at fork_trampoline+0x8
ino 118958, on dev ad0s1e
0xc2575000: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc186a900 (pid 9954)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 118970, on dev ad0s1e
0xc1cfe414: tag ufs, type VREG
usecount 0, writecount 0, refcount 529 mountedhere 0
flags (VV_SYSTEM)
v_object 0xc1eaf4a4 ref 0 pages 1464
lock type ufs: EXCL (count 1) by thread 0xc1b2e480 (pid 10028)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc076bcc5 at ffs_snapremove+0x4d
#2 0xc07719bc at softdep_releasefile+0x34
#3 0xc077e6d3 at ufs_inactive+0x67
#4 0xc080b32f at VOP_INACTIVE_APV+0x9b
#5 0xc068b2e1 at vinactive+0x8d
#6 0xc068b11b at vput+0x167
#7 0xc069070e at kern_unlink+0x19e
#8 0xc069056e at unlink+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 20, on dev ad0s1e
0xc2066570: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2885180 (pid 9937)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 26009, on dev ad0s1e
0xc1ff06cc: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2885600 (pid 9942)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 26021, on dev ad0s1e
0xc2433984: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc17ca480 (pid 9943)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 49626, on dev ad0s1e
0xc1cc2d98: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc20d0d80 (pid 9955)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 49634, on dev ad0s1e
0xc1de4c3c: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2886a80 (pid 9945)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 72137, on dev ad0s1e
0xc2078c3c: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b55d80 (pid 9931)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 72147, on dev ad0s1e
0xc29cbae0: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b55900 (pid 9966)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 72162, on dev ad0s1e
0xc2468570: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1864180 (pid 9962)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 72166, on dev ad0s1e
0xc26f8ae0: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc186a000 (pid 9961)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 72179, on dev ad0s1e
0xc20e5000: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e65180 (pid 9957)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 72187, on dev ad0s1e
0xc257c984: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1b99a80 (pid 9935)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 95687, on dev ad0s1e
0xc21d82b8: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e9e480 (pid 9968)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 95724, on dev ad0s1e
0xc2a34570: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2886900 (pid 9965)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 120365, on dev ad0s1e
0xc243d828: tag ufs, type VDIR
usecount 0, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1f9e480 (pid 9956)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 120383, on dev ad0s1e
0xc1e1a15c: tag ufs, type VDIR
usecount 2, writecount 0, refcount 3 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc2073480 (pid 9933)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc0684746 at lookup+0xce
#7 0xc0684416 at namei+0x37e
#8 0xc0692fd6 at kern_rmdir+0x3a
#9 0xc0692f9a at rmdir+0x12
#10 0xc07f978b at syscall+0x22f
#11 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 72468, on dev ad0s1e
0xc22a1570: tag ufs, type VDIR
usecount 2, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 2) by thread 0xc2073480 (pid 9933)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 72469, on dev ad0s1e
0xc1fad414: tag ufs, type VDIR
usecount 0, writecount 0, refcount 1 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e65480 (pid 9940)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 95896, on dev ad0s1e
0xc249a414: tag ufs, type VDIR
usecount 2, writecount 0, refcount 3 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc1e64c00 (pid 9960)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc0684746 at lookup+0xce
#7 0xc0684416 at namei+0x37e
#8 0xc0692fd6 at kern_rmdir+0x3a
#9 0xc0692f9a at rmdir+0x12
#10 0xc07f978b at syscall+0x22f
#11 0xc07e6a0f at Xint0x80_syscall+0x1f
ino 120515, on dev ad0s1e
0xc2666828: tag ufs, type VDIR
usecount 2, writecount 0, refcount 3 mountedhere 0
flags ()
lock type ufs: EXCL (count 2) by thread 0xc1e64c00 (pid 9960)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f
ino 120516, on dev ad0s1e
db> show lockedbufs
buf at 0xc65e636c
b_flags = 0xa0020020<remfree,vmio,clusterok,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc17f4ea4), b_data = 0xc66ef000, b_blkno = 754080
b_npages = 4, pages(OBJ, IDX, PA): (0xc17cc6b4, 0x17034, 0x398a000),(0xc17cc6b4, 0x17035, 0xa0eb000),(0xc17cc6b4, 0x17036, 0x44ac000),(0xc17cc6b4, 0x17037, 0xb62d000)
lock type getblk: EXCL (count 1) by thread 0xc2065480 (pid 9949)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc067da87 at getblk+0x15b
#2 0xc067aaf7 at breadn+0x5b
#3 0xc067aa98 at bread+0x20
#4 0xc0766248 at ffs_update+0x228
#5 0xc077e7f0 at ufs_inactive+0x184
#6 0xc080b32f at VOP_INACTIVE_APV+0x9b
#7 0xc068b2e1 at vinactive+0x8d
#8 0xc068b11b at vput+0x167
#9 0xc069319c at kern_rmdir+0x200
#10 0xc0692f9a at rmdir+0x12
#11 0xc07f978b at syscall+0x22f
#12 0xc07e6a0f at Xint0x80_syscall+0x1f
buf at 0xc65e69bc
b_flags = 0xa0020020<remfree,vmio,clusterok,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc17f4ea4), b_data = 0xc66ff000, b_blkno = 1506240
b_npages = 4, pages(OBJ, IDX, PA): (0xc17cc6b4, 0x2df78, 0xea2e000),(0xc17cc6b4, 0x2df79, 0x844f000),(0xc17cc6b4, 0x2df7a, 0xc420000),(0xc17cc6b4, 0x2df7b, 0xa6f1000)
lock type getblk: EXCL (count 1) by thread 0xc186b300 (pid 9932)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc067da87 at getblk+0x15b
#2 0xc067aaf7 at breadn+0x5b
#3 0xc067aa98 at bread+0x20
#4 0xc0766248 at ffs_update+0x228
#5 0xc077e7f0 at ufs_inactive+0x184
#6 0xc080b32f at VOP_INACTIVE_APV+0x9b
#7 0xc068b2e1 at vinactive+0x8d
#8 0xc068b11b at vput+0x167
#9 0xc069319c at kern_rmdir+0x200
#10 0xc0692f9a at rmdir+0x12
#11 0xc07f978b at syscall+0x22f
#12 0xc07e6a0f at Xint0x80_syscall+0x1f
buf at 0xc65f3878
b_flags = 0x80020024<remfree,clusterok,cache,async>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc1cfe520), b_data = 0xc690b000, b_blkno = 101216
b_npages = 4, pages(OBJ, IDX, PA): (0xc09be160, 0x6a0c, 0x804c000),(0xc09be160, 0x6a0d, 0xaded000),(0xc09be160, 0x6a0e, 0x36ae000),(0xc09be160, 0x6a0f, 0x836f000)
db> show alllocks
Process 9960 (mkdir) thread 0xc1e64c00 (100135)
exclusive sleep mutex vnode interlock r = 0 (0xc26668f0) locked @ ufs/ffs/ffs_softdep.c:2160
Process 9942 (mkdir) thread 0xc2885600 (100219)
exclusive sleep mutex vnode interlock r = 0 (0xc17f4e60) locked @ ufs/ffs/ffs_snapshot.c:2036
db> show pcpu
cpuid = 0
curthread = 0xc2885600: pid 9942 "mkdir"
curpcb = 0xcf512d90
fpcurthread = none
idlethread = 0xc155a780: pid 11 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
db> where 9960
Tracing pid 9960 tid 100135 td 0xc1e64c00
sched_switch(c1e64c00,0,2) at sched_switch+0x177
mi_switch(2,0,c0961fa0,0,c086611b) at mi_switch+0x270
critical_exit(c6649570,cf4368fc,c07e6fa0,0,c07e0008) at critical_exit+0x8b
lapic_handle_timer(0) at lapic_handle_timer+0xc9
Xtimerint(c6649570,c26668f0,1,72da8,0) at Xtimerint+0x30
softdep_setup_freeblocks(c25e6e70,0,0,800,12c) at softdep_setup_freeblocks+0x775
ffs_truncate(c2666828,0,0,0,c18ef180,c1e64c00,c09bd880,0,c087cc8f,d76) at ffs_truncate+0x569
handle_workitem_remove(c22d33a0,0,c09bd880,0,c087cc8f,c1c) at handle_workitem_remove+0x19a
softdep_setup_remove(c66a6bec,c264fc60,c25e6e70,1,c25e6e70) at softdep_setup_remove+0xc6
ufs_dirremove(c249a414,c25e6e70,100800c,1,16) at ufs_dirremove+0x12d
ufs_rmdir(cf436c1c,0,c2666828,cf436cc8,c069315a) at ufs_rmdir+0xc3
VOP_RMDIR_APV(c09051a0,cf436c1c) at VOP_RMDIR_APV+0xb5
kern_rmdir(c1e64c00,bfbd7110,0,cf436d30,c07f978b) at kern_rmdir+0x1be
rmdir(c1e64c00,cf436d04,1,19,296) at rmdir+0x12
syscall(2805003b,bfbf003b,bfbe003b,28050288,bfbfea7c) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280b6c0b, esp = 0xbfbd6ccc, ebp = 0xbfbd70f8 ---
db> ps
pid proc uid ppid pgrp flag stat wmesg wchan cmd
10028 c2887a3c 0 10017 10017 0004002 [SLPQ getblk 0xc66bdd94][SLP] rm
10017 c22a0000 0 7472 10017 0004002 [SLPQ wait 0xc22a0000][SLP] sh
9971 c2056624 1001 9928 7535 0000002 [SLPQ getblk 0xc6623964][SLP] mkdir
9970 c2888418 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9969 c2056a3c 1001 9928 7535 0000002 [RUNQ] mkdir
9968 c1fa6624 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9967 c1b16418 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9966 c1e4e20c 1001 9928 7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
9965 c2888000 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9964 c20d3624 1001 9928 7535 0000002 [SLPQ getblk 0xc660bea4][SLP] mkdir
9963 c186320c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9962 c1868418 1001 9928 7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
9961 c22a3418 1001 9928 7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
9960 c1e66418 1001 9928 7535 0000002 [RUNQ] mkdir
9959 c1e4ea3c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9958 c1b15830 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9957 c1e62c48 1001 9928 7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
9956 c1b16624 1001 9928 7535 0000002 [SLPQ getblk 0xc6696654][SLP] mkdir
9955 c205620c 1001 9928 7535 0000002 [SLPQ getblk 0xc662ea24][SLP] mkdir
9954 c2866624 1001 9928 7535 0000002 [SLPQ getblk 0xc6675280][SLP] mkdir
9953 c1b16830 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9952 c1fa620c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9951 c1869830 1001 9928 7535 0000002 [SLPQ getblk 0xc66268bc][SLP] mkdir
9950 c288820c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9949 c2867830 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9948 c1e66c48 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9947 c22a0624 1001 9928 7535 0000002 [SLPQ getblk 0xc661a378][SLP] mkdir
9946 c1e62a3c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9945 c2867c48 1001 9928 7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
9944 c1e66830 1001 9928 7535 0000002 [SLPQ getblk 0xc66268bc][SLP] mkdir
9943 c1869624 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9942 c20d3c48 1001 9928 7535 0000002 [CPU 0] mkdir
9941 c22a020c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9940 c2887418 1001 9928 7535 0000002 [RUNQ] mkdir
9938 c2888624 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9937 c2888c48 1001 9928 7535 0000002 [SLPQ getblk 0xc668fc98][SLP] mkdir
9936 c20d3a3c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9935 c20d320c 1001 9928 7535 0000002 [SLPQ getblk 0xc66bafd0][SLP] mkdir
9934 c2056830 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9933 c20d3830 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9932 c286620c 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9931 c2884418 1001 9928 7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
9930 c1e4d830 1001 9928 7535 0000002 [SLPQ getblk 0xc66268bc][SLP] mkdir
9929 c2884000 1001 9928 7535 0000002 [SLPQ getblk 0xc663cbd0][SLP] mkdir
9928 c22a3000 1001 9926 7535 0000002 [SLPQ wait 0xc22a3000][SLP] mkdir
9926 c1e4e624 1001 7544 7535 0004002 [RUNQ] mkdir
7544 c1b15418 1001 7543 7535 0000002 [SLPQ wait 0xc1b15418][SLP] run
7543 c1b16a3c 1001 7541 7535 0000002 [SLPQ wait 0xc1b16a3c][SLP] run
7541 c22a3624 1001 7535 7535 0004002 [RUNQ] run
7535 c1863418 1001 510 7535 0004002 [SLPQ wait 0xc1863418][SLP] sh
7472 c1e66624 0 7469 7472 0004002 [SLPQ wait 0xc1e66624][SLP] bash
7469 c1e66a3c 0 7468 7469 0004002 [SLPQ pause 0xc1e66a70][SLP] csh
7468 c22a0830 1001 7461 7468 0004102 [SLPQ wait 0xc22a0830][SLP] su
7461 c2866c48 1001 7460 7461 0004002 [SLPQ wait 0xc2866c48][SLP] bash
7460 c1e6220c 1001 7458 7458 0000100 [SLPQ select 0xc09afba4][SLP] sshd
7458 c1b1620c 0 469 7458 0004100 [SLPQ sbwait 0xc1bd4bc8][SLP] sshd
623 c17cb000 1001 1 610 0014003 [SLPQ select 0xc09afba4][SLP] gconfd-2
602 c1863624 0 1 602 0004002 [SLPQ ttyin 0xc16f9010][SLP][SWAP] getty
601 c1863a3c 0 1 601 0004002 [SLPQ ttyin 0xc16f9410][SLP][SWAP] getty
600 c1869000 0 1 600 0004002 [SLPQ ttyin 0xc16f0810][SLP][SWAP] getty
599 c1863830 0 1 599 0004002 [SLPQ ttyin 0xc16ef810][SLP][SWAP] getty
598 c1868624 0 1 598 0004002 [SLPQ ttyin 0xc16f0c10][SLP][SWAP] getty
597 c17c7a3c 0 1 597 0004002 [SLPQ ttyin 0xc16eec10][SLP][SWAP] getty
596 c17cb830 0 1 596 0004002 [SLPQ ttyin 0xc16ee810][SLP][SWAP] getty
595 c1868a3c 0 1 595 0004002 [SLPQ ttyin 0xc16e7410][SLP][SWAP] getty
581 c186820c 0 1 581 0000000 [SLPQ select 0xc09afba4][SLP][SWAP] inetd
510 c1863000 1001 502 510 0004002 [SLPQ wait 0xc1863000][SLP] bash
502 c1868830 1001 475 475 0000100 [SLPQ select 0xc09afba4][SLP] sshd
495 c1863c48 0 1 495 0000000 [SLPQ nanslp 0xc096288c][SLP] cron
483 c1868000 25 1 483 0000100 [SLPQ pause 0xc1868034][SLP] sendmail
477 c1868c48 0 1 477 0000100 [RUNQ] sendmail
475 c17c7830 0 469 475 0004100 [SLPQ sbwait 0xc184d20c][SLP][SWAP] sshd
469 c165e20c 0 1 469 0000100 [SLPQ select 0xc09afba4][SLP] sshd
451 c17cb624 0 1 451 0000000 [RUNQ] ntpd
422 c165e418 0 1 422 0000000 [SLPQ select 0xc09afba4][SLP] usbd
402 c17cb418 0 397 397 0000000 [SLPQ - 0xc1796c00][SLP][SWAP] nfsd
401 c17cba3c 0 397 397 0000000 [SLPQ - 0xc1796e00][SLP][SWAP] nfsd
400 c17c7624 0 397 397 0000000 [SLPQ - 0xc1822000][SLP][SWAP] nfsd
399 c17c7418 0 397 397 0000000 [SLPQ - 0xc1796a00][SLP][SWAP] nfsd
397 c17cb20c 0 1 397 0000000 [SLPQ select 0xc09afba4][SLP][SWAP] nfsd
395 c17cbc48 0 1 395 0000000 [SLPQ select 0xc09afba4][SLP] mountd
326 c17c7c48 0 1 326 0000000 [SLPQ select 0xc09afba4][SLP] rpcbind
298 c17c720c 0 1 298 0000000 [SLPQ select 0xc09afba4][SLP][SWAP] devd
271 c17c7000 0 1 271 0000000 [RUNQ] syslogd
61 c165e624 0 0 0 0000204 [SLPQ - 0xcca21d04][SLP] schedcpu
60 c165e830 0 0 0 0000204 [SLPQ - 0xc09b822c][SLP] nfsiod 3
59 c165ea3c 0 0 0 0000204 [SLPQ - 0xc09b8228][SLP] nfsiod 2
58 c165ec48 0 0 0 0000204 [SLPQ - 0xc09b8224][SLP] nfsiod 1
57 c1771000 0 0 0 0000204 [SLPQ - 0xc09b8220][SLP] nfsiod 0
56 c177120c 0 0 0 0000204 [SLPQ getblk 0xc6675280][SLP] syncer
55 c1771418 0 0 0 0000204 [RUNQ] vnlru
54 c1771624 0 0 0 0000204 [RUNQ] bufdaemon
53 c1771830 0 0 0 000020c [SLPQ pgzero 0xc09be824][SLP] pagezero
52 c1771a3c 0 0 0 0000204 [SLPQ psleep 0xc09be374][SLP] vmdaemon
51 c1771c48 0 0 0 0000204 [SLPQ psleep 0xc09be330][SLP] pagedaemon
50 c15ac624 0 0 0 0000204 [SLPQ - 0xc16a083c][SLP] fdc0
49 c15ac830 0 0 0 0000204 [IWAIT] swi0: sio
48 c15aca3c 0 0 0 0000204 [SLPQ usbevt 0xc1645210][SLP] usb4
47 c15acc48 0 0 0 0000204 [SLPQ usbevt 0xc1695210][SLP] usb3
46 c165d000 0 0 0 0000204 [SLPQ usbevt 0xc1696210][SLP] usb2
45 c165d20c 0 0 0 0000204 [SLPQ usbevt 0xc1682210][SLP] usb1
44 c165d418 0 0 0 0000204 [SLPQ usbtsk 0xc092b544][SLP] usbtask
43 c165d624 0 0 0 0000204 [SLPQ usbevt 0xc164e210][SLP] usb0
9 c165d830 0 0 0 0000204 [SLPQ - 0xc15cb800][SLP] thread taskq
42 c165da3c 0 0 0 0000204 [IWAIT] swi6:+
8 c165dc48 0 0 0 0000204 [SLPQ - 0xc15cb980][SLP] acpi_task2
7 c165e000 0 0 0 0000204 [SLPQ - 0xc15cb980][SLP] acpi_task1
6 c159cc48 0 0 0 0000204 [SLPQ - 0xc15cb980][SLP] acpi_task0
41 c15aa000 0 0 0 0000204 [IWAIT] swi6: task queue
5 c15aa20c 0 0 0 0000204 [SLPQ - 0xc15cbb80][SLP] kqueue taskq
40 c15aa418 0 0 0 0000204 [IWAIT] swi2: cambio
39 c15aa624 0 0 0 0000204 [IWAIT] swi5:+
38 c15aa830 0 0 0 0000204 [SLPQ - 0xc0929260][SLP] yarrow
4 c15aaa3c 0 0 0 0000204 [SLPQ - 0xc092bd88][SLP] g_down
3 c15aac48 0 0 0 0000204 [LOCK vnode interlock c1cdb840] g_up
2 c15ac000 0 0 0 0000204 [SLPQ - 0xc092bd7c][SLP] g_event
37 c15ac20c 0 0 0 0000204 [IWAIT] swi1: net
36 c15ac418 0 0 0 0000204 [IWAIT] swi3: vm
35 c1588624 0 0 0 000020c [IWAIT] swi4: clock sio
34 c1588830 0 0 0 0000204 [IWAIT] irq23: ehci0
33 c1588a3c 0 0 0 0000204 [IWAIT] irq22: rl0
32 c1588c48 0 0 0 0000204 [IWAIT] irq21:
31 c159c000 0 0 0 0000204 [IWAIT] irq20:
30 c159c20c 0 0 0 0000204 [IWAIT] irq19: uhci1
29 c159c418 0 0 0 0000204 [IWAIT] irq18: uhci2
28 c159c624 0 0 0 0000204 [IWAIT] irq17: pcm0
27 c159c830 0 0 0 0000204 [IWAIT] irq16: uhci0 uhci3
26 c159ca3c 0 0 0 0000204 [IWAIT] irq15: ata1
25 c155e20c 0 0 0 0000204 [IWAIT] irq14: ata0
24 c155e418 0 0 0 0000204 [IWAIT] irq13:
23 c155e624 0 0 0 0000204 [IWAIT] irq12:
22 c155e830 0 0 0 0000204 [IWAIT] irq11:
21 c155ea3c 0 0 0 0000204 [IWAIT] irq10:
20 c155ec48 0 0 0 0000204 [IWAIT] irq9: acpi0
19 c1588000 0 0 0 0000204 [IWAIT] irq8:
18 c158820c 0 0 0 0000204 [IWAIT] irq7: ppc0
17 c1588418 0 0 0 0000204 [IWAIT] irq6: fdc0
16 c1559000 0 0 0 0000204 [IWAIT] irq5:
15 c155920c 0 0 0 0000204 [IWAIT] irq4: sio0
14 c1559418 0 0 0 0000204 [IWAIT] irq3:
13 c1559624 0 0 0 0000204 [IWAIT] irq0:
12 c1559830 0 0 0 0000204 [IWAIT] irq1: atkbd0
11 c1559a3c 0 0 0 000020c [Can run] idle: cpu0
1 c1559c48 0 0 1 0004200 [SLPQ wait 0xc1559c48][SLP] init
10 c155e000 0 0 0 0000204 [SLPQ ktrace 0xc0960818][SLP] ktrace
0 c092be80 0 0 0 0000200 [IWAIT] swapper
db> call doadump
Dumping 254 MB (2 chunks)
chunk 0: 1MB (159 pages) ... ok
chunk 1: 255MB (65072 pages) 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 ... ok
Dump complete
= 0xf
db> reset
(kgdb) bt
#0 doadump () at pcpu.h:165
#1 0xc04698f3 in db_fncall (dummy1=0xc09dbd80, dummy2=0x0, dummy3=0x0,
dummy4=0xcf5128a8 "Ô(QÏغ}ÀÀ(QÏÄ(QÏ\220\a") at ../../../ddb/db_command.c:486
#2 0xc04696f8 in db_command (last_cmdp=0xc0914de4, cmd_table=0x0,
aux_cmd_tablep=0xc0890e24, aux_cmd_tablep_end=0xc0890e40)
at ../../../ddb/db_command.c:401
#3 0xc04697c0 in db_command_loop () at ../../../ddb/db_command.c:452
#4 0xc046b365 in db_trap (type=0xc, code=0x0) at ../../../ddb/db_main.c:228
#5 0xc064e32c in kdb_trap (type=0xc, code=0x0, tf=0xcf512a44)
at ../../../kern/subr_kdb.c:473
#6 0xc07f949c in trap_fatal (frame=0xcf512a44, eva=0xdeadc0e6)
at ../../../i386/i386/trap.c:822
#7 0xc07f91e3 in trap_pfault (frame=0xcf512a44, usermode=0x0, eva=0xdeadc0e6)
at ../../../i386/i386/trap.c:742
#8 0xc07f8e2d in trap (frame=
{tf_fs = 0xc0870008, tf_es = 0xcf510028, tf_ds = 0xc0620028, tf_edi = 0xdeadc0de, tf_esi = 0x5, tf_ebp = 0xcf512acc, tf_isp = 0xcf512a70, tf_ebx = 0xc17f4e60, tf_edx = 0xc27f2c00, tf_ecx = 0xc09af6e0, tf_eax = 0x10000, tf_trapno = 0xc, tf_err = 0x0, tf_eip = 0xc076d200, tf_cs = 0x20, tf_eflags = 0x10296, tf_esp = 0xc17f4e60, tf_ss = 0xc17f4e60})
at ../../../i386/i386/trap.c:432
#9 0xc07e69ba in calltrap () at ../../../i386/i386/exception.s:139
#10 0xc0870008 in ?? ()
#11 0xcf510028 in ?? ()
#12 0xc0620028 in exec_free_args (args=0xc17f4e60) at ../../../kern/kern_exec.c:1044
#13 0xc0779669 in ffs_geom_strategy (bo=0xdeadc0de, bp=0xc17f4ea4)
at ../../../ufs/ffs/ffs_vfsops.c:1686
#14 0xc067aeb2 in bufwrite (bp=0xc668fc38) at buf.h:415
#15 0xc07795d2 in ffs_bufwrite (bp=0xc668fc38) at ../../../ufs/ffs/ffs_vfsops.c:1663
#16 0xc07663d9 in ffs_update (vp=0xc1ff06cc, waitfor=0x0) at buf.h:401
#17 0xc077e7f0 in ufs_inactive (ap=0x0) at ../../../ufs/ufs/ufs_inode.c:118
#18 0xc080b32f in VOP_INACTIVE_APV (vop=0xc09056e0, a=0xcf512bb8) at vnode_if.c:1535
#19 0xc068b2e1 in vinactive (vp=0xc1ff06cc, td=0x0) at vnode_if.h:795
#20 0xc068b11b in vput (vp=0xc1ff06cc) at ../../../kern/vfs_subr.c:2107
#21 0xc069319c in kern_rmdir (td=0xc2885600, path=---Can't read userspace from dump, or kernel process---
) at ../../../kern/vfs_syscalls.c:3451
---Type <return> to continue, or q <return> to quit---
#22 0xc0692f9a in rmdir (td=0xc2885600, uap=0x0) at ../../../kern/vfs_syscalls.c:3386
#23 0xc07f978b in syscall (frame=
{tf_fs = 0x3b, tf_es = 0x3b, tf_ds = 0x3b, tf_edi = 0x28050288, tf_esi = 0xbfbfea7c, tf_ebp = 0xbfbe3e28, tf_isp = 0xcf512d64, tf_ebx = 0x1, tf_edx = 0xbfbd68b4, tf_ecx = 0x0, tf_eax = 0x89, tf_trapno = 0x0, tf_err = 0x2, tf_eip = 0x280b6c0b, tf_cs = 0x33, tf_eflags = 0x292, tf_esp = 0xbfbe39fc, tf_ss = 0x3b}) at ../../../i386/i386/trap.c:976
#24 0xc07e6a0f in Xint0x80_syscall () at ../../../i386/i386/exception.s:200
(kgdb) f 12
#12 0xc0620028 in exec_free_args (args=0xc17f4e60) at ../../../kern/kern_exec.c:1044
1044 kmem_free_wakeup(exec_map, (vm_offset_t)args->buf,
(kgdb) p *args
$3 = {buf = 0xc08cace4 "Z\003\210À\t", begin_argv = 0xc08682f2 "vnode interlock",
begin_envv = 0xc08682f2 "vnode interlock", endp = 0x30000---Can't read userspace from dump, or kernel process---
(kgdb) up
#13 0xc0779669 in ffs_geom_strategy (bo=0xdeadc0de, bp=0xc17f4ea4)
at ../../../ufs/ffs/ffs_vfsops.c:1686
1686 if ((vp->v_vflag & VV_COPYONWRITE) &&
(kgdb) info loc
vp = (struct vnode *) 0x5
error = 0x0
(kgdb) l
1681 if ((bp->b_flags & B_VALIDSUSPWRT) == 0 &&
1682 bp->b_vp != NULL && bp->b_vp->v_mount != NULL &&
1683 (bp->b_vp->v_mount->mnt_kern_flag & MNTK_SUSPENDED) != 0)
1684 panic("ffs_geom_strategy: bad I/O");
1685 bp->b_flags &= ~B_VALIDSUSPWRT;
1686 if ((vp->v_vflag & VV_COPYONWRITE) &&
1687 vp->v_rdev->si_snapdata != NULL &&
1688 (error = (ffs_copyonwrite)(vp, bp)) != 0 &&
1689 error != EOPNOTSUPP) {
1690 bp->b_error = error;
(kgdb) up
#14 0xc067aeb2 in bufwrite (bp=0xc668fc38) at buf.h:415
415 BO_STRATEGY(bp->b_bufobj, bp);
(kgdb) p *bp
$4 = {b_bufobj = 0xc17f4ea4, b_bcount = 0x4000, b_caller1 = 0x0,
b_data = 0xc81cb000 "øA\003", b_error = 0x0, b_iocmd = 0x2, b_ioflags = 0x2,
b_iooffset = 0xb878000, b_resid = 0x0, b_iodone = 0, b_blkno = 0x5c3c0,
b_offset = 0xb878000, b_bobufs = {tqe_next = 0xc6623dc0, tqe_prev = 0xc665221c},
b_left = 0xc667ada8, b_right = 0x0, b_vflags = 0x1, b_freelist = {
tqe_next = 0xc6683a1c, tqe_prev = 0xc660ef7c}, b_qindex = 0x2, b_flags = 0xa0020020,
b_xflags = 0x2, b_lock = {lk_interlock = 0xc09616d8, lk_flags = 0x2040000,
lk_sharecount = 0x0, lk_waitcount = 0x1, lk_exclusivecount = 0x1, lk_prio = 0x50,
lk_wmesg = 0xc086c7ec "getblk", lk_timo = 0x0, lk_lockholder = 0xc2885600,
lk_newlock = 0x0, lk_stack = {depth = 0xd, pcs = {0xc062b6c1, 0xc067da87,
0xc067aaf7, 0xc067aa98, 0xc0766248, 0xc077e7f0, 0xc080b32f, 0xc068b2e1,
0xc068b11b, 0xc069319c, 0xc0692f9a, 0xc07f978b, 0xc07e6a0f, 0x0, 0x0, 0x0, 0x0,
0x0}}}, b_bufsize = 0x4000, b_runningbufspace = 0x0,
b_kvabase = 0xc81cb000 "øA\003", b_kvasize = 0x4000, b_lblkno = 0x5c3c0,
b_vp = 0xc17f4d98, b_dirtyoff = 0x0, b_dirtyend = 0x0, b_rcred = 0x0, b_wcred = 0x0,
b_saveaddr = 0xc81cb000, b_pager = {pg_reqpage = 0x0}, b_cluster = {cluster_head = {
tqh_first = 0x0, tqh_last = 0xc6688224}, cluster_entry = {tqe_next = 0x0,
tqe_prev = 0xc6688224}}, b_pages = {0xc14067a8, 0xc13f74f0, 0xc13eca38,
0xc14cbf80, 0x0 <repeats 28 times>}, b_npages = 0x4, b_dep = {lh_first = 0xc2116480}}
(kgdb) l *0xc076d200
0xc076d200 is in ffs_copyonwrite (../../../ufs/ffs/ffs_snapshot.c:2014).
2009 /*
2010 * Not in the precomputed list, so check the snapshots.
2011 */
2012 retry:
2013 TAILQ_FOREACH(ip, &sn->sn_head, i_nextsnap) {
2014 vp = ITOV(ip);
2015 /*
2016 * We ensure that everything of our own that needs to be
2017 * copied will be done at the time that ffs_snapshot is
2018 * called. Thus we can skip the check here which can