GENERIC HEAD from Oct 2 08:57 UTC, vmcore.459

$ INCARNATIONS=100 ./run.sh ./mkdir.cfg
     run: run time  3+00:00:00, incarnations   1, load 100, verbose 1
13:07:32 Loop #1
   mkdir: run time  0+00:02:00, incarnations   4, load 100, verbose 1
13:09:41 Loop #2
   mkdir: run time  0+00:02:00, incarnations  83, load 100, verbose 1
13:12:10 Loop #3
   mkdir: run time  0+00:02:00, incarnations  64, load 100, verbose 1
13:14:21 Loop #4
   mkdir: run time  0+00:02:00, incarnations  92, load 100, verbose 1
13:17:38 Loop #5
   mkdir: run time  0+00:02:00, incarnations  41, load 100, verbose 1
13:20:46 Loop #6
   mkdir: run time  0+00:02:00, incarnations  40, load 100, verbose 1
13:23:05 Loop #7
   mkdir: run time  0+00:02:00, incarnations  33, load 100, verbose 1
13:26:22 Loop #8
   mkdir: run time  0+00:02:00, incarnations  46, load 100, verbose 1
13:29:53 Loop #9
   mkdir: run time  0+00:02:00, incarnations  14, load 100, verbose 1
13:32:39 Loop #10
   mkdir: run time  0+00:02:00, incarnations  26, load 100, verbose 1
13:35:55 Loop #11
   mkdir: run time  0+00:02:00, incarnations  81, load 100, verbose 1
13:39:17 Loop #12
   mkdir: run time  0+00:02:00, incarnations  10, load 100, verbose 1
13:41:23 Loop #13
   mkdir: run time  0+00:02:00, incarnations  29, load 100, verbose 1
13:44:42 Loop #14
   mkdir: run time  0+00:02:00, incarnations  92, load 100, verbose 1
13:52:36 Loop #15
   mkdir: run time  0+00:02:00, incarnations  42, load 100, verbose 1


GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Sun Oct  2 11:44:35 CEST 2005
    pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I  OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf13  Stepping = 3
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory  = 267583488 (255 MB)
avail memory = 251998208 (240 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
rl0: link state changed to DOWN
freebsd4_sigreturn: eflags = 0x0
freebsd4_sigreturn: eflags = 0x0
freebsd4_sigreturn: eflags = 0x0
ad0: FAILURE - out of memory in start
/tmp: mount pending error: blocks -20 files -54
/tmp: mount pending error: blocks -20 files -54
/tmp: mount pending error: blocks -20 files -54
:
/tmp: mount pending error: blocks -20 files -54
/tmp: mount pending error: blocks -20 files -54
lock order reversal:
 1st 0xc17f4e60 vnode interlock (vnode interlock) @ ufs/ffs/ffs_snapshot.c:2036
 2nd 0xc1060084 system map (system map) @ vm/vm_map.c:2997
KDB: stack backtrace:
kdb_backtrace(0,ffffffff,c0972e50,c0972ea0,c08cace4) at kdb_backtrace+0x29
witness_checkorder(c1060084,9,c08803fa,bb5) at witness_checkorder+0x578
_mtx_lock_flags(c1060084,0,c08803f1,bb5) at _mtx_lock_flags+0x5b
_vm_map_lock_read(c1060000,c08803f1,bb5,1647742,c) at _vm_map_lock_read+0x24
vm_map_lookup(cf51297c,deadc000,1,cf512980,cf512970) at vm_map_lookup+0x28
vm_fault(c1060000,deadc000,1,0,c2885600) at vm_fault+0x66
trap_pfault(cf512a44,0,deadc0e6) at trap_pfault+0x137
trap(c0870008,cf510028,c0620028,deadc0de,5) at trap+0x341
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc076d200, esp = 0xcf512a84, ebp = 0xcf512acc ---
ffs_copyonwrite(c17f4d98,c668fc38) at ffs_copyonwrite+0x1ac
ffs_geom_strategy(c17f4ea4,c668fc38) at ffs_geom_strategy+0x89
bufwrite(c668fc38,0,0,cf512b64,c07663d9) at bufwrite+0x172
ffs_bufwrite(c668fc38) at ffs_bufwrite+0x2ae
ffs_update(c1ff06cc,0,c1ff06cc,c176f400,1) at ffs_update+0x3b9
ufs_inactive(cf512bb8,c1ff0794,c1ff06cc,cf512bd0,c068b2e1) at ufs_inactive+0x184
VOP_INACTIVE_APV(c09051a0,cf512bb8) at VOP_INACTIVE_APV+0x9b
vinactive(c1ff06cc,c2885600) at vinactive+0x8d
vput(c1ff06cc) at vput+0x167
kern_rmdir(c2885600,bfbe3e40,0,cf512d30,c07f978b) at kern_rmdir+0x200
rmdir(c2885600,cf512d04,1,c,292) at rmdir+0x12
syscall(3b,3b,3b,28050288,bfbfea7c) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280b6c0b, esp = 0xbfbe39fc, ebp = 0xbfbe3e28 ---


Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address   = 0xdeadc0e6
fault code              = supervisor read, page not present
instruction pointer     = 0x20:0xc076d200
stack pointer           = 0x28:0xcf512a84
frame pointer           = 0x28:0xcf512acc
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 9942 (mkdir)
[thread pid 9942 tid 100219 ]
Stopped at      ffs_copyonwrite+0x1ac:  movl    0x8(%edi),%edx
db> where
Tracing pid 9942 tid 100219 td 0xc2885600
ffs_copyonwrite(c17f4d98,c668fc38) at ffs_copyonwrite+0x1ac
ffs_geom_strategy(c17f4ea4,c668fc38) at ffs_geom_strategy+0x89
bufwrite(c668fc38,0,0,cf512b64,c07663d9) at bufwrite+0x172
ffs_bufwrite(c668fc38) at ffs_bufwrite+0x2ae
ffs_update(c1ff06cc,0,c1ff06cc,c176f400,1) at ffs_update+0x3b9
ufs_inactive(cf512bb8,c1ff0794,c1ff06cc,cf512bd0,c068b2e1) at ufs_inactive+0x184
VOP_INACTIVE_APV(c09051a0,cf512bb8) at VOP_INACTIVE_APV+0x9b
vinactive(c1ff06cc,c2885600) at vinactive+0x8d
vput(c1ff06cc) at vput+0x167
kern_rmdir(c2885600,bfbe3e40,0,cf512d30,c07f978b) at kern_rmdir+0x200
rmdir(c2885600,cf512d04,1,c,292) at rmdir+0x12
syscall(3b,3b,3b,28050288,bfbfea7c) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280b6c0b, esp = 0xbfbe39fc, ebp = 0xbfbe3e28 ---
db> show lockedvnods
Locked vnodes

0xc2435c3c: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b2e480 (pid 10028)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc0680512 at cache_lookup+0x3c2
#8 0xc0680a9a at vfs_cache_lookup+0x92
#9 0xc08095f3 at VOP_LOOKUP_APV+0x87
#10 0xc0684a5a at lookup+0x3e2
#11 0xc0684416 at namei+0x37e
#12 0xc06905aa at kern_unlink+0x3a
#13 0xc069056e at unlink+0x12
#14 0xc07f978b at syscall+0x22f
#15 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 3, on dev ad0s1e

0xc2927c3c: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2885900 (pid 9950)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 48834, on dev ad0s1e

0xc19382b8: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2885780 (pid 9970)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 118152, on dev ad0s1e

0xc2a772b8: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2073000 (pid 9938)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 118392, on dev ad0s1e

0xc23bb6cc: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc186a300 (pid 9967)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 25494, on dev ad0s1e

0xc1c63ae0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e9f000 (pid 9971)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 25517, on dev ad0s1e

0xc2a336cc: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc20d0300 (pid 9934)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 118721, on dev ad0s1e

0xc1ba46cc: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b99900 (pid 9952)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 118954, on dev ad0s1e

0xc2a35ae0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b9b600 (pid 9936)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119030, on dev ad0s1e

0xc1d726cc: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e65c00 (pid 9947)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119126, on dev ad0s1e

0xc2b05414: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e9fa80 (pid 9953)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119146, on dev ad0s1e

0xc1763ae0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc20d0480 (pid 9941)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119403, on dev ad0s1e

0xc1dcb414: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e9f900 (pid 9964)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119421, on dev ad0s1e

0xc24542b8: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e9f480 (pid 9948)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119522, on dev ad0s1e

0xc1be0570: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc20d0000 (pid 9946)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119589, on dev ad0s1e

0xc2724000: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b55480 (pid 9958)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119682, on dev ad0s1e

0xc270f000: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc17ca300 (pid 9951)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119705, on dev ad0s1e

0xc2329414: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2065d80 (pid 9930)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119713, on dev ad0s1e

0xc24bfae0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2065000 (pid 9944)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119731, on dev ad0s1e

0xc218f15c: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b2ec00 (pid 9963)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119883, on dev ad0s1e

0xc2944414: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e9f180 (pid 9929)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 119918, on dev ad0s1e

0xc22cb2b8: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e64d80 (pid 9959)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 49150, on dev ad0s1e

0xc2843ae0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2065480 (pid 9949)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 49448, on dev ad0s1e

0xc22ca414: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc186b300 (pid 9932)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 95461, on dev ad0s1e

0xc25de000: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc165fa80 (pid 56)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc0771b1e at handle_workitem_remove+0x2e
#10 0xc076dbc1 at process_worklist_item+0x181
#11 0xc076d928 at softdep_process_worklist+0xf4
#12 0xc068a76e at sched_sync+0x266
#13 0xc0622c2c at fork_exit+0xa0
#14 0xc07e6a1c at fork_trampoline+0x8

        ino 118958, on dev ad0s1e

0xc2575000: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc186a900 (pid 9954)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 118970, on dev ad0s1e

0xc1cfe414: tag ufs, type VREG
    usecount 0, writecount 0, refcount 529 mountedhere 0
    flags (VV_SYSTEM)
    v_object 0xc1eaf4a4 ref 0 pages 1464
     lock type ufs: EXCL (count 1) by thread 0xc1b2e480 (pid 10028)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc076bcc5 at ffs_snapremove+0x4d
#2 0xc07719bc at softdep_releasefile+0x34
#3 0xc077e6d3 at ufs_inactive+0x67
#4 0xc080b32f at VOP_INACTIVE_APV+0x9b
#5 0xc068b2e1 at vinactive+0x8d
#6 0xc068b11b at vput+0x167
#7 0xc069070e at kern_unlink+0x19e
#8 0xc069056e at unlink+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 20, on dev ad0s1e

0xc2066570: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2885180 (pid 9937)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 26009, on dev ad0s1e

0xc1ff06cc: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2885600 (pid 9942)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 26021, on dev ad0s1e

0xc2433984: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc17ca480 (pid 9943)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 49626, on dev ad0s1e

0xc1cc2d98: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc20d0d80 (pid 9955)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 49634, on dev ad0s1e

0xc1de4c3c: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2886a80 (pid 9945)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 72137, on dev ad0s1e

0xc2078c3c: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b55d80 (pid 9931)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 72147, on dev ad0s1e

0xc29cbae0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b55900 (pid 9966)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 72162, on dev ad0s1e

0xc2468570: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1864180 (pid 9962)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 72166, on dev ad0s1e

0xc26f8ae0: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc186a000 (pid 9961)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 72179, on dev ad0s1e

0xc20e5000: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e65180 (pid 9957)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068af3a at vrele+0xd2
#7 0xc068e886 at kern_chdir+0x162
#8 0xc068e722 at chdir+0x12
#9 0xc07f978b at syscall+0x22f
#10 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 72187, on dev ad0s1e

0xc257c984: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1b99a80 (pid 9935)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 95687, on dev ad0s1e

0xc21d82b8: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e9e480 (pid 9968)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 95724, on dev ad0s1e

0xc2a34570: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2886900 (pid 9965)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 120365, on dev ad0s1e

0xc243d828: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1f9e480 (pid 9956)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 120383, on dev ad0s1e

0xc1e1a15c: tag ufs, type VDIR
    usecount 2, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc2073480 (pid 9933)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc0684746 at lookup+0xce
#7 0xc0684416 at namei+0x37e
#8 0xc0692fd6 at kern_rmdir+0x3a
#9 0xc0692f9a at rmdir+0x12
#10 0xc07f978b at syscall+0x22f
#11 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 72468, on dev ad0s1e

0xc22a1570: tag ufs, type VDIR
    usecount 2, writecount 0, refcount 2 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 2) by thread 0xc2073480 (pid 9933)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 72469, on dev ad0s1e

0xc1fad414: tag ufs, type VDIR
    usecount 0, writecount 0, refcount 1 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e65480 (pid 9940)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 95896, on dev ad0s1e

0xc249a414: tag ufs, type VDIR
    usecount 2, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc1e64c00 (pid 9960)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc0684746 at lookup+0xce
#7 0xc0684416 at namei+0x37e
#8 0xc0692fd6 at kern_rmdir+0x3a
#9 0xc0692f9a at rmdir+0x12
#10 0xc07f978b at syscall+0x22f
#11 0xc07e6a0f at Xint0x80_syscall+0x1f

        ino 120515, on dev ad0s1e

0xc2666828: tag ufs, type VDIR
    usecount 2, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 2) by thread 0xc1e64c00 (pid 9960)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc0682c8d at vop_stdlock+0x21
#2 0xc080b533 at VOP_LOCK_APV+0x87
#3 0xc0779a88 at ffs_lock+0x10
#4 0xc080b533 at VOP_LOCK_APV+0x87
#5 0xc0696a3c at vn_lock+0xac
#6 0xc068ad0e at vget+0x8e
#7 0xc06839a1 at vfs_hash_get+0x8d
#8 0xc0778917 at ffs_vget+0x27
#9 0xc077f165 at ufs_lookup+0x881
#10 0xc08096c7 at VOP_CACHEDLOOKUP_APV+0x9b
#11 0xc0680abd at vfs_cache_lookup+0xb5
#12 0xc08095f3 at VOP_LOOKUP_APV+0x87
#13 0xc0684a5a at lookup+0x3e2
#14 0xc0684416 at namei+0x37e
#15 0xc0692fd6 at kern_rmdir+0x3a
#16 0xc0692f9a at rmdir+0x12
#17 0xc07f978b at syscall+0x22f

        ino 120516, on dev ad0s1e
db> show lockedbufs
buf at 0xc65e636c
b_flags = 0xa0020020<remfree,vmio,clusterok,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc17f4ea4), b_data = 0xc66ef000, b_blkno = 754080
b_npages = 4, pages(OBJ, IDX, PA): (0xc17cc6b4, 0x17034, 0x398a000),(0xc17cc6b4, 0x17035, 0xa0eb000),(0xc17cc6b4, 0x17036, 0x44ac000),(0xc17cc6b4, 0x17037, 0xb62d000)
 lock type getblk: EXCL (count 1) by thread 0xc2065480 (pid 9949)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc067da87 at getblk+0x15b
#2 0xc067aaf7 at breadn+0x5b
#3 0xc067aa98 at bread+0x20
#4 0xc0766248 at ffs_update+0x228
#5 0xc077e7f0 at ufs_inactive+0x184
#6 0xc080b32f at VOP_INACTIVE_APV+0x9b
#7 0xc068b2e1 at vinactive+0x8d
#8 0xc068b11b at vput+0x167
#9 0xc069319c at kern_rmdir+0x200
#10 0xc0692f9a at rmdir+0x12
#11 0xc07f978b at syscall+0x22f
#12 0xc07e6a0f at Xint0x80_syscall+0x1f

buf at 0xc65e69bc
b_flags = 0xa0020020<remfree,vmio,clusterok,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc17f4ea4), b_data = 0xc66ff000, b_blkno = 1506240
b_npages = 4, pages(OBJ, IDX, PA): (0xc17cc6b4, 0x2df78, 0xea2e000),(0xc17cc6b4, 0x2df79, 0x844f000),(0xc17cc6b4, 0x2df7a, 0xc420000),(0xc17cc6b4, 0x2df7b, 0xa6f1000)
 lock type getblk: EXCL (count 1) by thread 0xc186b300 (pid 9932)#0 0xc062b6c1 at lockmgr+0x4ed
#1 0xc067da87 at getblk+0x15b
#2 0xc067aaf7 at breadn+0x5b
#3 0xc067aa98 at bread+0x20
#4 0xc0766248 at ffs_update+0x228
#5 0xc077e7f0 at ufs_inactive+0x184
#6 0xc080b32f at VOP_INACTIVE_APV+0x9b
#7 0xc068b2e1 at vinactive+0x8d
#8 0xc068b11b at vput+0x167
#9 0xc069319c at kern_rmdir+0x200
#10 0xc0692f9a at rmdir+0x12
#11 0xc07f978b at syscall+0x22f
#12 0xc07e6a0f at Xint0x80_syscall+0x1f

buf at 0xc65f3878
b_flags = 0x80020024<remfree,clusterok,cache,async>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc1cfe520), b_data = 0xc690b000, b_blkno = 101216
b_npages = 4, pages(OBJ, IDX, PA): (0xc09be160, 0x6a0c, 0x804c000),(0xc09be160, 0x6a0d, 0xaded000),(0xc09be160, 0x6a0e, 0x36ae000),(0xc09be160, 0x6a0f, 0x836f000)
db> show alllocks
Process 9960 (mkdir) thread 0xc1e64c00 (100135)
exclusive sleep mutex vnode interlock r = 0 (0xc26668f0) locked @ ufs/ffs/ffs_softdep.c:2160
Process 9942 (mkdir) thread 0xc2885600 (100219)
exclusive sleep mutex vnode interlock r = 0 (0xc17f4e60) locked @ ufs/ffs/ffs_snapshot.c:2036
db> show pcpu
cpuid        = 0
curthread    = 0xc2885600: pid 9942 "mkdir"
curpcb       = 0xcf512d90
fpcurthread  = none
idlethread   = 0xc155a780: pid 11 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:
db> where 9960
Tracing pid 9960 tid 100135 td 0xc1e64c00
sched_switch(c1e64c00,0,2) at sched_switch+0x177
mi_switch(2,0,c0961fa0,0,c086611b) at mi_switch+0x270
critical_exit(c6649570,cf4368fc,c07e6fa0,0,c07e0008) at critical_exit+0x8b
lapic_handle_timer(0) at lapic_handle_timer+0xc9
Xtimerint(c6649570,c26668f0,1,72da8,0) at Xtimerint+0x30
softdep_setup_freeblocks(c25e6e70,0,0,800,12c) at softdep_setup_freeblocks+0x775
ffs_truncate(c2666828,0,0,0,c18ef180,c1e64c00,c09bd880,0,c087cc8f,d76) at ffs_truncate+0x569
handle_workitem_remove(c22d33a0,0,c09bd880,0,c087cc8f,c1c) at handle_workitem_remove+0x19a
softdep_setup_remove(c66a6bec,c264fc60,c25e6e70,1,c25e6e70) at softdep_setup_remove+0xc6
ufs_dirremove(c249a414,c25e6e70,100800c,1,16) at ufs_dirremove+0x12d
ufs_rmdir(cf436c1c,0,c2666828,cf436cc8,c069315a) at ufs_rmdir+0xc3
VOP_RMDIR_APV(c09051a0,cf436c1c) at VOP_RMDIR_APV+0xb5
kern_rmdir(c1e64c00,bfbd7110,0,cf436d30,c07f978b) at kern_rmdir+0x1be
rmdir(c1e64c00,cf436d04,1,19,296) at rmdir+0x12
syscall(2805003b,bfbf003b,bfbe003b,28050288,bfbfea7c) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280b6c0b, esp = 0xbfbd6ccc, ebp = 0xbfbd70f8 ---
db> ps
  pid   proc     uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
10028 c2887a3c    0 10017 10017 0004002 [SLPQ getblk 0xc66bdd94][SLP] rm
10017 c22a0000    0  7472 10017 0004002 [SLPQ wait 0xc22a0000][SLP] sh
 9971 c2056624 1001  9928  7535 0000002 [SLPQ getblk 0xc6623964][SLP] mkdir
 9970 c2888418 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9969 c2056a3c 1001  9928  7535 0000002 [RUNQ] mkdir
 9968 c1fa6624 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9967 c1b16418 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9966 c1e4e20c 1001  9928  7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
 9965 c2888000 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9964 c20d3624 1001  9928  7535 0000002 [SLPQ getblk 0xc660bea4][SLP] mkdir
 9963 c186320c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9962 c1868418 1001  9928  7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
 9961 c22a3418 1001  9928  7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
 9960 c1e66418 1001  9928  7535 0000002 [RUNQ] mkdir
 9959 c1e4ea3c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9958 c1b15830 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9957 c1e62c48 1001  9928  7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
 9956 c1b16624 1001  9928  7535 0000002 [SLPQ getblk 0xc6696654][SLP] mkdir
 9955 c205620c 1001  9928  7535 0000002 [SLPQ getblk 0xc662ea24][SLP] mkdir
 9954 c2866624 1001  9928  7535 0000002 [SLPQ getblk 0xc6675280][SLP] mkdir
 9953 c1b16830 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9952 c1fa620c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9951 c1869830 1001  9928  7535 0000002 [SLPQ getblk 0xc66268bc][SLP] mkdir
 9950 c288820c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9949 c2867830 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9948 c1e66c48 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9947 c22a0624 1001  9928  7535 0000002 [SLPQ getblk 0xc661a378][SLP] mkdir
 9946 c1e62a3c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9945 c2867c48 1001  9928  7535 0000002 [SLPQ getblk 0xc66bbc70][SLP] mkdir
 9944 c1e66830 1001  9928  7535 0000002 [SLPQ getblk 0xc66268bc][SLP] mkdir
 9943 c1869624 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9942 c20d3c48 1001  9928  7535 0000002 [CPU 0] mkdir
 9941 c22a020c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9940 c2887418 1001  9928  7535 0000002 [RUNQ] mkdir
 9938 c2888624 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9937 c2888c48 1001  9928  7535 0000002 [SLPQ getblk 0xc668fc98][SLP] mkdir
 9936 c20d3a3c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9935 c20d320c 1001  9928  7535 0000002 [SLPQ getblk 0xc66bafd0][SLP] mkdir
 9934 c2056830 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9933 c20d3830 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9932 c286620c 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9931 c2884418 1001  9928  7535 0000002 [LOCK vnode interlock c1cdb840] mkdir
 9930 c1e4d830 1001  9928  7535 0000002 [SLPQ getblk 0xc66268bc][SLP] mkdir
 9929 c2884000 1001  9928  7535 0000002 [SLPQ getblk 0xc663cbd0][SLP] mkdir
 9928 c22a3000 1001  9926  7535 0000002 [SLPQ wait 0xc22a3000][SLP] mkdir
 9926 c1e4e624 1001  7544  7535 0004002 [RUNQ] mkdir
 7544 c1b15418 1001  7543  7535 0000002 [SLPQ wait 0xc1b15418][SLP] run
 7543 c1b16a3c 1001  7541  7535 0000002 [SLPQ wait 0xc1b16a3c][SLP] run
 7541 c22a3624 1001  7535  7535 0004002 [RUNQ] run
 7535 c1863418 1001   510  7535 0004002 [SLPQ wait 0xc1863418][SLP] sh
 7472 c1e66624    0  7469  7472 0004002 [SLPQ wait 0xc1e66624][SLP] bash
 7469 c1e66a3c    0  7468  7469 0004002 [SLPQ pause 0xc1e66a70][SLP] csh
 7468 c22a0830 1001  7461  7468 0004102 [SLPQ wait 0xc22a0830][SLP] su
 7461 c2866c48 1001  7460  7461 0004002 [SLPQ wait 0xc2866c48][SLP] bash
 7460 c1e6220c 1001  7458  7458 0000100 [SLPQ select 0xc09afba4][SLP] sshd
 7458 c1b1620c    0   469  7458 0004100 [SLPQ sbwait 0xc1bd4bc8][SLP] sshd
  623 c17cb000 1001     1   610 0014003 [SLPQ select 0xc09afba4][SLP] gconfd-2
  602 c1863624    0     1   602 0004002 [SLPQ ttyin 0xc16f9010][SLP][SWAP] getty
  601 c1863a3c    0     1   601 0004002 [SLPQ ttyin 0xc16f9410][SLP][SWAP] getty
  600 c1869000    0     1   600 0004002 [SLPQ ttyin 0xc16f0810][SLP][SWAP] getty
  599 c1863830    0     1   599 0004002 [SLPQ ttyin 0xc16ef810][SLP][SWAP] getty
  598 c1868624    0     1   598 0004002 [SLPQ ttyin 0xc16f0c10][SLP][SWAP] getty
  597 c17c7a3c    0     1   597 0004002 [SLPQ ttyin 0xc16eec10][SLP][SWAP] getty
  596 c17cb830    0     1   596 0004002 [SLPQ ttyin 0xc16ee810][SLP][SWAP] getty
  595 c1868a3c    0     1   595 0004002 [SLPQ ttyin 0xc16e7410][SLP][SWAP] getty
  581 c186820c    0     1   581 0000000 [SLPQ select 0xc09afba4][SLP][SWAP] inetd
  510 c1863000 1001   502   510 0004002 [SLPQ wait 0xc1863000][SLP] bash
  502 c1868830 1001   475   475 0000100 [SLPQ select 0xc09afba4][SLP] sshd
  495 c1863c48    0     1   495 0000000 [SLPQ nanslp 0xc096288c][SLP] cron
  483 c1868000   25     1   483 0000100 [SLPQ pause 0xc1868034][SLP] sendmail
  477 c1868c48    0     1   477 0000100 [RUNQ] sendmail
  475 c17c7830    0   469   475 0004100 [SLPQ sbwait 0xc184d20c][SLP][SWAP] sshd
  469 c165e20c    0     1   469 0000100 [SLPQ select 0xc09afba4][SLP] sshd
  451 c17cb624    0     1   451 0000000 [RUNQ] ntpd
  422 c165e418    0     1   422 0000000 [SLPQ select 0xc09afba4][SLP] usbd
  402 c17cb418    0   397   397 0000000 [SLPQ - 0xc1796c00][SLP][SWAP] nfsd
  401 c17cba3c    0   397   397 0000000 [SLPQ - 0xc1796e00][SLP][SWAP] nfsd
  400 c17c7624    0   397   397 0000000 [SLPQ - 0xc1822000][SLP][SWAP] nfsd
  399 c17c7418    0   397   397 0000000 [SLPQ - 0xc1796a00][SLP][SWAP] nfsd
  397 c17cb20c    0     1   397 0000000 [SLPQ select 0xc09afba4][SLP][SWAP] nfsd
  395 c17cbc48    0     1   395 0000000 [SLPQ select 0xc09afba4][SLP] mountd
  326 c17c7c48    0     1   326 0000000 [SLPQ select 0xc09afba4][SLP] rpcbind
  298 c17c720c    0     1   298 0000000 [SLPQ select 0xc09afba4][SLP][SWAP] devd
  271 c17c7000    0     1   271 0000000 [RUNQ] syslogd
   61 c165e624    0     0     0 0000204 [SLPQ - 0xcca21d04][SLP] schedcpu
   60 c165e830    0     0     0 0000204 [SLPQ - 0xc09b822c][SLP] nfsiod 3
   59 c165ea3c    0     0     0 0000204 [SLPQ - 0xc09b8228][SLP] nfsiod 2
   58 c165ec48    0     0     0 0000204 [SLPQ - 0xc09b8224][SLP] nfsiod 1
   57 c1771000    0     0     0 0000204 [SLPQ - 0xc09b8220][SLP] nfsiod 0
   56 c177120c    0     0     0 0000204 [SLPQ getblk 0xc6675280][SLP] syncer
   55 c1771418    0     0     0 0000204 [RUNQ] vnlru
   54 c1771624    0     0     0 0000204 [RUNQ] bufdaemon
   53 c1771830    0     0     0 000020c [SLPQ pgzero 0xc09be824][SLP] pagezero
   52 c1771a3c    0     0     0 0000204 [SLPQ psleep 0xc09be374][SLP] vmdaemon
   51 c1771c48    0     0     0 0000204 [SLPQ psleep 0xc09be330][SLP] pagedaemon
   50 c15ac624    0     0     0 0000204 [SLPQ - 0xc16a083c][SLP] fdc0
   49 c15ac830    0     0     0 0000204 [IWAIT] swi0: sio
   48 c15aca3c    0     0     0 0000204 [SLPQ usbevt 0xc1645210][SLP] usb4
   47 c15acc48    0     0     0 0000204 [SLPQ usbevt 0xc1695210][SLP] usb3
   46 c165d000    0     0     0 0000204 [SLPQ usbevt 0xc1696210][SLP] usb2
   45 c165d20c    0     0     0 0000204 [SLPQ usbevt 0xc1682210][SLP] usb1
   44 c165d418    0     0     0 0000204 [SLPQ usbtsk 0xc092b544][SLP] usbtask
   43 c165d624    0     0     0 0000204 [SLPQ usbevt 0xc164e210][SLP] usb0
    9 c165d830    0     0     0 0000204 [SLPQ - 0xc15cb800][SLP] thread taskq
   42 c165da3c    0     0     0 0000204 [IWAIT] swi6:+
    8 c165dc48    0     0     0 0000204 [SLPQ - 0xc15cb980][SLP] acpi_task2
    7 c165e000    0     0     0 0000204 [SLPQ - 0xc15cb980][SLP] acpi_task1
    6 c159cc48    0     0     0 0000204 [SLPQ - 0xc15cb980][SLP] acpi_task0
   41 c15aa000    0     0     0 0000204 [IWAIT] swi6: task queue
    5 c15aa20c    0     0     0 0000204 [SLPQ - 0xc15cbb80][SLP] kqueue taskq
   40 c15aa418    0     0     0 0000204 [IWAIT] swi2: cambio
   39 c15aa624    0     0     0 0000204 [IWAIT] swi5:+
   38 c15aa830    0     0     0 0000204 [SLPQ - 0xc0929260][SLP] yarrow
    4 c15aaa3c    0     0     0 0000204 [SLPQ - 0xc092bd88][SLP] g_down
    3 c15aac48    0     0     0 0000204 [LOCK vnode interlock c1cdb840] g_up
    2 c15ac000    0     0     0 0000204 [SLPQ - 0xc092bd7c][SLP] g_event
   37 c15ac20c    0     0     0 0000204 [IWAIT] swi1: net
   36 c15ac418    0     0     0 0000204 [IWAIT] swi3: vm
   35 c1588624    0     0     0 000020c [IWAIT] swi4: clock sio
   34 c1588830    0     0     0 0000204 [IWAIT] irq23: ehci0
   33 c1588a3c    0     0     0 0000204 [IWAIT] irq22: rl0
   32 c1588c48    0     0     0 0000204 [IWAIT] irq21:
   31 c159c000    0     0     0 0000204 [IWAIT] irq20:
   30 c159c20c    0     0     0 0000204 [IWAIT] irq19: uhci1
   29 c159c418    0     0     0 0000204 [IWAIT] irq18: uhci2
   28 c159c624    0     0     0 0000204 [IWAIT] irq17: pcm0
   27 c159c830    0     0     0 0000204 [IWAIT] irq16: uhci0 uhci3
   26 c159ca3c    0     0     0 0000204 [IWAIT] irq15: ata1
   25 c155e20c    0     0     0 0000204 [IWAIT] irq14: ata0
   24 c155e418    0     0     0 0000204 [IWAIT] irq13:
   23 c155e624    0     0     0 0000204 [IWAIT] irq12:
   22 c155e830    0     0     0 0000204 [IWAIT] irq11:
   21 c155ea3c    0     0     0 0000204 [IWAIT] irq10:
   20 c155ec48    0     0     0 0000204 [IWAIT] irq9: acpi0
   19 c1588000    0     0     0 0000204 [IWAIT] irq8:
   18 c158820c    0     0     0 0000204 [IWAIT] irq7: ppc0
   17 c1588418    0     0     0 0000204 [IWAIT] irq6: fdc0
   16 c1559000    0     0     0 0000204 [IWAIT] irq5:
   15 c155920c    0     0     0 0000204 [IWAIT] irq4: sio0
   14 c1559418    0     0     0 0000204 [IWAIT] irq3:
   13 c1559624    0     0     0 0000204 [IWAIT] irq0:
   12 c1559830    0     0     0 0000204 [IWAIT] irq1: atkbd0
   11 c1559a3c    0     0     0 000020c [Can run] idle: cpu0
    1 c1559c48    0     0     1 0004200 [SLPQ wait 0xc1559c48][SLP] init
   10 c155e000    0     0     0 0000204 [SLPQ ktrace 0xc0960818][SLP] ktrace
    0 c092be80    0     0     0 0000200 [IWAIT] swapper
db> call doadump
Dumping 254 MB (2 chunks)
  chunk 0: 1MB (159 pages) ... ok
  chunk 1: 255MB (65072 pages) 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 ... ok

Dump complete
= 0xf
db> reset

(kgdb) bt
#0  doadump () at pcpu.h:165
#1  0xc04698f3 in db_fncall (dummy1=0xc09dbd80, dummy2=0x0, dummy3=0x0,
    dummy4=0xcf5128a8 "(Qغ}(Q(Q\220\a") at ../../../ddb/db_command.c:486
#2  0xc04696f8 in db_command (last_cmdp=0xc0914de4, cmd_table=0x0,
    aux_cmd_tablep=0xc0890e24, aux_cmd_tablep_end=0xc0890e40)
    at ../../../ddb/db_command.c:401
#3  0xc04697c0 in db_command_loop () at ../../../ddb/db_command.c:452
#4  0xc046b365 in db_trap (type=0xc, code=0x0) at ../../../ddb/db_main.c:228
#5  0xc064e32c in kdb_trap (type=0xc, code=0x0, tf=0xcf512a44)
    at ../../../kern/subr_kdb.c:473
#6  0xc07f949c in trap_fatal (frame=0xcf512a44, eva=0xdeadc0e6)
    at ../../../i386/i386/trap.c:822
#7  0xc07f91e3 in trap_pfault (frame=0xcf512a44, usermode=0x0, eva=0xdeadc0e6)
    at ../../../i386/i386/trap.c:742
#8  0xc07f8e2d in trap (frame=
      {tf_fs = 0xc0870008, tf_es = 0xcf510028, tf_ds = 0xc0620028, tf_edi = 0xdeadc0de, tf_esi = 0x5, tf_ebp = 0xcf512acc, tf_isp = 0xcf512a70, tf_ebx = 0xc17f4e60, tf_edx = 0xc27f2c00, tf_ecx = 0xc09af6e0, tf_eax = 0x10000, tf_trapno = 0xc, tf_err = 0x0, tf_eip = 0xc076d200, tf_cs = 0x20, tf_eflags = 0x10296, tf_esp = 0xc17f4e60, tf_ss = 0xc17f4e60})
    at ../../../i386/i386/trap.c:432
#9  0xc07e69ba in calltrap () at ../../../i386/i386/exception.s:139
#10 0xc0870008 in ?? ()
#11 0xcf510028 in ?? ()
#12 0xc0620028 in exec_free_args (args=0xc17f4e60) at ../../../kern/kern_exec.c:1044
#13 0xc0779669 in ffs_geom_strategy (bo=0xdeadc0de, bp=0xc17f4ea4)
    at ../../../ufs/ffs/ffs_vfsops.c:1686
#14 0xc067aeb2 in bufwrite (bp=0xc668fc38) at buf.h:415
#15 0xc07795d2 in ffs_bufwrite (bp=0xc668fc38) at ../../../ufs/ffs/ffs_vfsops.c:1663
#16 0xc07663d9 in ffs_update (vp=0xc1ff06cc, waitfor=0x0) at buf.h:401
#17 0xc077e7f0 in ufs_inactive (ap=0x0) at ../../../ufs/ufs/ufs_inode.c:118
#18 0xc080b32f in VOP_INACTIVE_APV (vop=0xc09056e0, a=0xcf512bb8) at vnode_if.c:1535
#19 0xc068b2e1 in vinactive (vp=0xc1ff06cc, td=0x0) at vnode_if.h:795
#20 0xc068b11b in vput (vp=0xc1ff06cc) at ../../../kern/vfs_subr.c:2107
#21 0xc069319c in kern_rmdir (td=0xc2885600, path=---Can't read userspace from dump, or kernel process---

) at ../../../kern/vfs_syscalls.c:3451
---Type <return> to continue, or q <return> to quit---
#22 0xc0692f9a in rmdir (td=0xc2885600, uap=0x0) at ../../../kern/vfs_syscalls.c:3386
#23 0xc07f978b in syscall (frame=
      {tf_fs = 0x3b, tf_es = 0x3b, tf_ds = 0x3b, tf_edi = 0x28050288, tf_esi = 0xbfbfea7c, tf_ebp = 0xbfbe3e28, tf_isp = 0xcf512d64, tf_ebx = 0x1, tf_edx = 0xbfbd68b4, tf_ecx = 0x0, tf_eax = 0x89, tf_trapno = 0x0, tf_err = 0x2, tf_eip = 0x280b6c0b, tf_cs = 0x33, tf_eflags = 0x292, tf_esp = 0xbfbe39fc, tf_ss = 0x3b}) at ../../../i386/i386/trap.c:976
#24 0xc07e6a0f in Xint0x80_syscall () at ../../../i386/i386/exception.s:200
(kgdb) f 12
#12 0xc0620028 in exec_free_args (args=0xc17f4e60) at ../../../kern/kern_exec.c:1044
1044                    kmem_free_wakeup(exec_map, (vm_offset_t)args->buf,
(kgdb) p *args
$3 = {buf = 0xc08cace4 "Z\003\210\t", begin_argv = 0xc08682f2 "vnode interlock",
  begin_envv = 0xc08682f2 "vnode interlock", endp = 0x30000---Can't read userspace from dump, or kernel process---

(kgdb) up
#13 0xc0779669 in ffs_geom_strategy (bo=0xdeadc0de, bp=0xc17f4ea4)
    at ../../../ufs/ffs/ffs_vfsops.c:1686
1686                    if ((vp->v_vflag & VV_COPYONWRITE) &&
(kgdb) info loc
vp = (struct vnode *) 0x5
error = 0x0
(kgdb) l
1681                    if ((bp->b_flags & B_VALIDSUSPWRT) == 0 &&
1682                        bp->b_vp != NULL && bp->b_vp->v_mount != NULL &&
1683                        (bp->b_vp->v_mount->mnt_kern_flag & MNTK_SUSPENDED) != 0)
1684                            panic("ffs_geom_strategy: bad I/O");
1685                    bp->b_flags &= ~B_VALIDSUSPWRT;
1686                    if ((vp->v_vflag & VV_COPYONWRITE) &&
1687                        vp->v_rdev->si_snapdata != NULL &&
1688                        (error = (ffs_copyonwrite)(vp, bp)) != 0 &&
1689                        error != EOPNOTSUPP) {
1690                            bp->b_error = error;
(kgdb) up
#14 0xc067aeb2 in bufwrite (bp=0xc668fc38) at buf.h:415
415             BO_STRATEGY(bp->b_bufobj, bp);
(kgdb) p *bp
$4 = {b_bufobj = 0xc17f4ea4, b_bcount = 0x4000, b_caller1 = 0x0,
  b_data = 0xc81cb000 "A\003", b_error = 0x0, b_iocmd = 0x2, b_ioflags = 0x2,
  b_iooffset = 0xb878000, b_resid = 0x0, b_iodone = 0, b_blkno = 0x5c3c0,
  b_offset = 0xb878000, b_bobufs = {tqe_next = 0xc6623dc0, tqe_prev = 0xc665221c},
  b_left = 0xc667ada8, b_right = 0x0, b_vflags = 0x1, b_freelist = {
    tqe_next = 0xc6683a1c, tqe_prev = 0xc660ef7c}, b_qindex = 0x2, b_flags = 0xa0020020,
  b_xflags = 0x2, b_lock = {lk_interlock = 0xc09616d8, lk_flags = 0x2040000,
    lk_sharecount = 0x0, lk_waitcount = 0x1, lk_exclusivecount = 0x1, lk_prio = 0x50,
    lk_wmesg = 0xc086c7ec "getblk", lk_timo = 0x0, lk_lockholder = 0xc2885600,
    lk_newlock = 0x0, lk_stack = {depth = 0xd, pcs = {0xc062b6c1, 0xc067da87,
        0xc067aaf7, 0xc067aa98, 0xc0766248, 0xc077e7f0, 0xc080b32f, 0xc068b2e1,
        0xc068b11b, 0xc069319c, 0xc0692f9a, 0xc07f978b, 0xc07e6a0f, 0x0, 0x0, 0x0, 0x0,
        0x0}}}, b_bufsize = 0x4000, b_runningbufspace = 0x0,
  b_kvabase = 0xc81cb000 "A\003", b_kvasize = 0x4000, b_lblkno = 0x5c3c0,
  b_vp = 0xc17f4d98, b_dirtyoff = 0x0, b_dirtyend = 0x0, b_rcred = 0x0, b_wcred = 0x0,
  b_saveaddr = 0xc81cb000, b_pager = {pg_reqpage = 0x0}, b_cluster = {cluster_head = {
      tqh_first = 0x0, tqh_last = 0xc6688224}, cluster_entry = {tqe_next = 0x0,
      tqe_prev = 0xc6688224}}, b_pages = {0xc14067a8, 0xc13f74f0, 0xc13eca38,
    0xc14cbf80, 0x0 <repeats 28 times>}, b_npages = 0x4, b_dep = {lh_first = 0xc2116480}}
(kgdb) l *0xc076d200
0xc076d200 is in ffs_copyonwrite (../../../ufs/ffs/ffs_snapshot.c:2014).
2009            /*
2010             * Not in the precomputed list, so check the snapshots.
2011             */
2012    retry:
2013            TAILQ_FOREACH(ip, &sn->sn_head, i_nextsnap) {
2014                    vp = ITOV(ip);
2015                    /*
2016                     * We ensure that everything of our own that needs to be
2017                     * copied will be done at the time that ffs_snapshot is
2018                     * called. Thus we can skip the check here which can