GENERIC HEAD from Sep 11 09:02 UTC + Konstantin's vfs_mount.c patch, No vmcore
Panic while stress testing mksnap_ffs
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Tue Sep 12 08:44:40 CEST 2006
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1032458240 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
pKKaDBDnB:: istasct:ack clbok cakbackt r(cskltereapc emacu:te:e
xk)d b
Skodbf_t_dbbaeapc cLkkttroacrkc ancoet( eleo6(7ecdk6e69d1 ac@ec8 ,b8ucf,06c0f6f6s6/cfffcsf/aaf,,cf0sc9_09s2o24f4btbd1e1pee.,c,c0c:0599232642c44
3ccp,u4icd3 3=, f1cf3fd5
dK2DaB0b,:. de..n0t)e ,.ra:t kp.dab_n.i)cb aatck
[thread pid 50419 tid 100193 ]
Stopped at kdb_enter+0x2b: nop
db> where
Tracing pid 50419 tid 100193 td 0xc450b1b0
kdb_enter(c0919d8a) at kdb_enter+0x2b
panic(c091e90d,c0935f22,c09327d1,c09327e7,14f2,...) at panic+0x14b
witness_unlock(c0a631b8,8,c09327e7,14f2) at witness_unlock+0xcf
_mtx_unlock_flags(c0a631b8,0,c09327de,14f2,2,...) at _mtx_unlock_flags+0x8c
softdep_sync_metadata(c54bb7e0) at softdep_sync_metadata+0x62f
ffs_syncvnode(c54bb7e0,1) at ffs_syncvnode+0x33b
ffs_truncate(c54bb7e0,200,0,880,c46ad680,c450b1b0) at ffs_truncate+0x4d6
ufs_direnter(c54bb7e0,c4ec6540,e68269e4,e6826bc8,0,c4f5bd68,e6826bc8,e68269e4) at ufs_direnter+0x813
ufs_makeinode(a1ff,c54bb7e0,e6826bb4,e6826bc8) at ufs_makeinode+0x462
ufs_symlink(e6826b60) at ufs_symlink+0x2b
VOP_SYMLINK_APV(c09d7a40,e6826b60) at VOP_SYMLINK_APV+0x9b
kern_symlink(c450b1b0,bfbfe9d0,804bda0,0,e6826d30,...) at kern_symlink+0x333
symlink(c450b1b0,e6826d04) at symlink+0x15
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (57, FreeBSD ELF32, symlink), eip = 0x280cb523, esp = 0xbfbfe9bc, ebp = 0xbfbfea18 ---
db> show alllocks
Process 50432 (swap) thread 0xc46576c0 (100179)
exclusive sleep mutex vm object (standard object) r = 0 (0xc5d40438) locked @ vm/vm_fault.c:900
exclusive sx user map r = 0 (0xc47bca6c) locked @ vm/vm_map.c:3073
Process 50418 (swap) thread 0xc404a000 (100153)
exclusive sleep mutex pmap r = 0 (0xc47bcddc) locked @ i386/i386/pmap.c:2772
exclusive sleep mutex pmap r = 0 (0xc3c3cee0) locked @ i386/i386/pmap.c:2771
exclusive sleep mutex vm page queue mutex r = 0 (0xc0a639dc) locked @ i386/i386/pmap.c:2766
exclusive sx user map r = 0 (0xc3c3ce7c) locked @ vm/vm_map.c:2561
Process 4 (g_down) thread 0xc3c37bd0 (100014)
exclusive sleep mutex ATA queue lock r = 0 (0xc3d3a908) locked @ dev/ata/ata-queue.c:177
db> show lockedvnods
Locked vnodes
0xc54bb7e0: tag ufs, type VDIR
usecount 32, writecount 0, refcount 34 mountedhere 0
flags ()
v_object 0xc4d002d0 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xc450b1b0 (pid 50419) with 11 pending#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d91ea at ffs_lock+0x76
#2 0xc08a8127 at VOP_LOCK_APV+0x87
#3 0xc0700b1c at vn_lock+0xac
#4 0xc06ed512 at lookup+0xd2
#5 0xc06ed1de at namei+0x382
#6 0xc06fa444 at kern_symlink+0x90
#7 0xc06fa3b1 at symlink+0x15
#8 0xc08959f2 at syscall+0x256
#9 0xc08803df at Xint0x80_syscall+0x1f
ino 84, on dev ad0s1e
0xc5355bd0: tag ufs, type VREG
usecount 1, writecount 1, refcount 959 mountedhere 0
flags ()
v_object 0xc4991618 ref 0 pages 3828
lock type ufs: EXCL (count 1) by thread 0xc3ffd1b0 (pid 50416)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d91ea at ffs_lock+0x76
#2 0xc08a8127 at VOP_LOCK_APV+0x87
#3 0xc0700b1c at vn_lock+0xac
#4 0xc0700484 at vn_write+0x138
#5 0xc06c125b at dofilewrite+0x7b
#6 0xc06c10fe at kern_writev+0x36
#7 0xc06c1029 at write+0x45
#8 0xc08959f2 at syscall+0x256
#9 0xc08803df at Xint0x80_syscall+0x1f
ino 140, on dev ad0s1e
db> show lockedbufs
buf at 0xd7cff424
b_flags = 0xa0000020<remfree,vmio,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc40bd8e0), b_data = 0xd8e2c000, b_blkno = 3387360
b_npages = 4, pages(OBJ, IDX, PA): (0xc4062ca8, 0x675fc, 0x115e9000),(0xc4062ca8, 0x675fd, 0x6f2a000),(0xc4062ca8, 0x675fe, 0x3e9b000),(0xc4062ca8, 0x675ff, 0xe85c000)
lock type bufwait: EXCL (count 1) by thread 0xc3ffd1b0 (pid 50416)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d4af0 at getdirtybuf+0x60
#2 0xc07d3483 at softdep_sync_metadata+0x30f
#3 0xc07d910f at ffs_syncvnode+0x33b
#4 0xc07c3fc2 at ffs_truncate+0x4d6
#5 0xc07d9c87 at ffs_write+0x5ef
#6 0xc08a6e2a at VOP_WRITE_APV+0x132
#7 0xc0700536 at vn_write+0x1ea
#8 0xc06c125b at dofilewrite+0x7b
#9 0xc06c10fe at kern_writev+0x36
#10 0xc06c1029 at write+0x45
#11 0xc08959f2 at syscall+0x256
#12 0xc08803df at Xint0x80_syscall+0x1f
buf at 0xd7d93a20
b_flags = 0x20000000<vmio>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc40bd8e0), b_data = 0xda4c8000, b_blkno = 23488
b_npages = 4, pages(OBJ, IDX, PA): (0xc4062ca8, 0xb78, 0xce65000),(0xc4062ca8, 0xb79, 0x2a036000),(0xc4062ca8, 0xb7a, 0x36057000),(0xc4062ca8, 0xb7b, 0x14418000)
db> show allpcpu
Current CPU: 1
cpuid = 0
curthread = 0xc3c37bd0: pid 4 "g_down"
curpcb = 0xe26f1d90
fpcurthread = none
idlethread = 0xc3c36a20: pid 13 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc450b1b0: pid 50419 "swap"
curpcb = 0xe6826d90
fpcurthread = none
idlethread = 0xc3c36870: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc46576c0: pid 50432 "swap"
curpcb = 0xe67ded90
fpcurthread = none
idlethread = 0xc3c366c0: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc404a000: pid 50418 "swap"
curpcb = 0xe6691d90
fpcurthread = none
idlethread = 0xc3c36510: pid 10 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> ps
pid ppid pgrp uid state wmesg wchan cmd
50436 50418 50417 1001 R+ swap
50435 50418 50417 1001 R+ swap
50434 50418 50417 1001 R+ swap
50433 50418 50417 1001 R+ swap
50432 50418 50417 1001 R+ CPU 2 swap
50431 50418 50417 1001 S+ ufs 0xc54bb838 swap
50430 50418 50417 1001 S+ ufs 0xc54bb838 swap
50429 50418 50417 1001 S+ ufs 0xc54bb838 swap
50428 50418 50417 1001 S+ ufs 0xc54bb838 swap
50427 50418 50417 1001 S+ ufs 0xc54bb838 swap
50426 50418 50417 1001 S+ ufs 0xc54bb838 swap
50425 50418 50417 1001 S+ ufs 0xc54bb838 swap
50424 50418 50417 1001 S+ ufs 0xc54bb838 swap
50423 50418 50417 1001 S+ ufs 0xc54bb838 swap
50422 50418 50417 1001 S+ ufs 0xc54bb838 swap
50421 50418 50417 1001 S+ ufs 0xc54bb838 swap
50420 50418 50417 1001 S+ nanslp 0xc0a092a4 swap
50419 50418 50417 1001 R+ CPU 1 swap
50418 50417 50417 1001 RL+ CPU 3 swap
50417 1550 50417 1001 S+ nanslp 0xc0a092a4 swap
50416 50142 50142 0 S+ biowr 0xd7cff424 dd
50142 1013 50142 0 S+ wait 0xc3f76000 sh
50111 0 0 0 SL mdwait 0xc503a800 [md0]
19972 19871 19972 0 S+ select 0xc0a55d7c top
19871 19855 19871 0 SW+ wait 0xc4049d38 bash
19855 19854 19855 0 SW+ pause 0xc412ad6c csh
19854 857 19854 1001 SW+ wait 0xc4dd8468 su
1550 1542 1550 1001 Ss+ wait 0xc40f7000 bash
1542 1533 1533 1001 S select 0xc0a55d7c sshd
1533 841 1533 0 SWs sbwait 0xc45775cc sshd
1013 1010 1013 0 SW+ wait 0xc4129000 bash
1010 1009 1010 0 SW+ pause 0xc4040b38 csh
1009 1003 1009 1001 SW+ wait 0xc40f8468 su
1003 1002 1003 1001 SWs+ wait 0xc40028d0 bash
1002 1000 1000 1001 S select 0xc0a55d7c sshd
1000 841 1000 0 SWs sbwait 0xc41b0d94 sshd
999 993 999 1001 S+ kqread 0xc44f8e00 tail
993 992 993 1001 SWs+ wait 0xc4129234 bash
992 990 990 1001 S select 0xc0a55d7c sshd
990 841 990 0 SWs sbwait 0xc4124480 sshd
988 1 988 0 SWs+ ttyin 0xc3dfe810 getty
987 1 987 0 SWs+ ttyin 0xc3e02810 getty
986 1 986 0 SWs+ ttyin 0xc3e01810 getty
985 1 985 0 SWs+ ttyin 0xc3e01010 getty
984 1 984 0 SWs+ ttyin 0xc3e00c10 getty
983 1 983 0 SWs+ ttyin 0xc3e02c10 getty
982 1 982 0 SWs+ ttyin 0xc3e03010 getty
981 1 981 0 SWs+ ttyin 0xc3e01c10 getty
964 1 964 0 SWs select 0xc0a55d7c inetd
935 1 935 0 SWs select 0xc0a55d7c moused
917 1 917 0 Ss nanslp 0xc0a092a4 watchdogd
864 1 864 0 Ss nanslp 0xc0a092a4 cron
857 850 857 1001 SWs+ wait 0xc40f769c bash
855 1 855 25 SWs pause 0xc3f78d6c sendmail
851 1 851 0 Ss select 0xc0a55d7c sendmail
850 848 848 1001 S select 0xc0a55d7c sshd
848 841 848 0 SWs sbwait 0xc41245cc sshd
841 1 841 0 SWs select 0xc0a55d7c sshd
821 1 821 0 Ss select 0xc0a55d7c ntpd
761 757 757 0 SW - 0xc3facc00 nfsd
760 757 757 0 SW - 0xc3f69600 nfsd
759 757 757 0 SW - 0xc3f69800 nfsd
758 757 757 0 SW - 0xc3faca00 nfsd
757 1 757 0 SWs select 0xc0a55d7c nfsd
755 1 755 0 SWs select 0xc0a55d7c mountd
696 1 696 0 Ss select 0xc0a55d7c rpcbind
673 1 673 0 Ss select 0xc0a55d7c syslogd
563 1 563 0 SWs select 0xc0a55d7c devd
42 0 0 0 SL - 0xe43f4cfc [schedcpu]
41 0 0 0 SL sdflush 0xc0a631e0 [softdepflush]
40 0 0 0 SL vlruwt 0xc3f76d38 [vnlru]
39 0 0 0 SL syncer 0xc0a09070 [syncer]
38 0 0 0 SL psleep 0xc0a561f4 [bufdaemon]
37 0 0 0 SL pgzero 0xc0a6bb10 [pagezero]
36 0 0 0 SL psleep 0xc0a63a40 [vmdaemon]
35 0 0 0 SL psleep 0xc0a63a00 [pagedaemon]
34 0 0 0 WL [irq7: ppc0]
33 0 0 0 SL - 0xc3d7e03c [fdc0]
32 0 0 0 WL [swi0: sio]
31 0 0 0 WL [irq12: psm0]
30 0 0 0 WL [irq1: atkbd0]
29 0 0 0 WL [irq15: ata1]
28 0 0 0 WL [irq14: ata0]
27 0 0 0 WL [irq17: fxp0]
26 0 0 0 SL usbtsk 0xc0a06824 [usbtask]
25 0 0 0 SL usbevt 0xc3d6d210 [usb0]
24 0 0 0 WL [irq16: uhci0]
23 0 0 0 SL - 0xc3d5a280 [em0 taskq]
22 0 0 0 WL [irq9: acpi0]
21 0 0 0 WL [swi5: +]
9 0 0 0 SL - 0xc3c85700 [thread taskq]
20 0 0 0 WL [swi6: Giant taskq]
8 0 0 0 SL - 0xc3c85880 [acpi_task_2]
7 0 0 0 SL - 0xc3c85880 [acpi_task_1]
6 0 0 0 SL - 0xc3c85880 [acpi_task_0]
19 0 0 0 WL [swi6: task queue]
5 0 0 0 SL - 0xc3c85a00 [kqueue taskq]
18 0 0 0 WL [swi2: cambio]
17 0 0 0 SL - 0xc0a04460 [yarrow]
4 0 0 0 RL CPU 0 [g_down]
3 0 0 0 SL - 0xc0a06fd8 [g_up]
2 0 0 0 SL - 0xc0a06fd0 [g_event]
16 0 0 0 WL [swi3: vm]
15 0 0 0 RL [swi4: clock sio]
14 0 0 0 WL [swi1: net]
13 0 0 0 RL [idle: cpu0]
12 0 0 0 RL [idle: cpu1]
11 0 0 0 RL [idle: cpu2]
10 0 0 0 RL [idle: cpu3]
1 0 1 0 SLs wait 0xc3c3a000 [init]
0 0 0 0 WLs [swapper]
db> show sleepchain 50431
thread 100178 (pid 50431, swap) blocked on lk "ufs" EXCL (count 1)
thread 100193 (pid 50419, swap) running on CPU 1
db> show sleepchain 50416
thread 100071 (pid 50416, dd) sleeping on 0xd7cff424 "biowr"
db> allt
Tracing command swap pid 50436 tid 100166 td 0xc460f6c0
fork_trampoline() at fork_trampoline
Tracing command swap pid 50435 tid 100222 td 0xc4c4d000
fork_trampoline() at fork_trampoline
Tracing command swap pid 50434 tid 100103 td 0xc3f77d80
fork_trampoline() at fork_trampoline
Tracing command swap pid 50433 tid 100227 td 0xc4dcda20
fork_trampoline() at fork_trampoline
Tracing command swap pid 50432 tid 100179 td 0xc46576c0
cpustop_handler(e67de4a4,c0894c72,c46576b5,c46576c0,c46576c0,...) at cpustop_handler+0x31
ipi_nmi_handler(c46576b5,c46576c0,c46576c0,0,0,...) at ipi_nmi_handler+0x28
trap(c0a00008,c4650028,c0a70028,c46576c0,c0a785cc,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068d524, esp = 0xe67de4ec, ebp = 0xe67de4fc ---
_mtx_lock_spin(c0a785cc,c46576c0,0,c093c5b8,9fa) at _mtx_lock_spin+0x48
_mtx_lock_spin_flags(c0a785cc,0,c093c5b8,9fa,c0a897a0,...) at _mtx_lock_spin_flags+0xc2
sio_cnputc(c09e5140,74) at sio_cnputc+0x78
cnputc(74) at cnputc+0x4d
putchar(74,e67de678) at putchar+0x52
kvprintf(c0927d13,c06b5ad0,e67de678,a,e67de69c) at kvprintf+0x399
printf(c0927d13,74) at printf+0x43
db_putchar(74,0) at db_putchar+0x24
kvprintf(c08de0bc,c0477d98,0,10,e67de79c,...) at kvprintf+0x6fa
db_printf(c08de0bc,c0b6a8b6) at db_printf+0x1c
db_printsym(c06b3c6d,2,c093db0c,e67de810,e67de888,...) at db_printsym+0x79
db_print_stack_entry(c0b6a8b6,ffffffff,0,e67de890,c06b3c6d) at db_print_stack_entry+0x9c
db_backtrace(c46576c0,0,e67de888,c06b3c6d,3ff,...) at db_backtrace+0x20e
db_trace_self(c091c73b) at db_trace_self+0x2b
kdb_backtrace(e67de8b8,c06f6cfa,c0924b1e,c0924c43,c3fdd2a0,...) at kdb_backtrace+0x29
vfs_badlock(c0924b1e,c0924c43,c3fdd2a0) at vfs_badlock+0x11
assert_vop_locked(c3fdd2a0,c0924c43) at assert_vop_locked+0x4a
vop_lock_post(e67de904,0,1002,c3fdd2a0,e67de920,...) at vop_lock_post+0x2a
VOP_LOCK_APV(c09d7a40,e67de904) at VOP_LOCK_APV+0xa0
vn_lock(c3fdd2a0,1002,c46576c0) at vn_lock+0xac
lookup(e67deb7c) at lookup+0xd2
namei(e67deb7c) at namei+0x382
vn_open_cred(e67deb7c,e67dec7c,0,c46ad680,3,...) at vn_open_cred+0x28f
vn_open(e67deb7c,e67dec7c,0,3) at vn_open+0x1e
kern_open(c46576c0,2815d89f,0,1,0,...) at kern_open+0xb9
open(c46576c0,e67ded04) at open+0x18
syscall(3b,3b,3b,2805188c,2816904c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (5, FreeBSD ELF32, open), eip = 0x28144cb7, esp = 0xbfbfe99c, ebp = 0xbfbfea98 ---
Tracing command swap pid 50431 tid 100178 td 0xc4657870
sched_switch(c4657870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e67e1a80,40,60000,c4657870,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4657870) at lockmgr+0x46e
ffs_lock(e67e1ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e67e1ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4657870) at vn_lock+0xac
lookup(e67e1be8) at lookup+0xd2
namei(e67e1be8) at namei+0x382
kern_readlink(c4657870,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4657870,e67e1d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50430 tid 100136 td 0xc40fad80
sched_switch(c40fad80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6718a80,40,60000,c40fad80,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c40fad80) at lockmgr+0x46e
ffs_lock(e6718ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6718ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c40fad80) at vn_lock+0xac
lookup(e6718be8) at lookup+0xd2
namei(e6718be8) at namei+0x382
kern_readlink(c40fad80,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c40fad80,e6718d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50429 tid 100181 td 0xc4657360
sched_switch(c4657360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e67d8a80,40,60000,c4657360,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4657360) at lockmgr+0x46e
ffs_lock(e67d8ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e67d8ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4657360) at vn_lock+0xac
lookup(e67d8be8) at lookup+0xd2
namei(e67d8be8) at namei+0x382
kern_readlink(c4657360,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4657360,e67d8d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50428 tid 100212 td 0xc4666870
sched_switch(c4666870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e684da80,40,60000,c4666870,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4666870) at lockmgr+0x46e
ffs_lock(e684dad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e684dad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4666870) at vn_lock+0xac
lookup(e684dbe8) at lookup+0xd2
namei(e684dbe8) at namei+0x382
kern_readlink(c4666870,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4666870,e684dd04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50427 tid 100196 td 0xc4177bd0
sched_switch(c4177bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e681da80,40,60000,c4177bd0,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4177bd0) at lockmgr+0x46e
ffs_lock(e681dad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e681dad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4177bd0) at vn_lock+0xac
lookup(e681dbe8) at lookup+0xd2
namei(e681dbe8) at namei+0x382
kern_readlink(c4177bd0,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4177bd0,e681dd04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50426 tid 100204 td 0xc465bd80
sched_switch(c465bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6805a80,40,60000,c465bd80,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c465bd80) at lockmgr+0x46e
ffs_lock(e6805ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6805ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c465bd80) at vn_lock+0xac
lookup(e6805be8) at lookup+0xd2
namei(e6805be8) at namei+0x382
kern_readlink(c465bd80,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c465bd80,e6805d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50425 tid 100070 td 0xc3ffd360
sched_switch(c3ffd360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6637a80,40,60000,c3ffd360,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c3ffd360) at lockmgr+0x46e
ffs_lock(e6637ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6637ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c3ffd360) at vn_lock+0xac
lookup(e6637be8) at lookup+0xd2
namei(e6637be8) at namei+0x382
kern_readlink(c3ffd360,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c3ffd360,e6637d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50424 tid 100214 td 0xc4666510
sched_switch(c4666510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6847a80,40,60000,c4666510,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4666510) at lockmgr+0x46e
ffs_lock(e6847ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6847ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4666510) at vn_lock+0xac
lookup(e6847be8) at lookup+0xd2
namei(e6847be8) at namei+0x382
kern_readlink(c4666510,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4666510,e6847d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50423 tid 100172 td 0xc412ebd0
sched_switch(c412ebd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6778a80,40,60000,c412ebd0,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c412ebd0) at lockmgr+0x46e
ffs_lock(e6778ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6778ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c412ebd0) at vn_lock+0xac
lookup(e6778be8) at lookup+0xd2
namei(e6778be8) at namei+0x382
kern_readlink(c412ebd0,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c412ebd0,e6778d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50422 tid 100081 td 0xc40416c0
sched_switch(c40416c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6682a80,40,60000,c40416c0,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c40416c0) at lockmgr+0x46e
ffs_lock(e6682ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6682ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c40416c0) at vn_lock+0xac
lookup(e6682be8) at lookup+0xd2
namei(e6682be8) at namei+0x382
kern_readlink(c40416c0,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c40416c0,e6682d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50421 tid 100141 td 0xc404b510
sched_switch(c404b510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e66b5a80,40,60000,c404b510,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c404b510) at lockmgr+0x46e
ffs_lock(e66b5ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e66b5ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c404b510) at vn_lock+0xac
lookup(e66b5be8) at lookup+0xd2
namei(e66b5be8) at namei+0x382
kern_readlink(c404b510,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c404b510,e66b5d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---
Tracing command swap pid 50420 tid 100147 td 0xc404aa20
sched_switch(c404aa20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a092a4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a092a4) at sleepq_timedwait_sig+0x1e
msleep(c0a092a4,0,15c,c091aec1,3e9,...) at msleep+0x244
kern_nanosleep(c404aa20,e66a3c74,e66a3c6c) at kern_nanosleep+0xab
nanosleep(c404aa20,e66a3d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28137b2b, esp = 0xbfbfe98c, ebp = 0xbfbfe9b8 ---
Tracing command swap pid 50419 tid 100193 td 0xc450b1b0
kdb_enter(c0919d8a) at kdb_enter+0x2b
panic(c091e90d,c0935f22,c09327d1,c09327e7,14f2,...) at panic+0x14b
witness_unlock(c0a631b8,8,c09327e7,14f2) at witness_unlock+0xcf
_mtx_unlock_flags(c0a631b8,0,c09327de,14f2,2,...) at _mtx_unlock_flags+0x8c
softdep_sync_metadata(c54bb7e0) at softdep_sync_metadata+0x62f
ffs_syncvnode(c54bb7e0,1) at ffs_syncvnode+0x33b
ffs_truncate(c54bb7e0,200,0,880,c46ad680,c450b1b0) at ffs_truncate+0x4d6
ufs_direnter(c54bb7e0,c4ec6540,e68269e4,e6826bc8,0,c4f5bd68,e6826bc8,e68269e4) at ufs_direnter+0x813
ufs_makeinode(a1ff,c54bb7e0,e6826bb4,e6826bc8) at ufs_makeinode+0x462
ufs_symlink(e6826b60) at ufs_symlink+0x2b
VOP_SYMLINK_APV(c09d7a40,e6826b60) at VOP_SYMLINK_APV+0x9b
kern_symlink(c450b1b0,bfbfe9d0,804bda0,0,e6826d30,...) at kern_symlink+0x333
symlink(c450b1b0,e6826d04) at symlink+0x15
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (57, FreeBSD ELF32, symlink), eip = 0x280cb523, esp = 0xbfbfe9bc, ebp = 0xbfbfea18 ---
Tracing command swap pid 50418 tid 100153 td 0xc404a000
cpustop_handler(e6691724,c0894c72,c068d279,c3ff9200,c404a000,...) at cpustop_handler+0x31
ipi_nmi_handler(c068d279,c3ff9200,c404a000,0,0,...) at ipi_nmi_handler+0x28
trap(c0880008,e6690028,c0680028,1,e6691788,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc086d82c, esp = 0xe669176c, ebp = 0xe6691774 ---
siocnopen(e6691788,3f8,2580,30f97a0,e66917a4,...) at siocnopen+0x28
sio_cnputc(c09e5140,20) at sio_cnputc+0x8b
cnputc(20) at cnputc+0x4d
putchar(20,e66918c4) at putchar+0x52
kvprintf(c0927d13,c06b5ad0,e66918c4,a,e66918e8) at kvprintf+0x399
printf(c0927d13,20) at printf+0x43
db_putchar(20,0) at db_putchar+0x24
kvprintf(c093db0c,c0477d98,0,10,e66919e4,...) at kvprintf+0x81
db_printf(c093db0c,e6691a24,e6691a9c,c0ae1a70,e6691a70,...) at db_printf+0x1c
db_print_stack_entry(c0b6a8b6,ffffffff,0,e6691aa4,c06b3c6d) at db_print_stack_entry+0x92
db_backtrace(c404a000,0,e6691a9c,c06b3c6d,3ff,...) at db_backtrace+0x20e
db_trace_self(c091c73b) at db_trace_self+0x2b
kdb_backtrace(e6691acc,c06f6cfa,c0924b1e,c0924c43,c3ff5bd0,...) at kdb_backtrace+0x29
vfs_badlock(c0924b1e,c0924c43,c3ff5bd0) at vfs_badlock+0x11
assert_vop_locked(c3ff5bd0,c0924c43) at assert_vop_locked+0x4a
vop_lock_post(e6691b18,0,3041,c3ff5bd0,e6691b34,...) at vop_lock_post+0x2a
VOP_LOCK_APV(c09d7a40,e6691b18) at VOP_LOCK_APV+0xa0
vn_lock(c3ff5bd0,3041,c404a000,c3ff5bd0) at vn_lock+0xac
vget(c3ff5bd0,3041,c404a000) at vget+0xc2
vnode_pager_lock(c184c1e0,c184c1e0,c184c1e0,0,c0935a3c,...) at vnode_pager_lock+0x127
vm_fault(c3c3ce38,280be000,1,0,c404a000,...) at vm_fault+0x227
trap_pfault(e6691d38,1,280be68c,280be68c,280be68c,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfec04,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x280be68c, esp = 0xbfbfeabc, ebp = 0xbfbfead8 ---
Tracing command swap pid 50417 tid 100224 td 0xc3fe76c0
sched_switch(c3fe76c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a092a4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a092a4) at sleepq_timedwait_sig+0x1e
msleep(c0a092a4,0,15c,c091aec1,3e9,...) at msleep+0x244
kern_nanosleep(c3fe76c0,e6613c74,e6613c6c) at kern_nanosleep+0xab
nanosleep(c3fe76c0,e6613d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28137b2b, esp = 0xbfbfea9c, ebp = 0xbfbfeac8 ---
Tracing command dd pid 50416 tid 100071 td 0xc3ffd1b0
sched_switch(c3ffd1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7cff424,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(d7cff424,0,c09225f2,ead,8,...) at sleepq_wait+0x46
msleep(d7cff424,c0a56248,4c,c0922d0c,0) at msleep+0x27d
bwait(d7cff424,4c,c0922d0c) at bwait+0x47
bufwait(d7cff424,d7cff424,0,e663482c,c07d8c36,...) at bufwait+0x2d
bufwrite(d7cff424,d7cff424,0,e6634850,c07d34ed,...) at bufwrite+0x168
ffs_bufwrite(d7cff424) at ffs_bufwrite+0x282
softdep_sync_metadata(c5355bd0) at softdep_sync_metadata+0x379
ffs_syncvnode(c5355bd0,1) at ffs_syncvnode+0x33b
ffs_truncate(c5355bd0,e00000,0,800,c44f8780,...) at ffs_truncate+0x4d6
ffs_write(e6634b9c) at ffs_write+0x5ef
VOP_WRITE_APV(c09d7a40,e6634b9c) at VOP_WRITE_APV+0x132
vn_write(c3feb3f0,e6634c64,c44f8780,0,c3ffd1b0) at vn_write+0x1ea
dofilewrite(c3ffd1b0,4,c3feb3f0,e6634c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c3ffd1b0,4,e6634c64,84f0000,10000,...) at kern_writev+0x36
write(c3ffd1b0,e6634d04) at write+0x45
syscall(3b,bfbf003b,bfbf003b,8400000,100000,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x281530f7, esp = 0xbfbfea9c, ebp = 0xbfbfeac8 ---
Tracing command sh pid 50142 tid 100042 td 0xc3de5000
sched_switch(c3de5000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3f76000) at sleepq_switch+0xcd
sleepq_wait_sig(c3f76000) at sleepq_wait_sig+0x1d
msleep(c3f76000,c3f76068,15c,c091f239,0) at msleep+0x26e
kern_wait(c3de5000,ffffffff,e4392c2c,2,0) at kern_wait+0x7cf
wait4(c3de5000,e4392d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe688,c3de,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe63c, ebp = 0xbfbfe658 ---
Tracing command md0 pid 50111 tid 100169 td 0xc460f1b0
sched_switch(c460f1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c503a800,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c503a800,0,c08f9e1d,28f,8,...) at sleepq_wait+0x46
msleep(c503a800,c503a820,24c,c08f9e5e,0) at msleep+0x27d
md_kthread(c503a800,e679fd38) at md_kthread+0xb1
fork_exit(c057f134,c503a800,e679fd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe679fd6c, ebp = 0 ---
Tracing command top pid 19972 tid 100043 td 0xc3d46d80
sched_switch(c3d46d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a55d7c) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a55d7c,c0a55d7c,3e9,e4150ae8,246,...) at sleepq_timedwait_sig+0x1e
cv_timedwait_sig(c0a55d7c,c0a55d64,3e9) at cv_timedwait_sig+0x196
kern_select(c3d46d80,2,bfbfea50,0,0,...) at kern_select+0x4a9
select(c3d46d80,e4150d04) at select+0x44
syscall(3b,3b,bfbf003b,bfbfea50,1c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x281b3097, esp = 0xbfbfe9cc, ebp = 0xbfbfeb48 ---
Tracing command bash pid 19871 tid 100097 td 0xc404b870
db> bt 4
Tracing pid 4 tid 100014 td 0xc3c37bd0
cpustop_handler(e26f1b10,c0894c72,a,f000,da7a,...) at cpustop_handler+0x31
ipi_nmi_handler(a,f000,da7a,0,0,...) at ipi_nmi_handler+0x28
trap(8,c3c30028,28,c3c37bd0,c0a785cc,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068d524, esp = 0xe26f1b58, ebp = 0xe26f1b68 ---
_mtx_lock_spin(c0a785cc,c3c37bd0,0,c093c5b8,56e) at _mtx_lock_spin+0x48
_mtx_lock_spin_flags(c0a785cc,0,c093c5b8,56e,e26f1bc8,...) at _mtx_lock_spin_flags+0xc2
siointr(c3df3c00) at siointr+0x19
intr_execute_handlers(c3c2c0c4,e26f1be8,4,e26f1c44,c0880743,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc04e735b, esp = 0xe26f1c28, ebp = 0xe26f1c44 ---
ata_pci_dmastart(c3e12600) at ata_pci_dmastart+0x73
ata_begin_transaction(c61c8a80) at ata_begin_transaction+0x266
ata_start(c3d5bc80,c3d3a908,0,c08e85f7,5d) at ata_start+0x1a0
ata_queue_request(c61c8a80,c3e0c800,c5f28dec,c3e12500,0,...) at ata_queue_request+0x21a
ad_strategy(c5f28dec) at ad_strategy+0x161
g_disk_start(c61165ac) at g_disk_start+0x168
g_io_schedule_down(c3c37bd0) at g_io_schedule_down+0x15a
g_down_procbody(0,e26f1d38) at g_down_procbody+0x5a
fork_exit(c065f5d0,0,e26f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f1d6c, ebp = 0 ---
db> call doadump
Physical memory: 1007 MB
Dumping 201 MB:
(kgdb) l *_mtx_unlock_flags+0x8c
0xc068d098 is in _mtx_unlock_flags (../../../kern/kern_mutex.c:314).
309 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
310 ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
311 file, line));
312 curthread->td_locks--;
313 WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
314 LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file,
315 line);
316 mtx_assert(m, MA_OWNED);
317 #ifdef MUTEX_PROFILING
318 if (m->mtx_acqtime != 0) {
(kgdb) l *softdep_sync_metadata+0x62f
0xc07d37a3 is in softdep_sync_metadata (../../../ufs/ffs/ffs_softdep.c:5363).
5358 bp->b_lock.lk_flags &= ~LK_CANRECURSE;
5359 bawrite(bp);
5360 return (error);
5361 }
5362 FREE_LOCK(&lk);
5363 VI_LOCK(vp);
5364 while ((nbp = TAILQ_NEXT(bp, b_bobufs)) != NULL) {
5365 nbp = getdirtybuf(nbp, VI_MTX(vp), MNT_WAIT);
5366 if (nbp)
5367 break;