GENERIC HEAD from Sep 11 09:02 UTC + Konstantin's vfs_mount.c patch, No vmcore
Panic while stress testing mksnap_ffs

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Tue Sep 12 08:44:40 CEST 2006
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1032458240 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
pKKaDBDnB::  istasct:ack  clbok cakbackt r(cskltereapc emacu:te:e
xk)d b
Skodbf_t_dbbaeapc cLkkttroacrkc ancoet( eleo6(7ecdk6e69d1 ac@ec8 ,b8ucf,06c0f6f6s6/cfffcsf/aaf,,cf0sc9_09s2o24f4btbd1e1pee.,c,c0c:0599232642c44
3ccp,u4icd3 3=, f1cf3fd5
dK2DaB0b,:. de..n0t)e ,.ra:t  kp.dab_n.i)cb aatck
[thread pid 50419 tid 100193 ]
Stopped at      kdb_enter+0x2b: nop
db> where
Tracing pid 50419 tid 100193 td 0xc450b1b0
kdb_enter(c0919d8a) at kdb_enter+0x2b
panic(c091e90d,c0935f22,c09327d1,c09327e7,14f2,...) at panic+0x14b
witness_unlock(c0a631b8,8,c09327e7,14f2) at witness_unlock+0xcf
_mtx_unlock_flags(c0a631b8,0,c09327de,14f2,2,...) at _mtx_unlock_flags+0x8c
softdep_sync_metadata(c54bb7e0) at softdep_sync_metadata+0x62f
ffs_syncvnode(c54bb7e0,1) at ffs_syncvnode+0x33b
ffs_truncate(c54bb7e0,200,0,880,c46ad680,c450b1b0) at ffs_truncate+0x4d6
ufs_direnter(c54bb7e0,c4ec6540,e68269e4,e6826bc8,0,c4f5bd68,e6826bc8,e68269e4) at ufs_direnter+0x813
ufs_makeinode(a1ff,c54bb7e0,e6826bb4,e6826bc8) at ufs_makeinode+0x462
ufs_symlink(e6826b60) at ufs_symlink+0x2b
VOP_SYMLINK_APV(c09d7a40,e6826b60) at VOP_SYMLINK_APV+0x9b
kern_symlink(c450b1b0,bfbfe9d0,804bda0,0,e6826d30,...) at kern_symlink+0x333
symlink(c450b1b0,e6826d04) at symlink+0x15
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (57, FreeBSD ELF32, symlink), eip = 0x280cb523, esp = 0xbfbfe9bc, ebp = 0xbfbfea18 ---
db> show alllocks
Process 50432 (swap) thread 0xc46576c0 (100179)
exclusive sleep mutex vm object (standard object) r = 0 (0xc5d40438) locked @ vm/vm_fault.c:900
exclusive sx user map r = 0 (0xc47bca6c) locked @ vm/vm_map.c:3073
Process 50418 (swap) thread 0xc404a000 (100153)
exclusive sleep mutex pmap r = 0 (0xc47bcddc) locked @ i386/i386/pmap.c:2772
exclusive sleep mutex pmap r = 0 (0xc3c3cee0) locked @ i386/i386/pmap.c:2771
exclusive sleep mutex vm page queue mutex r = 0 (0xc0a639dc) locked @ i386/i386/pmap.c:2766
exclusive sx user map r = 0 (0xc3c3ce7c) locked @ vm/vm_map.c:2561
Process 4 (g_down) thread 0xc3c37bd0 (100014)
exclusive sleep mutex ATA queue lock r = 0 (0xc3d3a908) locked @ dev/ata/ata-queue.c:177
db> show lockedvnods
Locked vnodes

0xc54bb7e0: tag ufs, type VDIR
    usecount 32, writecount 0, refcount 34 mountedhere 0
    flags ()
    v_object 0xc4d002d0 ref 0 pages 1
     lock type ufs: EXCL (count 1) by thread 0xc450b1b0 (pid 50419) with 11 pending#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d91ea at ffs_lock+0x76
#2 0xc08a8127 at VOP_LOCK_APV+0x87
#3 0xc0700b1c at vn_lock+0xac
#4 0xc06ed512 at lookup+0xd2
#5 0xc06ed1de at namei+0x382
#6 0xc06fa444 at kern_symlink+0x90
#7 0xc06fa3b1 at symlink+0x15
#8 0xc08959f2 at syscall+0x256
#9 0xc08803df at Xint0x80_syscall+0x1f

        ino 84, on dev ad0s1e

0xc5355bd0: tag ufs, type VREG
    usecount 1, writecount 1, refcount 959 mountedhere 0
    flags ()
    v_object 0xc4991618 ref 0 pages 3828
     lock type ufs: EXCL (count 1) by thread 0xc3ffd1b0 (pid 50416)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d91ea at ffs_lock+0x76
#2 0xc08a8127 at VOP_LOCK_APV+0x87
#3 0xc0700b1c at vn_lock+0xac
#4 0xc0700484 at vn_write+0x138
#5 0xc06c125b at dofilewrite+0x7b
#6 0xc06c10fe at kern_writev+0x36
#7 0xc06c1029 at write+0x45
#8 0xc08959f2 at syscall+0x256
#9 0xc08803df at Xint0x80_syscall+0x1f

        ino 140, on dev ad0s1e
db> show lockedbufs
buf at 0xd7cff424
b_flags = 0xa0000020<remfree,vmio,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc40bd8e0), b_data = 0xd8e2c000, b_blkno = 3387360
b_npages = 4, pages(OBJ, IDX, PA): (0xc4062ca8, 0x675fc, 0x115e9000),(0xc4062ca8, 0x675fd, 0x6f2a000),(0xc4062ca8, 0x675fe, 0x3e9b000),(0xc4062ca8, 0x675ff, 0xe85c000)
 lock type bufwait: EXCL (count 1) by thread 0xc3ffd1b0 (pid 50416)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d4af0 at getdirtybuf+0x60
#2 0xc07d3483 at softdep_sync_metadata+0x30f
#3 0xc07d910f at ffs_syncvnode+0x33b
#4 0xc07c3fc2 at ffs_truncate+0x4d6
#5 0xc07d9c87 at ffs_write+0x5ef
#6 0xc08a6e2a at VOP_WRITE_APV+0x132
#7 0xc0700536 at vn_write+0x1ea
#8 0xc06c125b at dofilewrite+0x7b
#9 0xc06c10fe at kern_writev+0x36
#10 0xc06c1029 at write+0x45
#11 0xc08959f2 at syscall+0x256
#12 0xc08803df at Xint0x80_syscall+0x1f

buf at 0xd7d93a20
b_flags = 0x20000000<vmio>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc40bd8e0), b_data = 0xda4c8000, b_blkno = 23488
b_npages = 4, pages(OBJ, IDX, PA): (0xc4062ca8, 0xb78, 0xce65000),(0xc4062ca8, 0xb79, 0x2a036000),(0xc4062ca8, 0xb7a, 0x36057000),(0xc4062ca8, 0xb7b, 0x14418000)
db> show allpcpu
Current CPU: 1

cpuid        = 0
curthread    = 0xc3c37bd0: pid 4 "g_down"
curpcb       = 0xe26f1d90
fpcurthread  = none
idlethread   = 0xc3c36a20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc450b1b0: pid 50419 "swap"
curpcb       = 0xe6826d90
fpcurthread  = none
idlethread   = 0xc3c36870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc46576c0: pid 50432 "swap"
curpcb       = 0xe67ded90
fpcurthread  = none
idlethread   = 0xc3c366c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc404a000: pid 50418 "swap"
curpcb       = 0xe6691d90
fpcurthread  = none
idlethread   = 0xc3c36510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
50436 50418 50417  1001  R+                          swap
50435 50418 50417  1001  R+                          swap
50434 50418 50417  1001  R+                          swap
50433 50418 50417  1001  R+                          swap
50432 50418 50417  1001  R+      CPU 2               swap
50431 50418 50417  1001  S+      ufs      0xc54bb838 swap
50430 50418 50417  1001  S+      ufs      0xc54bb838 swap
50429 50418 50417  1001  S+      ufs      0xc54bb838 swap
50428 50418 50417  1001  S+      ufs      0xc54bb838 swap
50427 50418 50417  1001  S+      ufs      0xc54bb838 swap
50426 50418 50417  1001  S+      ufs      0xc54bb838 swap
50425 50418 50417  1001  S+      ufs      0xc54bb838 swap
50424 50418 50417  1001  S+      ufs      0xc54bb838 swap
50423 50418 50417  1001  S+      ufs      0xc54bb838 swap
50422 50418 50417  1001  S+      ufs      0xc54bb838 swap
50421 50418 50417  1001  S+      ufs      0xc54bb838 swap
50420 50418 50417  1001  S+      nanslp   0xc0a092a4 swap
50419 50418 50417  1001  R+      CPU 1               swap
50418 50417 50417  1001  RL+     CPU 3               swap
50417  1550 50417  1001  S+      nanslp   0xc0a092a4 swap
50416 50142 50142     0  S+      biowr    0xd7cff424 dd
50142  1013 50142     0  S+      wait     0xc3f76000 sh
50111     0     0     0  SL      mdwait   0xc503a800 [md0]
19972 19871 19972     0  S+      select   0xc0a55d7c top
19871 19855 19871     0  SW+     wait     0xc4049d38 bash
19855 19854 19855     0  SW+     pause    0xc412ad6c csh
19854   857 19854  1001  SW+     wait     0xc4dd8468 su
 1550  1542  1550  1001  Ss+     wait     0xc40f7000 bash
 1542  1533  1533  1001  S       select   0xc0a55d7c sshd
 1533   841  1533     0  SWs     sbwait   0xc45775cc sshd
 1013  1010  1013     0  SW+     wait     0xc4129000 bash
 1010  1009  1010     0  SW+     pause    0xc4040b38 csh
 1009  1003  1009  1001  SW+     wait     0xc40f8468 su
 1003  1002  1003  1001  SWs+    wait     0xc40028d0 bash
 1002  1000  1000  1001  S       select   0xc0a55d7c sshd
 1000   841  1000     0  SWs     sbwait   0xc41b0d94 sshd
  999   993   999  1001  S+      kqread   0xc44f8e00 tail
  993   992   993  1001  SWs+    wait     0xc4129234 bash
  992   990   990  1001  S       select   0xc0a55d7c sshd
  990   841   990     0  SWs     sbwait   0xc4124480 sshd
  988     1   988     0  SWs+    ttyin    0xc3dfe810 getty
  987     1   987     0  SWs+    ttyin    0xc3e02810 getty
  986     1   986     0  SWs+    ttyin    0xc3e01810 getty
  985     1   985     0  SWs+    ttyin    0xc3e01010 getty
  984     1   984     0  SWs+    ttyin    0xc3e00c10 getty
  983     1   983     0  SWs+    ttyin    0xc3e02c10 getty
  982     1   982     0  SWs+    ttyin    0xc3e03010 getty
  981     1   981     0  SWs+    ttyin    0xc3e01c10 getty
  964     1   964     0  SWs     select   0xc0a55d7c inetd
  935     1   935     0  SWs     select   0xc0a55d7c moused
  917     1   917     0  Ss      nanslp   0xc0a092a4 watchdogd
  864     1   864     0  Ss      nanslp   0xc0a092a4 cron
  857   850   857  1001  SWs+    wait     0xc40f769c bash
  855     1   855    25  SWs     pause    0xc3f78d6c sendmail
  851     1   851     0  Ss      select   0xc0a55d7c sendmail
  850   848   848  1001  S       select   0xc0a55d7c sshd
  848   841   848     0  SWs     sbwait   0xc41245cc sshd
  841     1   841     0  SWs     select   0xc0a55d7c sshd
  821     1   821     0  Ss      select   0xc0a55d7c ntpd
  761   757   757     0  SW      -        0xc3facc00 nfsd
  760   757   757     0  SW      -        0xc3f69600 nfsd
  759   757   757     0  SW      -        0xc3f69800 nfsd
  758   757   757     0  SW      -        0xc3faca00 nfsd
  757     1   757     0  SWs     select   0xc0a55d7c nfsd
  755     1   755     0  SWs     select   0xc0a55d7c mountd
  696     1   696     0  Ss      select   0xc0a55d7c rpcbind
  673     1   673     0  Ss      select   0xc0a55d7c syslogd
  563     1   563     0  SWs     select   0xc0a55d7c devd
   42     0     0     0  SL      -        0xe43f4cfc [schedcpu]
   41     0     0     0  SL      sdflush  0xc0a631e0 [softdepflush]
   40     0     0     0  SL      vlruwt   0xc3f76d38 [vnlru]
   39     0     0     0  SL      syncer   0xc0a09070 [syncer]
   38     0     0     0  SL      psleep   0xc0a561f4 [bufdaemon]
   37     0     0     0  SL      pgzero   0xc0a6bb10 [pagezero]
   36     0     0     0  SL      psleep   0xc0a63a40 [vmdaemon]
   35     0     0     0  SL      psleep   0xc0a63a00 [pagedaemon]
   34     0     0     0  WL                          [irq7: ppc0]
   33     0     0     0  SL      -        0xc3d7e03c [fdc0]
   32     0     0     0  WL                          [swi0: sio]
   31     0     0     0  WL                          [irq12: psm0]
   30     0     0     0  WL                          [irq1: atkbd0]
   29     0     0     0  WL                          [irq15: ata1]
   28     0     0     0  WL                          [irq14: ata0]
   27     0     0     0  WL                          [irq17: fxp0]
   26     0     0     0  SL      usbtsk   0xc0a06824 [usbtask]
   25     0     0     0  SL      usbevt   0xc3d6d210 [usb0]
   24     0     0     0  WL                          [irq16: uhci0]
   23     0     0     0  SL      -        0xc3d5a280 [em0 taskq]
   22     0     0     0  WL                          [irq9: acpi0]
   21     0     0     0  WL                          [swi5: +]
    9     0     0     0  SL      -        0xc3c85700 [thread taskq]
   20     0     0     0  WL                          [swi6: Giant taskq]
    8     0     0     0  SL      -        0xc3c85880 [acpi_task_2]
    7     0     0     0  SL      -        0xc3c85880 [acpi_task_1]
    6     0     0     0  SL      -        0xc3c85880 [acpi_task_0]
   19     0     0     0  WL                          [swi6: task queue]
    5     0     0     0  SL      -        0xc3c85a00 [kqueue taskq]
   18     0     0     0  WL                          [swi2: cambio]
   17     0     0     0  SL      -        0xc0a04460 [yarrow]
    4     0     0     0  RL      CPU 0               [g_down]
    3     0     0     0  SL      -        0xc0a06fd8 [g_up]
    2     0     0     0  SL      -        0xc0a06fd0 [g_event]
   16     0     0     0  WL                          [swi3: vm]
   15     0     0     0  RL                          [swi4: clock sio]
   14     0     0     0  WL                          [swi1: net]
   13     0     0     0  RL                          [idle: cpu0]
   12     0     0     0  RL                          [idle: cpu1]
   11     0     0     0  RL                          [idle: cpu2]
   10     0     0     0  RL                          [idle: cpu3]
    1     0     1     0  SLs     wait     0xc3c3a000 [init]
    0     0     0     0  WLs                         [swapper]
db> show sleepchain 50431
thread 100178 (pid 50431, swap) blocked on lk "ufs" EXCL (count 1)
thread 100193 (pid 50419, swap) running on CPU 1
db> show sleepchain 50416
thread 100071 (pid 50416, dd) sleeping on 0xd7cff424 "biowr"
db> allt

Tracing command swap pid 50436 tid 100166 td 0xc460f6c0
fork_trampoline() at fork_trampoline

Tracing command swap pid 50435 tid 100222 td 0xc4c4d000
fork_trampoline() at fork_trampoline

Tracing command swap pid 50434 tid 100103 td 0xc3f77d80
fork_trampoline() at fork_trampoline

Tracing command swap pid 50433 tid 100227 td 0xc4dcda20
fork_trampoline() at fork_trampoline

Tracing command swap pid 50432 tid 100179 td 0xc46576c0
cpustop_handler(e67de4a4,c0894c72,c46576b5,c46576c0,c46576c0,...) at cpustop_handler+0x31
ipi_nmi_handler(c46576b5,c46576c0,c46576c0,0,0,...) at ipi_nmi_handler+0x28
trap(c0a00008,c4650028,c0a70028,c46576c0,c0a785cc,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068d524, esp = 0xe67de4ec, ebp = 0xe67de4fc ---
_mtx_lock_spin(c0a785cc,c46576c0,0,c093c5b8,9fa) at _mtx_lock_spin+0x48
_mtx_lock_spin_flags(c0a785cc,0,c093c5b8,9fa,c0a897a0,...) at _mtx_lock_spin_flags+0xc2
sio_cnputc(c09e5140,74) at sio_cnputc+0x78
cnputc(74) at cnputc+0x4d
putchar(74,e67de678) at putchar+0x52
kvprintf(c0927d13,c06b5ad0,e67de678,a,e67de69c) at kvprintf+0x399
printf(c0927d13,74) at printf+0x43
db_putchar(74,0) at db_putchar+0x24
kvprintf(c08de0bc,c0477d98,0,10,e67de79c,...) at kvprintf+0x6fa
db_printf(c08de0bc,c0b6a8b6) at db_printf+0x1c
db_printsym(c06b3c6d,2,c093db0c,e67de810,e67de888,...) at db_printsym+0x79
db_print_stack_entry(c0b6a8b6,ffffffff,0,e67de890,c06b3c6d) at db_print_stack_entry+0x9c
db_backtrace(c46576c0,0,e67de888,c06b3c6d,3ff,...) at db_backtrace+0x20e
db_trace_self(c091c73b) at db_trace_self+0x2b
kdb_backtrace(e67de8b8,c06f6cfa,c0924b1e,c0924c43,c3fdd2a0,...) at kdb_backtrace+0x29
vfs_badlock(c0924b1e,c0924c43,c3fdd2a0) at vfs_badlock+0x11
assert_vop_locked(c3fdd2a0,c0924c43) at assert_vop_locked+0x4a
vop_lock_post(e67de904,0,1002,c3fdd2a0,e67de920,...) at vop_lock_post+0x2a
VOP_LOCK_APV(c09d7a40,e67de904) at VOP_LOCK_APV+0xa0
vn_lock(c3fdd2a0,1002,c46576c0) at vn_lock+0xac
lookup(e67deb7c) at lookup+0xd2
namei(e67deb7c) at namei+0x382
vn_open_cred(e67deb7c,e67dec7c,0,c46ad680,3,...) at vn_open_cred+0x28f
vn_open(e67deb7c,e67dec7c,0,3) at vn_open+0x1e
kern_open(c46576c0,2815d89f,0,1,0,...) at kern_open+0xb9
open(c46576c0,e67ded04) at open+0x18
syscall(3b,3b,3b,2805188c,2816904c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (5, FreeBSD ELF32, open), eip = 0x28144cb7, esp = 0xbfbfe99c, ebp = 0xbfbfea98 ---

Tracing command swap pid 50431 tid 100178 td 0xc4657870
sched_switch(c4657870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e67e1a80,40,60000,c4657870,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4657870) at lockmgr+0x46e
ffs_lock(e67e1ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e67e1ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4657870) at vn_lock+0xac
lookup(e67e1be8) at lookup+0xd2
namei(e67e1be8) at namei+0x382
kern_readlink(c4657870,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4657870,e67e1d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50430 tid 100136 td 0xc40fad80
sched_switch(c40fad80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6718a80,40,60000,c40fad80,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c40fad80) at lockmgr+0x46e
ffs_lock(e6718ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6718ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c40fad80) at vn_lock+0xac
lookup(e6718be8) at lookup+0xd2
namei(e6718be8) at namei+0x382
kern_readlink(c40fad80,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c40fad80,e6718d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50429 tid 100181 td 0xc4657360
sched_switch(c4657360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e67d8a80,40,60000,c4657360,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4657360) at lockmgr+0x46e
ffs_lock(e67d8ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e67d8ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4657360) at vn_lock+0xac
lookup(e67d8be8) at lookup+0xd2
namei(e67d8be8) at namei+0x382
kern_readlink(c4657360,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4657360,e67d8d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50428 tid 100212 td 0xc4666870
sched_switch(c4666870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e684da80,40,60000,c4666870,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4666870) at lockmgr+0x46e
ffs_lock(e684dad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e684dad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4666870) at vn_lock+0xac
lookup(e684dbe8) at lookup+0xd2
namei(e684dbe8) at namei+0x382
kern_readlink(c4666870,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4666870,e684dd04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50427 tid 100196 td 0xc4177bd0
sched_switch(c4177bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e681da80,40,60000,c4177bd0,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4177bd0) at lockmgr+0x46e
ffs_lock(e681dad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e681dad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4177bd0) at vn_lock+0xac
lookup(e681dbe8) at lookup+0xd2
namei(e681dbe8) at namei+0x382
kern_readlink(c4177bd0,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4177bd0,e681dd04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50426 tid 100204 td 0xc465bd80
sched_switch(c465bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6805a80,40,60000,c465bd80,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c465bd80) at lockmgr+0x46e
ffs_lock(e6805ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6805ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c465bd80) at vn_lock+0xac
lookup(e6805be8) at lookup+0xd2
namei(e6805be8) at namei+0x382
kern_readlink(c465bd80,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c465bd80,e6805d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50425 tid 100070 td 0xc3ffd360
sched_switch(c3ffd360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6637a80,40,60000,c3ffd360,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c3ffd360) at lockmgr+0x46e
ffs_lock(e6637ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6637ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c3ffd360) at vn_lock+0xac
lookup(e6637be8) at lookup+0xd2
namei(e6637be8) at namei+0x382
kern_readlink(c3ffd360,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c3ffd360,e6637d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50424 tid 100214 td 0xc4666510
sched_switch(c4666510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6847a80,40,60000,c4666510,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c4666510) at lockmgr+0x46e
ffs_lock(e6847ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6847ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c4666510) at vn_lock+0xac
lookup(e6847be8) at lookup+0xd2
namei(e6847be8) at namei+0x382
kern_readlink(c4666510,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c4666510,e6847d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50423 tid 100172 td 0xc412ebd0
sched_switch(c412ebd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6778a80,40,60000,c412ebd0,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c412ebd0) at lockmgr+0x46e
ffs_lock(e6778ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6778ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c412ebd0) at vn_lock+0xac
lookup(e6778be8) at lookup+0xd2
namei(e6778be8) at namei+0x382
kern_readlink(c412ebd0,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c412ebd0,e6778d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50422 tid 100081 td 0xc40416c0
sched_switch(c40416c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e6682a80,40,60000,c40416c0,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c40416c0) at lockmgr+0x46e
ffs_lock(e6682ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e6682ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c40416c0) at vn_lock+0xac
lookup(e6682be8) at lookup+0xd2
namei(e6682be8) at namei+0x382
kern_readlink(c40416c0,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c40416c0,e6682d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50421 tid 100141 td 0xc404b510
sched_switch(c404b510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c54bb838,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c54bb838,0,c54bb8a8,b7,c0917aae,...) at sleepq_wait+0x46
msleep(c54bb838,c0a080a0,50,c0922061,0,...) at msleep+0x27d
acquire(e66b5a80,40,60000,c404b510,0,...) at acquire+0x76
lockmgr(c54bb838,3002,c54bb8a8,c404b510) at lockmgr+0x46e
ffs_lock(e66b5ad8) at ffs_lock+0x76
VOP_LOCK_APV(c09d7a40,e66b5ad8) at VOP_LOCK_APV+0x87
vn_lock(c54bb7e0,1002,c404b510) at vn_lock+0xac
lookup(e66b5be8) at lookup+0xd2
namei(e66b5be8) at namei+0x382
kern_readlink(c404b510,804bda0,0,bfbfe9d0,0,...) at kern_readlink+0x35
readlink(c404b510,e66b5d04) at readlink+0x1a
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (58, FreeBSD ELF32, readlink), eip = 0x280e7373, esp = 0xbfbfe9bc, ebp = 0xbfbfea28 ---

Tracing command swap pid 50420 tid 100147 td 0xc404aa20
sched_switch(c404aa20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a092a4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a092a4) at sleepq_timedwait_sig+0x1e
msleep(c0a092a4,0,15c,c091aec1,3e9,...) at msleep+0x244
kern_nanosleep(c404aa20,e66a3c74,e66a3c6c) at kern_nanosleep+0xab
nanosleep(c404aa20,e66a3d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28137b2b, esp = 0xbfbfe98c, ebp = 0xbfbfe9b8 ---

Tracing command swap pid 50419 tid 100193 td 0xc450b1b0
kdb_enter(c0919d8a) at kdb_enter+0x2b
panic(c091e90d,c0935f22,c09327d1,c09327e7,14f2,...) at panic+0x14b
witness_unlock(c0a631b8,8,c09327e7,14f2) at witness_unlock+0xcf
_mtx_unlock_flags(c0a631b8,0,c09327de,14f2,2,...) at _mtx_unlock_flags+0x8c
softdep_sync_metadata(c54bb7e0) at softdep_sync_metadata+0x62f
ffs_syncvnode(c54bb7e0,1) at ffs_syncvnode+0x33b
ffs_truncate(c54bb7e0,200,0,880,c46ad680,c450b1b0) at ffs_truncate+0x4d6
ufs_direnter(c54bb7e0,c4ec6540,e68269e4,e6826bc8,0,c4f5bd68,e6826bc8,e68269e4) at ufs_direnter+0x813
ufs_makeinode(a1ff,c54bb7e0,e6826bb4,e6826bc8) at ufs_makeinode+0x462
ufs_symlink(e6826b60) at ufs_symlink+0x2b
VOP_SYMLINK_APV(c09d7a40,e6826b60) at VOP_SYMLINK_APV+0x9b
kern_symlink(c450b1b0,bfbfe9d0,804bda0,0,e6826d30,...) at kern_symlink+0x333
symlink(c450b1b0,e6826d04) at symlink+0x15
syscall(3b,3b,3b,2805188c,bfbfec04,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (57, FreeBSD ELF32, symlink), eip = 0x280cb523, esp = 0xbfbfe9bc, ebp = 0xbfbfea18 ---

Tracing command swap pid 50418 tid 100153 td 0xc404a000
cpustop_handler(e6691724,c0894c72,c068d279,c3ff9200,c404a000,...) at cpustop_handler+0x31
ipi_nmi_handler(c068d279,c3ff9200,c404a000,0,0,...) at ipi_nmi_handler+0x28
trap(c0880008,e6690028,c0680028,1,e6691788,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc086d82c, esp = 0xe669176c, ebp = 0xe6691774 ---
siocnopen(e6691788,3f8,2580,30f97a0,e66917a4,...) at siocnopen+0x28
sio_cnputc(c09e5140,20) at sio_cnputc+0x8b
cnputc(20) at cnputc+0x4d
putchar(20,e66918c4) at putchar+0x52
kvprintf(c0927d13,c06b5ad0,e66918c4,a,e66918e8) at kvprintf+0x399
printf(c0927d13,20) at printf+0x43
db_putchar(20,0) at db_putchar+0x24
kvprintf(c093db0c,c0477d98,0,10,e66919e4,...) at kvprintf+0x81
db_printf(c093db0c,e6691a24,e6691a9c,c0ae1a70,e6691a70,...) at db_printf+0x1c
db_print_stack_entry(c0b6a8b6,ffffffff,0,e6691aa4,c06b3c6d) at db_print_stack_entry+0x92
db_backtrace(c404a000,0,e6691a9c,c06b3c6d,3ff,...) at db_backtrace+0x20e
db_trace_self(c091c73b) at db_trace_self+0x2b
kdb_backtrace(e6691acc,c06f6cfa,c0924b1e,c0924c43,c3ff5bd0,...) at kdb_backtrace+0x29
vfs_badlock(c0924b1e,c0924c43,c3ff5bd0) at vfs_badlock+0x11
assert_vop_locked(c3ff5bd0,c0924c43) at assert_vop_locked+0x4a
vop_lock_post(e6691b18,0,3041,c3ff5bd0,e6691b34,...) at vop_lock_post+0x2a
VOP_LOCK_APV(c09d7a40,e6691b18) at VOP_LOCK_APV+0xa0
vn_lock(c3ff5bd0,3041,c404a000,c3ff5bd0) at vn_lock+0xac
vget(c3ff5bd0,3041,c404a000) at vget+0xc2
vnode_pager_lock(c184c1e0,c184c1e0,c184c1e0,0,c0935a3c,...) at vnode_pager_lock+0x127
vm_fault(c3c3ce38,280be000,1,0,c404a000,...) at vm_fault+0x227
trap_pfault(e6691d38,1,280be68c,280be68c,280be68c,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfec04,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x280be68c, esp = 0xbfbfeabc, ebp = 0xbfbfead8 ---

Tracing command swap pid 50417 tid 100224 td 0xc3fe76c0
sched_switch(c3fe76c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a092a4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a092a4) at sleepq_timedwait_sig+0x1e
msleep(c0a092a4,0,15c,c091aec1,3e9,...) at msleep+0x244
kern_nanosleep(c3fe76c0,e6613c74,e6613c6c) at kern_nanosleep+0xab
nanosleep(c3fe76c0,e6613d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28137b2b, esp = 0xbfbfea9c, ebp = 0xbfbfeac8 ---

Tracing command dd pid 50416 tid 100071 td 0xc3ffd1b0
sched_switch(c3ffd1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7cff424,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(d7cff424,0,c09225f2,ead,8,...) at sleepq_wait+0x46
msleep(d7cff424,c0a56248,4c,c0922d0c,0) at msleep+0x27d
bwait(d7cff424,4c,c0922d0c) at bwait+0x47
bufwait(d7cff424,d7cff424,0,e663482c,c07d8c36,...) at bufwait+0x2d
bufwrite(d7cff424,d7cff424,0,e6634850,c07d34ed,...) at bufwrite+0x168
ffs_bufwrite(d7cff424) at ffs_bufwrite+0x282
softdep_sync_metadata(c5355bd0) at softdep_sync_metadata+0x379
ffs_syncvnode(c5355bd0,1) at ffs_syncvnode+0x33b
ffs_truncate(c5355bd0,e00000,0,800,c44f8780,...) at ffs_truncate+0x4d6
ffs_write(e6634b9c) at ffs_write+0x5ef
VOP_WRITE_APV(c09d7a40,e6634b9c) at VOP_WRITE_APV+0x132
vn_write(c3feb3f0,e6634c64,c44f8780,0,c3ffd1b0) at vn_write+0x1ea
dofilewrite(c3ffd1b0,4,c3feb3f0,e6634c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c3ffd1b0,4,e6634c64,84f0000,10000,...) at kern_writev+0x36
write(c3ffd1b0,e6634d04) at write+0x45
syscall(3b,bfbf003b,bfbf003b,8400000,100000,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x281530f7, esp = 0xbfbfea9c, ebp = 0xbfbfeac8 ---

Tracing command sh pid 50142 tid 100042 td 0xc3de5000
sched_switch(c3de5000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3f76000) at sleepq_switch+0xcd
sleepq_wait_sig(c3f76000) at sleepq_wait_sig+0x1d
msleep(c3f76000,c3f76068,15c,c091f239,0) at msleep+0x26e
kern_wait(c3de5000,ffffffff,e4392c2c,2,0) at kern_wait+0x7cf
wait4(c3de5000,e4392d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe688,c3de,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe63c, ebp = 0xbfbfe658 ---

Tracing command md0 pid 50111 tid 100169 td 0xc460f1b0
sched_switch(c460f1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c503a800,c0a08a90,0,c091d176,211,...) at sleepq_switch+0xcd
sleepq_wait(c503a800,0,c08f9e1d,28f,8,...) at sleepq_wait+0x46
msleep(c503a800,c503a820,24c,c08f9e5e,0) at msleep+0x27d
md_kthread(c503a800,e679fd38) at md_kthread+0xb1
fork_exit(c057f134,c503a800,e679fd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe679fd6c, ebp = 0 ---

Tracing command top pid 19972 tid 100043 td 0xc3d46d80
sched_switch(c3d46d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a55d7c) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a55d7c,c0a55d7c,3e9,e4150ae8,246,...) at sleepq_timedwait_sig+0x1e
cv_timedwait_sig(c0a55d7c,c0a55d64,3e9) at cv_timedwait_sig+0x196
kern_select(c3d46d80,2,bfbfea50,0,0,...) at kern_select+0x4a9
select(c3d46d80,e4150d04) at select+0x44
syscall(3b,3b,bfbf003b,bfbfea50,1c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x281b3097, esp = 0xbfbfe9cc, ebp = 0xbfbfeb48 ---

Tracing command bash pid 19871 tid 100097 td 0xc404b870
db> bt 4
Tracing pid 4 tid 100014 td 0xc3c37bd0
cpustop_handler(e26f1b10,c0894c72,a,f000,da7a,...) at cpustop_handler+0x31
ipi_nmi_handler(a,f000,da7a,0,0,...) at ipi_nmi_handler+0x28
trap(8,c3c30028,28,c3c37bd0,c0a785cc,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc068d524, esp = 0xe26f1b58, ebp = 0xe26f1b68 ---
_mtx_lock_spin(c0a785cc,c3c37bd0,0,c093c5b8,56e) at _mtx_lock_spin+0x48
_mtx_lock_spin_flags(c0a785cc,0,c093c5b8,56e,e26f1bc8,...) at _mtx_lock_spin_flags+0xc2
siointr(c3df3c00) at siointr+0x19
intr_execute_handlers(c3c2c0c4,e26f1be8,4,e26f1c44,c0880743,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc04e735b, esp = 0xe26f1c28, ebp = 0xe26f1c44 ---
ata_pci_dmastart(c3e12600) at ata_pci_dmastart+0x73
ata_begin_transaction(c61c8a80) at ata_begin_transaction+0x266
ata_start(c3d5bc80,c3d3a908,0,c08e85f7,5d) at ata_start+0x1a0
ata_queue_request(c61c8a80,c3e0c800,c5f28dec,c3e12500,0,...) at ata_queue_request+0x21a
ad_strategy(c5f28dec) at ad_strategy+0x161
g_disk_start(c61165ac) at g_disk_start+0x168
g_io_schedule_down(c3c37bd0) at g_io_schedule_down+0x15a
g_down_procbody(0,e26f1d38) at g_down_procbody+0x5a
fork_exit(c065f5d0,0,e26f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f1d6c, ebp = 0 ---
db> call doadump
Physical memory: 1007 MB
Dumping 201 MB:

(kgdb) l *_mtx_unlock_flags+0x8c
0xc068d098 is in _mtx_unlock_flags (../../../kern/kern_mutex.c:314).
309             KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
310                 ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
311                 file, line));
312             curthread->td_locks--;
313             WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
314             LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file,
315                 line);
316             mtx_assert(m, MA_OWNED);
317     #ifdef MUTEX_PROFILING
318             if (m->mtx_acqtime != 0) {
(kgdb) l *softdep_sync_metadata+0x62f
0xc07d37a3 is in softdep_sync_metadata (../../../ufs/ffs/ffs_softdep.c:5363).
5358                    bp->b_lock.lk_flags &= ~LK_CANRECURSE;
5359                    bawrite(bp);
5360                    return (error);
5361            }
5362            FREE_LOCK(&lk);
5363            VI_LOCK(vp);
5364            while ((nbp = TAILQ_NEXT(bp, b_bobufs)) != NULL) {
5365                    nbp = getdirtybuf(nbp, VI_MTX(vp), MNT_WAIT);
5366                    if (nbp)
5367                            break;