GENERIC HEAD from Aug 12 12:55 UTC, vmcore.68
Livelock while accessing /tmp

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Sat Aug 12 15:06:41 CEST 2006
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1033048064 (985 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
mount option <export> is unknown
mount option <export> is unknown
mount option <export> is unknown
fsync: giving up on dirty
0xc400e30c: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 846 mountedhere 0xc3f9d300
    flags ()
    v_object 0xc402f5a0 ref 0 pages 3406
     lock type devfs: EXCL (count 1) by thread 0xc4e27bd0 (pid 7964)
        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc400e30c: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 134 mountedhere 0xc3f9d300
    flags ()
    v_object 0xc402f5a0 ref 0 pages 530
     lock type devfs: EXCL (count 1) by thread 0xc5d106c0 (pid 44592)
        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc400e30c: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 155 mountedhere 0xc3f9d300
    flags ()
    v_object 0xc402f5a0 ref 0 pages 710
     lock type devfs: EXCL (count 1) by thread 0xc4071360 (pid 47721)
        dev ad0s1e
fsync: giving up on dirty
0xc400e30c: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 155 mountedhere 0xc3f9d300
    flags ()
    v_object 0xc402f5a0 ref 0 pages 710
     lock type devfs: EXCL (count 1) by thread 0xc4071360 (pid 47721)
        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc400e30c: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 50 mountedhere 0xc3f9d300
    flags ()
    v_object 0xc402f5a0 ref 0 pages 730
     lock type devfs: EXCL (count 1) by thread 0xc44666c0 (pid 25905)
        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
fsync: giving up on dirty
0xc400e30c: tag devfs, type VCHR
    usecount 1, writecount 0, refcount 420 mountedhere 0xc3f9d300
    flags ()
    v_object 0xc402f5a0 ref 0 pages 1838
     lock type devfs: EXCL (count 1) by thread 0xc40f51b0 (pid 55079)
        dev ad0s1e
g_vfs_done():md0[READ(offset=65536, length=8192)]error = 5
~KDB: enter: Line break on console
[thread pid 13 tid 100003 ]
Stopped at      kdb_enter+0x2b: nop
db> show allpcpu
Current CPU: 0

cpuid        = 0
curthread    = 0xc3c2ea20: pid 13 "idle: cpu0"
curpcb       = 0xe2643d90
fpcurthread  = none
idlethread   = 0xc3c2ea20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc3c2e870: pid 12 "idle: cpu1"
curpcb       = 0xe2640d90
fpcurthread  = none
idlethread   = 0xc3c2e870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc3c2e6c0: pid 11 "idle: cpu2"
curpcb       = 0xe263dd90
fpcurthread  = none
idlethread   = 0xc3c2e6c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc3c2e510: pid 10 "idle: cpu3"
curpcb       = 0xe263ad90
fpcurthread  = none
idlethread   = 0xc3c2e510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> show alllocks
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
87855 87848 87855  1001  S+      suspfs   0xc40cdd14 ls
87848 87847 87848  1001  Ss+     ttyin    0xc3d44810 bash
87847 87845 87845  1001  S       select   0xc0a57d7c sshd
87845   837 87845     0  Ss      sbwait   0xc5874864 sshd
87741     0     0     0  SL      mdwait   0xc7262000 [md4]
87734     0     0     0  SL      mdwait   0xc7322000 [md3]
87727     0     0     0  SL      mdwait   0xc6bd2000 [md2]
87720     0     0     0  SL      mdwait   0xc7387000 [md1]
85476 85474 85472     0  S       suspfs   0xc40cdd14 mktemp
85474 85472 85472     0  S       piperd   0xc445f18c sh
85472 85470 85472     0  Ss      wait     0xc4d218d0 sh
85470   854   854     0  S       piperd   0xc445f948 cron
85054  1192  1192     0  S+      ufs      0xc45af97c mksnap_ffs
85021     0     0     0  SL      nfsreq   0xc63a5280 [nfsiod 5]
85020     0     0     0  SL      nfsreq   0xc63ce480 [nfsiod 4]
85019     0     0     0  SL      nfsreq   0xc624f780 [nfsiod 3]
85018     0     0     0  SL      nfsreq   0xc6246c00 [nfsiod 2]
85017     0     0     0  SL      nfsreq   0xc63e1b80 [nfsiod 1]
85016 85013 47642  1001  S+      getblk   0xd7e16788 rw
85015 85013 47642  1001  S+      getblk   0xd7e9b2a8 rw
85014 85013 47642  1001  S+      getblk   0xd7e452a8 rw
85013 85006 47642  1001  S+      wait     0xc55bc468 rw
85006 47650 47642  1001  S+      wait     0xc5d0eb04 rw
72009     0     0     0  SL      nfsreq   0xc6376700 [nfsiod 0]
47650 47649 47642  1001  S+      wait     0xc4e61468 run
47649 47648 47642  1001  SW+     wait     0xc542cd38 run
47648 47642 47642  1001  S+      nanslp   0xc0a0b8c4 run
47642  1069 47642  1001  SW+     wait     0xc4d63000 sh
 1192   999  1192     0  S+      wait     0xc4073d38 sh
 1069  1068  1069  1001  SWs+    wait     0xc40f3000 bash
 1068  1066  1066  1001  S       select   0xc0a57d7c sshd
 1066   837  1066     0  SWs     sbwait   0xc4552718 sshd
  999   996   999     0  SW+     wait     0xc3de08d0 bash
  996   995   996     0  SW+     pause    0xc40f3904 csh
  995   989   995  1001  SW+     wait     0xc40f28d0 su
  989   988   989  1001  SWs+    wait     0xc3f6f468 bash
  988   986   986  1001  S       select   0xc0a57d7c sshd
  986   837   986     0  SWs     sbwait   0xc4552334 sshd
  985   979   985  1001  S+      select   0xc0a57d7c top
  979   978   979  1001  SWs+    wait     0xc43fd69c bash
  978   976   976  1001  S       select   0xc0a57d7c sshd
  976   837   976     0  SWs     sbwait   0xc4116718 sshd
  975     1   975     0  SWs+    ttyin    0xc3df3810 getty
  974     1   974     0  SWs+    ttyin    0xc3dfb810 getty
  973     1   973     0  SWs+    ttyin    0xc3dfa810 getty
  972     1   972     0  SWs+    ttyin    0xc3dfa010 getty
  971     1   971     0  SWs+    ttyin    0xc3df5c10 getty
  970     1   970     0  SWs+    ttyin    0xc3dfbc10 getty
  969     1   969     0  SWs+    ttyin    0xc3dfc010 getty
  968     1   968     0  SWs+    ttyin    0xc3dfac10 getty
  951     1   951     0  SWs     select   0xc0a57d7c inetd
  922     1   922     0  SWs     select   0xc0a57d7c moused
  905     1   905     0  Ss      nanslp   0xc0a0b8c4 watchdogd
  854     1   854     0  Ss      nanslp   0xc0a0b8c4 cron
  848     1   848    25  Ss      pause    0xc4073904 sendmail
  844     1   844     0  Ss      select   0xc0a57d7c sendmail
  837     1   837     0  Ss      select   0xc0a57d7c sshd
  817     1   817     0  Ss      select   0xc0a57d7c ntpd
  759   755   755     0  S       suspfs   0xc40cdd14 nfsd
  758   755   755     0  S       ufs      0xc48f4468 nfsd
  757   755   755     0  S       suspfs   0xc40cdd14 nfsd
  756   755   755     0  S       suspfs   0xc40cdd14 nfsd
  755     1   755     0  SWs     select   0xc0a57d7c nfsd
  753     1   753     0  Ss      select   0xc0a57d7c mountd
  694     1   694     0  Ss      select   0xc0a57d7c rpcbind
  671     1   671     0  Ss      select   0xc0a57d7c syslogd
  561     1   561     0  SWs     select   0xc0a57d7c devd
   42     0     0     0  SL      -        0xe4364d04 [schedcpu]
   41     0     0     0  SL      sdflush  0xc0a65160 [softdepflush]
   40     0     0     0  SL      vlruwt   0xc3f6dd38 [vnlru]
   39     0     0     0  SL      syncer   0xc0a0b690 [syncer]
   38     0     0     0  SL      psleep   0xc0a581f4 [bufdaemon]
   37     0     0     0  SL      pgzero   0xc0a6da90 [pagezero]
   36     0     0     0  SL      psleep   0xc0a659c0 [vmdaemon]
   35     0     0     0  SL      psleep   0xc0a65980 [pagedaemon]
   34     0     0     0  WL                          [irq7: ppc0]
   33     0     0     0  SL      -        0xc3d7603c [fdc0]
   32     0     0     0  WL                          [swi0: sio]
   31     0     0     0  WL                          [irq12: psm0]
   30     0     0     0  WL                          [irq1: atkbd0]
   29     0     0     0  WL                          [irq15: ata1]
   28     0     0     0  WL                          [irq14: ata0]
   27     0     0     0  WL                          [irq17: fxp0]
   26     0     0     0  SL      usbtsk   0xc0a08e44 [usbtask]
   25     0     0     0  SL      usbevt   0xc3d67210 [usb0]
   24     0     0     0  WL                          [irq16: uhci0]
   23     0     0     0  SL      -        0xc3d51480 [em0 taskq]
   22     0     0     0  WL                          [irq9: acpi0]
    9     0     0     0  SL      -        0xc3d40980 [thread taskq]
   21     0     0     0  WL                          [swi6: Giant taskq]
   20     0     0     0  WL                          [swi6: task queue]
    8     0     0     0  SL      -        0xc3c7b280 [acpi_task_2]
    7     0     0     0  SL      -        0xc3c7b280 [acpi_task_1]
    6     0     0     0  SL      -        0xc3c7b280 [acpi_task_0]
    5     0     0     0  SL      -        0xc3c7b300 [kqueue taskq]
   19     0     0     0  WL                          [swi2: cambio]
   18     0     0     0  WL                          [swi5: +]
   17     0     0     0  SL      -        0xc0a06a80 [yarrow]
    4     0     0     0  SL      -        0xc0a095fc [g_down]
    3     0     0     0  SL      -        0xc0a095f8 [g_up]
    2     0     0     0  SL      -        0xc0a095f0 [g_event]
   16     0     0     0  WL                          [swi3: vm]
   15     0     0     0  WL                          [swi4: clock sio]
   14     0     0     0  WL                          [swi1: net]
   13     0     0     0  RL      CPU 0               [idle: cpu0]
   12     0     0     0  RL      CPU 1               [idle: cpu1]
   11     0     0     0  RL      CPU 2               [idle: cpu2]
   10     0     0     0  RL      CPU 3               [idle: cpu3]
    1     0     1     0  SLs     wait     0xc3c32000 [init]
    0     0     0     0  WLs                         [swapper]
85010 47650 47642  1001  Z+                          thr1
85009 47650 47642  1001  Z+                          mkdir
85011 47650 47642  1001  Z+                          udp
85012 47650 47642  1001  Z+                          tcp
85008 47650 47642  1001  Z+                          creat
85007 47650 47642  1001  Z+                          swap
db> where 756
Tracing pid 756 tid 100038 td 0xc3ddd6c0
sched_switch(c3ddd6c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c40cdd14,c0a0b0b0,0,c092000a,211,...) at sleepq_switch+0xcd
sleepq_wait(c40cdd14,0,c0927acf,3f3,c093229c,...) at sleepq_wait+0x46
msleep(c40cdd14,c40cdccc,29f,c0927b18,0,...) at msleep+0x27d
vn_start_secondary_write(c48f4410,e430e988,1) at vn_start_secondary_write+0x122
ufs_inactive(e430e9b8) at ufs_inactive+0x257
VOP_INACTIVE_APV(c09d9a00,e430e9b8) at VOP_INACTIVE_APV+0x7e
vinactive(c48f4410,c3ddd6c0) at vinactive+0x72
vput(c48f4410,c0a0b0c8,1,c0932293,407,...) at vput+0x1b3
nfsrv_read(c6dcb100,c3f95500,c3ddd6c0,e430ec40) at nfsrv_read+0xc21
nfssvc_nfsd(c3ddd6c0) at nfssvc_nfsd+0x409
nfssvc(c3ddd6c0,e430ed04) at nfssvc+0x18c
syscall(3b,3b,3b,1,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280cb193, esp = 0xbfbfeb2c, ebp = 0xbfbfeb48 ---
db> where 757
Tracing pid 757 tid 100072 td 0xc4014000
sched_switch(c4014000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c40cdd14,c0a0b0b0,0,c092000a,211,...) at sleepq_switch+0xcd
sleepq_wait(c40cdd14,0,c0927acf,3f3,c093229c,...) at sleepq_wait+0x46
msleep(c40cdd14,c40cdccc,29f,c0927b18,0,...) at msleep+0x27d
vn_start_secondary_write(c4acad34,e65a1988,1) at vn_start_secondary_write+0x122
ufs_inactive(e65a19b8) at ufs_inactive+0x257
VOP_INACTIVE_APV(c09d9a00,e65a19b8) at VOP_INACTIVE_APV+0x7e
vinactive(c4acad34,c4014000) at vinactive+0x72
vput(c4acad34,c0a0b0c8,1,c0932293,407,...) at vput+0x1b3
nfsrv_read(c52d2400,c3f95500,c4014000,e65a1c40) at nfsrv_read+0xc21
nfssvc_nfsd(c4014000) at nfssvc_nfsd+0x409
nfssvc(c4014000,e65a1d04) at nfssvc+0x18c
syscall(3b,3b,3b,1,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280cb193, esp = 0xbfbfeb2c, ebp = 0xbfbfeb48 ---
db> where 758
Tracing pid 758 tid 100056 td 0xc3fd6360
sched_switch(c3fd6360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c48f4468,c0a0b0b0,0,c092000a,211,...) at sleepq_switch+0xcd
sleepq_wait(c48f4468,0,c48f448c,b1,c091ab43,...) at sleepq_wait+0x46
msleep(c48f4468,c0a0a8e8,50,c0924f24,0,...) at msleep+0x27d
acquire(e657d828,40,60000,c3fd6360,0,...) at acquire+0x76
lockmgr(c48f4468,2002,c48f448c,c3fd6360) at lockmgr+0x44a
ffs_lock(e657d880) at ffs_lock+0x6e
VOP_LOCK_APV(c09d9a00,e657d880) at VOP_LOCK_APV+0x87
vn_lock(c48f4410,2002,c3fd6360,c48f4410) at vn_lock+0xa8
vget(c48f4410,2002,c3fd6360) at vget+0xba
vfs_hash_get(c40cdca8,22808,2,c3fd6360,e657d998,0,0) at vfs_hash_get+0x8a
ffs_vget(c40cdca8,22808,2,e657d998) at ffs_vget+0x27
ufs_fhtovp(c40cdca8,e657db8c,e657da78,c40cdca8,e657da00,...) at ufs_fhtovp+0x22
ffs_fhtovp(c40cdca8,e657db8c,e657da78) at ffs_fhtovp+0x3a
nfsrv_fhtovp(e657db84,1,e657da78,c6376280,c3f95500,c4063a70,e657da7c,1) at nfsrv_fhtovp+0xf9
nfsrv_read(c3fda500,c3f95500,c3fd6360,e657dc40) at nfsrv_read+0x275
nfssvc_nfsd(c3fd6360) at nfssvc_nfsd+0x409
nfssvc(c3fd6360,e657dd04) at nfssvc+0x18c
syscall(3b,3b,3b,1,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280cb193, esp = 0xbfbfeb2c, ebp = 0xbfbfeb48 ---
db> where 85476
Tracing pid 85476 tid 100202 td 0xc4400870
sched_switch(c4400870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c40cdd14) at sleepq_switch+0xcd
sleepq_wait_sig(c40cdd14) at sleepq_wait_sig+0x1d
msleep(c40cdd14,c40cdccc,19f,c0927b18,0) at msleep+0x26e
vn_start_write(0,e670fa10,104) at vn_start_write+0xbf
vn_open_cred(e670fb7c,e670fc7c,180,c6366a80,3,...) at vn_open_cred+0x12f
vn_open(e670fb7c,e670fc7c,180,3) at vn_open+0x1e
kern_open(c4400870,8208140,0,a03,180,...) at kern_open+0xb9
open(c4400870,e670fd04) at open+0x18
syscall(3b,3b,3b,8208140,8208144,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (5, FreeBSD ELF32, open), eip = 0x28142c5f, esp = 0xbfbfed0c, ebp = 0xbfbfedb8 ---
db> where 759
Tracing pid 759 tid 100104 td 0xc3f6ebd0
sched_switch(c3f6ebd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c40cdd14,c0a0b0b0,0,c092000a,211,...) at sleepq_switch+0xcd
sleepq_wait(c40cdd14,0,c0927acf,3f3,c093229c,...) at sleepq_wait+0x46
msleep(c40cdd14,c40cdccc,29f,c0927b18,0,...) at msleep+0x27d
vn_start_secondary_write(c45af924,e436a988,1) at vn_start_secondary_write+0x122
ufs_inactive(e436a9b8) at ufs_inactive+0x257
VOP_INACTIVE_APV(c09d9a00,e436a9b8) at VOP_INACTIVE_APV+0x7e
vinactive(c45af924,c3f6ebd0) at vinactive+0x72
vput(c45af924,c0a0b0c8,1,c0932293,407,...) at vput+0x1b3
nfsrv_read(c55b7900,c3f95500,c3f6ebd0,e436ac40) at nfsrv_read+0xc21
nfssvc_nfsd(c3f6ebd0) at nfssvc_nfsd+0x409
nfssvc(c3f6ebd0,e436ad04) at nfssvc+0x18c
syscall(3b,3b,3b,1,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280cb193, esp = 0xbfbfeb2c, ebp = 0xbfbfeb48 ---
db> call doadump
Physical memory: 1007 MB
Dumping 158 MB: 143 127 111 95 79 63 47 31 15
Dump complete
= 0xf
db> reset
GDB: no debug ports present

(kgdb) p **(struct mount **)0xe670fa10
$2 = {mnt_lock = {lk_interlock = 0xc0a0a9f0, lk_flags = 0x1000000, lk_sharecount = 0x1, lk_waitcount = 0x0, lk_exclusivecount = 0x0,
    lk_prio = 0x50, lk_wmesg = 0xc0926925 "vfslock", lk_timo = 0x0, lk_lockholder = 0xffffffff, lk_newlock = 0x0}, mnt_mtx = {
    mtx_object = {lo_name = 0xc0926914 "struct mount mtx", lo_type = 0xc0926914 "struct mount mtx", lo_flags = 0x1030000,
      lo_witness_data = {lod_list = {stqe_next = 0xc0a197e8}, lod_witness = 0xc0a197e8}}, mtx_lock = 0x4, mtx_recurse = 0x0},
  mnt_list = {tqe_next = 0xc40cda20, tqe_prev = 0xc40ca54c}, mnt_op = 0xc09d96a0, mnt_vfc = 0xc09d96e0,
  mnt_vnodecovered = 0xc3ffa71c, mnt_syncer = 0xc40cbb2c, mnt_ref = 0x9f, mnt_nvnodelist = {tqh_first = 0xc40c5104,
    tqh_last = 0xc4d44d48}, mnt_nvnodelistsize = 0x98, mnt_writeopcount = 0x0, mnt_kern_flag = 0x2c000000, mnt_flag = 0x1211300,
  mnt_opt = 0xc3dffa30, mnt_optnew = 0xc4063bf0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x6,
    f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0xf20a7, f_bfree = 0x59933, f_bavail = 0x46364,
    f_files = 0x3f3fe, f_ffree = 0x3f219, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {
      0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x434f6181, 0xa1e353a3}},
    f_charspare = '\0' <repeats 79 times>, f_fstypename = "ufs", '\0' <repeats 12 times>,
    f_mntfromname = "/dev/ad0s1e", '\0' <repeats 76 times>, f_mntonname = "/tmp", '\0' <repeats 83 times>}, mnt_cred = 0xc3f95300,
  mnt_data = 0xc3f9c800, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc453dc00, mnt_mntlabel = 0x0, mnt_fslabel = 0x0,
  mnt_hashseed = 0xea73cfb7, mnt_markercnt = 0x1, mnt_holdcnt = 0x1, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0,
  mnt_secondary_accwrites = 0x529622}
(kgdb) p **(struct mount **)0xe430e988
$5 = {mnt_lock = {lk_interlock = 0xc0a0a9f0, lk_flags = 0x1000000, lk_sharecount = 0x1, lk_waitcount = 0x0, lk_exclusivecount = 0x0,
    lk_prio = 0x50, lk_wmesg = 0xc0926925 "vfslock", lk_timo = 0x0, lk_lockholder = 0xffffffff, lk_newlock = 0x0}, mnt_mtx = {
    mtx_object = {lo_name = 0xc0926914 "struct mount mtx", lo_type = 0xc0926914 "struct mount mtx", lo_flags = 0x1030000,
      lo_witness_data = {lod_list = {stqe_next = 0xc0a197e8}, lod_witness = 0xc0a197e8}}, mtx_lock = 0x4, mtx_recurse = 0x0},
  mnt_list = {tqe_next = 0xc40cda20, tqe_prev = 0xc40ca54c}, mnt_op = 0xc09d96a0, mnt_vfc = 0xc09d96e0,
  mnt_vnodecovered = 0xc3ffa71c, mnt_syncer = 0xc40cbb2c, mnt_ref = 0x9f, mnt_nvnodelist = {tqh_first = 0xc40c5104,
    tqh_last = 0xc4d44d48}, mnt_nvnodelistsize = 0x98, mnt_writeopcount = 0x0, mnt_kern_flag = 0x2c000000, mnt_flag = 0x1211300,
  mnt_opt = 0xc3dffa30, mnt_optnew = 0xc4063bf0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x6,
    f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0xf20a7, f_bfree = 0x59933, f_bavail = 0x46364,
    f_files = 0x3f3fe, f_ffree = 0x3f219, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {
      0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x434f6181, 0xa1e353a3}},
    f_charspare = '\0' <repeats 79 times>, f_fstypename = "ufs", '\0' <repeats 12 times>,
    f_mntfromname = "/dev/ad0s1e", '\0' <repeats 76 times>, f_mntonname = "/tmp", '\0' <repeats 83 times>}, mnt_cred = 0xc3f95300,
  mnt_data = 0xc3f9c800, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc453dc00, mnt_mntlabel = 0x0, mnt_fslabel = 0x0,
  mnt_hashseed = 0xea73cfb7, mnt_markercnt = 0x1, mnt_holdcnt = 0x1, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0,
  mnt_secondary_accwrites = 0x529622}
(kgdb) p **(struct mount **)0xe65a1988
$6 = {mnt_lock = {lk_interlock = 0xc0a0a9f0, lk_flags = 0x1000000, lk_sharecount = 0x1, lk_waitcount = 0x0, lk_exclusivecount = 0x0,
    lk_prio = 0x50, lk_wmesg = 0xc0926925 "vfslock", lk_timo = 0x0, lk_lockholder = 0xffffffff, lk_newlock = 0x0}, mnt_mtx = {
    mtx_object = {lo_name = 0xc0926914 "struct mount mtx", lo_type = 0xc0926914 "struct mount mtx", lo_flags = 0x1030000,
      lo_witness_data = {lod_list = {stqe_next = 0xc0a197e8}, lod_witness = 0xc0a197e8}}, mtx_lock = 0x4, mtx_recurse = 0x0},
  mnt_list = {tqe_next = 0xc40cda20, tqe_prev = 0xc40ca54c}, mnt_op = 0xc09d96a0, mnt_vfc = 0xc09d96e0,
  mnt_vnodecovered = 0xc3ffa71c, mnt_syncer = 0xc40cbb2c, mnt_ref = 0x9f, mnt_nvnodelist = {tqh_first = 0xc40c5104,
    tqh_last = 0xc4d44d48}, mnt_nvnodelistsize = 0x98, mnt_writeopcount = 0x0, mnt_kern_flag = 0x2c000000, mnt_flag = 0x1211300,
  mnt_opt = 0xc3dffa30, mnt_optnew = 0xc4063bf0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x6,
    f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0xf20a7, f_bfree = 0x59933, f_bavail = 0x46364,
    f_files = 0x3f3fe, f_ffree = 0x3f219, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {
      0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x434f6181, 0xa1e353a3}},
    f_charspare = '\0' <repeats 79 times>, f_fstypename = "ufs", '\0' <repeats 12 times>,
    f_mntfromname = "/dev/ad0s1e", '\0' <repeats 76 times>, f_mntonname = "/tmp", '\0' <repeats 83 times>}, mnt_cred = 0xc3f95300,
  mnt_data = 0xc3f9c800, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc453dc00, mnt_mntlabel = 0x0, mnt_fslabel = 0x0,
  mnt_hashseed = 0xea73cfb7, mnt_markercnt = 0x1, mnt_holdcnt = 0x1, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0,
  mnt_secondary_accwrites = 0x529622}
(kgdb) p **(struct mount **)0xe436a988
$7 = {mnt_lock = {lk_interlock = 0xc0a0a9f0, lk_flags = 0x1000000, lk_sharecount = 0x1, lk_waitcount = 0x0, lk_exclusivecount = 0x0,
    lk_prio = 0x50, lk_wmesg = 0xc0926925 "vfslock", lk_timo = 0x0, lk_lockholder = 0xffffffff, lk_newlock = 0x0}, mnt_mtx = {
    mtx_object = {lo_name = 0xc0926914 "struct mount mtx", lo_type = 0xc0926914 "struct mount mtx", lo_flags = 0x1030000,
      lo_witness_data = {lod_list = {stqe_next = 0xc0a197e8}, lod_witness = 0xc0a197e8}}, mtx_lock = 0x4, mtx_recurse = 0x0},
  mnt_list = {tqe_next = 0xc40cda20, tqe_prev = 0xc40ca54c}, mnt_op = 0xc09d96a0, mnt_vfc = 0xc09d96e0,
  mnt_vnodecovered = 0xc3ffa71c, mnt_syncer = 0xc40cbb2c, mnt_ref = 0x9f, mnt_nvnodelist = {tqh_first = 0xc40c5104,
    tqh_last = 0xc4d44d48}, mnt_nvnodelistsize = 0x98, mnt_writeopcount = 0x0, mnt_kern_flag = 0x2c000000, mnt_flag = 0x1211300,
  mnt_opt = 0xc3dffa30, mnt_optnew = 0xc4063bf0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x6,
    f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0xf20a7, f_bfree = 0x59933, f_bavail = 0x46364,
    f_files = 0x3f3fe, f_ffree = 0x3f219, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {
      0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x434f6181, 0xa1e353a3}},
    f_charspare = '\0' <repeats 79 times>, f_fstypename = "ufs", '\0' <repeats 12 times>,
    f_mntfromname = "/dev/ad0s1e", '\0' <repeats 76 times>, f_mntonname = "/tmp", '\0' <repeats 83 times>}, mnt_cred = 0xc3f95300,
  mnt_data = 0xc3f9c800, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc453dc00, mnt_mntlabel = 0x0, mnt_fslabel = 0x0,
  mnt_hashseed = 0xea73cfb7, mnt_markercnt = 0x1, mnt_holdcnt = 0x1, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0,
  mnt_secondary_accwrites = 0x529622}
(kgdb) p *(struct vnode *)0xc48f4410
$8 = {v_type = VREG, v_tag = 0xc0924f24 "ufs", v_op = 0xc09d9a00, v_data = 0xc5bb2210, v_mount = 0xc40cdca8, v_nmntvnodes = {
    tqe_next = 0xc4a3cc30, tqe_prev = 0xc4acad48}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0},
  v_hashlist = {le_next = 0x0, le_prev = 0xc641d840}, v_hash = 0x22808, v_cache_src = {lh_first = 0x0}, v_cache_dst = {
    tqh_first = 0xc4d36198, tqh_last = 0xc4d361a8}, v_dd = 0x0, v_cstart = 0xfa0, v_lasta = 0x163a0, v_lastw = 0xfa2, v_clen = 0x7,
  v_lock = {lk_interlock = 0xc0a0a8e8, lk_flags = 0x2040040, lk_sharecount = 0x0, lk_waitcount = 0x1, lk_exclusivecount = 0x1,
    lk_prio = 0x50, lk_wmesg = 0xc0924f24 "ufs", lk_timo = 0x33, lk_lockholder = 0xc3ddd6c0, lk_newlock = 0x0}, v_interlock = {
    mtx_object = {lo_name = 0xc09210e6 "vnode interlock", lo_type = 0xc09210e6 "vnode interlock", lo_flags = 0x1030000,
      lo_witness_data = {lod_list = {stqe_next = 0xc0a1b070}, lod_witness = 0xc0a1b070}}, mtx_lock = 0x4, mtx_recurse = 0x0},
  v_vnlock = 0xc48f4468, v_holdcnt = 0x3, v_usecount = 0x0, v_iflag = 0x800, v_vflag = 0x0, v_writecount = 0x0, v_freelist = {
    tqe_next = 0x0, tqe_prev = 0xc50eb0ac}, v_bufobj = {bo_mtx = 0xc48f448c, bo_clean = {bv_hd = {tqh_first = 0x0,
        tqh_last = 0xc48f44c8}, bv_root = 0x0, bv_cnt = 0x0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc48f44d8},
      bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x0, bo_ops = 0xc09cd0a4, bo_bsize = 0x4000,
    bo_object = 0xc4a7c258, bo_synclist = {le_next = 0x0, le_prev = 0xc400e2f4}, bo_private = 0xc48f4410, __bo_vnode = 0xc48f4410},
  v_pollinfo = 0x0, v_label = 0x0}
(kgdb) p *(struct vnode *)0xc48f4410
$8 = {v_type = VREG, v_tag = 0xc0924f24 "ufs", v_op = 0xc09d9a00, v_data = 0xc5bb2210, v_mount = 0xc40cdca8, v_nmntvnodes = {
    tqe_next = 0xc4a3cc30, tqe_prev = 0xc4acad48}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0},
  v_hashlist = {le_next = 0x0, le_prev = 0xc641d840}, v_hash = 0x22808, v_cache_src = {lh_first = 0x0}, v_cache_dst = {
    tqh_first = 0xc4d36198, tqh_last = 0xc4d361a8}, v_dd = 0x0, v_cstart = 0xfa0, v_lasta = 0x163a0, v_lastw = 0xfa2, v_clen = 0x7,
  v_lock = {lk_interlock = 0xc0a0a8e8, lk_flags = 0x2040040, lk_sharecount = 0x0, lk_waitcount = 0x1, lk_exclusivecount = 0x1,
    lk_prio = 0x50, lk_wmesg = 0xc0924f24 "ufs", lk_timo = 0x33, lk_lockholder = 0xc3ddd6c0, lk_newlock = 0x0}, v_interlock = {
    mtx_object = {lo_name = 0xc09210e6 "vnode interlock", lo_type = 0xc09210e6 "vnode interlock", lo_flags = 0x1030000,
      lo_witness_data = {lod_list = {stqe_next = 0xc0a1b070}, lod_witness = 0xc0a1b070}}, mtx_lock = 0x4, mtx_recurse = 0x0},
  v_vnlock = 0xc48f4468, v_holdcnt = 0x3, v_usecount = 0x0, v_iflag = 0x800, v_vflag = 0x0, v_writecount = 0x0, v_freelist = {
    tqe_next = 0x0, tqe_prev = 0xc50eb0ac}, v_bufobj = {bo_mtx = 0xc48f448c, bo_clean = {bv_hd = {tqh_first = 0x0,
        tqh_last = 0xc48f44c8}, bv_root = 0x0, bv_cnt = 0x0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc48f44d8},
      bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x0, bo_ops = 0xc09cd0a4, bo_bsize = 0x4000,
    bo_object = 0xc4a7c258, bo_synclist = {le_next = 0x0, le_prev = 0xc400e2f4}, bo_private = 0xc48f4410, __bo_vnode = 0xc48f4410},
  v_pollinfo = 0x0, v_label = 0x0}
(kgdb) p *(struct thread *)0xc3ddd6c0
$9 = {td_proc = 0xc3de069c, td_ksegrp = 0xc3c300c0, td_plist = {tqe_next = 0x0, tqe_prev = 0xc3de06ac}, td_kglist = {tqe_next = 0x0,
    tqe_prev = 0xc3c300cc}, td_slpq = {tqe_next = 0xc4014000, tqe_prev = 0xc6a34c20}, td_lockq = {tqe_next = 0x0,
    tqe_prev = 0xe657daa4}, td_runq = {tqe_next = 0x0, tqe_prev = 0x0}, td_selq = {tqh_first = 0x0, tqh_last = 0x0},
  td_sleepqueue = 0x0, td_turnstile = 0xc66bf940, td_umtxq = 0xc3d7c0c0, td_tid = 0x186c6, td_sigqueue = {sq_signals = {__bits = {
        0x0, 0x0, 0x0, 0x0}}, sq_kill = {__bits = {0x0, 0x0, 0x0, 0x0}}, sq_list = {tqh_first = 0x0, tqh_last = 0xc3ddd728},
    sq_proc = 0xc3de069c, sq_flags = 0x1}, td_flags = 0x0, td_inhibitors = 0x2, td_pflags = 0x0, td_dupfd = 0x0,
  td_wchan = 0xc40cdd14, td_wmesg = 0xc0927b18 "suspfs", td_lastcpu = 0x0, td_oncpu = 0xff, td_owepreempt = 0x0, td_locks = 0x1,
  td_tsqueue = 0xff, td_blocked = 0x0, td_lockname = 0x0, td_contested = {lh_first = 0x0}, td_sleeplocks = 0x0,
  td_intr_nesting_level = 0x0, td_pinned = 0x0, td_mailbox = 0x0, td_ucred = 0xc3c2b780, td_standin = 0x0, td_upcall = 0x0,
  td_pticks = 0x332b7, td_sticks = 0x0, td_iticks = 0x0, td_uticks = 0x0, td_uuticks = 0x0, td_usticks = 0x0, td_intrval = 0x0,
  td_oldsigmask = {__bits = {0x0, 0x0, 0x0, 0x0}}, td_sigmask = {__bits = {0x0, 0x0, 0x0, 0x0}}, td_generation = 0x53209f,
  td_sigstk = {ss_sp = 0x0, ss_size = 0x0, ss_flags = 0x4}, td_kflags = 0x0, td_xsig = 0x0, td_profil_addr = 0x0,
  td_profil_ticks = 0x0, td_name = '\0' <repeats 19 times>, td_base_pri = 0x9f, td_priority = 0x9f, td_pcb = 0xe430ed90,
  td_state = TDS_INHIBITED, td_retval = {0x0, 0x281661b8}, td_slpcallout = {c_links = {sle = {sle_next = 0xc3c2fd14}, tqe = {
        tqe_next = 0xc3c2fd14, tqe_prev = 0xd7c565e8}}, c_time = 0x174b3b3, c_arg = 0xc3ddd6c0, c_func = 0xc06c0384 <sleepq_timeout>,
    c_mtx = 0x0, c_flags = 0x10}, td_frame = 0xe430ed38, td_kstack_obj = 0xc185aca8, td_kstack = 0xe430d000, td_kstack_pages = 0x2,
  td_altkstack_obj = 0x0, td_altkstack = 0x0, td_altkstack_pages = 0x0, td_critnest = 0x1, td_md = {md_spinlock_count = 0x1,
    md_saved_flags = 0x246}, td_sched = 0xc3ddd850, td_ar = 0x0}
(kgdb) p/d (*(struct thread *)0xc3ddd6c0)->td_proc.p_pid
$10 = 756
(kgdb) p *(*(struct vnode *)0xc48f4410)->v_mount
$14 = {mnt_lock = {lk_interlock = 0xc0a0a9f0, lk_flags = 0x1000000, lk_sharecount = 0x1, lk_waitcount = 0x0, lk_exclusivecount = 0x0,
    lk_prio = 0x50, lk_wmesg = 0xc0926925 "vfslock", lk_timo = 0x0, lk_lockholder = 0xffffffff, lk_newlock = 0x0}, mnt_mtx = {
    mtx_object = {lo_name = 0xc0926914 "struct mount mtx", lo_type = 0xc0926914 "struct mount mtx", lo_flags = 0x1030000,
      lo_witness_data = {lod_list = {stqe_next = 0xc0a197e8}, lod_witness = 0xc0a197e8}}, mtx_lock = 0x4, mtx_recurse = 0x0},
  mnt_list = {tqe_next = 0xc40cda20, tqe_prev = 0xc40ca54c}, mnt_op = 0xc09d96a0, mnt_vfc = 0xc09d96e0,
  mnt_vnodecovered = 0xc3ffa71c, mnt_syncer = 0xc40cbb2c, mnt_ref = 0x9f, mnt_nvnodelist = {tqh_first = 0xc40c5104,
    tqh_last = 0xc4d44d48}, mnt_nvnodelistsize = 0x98, mnt_writeopcount = 0x0, mnt_kern_flag = 0x2c000000, mnt_flag = 0x1211300,
  mnt_opt = 0xc3dffa30, mnt_optnew = 0xc4063bf0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x6,
    f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0xf20a7, f_bfree = 0x59933, f_bavail = 0x46364,
    f_files = 0x3f3fe, f_ffree = 0x3f219, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {
      0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x434f6181, 0xa1e353a3}},
    f_charspare = '\0' <repeats 79 times>, f_fstypename = "ufs", '\0' <repeats 12 times>,
    f_mntfromname = "/dev/ad0s1e", '\0' <repeats 76 times>, f_mntonname = "/tmp", '\0' <repeats 83 times>}, mnt_cred = 0xc3f95300,
  mnt_data = 0xc3f9c800, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc453dc00, mnt_mntlabel = 0x0, mnt_fslabel = 0x0,
  mnt_hashseed = 0xea73cfb7, mnt_markercnt = 0x1, mnt_holdcnt = 0x1, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0,
  mnt_secondary_accwrites = 0x529622}