GENERIC HEAD from Jun 19 20:17 UTC, vmcore.240

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2008 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 8.0-CURRENT #1: Fri Jun 20 16:11:31 CEST 2008
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1037516800 (989 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
lock order reversal: (sleepable after non-sleepable)
 1st 0xc4346e64 bufobj interlock (bufobj interlock) @ kern/vfs_bio.c:2442
 2nd 0xd80ea918 bufwait (bufwait) @ kern/vfs_bio.c:2456
KDB: stack backtrace:
db_trace_self_wrapper(c0b38a32,c3da4784,c07c5b5e,c0b3b26f,d80ea918,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0b3b26f,d80ea918,c0b3fe81,c0b3fe81,c0b3f437,...) at kdb_backtrace+0x29
witness_checkorder(d80ea918,9,c0b3f42e,998,c0c84a34,...) at witness_checkorder+0x6de
__lockmgr_args(d80ea918,81900,c4346e64,c0b3fdb6,50,...) at __lockmgr_args+0x777
getblk(c4346d70,0,0,800,0,...) at getblk+0x163
breadn(c4346d70,0,0,800,0,...) at breadn+0x44
bread(c4346d70,0,0,800,0,...) at bread+0x4c
ffs_blkatoff(c4346d70,0,0,0,c3da49a4,...) at ffs_blkatoff+0xd1
ufs_lookup(c3da49ec,c0b6beb4,c4346d70,c4346d70,c3da4b50,...) at ufs_lookup+0x2e6
VOP_CACHEDLOOKUP_APV(c0c2c860,c3da49ec,c3da4b50,c3da4b3c,c40a9800,...) at VOP_CACHEDLOOKUP_APV+0xc5
vfs_cache_lookup(c3da4a6c,c0b421e3,c0c44b60,80000,c4346d70,...) at vfs_cache_lookup+0xd3
VOP_LOOKUP_APV(c0c2c860,c3da4a6c,c0b40c5b,1b0,c3da4b3c,...) at VOP_LOOKUP_APV+0xe5
lookup(c3da4b24,c0b40c5b,d8,c0,c407322c,...) at lookup+0x58e
namei(c3da4b24,c3da4b28,c07c533c,c0b40fa3,c0ccc0e8,...) at namei+0x45b
kern_unlinkat(c40a4d20,ffffff9c,c0b412ea,1,c3da4c5c,...) at kern_unlinkat+0x46
kern_unlink(c40a4d20,c0b412ea,1,62c,0,...) at kern_unlink+0x27
vfs_mountroot_try(c0b414a4,c0b2f2c6,c0b27df3,1,c0802d20,...) at vfs_mountroot_try+0x49f
vfs_mountroot(c0c77bb0,4,c0b305e3,264,0,...) at vfs_mountroot+0x420
start_init(0,c3da4d38,c0b31f64,324,c40a2d0c,...) at start_init+0x65
fork_exit(c074f860,0,c3da4d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc3da4d70, ebp = 0 ---
fxp0: link state changed to UP
softdep_waitidle: Failed to flush worklist for 0xc4572338
softdep_waitidle: Failed to flush worklist for 0xc4572338
softdep_waitidle: Failed to flush worklist for 0xc4572338
~KDB: enter: Line break on console
[thread pid 11 tid 100006 ]
Stopped at      kdb_enter+0x3a: movl    $0,kdb_why
db> b ffs_unmount
db> c
[thread pid 964 tid 100114 ]
Breakpoint at   ffs_unmount:    pushl   %ebp
db> bt
Tracing pid 964 tid 100114 td 0xc46fc460
ffs_unmount(c4572338,8080000,c46fc460,484,63902ae,...) at ffs_unmount
unmount(c46fc460,e690dcfc,8,c46fc460,c0c09150,...) at unmount+0x2e9
syscall(e690dd38) at syscall+0x2a3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (22, FreeBSD ELF32, unmount), eip = 0x280cf7ff, esp = 0xbfbfe3cc, ebp = 0xbfbfe488 ---
db> show mount 0xc4572338
0xc4572338 /dev/md5c on /mnt (ufs)
    mnt_flag = SOFTDEP, EXPORTED, DEFEXPORTED, LOCAL
    mnt_kern_flag = UNMOUNTF, SOFTDEP, NOINSMNTQ, UNMOUNT, MPSAFE
    mnt_stat = { version=537068824 type=5 flags=0x0000000000201300 bsize=2048 iosize=16384 blocks=9783 bfree=9780 bavail=8998 files=2814 ffree=2811 syncwrites=0 asyncwrites=0 syncreads=0 asyncreads=0 namemax=255 owner=0 fsid=[1213972983, 104399534] }
    mnt_cred = { uid=0 ruid=0 }
    mnt_ref = 0
    mnt_gen = 1
    mnt_nvnodelistsize = 0
    mnt_writeopcount = 1
    mnt_noasync = 1
    mnt_maxsymlinklen = 120
    mnt_iosize_max = 131072
    mnt_hashseed = 2741906930
    mnt_markercnt = 0
    mnt_holdcnt = 0
    mnt_holdcntwaiters = 0
    mnt_secondary_writes = 0
    mnt_secondary_accwrites = 6033
    mnt_gjprovider = NULL

db> set $lines 0
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
  964   931   964     0  R+      CPU 0               umount
  949     0     0     0  SL      mdwait   0xc4310000 [md5]
  931   927   931     0  S+      wait     0xc49e9000 bash
  927   926   927     0  S+      pause    0xc4ba4ac8 csh
  926   922   926  1001  S+      wait     0xc45f0000 su
  922   920   922  1001  Ss+     wait     0xc4ba4d0c bash
  920   918   918  1001  S       select   0xc44f1124 sshd
  918   794   918     0  Ss      sbwait   0xc4ba2548 sshd
  917     1   917     0  Ss+     ttyin    0xc4338c10 getty
  916     1   916     0  Ss+     ttyin    0xc4338810 getty
  915     1   915     0  Ss+     ttyin    0xc4339810 getty
  914     1   914     0  Ss+     ttyin    0xc4339410 getty
  913     1   913     0  Ss+     ttyin    0xc4332810 getty
  912     1   912     0  Ss+     ttyin    0xc4332c10 getty
  911     1   911     0  Ss+     ttyin    0xc4338010 getty
  910     1   910     0  Ss+     ttyin    0xc4338410 getty
  893     1   893     0  Ss      select   0xc4528d64 inetd
  866     1   866     0  Ss      select   0xc43353e4 moused
  852     1   852     0  Ss      nanslp   0xc0c78324 watchdogd
  810     1   810     0  Ss      nanslp   0xc0c78324 cron
  803     1   803    25  Ss      pause    0xc48fd058 sendmail
  799     1   799     0  Ss      select   0xc4575364 sendmail
  794     1   794     0  Ss      select   0xc4575524 sshd
  775     1   775     0  Ss      select   0xc44f1824 ntpd
  715     1   714     0  S       nanslp   0xc0c78324 smartd
  691   675   675     0  S       -        0xc45a4400 nfsd
  690   675   675     0  S       -        0xc44eae00 nfsd
  689   675   675     0  S       -        0xc45a4600 nfsd
  688   675   675     0  S       -        0xc4531000 nfsd
  687   675   675     0  S       -        0xc45a4800 nfsd
  686   675   675     0  S       -        0xc4531200 nfsd
  685   675   675     0  S       -        0xc45a5400 nfsd
  684   675   675     0  S       -        0xc4531400 nfsd
  683   675   675     0  S       -        0xc45a5200 nfsd
  682   675   675     0  S       -        0xc4531600 nfsd
  680   675   675     0  S       -        0xc45a5000 nfsd
  679   675   675     0  S       -        0xc46eb600 nfsd
  678   675   675     0  S       -        0xc45a4e00 nfsd
  677   675   675     0  S       -        0xc4532600 nfsd
  676   675   675     0  S       -        0xc45a4c00 nfsd
  675     1   675     0  Ss      select   0xc432cde4 nfsd
  673     1   673     0  Ss      select   0xc45283e4 mountd
  620     1   620     0  Ss      select   0xc44f1524 rpcbind
  603     1   603     0  Ss      select   0xc44f1424 syslogd
  546     1   546     0  Ss      select   0xc44f21e4 devd
   21     0     0     0  SL      sdflush  0xc0cd8080 [softdepflush]
   20     0     0     0  SL      vlruwt   0xc450329c [vnlru]
   19     0     0     0  SL      syncer   0xc0c78140 [syncer]
   18     0     0     0  SL      psleep   0xc0ccc624 [bufdaemon]
   17     0     0     0  SL      pgzero   0xc0cd8c74 [pagezero]
    9     0     0     0  SL      psleep   0xc0cd889c [vmdaemon]
    8     0     0     0  SL      psleep   0xc0cd8864 [pagedaemon]
    7     0     0     0  SL      waiting_ 0xc0cce2e0 [sctp_iterator]
    6     0     0     0  SL      -        0xc42ae23c [fdc0]
   16     0     0     0  SL      usbtsk   0xc0c75814 [usbtask-dr]
   15     0     0     0  SL      usbtsk   0xc0c75800 [usbtask-hc]
   14     0     0     0  SL      usbevt   0xc429f210 [usb0]
    5     0     0     0  SL      ccb_scan 0xc0c45654 [xpt_thrd]
   13     0     0     0  SL      -        0xc0c78148 [yarrow]
    4     0     0     0  SL      -        0xc0c75f24 [g_down]
    3     0     0     0  SL      -        0xc0c75f20 [g_up]
    2     0     0     0  SL      -        0xc0c75f18 [g_event]
   12     0     0     0  WL      (threaded)          intr
100041                   I                           [irq7: ppbus0 ppc0]
100039                   I                           [swi0: sio]
100038                   I                           [irq12: psm0]
100037                   I                           [irq1: atkbd0]
100036                   I                           [irq15: ata1]
100035                   I                           [irq14: ata0]
100034                   I                           [irq17: fxp0]
100030                   I                           [irq16: uhci0]
100028                   I                           [irq9: acpi0]
100027                   I                           [swi6: task queue]
100026                   I                           [swi6: Giant taskq]
100024                   I                           [swi5: +]
100020                   I                           [swi2: cambio]
100012                   I                           [swi1: net]
100011                   I                           [swi3: vm]
100010                   I                           [swi4: clock]
100009                   I                           [swi4: clock]
100008                   I                           [swi4: clock]
100007                   I                           [swi4: clock sio]
   11     0     0     0  RL      (threaded)          idle
100006                   CanRun                      [idle: cpu0]
100005                   Run     CPU 1               [idle: cpu1]
100004                   Run     CPU 2               [idle: cpu2]
100003                   Run     CPU 3               [idle: cpu3]
    1     0     1     0  SLs     wait     0xc40a2d0c [init]
   10     0     0     0  SL      audit_wo 0xc0cd7a7c [audit]
    0     0     0     0  SLs     (threaded)          kernel
100029                   D       -        0xc4298500 [em0 taskq]
100025                   D       -        0xc4253480 [thread taskq]
100023                   D       -        0xc4253540 [acpi_task_2]
100022                   D       -        0xc4253540 [acpi_task_1]
100021                   D       -        0xc4253540 [acpi_task_0]
100018                   D       -        0xc4253840 [kqueue taskq]
100016                   D       -        0xc408a980 [firmware taskq]
100000                   I                           [swapper]
db> call doadump
Physical memory: 1011 MB
Dumping 90 MB: 75 59 43 27 11
Dump complete
= 0xf
db> reset

(kgdb) bt
#0  doadump () at pcpu.h:196
#1  0xc04955b9 in db_fncall (dummy1=0x3f9, dummy2=0x0, dummy3=0x3fb, dummy4=0xe690da24 "\003") at ../../../ddb/db_command.c:516
#2  0xc0495b3c in db_command (last_cmdp=0xc0c462f0, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:413
#3  0xc0495c4a in db_command_loop () at ../../../ddb/db_command.c:466
#4  0xc049743d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5  0xc07b45e6 in kdb_trap (type=0x3, code=0x0, tf=0xe690dbcc) at ../../../kern/subr_kdb.c:534
#6  0xc0aa437b in trap (frame=0xe690dbcc) at ../../../i386/i386/trap.c:683
#7  0xc0a882bb in calltrap () at ../../../i386/i386/exception.s:165
#8  0xc09980d0 in ffs_sync () at ../../../ufs/ffs/ffs_vfsops.c:1290
#9  0xc0806869 in unmount (td=0xc46fc460, uap=0xe690dcfc) at ../../../kern/vfs_mount.c:1184
#10 0xc0aa3b03 in syscall (frame=0xe690dd38) at ../../../i386/i386/trap.c:1081
#11 0xc0a88320 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261
#12 0x00000033 in ?? ()
(kgdb) f 8
#8  0xc09980d0 in ffs_sync () at ../../../ufs/ffs/ffs_vfsops.c:1290
1290    }
(kgdb) up
#9  0xc0806869 in unmount (td=0xc46fc460, uap=0xe690dcfc) at ../../../kern/vfs_mount.c:1184
1184            error = dounmount(mp, uap->flags, td);
(kgdb) p *mp
$2 = {mnt_lock = {lock_object = {lo_name = 0xc0b40fa3 "vfslock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc0c8d618}, lk_lock = 0xc46fc460, lk_timo = 0x0,
    lk_pri = 0x50, lk_stack = {depth = 0x5, pcs = {0xc0774663, 0xc0805fe7, 0xc0806869, 0xc0aa3b03, 0xc0a88320, 0x0 <repeats 13 times>}}}, mnt_mtx = {lock_object = {
      lo_name = 0xc0b3b3c4 "struct mount mtx", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc0c8f3c8}, mtx_lock = 0x4}, mnt_gen = 0x1, mnt_list = {tqe_next = 0x0,
    tqe_prev = 0xc45726f0}, mnt_op = 0xc0c2c660, mnt_vfc = 0xc0c2c620, mnt_vnodecovered = 0xc46e4560, mnt_syncer = 0x0, mnt_ref = 0x0, mnt_nvnodelist = {tqh_first = 0x0,
    tqh_last = 0xc45723d4}, mnt_nvnodelistsize = 0x0, mnt_writeopcount = 0x1, mnt_kern_flag = 0x2100000d, mnt_flag = 0x201300, mnt_noasync = 0x1, mnt_opt = 0xc457b6b0,
  mnt_optnew = 0x0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x5, f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0x2637,
    f_bfree = 0x2634, f_bavail = 0x2326, f_files = 0xafe, f_ffree = 0xafb, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {0x0,
      0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x485bc1f7, 0x63902ae}}, f_charspare = '\0' <repeats 79 times>,
    f_fstypename = "ufs", '\0' <repeats 12 times>, f_mntfromname = "/dev/md5c", '\0' <repeats 78 times>, f_mntonname = "/mnt", '\0' <repeats 83 times>},
  mnt_cred = 0xc46ca300, mnt_data = 0xc4c81400, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc46eb800, mnt_label = 0x0, mnt_hashseed = 0xa36e2df2,
  mnt_markercnt = 0x0, mnt_holdcnt = 0x0, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0, mnt_secondary_accwrites = 0x1791, mnt_gjprovider = 0x0, mnt_explock = {
    lock_object = {lo_name = 0xc0b40fab "explock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc0c8d5f0}, lk_lock = 0x1, lk_timo = 0x0, lk_pri = 0x50, lk_stack = {
      depth = 0x6, pcs = {0xc0774020, 0xc07fd38d, 0xc0804af6, 0xc0805d93, 0xc0aa3b03, 0xc0a88320, 0x0 <repeats 12 times>}}}}
(kgdb) p *((struct ufsmount *)((mp)->mnt_data))
$3 = {um_mountp = 0xc4572338, um_dev = 0xc4ad1e00, um_cp = 0xc4cde8c0, um_bo = 0xc4c340f4, um_devvp = 0xc4c34000, um_fstype = 0x2, um_fs = 0xc458c800, um_extattr = {
    uepm_lock = {lock_object = {lo_name = 0x0, lo_flags = 0x0, lo_data = 0x0, lo_witness = 0x0}, sx_lock = 0x0}, uepm_list = {lh_first = 0x0}, uepm_ucred = 0x0,
    uepm_flags = 0x0}, um_nindir = 0x800, um_bptrtodb = 0x2, um_seqinc = 0x8, um_lock = {lock_object = {lo_name = 0xc0b5908e "FFS", lo_flags = 0x1030000, lo_data = 0x0,
      lo_witness = 0xc0c8d578}, mtx_lock = 0x4}, um_numindirdeps = 0x0, softdep_workitem_pending = {lh_first = 0x0}, softdep_worklist_tail = 0x0, softdep_on_worklist = 0x0,
  softdep_on_worklist_inprogress = 0x0, softdep_deps = 0x2e, softdep_accdeps = 0x1559, softdep_req = 0x0, um_quotas = {0x0, 0x0}, um_cred = {0x0, 0x0}, um_btime = {0x0,
    0x0}, um_itime = {0x0, 0x0}, um_qflags = "\000", um_savedmaxfilesize = 0x0, um_balloc = 0xc097d170 <ffs_balloc_ufs2>, um_blkatoff = 0xc09963c0 <ffs_blkatoff>,
  um_truncate = 0xc0981290 <ffs_truncate>, um_update = 0xc0980f10 <ffs_update>, um_valloc = 0xc09784b0 <ffs_valloc>, um_vfree = 0xc0977120 <ffs_vfree>,
  um_ifree = 0xc0996e50 <ffs_ifree>}
(kgdb)
--
Test scenario:

$ ./mountro2.sh
mount: /dev/md5c : Device busy
mdconfig: ioctl(/dev/mdctl): Device busy
$ mount | grep md5
/dev/md5c on /mnt (ufs, NFS exported, local, soft-updates)
$ umount -f /mnt
umount: unmount of /mnt failed: Device busy
$ umount -f /mnt