GENERIC HEAD from Jun 19 20:17 UTC, vmcore.240
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2008 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 8.0-CURRENT #1: Fri Jun 20 16:11:31 CEST 2008
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1037516800 (989 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
lock order reversal: (sleepable after non-sleepable)
1st 0xc4346e64 bufobj interlock (bufobj interlock) @ kern/vfs_bio.c:2442
2nd 0xd80ea918 bufwait (bufwait) @ kern/vfs_bio.c:2456
KDB: stack backtrace:
db_trace_self_wrapper(c0b38a32,c3da4784,c07c5b5e,c0b3b26f,d80ea918,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c0b3b26f,d80ea918,c0b3fe81,c0b3fe81,c0b3f437,...) at kdb_backtrace+0x29
witness_checkorder(d80ea918,9,c0b3f42e,998,c0c84a34,...) at witness_checkorder+0x6de
__lockmgr_args(d80ea918,81900,c4346e64,c0b3fdb6,50,...) at __lockmgr_args+0x777
getblk(c4346d70,0,0,800,0,...) at getblk+0x163
breadn(c4346d70,0,0,800,0,...) at breadn+0x44
bread(c4346d70,0,0,800,0,...) at bread+0x4c
ffs_blkatoff(c4346d70,0,0,0,c3da49a4,...) at ffs_blkatoff+0xd1
ufs_lookup(c3da49ec,c0b6beb4,c4346d70,c4346d70,c3da4b50,...) at ufs_lookup+0x2e6
VOP_CACHEDLOOKUP_APV(c0c2c860,c3da49ec,c3da4b50,c3da4b3c,c40a9800,...) at VOP_CACHEDLOOKUP_APV+0xc5
vfs_cache_lookup(c3da4a6c,c0b421e3,c0c44b60,80000,c4346d70,...) at vfs_cache_lookup+0xd3
VOP_LOOKUP_APV(c0c2c860,c3da4a6c,c0b40c5b,1b0,c3da4b3c,...) at VOP_LOOKUP_APV+0xe5
lookup(c3da4b24,c0b40c5b,d8,c0,c407322c,...) at lookup+0x58e
namei(c3da4b24,c3da4b28,c07c533c,c0b40fa3,c0ccc0e8,...) at namei+0x45b
kern_unlinkat(c40a4d20,ffffff9c,c0b412ea,1,c3da4c5c,...) at kern_unlinkat+0x46
kern_unlink(c40a4d20,c0b412ea,1,62c,0,...) at kern_unlink+0x27
vfs_mountroot_try(c0b414a4,c0b2f2c6,c0b27df3,1,c0802d20,...) at vfs_mountroot_try+0x49f
vfs_mountroot(c0c77bb0,4,c0b305e3,264,0,...) at vfs_mountroot+0x420
start_init(0,c3da4d38,c0b31f64,324,c40a2d0c,...) at start_init+0x65
fork_exit(c074f860,0,c3da4d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc3da4d70, ebp = 0 ---
fxp0: link state changed to UP
softdep_waitidle: Failed to flush worklist for 0xc4572338
softdep_waitidle: Failed to flush worklist for 0xc4572338
softdep_waitidle: Failed to flush worklist for 0xc4572338
~KDB: enter: Line break on console
[thread pid 11 tid 100006 ]
Stopped at kdb_enter+0x3a: movl $0,kdb_why
db> b ffs_unmount
db> c
[thread pid 964 tid 100114 ]
Breakpoint at ffs_unmount: pushl %ebp
db> bt
Tracing pid 964 tid 100114 td 0xc46fc460
ffs_unmount(c4572338,8080000,c46fc460,484,63902ae,...) at ffs_unmount
unmount(c46fc460,e690dcfc,8,c46fc460,c0c09150,...) at unmount+0x2e9
syscall(e690dd38) at syscall+0x2a3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (22, FreeBSD ELF32, unmount), eip = 0x280cf7ff, esp = 0xbfbfe3cc, ebp = 0xbfbfe488 ---
db> show mount 0xc4572338
0xc4572338 /dev/md5c on /mnt (ufs)
mnt_flag = SOFTDEP, EXPORTED, DEFEXPORTED, LOCAL
mnt_kern_flag = UNMOUNTF, SOFTDEP, NOINSMNTQ, UNMOUNT, MPSAFE
mnt_stat = { version=537068824 type=5 flags=0x0000000000201300 bsize=2048 iosize=16384 blocks=9783 bfree=9780 bavail=8998 files=2814 ffree=2811 syncwrites=0 asyncwrites=0 syncreads=0 asyncreads=0 namemax=255 owner=0 fsid=[1213972983, 104399534] }
mnt_cred = { uid=0 ruid=0 }
mnt_ref = 0
mnt_gen = 1
mnt_nvnodelistsize = 0
mnt_writeopcount = 1
mnt_noasync = 1
mnt_maxsymlinklen = 120
mnt_iosize_max = 131072
mnt_hashseed = 2741906930
mnt_markercnt = 0
mnt_holdcnt = 0
mnt_holdcntwaiters = 0
mnt_secondary_writes = 0
mnt_secondary_accwrites = 6033
mnt_gjprovider = NULL
db> set $lines 0
db> ps
pid ppid pgrp uid state wmesg wchan cmd
964 931 964 0 R+ CPU 0 umount
949 0 0 0 SL mdwait 0xc4310000 [md5]
931 927 931 0 S+ wait 0xc49e9000 bash
927 926 927 0 S+ pause 0xc4ba4ac8 csh
926 922 926 1001 S+ wait 0xc45f0000 su
922 920 922 1001 Ss+ wait 0xc4ba4d0c bash
920 918 918 1001 S select 0xc44f1124 sshd
918 794 918 0 Ss sbwait 0xc4ba2548 sshd
917 1 917 0 Ss+ ttyin 0xc4338c10 getty
916 1 916 0 Ss+ ttyin 0xc4338810 getty
915 1 915 0 Ss+ ttyin 0xc4339810 getty
914 1 914 0 Ss+ ttyin 0xc4339410 getty
913 1 913 0 Ss+ ttyin 0xc4332810 getty
912 1 912 0 Ss+ ttyin 0xc4332c10 getty
911 1 911 0 Ss+ ttyin 0xc4338010 getty
910 1 910 0 Ss+ ttyin 0xc4338410 getty
893 1 893 0 Ss select 0xc4528d64 inetd
866 1 866 0 Ss select 0xc43353e4 moused
852 1 852 0 Ss nanslp 0xc0c78324 watchdogd
810 1 810 0 Ss nanslp 0xc0c78324 cron
803 1 803 25 Ss pause 0xc48fd058 sendmail
799 1 799 0 Ss select 0xc4575364 sendmail
794 1 794 0 Ss select 0xc4575524 sshd
775 1 775 0 Ss select 0xc44f1824 ntpd
715 1 714 0 S nanslp 0xc0c78324 smartd
691 675 675 0 S - 0xc45a4400 nfsd
690 675 675 0 S - 0xc44eae00 nfsd
689 675 675 0 S - 0xc45a4600 nfsd
688 675 675 0 S - 0xc4531000 nfsd
687 675 675 0 S - 0xc45a4800 nfsd
686 675 675 0 S - 0xc4531200 nfsd
685 675 675 0 S - 0xc45a5400 nfsd
684 675 675 0 S - 0xc4531400 nfsd
683 675 675 0 S - 0xc45a5200 nfsd
682 675 675 0 S - 0xc4531600 nfsd
680 675 675 0 S - 0xc45a5000 nfsd
679 675 675 0 S - 0xc46eb600 nfsd
678 675 675 0 S - 0xc45a4e00 nfsd
677 675 675 0 S - 0xc4532600 nfsd
676 675 675 0 S - 0xc45a4c00 nfsd
675 1 675 0 Ss select 0xc432cde4 nfsd
673 1 673 0 Ss select 0xc45283e4 mountd
620 1 620 0 Ss select 0xc44f1524 rpcbind
603 1 603 0 Ss select 0xc44f1424 syslogd
546 1 546 0 Ss select 0xc44f21e4 devd
21 0 0 0 SL sdflush 0xc0cd8080 [softdepflush]
20 0 0 0 SL vlruwt 0xc450329c [vnlru]
19 0 0 0 SL syncer 0xc0c78140 [syncer]
18 0 0 0 SL psleep 0xc0ccc624 [bufdaemon]
17 0 0 0 SL pgzero 0xc0cd8c74 [pagezero]
9 0 0 0 SL psleep 0xc0cd889c [vmdaemon]
8 0 0 0 SL psleep 0xc0cd8864 [pagedaemon]
7 0 0 0 SL waiting_ 0xc0cce2e0 [sctp_iterator]
6 0 0 0 SL - 0xc42ae23c [fdc0]
16 0 0 0 SL usbtsk 0xc0c75814 [usbtask-dr]
15 0 0 0 SL usbtsk 0xc0c75800 [usbtask-hc]
14 0 0 0 SL usbevt 0xc429f210 [usb0]
5 0 0 0 SL ccb_scan 0xc0c45654 [xpt_thrd]
13 0 0 0 SL - 0xc0c78148 [yarrow]
4 0 0 0 SL - 0xc0c75f24 [g_down]
3 0 0 0 SL - 0xc0c75f20 [g_up]
2 0 0 0 SL - 0xc0c75f18 [g_event]
12 0 0 0 WL (threaded) intr
100041 I [irq7: ppbus0 ppc0]
100039 I [swi0: sio]
100038 I [irq12: psm0]
100037 I [irq1: atkbd0]
100036 I [irq15: ata1]
100035 I [irq14: ata0]
100034 I [irq17: fxp0]
100030 I [irq16: uhci0]
100028 I [irq9: acpi0]
100027 I [swi6: task queue]
100026 I [swi6: Giant taskq]
100024 I [swi5: +]
100020 I [swi2: cambio]
100012 I [swi1: net]
100011 I [swi3: vm]
100010 I [swi4: clock]
100009 I [swi4: clock]
100008 I [swi4: clock]
100007 I [swi4: clock sio]
11 0 0 0 RL (threaded) idle
100006 CanRun [idle: cpu0]
100005 Run CPU 1 [idle: cpu1]
100004 Run CPU 2 [idle: cpu2]
100003 Run CPU 3 [idle: cpu3]
1 0 1 0 SLs wait 0xc40a2d0c [init]
10 0 0 0 SL audit_wo 0xc0cd7a7c [audit]
0 0 0 0 SLs (threaded) kernel
100029 D - 0xc4298500 [em0 taskq]
100025 D - 0xc4253480 [thread taskq]
100023 D - 0xc4253540 [acpi_task_2]
100022 D - 0xc4253540 [acpi_task_1]
100021 D - 0xc4253540 [acpi_task_0]
100018 D - 0xc4253840 [kqueue taskq]
100016 D - 0xc408a980 [firmware taskq]
100000 I [swapper]
db> call doadump
Physical memory: 1011 MB
Dumping 90 MB: 75 59 43 27 11
Dump complete
= 0xf
db> reset
(kgdb) bt
#0 doadump () at pcpu.h:196
#1 0xc04955b9 in db_fncall (dummy1=0x3f9, dummy2=0x0, dummy3=0x3fb, dummy4=0xe690da24 "ø\003") at ../../../ddb/db_command.c:516
#2 0xc0495b3c in db_command (last_cmdp=0xc0c462f0, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:413
#3 0xc0495c4a in db_command_loop () at ../../../ddb/db_command.c:466
#4 0xc049743d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5 0xc07b45e6 in kdb_trap (type=0x3, code=0x0, tf=0xe690dbcc) at ../../../kern/subr_kdb.c:534
#6 0xc0aa437b in trap (frame=0xe690dbcc) at ../../../i386/i386/trap.c:683
#7 0xc0a882bb in calltrap () at ../../../i386/i386/exception.s:165
#8 0xc09980d0 in ffs_sync () at ../../../ufs/ffs/ffs_vfsops.c:1290
#9 0xc0806869 in unmount (td=0xc46fc460, uap=0xe690dcfc) at ../../../kern/vfs_mount.c:1184
#10 0xc0aa3b03 in syscall (frame=0xe690dd38) at ../../../i386/i386/trap.c:1081
#11 0xc0a88320 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261
#12 0x00000033 in ?? ()
(kgdb) f 8
#8 0xc09980d0 in ffs_sync () at ../../../ufs/ffs/ffs_vfsops.c:1290
1290 }
(kgdb) up
#9 0xc0806869 in unmount (td=0xc46fc460, uap=0xe690dcfc) at ../../../kern/vfs_mount.c:1184
1184 error = dounmount(mp, uap->flags, td);
(kgdb) p *mp
$2 = {mnt_lock = {lock_object = {lo_name = 0xc0b40fa3 "vfslock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc0c8d618}, lk_lock = 0xc46fc460, lk_timo = 0x0,
lk_pri = 0x50, lk_stack = {depth = 0x5, pcs = {0xc0774663, 0xc0805fe7, 0xc0806869, 0xc0aa3b03, 0xc0a88320, 0x0 <repeats 13 times>}}}, mnt_mtx = {lock_object = {
lo_name = 0xc0b3b3c4 "struct mount mtx", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc0c8f3c8}, mtx_lock = 0x4}, mnt_gen = 0x1, mnt_list = {tqe_next = 0x0,
tqe_prev = 0xc45726f0}, mnt_op = 0xc0c2c660, mnt_vfc = 0xc0c2c620, mnt_vnodecovered = 0xc46e4560, mnt_syncer = 0x0, mnt_ref = 0x0, mnt_nvnodelist = {tqh_first = 0x0,
tqh_last = 0xc45723d4}, mnt_nvnodelistsize = 0x0, mnt_writeopcount = 0x1, mnt_kern_flag = 0x2100000d, mnt_flag = 0x201300, mnt_noasync = 0x1, mnt_opt = 0xc457b6b0,
mnt_optnew = 0x0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x5, f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0x2637,
f_bfree = 0x2634, f_bavail = 0x2326, f_files = 0xafe, f_ffree = 0xafb, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x485bc1f7, 0x63902ae}}, f_charspare = '\0' <repeats 79 times>,
f_fstypename = "ufs", '\0' <repeats 12 times>, f_mntfromname = "/dev/md5c", '\0' <repeats 78 times>, f_mntonname = "/mnt", '\0' <repeats 83 times>},
mnt_cred = 0xc46ca300, mnt_data = 0xc4c81400, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc46eb800, mnt_label = 0x0, mnt_hashseed = 0xa36e2df2,
mnt_markercnt = 0x0, mnt_holdcnt = 0x0, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0, mnt_secondary_accwrites = 0x1791, mnt_gjprovider = 0x0, mnt_explock = {
lock_object = {lo_name = 0xc0b40fab "explock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc0c8d5f0}, lk_lock = 0x1, lk_timo = 0x0, lk_pri = 0x50, lk_stack = {
depth = 0x6, pcs = {0xc0774020, 0xc07fd38d, 0xc0804af6, 0xc0805d93, 0xc0aa3b03, 0xc0a88320, 0x0 <repeats 12 times>}}}}
(kgdb) p *((struct ufsmount *)((mp)->mnt_data))
$3 = {um_mountp = 0xc4572338, um_dev = 0xc4ad1e00, um_cp = 0xc4cde8c0, um_bo = 0xc4c340f4, um_devvp = 0xc4c34000, um_fstype = 0x2, um_fs = 0xc458c800, um_extattr = {
uepm_lock = {lock_object = {lo_name = 0x0, lo_flags = 0x0, lo_data = 0x0, lo_witness = 0x0}, sx_lock = 0x0}, uepm_list = {lh_first = 0x0}, uepm_ucred = 0x0,
uepm_flags = 0x0}, um_nindir = 0x800, um_bptrtodb = 0x2, um_seqinc = 0x8, um_lock = {lock_object = {lo_name = 0xc0b5908e "FFS", lo_flags = 0x1030000, lo_data = 0x0,
lo_witness = 0xc0c8d578}, mtx_lock = 0x4}, um_numindirdeps = 0x0, softdep_workitem_pending = {lh_first = 0x0}, softdep_worklist_tail = 0x0, softdep_on_worklist = 0x0,
softdep_on_worklist_inprogress = 0x0, softdep_deps = 0x2e, softdep_accdeps = 0x1559, softdep_req = 0x0, um_quotas = {0x0, 0x0}, um_cred = {0x0, 0x0}, um_btime = {0x0,
0x0}, um_itime = {0x0, 0x0}, um_qflags = "\000", um_savedmaxfilesize = 0x0, um_balloc = 0xc097d170 <ffs_balloc_ufs2>, um_blkatoff = 0xc09963c0 <ffs_blkatoff>,
um_truncate = 0xc0981290 <ffs_truncate>, um_update = 0xc0980f10 <ffs_update>, um_valloc = 0xc09784b0 <ffs_valloc>, um_vfree = 0xc0977120 <ffs_vfree>,
um_ifree = 0xc0996e50 <ffs_ifree>}
(kgdb)
--
Test scenario:
$ ./mountro2.sh
mount: /dev/md5c : Device busy
mdconfig: ioctl(/dev/mdctl): Device busy
$ mount | grep md5
/dev/md5c on /mnt (ufs, NFS exported, local, soft-updates)
$ umount -f /mnt
umount: unmount of /mnt failed: Device busy
$ umount -f /mnt