GENERIC HEAD from Jun 19 20:17 UTC, vmcore.240 GDB: no debug ports present KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2008 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 8.0-CURRENT #1: Fri Jun 20 16:11:31 CEST 2008 pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0xf24 Stepping = 4 Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM> Logical CPUs per core: 2 real memory = 1073676288 (1023 MB) avail memory = 1037516800 (989 MB) : Trying to mount root from ufs:/dev/ad0s1a lock order reversal: (sleepable after non-sleepable) 1st 0xc4346e64 bufobj interlock (bufobj interlock) @ kern/vfs_bio.c:2442 2nd 0xd80ea918 bufwait (bufwait) @ kern/vfs_bio.c:2456 KDB: stack backtrace: db_trace_self_wrapper(c0b38a32,c3da4784,c07c5b5e,c0b3b26f,d80ea918,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0b3b26f,d80ea918,c0b3fe81,c0b3fe81,c0b3f437,...) at kdb_backtrace+0x29 witness_checkorder(d80ea918,9,c0b3f42e,998,c0c84a34,...) at witness_checkorder+0x6de __lockmgr_args(d80ea918,81900,c4346e64,c0b3fdb6,50,...) at __lockmgr_args+0x777 getblk(c4346d70,0,0,800,0,...) at getblk+0x163 breadn(c4346d70,0,0,800,0,...) at breadn+0x44 bread(c4346d70,0,0,800,0,...) at bread+0x4c ffs_blkatoff(c4346d70,0,0,0,c3da49a4,...) at ffs_blkatoff+0xd1 ufs_lookup(c3da49ec,c0b6beb4,c4346d70,c4346d70,c3da4b50,...) at ufs_lookup+0x2e6 VOP_CACHEDLOOKUP_APV(c0c2c860,c3da49ec,c3da4b50,c3da4b3c,c40a9800,...) at VOP_CACHEDLOOKUP_APV+0xc5 vfs_cache_lookup(c3da4a6c,c0b421e3,c0c44b60,80000,c4346d70,...) at vfs_cache_lookup+0xd3 VOP_LOOKUP_APV(c0c2c860,c3da4a6c,c0b40c5b,1b0,c3da4b3c,...) at VOP_LOOKUP_APV+0xe5 lookup(c3da4b24,c0b40c5b,d8,c0,c407322c,...) at lookup+0x58e namei(c3da4b24,c3da4b28,c07c533c,c0b40fa3,c0ccc0e8,...) at namei+0x45b kern_unlinkat(c40a4d20,ffffff9c,c0b412ea,1,c3da4c5c,...) at kern_unlinkat+0x46 kern_unlink(c40a4d20,c0b412ea,1,62c,0,...) at kern_unlink+0x27 vfs_mountroot_try(c0b414a4,c0b2f2c6,c0b27df3,1,c0802d20,...) at vfs_mountroot_try+0x49f vfs_mountroot(c0c77bb0,4,c0b305e3,264,0,...) at vfs_mountroot+0x420 start_init(0,c3da4d38,c0b31f64,324,c40a2d0c,...) at start_init+0x65 fork_exit(c074f860,0,c3da4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3da4d70, ebp = 0 --- fxp0: link state changed to UP softdep_waitidle: Failed to flush worklist for 0xc4572338 softdep_waitidle: Failed to flush worklist for 0xc4572338 softdep_waitidle: Failed to flush worklist for 0xc4572338 ~KDB: enter: Line break on console [thread pid 11 tid 100006 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> b ffs_unmount db> c [thread pid 964 tid 100114 ] Breakpoint at ffs_unmount: pushl %ebp db> bt Tracing pid 964 tid 100114 td 0xc46fc460 ffs_unmount(c4572338,8080000,c46fc460,484,63902ae,...) at ffs_unmount unmount(c46fc460,e690dcfc,8,c46fc460,c0c09150,...) at unmount+0x2e9 syscall(e690dd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280cf7ff, esp = 0xbfbfe3cc, ebp = 0xbfbfe488 --- db> show mount 0xc4572338 0xc4572338 /dev/md5c on /mnt (ufs) mnt_flag = SOFTDEP, EXPORTED, DEFEXPORTED, LOCAL mnt_kern_flag = UNMOUNTF, SOFTDEP, NOINSMNTQ, UNMOUNT, MPSAFE mnt_stat = { version=537068824 type=5 flags=0x0000000000201300 bsize=2048 iosize=16384 blocks=9783 bfree=9780 bavail=8998 files=2814 ffree=2811 syncwrites=0 asyncwrites=0 syncreads=0 asyncreads=0 namemax=255 owner=0 fsid=[1213972983, 104399534] } mnt_cred = { uid=0 ruid=0 } mnt_ref = 0 mnt_gen = 1 mnt_nvnodelistsize = 0 mnt_writeopcount = 1 mnt_noasync = 1 mnt_maxsymlinklen = 120 mnt_iosize_max = 131072 mnt_hashseed = 2741906930 mnt_markercnt = 0 mnt_holdcnt = 0 mnt_holdcntwaiters = 0 mnt_secondary_writes = 0 mnt_secondary_accwrites = 6033 mnt_gjprovider = NULL db> set $lines 0 db> ps pid ppid pgrp uid state wmesg wchan cmd 964 931 964 0 R+ CPU 0 umount 949 0 0 0 SL mdwait 0xc4310000 [md5] 931 927 931 0 S+ wait 0xc49e9000 bash 927 926 927 0 S+ pause 0xc4ba4ac8 csh 926 922 926 1001 S+ wait 0xc45f0000 su 922 920 922 1001 Ss+ wait 0xc4ba4d0c bash 920 918 918 1001 S select 0xc44f1124 sshd 918 794 918 0 Ss sbwait 0xc4ba2548 sshd 917 1 917 0 Ss+ ttyin 0xc4338c10 getty 916 1 916 0 Ss+ ttyin 0xc4338810 getty 915 1 915 0 Ss+ ttyin 0xc4339810 getty 914 1 914 0 Ss+ ttyin 0xc4339410 getty 913 1 913 0 Ss+ ttyin 0xc4332810 getty 912 1 912 0 Ss+ ttyin 0xc4332c10 getty 911 1 911 0 Ss+ ttyin 0xc4338010 getty 910 1 910 0 Ss+ ttyin 0xc4338410 getty 893 1 893 0 Ss select 0xc4528d64 inetd 866 1 866 0 Ss select 0xc43353e4 moused 852 1 852 0 Ss nanslp 0xc0c78324 watchdogd 810 1 810 0 Ss nanslp 0xc0c78324 cron 803 1 803 25 Ss pause 0xc48fd058 sendmail 799 1 799 0 Ss select 0xc4575364 sendmail 794 1 794 0 Ss select 0xc4575524 sshd 775 1 775 0 Ss select 0xc44f1824 ntpd 715 1 714 0 S nanslp 0xc0c78324 smartd 691 675 675 0 S - 0xc45a4400 nfsd 690 675 675 0 S - 0xc44eae00 nfsd 689 675 675 0 S - 0xc45a4600 nfsd 688 675 675 0 S - 0xc4531000 nfsd 687 675 675 0 S - 0xc45a4800 nfsd 686 675 675 0 S - 0xc4531200 nfsd 685 675 675 0 S - 0xc45a5400 nfsd 684 675 675 0 S - 0xc4531400 nfsd 683 675 675 0 S - 0xc45a5200 nfsd 682 675 675 0 S - 0xc4531600 nfsd 680 675 675 0 S - 0xc45a5000 nfsd 679 675 675 0 S - 0xc46eb600 nfsd 678 675 675 0 S - 0xc45a4e00 nfsd 677 675 675 0 S - 0xc4532600 nfsd 676 675 675 0 S - 0xc45a4c00 nfsd 675 1 675 0 Ss select 0xc432cde4 nfsd 673 1 673 0 Ss select 0xc45283e4 mountd 620 1 620 0 Ss select 0xc44f1524 rpcbind 603 1 603 0 Ss select 0xc44f1424 syslogd 546 1 546 0 Ss select 0xc44f21e4 devd 21 0 0 0 SL sdflush 0xc0cd8080 [softdepflush] 20 0 0 0 SL vlruwt 0xc450329c [vnlru] 19 0 0 0 SL syncer 0xc0c78140 [syncer] 18 0 0 0 SL psleep 0xc0ccc624 [bufdaemon] 17 0 0 0 SL pgzero 0xc0cd8c74 [pagezero] 9 0 0 0 SL psleep 0xc0cd889c [vmdaemon] 8 0 0 0 SL psleep 0xc0cd8864 [pagedaemon] 7 0 0 0 SL waiting_ 0xc0cce2e0 [sctp_iterator] 6 0 0 0 SL - 0xc42ae23c [fdc0] 16 0 0 0 SL usbtsk 0xc0c75814 [usbtask-dr] 15 0 0 0 SL usbtsk 0xc0c75800 [usbtask-hc] 14 0 0 0 SL usbevt 0xc429f210 [usb0] 5 0 0 0 SL ccb_scan 0xc0c45654 [xpt_thrd] 13 0 0 0 SL - 0xc0c78148 [yarrow] 4 0 0 0 SL - 0xc0c75f24 [g_down] 3 0 0 0 SL - 0xc0c75f20 [g_up] 2 0 0 0 SL - 0xc0c75f18 [g_event] 12 0 0 0 WL (threaded) intr 100041 I [irq7: ppbus0 ppc0] 100039 I [swi0: sio] 100038 I [irq12: psm0] 100037 I [irq1: atkbd0] 100036 I [irq15: ata1] 100035 I [irq14: ata0] 100034 I [irq17: fxp0] 100030 I [irq16: uhci0] 100028 I [irq9: acpi0] 100027 I [swi6: task queue] 100026 I [swi6: Giant taskq] 100024 I [swi5: +] 100020 I [swi2: cambio] 100012 I [swi1: net] 100011 I [swi3: vm] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi4: clock] 100007 I [swi4: clock sio] 11 0 0 0 RL (threaded) idle 100006 CanRun [idle: cpu0] 100005 Run CPU 1 [idle: cpu1] 100004 Run CPU 2 [idle: cpu2] 100003 Run CPU 3 [idle: cpu3] 1 0 1 0 SLs wait 0xc40a2d0c [init] 10 0 0 0 SL audit_wo 0xc0cd7a7c [audit] 0 0 0 0 SLs (threaded) kernel 100029 D - 0xc4298500 [em0 taskq] 100025 D - 0xc4253480 [thread taskq] 100023 D - 0xc4253540 [acpi_task_2] 100022 D - 0xc4253540 [acpi_task_1] 100021 D - 0xc4253540 [acpi_task_0] 100018 D - 0xc4253840 [kqueue taskq] 100016 D - 0xc408a980 [firmware taskq] 100000 I [swapper] db> call doadump Physical memory: 1011 MB Dumping 90 MB: 75 59 43 27 11 Dump complete = 0xf db> reset (kgdb) bt #0 doadump () at pcpu.h:196 #1 0xc04955b9 in db_fncall (dummy1=0x3f9, dummy2=0x0, dummy3=0x3fb, dummy4=0xe690da24 "ø\003") at ../../../ddb/db_command.c:516 #2 0xc0495b3c in db_command (last_cmdp=0xc0c462f0, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:413 #3 0xc0495c4a in db_command_loop () at ../../../ddb/db_command.c:466 #4 0xc049743d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228 #5 0xc07b45e6 in kdb_trap (type=0x3, code=0x0, tf=0xe690dbcc) at ../../../kern/subr_kdb.c:534 #6 0xc0aa437b in trap (frame=0xe690dbcc) at ../../../i386/i386/trap.c:683 #7 0xc0a882bb in calltrap () at ../../../i386/i386/exception.s:165 #8 0xc09980d0 in ffs_sync () at ../../../ufs/ffs/ffs_vfsops.c:1290 #9 0xc0806869 in unmount (td=0xc46fc460, uap=0xe690dcfc) at ../../../kern/vfs_mount.c:1184 #10 0xc0aa3b03 in syscall (frame=0xe690dd38) at ../../../i386/i386/trap.c:1081 #11 0xc0a88320 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261 #12 0x00000033 in ?? () (kgdb) f 8 #8 0xc09980d0 in ffs_sync () at ../../../ufs/ffs/ffs_vfsops.c:1290 1290 } (kgdb) up #9 0xc0806869 in unmount (td=0xc46fc460, uap=0xe690dcfc) at ../../../kern/vfs_mount.c:1184 1184 error = dounmount(mp, uap->flags, td); (kgdb) p *mp $2 = {mnt_lock = {lock_object = {lo_name = 0xc0b40fa3 "vfslock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc0c8d618}, lk_lock = 0xc46fc460, lk_timo = 0x0, lk_pri = 0x50, lk_stack = {depth = 0x5, pcs = {0xc0774663, 0xc0805fe7, 0xc0806869, 0xc0aa3b03, 0xc0a88320, 0x0 <repeats 13 times>}}}, mnt_mtx = {lock_object = { lo_name = 0xc0b3b3c4 "struct mount mtx", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc0c8f3c8}, mtx_lock = 0x4}, mnt_gen = 0x1, mnt_list = {tqe_next = 0x0, tqe_prev = 0xc45726f0}, mnt_op = 0xc0c2c660, mnt_vfc = 0xc0c2c620, mnt_vnodecovered = 0xc46e4560, mnt_syncer = 0x0, mnt_ref = 0x0, mnt_nvnodelist = {tqh_first = 0x0, tqh_last = 0xc45723d4}, mnt_nvnodelistsize = 0x0, mnt_writeopcount = 0x1, mnt_kern_flag = 0x2100000d, mnt_flag = 0x201300, mnt_noasync = 0x1, mnt_opt = 0xc457b6b0, mnt_optnew = 0x0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x5, f_flags = 0x201300, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0x2637, f_bfree = 0x2634, f_bavail = 0x2326, f_files = 0xafe, f_ffree = 0xafb, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x485bc1f7, 0x63902ae}}, f_charspare = '\0' <repeats 79 times>, f_fstypename = "ufs", '\0' <repeats 12 times>, f_mntfromname = "/dev/md5c", '\0' <repeats 78 times>, f_mntonname = "/mnt", '\0' <repeats 83 times>}, mnt_cred = 0xc46ca300, mnt_data = 0xc4c81400, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc46eb800, mnt_label = 0x0, mnt_hashseed = 0xa36e2df2, mnt_markercnt = 0x0, mnt_holdcnt = 0x0, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0, mnt_secondary_accwrites = 0x1791, mnt_gjprovider = 0x0, mnt_explock = { lock_object = {lo_name = 0xc0b40fab "explock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc0c8d5f0}, lk_lock = 0x1, lk_timo = 0x0, lk_pri = 0x50, lk_stack = { depth = 0x6, pcs = {0xc0774020, 0xc07fd38d, 0xc0804af6, 0xc0805d93, 0xc0aa3b03, 0xc0a88320, 0x0 <repeats 12 times>}}}} (kgdb) p *((struct ufsmount *)((mp)->mnt_data)) $3 = {um_mountp = 0xc4572338, um_dev = 0xc4ad1e00, um_cp = 0xc4cde8c0, um_bo = 0xc4c340f4, um_devvp = 0xc4c34000, um_fstype = 0x2, um_fs = 0xc458c800, um_extattr = { uepm_lock = {lock_object = {lo_name = 0x0, lo_flags = 0x0, lo_data = 0x0, lo_witness = 0x0}, sx_lock = 0x0}, uepm_list = {lh_first = 0x0}, uepm_ucred = 0x0, uepm_flags = 0x0}, um_nindir = 0x800, um_bptrtodb = 0x2, um_seqinc = 0x8, um_lock = {lock_object = {lo_name = 0xc0b5908e "FFS", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc0c8d578}, mtx_lock = 0x4}, um_numindirdeps = 0x0, softdep_workitem_pending = {lh_first = 0x0}, softdep_worklist_tail = 0x0, softdep_on_worklist = 0x0, softdep_on_worklist_inprogress = 0x0, softdep_deps = 0x2e, softdep_accdeps = 0x1559, softdep_req = 0x0, um_quotas = {0x0, 0x0}, um_cred = {0x0, 0x0}, um_btime = {0x0, 0x0}, um_itime = {0x0, 0x0}, um_qflags = "\000", um_savedmaxfilesize = 0x0, um_balloc = 0xc097d170 <ffs_balloc_ufs2>, um_blkatoff = 0xc09963c0 <ffs_blkatoff>, um_truncate = 0xc0981290 <ffs_truncate>, um_update = 0xc0980f10 <ffs_update>, um_valloc = 0xc09784b0 <ffs_valloc>, um_vfree = 0xc0977120 <ffs_vfree>, um_ifree = 0xc0996e50 <ffs_ifree>} (kgdb) -- Test scenario: $ ./mountro2.sh mount: /dev/md5c : Device busy mdconfig: ioctl(/dev/mdctl): Device busy $ mount | grep md5 /dev/md5c on /mnt (ufs, NFS exported, local, soft-updates) $ umount -f /mnt umount: unmount of /mnt failed: Device busy $ umount -f /mnt