-bash-4.2$ sudo kgdb /boot/kernel/kernel.YAHOO /var/crash/vmcore.0 We trust you have received the usual lecture from the local System Administrator. It usually boils down to these three things: #1) Respect the privacy of others. #2) Think before you type. #3) With great power comes great responsibility. Password: GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd"... Unread portion of the kernel message buffer: processor eflags = interrupt enabled, resume, IOPL = 0 current process = 59996 (qmail-send) trap number = 12 panic: page fault cpuid = 5 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe048c9672a0 kdb_backtrace() at kdb_backtrace+0x39/frame 0xfffffe048c967350 vpanic() at vpanic+0x126/frame 0xfffffe048c967390 panic() at panic+0x43/frame 0xfffffe048c9673f0 trap_fatal() at trap_fatal+0x3a2/frame 0xfffffe048c967450 trap_pfault() at trap_pfault+0x22f/frame 0xfffffe048c9674f0 trap() at trap+0x670/frame 0xfffffe048c967710 calltrap() at calltrap+0x8/frame 0xfffffe048c967710 --- trap 0xc, rip = 0xffffffff805e759b, rsp = 0xfffffe048c9677d0, rbp = 0xfffffe048c9677e0 --- __mtx_assert() at __mtx_assert+0x3b/frame 0xfffffe048c9677e0 knote() at knote+0x39/frame 0xfffffe048c967830 pipeclose() at pipeclose+0xbe/frame 0xfffffe048c967860 pipe_dtor() at pipe_dtor+0x49/frame 0xfffffe048c967880 fifo_close() at fifo_close+0x144/frame 0xfffffe048c9678c0 VOP_CLOSE_APV() at VOP_CLOSE_APV+0xea/frame 0xfffffe048c9678f0 vn_close() at vn_close+0x139/frame 0xfffffe048c967960 vn_closefile() at vn_closefile+0x48/frame 0xfffffe048c9679e0 _fdrop() at _fdrop+0x29/frame 0xfffffe048c967a00 closef() at closef+0x1d1/frame 0xfffffe048c967a90 closefp() at closefp+0xa0/frame 0xfffffe048c967ae0 ia32_syscall() at ia32_syscall+0x255/frame 0xfffffe048c967bf0 Xint0x80_syscall() at Xint0x80_syscall+0x95/frame 0xfffffe048c967bf0 --- syscall (6, FreeBSD ELF32, sys_close), rip = 0x210f7804, rsp = 0xffffcae4, rbp = 0xffffcb00 --- Uptime: 30d22h18m58s Dumping 2251 out of 16358 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91% Reading symbols from /boot/modules/dscp_rewrite.ko...done. Loaded symbols for /boot/modules/dscp_rewrite.ko Reading symbols from /boot/modules/markdev_mod.ko...done. Loaded symbols for /boot/modules/markdev_mod.ko Reading symbols from /boot/modules/ylock_mod.ko...done. Loaded symbols for /boot/modules/ylock_mod.ko #0 doadump (textdump=1) at pcpu.h:219 219 pcpu.h: No such file or directory. in pcpu.h (kgdb) bt full #0 doadump (textdump=1) at pcpu.h:219 coredump = #1 0xffffffff805fb2a7 in kern_reboot (howto=260) at ../../../kern/kern_shutdown.c:452 _el = #2 0xffffffff805fb7b5 in vpanic (fmt=, ap=) at ../../../kern/kern_shutdown.c:759 buf = "page fault", '\0' #3 0xffffffff805fb803 in panic (fmt=) at ../../../kern/kern_shutdown.c:688 ap = {{gp_offset = 16, fp_offset = 48, overflow_arg_area = 0xfffffe048c967400, reg_save_area = 0xfffffe048c9673a0}} #4 0xffffffff80977852 in trap_fatal (frame=, eva=) at ../../../amd64/amd64/trap.c:882 softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27, ssd_dpl = 0, ssd_p = 1, ssd_long = 1, ssd_def32 = 0, ssd_gran = 1} #5 0xffffffff80977a8f in trap_pfault (frame=0x0, usermode=) at ../../../amd64/amd64/trap.c:699 rv = Cannot access memory at address 0x0 Current language: auto; currently minimal (kgdb) bt #0 doadump (textdump=1) at pcpu.h:219 #1 0xffffffff805fb2a7 in kern_reboot (howto=260) at ../../../kern/kern_shutdown.c:452 #2 0xffffffff805fb7b5 in vpanic (fmt=, ap=) at ../../../kern/kern_shutdown.c:759 #3 0xffffffff805fb803 in panic (fmt=) at ../../../kern/kern_shutdown.c:688 #4 0xffffffff80977852 in trap_fatal (frame=, eva=) at ../../../amd64/amd64/trap.c:882 #5 0xffffffff80977a8f in trap_pfault (frame=0x0, usermode=) at ../../../amd64/amd64/trap.c:699 #6 0xffffffff809772a0 in trap (frame=0xfffffe048c967720) at ../../../amd64/amd64/trap.c:463 #7 0xffffffff8095c7e2 in calltrap () at ../../../amd64/amd64/exception.S:232 #8 0xffffffff805e759b in __mtx_assert (c=0x18, what=4, file=0xffffffff80ca0bcb "../../../kern/kern_event.c", line=1960) at ../../../kern/kern_mutex.c:791 #9 0xffffffff805c2099 in knote (list=0xfffff802ffcadae0, hint=0, lockflags=1) at ../../../kern/kern_event.c:1822 #10 0xffffffff806510fe in pipeclose (cpipe=0xfffff802ffcada18) at ../../../kern/sys_pipe.c:1655 #11 0xffffffff80651019 in pipe_dtor (dpipe=) at ../../../kern/sys_pipe.c:395 #12 0xffffffff80559924 in fifo_close (ap=) at ../../../fs/fifofs/fifo_vnops.c:115 #13 0xffffffff80a6ba6a in VOP_CLOSE_APV (vop=, a=) at vnode_if.c:535 #14 0xffffffff806acc09 in vn_close (vp=0xfffff80386042588, flags=6, file_cred=0xfffff801b3fa9900, td=0xfffff8010b76d490) at vnode_if.h:225 #15 0xffffffff806abad8 in vn_closefile (fp=0xfffff8034f9757d0, td=0xfffff8010b76d490) at ../../../kern/vfs_vnops.c:1481 #16 0xffffffff805b8789 in _fdrop (fp=0xfffff8034f9757d0, td=0x4) at file.h:342 #17 0xffffffff805bb0e1 in closef (fp=0xfffff8034f9757d0, td=0xfffff8010b76d490) at ../../../kern/kern_descrip.c:2415 #18 0xffffffff805b8bf0 in closefp (fdp=0xfffff80099c4a000, fd=, fp=0xfffff8034f9757d0, td=0xfffff8010b76d490, holdleaders=) at ../../../kern/kern_descrip.c:1257 #19 0xffffffff80a332b5 in ia32_syscall (frame=0xfffffe048c967c00) at subr_syscall.c:135 #20 0xffffffff8095cdc5 in Xint0x80_syscall () at ia32_exception.S:73 #21 0x00000000210f7804 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) where #0 doadump (textdump=1) at pcpu.h:219 #1 0xffffffff805fb2a7 in kern_reboot (howto=260) at ../../../kern/kern_shutdown.c:452 #2 0xffffffff805fb7b5 in vpanic (fmt=, ap=) at ../../../kern/kern_shutdown.c:759 #3 0xffffffff805fb803 in panic (fmt=) at ../../../kern/kern_shutdown.c:688 #4 0xffffffff80977852 in trap_fatal (frame=, eva=) at ../../../amd64/amd64/trap.c:882 #5 0xffffffff80977a8f in trap_pfault (frame=0x0, usermode=) at ../../../amd64/amd64/trap.c:699 #6 0xffffffff809772a0 in trap (frame=0xfffffe048c967720) at ../../../amd64/amd64/trap.c:463 #7 0xffffffff8095c7e2 in calltrap () at ../../../amd64/amd64/exception.S:232 #8 0xffffffff805e759b in __mtx_assert (c=0x18, what=4, file=0xffffffff80ca0bcb "../../../kern/kern_event.c", line=1960) at ../../../kern/kern_mutex.c:791 #9 0xffffffff805c2099 in knote (list=0xfffff802ffcadae0, hint=0, lockflags=1) at ../../../kern/kern_event.c:1822 #10 0xffffffff806510fe in pipeclose (cpipe=0xfffff802ffcada18) at ../../../kern/sys_pipe.c:1655 #11 0xffffffff80651019 in pipe_dtor (dpipe=) at ../../../kern/sys_pipe.c:395 #12 0xffffffff80559924 in fifo_close (ap=) at ../../../fs/fifofs/fifo_vnops.c:115 #13 0xffffffff80a6ba6a in VOP_CLOSE_APV (vop=, a=) at vnode_if.c:535 #14 0xffffffff806acc09 in vn_close (vp=0xfffff80386042588, flags=6, file_cred=0xfffff801b3fa9900, td=0xfffff8010b76d490) at vnode_if.h:225 #15 0xffffffff806abad8 in vn_closefile (fp=0xfffff8034f9757d0, td=0xfffff8010b76d490) at ../../../kern/vfs_vnops.c:1481 #16 0xffffffff805b8789 in _fdrop (fp=0xfffff8034f9757d0, td=0x4) at file.h:342 #17 0xffffffff805bb0e1 in closef (fp=0xfffff8034f9757d0, td=0xfffff8010b76d490) at ../../../kern/kern_descrip.c:2415 #18 0xffffffff805b8bf0 in closefp (fdp=0xfffff80099c4a000, fd=, fp=0xfffff8034f9757d0, td=0xfffff8010b76d490, holdleaders=) at ../../../kern/kern_descrip.c:1257 #19 0xffffffff80a332b5 in ia32_syscall (frame=0xfffffe048c967c00) at subr_syscall.c:135 #20 0xffffffff8095cdc5 in Xint0x80_syscall () at ia32_exception.S:73 #21 0x00000000210f7804 in ?? () (kgdb) f 12 #12 0xffffffff80559924 in fifo_close (ap=) at ../../../fs/fifofs/fifo_vnops.c:115 115 ../../../fs/fifofs/fifo_vnops.c: No such file or directory. in ../../../fs/fifofs/fifo_vnops.c (kgdb) l 110 in ../../../fs/fifofs/fifo_vnops.c (kgdb) p *vp $1 = {v_tag = 0xffffffff80c96b7e "ufs", v_op = 0xffffffff8111e470, v_data = 0xfffff803311f65e8, v_mount = 0xfffff80011311000, v_nmntvnodes = { tqe_next = 0xfffff80333118ce8, tqe_prev = 0xfffff801a6e39958}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0}, v_hashlist = { le_next = 0x0, le_prev = 0xfffffe0001294a38}, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0xfffff80355081230, tqh_last = 0xfffff80355081250}, v_cache_dd = 0x0, v_lock = {lock_object = {lo_name = 0xffffffff80c96b7e "ufs", lo_flags = 117112840, lo_data = 0, lo_witness = 0x0}, lk_lock = 18446735282103833744, lk_exslpfail = 0, lk_timo = 21, lk_pri = 96}, v_interlock = {lock_object = { lo_name = 0xffffffff80cb7fa9 "vnode interlock", lo_flags = 16973824, lo_data = 0, lo_witness = 0x0}, mtx_lock = 4}, v_vnlock = 0xfffff803860425f0, v_actfreelist = {tqe_next = 0xfffff802b894e3b0, tqe_prev = 0xfffff801fe2dada8}, v_bufobj = {bo_lock = {lock_object = { lo_name = 0xffffffff80cb7fb9 "bufobj interlock", lo_flags = 86179840, lo_data = 0, lo_witness = 0x0}, rw_lock = 1}, bo_ops = 0xffffffff81105f80, bo_object = 0x0, bo_synclist = {le_next = 0x0, le_prev = 0x0}, bo_private = 0xfffff80386042588, __bo_vnode = 0xfffff80386042588, bo_clean = {bv_hd = { tqh_first = 0x0, tqh_last = 0xfffff803860426a8}, bv_root = {pt_root = 0}, bv_cnt = 0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xfffff803860426c8}, bv_root = {pt_root = 0}, bv_cnt = 0}, bo_numoutput = 0, bo_flag = 0, bo_bsize = 16384}, v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0, v_rl = {rl_waiters = {tqh_first = 0x0, tqh_last = 0xfffff80386042710}, rl_currdep = 0x0}, v_cstart = 0, v_lasta = 0, v_lastw = 0, v_clen = 0, v_holdcnt = 1, v_usecount = 1, v_iflag = 512, v_vflag = 0, v_writecount = 0, v_hash = 57577702, v_type = VFIFO} (kgdb) p *fip $2 = {fi_pipe = 0xffffffff80c96b7e, fi_readers = -2129533840, fi_writers = -8782383979032} (kgdb) f 11 #11 0xffffffff80651019 in pipe_dtor (dpipe=) at ../../../kern/sys_pipe.c:395 395 ../../../kern/sys_pipe.c: No such file or directory. in ../../../kern/sys_pipe.c (kgdb) l 390 in ../../../kern/sys_pipe.c (kgdb) p *dpipe $3 = {pipe_buffer = {cnt = 2160683902, in = 4294967295, out = 2165433456, size = 4294967295, buffer = 0xfffff803311f65e8 ""}, pipe_map = { cnt = 18446735277904957440, pos = 18446735291358219496, npages = -1495033512, ms = {0x0, 0x0, 0xfffffe0001294a38, 0x0, 0xfffff80355081230, 0xfffff80355081250, 0x0, 0xffffffff80c96b7e, 0x6fb0008, 0x0, 0xfffff8010b76d490, 0x1500000000, 0x60, 0xffffffff80cb7fa9, 0x1030000, 0x0, 0x4}}, pipe_sel = {si_tdlist = {tqh_first = 0xfffff803860425f0, tqh_last = 0xfffff802b894e3b0}, si_note = {kl_list = {slh_first = 0xfffff801fe2dada8}, kl_lock = 0xffffffff80cb7fb9 , kl_unlock = 0x5230000, kl_assert_locked = 0, kl_assert_unlocked = 0x1, kl_lockarg = 0xffffffff81105f80}, si_mtx = 0x0}, pipe_atime = {tv_sec = 0, tv_nsec = 0}, pipe_mtime = {tv_sec = -8780959701624, tv_nsec = -8780959701624}, pipe_ctime = {tv_sec = 0, tv_nsec = -8780959701336}, pipe_sigio = 0x0, pipe_peer = 0x0, pipe_pair = 0x0, pipe_state = 2248419016, pipe_busy = -2045, pipe_present = 0, pipe_wgen = 0, pipe_ino = 0} (kgdb) f 10 #10 0xffffffff806510fe in pipeclose (cpipe=0xfffff802ffcada18) at ../../../kern/sys_pipe.c:1655 1655 in ../../../kern/sys_pipe.c (kgdb) l 1650 in ../../../kern/sys_pipe.c (kgdb) p *cpipe $4 = {pipe_buffer = {cnt = 0, in = 0, out = 0, size = 0, buffer = 0x0}, pipe_map = {cnt = 0, pos = 0, npages = 0, ms = {0x0 }}, pipe_sel = { si_tdlist = {tqh_first = 0x0, tqh_last = 0x0}, si_note = {kl_list = {slh_first = 0x0}, kl_lock = 0, kl_unlock = 0, kl_assert_locked = 0xffffffff805c25c0 , kl_assert_unlocked = 0xffffffff805c25e0 , kl_lockarg = 0x0}, si_mtx = 0x0}, pipe_atime = {tv_sec = 1399583015, tv_nsec = 0}, pipe_mtime = {tv_sec = 1399583015, tv_nsec = 0}, pipe_ctime = { tv_sec = 1399583015, tv_nsec = 0}, pipe_sigio = 0x0, pipe_peer = 0xfffff802ffcad8b8, pipe_pair = 0xfffff802ffcad8b8, pipe_state = 2432, pipe_busy = 0, pipe_present = 3, pipe_wgen = 0, pipe_ino = 4294967295} (kgdb) p *ppipe $5 = {pipe_buffer = {cnt = 0, in = 0, out = 0, size = 0, buffer = 0x0}, pipe_map = {cnt = 0, pos = 0, npages = 0, ms = {0x0 }}, pipe_sel = { si_tdlist = {tqh_first = 0x0, tqh_last = 0x0}, si_note = {kl_list = {slh_first = 0x0}, kl_lock = 0, kl_unlock = 0, kl_assert_locked = 0xffffffff805c25c0 , kl_assert_unlocked = 0xffffffff805c25e0 , kl_lockarg = 0x0}, si_mtx = 0x0}, pipe_atime = {tv_sec = 1399583015, tv_nsec = 0}, pipe_mtime = {tv_sec = 1399583015, tv_nsec = 0}, pipe_ctime = { tv_sec = 1399583015, tv_nsec = 0}, pipe_sigio = 0x0, pipe_peer = 0xfffff802ffcad8b8, pipe_pair = 0xfffff802ffcad8b8, pipe_state = 2432, pipe_busy = 0, pipe_present = 3, pipe_wgen = 0, pipe_ino = 4294967295} (kgdb) q -bash-4.2$ -bash-4.2$ # time for some science (kgdb drama) -bash-4.2$ -bash-4.2$ sudo kgdb /boot/kernel/kernel.YAHOO /var/crash/vmcore.0 Password: GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd"... Unread portion of the kernel message buffer: processor eflags = interrupt enabled, resume, IOPL = 0 current process = 59996 (qmail-send) trap number = 12 panic: page fault cpuid = 5 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe048c9672a0 kdb_backtrace() at kdb_backtrace+0x39/frame 0xfffffe048c967350 vpanic() at vpanic+0x126/frame 0xfffffe048c967390 panic() at panic+0x43/frame 0xfffffe048c9673f0 trap_fatal() at trap_fatal+0x3a2/frame 0xfffffe048c967450 trap_pfault() at trap_pfault+0x22f/frame 0xfffffe048c9674f0 trap() at trap+0x670/frame 0xfffffe048c967710 calltrap() at calltrap+0x8/frame 0xfffffe048c967710 --- trap 0xc, rip = 0xffffffff805e759b, rsp = 0xfffffe048c9677d0, rbp = 0xfffffe048c9677e0 --- __mtx_assert() at __mtx_assert+0x3b/frame 0xfffffe048c9677e0 knote() at knote+0x39/frame 0xfffffe048c967830 pipeclose() at pipeclose+0xbe/frame 0xfffffe048c967860 pipe_dtor() at pipe_dtor+0x49/frame 0xfffffe048c967880 fifo_close() at fifo_close+0x144/frame 0xfffffe048c9678c0 VOP_CLOSE_APV() at VOP_CLOSE_APV+0xea/frame 0xfffffe048c9678f0 vn_close() at vn_close+0x139/frame 0xfffffe048c967960 vn_closefile() at vn_closefile+0x48/frame 0xfffffe048c9679e0 _fdrop() at _fdrop+0x29/frame 0xfffffe048c967a00 closef() at closef+0x1d1/frame 0xfffffe048c967a90 closefp() at closefp+0xa0/frame 0xfffffe048c967ae0 ia32_syscall() at ia32_syscall+0x255/frame 0xfffffe048c967bf0 Xint0x80_syscall() at Xint0x80_syscall+0x95/frame 0xfffffe048c967bf0 --- syscall (6, FreeBSD ELF32, sys_close), rip = 0x210f7804, rsp = 0xffffcae4, rbp = 0xffffcb00 --- Uptime: 30d22h18m58s Dumping 2251 out of 16358 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91% Reading symbols from /boot/modules/dscp_rewrite.ko...done. Loaded symbols for /boot/modules/dscp_rewrite.ko Reading symbols from /boot/modules/markdev_mod.ko...done. Loaded symbols for /boot/modules/markdev_mod.ko Reading symbols from /boot/modules/ylock_mod.ko...done. Loaded symbols for /boot/modules/ylock_mod.ko #0 doadump (textdump=1) at pcpu.h:219 219 pcpu.h: No such file or directory. in pcpu.h (kgdb) f 12 #12 0xffffffff80559924 in fifo_close (ap=) at ../../../fs/fifofs/fifo_vnops.c:115 115 ../../../fs/fifofs/fifo_vnops.c: No such file or directory. in ../../../fs/fifofs/fifo_vnops.c Current language: auto; currently minimal (kgdb) p *fip Cannot access memory at address 0xa00000038 (kgdb) p *vp $1 = {v_tag = 0xffffffff80c96b7e "ufs", v_op = 0xffffffff8111e470, v_data = 0xfffff803311f65e8, v_mount = 0xfffff80011311000, v_nmntvnodes = { tqe_next = 0xfffff80333118ce8, tqe_prev = 0xfffff801a6e39958}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0}, v_hashlist = { le_next = 0x0, le_prev = 0xfffffe0001294a38}, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0xfffff80355081230, tqh_last = 0xfffff80355081250}, v_cache_dd = 0x0, v_lock = {lock_object = {lo_name = 0xffffffff80c96b7e "ufs", lo_flags = 117112840, lo_data = 0, lo_witness = 0x0}, lk_lock = 18446735282103833744, lk_exslpfail = 0, lk_timo = 21, lk_pri = 96}, v_interlock = {lock_object = { lo_name = 0xffffffff80cb7fa9 "vnode interlock", lo_flags = 16973824, lo_data = 0, lo_witness = 0x0}, mtx_lock = 4}, v_vnlock = 0xfffff803860425f0, v_actfreelist = {tqe_next = 0xfffff802b894e3b0, tqe_prev = 0xfffff801fe2dada8}, v_bufobj = {bo_lock = {lock_object = { lo_name = 0xffffffff80cb7fb9 "bufobj interlock", lo_flags = 86179840, lo_data = 0, lo_witness = 0x0}, rw_lock = 1}, bo_ops = 0xffffffff81105f80, bo_object = 0x0, bo_synclist = {le_next = 0x0, le_prev = 0x0}, bo_private = 0xfffff80386042588, __bo_vnode = 0xfffff80386042588, bo_clean = {bv_hd = { tqh_first = 0x0, tqh_last = 0xfffff803860426a8}, bv_root = {pt_root = 0}, bv_cnt = 0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xfffff803860426c8}, bv_root = {pt_root = 0}, bv_cnt = 0}, bo_numoutput = 0, bo_flag = 0, bo_bsize = 16384}, v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0, v_rl = {rl_waiters = {tqh_first = 0x0, tqh_last = 0xfffff80386042710}, rl_currdep = 0x0}, v_cstart = 0, v_lasta = 0, v_lastw = 0, v_clen = 0, v_holdcnt = 1, v_usecount = 1, v_iflag = 512, v_vflag = 0, v_writecount = 0, v_hash = 57577702, v_type = VFIFO} (kgdb) p *fip $2 = {fi_pipe = 0xffffffff80c96b7e, fi_readers = -2129533840, fi_writers = -8782383979032} (kgdb)