GENERIC HEAD from 2008-11-26 13:44:11 UTC, r185336M, no vmcore GDB: no debug ports present KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2008 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 8.0-CURRENT #1 r185336M: Thu Nov 27 09:53:19 CET 2008 pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0xf24 Stepping = 4 Features=0x3febfbff Logical CPUs per core: 2 real memory = 1073676288 (1023 MB) avail memory = 1032540160 (984 MB) ACPI APIC Table: FreeBSD/SMP: Multiprocessor System Detected: 4 CPUs cpu0 (BSP): APIC ID: 0 cpu1 (AP): APIC ID: 1 cpu2 (AP): APIC ID: 6 cpu3 (AP): APIC ID: 7 ioapic0 irqs 0-23 on motherboard : Trying to mount root from ufs:/dev/ad0s1a WARNING: / was not properly dismounted lock order reversal: 1st 0xc456a044 user map (user map) @ vm/vm_map.c:3115 2nd 0xc488e9c0 ufs (ufs) @ kern/vfs_subr.c:2079 KDB: stack backtrace: db_trace_self_wrapper(c0bd386d,c41c4914,c0842b45,4,c0bcee72,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bcee72,c451f728,c45242b8,c41c4970,...) at kdb_backtrace+0x29 _witness_debugger(c0bd64c8,c488e9c0,c0bca359,c45242b8,c0bdd21a,...) at _witness_debugger+0x25 witness_checkorder(c488e9c0,1,c0bdd211,81f,0,...) at witness_checkorder+0x839 __lockmgr_args(c488e9c0,200501,c488ea28,0,0,...) at __lockmgr_args+0x237 ffs_lock(c41c4a78,c0bf90bf,c0bc9a86,200501,c488e968,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0cd0c60,c41c4a78,c0cec660,c488e968,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c488e968,200501,c0bdd211,81f,4,...) at _vn_lock+0x78 vget(c488e968,200501,c4566d80,4b4,0,...) at vget+0xc9 vnode_pager_lock(c1876d14,0,c0bf66d0,127,c41c4c18,...) at vnode_pager_lock+0x1e3 vm_fault(c456a000,80db000,2,8,80db160,...) at vm_fault+0x1df trap_pfault(5,0,c0c064bc,2e7,c4564d0c,...) at trap_pfault+0x118 trap(c41c4d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80480e5, esp = 0xbfbfeef0, ebp = 0xbfbfef10 --- fxp0: link state changed to UP lock order reversal: 1st 0xd8510b0c bufwait (bufwait) @ kern/vfs_bio.c:2443 2nd 0xc49fee00 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:263 KDB: stack backtrace: db_trace_self_wrapper(c0bd386d,e6bcc778,c0842b45,4,c0bcee72,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bcee72,c45216d8,c4524320,e6bcc7d4,...) at kdb_backtrace+0x29 _witness_debugger(c0bd64c8,c49fee00,c0bf51cf,c4524320,c0bf4e75,...) at _witness_debugger+0x25 witness_checkorder(c49fee00,9,c0bf4e6c,107,0,...) at witness_checkorder+0x839 _sx_xlock(c49fee00,0,c0bf4e6c,107,c4c062d0,...) at _sx_xlock+0x85 ufsdirhash_acquire(d8510aac,e6bcc8ec,8c,d8daad84,e6bcc8a4,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(c4c062d0,e6bcc8ec,d84,e6bcc890,e6bcc894,...) at ufsdirhash_add+0x13 ufs_direnter(c4bebd70,c4c0cac0,e6bcc8ec,e6bccbd4,0,...) at ufs_direnter+0x779 ufs_makeinode(e6bccbd4,c0cd1180,e6bccacc,e6bcca34,c0b2fe45,...) at ufs_makeinode+0x584 ufs_create(e6bccacc,c0c0903c,0,e6bccacc,e6bccba8,...) at ufs_create+0x30 VOP_CREATE_APV(c0cd0c60,e6bccacc,2,c0bc9a86,3,...) at VOP_CREATE_APV+0xc5 vn_open_cred(e6bccba8,e6bccc5c,180,c456b400,c4be62d8,...) at vn_open_cred+0x1d0 vn_open(e6bccba8,e6bccc5c,180,c4be62d8,246,...) at vn_open+0x33 kern_openat(c4a936c0,ffffff9c,28300000,0,602,...) at kern_openat+0x110 kern_open(c4a936c0,28300000,0,601,180,...) at kern_open+0x35 open(c4a936c0,e6bcccf8,c,c0bef8d9,c0cad678,...) at open+0x30 syscall(e6bccd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x281792b3, esp = 0xbfbfe32c, ebp = 0xbfbfee48 --- lock order reversal: 1st 0xc6544dc8 ufs (ufs) @ kern/vfs_lookup.c:442 2nd 0xd8683598 bufwait (bufwait) @ ufs/ffs/ffs_softdep.c:6150 3rd 0xc5af1868 ufs (ufs) @ kern/vfs_subr.c:2079 KDB: stack backtrace: db_trace_self_wrapper(c0bd386d,e6cad160,c0842b45,4,c0bcee72,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bcee72,c45216d8,c45242b8,e6cad1bc,...) at kdb_backtrace+0x29 _witness_debugger(c0bd64e1,c5af1868,c0bca359,c45242b8,c0bdd21a,...) at _witness_debugger+0x25 witness_checkorder(c5af1868,9,c0bdd211,81f,0,...) at witness_checkorder+0x839 __lockmgr_args(c5af1868,80100,c5af18d0,0,0,...) at __lockmgr_args+0x7a7 ffs_lock(e6cad2c4,c0bdc805,c0bc9a86,80100,c5af1810,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0cd0c60,e6cad2c4,c0cec660,c5af1810,80100,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c5af1810,80100,c0bdd211,81f,4,...) at _vn_lock+0x78 vget(c5af1810,80100,c4d4d6c0,50,0,...) at vget+0xc9 vfs_hash_get(c4b412cc,c299,80000,c4d4d6c0,e6cad420,...) at vfs_hash_get+0xed ffs_vgetf(c4b412cc,c299,80000,e6cad420,1,...) at ffs_vgetf+0x49 softdep_sync_metadata(c6544d70,0,c0bf4b2a,131,0,...) at softdep_sync_metadata+0x5ba ffs_syncvnode(c6544d70,1,c07f5534,c0d33d5c,4,...) at ffs_syncvnode+0x3e2 ffs_truncate(c6544d70,400,0,880,c456b400,...) at ffs_truncate+0x616 ufs_direnter(c6544d70,c5af1810,e6cad790,e6cada90,d85ea6f8,...) at ufs_direnter+0x946 ufs_mkdir(e6cadab4,c0c08e76,e6cad9fc,e6cada64,e6cadab4,...) at ufs_mkdir+0x977 VOP_MKDIR_APV(c0cd0c60,e6cadab4,e6cadae0,8,0,...) at VOP_MKDIR_APV+0xc5 nfsrv_mkdir(e6cadbb4,0,e6cadbac,0,c4a09800,...) at nfsrv_mkdir+0x60f nfssvc_program(c4a09800,c60de600,c0bf0f90,488,e6cadcdc,...) at nfssvc_program+0x379 svc_run_internal(e6cadd24,c07e2718,c48cf780,e6cadd38,c0bcc892,...) at svc_run_internal+0x93f svc_thread_start(c48cf780,e6cadd38,c0bcc892,327,c4a9229c,...) at svc_thread_start+0x10 fork_exit(c09d8130,c48cf780,e6cadd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0xc, esp = 0x33, ebp = 0 --- Kernel page fault with the following non-sleepable locks held: exclusive sleep mutex sp_lock (sp_lock) r = 0 (0xc48cf780) locked @ rpc/svc.c:1090 KDB: stack backtrace: db_trace_self_wrapper(c0bd386d,e6cb0b24,c0842b45,c0bf0f99,442,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0bf0f99,442,ffffffff,c0e60544,e6cb0b5c,...) at kdb_backtrace+0x29 _witness_debugger(c0bd5a93,e6cb0b70,4,1,0,...) at _witness_debugger+0x25 witness_warn(5,0,c0c064bc,c0bf5f09,c4a9229c,...) at witness_warn+0x1fd trap(e6cb0bfc) at trap+0x152 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0xc09d607b, esp = 0xe6cb0c3c, ebp = 0xe6cb0c40 --- xprt_assignthread(c48cf780,0,c0bf0f90,442,e6cb0cdc,...) at xprt_assignthread+0xb svc_run_internal(e6cb0d24,c07e2718,c48cf780,e6cb0d38,c0bcc892,...) at svc_run_internal+0x7a8 svc_thread_start(c48cf780,e6cb0d38,c0bcc892,327,c4a9229c,...) at svc_thread_start+0x10 fork_exit(c09d8130,c48cf780,e6cb0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0xc, esp = 0x33, ebp = 0 --- Fatal trap 12: page fault while in kernel mode cpuid = 2; apic id = 06 fault virtual address = 0xdeadc11a fault code = supervisor read, page not present instruction pointer = 0x20:0xc09d607b stack pointer = 0x28:0xe6cb0c3c frame pointer = 0x28:0xe6cb0c40 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 913 (nfsd: service) [thread pid 913 tid 100102 ] Stopped at xprt_assignthread+0xb: movl 0x3c(%edx),%eax db> lock order reversal: (Giant after non-sleepable) 1st 0xc48cf780 sp_lock (sp_lock) @ rpc/svc.c:1090 2nd 0xc0d201b0 Giant (Giant) @ dev/kbdmux/kbdmux.c:1044 KDB: stack backtrace: db_trace_self_wrapper(c0bd386d,e6cb096c,c0842b45,4,c0bcee72,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bcee72,c4523348,c451e1a0,e6cb09c8,...) at kdb_backtrace+0x29 _witness_debugger(c0bd64c8,c0d201b0,c0befdde,c451e1a0,c0baa3fe,...) at _witness_debugger+0x25 witness_checkorder(c0d201b0,9,c0baa3f5,414,0,...) at witness_checkorder+0x839 _mtx_lock_flags(c0d201b0,0,c0baa3f5,414,765,...) at _mtx_lock_flags+0xc4 kbdmux_ioctl(c46f6a00,20044b07,c0d1bec4,c0d1ebc8,1,...) at kbdmux_ioctl+0x598 sc_cngetc(c0c69fe0,78,e6cb0a68,c07cec76,e6cb0a88,...) at sc_cngetc+0xa5 cncheckc(e6cb0a88,c04bdd25,c0b81d02,c04befd0,e6cb0a84,...) at cncheckc+0x48 cngetc(c0b81d02,c04befd0,e6cb0a84,e6cb0ac0,1,...) at cngetc+0x16 db_readline(c0cee640,78,e6cb0aa4,c04bc9a6,c0b81d02,...) at db_readline+0x75 db_read_line(c0b81d02,e6cb0af8,c04be81d,c0c02b55,d31206e,...) at db_read_line+0x1a db_command_loop(c0c02b55,d31206e,2c66000a,3,0,...) at db_command_loop+0x46 db_trap(c,0,b,28,e6cb0bfc,...) at db_trap+0xdd kdb_trap(c,0,e6cb0bfc,1,1,...) at kdb_trap+0x96 trap_fatal(5,0,c0c064bc,c0bf5f09,c4a9229c,...) at trap_fatal+0x2ef trap(e6cb0bfc) at trap+0x160 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0xc09d607b, esp = 0xe6cb0c3c, ebp = 0xe6cb0c40 --- xprt_assignthread(c48cf780,0,c0bf0f90,442,e6cb0cdc,...) at xprt_assignthread+0xb svc_run_internal(e6cb0d24,c07e2718,c48cf780,e6cb0d38,c0bcc892,...) at svc_run_internal+0x7a8 svc_thread_start(c48cf780,e6cb0d38,c0bcc892,327,c4a9229c,...) at svc_thread_start+0x10 fork_exit(c09d8130,c48cf780,e6cb0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0xc, esp = 0x33, ebp = 0 --- ~~ (kgdb) l *0xc09d607b 0xc09d607b is in xprt_assignthread (../../../rpc/svc.c:327). 322 323 /* 324 * Attempt to assign a service thread to this 325 * transport. 326 */ 327 LIST_FOREACH(st, &pool->sp_idlethreads, st_ilink) { 328 if (st->st_xprt == NULL && STAILQ_EMPTY(&st->st_reqs)) 329 break; 330 } 331 if (st) { (kgdb) -- Test scenario: lockf.sh