GENERIC HEAD from Oct 6 15:25 UTC + ufs_lookup patch, vmcore.288 GDB: no debug ports present KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2008 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 8.0-CURRENT #0: Mon Oct 6 17:51:19 CEST 2008 pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.81-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0xf24 Stepping = 4 Features=0x3febfbff Logical CPUs per core: 2 real memory = 1073676288 (1023 MB) avail memory = 1036734464 (988 MB) : Trying to mount root from ufs:/dev/ad0s1a lock order reversal: 1st 0xc4619000 vfslock (vfslock) @ kern/vfs_subr.c:372 2nd 0xc4621308 devfs (devfs) @ kern/vfs_lookup.c:428 3rd 0xc46189b4 vfslock (vfslock) @ kern/vfs_subr.c:372 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,c3dc4824,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c46189b4,c4122fe0,c3dc487c,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaad1,c46189b4,c0bd1323,c4122fe0,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(c46189b4,1,c0bd18c8,174,0,...) at witness_checkorder+0x800 __lockmgr_args(c46189b4,200100,c4618a1c,0,0,...) at __lockmgr_args+0x237 vfs_busy(c46189b4,200,0,c4621370,c0cc18e0,...) at vfs_busy+0x1d3 vfs_mount_alloc(c46212b0,c0cc18e0,c4622490,c4169400,0,...) at vfs_mount_alloc+0x86 vfs_donmount(c4164d20,4001,c3dc4b50,c461e480,6,...) at vfs_donmount+0x127b kernel_mount(c46224f0,4001,c4628080,ffffffff,c3dc4bc0) at kernel_mount+0x84 kernel_vmount(4001,c0bd1512,c4622500,c0bd1521,c0bbe8c2,...) at kernel_vmount+0x63 vfs_mountroot_try(c0bd1824,c0bbe8c2,c0bb6a40,1,c0878ad0,...) at vfs_mountroot_try+0x132 vfs_mountroot(c0d11030,4,c0bbfbef,265,0,...) at vfs_mountroot+0x423 start_init(0,c3dc4d38,c0bc15d4,322,c4162d0c,...) at start_init+0x65 fork_exit(c07bfd70,0,c3dc4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dc4d70, ebp = 0 --- WARNING: / was not properly dismounted lock order reversal: 1st 0xc43fac70 ufs (ufs) @ kern/vfs_subr.c:2051 2nd 0xc4619000 vfslock (vfslock) @ kern/vfs_subr.c:372 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,c3dc48f4,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0bd18d1,c4122fe0,c3dc494c,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaab8,c4619000,c0bd1323,c4122fe0,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(c4619000,1,c0bd18c8,174,0,...) at witness_checkorder+0x800 __lockmgr_args(c4619000,200100,c4619068,0,0,...) at __lockmgr_args+0x237 vfs_busy(c4619000,0,0,1ac,c3dc4b24,...) at vfs_busy+0x1d3 lookup(c3dc4b0c,c0bd0fd7,d4,bc,c416952c,...) at lookup+0x7d8 namei(c3dc4b0c,c3dc4b24,c083827c,c0e51910,c0bd166a,...) at namei+0x45b kern_unlinkat(c4164d20,ffffff9c,c0bd166a,1,c3dc4c5c,...) at kern_unlinkat+0x46 kern_unlink(c4164d20,c0bd166a,1,631,0,...) at kern_unlink+0x27 vfs_mountroot_try(c0bd1824,c0bbe8c2,c0bb6a40,1,c0878ad0,...) at vfs_mountroot_try+0x4a5 vfs_mountroot(c0d11030,4,c0bbfbef,265,0,...) at vfs_mountroot+0x423 start_init(0,c3dc4d38,c0bc15d4,322,c4162d0c,...) at start_init+0x65 fork_exit(c07bfd70,0,c3dc4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dc4d70, ebp = 0 --- lock order reversal: 1st 0xc4168044 user map (user map) @ vm/vm_map.c:3115 2nd 0xc43fa9c0 ufs (ufs) @ kern/vfs_subr.c:2051 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,c3dc4914,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0bd18d1,c41231e8,c3dc496c,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaab8,c43fa9c0,c0bbf193,c41231e8,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(c43fa9c0,1,c0bd18c8,803,0,...) at witness_checkorder+0x800 __lockmgr_args(c43fa9c0,200501,c43faa28,0,0,...) at __lockmgr_args+0x237 ffs_lock(c3dc4a78,c0beceb2,c0bbe8c0,200501,c43fa968,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,c3dc4a78,c0cdd660,c43fa968,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c43fa968,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c43fa968,200501,c4164d20,4b4,0,...) at vget+0xc9 vnode_pager_lock(c1476d14,0,c0bea4a7,127,c3dc4c18,...) at vnode_pager_lock+0x1e6 vm_fault(c4168000,80db000,2,8,80db140,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c4162d0c,...) at trap_pfault+0x118 trap(c3dc4d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80480e5, esp = 0xbfbfeef0, ebp = 0xbfbfef10 --- fxp0: link state changed to UP lock order reversal: 1st 0xc4729710 syncer (syncer) @ kern/vfs_subr.c:1659 2nd 0xc472c000 vfslock (vfslock) @ kern/vfs_subr.c:372 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,e4603aec,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0bd18d1,c4122fe0,e4603b44,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaab8,c472c000,c0bd1323,c4122fe0,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(c472c000,1,c0bd18c8,174,0,...) at witness_checkorder+0x800 __lockmgr_args(c472c000,200100,c472c068,0,0,...) at __lockmgr_args+0x237 vfs_busy(c472c000,80200,c0e7de5c,d1c,c472c000,...) at vfs_busy+0x1d3 sync_fsync(e4603c7c,c0bfbb8f,c4729778,e4603c7c,c47296b8,...) at sync_fsync+0xb5 VOP_FSYNC_APV(c0caf360,e4603c7c,c0bd18c8,67b,c4337d20,...) at VOP_FSYNC_APV+0xc5 sync_vnode(c0e7df50,c0e7df3c,3e8,6aa,4e20,...) at sync_vnode+0x16b sched_sync(0,e4603d38,c0bc15d4,322,c45a6538,...) at sched_sync+0x273 fork_exit(c0882f10,0,e4603d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4603d70, ebp = 0 --- lock order reversal: 1st 0xd8110e44 bufwait (bufwait) @ kern/vfs_bio.c:2443 2nd 0xc45ca800 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:254 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,e68b577c,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0be8c4c,c4123250,e68b57d4,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaab8,c45ca800,c0be8fa6,c4123250,c0be8c4c,...) at _witness_debugger+0x25 witness_checkorder(c45ca800,9,c0be8c43,fe,0,...) at witness_checkorder+0x800 _sx_xlock(c45ca800,0,c0be8c43,fe,c473fa50,...) at _sx_xlock+0x85 ufsdirhash_acquire(d8110de4,e68b58ec,11c,d89b58f4,e68b58a4,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(c473fa50,e68b58ec,8f4,e68b5890,e68b5894,...) at ufsdirhash_add+0x13 ufs_direnter(c4790000,c47ae000,e68b58ec,e68b5bd4,0,...) at ufs_direnter+0x7a1 ufs_makeinode(e68b5bd4,c0cc21c0,e68b5acc,e68b5a34,c0b25c25,...) at ufs_makeinode+0x58a ufs_create(e68b5acc,c0bfc55d,0,e68b5acc,e68b5ba8,...) at ufs_create+0x30 VOP_CREATE_APV(c0cc1ca0,e68b5acc,2,c0bbe8c0,3,...) at VOP_CREATE_APV+0xc5 vn_open_cred(e68b5ba8,e68b5c5c,180,c4169400,c4699d90,...) at vn_open_cred+0x1d0 vn_open(e68b5ba8,e68b5c5c,180,c4699d90,246,...) at vn_open+0x33 kern_openat(c475d000,ffffff9c,8200000,0,602,...) at kern_openat+0x106 kern_open(c475d000,8200000,0,601,180,...) at kern_open+0x35 open(c475d000,e68b5cf8,c,c0be3d12,c0c9e738,...) at open+0x30 syscall(e68b5d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x28178e73, esp = 0xbfbfe32c, ebp = 0xbfbfee48 --- lock order reversal: 1st 0xc4732b18 pseudofs (pseudofs) @ kern/vfs_subr.c:2051 2nd 0xc4618678 vfslock (vfslock) @ kern/vfs_subr.c:372 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,e6808964,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0bd18d1,c4122fe0,e68089bc,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaab8,c4618678,c0bd1323,c4122fe0,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(c4618678,1,c0bd18c8,174,0,...) at witness_checkorder+0x800 __lockmgr_args(c4618678,200100,c46186e0,0,0,...) at __lockmgr_args+0x237 vfs_busy(c4618678,200,0,399,246,...) at vfs_busy+0x1d3 vfs_donmount(c46798c0,11000,c47b8a80,c47b8a80,811b040,...) at vfs_donmount+0xd77 nmount(c46798c0,e6808cf8,c,c0bcbba3,c0ca0a30,...) at nmount+0xbe syscall(e6808d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (378, FreeBSD ELF32, nmount), eip = 0x280eda9b, esp = 0xbfbfe96c, ebp = 0xbfbfedb8 --- lock order reversal: 1st 0xc4f66710 ufs (ufs) @ kern/vfs_lookup.c:428 2nd 0xd8252de0 bufwait (bufwait) @ ufs/ffs/ffs_softdep.c:6150 3rd 0xc47dfdc8 ufs (ufs) @ kern/vfs_subr.c:2051 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,e6d083f0,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c47dfdc8,c41231e8,e6d08448,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaad1,c47dfdc8,c0bbf193,c41231e8,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(c47dfdc8,9,c0bd18c8,803,0,...) at witness_checkorder+0x800 __lockmgr_args(c47dfdc8,80100,c47dfe30,0,0,...) at __lockmgr_args+0x7a7 ffs_lock(e6d08554,c0bd0ec8,c0bbe8c0,80100,c47dfd70,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6d08554,c0cdd660,c47dfd70,80100,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c47dfd70,80100,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c47dfd70,80100,c510b690,50,0,...) at vget+0xc9 vfs_hash_get(c472c33c,bde2,80000,c510b690,e6d086b0,...) at vfs_hash_get+0xed ffs_vgetf(c472c33c,bde2,80000,e6d086b0,1,...) at ffs_vgetf+0x49 softdep_sync_metadata(c4f666b8,0,c0be8930,131,0,...) at softdep_sync_metadata+0x5ba ffs_syncvnode(c4f666b8,1,e6d0874c,c07ebfd4,c0d24b9c,...) at ffs_syncvnode+0x3e2 ffs_truncate(c4f666b8,200,0,880,c4778500,...) at ffs_truncate+0x639 ufs_direnter(c4f666b8,c47dfd70,e6d08a20,e6d08c04,d832c694,...) at ufs_direnter+0x973 ufs_mkdir(e6d08c28,c0bfc397,0,e6d08c28,e6d08bd8,...) at ufs_mkdir+0x997 VOP_MKDIR_APV(c0cc1ca0,e6d08c28,e99,e97,0,...) at VOP_MKDIR_APV+0xc5 kern_mkdirat(c510b690,ffffff9c,804d7e0,0,1f8,...) at kern_mkdirat+0x276 kern_mkdir(c510b690,804d7e0,0,1f8,e6d08d2c,...) at kern_mkdir+0x2e mkdir(c510b690,e6d08cf8,8,c0bcb5ff,c0c9f380,...) at mkdir+0x29 syscall(e6d08d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (136, FreeBSD ELF32, mkdir), eip = 0x28176e33, esp = 0xbfbfe9dc, ebp = 0xbfbfea78 --- kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) acquiring duplicate lock of same type: "dirhash" 1st dirhash @ ufs/ufs/ufs_dirhash.c:185 2nd dirhash @ ufs/ufs/ufs_dirhash.c:204 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,e69b0764,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0be8c4c,c4123250,e69b07bc,...) at kdb_backtrace+0x29 _witness_debugger(c0bca9ab,c0be8fa6,c0be8c4c,cc,3,...) at _witness_debugger+0x25 witness_checkorder(c7d2f400,1,c0be8c43,cc,0,...) at witness_checkorder+0x425 _sx_slock(c7d2f400,0,c0be8c43,cc,e69b0814,...) at _sx_slock+0x85 ufsdirhash_build(c4f61bb8,1200,0,c4a43af0,c0b44b66,...) at ufsdirhash_build+0x1b9 ufs_lookup(e69b0928,c0bfc0a0,c4f666b8,c4f666b8,e69b0bd4,...) at ufs_lookup+0xf3 VOP_CACHEDLOOKUP_APV(c0cc1ca0,e69b0928,e69b0bd4,e69b0bc0,c4778500,...) at VOP_CACHEDLOOKUP_APV+0xc5 vfs_cache_lookup(e69b09a8,c0bd2504,c0cdd2c0,80000,c4f666b8,...) at vfs_cache_lookup+0xcf VOP_LOOKUP_APV(c0cc1ca0,e69b09a8,c0bd0fd7,1ac,e69b0bc0,...) at VOP_LOOKUP_APV+0xe5 lookup(e69b0ba8,c0bd0fd7,d4,bc,c5f0312c,...) at lookup+0x58e namei(e69b0ba8,c08381db,c0bc0bb9,c0bbe8c0,3,...) at namei+0x45b vn_open_cred(e69b0ba8,e69b0c5c,180,c4778500,c4660b60,...) at vn_open_cred+0x2ba vn_open(e69b0ba8,e69b0c5c,180,c4660b60,0,...) at vn_open+0x33 kern_openat(c4a43af0,ffffff9c,804d2a0,0,3,...) at kern_openat+0x106 kern_open(c4a43af0,804d2a0,0,2,180,...) at kern_open+0x35 open(c4a43af0,e69b0cf8,c,e69b0d38,c0c9e738,...) at open+0x30 syscall(e69b0d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x28176e73, esp = 0xbfbfe8ac, ebp = 0xbfbfe8d8 --- kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) kern.ipc.maxpipekva exceeded; see tuning(7) sigreturn: eflags = 0x0 sigreturn: eflags = 0x0 sigreturn: eflags = 0x0 sigreturn: eflags = 0x0 : freebsd4_sigreturn: eflags = 0x0 freebsd4_sigreturn: eflags = 0x0 lock order reversal: 1st 0xc721c710 ufs (ufs) @ kern/vfs_mount.c:1214 2nd 0xca009b18 devfs (devfs) @ kern/vfs_subr.c:2151 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,e6c29a20,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0bd18d1,c4123118,e6c29a78,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaab8,ca009b18,c0bbb5b7,c4123118,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(ca009b18,9,c0bd18c8,867,0,...) at witness_checkorder+0x800 __lockmgr_args(ca009b18,80100,ca009b80,0,0,...) at __lockmgr_args+0x7a7 vop_stdlock(e6c29b7c,c0bc3c47,ca,80100,ca009ac0,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0c9b180,e6c29b7c,c0cdd660,ca009ac0,80100,...) at VOP_LOCK1_APV+0xa5 _vn_lock(ca009ac0,80100,c0bd18c8,867,c0be8467,...) at _vn_lock+0x5e vrele(ca009ac0,c0bd1237,47a,44f,c07caf74,...) at vrele+0x142 ffs_unmount(c4618000,8080000,c5094d20,c5094d20,0,...) at ffs_unmount+0x443 dounmount(c4618000,8080000,c5094d20,489,bf18afe6,...) at dounmount+0x474 unmount(c5094d20,e6c29cf8,8,e6c29d38,c0c9e8d0,...) at unmount+0x2e9 syscall(e6c29d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280dafff, esp = 0xbfbfe1ec, ebp = 0xbfbfe2b8 --- lock order reversal: 1st 0xc54cbd2c filedesc structure (filedesc structure) @ kern/kern_descrip.c:1076 2nd 0xc4e7b1b0 ufs (ufs) @ kern/vfs_subr.c:4028 KDB: stack backtrace: db_trace_self_wrapper(c0bc8233,e6f0aa38,c0838435,4,c0bc3c47,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bc3c47,c0bd18d1,c41231e8,e6f0aa90,...) at kdb_backtrace+0x29 _witness_debugger(c0bcaab8,c4e7b1b0,c0bbf193,c41231e8,c0bd18d1,...) at _witness_debugger+0x25 witness_checkorder(c4e7b1b0,9,c0bd18c8,fbc,c4e7b218,...) at witness_checkorder+0x800 __lockmgr_args(c4e7b1b0,80400,c4e7b218,0,0,...) at __lockmgr_args+0x7a7 ffs_lock(e6f0ab9c,c,c5314964,80400,c4e7b158,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6f0ab9c,c0cdd660,c4e7b158,80400,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4e7b158,80400,c0bd18c8,fbc,e6f0abf8,...) at _vn_lock+0x5e vfs_knllock(c4e7b158,0,c0bc1077,68e,c49e7a5c,...) at vfs_knllock+0x29 knlist_remove_kq(0,e6f0ac18,c087d9f9,c571801c,c49e7a5c,...) at knlist_remove_kq+0xad knlist_remove(c571801c,c49e7a5c,0,e6f0ac44,c07d2a55,...) at knlist_remove+0x1b filt_vfsdetach(c49e7a5c,0,c0bc1077,75e,3,...) at filt_vfsdetach+0x39 knote_fdclose(c53148c0,3,c0bc0bb0,434,c4d17070,...) at knote_fdclose+0xf5 kern_close(c53148c0,3,e6f0ad2c,c0b170c3,c53148c0,...) at kern_close+0xd5 close(c53148c0,e6f0acf8,4,e6f0ad38,c0c9e750,...) at close+0x1a syscall(e6f0ad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (6, FreeBSD ELF32, close), eip = 0x28177a63, esp = 0xbfbfeb6c, ebp = 0xbfbfeb78 --- panic: Lock ufs not locked @ ../../../kern/vfs_default.c:323 cpuid = 1 KDB: enter: panic [thread pid 77717 tid 100444 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> set $lines 0 db> bt Tracing pid 77717 tid 100444 td 0xc5475690 kdb_enter(c0bc50cf,c0bc50cf,c0bc2428,e6e218ac,1,...) at kdb_enter+0x3a panic(c0bc2428,c0bbf193,c0bb4df6,c0bd0d18,143,...) at panic+0x136 _lockmgr_assert(c4729b18,1,c0bd0d18,143,c5475690,...) at _lockmgr_assert+0xb2 __lockmgr_args(c4729b18,100100,c4729b80,0,0,0,c0bd0d18,143) at __lockmgr_args+0xc2a vop_stdunlock(e6e219c8,ca,c4729ac0,c4729ac0,c6455cd8,...) at vop_stdunlock+0x65 VOP_UNLOCK_APV(c0cc1ca0,e6e219c8,c5723d82,25c,2,...) at VOP_UNLOCK_APV+0xb5 null_unlock(e6e21a70,c4729b18,c77daa60,e6e21a20,c5362000,...) at null_unlock+0x102 VOP_UNLOCK_APV(c5724240,e6e21a70,e6e21a80,e6e21a7c,c0bd1237,...) at VOP_UNLOCK_APV+0xb5 nullfs_mount(c5362000,c5475690,c0bd122e,3f0,0,...) at nullfs_mount+0x219 vfs_donmount(c5475690,0,c629bd00,c629bd00,bfbfebac,...) at vfs_donmount+0x1338 nmount(c5475690,e6e21cf8,c,e6e21d38,c0ca0a30,...) at nmount+0xbe syscall(e6e21d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (378, FreeBSD ELF32, nmount), eip = 0x280d8a9b, esp = 0xbfbfe2fc, ebp = 0xbfbfeb68 --- db> show allpcpu Current CPU: 1 cpuid = 0 curthread = 0xc4164000: pid 12 "swi4: clock" curpcb = 0xc3ddad90 fpcurthread = none idlethread = 0xc4164460: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 curthread = 0xc5475690: pid 77717 "mount_nullfs" curpcb = 0xe6e21d90 fpcurthread = none idlethread = 0xc4164690: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 curthread = 0xc41648c0: pid 11 "idle: cpu2" curpcb = 0xc3dcbd90 fpcurthread = none idlethread = 0xc41648c0: pid 11 "idle: cpu2" APIC ID = 6 currentldt = 0x50 spin locks held: cpuid = 3 curthread = 0xc4164af0: pid 11 "idle: cpu3" curpcb = 0xc3dc8d90 fpcurthread = none idlethread = 0xc4164af0: pid 11 "idle: cpu3" APIC ID = 7 currentldt = 0x50 spin locks held: db> show alllocks Process 77737 (grep) thread 0xc51e4000 (100424) exclusive sx user map (user map) r = 0 (0xc50c8ba4) locked @ vm/vm_map.c:2425 Process 77732 (mount_nullfs) thread 0xc510baf0 (100350) exclusive sx user map (user map) r = 0 (0xc467a9e4) locked @ vm/vm_map.c:3115 Process 77730 (mount_nullfs) thread 0xc527ad20 (100392) shared lockmgr ufs (ufs) r = 0 (0xc46215b8) locked @ kern/vfs_subr.c:2051 Process 77727 (grep) thread 0xc4a43d20 (100147) exclusive sx user map (user map) r = 0 (0xc4f823c4) locked @ vm/vm_map.c:3115 Process 77725 (grep) thread 0xc50c5000 (100298) exclusive sx user map (user map) r = 0 (0xc4a482e4) locked @ vm/vm_map.c:3115 Process 77723 (mount) thread 0xc546c690 (100468) exclusive sleep mutex vnode interlock (vnode interlock) r = 0 (0xc4673b80) locked @ kern/vfs_subr.c:2180 Process 77717 (mount_nullfs) thread 0xc5475690 (100444) exclusive sleep mutex vnode interlock (vnode interlock) r = 0 (0xc4729b80) locked @ /usr/src/sys/modules/nullfs/../../fs/nullfs/null_vnops.c:601 shared lockmgr vfslock (vfslock) r = 0 (0xc5362000) locked @ kern/vfs_subr.c:372 exclusive sleep mutex Giant (Giant) r = 0 (0xc0d11030) locked @ kern/vfs_mount.c:734 Process 77716 (mount_nullfs) thread 0xc4faf460 (100189) exclusive lockmgr ufs (ufs) r = 0 (0xc4729b18) locked @ /usr/src/sys/modules/nullfs/../../fs/nullfs/null_vnops.c:546 Process 77715 (mount_nullfs) thread 0xc510d690 (100345) exclusive sx user map (user map) r = 0 (0xc50c6904) locked @ vm/vm_map.c:3115 Process 77712 (mount) thread 0xc529a230 (100467) exclusive sx user map (user map) r = 0 (0xc50c8744) locked @ vm/vm_map.c:3115 Process 77694 (grep) thread 0xc4ff2230 (100226) exclusive sx user map (user map) r = 0 (0xc465be44) locked @ vm/vm_map.c:3115 Process 77690 (mount) thread 0xc4ff1af0 (100229) exclusive sx user map (user map) r = 0 (0xc5085584) locked @ vm/vm_map.c:3115 Process 77679 (mount_nullfs) thread 0xc50368c0 (100259) exclusive sx user map (user map) r = 0 (0xc4fee824) locked @ vm/vm_map.c:3115 Process 3073 (sshd) thread 0xc5286d20 (100400) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc4eb46a4) locked @ kern/uipc_sockbuf.c:148 Process 3072 (sshd) thread 0xc528a460 (100366) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc51b26a4) locked @ kern/uipc_sockbuf.c:148 Process 3069 (sshd) thread 0xc5038000 (100256) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc6bc251c) locked @ kern/uipc_sockbuf.c:148 Process 1258 (sshd) thread 0xc4b30af0 (100141) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc4852394) locked @ kern/uipc_sockbuf.c:148 db> show lockedvnods Locked vnodes 0xc4621560: tag ufs, type VDIR usecount 1, writecount 0, refcount 4 mountedhere 0 flags () v_object 0xc462ab24 ref 0 pages 1 lock type ufs: SHARED (count 1) #0 0xc07e4de2 at __lockmgr_args+0x582 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc088e7fe at _vn_lock+0x5e #4 0xc08827b9 at vget+0xc9 #5 0xc086ff90 at cache_lookup+0x510 #6 0xc08700d6 at vfs_cache_lookup+0xa6 #7 0xc0b25dd5 at VOP_LOOKUP_APV+0xe5 #8 0xc087666e at lookup+0x58e #9 0xc08772bb at namei+0x45b #10 0xc088e50a at vn_open_cred+0x2ba #11 0xc088e793 at vn_open+0x33 #12 0xc088d176 at kern_openat+0x106 #13 0xc088d6a5 at kern_open+0x35 #14 0xc088d6e0 at open+0x30 #15 0xc0b170c3 at syscall+0x2a3 #16 0xc0afb8e0 at Xint0x80_syscall+0x20 ino 5, on dev ad0s1a 0xc4673ac0: tag ufs, type VREG usecount 198, writecount 0, refcount 210 mountedhere 0 flags () v_object 0xc462a7c0 ref 196 pages 254 lock type ufs: SHARED (count 0) with shared waiters pending #0 0xc07e4de2 at __lockmgr_args+0x582 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc088e7fe at _vn_lock+0x5e #4 0xc08827b9 at vget+0xc9 #5 0xc0a35fa6 at vnode_pager_lock+0x1e6 #6 0xc0a1d7df at vm_fault+0x1df #7 0xc0b16d28 at trap_pfault+0x118 #8 0xc0b17629 at trap+0x289 #9 0xc0afb87b at calltrap+0x6 ino 1714, on dev ad0s1a 0xc4729ac0: tag ufs, type VDIR usecount 15, writecount 0, refcount 19 mountedhere 0 flags (VV_ROOT) VI_LOCKed v_object 0xc479dba0 ref 0 pages 4 lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 ino 2, on dev ad0s1e 0xc59b32b0: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc59b32b0, lowervp=0xc4729ac0 0xca009408: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xca009408, lowervp=0xc4729ac0 0xc59b3968: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc59b3968, lowervp=0xc4729ac0 0xc5c18c18: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc5c18c18, lowervp=0xc4729ac0 0xc5cbd6b8: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc5cbd6b8, lowervp=0xc4729ac0 0xc9460d70: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc9460d70, lowervp=0xc4729ac0 0xc7cb1000: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc7cb1000, lowervp=0xc4729ac0 0xc71132b0: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc71132b0, lowervp=0xc4729ac0 0xc9b9ec18: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc9b9ec18, lowervp=0xc4729ac0 0xc70dbc18: tag null, type VDIR usecount 2, writecount 0, refcount 2 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc70dbc18, lowervp=0xc4729ac0 0xc4732560: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc4732560, lowervp=0xc4729ac0 0xc70da968: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc70da968, lowervp=0xc4729ac0 0xc6455c18: tag null, type VDIR usecount 2, writecount 0, refcount 2 mountedhere 0 flags (VV_ROOT|VI_MOUNT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc6455c18, lowervp=0xc4729ac0 0xc5a49000: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc5a49000, lowervp=0xc4729ac0 0xc5c5e968: tag null, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags (VV_ROOT) lock type ufs: EXCL by thread 0xc4faf460 (pid 77716) #0 0xc07e53e0 at __lockmgr_args+0xb80 #1 0xc0a0487a at ffs_lock+0x8a #2 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #3 0xc57232e8 at null_lock+0xb8 #4 0xc0b24c55 at VOP_LOCK1_APV+0xa5 #5 0xc088e7fe at _vn_lock+0x5e #6 0xc5722c32 at nullfs_root+0x42 #7 0xc08769b6 at lookup+0x8d6 #8 0xc08772bb at namei+0x45b #9 0xc0885be4 at kern_statat+0x64 #10 0xc0885e56 at kern_stat+0x36 #11 0xc0885eff at stat+0x2f #12 0xc0b170c3 at syscall+0x2a3 #13 0xc0afb8e0 at Xint0x80_syscall+0x20 vp=0xc5c5e968, lowervp=0xc4729ac0 db> show mount 0xc46189b4 /dev/ad0s1a on / (ufs) 0xc4619000 devfs on /dev (devfs) 0xc472c678 /dev/ad0s1f on /home (ufs) 0xc472c33c /dev/ad0s1e on /tmp (ufs) 0xc472c000 /dev/ad0s1g on /usr (ufs) 0xc46199b4 /dev/ad0s1d on /var (ufs) 0xc4618678 procfs on /proc (procfs) 0xc472c9b4 /tmp on /home/pho/stress2/misc/7 (nullfs) 0xc534233c /tmp on /home/pho/stress2/misc/1 (nullfs) 0xc53649b4 /tmp on /home/pho/stress2/misc/6 (nullfs) 0xc53ce000 /tmp on /home/pho/stress2/misc/15 (nullfs) 0xc53b333c /tmp on /home/pho/stress2/misc/13 (nullfs) 0xc53a333c /tmp on /home/pho/stress2/misc/10 (nullfs) 0xc5351678 /tmp on /home/pho/stress2/misc/5 (nullfs) 0xc53c433c /tmp on /home/pho/stress2/misc/9 (nullfs) 0xc53cc000 /tmp on /home/pho/stress2/misc/11 (nullfs) 0xc53b3678 /tmp on /home/pho/stress2/misc/8 (nullfs) 0xc5342000 /tmp on /home/pho/stress2/misc/12 (nullfs) 0xc535133c /tmp on /home/pho/stress2/misc/3 (nullfs) 0xc537a000 /tmp on /home/pho/stress2/misc/2 (nullfs) 0xc53c4678 /tmp on /home/pho/stress2/misc/4 (nullfs) 0xc536f9b4 /tmp on /home/pho/stress2/misc/14 (nullfs) More info: show mount db> ps pid ppid pgrp uid state wmesg wchan cmd 77738 70600 70531 0 R+ sh 77737 70600 70531 0 R+ grep 77736 70600 70531 0 R+ sh 77735 70608 70531 0 R+ sh 77732 70599 70531 0 RL+ mount_nullfs 77731 70601 70531 0 R+ mount_nullfs 77730 70607 70531 0 R+ mount_nullfs 77729 3083 3080 1001 S nanslp 0xc0d11784 sleep 77727 70604 70531 0 SL+ ufs 0xc4673b18 grep 77725 70604 70531 0 SL+ ufs 0xc4673b18 grep 77723 70604 70531 0 LL+ *vm page 0xc6ebc640 mount 77717 70596 70531 0 R+ CPU 1 mount_nullfs 77716 70602 70531 0 R+ mount_nullfs 77715 70595 70531 0 RL+ mount_nullfs 77714 70609 70531 0 R+ grep 77713 70609 70531 0 S+ piperd 0xc4fe7480 grep 77712 70609 70531 0 SL+ ufs 0xc4673b18 mount 77707 70605 70531 0 R+ mount_nullfs 77695 70603 70531 0 S+ piperd 0xc5057c00 grep 77694 70603 70531 0 RL+ grep 77693 70598 70531 0 S+ piperd 0xc49ef300 grep 77691 70598 70531 0 S+ piperd 0xc4d89480 grep 77690 70598 70531 0 RL+ mount 77679 70597 70531 0 RL+ mount_nullfs 70609 1 70531 0 S+ wait 0xc508d538 sh 70608 1 70531 0 S+ wait 0xc52c9d0c sh 70607 1 70531 0 S+ wait 0xc52b5000 sh 70606 1 70531 0 R+ sh 70605 1 70531 0 S+ wait 0xc52b8d0c sh 70604 1 70531 0 S+ wait 0xc480829c sh 70603 1 70531 0 S+ wait 0xc4aad29c sh 70602 1 70531 0 S+ wait 0xc4839000 sh 70601 1 70531 0 S+ wait 0xc503d7d4 sh 70600 1 70531 0 S+ wait 0xc5225a70 sh 70599 1 70531 0 S+ wait 0xc523c7d4 sh 70598 1 70531 0 S+ wait 0xc483a7d4 sh 70597 1 70531 0 S+ wait 0xc50bad0c sh 70596 1 70531 0 S+ wait 0xc50fe538 sh 70595 1 70531 0 S+ wait 0xc5174a70 sh 70510 70506 70510 0 S+ tty inpu 0xc4651278 bash 70506 70505 70506 0 S+ pause 0xc4e37d64 csh 70505 1261 70505 1001 S+ wait 0xc503b29c su 3084 3080 3080 1001 S piperd 0xc4d8e900 awk 3083 3080 3080 1001 S wait 0xc52c0538 sh 3082 3078 3082 1001 Ss+ select 0xc524f7e4 top 3081 3079 3081 1001 Ss kqread 0xc504c400 tail 3080 3077 3080 1001 Ss wait 0xc52be000 sh 3079 3073 3073 1001 S select 0xc5581024 sshd 3078 3072 3072 1001 S select 0xc5277e64 sshd 3077 3069 3069 1001 S select 0xc513bc64 sshd 3073 1099 3073 0 Ss sbwait 0xc4eb46d0 sshd 3072 1099 3072 0 Ss sbwait 0xc51b26d0 sshd 3069 1099 3069 0 Ss sbwait 0xc6bc2548 sshd 1261 1260 1261 1001 Ss+ wait 0xc48087d4 bash 1260 1258 1258 1001 S select 0xc47c8324 sshd 1258 1099 1258 0 Ss sbwait 0xc48523c0 sshd 1233 1 1233 0 Ss+ tty inpu 0xc4361a78 getty 1232 1 1232 0 Ss+ tty inpu 0xc4361c78 getty 1231 1 1231 0 Ss+ tty inpu 0xc4361e78 getty 1230 1 1230 0 Ss+ tty inpu 0xc43f2078 getty 1229 1 1229 0 Ss+ tty inpu 0xc43f2878 getty 1228 1 1228 0 Ss+ tty inpu 0xc43f2678 getty 1227 1 1227 0 Ss+ tty inpu 0xc43f2478 getty 1226 1 1226 0 Ss+ tty inpu 0xc43f2278 getty 1200 1 1200 0 Ss select 0xc47c72e4 inetd 1176 1 1176 0 Ss select 0xc47c53a4 moused 1163 1 1163 0 Ss nanslp 0xc0d11784 watchdogd 1117 1 1117 0 Ss nanslp 0xc0d11784 cron 1110 1 1110 25 Ss pause 0xc4674d64 sendmail 1104 1 1104 0 Ss select 0xc47c2ca4 sendmail 1099 1 1099 0 Ss select 0xc47c7e64 sshd 1062 1 1062 0 Ss select 0xc47c1e64 ntpd 927 911 911 0 S - 0xc484d400 nfsd 926 911 911 0 S - 0xc484d200 nfsd 925 911 911 0 S - 0xc45f5400 nfsd 924 911 911 0 S - 0xc484d600 nfsd 923 911 911 0 S - 0xc43f7c00 nfsd 922 911 911 0 S - 0xc45f5600 nfsd 921 911 911 0 S - 0xc45caa00 nfsd 919 911 911 0 S - 0xc45f5800 nfsd 918 911 911 0 S - 0xc45ca400 nfsd 917 911 911 0 S - 0xc45cb400 nfsd 916 911 911 0 S - 0xc45cb000 nfsd 915 911 911 0 S - 0xc43f7800 nfsd 914 911 911 0 S - 0xc43f7a00 nfsd 913 911 911 0 S - 0xc45cb600 nfsd 912 911 911 0 S - 0xc4651800 nfsd 911 1 911 0 Ss select 0xc47c68a4 nfsd 909 1 909 0 Ss select 0xc47c2024 mountd 827 1 827 0 Ss select 0xc47bf224 rpcbind 808 1 808 0 Ss select 0xc45a2aa4 syslogd 613 1 613 0 Ss select 0xc45d89e4 devd 21 0 0 0 SL sdflush 0xc0e89bc0 [softdepflush] 20 0 0 0 SL vlruwt 0xc45a629c [vnlru] 19 0 0 0 SL syncer 0xc0e7df50 [syncer] 18 0 0 0 SL psleep 0xc0e7dca4 [bufdaemon] 17 0 0 0 SL pgzero 0xc0e8a7b4 [pagezero] 9 0 0 0 SL psleep 0xc0e8a3dc [vmdaemon] 8 0 0 0 SL psleep 0xc0e8a3a4 [pagedaemon] 7 0 0 0 SL waiting_ 0xc0e7f900 [sctp_iterator] 6 0 0 0 SL - 0xc428763c [fdc0] 16 0 0 0 SL usbtsk 0xc0d0ec94 [usbtask-dr] 15 0 0 0 SL usbtsk 0xc0d0ec80 [usbtask-hc] 14 0 0 0 SL usbevt 0xc435c210 [usb0] 5 0 0 0 SL ccb_scan 0xc0cdddd4 [xpt_thrd] 13 0 0 0 RL [yarrow] 4 0 0 0 RL [g_down] 3 0 0 0 SL - 0xc0d0f3c0 [g_up] 2 0 0 0 SL - 0xc0d0f3b8 [g_event] 12 0 0 0 RL (threaded) intr 100041 I [irq7: ppc0] 100039 I [swi0: uart uart] 100038 I [irq12: psm0] 100037 I [irq1: atkbd0] 100036 I [irq15: ata1] 100035 I [irq14: ata0] 100034 I [irq17: fxp0] 100030 I [irq16: uhci0] 100028 I [irq9: acpi0] 100027 I [swi5: +] 100025 I [swi2: cambio] 100020 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100012 I [swi3: vm] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 Run CPU 0 [swi4: clock] 100007 I [swi1: net] 11 0 0 0 RL (threaded) idle 100006 CanRun [idle: cpu0] 100005 CanRun [idle: cpu1] 100004 Run CPU 2 [idle: cpu2] 100003 Run CPU 3 [idle: cpu3] 1 0 1 0 SLs wait 0xc4162d0c [init] 10 0 0 0 SL audit_wo 0xc0e895c0 [audit] 0 0 0 0 SLs (threaded) kernel 100029 D - 0xc4354d40 [em0 taskq] 100026 D - 0xc4315e80 [kqueue taskq] 100023 D - 0xc4316100 [acpi_task_2] 100022 D - 0xc4316100 [acpi_task_1] 100021 D - 0xc4316100 [acpi_task_0] 100018 D - 0xc4316400 [thread taskq] 100016 D - 0xc414a840 [firmware taskq] 100000 D sched 0xc0d0f480 [swapper] 77734 70606 70531 0 Z+ sh db> allt Tracing command sh pid 77738 tid 100341 td 0xc510e000 fork_trampoline() at fork_trampoline Tracing command grep pid 77737 tid 100424 td 0xc51e4000 sched_switch(c51e4000,0,60b,17b,49e7bad4,...) at sched_switch+0x437 mi_switch(60b,0,c0bc6c0c,804,3,...) at mi_switch+0x200 sched_preempt(c51e4000,3,e6dd2988,c0afbf6e,e6dd0008,...) at sched_preempt+0xb5 ipi_bitmap_handler(e6dd0008,c07e0028,c5390028,0,e133d000,...) at ipi_bitmap_handler+0x30 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0b07edb, esp = 0xe6dd2984, ebp = 0xe6dd2988 --- spinlock_exit(c0d23cb4,4,c0bc3c47,f4) at spinlock_exit+0x2b _mtx_unlock_spin_flags(c0d23cb4,0,c0bf7ce5,3eb,f4) at _mtx_unlock_spin_flags+0xf4 smp_tlb_shootdown(e6dd29ec,c0b0ed78,e133d000,c0eac0a0,0,...) at smp_tlb_shootdown+0xc0 smp_invlpg(e133d000,c0eac0a0,0,bff84cf4,e6dd2a14,...) at smp_invlpg+0x1e pmap_invalidate_page(c0eac0a0,e133d000,c0bf870e,997,e133d000,...) at pmap_invalidate_page+0x38 pmap_remove_pte(e6dd2a34,0,c0bf870e,9d7,c0eac0a0,...) at pmap_remove_pte+0x84 pmap_remove(c0eac0a0,e133d000,e137f000,e6dd2a80,c08015c1,...) at pmap_remove+0x17a vm_map_delete(c1490294,e133d000,e137f000,e6dd2c58,0,...) at vm_map_delete+0x17c kmem_free_wakeup(c1490294,e133d000,41400,c539ca70,e6dd2c3c,...) at kmem_free_wakeup+0x4b exec_free_args(c6f619b0,e6dd2c0c,c0bc140a,348,c539ca70,...) at exec_free_args+0x28 kern_execve(c51e4000,e6dd2c58,0,8145e20,8145eb8,e133d000,e133d000,e133d006,e133d2df,e137d000,3fd21,2,23,0) at kern_execve+0x112c execve(c51e4000,e6dd2cf8,c,e6dd2d38,c0c9ec48,...) at execve+0x4c syscall(e6dd2d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (0, FreeBSD ELF32, nosys), eip = 0x2805c460, esp = 0xbfbfebb8, ebp = 0 --- Tracing command sh pid 77736 tid 100418 td 0xc538cd20 fork_trampoline() at fork_trampoline Tracing command sh pid 77735 tid 100378 td 0xc4e70690 fork_trampoline() at fork_trampoline Tracing command mount_nullfs pid 77732 tid 100350 td 0xc510baf0 sched_switch(c510baf0,0,104,180,4120acfc,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c510baf0,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e6cc0a78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6cc0a78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c510baf0,4b4,40000,...) at vget+0xc9 vnode_pager_lock(c8a176c8,0,c0bea4a7,127,e6cc0c18,...) at vnode_pager_lock+0x1e6 vm_fault(c467a9a0,280c6000,2,8,280c6d06,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c50d3538,...) at trap_pfault+0x118 trap(e6cc0d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804b9c6, esp = 0xbfbfe950, ebp = 0xbfbfe9a8 --- Tracing command mount_nullfs pid 77731 tid 100428 td 0xc4a41000 sched_switch(c4a41000,0,104,180,414266b4,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c4a41000,0,c0bc90d6,247,50,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,0,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,80400,c4673b80,0,0,...) at __lockmgr_args+0xa9f ffs_lock(e6de9b8c,c0bc0bb9,e6de9b94,80400,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6de9b8c,c0cdd660,c4673ac0,80400,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,80400,c0bd2cfc,2a3,c5d93a2c,...) at _vn_lock+0x5e vn_statfile(c4e00498,e6de9c18,c5482d00,c4a41000,c4e00498,...) at vn_statfile+0x103 kern_fstat(c4a41000,3,e6de9c18,c4a41000,e6de9c28,...) at kern_fstat+0x7a fstat(c4a41000,e6de9cf8,8,c0bb2b1c,c0c9f878,...) at fstat+0x27 syscall(e6de9d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (189, FreeBSD ELF32, fstat), eip = 0x28052e33, esp = 0xbfbfe73c, ebp = 0xbfbfe9b8 --- Tracing command mount_nullfs pid 77730 tid 100392 td 0xc527ad20 sched_switch(c527ad20,0,104,180,411e2b30,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c527ad20,0,c0bc90d6,247,50,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,0,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,80100,c4673b80,0,0,...) at __lockmgr_args+0xa9f ffs_lock(e6d5a88c,c0bd08d2,c0bbe8c0,80100,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6d5a88c,c0cdd660,c4673ac0,80100,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,80100,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,80100,c527ad20,1c5,c0b23f8f,...) at vget+0xc9 cache_lookup(c4621560,e6d5abc0,e6d5abd4,e6d5abc0,c5482d00,...) at cache_lookup+0x510 vfs_cache_lookup(e6d5a9a8,c0bd2504,c0cdd2c0,80000,c4621560,...) at vfs_cache_lookup+0xa6 VOP_LOOKUP_APV(c0cc1ca0,e6d5a9a8,c0bd0fd7,1ac,e6d5abc0,...) at VOP_LOOKUP_APV+0xe5 lookup(e6d5aba8,c0bd0fd7,d4,bc,c523822c,...) at lookup+0x58e namei(e6d5aba8,c08381db,c0bc0bb9,c0bbe8c0,3,...) at namei+0x45b vn_open_cred(e6d5aba8,e6d5ac5c,0,c5482d00,c4eeb818,...) at vn_open_cred+0x2ba vn_open(e6d5aba8,e6d5ac5c,0,c4eeb818,c4fb8700,...) at vn_open+0x33 kern_openat(c527ad20,ffffff9c,28080940,0,1,...) at kern_openat+0x106 kern_open(c527ad20,28080940,0,0,0,...) at kern_open+0x35 open(c527ad20,e6d5acf8,c,c0d16880,c0c9e738,...) at open+0x30 syscall(e6d5ad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x28052d93, esp = 0xbfbfe73c, ebp = 0xbfbfe9b8 --- Tracing command sleep pid 77729 tid 100083 td 0xc4676000 sched_switch(c4676000,0,104,180,3e51ed28,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4676000,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(2711,c082ee90,c4676000,3,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0d11784,5c,c0bc6335,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0d11784,0,15c,c0bc6335,2711,...) at _sleep+0x30e kern_nanosleep(c4676000,e689fc64,e689fc6c,a,0,...) at kern_nanosleep+0xc1 nanosleep(c4676000,e689fcf8,8,e689fd38,c0c9fd40,...) at nanosleep+0x6f syscall(e689fd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815540f, esp = 0xbfbfed0c, ebp = 0xbfbfed48 --- Tracing command grep pid 77727 tid 100147 td 0xc4a43d20 sched_switch(c4a43d20,0,104,180,4142bdd8,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c4a43d20,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e69ada78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e69ada78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c4a43d20,4b2,200000,...) at vget+0xc9 vnode_pager_lock(c462a7c0,0,c0bea4a7,127,e69adc18,...) at vnode_pager_lock+0x1e6 vm_fault(c4f82380,280da000,1,0,280da680,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c4e3829c,...) at trap_pfault+0x118 trap(e69add38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2805e618, esp = 0xbfbfe7f0, ebp = 0xbfbfe838 --- Tracing command grep pid 77725 tid 100298 td 0xc50c5000 sched_switch(c50c5000,0,104,180,414345d0,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c50c5000,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e6c09a78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6c09a78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c50c5000,4b4,200000,...) at vget+0xc9 vnode_pager_lock(c5e9b64c,0,c0bea4a7,127,e6c09c18,...) at vnode_pager_lock+0x1e6 vm_fault(c4a482a0,2810d000,2,8,2810d0b1,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c50bb538,...) at trap_pfault+0x118 trap(e6c09d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2805c9c6, esp = 0xbfbfe960, ebp = 0xbfbfe9b8 --- Tracing command mount pid 77723 tid 100468 td 0xc546c690 sched_switch(c546c690,0,103,17b,49e80270,...) at sched_switch+0x437 mi_switch(103,0,c0bc991f,2e1,c6ebc640,...) at mi_switch+0x200 turnstile_wait(c6ebc640,c51e4000,0,188,c0e8a384,...) at turnstile_wait+0x48a _mtx_lock_sleep(c0e8a384,c546c690,0,c0bea4a7,380,...) at _mtx_lock_sleep+0x18e _mtx_lock_flags(c0e8a384,0,c0bea4a7,380,7,...) at _mtx_lock_flags+0xf7 vm_fault(c50c6540,28120000,2,8,2812015f,...) at vm_fault+0x1b79 trap_pfault(5,0,c0bf9a36,2f5,c5313a70,...) at trap_pfault+0x118 trap(e6e69d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804d9c6, esp = 0xbfbfe960, ebp = 0xbfbfe9b8 --- Tracing command mount_nullfs pid 77717 tid 100444 td 0xc5475690 kdb_enter(c0bc50cf,c0bc50cf,c0bc2428,e6e218ac,1,...) at kdb_enter+0x3a panic(c0bc2428,c0bbf193,c0bb4df6,c0bd0d18,143,...) at panic+0x136 _lockmgr_assert(c4729b18,1,c0bd0d18,143,c5475690,...) at _lockmgr_assert+0xb2 __lockmgr_args(c4729b18,100100,c4729b80,0,0,0,c0bd0d18,143) at __lockmgr_args+0xc2a vop_stdunlock(e6e219c8,ca,c4729ac0,c4729ac0,c6455cd8,...) at vop_stdunlock+0x65 VOP_UNLOCK_APV(c0cc1ca0,e6e219c8,c5723d82,25c,2,...) at VOP_UNLOCK_APV+0xb5 null_unlock(e6e21a70,c4729b18,c77daa60,e6e21a20,c5362000,...) at null_unlock+0x102 VOP_UNLOCK_APV(c5724240,e6e21a70,e6e21a80,e6e21a7c,c0bd1237,...) at VOP_UNLOCK_APV+0xb5 nullfs_mount(c5362000,c5475690,c0bd122e,3f0,0,...) at nullfs_mount+0x219 vfs_donmount(c5475690,0,c629bd00,c629bd00,bfbfebac,...) at vfs_donmount+0x1338 nmount(c5475690,e6e21cf8,c,e6e21d38,c0ca0a30,...) at nmount+0xbe syscall(e6e21d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (378, FreeBSD ELF32, nmount), eip = 0x280d8a9b, esp = 0xbfbfe2fc, ebp = 0xbfbfeb68 --- Tracing command mount_nullfs pid 77716 tid 100189 td 0xc4faf460 sched_switch(c4faf460,0,60b,17b,467eced0,...) at sched_switch+0x437 mi_switch(60b,0,c0bc6c0c,804,0,...) at mi_switch+0x200 sched_preempt(c4faf460,0,e6a5b924,c0afbf6e,8,...) at sched_preempt+0xb5 ipi_bitmap_handler(8,c4110028,c0bd0028,c4faf460,0,...) at ipi_bitmap_handler+0x30 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc07ec599, esp = 0xe6a5b908, ebp = 0xe6a5b924 --- _mtx_lock_sleep(c4729b80,c4faf460,0,c0be94cb,19a,...) at _mtx_lock_sleep+0x99 _mtx_lock_flags(c4729b80,0,c0be94cb,19a,c4729ac0,...) at _mtx_lock_flags+0xf7 ufs_getattr(e6a5bae8,c0bfbfd2,c4faf460,1,0,...) at ufs_getattr+0x40 VOP_GETATTR_APV(c0cc1ca0,e6a5bae8,e6a5ba44,c5723603,e6a5bae8,...) at VOP_GETATTR_APV+0xc5 VOP_GETATTR_AP(e6a5bae8,4,c0bc3c47,ca,e6a5baec,...) at VOP_GETATTR_AP+0x1b null_bypass(e6a5bae8,c5724240,e6a5ba70,c0b23e05,e6a5bae8,...) at null_bypass+0xb3 null_getattr(e6a5bae8,c0bfbfd2,8d3,e6a5bb24,c70dbc18,...) at null_getattr+0x12 VOP_GETATTR_APV(c5724240,e6a5bae8,c14726d4,c1481800,0,...) at VOP_GETATTR_APV+0xc5 vn_stat(c70dbc18,e6a5bb24,c5482d00,0,c4faf460,...) at vn_stat+0x56 kern_statat(c4faf460,0,ffffff9c,bfbfe754,0,...) at kern_statat+0xa8 kern_stat(c4faf460,bfbfe754,0,e6a5bc18,800ff0b,...) at kern_stat+0x36 stat(c4faf460,e6a5bcf8,8,c0bcb647,c0c9f860,...) at stat+0x2f syscall(e6a5bd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (188, FreeBSD ELF32, stat), eip = 0x28164e13, esp = 0xbfbfe27c, ebp = 0xbfbfe2f8 --- Tracing command mount_nullfs pid 77715 tid 100345 td 0xc510d690 sched_switch(c510d690,0,104,180,4121a454,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c510d690,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e6cb1a78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6cb1a78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c510d690,4b4,200000,...) at vget+0xc9 vnode_pager_lock(c9cb3000,0,c0bea4a7,127,e6cb1c18,...) at vnode_pager_lock+0x1e6 vm_fault(c50c68c0,28107000,2,8,28107c8a,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,14,c50fe29c,...) at trap_pfault+0x118 trap(e6cb1d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804b9c6, esp = 0xbfbfe950, ebp = 0xbfbfe9a8 --- Tracing command grep pid 77714 tid 100380 td 0xc4e70230 sched_switch(c4e70230,0,207,17b,41282fdc,...) at sched_switch+0x437 mi_switch(207,0,c0bc98d9,d4,c53f829c,...) at mi_switch+0x200 ast(e6d30d38) at ast+0x2cd doreti_ast() at doreti_ast+0x17 Tracing command grep pid 77713 tid 100115 td 0xc43dd230 sched_switch(c43dd230,0,104,180,365d2be2,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c43dd230,0,c0bc90d6,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4fe7480,4c,c0bcb3c3,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4fe7480,c4fe75e8,14c,c0bcb3c3,0,...) at _sleep+0x344 pipe_read(c4e00e70,e6934c58,c5482d00,0,c43dd230,...) at pipe_read+0x3f7 dofileread(e6934c58,ffffffff,ffffffff,0,c4e00e70,...) at dofileread+0x96 kern_readv(c43dd230,0,e6934c58,e6934c78,1,...) at kern_readv+0x58 read(c43dd230,e6934cf8,c,e6934ca0,c0c9e708,...) at read+0x4f syscall(e6934d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281baaa3, esp = 0xbfbfe95c, ebp = 0xbfbfe9e8 --- Tracing command mount pid 77712 tid 100467 td 0xc529a230 sched_switch(c529a230,0,104,180,41430758,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c529a230,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e6e66a78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6e66a78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c529a230,4b4,200000,...) at vget+0xc9 vnode_pager_lock(c8007aa8,0,c0bea4a7,127,e6e66c18,...) at vnode_pager_lock+0x1e6 vm_fault(c50c8700,28115000,2,8,2811599f,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c5244538,...) at trap_pfault+0x118 trap(e6e66d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804d9c6, esp = 0xbfbfe960, ebp = 0xbfbfe9b8 --- Tracing command mount_nullfs pid 77707 tid 100532 td 0xc538ed20 sched_switch(c538ed20,0,207,17b,40bec9bc,...) at sched_switch+0x437 mi_switch(207,0,c0bc98d9,d4,c524129c,...) at mi_switch+0x200 ast(e6f31d38) at ast+0x2cd doreti_ast() at doreti_ast+0x17 Tracing command grep pid 77695 tid 100425 td 0xc4a41690 sched_switch(c4a41690,0,104,180,30e8bd48,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c4a41690,0,c0bc90d6,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c5057c00,4c,c0bcb3c3,100,0,...) at sleepq_wait_sig+0x17 _sleep(c5057c00,c5057d68,14c,c0bcb3c3,0,...) at _sleep+0x344 pipe_read(c4d84d58,e6de0c58,c5482d00,0,c4a41690,...) at pipe_read+0x3f7 dofileread(e6de0c58,ffffffff,ffffffff,0,c4d84d58,...) at dofileread+0x96 kern_readv(c4a41690,0,e6de0c58,e6de0c78,1,...) at kern_readv+0x58 read(c4a41690,e6de0cf8,c,c0bcb6d0,c0c9e708,...) at read+0x4f syscall(e6de0d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281baaa3, esp = 0xbfbfe95c, ebp = 0xbfbfe9e8 --- Tracing command grep pid 77694 tid 100226 td 0xc4ff2230 sched_switch(c4ff2230,0,104,180,411ea684,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c4ff2230,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e6aefa78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6aefa78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c4ff2230,4b4,40000,...) at vget+0xc9 vnode_pager_lock(c9975000,0,c0bea4a7,127,e6aefc18,...) at vnode_pager_lock+0x1e6 vm_fault(c465be00,28163000,2,8,28163714,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c4ff7a70,...) at trap_pfault+0x118 trap(e6aefd38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2805c9c6, esp = 0xbfbfe960, ebp = 0xbfbfe9b8 --- Tracing command grep pid 77693 tid 100173 td 0xc4f8d000 sched_switch(c4f8d000,0,104,180,347b0390,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c4f8d000,0,c0bc90d6,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c49ef300,4c,c0bcb3c3,100,0,...) at sleepq_wait_sig+0x17 _sleep(c49ef300,c49ef468,14c,c0bcb3c3,0,...) at _sleep+0x344 pipe_read(c4e009a0,e6a1bc58,c5482d00,0,c4f8d000,...) at pipe_read+0x3f7 dofileread(e6a1bc58,ffffffff,ffffffff,0,c4e009a0,...) at dofileread+0x96 kern_readv(c4f8d000,0,e6a1bc58,e6a1bc78,1,...) at kern_readv+0x58 read(c4f8d000,e6a1bcf8,c,c0bcb6d0,c0c9e708,...) at read+0x4f syscall(e6a1bd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281baaa3, esp = 0xbfbfe95c, ebp = 0xbfbfe9e8 --- Tracing command grep pid 77691 tid 100257 td 0xc5036d20 sched_switch(c5036d20,0,104,180,33d8453e,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c5036d20,0,c0bc90d6,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4d89480,4c,c0bcb3c3,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4d89480,c4d895e8,14c,c0bcb3c3,0,...) at _sleep+0x344 pipe_read(c4e000e0,e6b6bc58,c5482d00,0,c5036d20,...) at pipe_read+0x3f7 dofileread(e6b6bc58,ffffffff,ffffffff,0,c4e000e0,...) at dofileread+0x96 kern_readv(c5036d20,0,e6b6bc58,e6b6bc78,1,...) at kern_readv+0x58 read(c5036d20,e6b6bcf8,c,c0bcb6d0,c0c9e708,...) at read+0x4f syscall(e6b6bd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281baaa3, esp = 0xbfbfe95c, ebp = 0xbfbfe9e8 --- Tracing command mount pid 77690 tid 100229 td 0xc4ff1af0 sched_switch(c4ff1af0,0,104,180,4120e674,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c4ff1af0,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e6afba78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6afba78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c4ff1af0,4b4,200000,...) at vget+0xc9 vnode_pager_lock(ca084934,0,c0bea4a7,127,e6afbc18,...) at vnode_pager_lock+0x1e6 vm_fault(c5085540,28192000,1,0,281920ee,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c4ff729c,...) at trap_pfault+0x118 trap(e6afbd38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280c630c, esp = 0xbfbfeac0, ebp = 0xbfbfeae8 --- Tracing command mount_nullfs pid 77679 tid 100259 td 0xc50368c0 sched_switch(c50368c0,0,104,180,41207344,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c50368c0,0,c0bc90d6,247,c4673b18,...) at sleepq_switch+0x15f sleepq_wait(c4673b18,50,c0bbf193,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c4673b18,200501,c4673b80,0,0,...) at __lockmgr_args+0x4a5 ffs_lock(e6b73a78,c0beceb2,c0bbe8c0,200501,c4673ac0,...) at ffs_lock+0x8a VOP_LOCK1_APV(c0cc1ca0,e6b73a78,c0cdd660,c4673ac0,200501,...) at VOP_LOCK1_APV+0xa5 _vn_lock(c4673ac0,200501,c0bd18c8,803,4,...) at _vn_lock+0x5e vget(c4673ac0,200501,c50368c0,4b4,200000,...) at vget+0xc9 vnode_pager_lock(c87eb64c,0,c0bea4a7,127,e6b73c18,...) at vnode_pager_lock+0x1e6 vm_fault(c4fee7e0,28104000,2,8,2810499f,...) at vm_fault+0x1df trap_pfault(5,0,c0bf9a36,2f5,c503cd0c,...) at trap_pfault+0x118 trap(e6b73d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804b9c6, esp = 0xbfbfe950, ebp = 0xbfbfe9a8 --- Tracing command sh pid 70609 tid 100284 td 0xc5093690 sched_switch(c5093690,0,104,180,1f1b305e,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c5093690,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c508d538,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c508d538,c508d5c0,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c5093690,ffffffff,e6bd1c74,2,0,...) at kern_wait+0xab6 wait4(c5093690,e6bd1cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6bd1d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe85c, ebp = 0xbfbfe878 --- Tracing command sh pid 70608 tid 100396 td 0xc527a8c0 sched_switch(c527a8c0,0,104,180,4064d9a4,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c527a8c0,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c52c9d0c,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c52c9d0c,c52c9d94,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c527a8c0,ffffffff,e6d6ac74,2,0,...) at kern_wait+0xab6 wait4(c527a8c0,e6d6acf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6d6ad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70607 tid 100387 td 0xc52dd8c0 sched_switch(c52dd8c0,0,104,180,3b07eb6c,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c52dd8c0,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c52b5000,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c52b5000,c52b5088,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c52dd8c0,ffffffff,e6d49c74,2,0,...) at kern_wait+0xab6 wait4(c52dd8c0,e6d49cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6d49d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70606 tid 100332 td 0xc5016690 sched_switch(c5016690,0,104,180,404085f0,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c5016690,0,c0bc90d6,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4d9e600,4c,c0bcb3c3,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4d9e600,c4d9e768,14c,c0bcb3c3,0,...) at _sleep+0x344 pipe_read(c47a6af0,e6c8ac58,c5482d00,0,c5016690,...) at pipe_read+0x3f7 dofileread(e6c8ac58,ffffffff,ffffffff,0,c47a6af0,...) at dofileread+0x96 kern_readv(c5016690,3,e6c8ac58,e6c8ac78,1,...) at kern_readv+0x58 read(c5016690,e6c8acf8,c,e6c8ad38,c0c9e708,...) at read+0x4f syscall(e6c8ad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281e3aa3, esp = 0xbfbfe66c, ebp = 0xbfbfe7c8 --- Tracing command sh pid 70605 tid 100432 td 0xc52e1d20 sched_switch(c52e1d20,0,104,180,19dc2690,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c52e1d20,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c52b8d0c,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c52b8d0c,c52b8d94,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c52e1d20,ffffffff,e6dfdc74,2,0,...) at kern_wait+0xab6 wait4(c52e1d20,e6dfdcf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6dfdd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70604 tid 100109 td 0xc4811460 sched_switch(c4811460,0,104,180,35815d2c,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4811460,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c480829c,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c480829c,c4808324,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c4811460,ffffffff,e691bc74,2,0,...) at kern_wait+0xab6 wait4(c4811460,e691bcf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e691bd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe85c, ebp = 0xbfbfe878 --- Tracing command sh pid 70603 tid 100133 td 0xc4810000 sched_switch(c4810000,0,104,180,386a3484,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4810000,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4aad29c,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4aad29c,c4aad324,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c4810000,ffffffff,e6981c74,2,0,...) at kern_wait+0xab6 wait4(c4810000,e6981cf8,10,c0bcb5bc,c0c9e768,...) at wait4+0x3b syscall(e6981d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe85c, ebp = 0xbfbfe878 --- Tracing command sh pid 70602 tid 100127 td 0xc4a418c0 sched_switch(c4a418c0,0,104,180,215561bc,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4a418c0,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4839000,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4839000,c4839088,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c4a418c0,ffffffff,e6961c74,2,0,...) at kern_wait+0xab6 wait4(c4a418c0,e6961cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6961d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70601 tid 100275 td 0xc5083230 sched_switch(c5083230,0,104,180,3cc868b4,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c5083230,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c503d7d4,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c503d7d4,c503d85c,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c5083230,ffffffff,e6badc74,2,0,...) at kern_wait+0xab6 wait4(c5083230,e6badcf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6badd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70600 tid 100495 td 0xc5308000 sched_switch(c5308000,0,104,180,411db1d4,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c5308000,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c5225a70,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c5225a70,c5225af8,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c5308000,ffffffff,e6ec2c74,2,0,...) at kern_wait+0xab6 wait4(c5308000,e6ec2cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6ec2d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe85c, ebp = 0xbfbfe878 --- Tracing command sh pid 70599 tid 100539 td 0xc538e460 sched_switch(c538e460,0,104,180,3d9b5ad4,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c538e460,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c523c7d4,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c523c7d4,c523c85c,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c538e460,ffffffff,e6f46c74,2,0,...) at kern_wait+0xab6 wait4(c538e460,e6f46cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6f46d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70598 tid 100118 td 0xc4a42d20 sched_switch(c4a42d20,0,104,180,12b3c444,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4a42d20,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c483a7d4,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c483a7d4,c483a85c,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c4a42d20,ffffffff,e6946c74,2,0,...) at kern_wait+0xab6 wait4(c4a42d20,e6946cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6946d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe85c, ebp = 0xbfbfe878 --- Tracing command sh pid 70597 tid 100301 td 0xc50c28c0 sched_switch(c50c28c0,0,104,180,ca4887a,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c50c28c0,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c50bad0c,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c50bad0c,c50bad94,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c50c28c0,ffffffff,e6c15c74,2,0,...) at kern_wait+0xab6 wait4(c50c28c0,e6c15cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6c15d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70596 tid 100344 td 0xc510d8c0 sched_switch(c510d8c0,0,104,180,264308d6,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c510d8c0,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c50fe538,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c50fe538,c50fe5c0,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c510d8c0,ffffffff,e6caec74,2,0,...) at kern_wait+0xab6 wait4(c510d8c0,e6caecf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6caed38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command sh pid 70595 tid 100356 td 0xc5109d20 sched_switch(c5109d20,0,104,180,1fffd620,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c5109d20,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c5174a70,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c5174a70,c5174af8,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c5109d20,ffffffff,e6cdec74,2,0,...) at kern_wait+0xab6 wait4(c5109d20,e6cdecf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6cded38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command bash pid 70510 tid 100434 td 0xc5476af0 sched_switch(c5476af0,0,104,180,1131e8,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c5476af0,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c4651204,0,c0bc3c47,c5476af0,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4651278,0,e6e03b0c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c4651278,c4651204,c0bcce73,4a7,0,...) at _cv_wait_sig+0x240 tty_wait(c4651200,c4651278,e6e03c58,1,0,...) at tty_wait+0x41 ttydisc_read(c4651200,e6e03c58,0,92,0,...) at ttydisc_read+0x22b ttydev_read(c4cd4200,e6e03c58,0,0,1,...) at ttydev_read+0x7a devfs_read_f(c4680d20,e6e03c58,c5482d00,0,c5476af0,...) at devfs_read_f+0x7e dofileread(e6e03c58,ffffffff,ffffffff,0,c4680d20,...) at dofileread+0x96 kern_readv(c5476af0,0,e6e03c58,e6e03c78,1,...) at kern_readv+0x58 read(c5476af0,e6e03cf8,c,16,c0c9e708,...) at read+0x4f syscall(e6e03d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x28336aa3, esp = 0xbfbfe63c, ebp = 0xbfbfe668 --- Tracing command csh pid 70506 tid 100149 td 0xc4a438c0 sched_switch(c4a438c0,0,104,180,afa0b3de,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,68,...) at mi_switch+0x200 sleepq_switch(c4a438c0,0,c0bc90d6,189,68,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4e37d64,68,c0b826dd,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4e37d64,c4e37d94,168,c0b826dd,0,...) at _sleep+0x344 kern_sigsuspend(c4a438c0,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c4a438c0,e69b3cf8,4,c0bcb67b,c0ca06b8,...) at sigsuspend+0x4d syscall(e69b3d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x2817adbb, esp = 0xbfbfe62c, ebp = 0xbfbfe698 --- Tracing command su pid 70505 tid 100334 td 0xc510ad20 sched_switch(c510ad20,0,104,180,1419468a,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c510ad20,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c503b29c,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c503b29c,c503b324,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c510ad20,1136a,e6c90c74,2,0,...) at kern_wait+0xab6 wait4(c510ad20,e6c90cf8,10,e6c90d38,c0c9e768,...) at wait4+0x3b syscall(e6c90d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2811baeb, esp = 0xbfbfe67c, ebp = 0xbfbfe698 --- Tracing command awk pid 3084 tid 100354 td 0xc510a230 sched_switch(c510a230,0,104,180,3a8cf130,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c510a230,0,c0bc90d6,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4d8e900,4c,c0bcb3c3,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4d8e900,c4d8ea68,14c,c0bcb3c3,0,...) at _sleep+0x344 pipe_read(c7b28968,e6cd8c58,c6757d00,0,c510a230,...) at pipe_read+0x3f7 dofileread(e6cd8c58,ffffffff,ffffffff,0,c7b28968,...) at dofileread+0x96 kern_readv(c510a230,0,e6cd8c58,e6cd8c78,1,...) at kern_readv+0x58 read(c510a230,e6cd8cf8,c,16,c0c9e708,...) at read+0x4f syscall(e6cd8d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281abaa3, esp = 0xbfbfe9cc, ebp = 0xbfbfe9e8 --- Tracing command sh pid 3083 tid 100395 td 0xc52dd000 sched_switch(c52dd000,0,104,180,3aebab34,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c52dd000,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c52c0538,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c52c0538,c52c05c0,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c52dd000,ffffffff,e6d67c74,2,0,...) at kern_wait+0xab6 wait4(c52dd000,e6d67cf8,10,c0bcb780,c0c9e768,...) at wait4+0x3b syscall(e6d67d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfe99c, ebp = 0xbfbfe9b8 --- Tracing command top pid 3082 tid 100518 td 0xc5314af0 sched_switch(c5314af0,0,104,180,400f97fc,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c5314af0,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e6f07a4c,c07ec36a,c524f7d0,0,c5314af0,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c524f7e4,0,e6f07a7c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c524f7e4,c524f7d0,3e8,570,c7a59380,...) at _cv_timedwait_sig+0x250 seltdwait(e6f07c2c,e6f07c34,3d6,c5314af0,343a3631,...) at seltdwait+0x8a kern_select(c5314af0,2,bfbfec3c,0,0,e6f07c70,1,0) at kern_select+0x4cf select(c5314af0,e6f07cf8,14,c0bb3a5a,c0c9ef78,...) at select+0x5e syscall(e6f07d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281e0a23, esp = 0xbfbfebdc, ebp = 0xbfbfed38 --- Tracing command tail pid 3081 tid 100519 td 0xc53148c0 sched_switch(c53148c0,0,104,180,f562aeb8,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,58,...) at mi_switch+0x200 sleepq_switch(c53148c0,0,c0bc90d6,189,58,...) at sleepq_switch+0x15f sleepq_catch_signals(3e9,c082ee90,c53148c0,1,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c504c400,58,c0bc133b,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c504c400,c504c400,158,c0bc133b,3e9,...) at _sleep+0x30e kern_kevent(c53148c0,4,0,1,e6f0ac58,...) at kern_kevent+0x357 kevent(c53148c0,e6f0acf8,18,c0b6e282,c0ca08c8,...) at kevent+0x19b syscall(e6f0ad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (363, FreeBSD ELF32, kevent), eip = 0x281595fb, esp = 0xbfbfebcc, ebp = 0xbfbfec78 --- Tracing command sh pid 3080 tid 100422 td 0xc538caf0 sched_switch(c538caf0,0,104,180,640d147a,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c538caf0,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c52be000,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c52be000,c52be088,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c538caf0,ffffffff,e6dcbc74,2,0,...) at kern_wait+0xab6 wait4(c538caf0,e6dcbcf8,10,e6dcbd38,c0c9e768,...) at wait4+0x3b syscall(e6dcbd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815caeb, esp = 0xbfbfeb9c, ebp = 0xbfbfebb8 --- Tracing command sshd pid 3079 tid 100541 td 0xc538e000 sched_switch(c538e000,0,104,180,147413be,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c538e000,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c5581010,0,c0bc3c47,c538e000,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c5581024,0,e6f4ca7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c5581024,c5581010,c0bcb1eb,570,c4f9b348,...) at _cv_wait_sig+0x240 seltdwait(c670db2c,c0bcb1eb,3d6,c538e000,200246,...) at seltdwait+0xa2 kern_select(c538e000,c,81030b8,81030e4,0,0,1,280fe818) at kern_select+0x4cf select(c538e000,e6f4ccf8,14,c0bb3a5a,c0c9ef78,...) at select+0x5e syscall(e6f4cd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283bfa23, esp = 0xbfbfdeac, ebp = 0xbfbfdef8 --- Tracing command sshd pid 3078 tid 100249 td 0xc5013690 sched_switch(c5013690,0,104,180,4019ea50,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c5013690,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c5277e50,0,c0bc3c47,c5013690,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c5277e64,0,e6b4ba7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c5277e64,c5277e50,c0bcb1eb,570,c4d16700,...) at _cv_wait_sig+0x240 seltdwait(c503fe2c,c0bcb1eb,3d6,c5013690,200246,...) at seltdwait+0xa2 kern_select(c5013690,a,81030b8,81030e4,0,0,1,280fe818) at kern_select+0x4cf select(c5013690,e6b4bcf8,14,c0bb3a5a,c0c9ef78,...) at select+0x5e syscall(e6b4bd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283bfa23, esp = 0xbfbfdeac, ebp = 0xbfbfdef8 --- Tracing command sshd pid 3077 tid 100312 td 0xc50d88c0 Tracing command sshd pid 3073 tid 100400 td 0xc5286d20 Tracing command sshd pid 3072 tid 100366 td 0xc528a460 Tracing command sshd pid 3069 tid 100256 td 0xc5038000 Tracing command bash pid 1261 tid 100107 td 0xc48118c0 sched_switch(c48118c0,0,104,180,ef4b1b44,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c48118c0,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c48087d4,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c48087d4,c480885c,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c48118c0,ffffffff,e6915c74,6,0,...) at kern_wait+0xab6 wait4(c48118c0,e6915cf8,10,c0bcb67b,c0c9e768,...) at wait4+0x3b syscall(e6915d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282afaeb, esp = 0xbfbfe8ec, ebp = 0xbfbfe908 --- Tracing command sshd pid 1260 tid 100122 td 0xc4a42460 sched_switch(c4a42460,0,104,180,40c7cf38,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4a42460,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c47c8310,0,c0bc3c47,c4a42460,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c47c8324,0,e6952a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c47c8324,c47c8310,c0bcb1eb,570,c4680c40,...) at _cv_wait_sig+0x240 seltdwait(c4cd4b2c,c0bcb1eb,3d6,c4a42460,200246,...) at seltdwait+0xa2 kern_select(c4a42460,a,81030b8,81030e4,0,0,1,280fe818) at kern_select+0x4cf select(c4a42460,e6952cf8,14,c0bb3a5a,c0c9ef78,...) at select+0x5e syscall(e6952d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283bfa23, esp = 0xbfbfdeac, ebp = 0xbfbfdef8 --- Tracing command sshd pid 1258 tid 100141 td 0xc4b30af0 Tracing command getty pid 1233 tid 100062 td 0xc4679000 Tracing command getty pid 1232 tid 100100 td 0xc475c000 Tracing command getty pid 1231 tid 100054 td 0xc46798c0 Tracing command getty pid 1230 tid 100066 td 0xc4657230 Tracing command getty pid 1229 tid 100111 td 0xc4811000 Tracing command getty pid 1228 tid 100104 td 0xc46c4690 Tracing command getty pid 1227 tid 100108 td 0xc4811690 Tracing command getty pid 1226 tid 100097 td 0xc46bbd20 Tracing command inetd pid 1200 tid 100080 td 0xc46c1690 Tracing command moused pid 1176 tid 100139 td 0xc4b31000 Tracing command watchdogd pid 1163 tid 100134 td 0xc475dd20 sched_switch(c475dd20,0,104,180,7313c0ea,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c475dd20,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c082ee90,c475dd20,0,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0d11784,5c,c0bc6335,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0d11784,0,15c,c0bc6335,ea61,...) at _sleep+0x30e kern_nanosleep(c475dd20,e6984c64,e6984c6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c475dd20,e6984cf8,8,c0bfc659,c0c9fd40,...) at nanosleep+0x6f syscall(e6984d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2817f40f, esp = 0xbfbfed4c, ebp = 0xbfbfed78 --- Tracing command cron pid 1117 tid 100089 td 0xc475cd20 sched_switch(c475cd20,0,104,180,5584c542,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c475cd20,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c082ee90,c475cd20,1,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0d11784,5c,c0bc6335,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0d11784,0,15c,c0bc6335,ea61,...) at _sleep+0x30e kern_nanosleep(c475cd20,e68b8c64,e68b8c6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c475cd20,e68b8cf8,8,c0bcb8e5,c0c9fd40,...) at nanosleep+0x6f syscall(e68b8d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2817240f, esp = 0xbfbfecdc, ebp = 0xbfbfed08 --- Tracing command sendmail pid 1110 tid 100058 td 0xc4679690 sched_switch(c4679690,0,104,180,c80e00c6,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,68,...) at mi_switch+0x200 sleepq_switch(c4679690,0,c0bc90d6,189,68,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4674d64,68,c0b826dd,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4674d64,c4674d94,168,c0b826dd,0,...) at _sleep+0x344 kern_sigsuspend(c4679690,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c4679690,e6818cf8,4,c0bcb67b,c0ca06b8,...) at sigsuspend+0x4d syscall(e6818d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x2832ddbb, esp = 0xbfbfd00c, ebp = 0xbfbfd038 --- Tracing command sendmail pid 1104 tid 100105 td 0xc4811d20 sched_switch(c4811d20,0,104,180,b289bea6,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4811d20,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e690fa4c,c07ec36a,c47c2c90,0,c4811d20,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c47c2ca4,0,e690fa7c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c47c2ca4,c47c2c90,1389,570,c4d58460,...) at _cv_timedwait_sig+0x250 seltdwait(e690fc2c,e690fc34,3d6,c4811d20,c148ca38,...) at seltdwait+0x8a kern_select(c4811d20,7,bfbfc5b0,0,0,e690fc70,5,0) at kern_select+0x4cf select(c4811d20,e690fcf8,14,c0bcbba3,c0c9ef78,...) at select+0x5e syscall(e690fd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283caa23, esp = 0xbfbfc51c, ebp = 0xbfbfd048 --- Tracing command sshd pid 1099 tid 100126 td 0xc4a41af0 Tracing command ntpd pid 1062 tid 100110 td 0xc4811230 sched_switch(c4811230,0,104,180,fc127fc2,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4811230,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c47c1e50,0,c0bc3c47,c4811230,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c47c1e64,0,e691ea7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c47c1e64,c47c1e50,c0bcb1eb,570,c465f038,...) at _cv_wait_sig+0x240 seltdwait(c4a46e2c,c0bcb1eb,3d6,c4811230,0,...) at seltdwait+0xa2 kern_select(c4811230,1b,bfbfed78,0,0,0,246,0) at kern_select+0x4cf select(c4811230,e691ecf8,14,16,c0c9ef78,...) at select+0x5e syscall(e691ed38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2834ba23, esp = 0xbfbfed4c, ebp = 0xbfbfee18 --- Tracing command nfsd pid 927 tid 100114 td 0xc43dd460 Tracing command nfsd pid 926 tid 100113 td 0xc43dd690 Tracing command nfsd pid 925 tid 100057 td 0xc4657690 Tracing command nfsd pid 924 tid 100085 td 0xc475d690 Tracing command nfsd pid 923 tid 100077 td 0xc43ddaf0 Tracing command nfsd pid 922 tid 100076 td 0xc43ddd20 Tracing command nfsd pid 921 tid 100075 td 0xc4657000 Tracing command nfsd pid 919 tid 100078 td 0xc43dd8c0 Tracing command nfsd pid 918 tid 100055 td 0xc4657af0 Tracing command nfsd pid 917 tid 100056 td 0xc46578c0 Tracing command nfsd pid 916 tid 100061 td 0xc4657460 Tracing command nfsd pid 915 tid 100064 td 0xc46bb000 Tracing command nfsd pid 914 tid 100103 td 0xc46c48c0 Tracing command nfsd pid 913 tid 100106 td 0xc4811af0 Tracing command nfsd pid 912 tid 100087 td 0xc475d230 Tracing command nfsd pid 911 tid 100099 td 0xc475c230 Tracing command mountd pid 909 tid 100102 td 0xc46c4af0 sched_switch(c46c4af0,0,104,180,abe26afc,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c46c4af0,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c47c2010,0,c0bc3c47,c46c4af0,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c47c2024,0,e6906a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c47c2024,c47c2010,c0bcb1eb,570,c465f9a0,...) at _cv_wait_sig+0x240 seltdwait(c45a752c,c0bcb1eb,3d6,c46c4af0,c0bd2d05,...) at seltdwait+0xa2 kern_select(c46c4af0,9,bfbfedec,0,0,0,e6906c80,2819d9e0) at kern_select+0x4cf select(c46c4af0,e6906cf8,14,c0be3d12,c0c9ef78,...) at select+0x5e syscall(e6906d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2818aa23, esp = 0xbfbfedbc, ebp = 0xbfbfee88 --- Tracing command rpcbind pid 827 tid 100101 td 0xc46c4d20 sched_switch(c46c4d20,0,104,180,9e7fd606,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c46c4d20,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e6903aa8,c07ec36a,c47bf210,0,c46c4d20,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c47bf224,0,e6903ad8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c47bf224,c47bf210,7531,570,e6903b8c,...) at _cv_timedwait_sig+0x250 seltdwait(e6903c5c,e6903c64,480,c46c4d20,e6903b5c,...) at seltdwait+0x8a poll(c46c4d20,e6903cf8,c,c0bcb8e5,c0c9fa58,...) at poll+0x2f9 syscall(e6903d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (209, FreeBSD ELF32, poll), eip = 0x2813b40f, esp = 0xbfbfcc7c, ebp = 0xbfbfee48 --- Tracing command syslogd pid 808 tid 100086 td 0xc475d460 sched_switch(c475d460,0,104,180,8bde300,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c475d460,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c45a2a90,0,c0bc3c47,c475d460,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c45a2aa4,0,e68aca7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c45a2aa4,c45a2a90,c0bcb1eb,570,c465fab8,...) at _cv_wait_sig+0x240 seltdwait(c45a8d2c,c0bcb1eb,3d6,c475d460,0,...) at seltdwait+0xa2 kern_select(c475d460,9,81290ac,0,0,0,16f4e,2819d9e0) at kern_select+0x4cf select(c475d460,e68accf8,14,c0bcb790,c0c9ef78,...) at select+0x5e syscall(e68acd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2818aa23, esp = 0xbfbfdedc, ebp = 0xbfbfee88 --- Tracing command devd pid 613 tid 100090 td 0xc475caf0 sched_switch(c475caf0,0,104,180,dfeb478c,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c475caf0,0,c0bc90d6,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c07ec36a,c45d89d0,0,c0bc3c47,c475caf0,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c45d89e4,0,e68bba7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c45d89e4,c45d89d0,c0bcb1eb,570,c465fa10,...) at _cv_wait_sig+0x240 seltdwait(c461b32c,c0bcb1eb,3d6,c475caf0,c0bc012d,...) at seltdwait+0xa2 kern_select(c475caf0,5,bfbfe9f0,0,0,0,0,8) at kern_select+0x4cf select(c475caf0,e68bbcf8,14,c0b6e282,c0c9ef78,...) at select+0x5e syscall(e68bbd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x808806f, esp = 0xbfbfe9bc, ebp = 0xbfbfee98 --- Tracing command softdepflush pid 21 tid 100049 td 0xc43378c0 sched_switch(c43378c0,0,104,180,8655b4,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,44,...) at mi_switch+0x200 sleepq_switch(c43378c0,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e89bc0,44,c0be7e10,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e89bc0,c0e89b64,44,c0be7e10,3e8,...) at _sleep+0x329 softdep_flush(0,e4609d38,c0bc15d4,322,c45a6000,...) at softdep_flush+0x2c1 fork_exit(c09fac80,0,e4609d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4609d70, ebp = 0 --- Tracing command vnlru pid 20 tid 100048 td 0xc4337af0 sched_switch(c4337af0,0,104,180,ee893d5a,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,50,...) at mi_switch+0x200 sleepq_switch(c4337af0,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c45a629c,50,c0bd282a,0,0,...) at sleepq_timedwait+0x6b _sleep(c45a629c,c0e7df10,250,c0bd282a,3e8,...) at _sleep+0x329 vnlru_proc(0,e4606d38,c0bc15d4,322,c45a629c,...) at vnlru_proc+0x10b fork_exit(c0883aa0,0,e4606d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4606d70, ebp = 0 --- Tracing command syncer pid 19 tid 100047 td 0xc4337d20 sched_switch(c4337d20,0,104,180,2715d428,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4337d20,0,c0bc90d6,26a,c4337d20,...) at sleepq_switch+0x15f sleepq_timedwait(c0e7df50,0,e4603c88,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0e7df50,c0e7df3c,3e8,6aa,4e20,...) at _cv_timedwait+0x250 sched_sync(0,e4603d38,c0bc15d4,322,c45a6538,...) at sched_sync+0x502 fork_exit(c0882f10,0,e4603d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4603d70, ebp = 0 --- Tracing command bufdaemon pid 18 tid 100046 td 0xc43dc000 sched_switch(c43dc000,0,104,180,f583733c,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,44,...) at mi_switch+0x200 sleepq_switch(c43dc000,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e7dca4,44,c0bcfd8c,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e7dca4,c0e7dca8,44,c0bcfd8c,3e8,...) at _sleep+0x329 buf_daemon(0,e4600d38,c0bc15d4,322,c45a67d4,...) at buf_daemon+0x21e fork_exit(c086c110,0,e4600d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4600d70, ebp = 0 --- Tracing command pagezero pid 17 tid 100045 td 0xc43dc230 sched_switch(c43dc230,0,104,180,b28e212a,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c43dc230,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e8a7b4,0,c0bece55,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e8a7b4,c0e8a370,0,c0bece55,493e0,...) at _sleep+0x329 vm_pagezero(0,e45fdd38,c0bc15d4,322,c45a6a70,...) at vm_pagezero+0xdc fork_exit(c0a35150,0,e45fdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45fdd70, ebp = 0 --- Tracing command vmdaemon pid 9 tid 100044 td 0xc43dc460 sched_switch(c43dc460,0,104,180,dc5c3134,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,68,...) at mi_switch+0x200 sleepq_switch(c43dc460,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0e8a3dc,68,c0bcfd8c,0,0,...) at sleepq_wait+0x63 _sleep(c0e8a3dc,c0e8a3e0,68,c0bcfd8c,0,...) at _sleep+0x35b vm_daemon(0,e45fad38,c0bc15d4,322,c45a6d0c,...) at vm_daemon+0x59 fork_exit(c0a2f5e0,0,e45fad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45fad70, ebp = 0 --- Tracing command pagedaemon pid 8 tid 100043 td 0xc43dc690 sched_switch(c43dc690,0,104,180,37d85060,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,44,...) at mi_switch+0x200 sleepq_switch(c43dc690,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e8a3a4,44,c0bcfd8c,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e8a3a4,c0e8a370,44,c0bcfd8c,1388,...) at _sleep+0x329 vm_pageout(0,e45f7d38,c0bc15d4,322,c416329c,...) at vm_pageout+0x2bb fork_exit(c0a30480,0,e45f7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45f7d70, ebp = 0 --- Tracing command sctp_iterator pid 7 tid 100042 td 0xc43dc8c0 sched_switch(c43dc8c0,0,104,180,434d0ed2,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c43dc8c0,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0e7f900,0,c0bdb7cb,0,0,...) at sleepq_wait+0x63 _sleep(c0e7f900,c0e7f814,0,c0bdb7cb,0,...) at _sleep+0x35b sctp_iterator_thread(0,e45f4d38,c0bc15d4,322,c4163538,...) at sctp_iterator_thread+0x60 fork_exit(c08f6f90,0,e45f4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45f4d70, ebp = 0 --- Tracing command fdc0 pid 6 tid 100040 td 0xc43dcd20 sched_switch(c43dcd20,0,104,180,f2ce36de,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c43dcd20,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c428763c,4c,c0bbc376,0,0,...) at sleepq_timedwait+0x6b _sleep(c428763c,c42876f0,4c,c0bbc376,3e8,...) at _sleep+0x329 fdc_thread(c4287600,e45ebd38,c0bc15d4,322,c41637d4,...) at fdc_thread+0x2b8 fork_exit(c0ad5f30,c4287600,e45ebd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45ebd70, ebp = 0 --- Tracing command usbtask-dr pid 16 tid 100033 td 0xc43368c0 sched_switch(c43368c0,0,104,180,a57953c,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c43368c0,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0d0ec94,5c,c0bb858f,0,0,...) at sleepq_wait+0x63 _sleep(c0d0ec94,0,5c,c0bb858f,0,...) at _sleep+0x35b usb_task_thread(c0d0ec94,c3e4cd38,c0bc15d4,322,c4163a70,...) at usb_task_thread+0x62 fork_exit(c0761cc0,c0d0ec94,c3e4cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e4cd70, ebp = 0 --- Tracing command usbtask-hc pid 15 tid 100032 td 0xc4336af0 sched_switch(c4336af0,0,104,180,a576684,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4336af0,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0d0ec80,5c,c0bb858f,0,0,...) at sleepq_wait+0x63 _sleep(c0d0ec80,0,5c,c0bb858f,0,...) at _sleep+0x35b usb_task_thread(c0d0ec80,c3e49d38,c0bc15d4,322,c4163d0c,...) at usb_task_thread+0x62 fork_exit(c0761cc0,c0d0ec80,c3e49d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e49d70, ebp = 0 --- Tracing command usb0 pid 14 tid 100031 td 0xc4336d20 sched_switch(c4336d20,0,104,180,86db3f92,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4336d20,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c435c210,5c,c0bb859d,0,0,...) at sleepq_timedwait+0x6b _sleep(c435c210,0,5c,c0bb859d,ea60,...) at _sleep+0x329 usb_event_thread(c4354180,c3e46d38,c0bc15d4,322,c429e000,...) at usb_event_thread+0xca fork_exit(c0761e00,c4354180,c3e46d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e46d70, ebp = 0 --- Tracing command xpt_thrd pid 5 tid 100024 td 0xc41abd20 sched_switch(c41abd20,0,104,180,a56fa40,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c41abd20,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0cdddd4,4c,c0b69f3d,0,0,...) at sleepq_wait+0x63 _sleep(c0cdddd4,c0cdddec,4c,c0b69f3d,0,...) at _sleep+0x35b xpt_scanner_thread(0,c3e0ad38,c0bc15d4,322,c429e29c,...) at xpt_scanner_thread+0x41 fork_exit(c0479780,0,c3e0ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e0ad70, ebp = 0 --- Tracing command yarrow pid 13 tid 100017 td 0xc4166230 sched_switch(c4166230,0,104,180,3ab3dba6,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4166230,0,c0bc90d6,26a,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0d115c4,0,c0bbc376,2,0,...) at sleepq_timedwait+0x6b _sleep(c0d115c4,0,0,c0bbc376,64,...) at _sleep+0x329 pause(c0bbc376,64,c0bae111,112,0,...) at pause+0x47 random_kthread(0,c3df5d38,c0bc15d4,322,c429e538,...) at random_kthread+0x1d9 fork_exit(c06d0780,0,c3df5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3df5d70, ebp = 0 --- Tracing command g_down pid 4 tid 100015 td 0xc4166690 sched_switch(c4166690,0,104,180,3ab29aaa,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c4166690,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d0f3c4,4c,c0bbc376,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d0f3c4,c0d0f328,24c,c0bbc376,64,...) at _sleep+0x329 g_io_schedule_down(c4166690,0,c0bbda43,74,0,...) at g_io_schedule_down+0x6b g_down_procbody(0,c3defd38,c0bc15d4,322,c4162000,...) at g_down_procbody+0x8d fork_exit(c07a5d10,0,c3defd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3defd70, ebp = 0 --- Tracing command g_up pid 3 tid 100014 td 0xc41668c0 sched_switch(c41668c0,0,104,180,49e8eccc,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c41668c0,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d0f3c0,4c,c0bbc376,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d0f3c0,c0d0f348,24c,c0bbc376,64,...) at _sleep+0x329 g_io_schedule_up(c41668c0,0,c0bbda43,5d,0,...) at g_io_schedule_up+0x133 g_up_procbody(0,c3decd38,c0bc15d4,322,c416229c,...) at g_up_procbody+0x8d fork_exit(c07a5da0,0,c3decd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3decd70, ebp = 0 --- Tracing command g_event pid 2 tid 100013 td 0xc4166af0 sched_switch(c4166af0,0,104,180,4b72eb18,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c4166af0,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d0f3b8,4c,c0bbc376,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d0f3b8,0,4c,c0bbc376,64,...) at _sleep+0x329 g_event_procbody(0,c3de9d38,c0bc15d4,322,c4162538,...) at g_event_procbody+0xcb fork_exit(c07a5e30,0,c3de9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3de9d70, ebp = 0 --- Tracing command intr pid 12 tid 100041 td 0xc43dcaf0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100039 td 0xc43dd000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100038 td 0xc4327d20 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100037 td 0xc4336000 sched_switch(c4336000,0,109,180,a5141ec,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c415fd70,...) at mi_switch+0x200 ithread_loop(c43cdca0,e45d8d38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c43cdca0,e45d8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45d8d70, ebp = 0 --- Tracing command intr pid 12 tid 100036 td 0xc4336230 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100035 td 0xc4336460 sched_switch(c4336460,0,109,180,60af5142,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c41a9070,...) at mi_switch+0x200 ithread_loop(c43cd5c0,e45ced38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c43cd5c0,e45ced38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45ced70, ebp = 0 --- Tracing command intr pid 12 tid 100034 td 0xc4336690 sched_switch(c4336690,0,109,180,4b6e1890,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c41a8df0,...) at mi_switch+0x200 ithread_loop(c4373a80,e45b7d38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c4373a80,e45b7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe45b7d70, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc4337000 sched_switch(c4337000,0,109,180,a50e3f0,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c41a8e70,...) at mi_switch+0x200 ithread_loop(c4353370,c3e42d38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c4353370,c3e42d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e42d70, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc4337460 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100027 td 0xc41ab690 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100025 td 0xc41abaf0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100020 td 0xc4327690 sched_switch(c4327690,0,109,180,46e4ffda,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c426f370,...) at mi_switch+0x200 ithread_loop(c42d2ad0,c3dfed38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c42d2ad0,c3dfed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dfed70, ebp = 0 --- Tracing command intr pid 12 tid 100019 td 0xc43278c0 sched_switch(c43278c0,0,109,180,e21ef37e,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c426f3f0,...) at mi_switch+0x200 ithread_loop(c42d2ae0,c3dfbd38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c42d2ae0,c3dfbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dfbd70, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc4166d20 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100011 td 0xc41ab000 sched_switch(c41ab000,0,109,180,3c871fe2,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c41a98f0,...) at mi_switch+0x200 ithread_loop(c41610c0,c3de3d38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c41610c0,c3de3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3de3d70, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc41ab230 sched_switch(c41ab230,0,109,180,49e8e2fc,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c41a9970,...) at mi_switch+0x200 ithread_loop(c41610d0,c3de0d38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c41610d0,c3de0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3de0d70, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc41ab460 sched_switch(c41ab460,0,109,180,4b729388,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c41a99f0,...) at mi_switch+0x200 ithread_loop(c41610e0,c3dddd38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c41610e0,c3dddd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dddd70, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc4164000 cpustop_handler(1,c3ddaadc,c0b173cd,1f,f8,...) at cpustop_handler+0x32 ipi_nmi_handler(1f,f8,8,c3ddaa78,c41627d4,...) at ipi_nmi_handler+0x2f trap(c3ddaae8) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0819d49, esp = 0xc3ddab28, ebp = 0xc3ddab2c --- sched_tick(c4164000,0,c0bbff82,11e,ae56b959,...) at sched_tick+0x19 hardclock_cpu(0,c3ddab84,c07c2400,c0d15500,0,...) at hardclock_cpu+0x12f hardclock(0,c07ec599,0,c0d11030,0,...) at hardclock+0x22 lapic_handle_timer(c3ddabac) at lapic_handle_timer+0x9c Xtimerint() at Xtimerint+0x1f --- interrupt, eip = 0xc07ec599, esp = 0xc3ddabec, ebp = 0xc3ddac08 --- _mtx_lock_sleep(c0d11030,c4164000,0,c0bc3c47,89,...) at _mtx_lock_sleep+0x99 _mtx_lock_flags(c0d11030,0,c0bc3c47,89,c3ddaca0,...) at _mtx_lock_flags+0xf7 lock_mtx(c0d11030,1,c0bc644e,16b,c0d117f4,...) at lock_mtx+0x29 softclock(c0d117c0,c3ddacc8,c07ebfd4,c0d15500,c41a9a38,...) at softclock+0x1e5 intr_event_execute_handlers(c41627d4,c41a9a00,c0bc185a,4dd,c41a9a70,...) at intr_event_execute_handlers+0x125 ithread_loop(c41610f0,c3ddad38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x9f fork_exit(c07db9f0,c41610f0,c3ddad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3ddad70, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc4164230 sched_switch(c4164230,0,109,180,5be98de2,...) at sched_switch+0x437 mi_switch(109,0,c0bc185a,4d7,c41a9af0,...) at mi_switch+0x200 ithread_loop(c4161100,c3dd7d38,c0bc15d4,322,c41627d4,...) at ithread_loop+0x1f6 fork_exit(c07db9f0,c4161100,c3dd7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dd7d70, ebp = 0 --- Tracing command idle pid 11 tid 100006 td 0xc4164460 sched_switch(c4164460,0,108,17b,40ba1c64,...) at sched_switch+0x437 mi_switch(108,0,c0bc6c0c,3a1,c4164460,...) at mi_switch+0x200 sched_idletd(0,c3dd1d38,c0bc15d4,322,c4162a70,...) at sched_idletd+0x178 fork_exit(c081afa0,0,c3dd1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dd1d70, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc4164690 sched_switch(c4164690,0,108,17b,3f4ad370,...) at sched_switch+0x437 mi_switch(108,0,c0bc6c0c,3a1,c4164690,...) at mi_switch+0x200 sched_idletd(0,c3dced38,c0bc15d4,322,c4162a70,...) at sched_idletd+0x178 fork_exit(c081afa0,0,c3dced38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dced70, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc41648c0 cpustop_handler(4,c3dcbc40,c0b173cd,46,c0d11830,...) at cpustop_handler+0x32 ipi_nmi_handler(46,c0d11830,c3dcbbd8,c07ebfd4,c4162a70,...) at ipi_nmi_handler+0x2f trap(c3dcbc4c) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc081af88, esp = 0xc3dcbc8c, ebp = 0xc3dcbcb4 --- cpu_search_highest(c0d23d20,c3dcbcd8,c0bc6c0c,3a3,c41648c0,...) at cpu_search_highest+0x108 sched_idletd(0,c3dcbd38,c0bc15d4,322,c4162a70,...) at sched_idletd+0xcf fork_exit(c081afa0,0,c3dcbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dcbd70, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc4164af0 cpustop_handler(8,c3dc8c10,c0b173cd,1ca,3,...) at cpustop_handler+0x32 ipi_nmi_handler(1ca,3,53cdfbf,c4164af0,c4162a70,...) at ipi_nmi_handler+0x2f trap(c3dc8c1c) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc081af66, esp = 0xc3dc8c5c, ebp = 0xc3dc8c84 --- cpu_search_highest(c0d23d20,c3dc8c98,b,fffffff7,6,...) at cpu_search_highest+0xe6 cpu_search_highest(c0d23d00,c3dc8cd8,c0bc6c0c,3a3,c4164af0,...) at cpu_search_highest+0x71 sched_idletd(0,c3dc8d38,c0bc15d4,322,c4162a70,...) at sched_idletd+0xcf fork_exit(c081afa0,0,c3dc8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dc8d70, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc4164d20 sched_switch(c4164d20,0,104,180,e6d02238,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c4164d20,0,c0bc90d6,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bc90d6,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c4162d0c,5c,c0bcb7a1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4162d0c,c4162d94,15c,c0bcb7a1,0,...) at _sleep+0x344 kern_wait(c4164d20,ffffffff,c3dc4c74,0,0,...) at kern_wait+0xab6 wait4(c4164d20,c3dc4cf8,10,c0bcb5bc,c0c9e768,...) at wait4+0x3b syscall(c3dc4d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x8054d4f, esp = 0xbfbfe95c, ebp = 0xbfbfe978 --- Tracing command audit pid 10 tid 100001 td 0xc4166000 sched_switch(c4166000,0,104,180,a518474,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4166000,0,c0bc90d6,247,c4166000,...) at sleepq_switch+0x15f sleepq_wait(c0e895c0,0,c3dc1c9c,1,0,...) at sleepq_wait+0x63 _cv_wait(c0e895c0,c0e895a4,c0be5d26,18d,0,...) at _cv_wait+0x240 audit_worker(0,c3dc1d38,c0bc15d4,322,c4163000,...) at audit_worker+0x84 fork_exit(c09de5e0,0,c3dc1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3dc1d70, ebp = 0 --- Tracing command kernel pid 0 tid 100029 td 0xc4337230 sched_switch(c4337230,0,104,180,a508828,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4337230,0,c0bc90d6,247,c4337230,...) at sleepq_switch+0x15f sleepq_wait(c4354d40,0,c0bc5c38,c0bbc376,0,...) at sleepq_wait+0x63 msleep_spin(c4354d40,c4354d5c,c0bbc376,0,c0bc3c47,...) at msleep_spin+0x21d taskqueue_thread_loop(c4321b6c,c3e3fd38,c0bc15d4,322,c0d0f480,...) at taskqueue_thread_loop+0x8a fork_exit(c08316c0,c4321b6c,c3e3fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e3fd70, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc41ab8c0 sched_switch(c41ab8c0,0,104,180,a8b7cec,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c41ab8c0,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c4315e80,0,c0bbc376,0,0,...) at sleepq_wait+0x63 _sleep(c4315e80,c4315e9c,0,c0bbc376,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d0fbc4,c3e10d38,c0bc15d4,322,c0d0f480,...) at taskqueue_thread_loop+0xb4 fork_exit(c08316c0,c0d0fbc4,c3e10d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e10d70, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc4327000 sched_switch(c4327000,0,104,180,a8b4e0c,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4327000,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c4316100,0,c0bbc376,0,0,...) at sleepq_wait+0x63 _sleep(c4316100,c431611c,0,c0bbc376,0,...) at _sleep+0x35b taskqueue_thread_loop(c0ce0a84,c3e07d38,c0bc15d4,322,c0d0f480,...) at taskqueue_thread_loop+0xb4 fork_exit(c08316c0,c0ce0a84,c3e07d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e07d70, ebp = 0 --- Tracing command kernel pid 0 tid 100022 td 0xc4327230 sched_switch(c4327230,0,104,180,a8b1ff0,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4327230,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c4316100,0,c0bbc376,0,0,...) at sleepq_wait+0x63 _sleep(c4316100,c431611c,0,c0bbc376,0,...) at _sleep+0x35b taskqueue_thread_loop(c0ce0a84,c3e04d38,c0bc15d4,322,c0d0f480,...) at taskqueue_thread_loop+0xb4 fork_exit(c08316c0,c0ce0a84,c3e04d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e04d70, ebp = 0 --- Tracing command kernel pid 0 tid 100021 td 0xc4327460 sched_switch(c4327460,0,104,180,a8af310,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4327460,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c4316100,0,c0bbc376,0,0,...) at sleepq_wait+0x63 _sleep(c4316100,c431611c,0,c0bbc376,0,...) at _sleep+0x35b taskqueue_thread_loop(c0ce0a84,c3e01d38,c0bc15d4,322,c0d0f480,...) at taskqueue_thread_loop+0xb4 fork_exit(c08316c0,c0ce0a84,c3e01d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3e01d70, ebp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xc4327af0 sched_switch(c4327af0,0,104,180,a879844,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4327af0,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c4316400,0,c0bbc376,0,0,...) at sleepq_wait+0x63 _sleep(c4316400,c431641c,0,c0bbc376,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d23f08,c3df8d38,c0bc15d4,322,c0d0f480,...) at taskqueue_thread_loop+0xb4 fork_exit(c08316c0,c0d23f08,c3df8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3df8d70, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc4166460 sched_switch(c4166460,0,104,180,a465e5c,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,0,...) at mi_switch+0x200 sleepq_switch(c4166460,0,c0bc90d6,247,0,...) at sleepq_switch+0x15f sleepq_wait(c414a840,0,c0bbc376,0,0,...) at sleepq_wait+0x63 _sleep(c414a840,c414a85c,0,c0bbc376,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d22a40,c3df2d38,c0bc15d4,322,c0d0f480,...) at taskqueue_thread_loop+0xb4 fork_exit(c08316c0,c0d22a40,c3df2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3df2d70, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc0d0f720 sched_switch(c0d0f720,0,104,180,39a30e06,...) at sched_switch+0x437 mi_switch(104,0,c0bc90d6,1d2,44,...) at mi_switch+0x200 sleepq_switch(c0d0f720,0,c0bc90d6,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d0f480,44,c0bc7359,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d0f480,0,44,c0bc7359,2710,...) at _sleep+0x329 scheduler(0,101ec00,101ec00,101e000,1025000,...) at scheduler+0x23e mi_startup() at mi_startup+0x96 begin() at begin+0x2c db> call doadump Physical memory: 1011 MB Dumping 110 MB: 95 79 63 47 31 15 Dump complete = 0xf db> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 1 (kgdb) bt #0 doadump () at pcpu.h:221 #1 0xc04bc049 in db_fncall (dummy1=0xc05b0b88, dummy2=0x0, dummy3=0xe6e21694, dummy4=0xe6e21684 "À\204ÎÀê~¼ÀØ") at ../../../ddb/db_command.c:548 #2 0xc04bc441 in db_command (last_cmdp=0xc0cded5c, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:445 #3 0xc04bc59a in db_command_loop () at ../../../ddb/db_command.c:498 #4 0xc04be3fd in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:229 #5 0xc0826b86 in kdb_trap (type=0x3, code=0x0, tf=0xe6e2182c) at ../../../kern/subr_kdb.c:534 #6 0xc0b1796b in trap (frame=0xe6e2182c) at ../../../i386/i386/trap.c:694 #7 0xc0afb87b in calltrap () at ../../../i386/i386/exception.s:165 #8 0xc0826d0a in kdb_enter (why=0xc0bc50cf "panic", msg=0xc0bc50cf "panic") at cpufunc.h:70 #9 0xc07f9926 in panic (fmt=0xc0bc2428 "Lock %s not %slocked @ %s:%d\n") at ../../../kern/kern_shutdown.c:559 #10 0xc07e4222 in _lockmgr_assert (lk=0xc4729b18, what=0x1, file=0xc0bd0d18 "../../../kern/vfs_default.c", line=0x143) at ../../../kern/kern_lock.c:977 #11 0xc07e548a in __lockmgr_args (lk=0xc4729b18, flags=0x100100, ilk=0xc4729b80, wmesg=0x0, pri=0x0, timo=0x0, file=0xc0bd0d18 "../../../kern/vfs_default.c", line=0x143) at ../../../kern/kern_lock.c:659 #12 0xc0872645 in vop_stdunlock (ap=0xe6e219c8) at lockmgr.h:93 #13 0xc0b24b65 in VOP_UNLOCK_APV (vop=0xc0caedc0, a=0xe6e219c8) at vnode_if.c:1667 #14 0xc5723b92 in null_unlock (ap=0xe6e21a70) at vnode_if.h:862 #15 0xc0b24b65 in VOP_UNLOCK_APV (vop=0xc5724240, a=0xe6e21a70) at vnode_if.c:1667 #16 0xc5722ed9 in nullfs_mount (mp=0xc5362000, td=0xc5475690) at vnode_if.h:862 #17 0xc087a808 in vfs_donmount (td=0xc5475690, fsflags=0x0, fsoptions=0xc629bd00) at ../../../kern/vfs_mount.c:1014 #18 0xc087bafe in nmount (td=0xc5475690, uap=0xe6e21cf8) at ../../../kern/vfs_mount.c:417 #19 0xc0b170c3 in syscall (frame=0xe6e21d38) at ../../../i386/i386/trap.c:1090 #20 0xc0afb8e0 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261 #21 0x00000033 in ?? () (kgdb) f 16 #16 0xc5722ed9 in nullfs_mount (mp=0xc5362000, td=0xc5475690) at vnode_if.h:862 862 return (VOP_UNLOCK_APV(vp->v_op, &a)); (kgdb) p *mp $4 = {mnt_lock = {lock_object = {lo_name = 0xc0bd1323 "vfslock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc4122fe0}, lk_lock = 0x9, lk_timo = 0x0, lk_pri = 0x50, lk_stack = {depth = 0x7, pcs = {0xc07e4de2, 0xc0880fa3, 0xc08791f6, 0xc087a74b, 0xc087bafe, 0xc0b170c3, 0xc0afb8e0, 0x0 }}}, mnt_mtx = {lock_object = { lo_name = 0xc0bcb053 "struct mount mtx", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc411e1e0}, mtx_lock = 0x4}, mnt_gen = 0xd1, mnt_list = {tqe_next = 0x0, tqe_prev = 0x0}, mnt_op = 0xc5724200, mnt_vfc = 0xc57241c0, mnt_vnodecovered = 0xc6455c18, mnt_syncer = 0x0, mnt_ref = 0x1, mnt_nvnodelist = {tqh_first = 0x0, tqh_last = 0xc536209c}, mnt_nvnodelistsize = 0x0, mnt_writeopcount = 0x0, mnt_kern_flag = 0x0, mnt_flag = 0x0, mnt_noasync = 0x0, mnt_opt = 0x0, mnt_optnew = 0xc9c026a0, mnt_maxsymlinklen = 0x0, mnt_stat = {f_version = 0x0, f_type = 0x8, f_flags = 0x0, f_bsize = 0x0, f_iosize = 0x0, f_blocks = 0x0, f_bfree = 0x0, f_bavail = 0x0, f_files = 0x0, f_ffree = 0x0, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0x0, f_owner = 0x0, f_fsid = {val = {0x0, 0x0}}, f_charspare = '\0' , f_fstypename = "nullfs\000\000\000\000\000\000\000\000\000", f_mntfromname = '\0' , f_mntonname = "/home/pho/stress2/misc/2", '\0' }, mnt_cred = 0xc4ac0600, mnt_data = 0x0, mnt_time = 0x0, mnt_iosize_max = 0x10000, mnt_export = 0x0, mnt_label = 0x0, mnt_hashseed = 0x49704df2, mnt_markercnt = 0x0, mnt_holdcnt = 0x0, mnt_holdcntwaiters = 0x0, mnt_secondary_writes = 0x0, mnt_secondary_accwrites = 0x0, mnt_susp_owner = 0x0, mnt_gjprovider = 0x0, mnt_explock = {lock_object = {lo_name = 0xc0bd132b "explock", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc4123048}, lk_lock = 0x1, lk_timo = 0x0, lk_pri = 0x50, lk_stack = {depth = 0x6, pcs = {0xc07e53e0, 0xc0872e1d, 0xc087a85a, 0xc087bafe, 0xc0b170c3, 0xc0afb8e0, 0x0 }}}} -- testscenario: nullfs.sh