GENERIC HEAD from 2008-11-20 03:34:36 UTC, svn Revision: 185120, vmcore.10 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2008 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 8.0-CURRENT #0 r185120M: Thu Nov 20 08:36:07 CET 2008 pho@x4.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1799.99-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff,,,Prefetch,,> TSC: P-state invariant Cores per package: 4 real memory = 3623682048 (3455 MB) avail memory = 3540017152 (3376 MB) : Trying to mount root from ufs:/dev/ad4s1a lock order reversal: 1st 0xc6d6a044 user map (user map) @ vm/vm_map.c:3115 2nd 0xc715f9c0 ufs (ufs) @ kern/vfs_subr.c:2053 KDB: stack backtrace: db_trace_self_wrapper(c0bd2b04,c6a8f914,c0841ab5,4,c0bce135,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bce135,c6d1f728,c6d243f0,c6a8f970,...) at kdb_backtrace+0x29 _witness_debugger(c0bd575f,c715f9c0,c0bc9631,c6d243f0,c0bdc4b1,...) at _witness_debugger+0x25 witness_checkorder(c715f9c0,1,c0bdc4a8,805,0,...) at witness_checkorder+0x839 __lockmgr_args(c715f9c0,200501,c715fa28,0,0,...) at __lockmgr_args+0x237 ffs_lock(c6a8fa78,c0bf83e2,c0bc8d5e,200501,c715f968,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0ccffc0,c6a8fa78,c0ceb9e0,c715f968,200501,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c715f968,200501,c0bdc4a8,805,4,...) at _vn_lock+0x78 vget(c715f968,200501,c6d66d80,4b4,0,...) at vget+0xc9 vnode_pager_lock(c1875d14,0,c0bf59f3,127,c6a8fc18,...) at vnode_pager_lock+0x1e3 vm_fault(c6d6a000,80db000,2,8,80db0c0,...) at vm_fault+0x1df trap_pfault(5,0,c0c057df,2e7,c6d64d0c,...) at trap_pfault+0x118 trap(c6a8fd38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80480e5, esp = 0xbfbfeef0, ebp = 0xbfbfef10 --- Entropy harvesting: interrupts ethernet point_to_point kickstart. /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 800127 free (1855 frags, 99784 blocks, 0.2% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 50734114 free (194 frags, 6341740 blocks, 0.0% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 18182492 free (64004 frags, 2264811 blocks, 0.3% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 44360769 free (2793 frags, 5544747 blocks, 0.0% fragmentation) re0: link state changed to DOWN Starting Network: lo0 re0. add net default: gateway 192.168.1.1 Additional ABI support: linux. lock order reversal: 1st 0xc73a5460 ufs (ufs) @ kern/vfs_subr.c:2053 2nd 0xdad12668 bufwait (bufwait) @ ufs/ffs/ffs_softdep.c:6150 3rd 0xc75bf460 ufs (ufs) @ kern/vfs_subr.c:2053 KDB: stack backtrace: db_trace_self_wrapper(c0bd2b04,e99973f0,c0841ab5,4,c0bce135,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bce135,c6d216d8,c6d243f0,e999744c,...) at kdb_backtrace+0x29 _witness_debugger(c0bd5778,c75bf460,c0bc9631,c6d243f0,c0bdc4b1,...) at _witness_debugger+0x25 witness_checkorder(c75bf460,9,c0bdc4a8,805,0,...) at witness_checkorder+0x839 __lockmgr_args(c75bf460,80100,c75bf4c8,0,0,...) at __lockmgr_args+0x7a7 ffs_lock(e9997554,c0bdba9c,c0bc8d5e,80100,c75bf408,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0ccffc0,e9997554,c0ceb9e0,c75bf408,80100,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c75bf408,80100,c0bdc4a8,805,4,...) at _vn_lock+0x78 vget(c75bf408,80100,c6fcd240,50,0,...) at vget+0xc9 vfs_hash_get(c74a4598,79d400,80000,c6fcd240,e99976b0,...) at vfs_hash_get+0xed ffs_vgetf(c74a4598,79d400,80000,e99976b0,1,...) at ffs_vgetf+0x49 softdep_sync_metadata(c73a5408,0,c0bf3e54,131,0,...) at softdep_sync_metadata+0x5ba ffs_syncvnode(c73a5408,1,c07f4564,c0d330dc,4,...) at ffs_syncvnode+0x3e2 ffs_truncate(c73a5408,400,0,880,c6d6b400,...) at ffs_truncate+0x616 ufs_direnter(c73a5408,c75bf408,e9997a20,e9997c04,dad12adc,...) at ufs_direnter+0x946 ufs_mkdir(e9997c28,c0c081e6,0,e9997c28,e9997bd8,...) at ufs_mkdir+0x977 VOP_MKDIR_APV(c0ccffc0,e9997c28,eaa,ea8,0,...) at VOP_MKDIR_APV+0xc5 kern_mkdirat(c6fcd240,ffffff9c,bfbfef66,0,1ff,...) at kern_mkdirat+0x278 kern_mkdir(c6fcd240,bfbfef66,0,1ff,e9997d2c,...) at kern_mkdir+0x2e mkdir(c6fcd240,e9997cf8,8,c0bd5ff1,c0cad620,...) at mkdir+0x29 syscall(e9997d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (136, FreeBSD ELF32, mkdir), eip = 0x2815aed3, esp = 0xbfbfedac, ebp = 0xbfbfee78 --- Starting mountd. Configuring syscons: keymap blanktime. Thu Nov 20 08:48:10 CET 2008 lock order reversal: 1st 0xdad2ec40 bufwait (bufwait) @ kern/vfs_bio.c:2443 2nd 0xc77d3a00 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:263 KDB: stack backtrace: db_trace_self_wrapper(c0bd2b04,e9adaa78,c0841ab5,4,c0bce135,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bce135,c6d216d8,c6d24458,e9adaad4,...) at kdb_backtrace+0x29 _witness_debugger(c0bd575f,c77d3a00,c0bf44f9,c6d24458,c0bf419f,...) at _witness_debugger+0x25 witness_checkorder(c77d3a00,9,c0bf4196,107,0,...) at witness_checkorder+0x839 _sx_xlock(c77d3a00,0,c0bf4196,107,dba61018,...) at _sx_xlock+0x85 ufsdirhash_acquire(0,e,c7253800,dad2ebe0,dba61018,...) at ufsdirhash_acquire+0x48 ufsdirhash_remove(c7389e10,dba61018,18,e9adab64,e9adab60,...) at ufsdirhash_remove+0x14 ufs_dirremove(c783b000,c77dc3c0,500800c,1,c78deac0,...) at ufs_dirremove+0xe5 ufs_rmdir(e9adac30,c0c081a5,0,e9adac30,c78deac0,...) at ufs_rmdir+0xd6 VOP_RMDIR_APV(c0ccffc0,e9adac30,2,0,81471b8,...) at VOP_RMDIR_APV+0xf6 kern_rmdirat(c78d6d80,ffffff9c,81471b8,0,e9adac80,...) at kern_rmdirat+0x1cc kern_rmdir(c78d6d80,81471b8,0,e9adad2c,c0b205b3,...) at kern_rmdir+0x27 rmdir(c78d6d80,e9adacf8,4,c0beec03,c0cad638,...) at rmdir+0x22 syscall(e9adad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280d7a5b, esp = 0xbfbfe9ec, ebp = 0xbfbfea08 --- Limiting icmp unreach response from 106644 to 200 packets/sec Limiting icmp unreach response from 102282 to 200 packets/sec Limiting icmp unreach response from 101926 to 200 packets/sec : Limiting icmp unreach response from 80034 to 200 packets/sec Limiting icmp unreach response from 44745 to 200 packets/sec Limiting icmp unreach response from 61485 to 200 packets/sec Nov 20 14:07:53 x4 su: pho to root on /dev/pts/0 lock order reversal: 1st 0xdad66b10 bufwait (bufwait) @ nfsclient/nfs_vnops.c:2836 2nd 0xc800041c so_snd_sx (so_snd_sx) @ kern/uipc_sockbuf.c:148 KDB: stack backtrace: db_trace_self_wrapper(c0bd2b04,e9dd16dc,c0841ab5,4,c0bce135,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bce135,c6d216d8,c6d24528,e9dd1738,...) at kdb_backtrace+0x29 _witness_debugger(c0bd575f,c800041c,c0bd9800,c6d24528,c0bd94d0,...) at _witness_debugger+0x25 witness_checkorder(c800041c,9,c0bd94c7,94,0,...) at witness_checkorder+0x839 _sx_xlock(c800041c,40,c0bd94c7,94,e9dd17d0,...) at _sx_xlock+0x85 sblock(c80003e8,1,c0bc8d5e,3,c8057900,...) at sblock+0x85 sosend_generic(c8000310,0,0,c7b3f400,0,...) at sosend_generic+0xc2 sosend(c8000310,0,0,c7b3f400,0,...) at sosend+0x3f clnt_vc_call(c888a740,e9dd1934,15,c7b3f900,e9dd1970,...) at clnt_vc_call+0x43c clnt_reconnect_call(c8eab180,e9dd1934,15,c7b3f900,e9dd1970,...) at clnt_reconnect_call+0x5a0 nfs_request(c7245968,c7b3f900,15,c8057900,c9a9e000,...) at nfs_request+0x1dd nfs_commit(c7245968,0,0,22,c9a9e000,...) at nfs_commit+0x172 nfs_flush(e9dd1aec,c0b2bf25,e9dd1b1c,c0c07991,0,...) at nfs_flush+0x4b8 nfs_fsync(e9dd1b1c,c0c07991,0,0,e9dd1ba8,...) at nfs_fsync+0x16 VOP_FSYNC_APV(c0cccd40,e9dd1b1c,c0bee9d6,663,e9dd1b24,...) at VOP_FSYNC_APV+0xc5 nfs_rename(e9dd1c1c,c0c076eb,e9dd1bcc,e9dd1b7c,e9dd1bcc,...) at nfs_rename+0xe1 VOP_RENAME_APV(c0cccd40,e9dd1c1c,101,c1894b88,5009c10,...) at VOP_RENAME_APV+0xa5 kern_renameat(c8057900,ffffff9c,bfbfe540,ffffff9c,bfbfeb23,...) at kern_renameat+0x2b7 kern_rename(c8057900,bfbfe540,bfbfeb23,0,e9dd1d2c,...) at kern_rename+0x36 rename(c8057900,e9dd1cf8,8,c0beec03,c0cad560,...) at rename+0x29 syscall(e9dd1d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (128, FreeBSD ELF32, rename), eip = 0x280cfabb, esp = 0xbfbfe51c, ebp = 0xbfbfe948 --- lock order reversal: 1st 0xc8b2a5b8 ufs (ufs) @ kern/vfs_mount.c:1207 2nd 0xc84469c0 syncer (syncer) @ kern/vfs_subr.c:2153 KDB: stack backtrace: db_trace_self_wrapper(c0bd2b04,e9b72a64,c0841ab5,4,c0bce135,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bce135,c6d243f0,c6d244c0,e9b72ac0,...) at kdb_backtrace+0x29 _witness_debugger(c0bd575f,c84469c0,c0bdc60e,c6d244c0,c0bdc4b1,...) at _witness_debugger+0x25 witness_checkorder(c84469c0,9,c0bdc4a8,869,0,...) at witness_checkorder+0x839 __lockmgr_args(c84469c0,80100,c8446a28,0,0,...) at __lockmgr_args+0x7a7 vop_stdlock(e9b72bc4,c0bdc4b1,c712580c,80100,c8446968,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0cbd7a0,e9b72bc4,c0ceb9e0,c8446968,80100,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c8446968,80100,c0bdc4a8,869,0,...) at _vn_lock+0x78 vrele(c8446968,0,c0bdbe02,4fc,1,...) at vrele+0x142 dounmount(c7abd000,8000000,c7339480,482,4,...) at dounmount+0x3ce unmount(c7339480,e9b72cf8,8,e9b72d38,c0cacb70,...) at unmount+0x2e0 syscall(e9b72d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d1fff, esp = 0xbfbfe1dc, ebp = 0xbfbfe2a8 --- lock order reversal: 1st 0xc84109c0 ufs (ufs) @ kern/vfs_mount.c:1207 2nd 0xc9a21b18 devfs (devfs) @ ufs/ffs/ffs_vfsops.c:1325 KDB: stack backtrace: db_trace_self_wrapper(c0bd2b04,e99d3a2c,c0841ab5,4,c0bce135,...) at db_trace_self_wrapper+0x26 kdb_backtrace(4,c0bce135,c6d243f0,c6d24320,e99d3a88,...) at kdb_backtrace+0x29 _witness_debugger(c0bd575f,c9a21b18,c0bc5a55,c6d24320,c0bf3994,...) at _witness_debugger+0x25 witness_checkorder(c9a21b18,9,c0bf398b,52d,c9a21b80,...) at witness_checkorder+0x839 __lockmgr_args(c9a21b18,80400,c9a21b80,0,0,...) at __lockmgr_args+0x7a7 vop_stdlock(e99d3b8c,c73f20a4,c0e5d500,80400,c9a21ac0,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0ca9480,e99d3b8c,c0ceb9e0,c9a21ac0,80400,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c9a21ac0,80400,c0bf398b,52d,c9946e30,...) at _vn_lock+0x78 ffs_sync(ce0812cc,1,c73f2000,c73f2000,1,...) at ffs_sync+0x3c5 dounmount(ce0812cc,8080000,c73f2000,482,f3d70e50,...) at dounmount+0x45c unmount(c73f2000,e99d3cf8,8,e99d3d38,c0cacb70,...) at unmount+0x2e0 syscall(e99d3d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d1fff, esp = 0xbfbfe1dc, ebp = 0xbfbfe2a8 --- Nov 20 14:12:54 x4 mountd[878]: can't delete exports for /mnt19: Invalid argument Nov 20 14:12:54 x4 mountd[878]: can't delete exports for /mnt10: Invalid argument Nov 20 14:12:57 x4 mountd[878]: can't delete exports for /mnt9: Invalid argument Nov 20 14:12:57 x4 mountd[878]: can't delete exports for /mnt14: Invalid argument Nov 20 14:12:57 x4 mountd[878]: can't delete exports for /mnt12: Invalid argument Nov 20 14:12:59 x4 mountd[878]: can't delete exports for /mnt5: Device busy Nov 20 14:12:59 x4 mountd[878]: can't delete exports for /mnt8: Invalid argument Nov 20 14:12:59 x4 mountd[878]: can't delete exports for /mnt9: Invalid argument Nov 20 14:12:59 x4 mountd[878]: can't delete exports for /mnt12: Invalid argument Nov 20 14:12:59 x4 mountd[878]: can't delete exports for /mnt10: Invalid argument Nov 20 14:13:00 x4 mountd[878]: can't delete exports for /mnt15: Invalid argument Nov 20 14:13:00 x4 mountd[878]: can't delete exports for /mnt17: Invalid argument Nov 20 14:13:00 x4 mountd[878]: can't delete exports for /mnt18: Invalid argument Nov 20 14:13:00 x4 mountd[878]: can't delete exports for /mnt13: Invalid argument Nov 20 14:13:00 x4 mountd[878]: can't delete exports for /mnt16: Invalid argument Nov 20 14:13:00 x4 mountd[878]: can't delete exports for /mnt14: Invalid argument Nov 20 14:13:04 x4 mountd[878]: can't delete exports for /mnt17: Invalid argument Nov 20 14:13:04 x4 mountd[878]: can't delete exports for /mnt6: Invalid argument Nov 20 14:13:07 x4 mountd[878]: can't delete exports for /mnt10: Device busy Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt11: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt12: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt18: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt5: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt8: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt9: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt15: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt7: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt14: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt6: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt19: Invalid argument Nov 20 14:13:08 x4 mountd[878]: can't delete exports for /mnt16: Invalid argument Nov 20 14:13:09 x4 mountd[878]: can't delete exports for /mnt6: Invalid argument Nov 20 14:13:09 x4 mountd[878]: can't delete exports for /mnt11: Invalid argument Nov 20 14:13:09 x4 mountd[878]: can't delete exports for /mnt9: Invalid argument fsync: giving up on dirty 0xc95ee408: tag devfs, type VCHR usecount 1, writecount 0, refcount 5 mountedhere 0xc986e900 flags () v_object 0xc7e4dc98 ref 0 pages 6 lock type devfs: EXCL by thread 0xc798f900 (pid 54197) #0 0xc07ed8f0 at __lockmgr_args+0xb80 #1 0xc087b7d5 at vop_stdlock+0x65 #2 0xc0b2e305 at VOP_LOCK1_APV+0xb5 #3 0xc08976d8 at _vn_lock+0x78 #4 0xc0a08065 at ffs_sync+0x3c5 #5 0xc0884fec at dounmount+0x45c #6 0xc0885590 at unmount+0x2e0 #7 0xc0b205b3 at syscall+0x2a3 #8 0xc0b04d00 at Xint0x80_syscall+0x20 dev md5c Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt6: Invalid argument Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt12: Invalid argument Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt13: Invalid argument Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt16: Invalid argument Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt18: Invalid argument Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt8: Invalid argument Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt15: Invalid argument Nov 20 14:13:16 x4 mountd[878]: can't delete exports for /mnt11: Invalid argument : Nov 20 14:28:53 x4 mountd[878]: can't delete exports for /mnt5: Invalid argument Nov 20 14:28:53 x4 mountd[878]: can't delete exports for /mnt16: Invalid argument Nov 20 14:28:53 x4 mountd[878]: can't delete exports for /mnt6: Invalid argument Nov 20 14:57:02 x4 mountd[878]: can't delete exports for /mnt8: Invalid argument Nov 20 14:57:02 x4 mountd[878]: can't delete exports for /mnt14: Invalid argument Nov 20 14:57:02 x4 mountd[878]: can't delete exports for /mnt18: Invalid argument Nov 20 14:57:02 x4 mountd[878]: can't delete exports for /mnt11: Invalid argument Nov 20 14:57:04 x4 mountd[878]: can't delete exports for /mnt14: Invalid argument Nov 20 14:57:04 x4 mountd[878]: can't delete exports for /mnt10: Invalid argument Nov 20 14:57:10 x4 mountd[878]: can't delete exports for /mnt10: Invalid argument Nov 20 14:57:14 x4 mountd[878]: can't delete exports for /mnt13: Invalid argument Nov 20 14:57:15 x4 mountd[878]: can't delete exports for /mnt17: Invalid argument Nov 20 14:57:19 x4 mountd[878]: can't delete exports for /mnt8: Device busy Nov 20 14:57:21 x4 mountd[878]: can't delete exports for /mnt6: Device busy fsync: giving up on dirty 0xc957e6b8: tag devfs, type VCHR usecount 1, writecount 0, refcount 5 mountedhere 0xcef41e00 flags () v_object 0xcd7896c8 ref 0 pages 6 lock type devfs: EXCL by thread 0xc7339b40 (pid 98900) #0 0xc07ed8f0 at __lockmgr_args+0xb80 #1 0xc087b7d5 at vop_stdlock+0x65 #2 0xc0b2e305 at VOP_LOCK1_APV+0xb5 #3 0xc08976d8 at _vn_lock+0x78 #4 0xc0a08065 at ffs_sync+0x3c5 #5 0xc0884fec at dounmount+0x45c #6 0xc0885590 at unmount+0x2e0 #7 0xc0b205b3 at syscall+0x2a3 #8 0xc0b04d00 at Xint0x80_syscall+0x20 dev md6c Nov 20 14:57:36 x4 mountd[878]: can't delete exports for /mnt10: Invalid argument Nov 20 14:57:36 x4 mountd[878]: can't delete exports for /mnt7: Invalid argument Nov 20 14:57:36 x4 mountd[878]: can't delete exports for /mnt19: Invalid argument Nov 20 14:57:36 x4 mountd[878]: can't delete exports for /mnt14: Invalid argument : Nov 20 15:29:43 x4 mountd[878]: can't delete exports for /mnt8: Invalid argument Nov 20 15:29:49 x4 mountd[878]: can't delete exports for /mnt5: Invalid argument Nov 20 15:29:49 x4 mountd[878]: can't delete exports for /mnt13: Invalid argument Nov 20 15:29:51 x4 mountd[878]: can't delete exports for /mnt8: Invalid argument Nov 20 15:29:51 x4 mountd[878]: can't delete exports for /mnt15: Invalid argument Nov 20 15:29:51 x4 mountd[878]: can't delete exports for /mnt5: Invalid argument Nov 20 15:29:53 x4 mountd[878]: can't delete exports for /mnt12: Invalid argument Nov 20 15:29:53 x4 mountd[878]: can't delete exports for /mnt5: Invalid argument Nov 20 15:29:55 x4 mountd[878]: can't delete exports for /mnt17: Invalid argument Nov 20 15:29:56 x4 mountd[878]: can't delete exports for /mnt9: Invalid argument Nov 20 15:30:03 x4 mountd[878]: can't delete exports for /mnt5: Invalid argument Nov 20 15:30:09 x4 mountd[878]: can't delete exports for /mnt6: Invalid argument Nov 20 15:30:09 x4 mountd[878]: can't delete exports for /mnt7: Invalid argument Nov 20 15:30:09 x4 mountd[878]: can't delete exports for /mnt8: Invalid argument Waiting for mount point write ops mount point write ops completed panic: Bad link elm 0xc735f800 prev->next != elm cpuid = 3 KDB: enter: panic [thread pid 881 tid 100126 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> set $lines 0 db> bt Tracing pid 881 tid 100126 td 0xc74a3d80 kdb_enter(c0bcf865,c0bcf865,c0b73411,e9a99c14,3,...) at kdb_enter+0x3a panic(c0b73411,c735f800,c71a0900,c735f800,e9a99c40,...) at panic+0x136 xprt_unregister_locked(c71a0900,0,c0bf02ba,134,0,...) at xprt_unregister_locked+0xa9 xprt_unregister(c735f800,e9a99ca4,c82f3848,e9a99ce0,e9a99cdc,...) at xprt_unregister+0x35 svc_run_internal(e9a99d24,c07e1748,c71a0900,e9a99d38,c0bcbb55,...) at svc_run_internal+0x6da svc_thread_start(c71a0900,e9a99d38,c0bcbb55,327,c733229c,...) at svc_thread_start+0x10 fork_exit(c09d75c0,c71a0900,e9a99d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2, eip = 0xc, esp = 0x33, ebp = 0 --- db> show allpcpu Current CPU: 3 cpuid = 0 curthread = 0xc74a2d80: pid 878 "mountd" curpcb = 0xe9a08d90 fpcurthread = none idlethread = 0xc6d66480: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 curthread = 0xc6d666c0: pid 11 "idle: cpu1" curpcb = 0xc6a99d90 fpcurthread = none idlethread = 0xc6d666c0: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 curthread = 0xc74d4000: pid 881 "nfsd: service" curpcb = 0xe9a96d90 fpcurthread = none idlethread = 0xc6d66900: pid 11 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 curthread = 0xc74a3d80: pid 881 "nfsd: service" curpcb = 0xe9a99d90 fpcurthread = none idlethread = 0xc6d66b40: pid 11 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db> show alllocks Process 34154 (mount) thread 0xc73ef240 (100119) exclusive sx user map (user map) r = 0 (0xc733a904) locked @ vm/vm_map.c:3115 Process 34152 (mount) thread 0xc7fe2240 (100320) exclusive sleep mutex vm object (standard object) r = 0 (0xcc5a2934) locked @ vm/vnode_pager.c:1224 shared lockmgr ufs (ufs) r = 0 (0xc7327710) locked @ kern/vfs_subr.c:2053 exclusive sx user map (user map) r = 0 (0xc79afac4) locked @ vm/vm_map.c:3115 Process 34151 (mount) thread 0xc8057b40 (100326) exclusive sleep mutex vm object (standard object) r = 0 (0xcc75a174) locked @ vm/vnode_pager.c:1224 shared lockmgr ufs (ufs) r = 0 (0xc7327710) locked @ kern/vfs_subr.c:2053 exclusive sx user map (user map) r = 0 (0xc784e4a4) locked @ vm/vm_map.c:3115 Process 34149 (mount_nfs) thread 0xc7333900 (100084) exclusive sleep mutex vm object (standard object) r = 0 (0xccc4d174) locked @ vm/vnode_pager.c:1224 shared lockmgr ufs (ufs) r = 0 (0xc7327710) locked @ kern/vfs_subr.c:2053 exclusive sx user map (user map) r = 0 (0xc784dc84) locked @ vm/vm_map.c:3115 Process 34147 (mount) thread 0xc73f2b40 (100099) exclusive sleep mutex SYSMAPS (SYSMAPS) r = 0 (0xc0eba728) locked @ i386/i386/pmap.c:3765 exclusive sleep mutex vm object (standard object) r = 0 (0xc72d7a2c) locked @ vm/vm_fault.c:676 shared lockmgr ufs (ufs) r = 0 (0xc7327710) locked @ kern/vfs_subr.c:2053 exclusive sx user map (user map) r = 0 (0xc7fd6904) locked @ vm/vm_map.c:3115 Process 34142 (mount_nfs) thread 0xc799c000 (100193) exclusive sx user map (user map) r = 0 (0xc6d6a4a4) locked @ vm/vm_map.c:3115 Process 34139 (mount_nfs) thread 0xc79b8b40 (100181) exclusive sx user map (user map) r = 0 (0xc784d584) locked @ vm/vm_map.c:3115 Process 1122 (sshd) thread 0xc74a2900 (100093) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc74e682c) locked @ kern/uipc_sockbuf.c:148 Process 1121 (sshd) thread 0xc74a3240 (100089) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc770ab3c) locked @ kern/uipc_sockbuf.c:148 Process 1120 (sshd) thread 0xc74a2b40 (100092) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc770ae4c) locked @ kern/uipc_sockbuf.c:148 Process 1113 (sshd) thread 0xc7314d80 (100128) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc768ccc4) locked @ kern/uipc_sockbuf.c:148 Process 881 (nfsd) thread 0xc74a3d80 (100126) exclusive sleep mutex sp_lock (sp_lock) r = 0 (0xc71a0900) locked @ rpc/svc.c:308 Process 881 (nfsd) thread 0xc74d4000 (100125) exclusive sleep mutex radix node head (radix node head) r = 0 (0xc70dd180) locked @ net/route.c:283 exclusive rw tcpinp (tcpinp) r = 0 (0xcf219c94) locked @ netinet/tcp_usrreq.c:585 exclusive rw tcp (tcp) r = 0 (0xc0e8e5ac) locked @ netinet/tcp_usrreq.c:582 db> show lockedvnods Locked vnodes 0xc73276b8: tag ufs, type VREG usecount 175, writecount 0, refcount 178 mountedhere 0 flags () v_object 0xc72d7a2c ref 171 pages 256 lock type ufs: SHARED (count 4) #0 0xc07ed2f2 at __lockmgr_args+0x582 #1 0xc0a0c501 at ffs_lock+0xa1 #2 0xc0b2e305 at VOP_LOCK1_APV+0xb5 #3 0xc08976d8 at _vn_lock+0x78 #4 0xc088b749 at vget+0xc9 #5 0xc0a3d983 at vnode_pager_lock+0x1e3 #6 0xc0a251df at vm_fault+0x1df #7 0xc0b20218 at trap_pfault+0x118 #8 0xc0b20b19 at trap+0x289 #9 0xc0b04c9b at calltrap+0x6 ino 212721, on dev ad4s1a db> show mount 0xc72c9b30 /dev/ad4s1a on / (ufs) 0xc72ca000 devfs on /dev (devfs) 0xc74a4598 /dev/ad4s1e on /tmp (ufs) 0xc74a42cc /dev/ad4s1f on /usr (ufs) 0xc74a4000 /dev/ad4s1d on /var (ufs) More info: show mount db> ps pid ppid pgrp uid state wmesg wchan cmd 34154 30799 41867 0 RL+ mount 34153 30787 41867 0 R+ grep 34152 30787 41867 0 RL+ mount 34151 30805 41867 0 RL+ mount 34150 34145 41867 0 R+ mount_nfs 34149 34146 41867 0 RL+ mount_nfs 34147 30791 41867 0 RL+ mount 34146 30797 41867 0 S+ wait 0xc78d5a70 mount 34145 30793 41867 0 S+ wait 0xc82a0a70 mount 34142 34131 41867 0 RL+ mount_nfs 34141 34126 41867 0 R+ mount_nfs 34139 34130 41867 0 S+ kqread 0xcf2e2500 mount_nfs 34131 30803 41867 0 S+ wait 0xc799ad0c mount 34130 30789 41867 0 S+ wait 0xc82a029c mount 34126 30801 41867 0 S+ wait 0xc79e3d0c mount 34122 30795 41867 0 R+ mount 32007 1132 1129 1001 S nanslp 0xc0d1fcc4 sleep 30805 30761 41867 0 S+ wait 0xc79e2a70 sh 30803 30761 41867 0 S+ wait 0xc79a229c sh 30801 30761 41867 0 S+ wait 0xc7f257d4 sh 30799 30761 41867 0 S+ wait 0xc799a538 sh 30797 30761 41867 0 S+ wait 0xc78d4d0c sh 30795 30761 41867 0 S+ wait 0xc72b1a70 sh 30793 30761 41867 0 S+ wait 0xc79b3d0c sh 30791 30761 41867 0 S+ wait 0xcc032538 sh 30789 30761 41867 0 S+ wait 0xc799929c sh 30787 30761 41867 0 S+ wait 0xcc03629c sh 30761 41867 41867 0 S+ wait 0xc798ba70 sh 24741 0 0 0 SL - 0xc0e92160 [nfsiod 0] 41867 41801 41867 0 S+ wait 0xcc03229c sh 41801 41797 41801 0 S+ wait 0xc78d47d4 bash 41797 41796 41797 0 S+ pause 0xcc033590 csh 41796 1116 41796 1001 S+ wait 0xc78d429c su 1133 1129 1129 1001 S piperd 0xc732dc00 awk 1132 1129 1129 1001 S wait 0xc74d27d4 sh 1131 1130 1131 1001 Rs tail 1130 1122 1122 1001 S select 0xc75c81a4 sshd 1129 1127 1129 1001 Ss wait 0xc74a07d4 sh 1128 1126 1128 1001 Ss+ select 0xc729fe64 top 1127 1121 1121 1001 S select 0xc729d024 sshd 1126 1120 1120 1001 S select 0xc75c8064 sshd 1122 1037 1122 0 Ss sbwait 0xc74e6858 sshd 1121 1037 1121 0 Ss sbwait 0xc770ab68 sshd 1120 1037 1120 0 Ss sbwait 0xc770ae78 sshd 1116 1115 1116 1001 Ss+ wait 0xc74d1a70 bash 1115 1113 1113 1001 S select 0xc72a0ba4 sshd 1113 1037 1113 0 Ss sbwait 0xc768ccf0 sshd 1109 1 1109 0 Ss+ tty inpu 0xc70db078 getty 1108 1 1108 0 Ss+ tty inpu 0xc70db278 getty 1107 1 1107 0 Ss+ tty inpu 0xc70db478 getty 1106 1 1106 0 Ss+ tty inpu 0xc70db678 getty 1105 1 1105 0 Ss+ tty inpu 0xc70db878 getty 1104 1 1104 0 Ss+ tty inpu 0xc6fc9e78 getty 1103 1 1103 0 Ss+ tty inpu 0xc6fd8078 getty 1102 1 1102 0 Ss+ tty inpu 0xc6fd8678 getty 1076 1 1076 0 Ss select 0xc73a2aa4 inetd 1054 1 1054 0 Ss nanslp 0xc0d1fcc4 cron 1048 1 1048 25 Ss pause 0xc72b12f4 sendmail 1044 1 1044 0 Ss select 0xc72a00a4 sendmail 1037 1 1037 0 Ss select 0xc7337524 sshd 1000 1 1000 0 Ss select 0xc75c7464 ntpd 881 880 880 0 R (threaded) nfsd 100127 S rpcsvc 0xc72a0110 nfsd: service 100126 Run CPU 3 nfsd: service 100125 Run CPU 2 nfsd: service 100065 S rpcsvc 0xc75c7a10 nfsd: master 880 1 880 0 Ss select 0xc70c99a4 nfsd 878 1 878 0 Rs CPU 0 mountd 796 1 796 0 Ss select 0xc7336724 rpcbind 767 1 767 0 Ss select 0xc72a0da4 syslogd 572 1 572 0 Ss select 0xc73360e4 devd 27 0 0 0 SL sdflush 0xc0e981a0 [softdepflush] 26 0 0 0 SL syncer 0xc0e8c350 [syncer] 25 0 0 0 SL vlruwt 0xc72b27d4 [vnlru] 24 0 0 0 SL psleep 0xc0e8c0a4 [bufdaemon] 23 0 0 0 SL pgzero 0xc0e98d94 [pagezero] 22 0 0 0 SL psleep 0xc0e989bc [vmdaemon] 9 0 0 0 SL psleep 0xc0e98984 [pagedaemon] 8 0 0 0 SL waiting_ 0xc0e8dd80 [sctp_iterator] 7 0 0 0 SL - 0xc6fd823c [fdc0] 6 0 0 0 SL - 0xc6fd9000 [fw0_probe] 21 0 0 0 SL usbevt 0xc6fa9a10 [usb5] 20 0 0 0 SL usbevt 0xc6fc6210 [usb4] 19 0 0 0 SL usbevt 0xc6fb2210 [usb3] 18 0 0 0 SL usbevt 0xc6fb1210 [usb2] 17 0 0 0 SL usbevt 0xc6f83210 [usb1] 16 0 0 0 SL usbtsk 0xc0d1d034 [usbtask-dr] 15 0 0 0 SL usbtsk 0xc0d1d020 [usbtask-hc] 14 0 0 0 SL usbevt 0xc6f8a210 [usb0] 5 0 0 0 SL ccb_scan 0xc0cec154 [xpt_thrd] 13 0 0 0 RL [yarrow] 4 0 0 0 SL - 0xc0d1d764 [g_down] 3 0 0 0 SL - 0xc0d1d760 [g_up] 2 0 0 0 RL [g_event] 12 0 0 0 RL (threaded) intr 100048 I [irq12: psm0] 100047 I [irq1: atkbd0] 100045 I [swi0: uart] 100042 I [irq14: ata0] 100040 I [irq19: re0 ehci0] 100036 I [irq18: ohci2 ohci4] 100034 I [irq17: ohci1 ohci3] 100030 I [irq16: ohci0] 100029 I [irq22: atapci0] 100028 I [irq9: acpi0] 100027 I [swi6: Giant taskq] 100025 I [swi5: +] 100021 I [swi2: cambio] 100018 I [swi6: task queue] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 RunQ [swi4: clock] 100008 RunQ [swi1: net] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) idle 100006 CanRun [idle: cpu0] 100005 Run CPU 1 [idle: cpu1] 100004 CanRun [idle: cpu2] 100003 CanRun [idle: cpu3] 1 0 1 0 SLs wait 0xc6d64d0c [init] 10 0 0 0 SL audit_wo 0xc0e97b60 [audit] 0 0 0 0 SLs (threaded) kernel 100043 D - 0xc6fbbe80 [fw0_taskq] 100026 D - 0xc6f1e500 [thread taskq] 100024 D - 0xc6f1e5c0 [acpi_task_2] 100023 D - 0xc6f1e5c0 [acpi_task_1] 100022 D - 0xc6f1e5c0 [acpi_task_0] 100019 D - 0xc6f1e700 [kqueue taskq] 100016 D - 0xc6d4bd80 [firmware taskq] 100000 D sched 0xc0d1d820 [swapper] db> allt Tracing command mount pid 34154 tid 100119 td 0xc73ef240 sched_switch(c73ef240,0,103,181,89463b54,...) at sched_switch+0x437 mi_switch(103,0,c0bd4254,2e1,c7317eb0,...) at mi_switch+0x200 turnstile_wait(c7317eb0,c7fe2240,0,188,c0e98964,...) at turnstile_wait+0x48a _mtx_lock_sleep(c0e98964,c73ef240,0,c0c044b7,bd4,...) at _mtx_lock_sleep+0x18e _mtx_lock_flags(c0e98964,0,c0c044b7,bd4,c0eba728,...) at _mtx_lock_flags+0xf7 pmap_enter(c733a968,28084000,2,c1c74748,7,...) at pmap_enter+0x75 vm_fault(c733a8c0,28084000,2,8,2808400c,...) at vm_fault+0x16b5 trap_pfault(5,0,c0c057df,2e7,c73ed29c,...) at trap_pfault+0x118 trap(e9a76d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x28053d56, esp = 0xbfbfe710, ebp = 0xbfbfe738 --- Tracing command grep pid 34153 tid 100283 td 0xc837a480 sched_switch(c837a480,0,60b,181,894eab2f,...) at sched_switch+0x437 mi_switch(60b,0,c0bd13a2,808,3,...) at mi_switch+0x200 sched_preempt(c837a480,3,e9d14d2c,c0b0538e,8,...) at sched_preempt+0xb5 ipi_bitmap_handler(8,28,28,c837a480,e9d14d38,...) at ipi_bitmap_handler+0x30 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0b209f6, esp = 0xe9d14ca0, ebp = 0xe9d14d2c --- trap(e9d14d38) at trap+0x166 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2805c8f6, esp = 0xbfbfe770, ebp = 0xbfbfe7c8 --- Tracing command mount pid 34152 tid 100320 td 0xc7fe2240 sched_switch(c7fe2240,0,103,181,894dedef,...) at sched_switch+0x437 mi_switch(103,0,c0bd4254,2e1,c7317230,...) at mi_switch+0x200 turnstile_wait(c7317230,c73f2b40,0,188,c0e98950,...) at turnstile_wait+0x48a _mtx_lock_sleep(c0e98950,c7fe2240,0,c0bf6e78,418,...) at _mtx_lock_sleep+0x18e _mtx_lock_flags(c0e98950,0,c0bf6e78,418,0,...) at _mtx_lock_flags+0xf7 vm_page_alloc(cc5a2934,3e,0,0,e9db6c18,...) at vm_page_alloc+0xb4 vm_fault(c79afa80,280d6000,2,8,280d6d93,...) at vm_fault+0x6d4 trap_pfault(5,0,c0c057df,2e7,cc0367d4,...) at trap_pfault+0x118 trap(e9db6d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804d9c6, esp = 0xbfbfe780, ebp = 0xbfbfe7d8 --- Tracing command mount pid 34151 tid 100326 td 0xc8057b40 sched_switch(c8057b40,0,103,181,894e03c8,...) at sched_switch+0x437 mi_switch(103,0,c0bd4254,2e1,c7317230,...) at mi_switch+0x200 turnstile_wait(c7317230,c73f2b40,0,188,c0e98950,...) at turnstile_wait+0x48a _mtx_lock_sleep(c0e98950,c8057b40,0,c0bf6e78,418,...) at _mtx_lock_sleep+0x18e _mtx_lock_flags(c0e98950,0,c0bf6e78,418,0,...) at _mtx_lock_flags+0xf7 vm_page_alloc(cc75a174,5a,0,0,e9dc8c18,...) at vm_page_alloc+0xb4 vm_fault(c784e460,280f2000,2,8,280f21ae,...) at vm_fault+0x6d4 trap_pfault(5,0,c0c057df,2e7,cc0357d4,...) at trap_pfault+0x118 trap(e9dc8d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804d9c6, esp = 0xbfbfe6f0, ebp = 0xbfbfe748 --- Tracing command mount_nfs pid 34150 tid 100197 td 0xc7990b40 sched_switch(c7990b40,0,60b,181,894e31ef,...) at sched_switch+0x437 mi_switch(60b,0,c0bd13a2,808,1,...) at mi_switch+0x200 sched_preempt(c7990b40,1,0,c0b0538e,3b,...) at sched_preempt+0xb5 ipi_bitmap_handler(3b,3b,3b,0,bfbfe954,...) at ipi_bitmap_handler+0x30 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0x28053f71, esp = 0xbfbfe928, ebp = 0 --- Tracing command mount_nfs pid 34149 tid 100084 td 0xc7333900 sched_switch(c7333900,0,103,181,894e83ab,...) at sched_switch+0x437 mi_switch(103,0,c0bd4254,2e1,c7317230,...) at mi_switch+0x200 turnstile_wait(c7317230,c73f2b40,0,188,c0e98950,...) at turnstile_wait+0x48a _mtx_lock_sleep(c0e98950,c7333900,0,c0bf6e78,418,...) at _mtx_lock_sleep+0x18e _mtx_lock_flags(c0e98950,0,c0bf6e78,418,0,...) at _mtx_lock_flags+0xf7 vm_page_alloc(ccc4d174,0,0,0,e99e9c18,...) at vm_page_alloc+0xb4 vm_fault(c784dc40,2817f000,2,8,2817f000,...) at vm_fault+0x6d4 trap_pfault(5,0,c0c057df,2e7,c7313a70,...) at trap_pfault+0x118 trap(e99e9d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804f8f6, esp = 0xbfbfe700, ebp = 0xbfbfe758 --- Tracing command mount pid 34147 tid 100099 td 0xc73f2b40 sched_switch(c73f2b40,0,60b,181,89545d02,...) at sched_switch+0x437 mi_switch(60b,0,c0bd13a2,808,2,...) at mi_switch+0x200 sched_preempt(c73f2b40,2,e9a20afc,c0b0538e,8,...) at sched_preempt+0xb5 ipi_bitmap_handler(8,28,28,c18053d0,c18043d0,...) at ipi_bitmap_handler+0x30 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0b1dce6, esp = 0xe9a20ad4, ebp = 0xe9a20afc --- generic_bcopy(c1ca12e0,c4bafdac,c0bf59f3,192,e9a20c18,...) at generic_bcopy+0x1a vm_fault(c7fd68c0,280ce000,2,8,280ced06,...) at vm_fault+0x1297 trap_pfault(5,0,c0c057df,2e7,c73eea70,...) at trap_pfault+0x118 trap(e9a20d38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2804d9c6, esp = 0xbfbfe6f0, ebp = 0xbfbfe748 --- Tracing command mount pid 34146 tid 100205 td 0xc78d9b40 sched_switch(c78d9b40,0,104,186,88067675,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c78d9b40,0,c0bd3a0b,189,c78d5a70,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c78d5a70,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c78d5a70,c78d5af8,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c78d9b40,8565,e9bdcc74,0,0,...) at kern_wait+0xab6 wait4(c78d9b40,e9bdccf8,10,e9bdcd38,c0caca08,...) at wait4+0x3b syscall(e9bdcd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f6aeb, esp = 0xbfbfdc8c, ebp = 0xbfbfdca8 --- Tracing command mount pid 34145 tid 100270 td 0xc83956c0 sched_switch(c83956c0,0,104,186,881d8733,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c83956c0,0,c0bd3a0b,189,c82a0a70,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c82a0a70,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c82a0a70,c82a0af8,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c83956c0,8566,e9cdec74,0,0,...) at kern_wait+0xab6 wait4(c83956c0,e9cdecf8,10,e9cded38,c0caca08,...) at wait4+0x3b syscall(e9cded38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f6aeb, esp = 0xbfbfdc8c, ebp = 0xbfbfdca8 --- Tracing command mount_nfs pid 34142 tid 100193 td 0xc799c000 sched_switch(c799c000,0,103,181,8952faf5,...) at sched_switch+0x437 mi_switch(103,0,c0bd4254,2e1,c6d67460,...) at mi_switch+0x200 turnstile_wait(c6d67460,c73f2b40,0,188,c0e98964,...) at turnstile_wait+0x48a _mtx_lock_sleep(c0e98964,c799c000,0,c0c044b7,bd4,...) at _mtx_lock_sleep+0x18e _mtx_lock_flags(c0e98964,0,c0c044b7,bd4,c0eba704,...) at _mtx_lock_flags+0xf7 pmap_enter(c6d6a508,815a000,2,c3aa419c,7,...) at pmap_enter+0x75 vm_fault(c6d6a460,815a000,2,8,815a000,...) at vm_fault+0x16b5 trap_pfault(5,0,c0c057df,2e7,c798cd0c,...) at trap_pfault+0x118 trap(e9bafd38) at trap+0x289 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x28170073, esp = 0xbfbfe0d0, ebp = 0xbfbfe148 --- Tracing command mount_nfs pid 34141 tid 100199 td 0xc79a6480 sched_switch(c79a6480,0,104,186,894f58ff,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,58,...) at mi_switch+0x200 sleepq_switch(c79a6480,0,c0bd3a0b,189,c894a7e2,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c894a7e2,58,c0bd9b8f,100,0,...) at sleepq_wait_sig+0x17 _sleep(c894a7e2,c894a818,158,c0bd9b8f,0,...) at _sleep+0x344 kern_connect(c79a6480,3,ca4f43f0,ca4f43f0,281839e0,...) at kern_connect+0x10c connect(c79a6480,e9bc6cf8,c,c0bd62c7,c0cad290,...) at connect+0x46 syscall(e9bc6d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (98, FreeBSD ELF32, connect), eip = 0x281526eb, esp = 0xbfbfdfdc, ebp = 0xbfbfe138 --- Tracing command mount_nfs pid 34139 tid 100181 td 0xc79b8b40 sched_switch(c79b8b40,0,104,186,8992b953,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,58,...) at mi_switch+0x200 sleepq_switch(c79b8b40,0,c0bd3a0b,189,cf2e2500,...) at sleepq_switch+0x15f sleepq_catch_signals(3a99,c0838610,c79b8b40,1,100,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(cf2e2500,58,c0bcb8bc,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(cf2e2500,cf2e2500,158,c0bcb8bc,3a99,...) at _sleep+0x30e kern_kevent(c79b8b40,4,0,1,e9b84c58,...) at kern_kevent+0x357 kevent(c79b8b40,e9b84cf8,18,c0bd62a6,c0caeb68,...) at kevent+0x19b syscall(e9b84d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (363, FreeBSD ELF32, kevent), eip = 0x281526ab, esp = 0xbfbfdefc, ebp = 0xbfbfe038 --- Tracing command mount pid 34131 tid 100224 td 0xc79a5480 sched_switch(c79a5480,0,104,186,84a995c0,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c79a5480,0,c0bd3a0b,189,c799ad0c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c799ad0c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c799ad0c,c799ad94,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c79a5480,855e,e9c24c74,0,0,...) at kern_wait+0xab6 wait4(c79a5480,e9c24cf8,10,e9c24d38,c0caca08,...) at wait4+0x3b syscall(e9c24d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f6aeb, esp = 0xbfbfdc8c, ebp = 0xbfbfdca8 --- Tracing command mount pid 34130 tid 100273 td 0xc8395000 sched_switch(c8395000,0,104,186,8472abc6,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c8395000,0,c0bd3a0b,189,c82a029c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c82a029c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c82a029c,c82a0324,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c8395000,855b,e9cebc74,0,0,...) at kern_wait+0xab6 wait4(c8395000,e9cebcf8,10,e9cebd38,c0caca08,...) at wait4+0x3b syscall(e9cebd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f6aeb, esp = 0xbfbfdc8c, ebp = 0xbfbfdca8 --- Tracing command mount pid 34126 tid 100202 td 0xc79a6000 sched_switch(c79a6000,0,104,186,849c646c,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c79a6000,0,c0bd3a0b,189,c79e3d0c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c79e3d0c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c79e3d0c,c79e3d94,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c79a6000,855d,e9bd2c74,0,0,...) at kern_wait+0xab6 wait4(c79a6000,e9bd2cf8,10,e9bd2d38,c0caca08,...) at wait4+0x3b syscall(e9bd2d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f6aeb, esp = 0xbfbfdc8c, ebp = 0xbfbfdca8 --- Tracing command mount pid 34122 tid 100281 td 0xc72476c0 sched_switch(c72476c0,0,60b,181,89517bd8,...) at sched_switch+0x437 mi_switch(60b,0,c0bd13a2,808,0,...) at mi_switch+0x200 sched_preempt(c72476c0,0,e9d0cbec,c0b0538e,8,...) at sched_preempt+0xb5 ipi_bitmap_handler(8,28,28,3d5,c6d1e340,...) at ipi_bitmap_handler+0x30 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc08424cc, esp = 0xe9d0cb98, ebp = 0xe9d0cbec --- witness_checkorder(c7f04d94,9,c0bcf8d4,3d5,0,...) at witness_checkorder+0x7c _mtx_lock_flags(c7f04d94,0,c0bcf8d4,3d5,e9d0cc64,...) at _mtx_lock_flags+0xc4 kern_sigprocmask(c72476c0,3,e9d0cc64,0,0,...) at kern_sigprocmask+0x3a sigprocmask(c72476c0,e9d0ccf8,c,c0bd5fb0,c0cae940,...) at sigprocmask+0x6e syscall(e9d0cd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (340, FreeBSD ELF32, sigprocmask), eip = 0x28054f43, esp = 0xbfbfe7bc, ebp = 0xbfbfe7d8 --- Tracing command sleep pid 32007 tid 100298 td 0xc7b9a480 sched_switch(c7b9a480,0,104,186,d794e01c,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c7b9a480,0,c0bd3a0b,189,c0d1fcc4,...) at sleepq_switch+0x15f sleepq_catch_signals(2711,c0838610,c7b9a480,2,100,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c0d1fcc4,5c,c0bd0acb,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0d1fcc4,0,15c,c0bd0acb,2711,...) at _sleep+0x30e kern_nanosleep(c7b9a480,e9d5dc64,e9d5dc6c,a,0,...) at kern_nanosleep+0xc1 nanosleep(c7b9a480,e9d5dcf8,8,e9d5dd38,c0cadfe0,...) at nanosleep+0x6f syscall(e9d5dd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2814c4bf, esp = 0xbfbfecfc, ebp = 0xbfbfed38 --- Tracing command sh pid 30805 tid 100213 td 0xc79fdd80 sched_switch(c79fdd80,0,104,186,882de99b,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c79fdd80,0,c0bd3a0b,189,c79e2a70,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c79e2a70,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c79e2a70,c79e2af8,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c79fdd80,ffffffff,e9bfdc74,2,0,...) at kern_wait+0xab6 wait4(c79fdd80,e9bfdcf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9bfdd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30803 tid 100216 td 0xc79a7900 sched_switch(c79a7900,0,104,186,827c3e1b,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c79a7900,0,c0bd3a0b,189,c79a229c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c79a229c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c79a229c,c79a2324,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c79a7900,ffffffff,e9c07c74,2,0,...) at kern_wait+0xab6 wait4(c79a7900,e9c07cf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9c07d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30801 tid 100237 td 0xc7f28900 sched_switch(c7f28900,0,104,186,8062672d,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c7f28900,0,c0bd3a0b,189,c7f257d4,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c7f257d4,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7f257d4,c7f2585c,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c7f28900,ffffffff,e9c60c74,2,0,...) at kern_wait+0xab6 wait4(c7f28900,e9c60cf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9c60d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30799 tid 100155 td 0xc7333480 sched_switch(c7333480,0,104,186,8929fdd4,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c7333480,0,c0bd3a0b,189,c799a538,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c799a538,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c799a538,c799a5c0,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c7333480,ffffffff,e9b2cc74,2,0,...) at kern_wait+0xab6 wait4(c7333480,e9b2ccf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9b2cd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30797 tid 100133 td 0xc78d9000 sched_switch(c78d9000,0,104,186,8594dbb8,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c78d9000,0,c0bd3a0b,189,c78d4d0c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c78d4d0c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c78d4d0c,c78d4d94,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c78d9000,ffffffff,e9abec74,2,0,...) at kern_wait+0xab6 wait4(c78d9000,e9abecf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9abed38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30795 tid 100061 td 0xc6fcdb40 sched_switch(c6fcdb40,0,104,186,7ed7beda,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6fcdb40,0,c0bd3a0b,189,c72b1a70,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c72b1a70,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c72b1a70,c72b1af8,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c6fcdb40,ffffffff,e9983c74,2,0,...) at kern_wait+0xab6 wait4(c6fcdb40,e9983cf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9983d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30793 tid 100227 td 0xc79fd240 sched_switch(c79fd240,0,104,186,84f847bc,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c79fd240,0,c0bd3a0b,189,c79b3d0c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c79b3d0c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c79b3d0c,c79b3d94,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c79fd240,ffffffff,e9c2dc74,2,0,...) at kern_wait+0xab6 wait4(c79fd240,e9c2dcf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9c2dd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30791 tid 100345 td 0xc83c0000 sched_switch(c83c0000,0,104,186,869f52aa,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c83c0000,0,c0bd3a0b,189,cc032538,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(cc032538,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc032538,cc0325c0,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c83c0000,ffffffff,e9e01c74,2,0,...) at kern_wait+0xab6 wait4(c83c0000,e9e01cf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9e01d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30789 tid 100168 td 0xc799d240 sched_switch(c799d240,0,104,186,8221d10b,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c799d240,0,c0bd3a0b,189,c799929c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c799929c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c799929c,c7999324,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c799d240,ffffffff,e9b5cc74,2,0,...) at kern_wait+0xab6 wait4(c799d240,e9b5ccf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9b5cd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe47c, ebp = 0xbfbfe498 --- Tracing command sh pid 30787 tid 100322 td 0xc7fe2b40 sched_switch(c7fe2b40,0,104,186,885a1485,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c7fe2b40,0,c0bd3a0b,189,cc03629c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(cc03629c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc03629c,cc036324,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c7fe2b40,ffffffff,e9dbcc74,2,0,...) at kern_wait+0xab6 wait4(c7fe2b40,e9dbccf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9dbcd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe61c, ebp = 0xbfbfe638 --- Tracing command sh pid 30761 tid 100154 td 0xc798f900 sched_switch(c798f900,0,104,186,9e2663f1,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c798f900,0,c0bd3a0b,189,c798ba70,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c798ba70,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c798ba70,c798baf8,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c798f900,ffffffff,e9b28c74,2,0,...) at kern_wait+0xab6 wait4(c798f900,e9b28cf8,10,c0bd5ef1,c0caca08,...) at wait4+0x3b syscall(e9b28d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe78c, ebp = 0xbfbfe7a8 --- Tracing command nfsiod 0 pid 24741 tid 100303 td 0xc8395b40 sched_switch(c8395b40,0,104,186,2f8490a4,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c8395b40,0,c0bd3a0b,189,c0e92160,...) at sleepq_switch+0x15f sleepq_catch_signals(1d4c0,c0838610,c8395b40,2,100,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c0e92160,5c,c0bc6814,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0e92160,c0e92130,15c,c0bc6814,1d4c0,...) at _sleep+0x30e nfssvc_iod(c0e91ae0,e9d69d38,c0bcbb55,327,c8c98a70,...) at nfssvc_iod+0xcf fork_exit(c09a2040,c0e91ae0,e9d69d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe9d69d70, ebp = 0 --- Tracing command sh pid 41867 tid 100346 td 0xc83c0900 sched_switch(c83c0900,0,104,186,ce9e6f25,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c83c0900,0,c0bd3a0b,189,cc03229c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(cc03229c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc03229c,cc032324,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c83c0900,ffffffff,e9e04c74,2,0,...) at kern_wait+0xab6 wait4(c83c0900,e9e04cf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9e04d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe73c, ebp = 0xbfbfe758 --- Tracing command bash pid 41801 tid 100135 td 0xc78d7b40 sched_switch(c78d7b40,0,104,186,bf23eb89,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c78d7b40,0,c0bd3a0b,189,c78d47d4,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c78d47d4,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c78d47d4,c78d485c,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c78d7b40,ffffffff,e9ac4c74,6,0,...) at kern_wait+0xab6 wait4(c78d7b40,e9ac4cf8,10,c0bd5fb0,c0caca08,...) at wait4+0x3b syscall(e9ac4d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282a6aeb, esp = 0xbfbfe84c, ebp = 0xbfbfe868 --- Tracing command csh pid 41797 tid 100339 td 0xc74936c0 sched_switch(c74936c0,0,104,186,149ce7f6,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,68,...) at mi_switch+0x200 sleepq_switch(c74936c0,0,c0bd3a0b,189,cc033590,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(cc033590,68,c0b8bf3c,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc033590,cc0335c0,168,c0b8bf3c,0,...) at _sleep+0x344 kern_sigsuspend(c74936c0,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c74936c0,e9defcf8,4,c0bd5fb0,c0cae958,...) at sigsuspend+0x4d syscall(e9defd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x28171dbb, esp = 0xbfbfe62c, ebp = 0xbfbfe698 --- Tracing command su pid 41796 tid 100137 td 0xc78d76c0 sched_switch(c78d76c0,0,104,186,6ca421ff,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c78d76c0,0,c0bd3a0b,189,c78d429c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c78d429c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c78d429c,c78d4324,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c78d76c0,a345,e9acac74,2,0,...) at kern_wait+0xab6 wait4(c78d76c0,e9acacf8,10,e9acad38,c0caca08,...) at wait4+0x3b syscall(e9acad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28111aeb, esp = 0xbfbfe66c, ebp = 0xbfbfe688 --- Tracing command awk pid 1133 tid 100107 td 0xc74d6240 sched_switch(c74d6240,0,104,186,d4b26bbd,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,4c,...) at mi_switch+0x200 sleepq_switch(c74d6240,0,c0bd3a0b,189,c732dc00,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c732dc00,4c,c0bd5cf8,100,0,...) at sleepq_wait_sig+0x17 _sleep(c732dc00,c732dd68,14c,c0bd5cf8,0,...) at _sleep+0x344 pipe_read(c73e44d0,e9a38c58,c76b6200,0,c74d6240,...) at pipe_read+0x3f7 dofileread(e9a38c58,ffffffff,ffffffff,0,c73e44d0,...) at dofileread+0x96 kern_readv(c74d6240,0,e9a38c58,e9a38c78,1,...) at kern_readv+0x58 read(c74d6240,e9a38cf8,c,16,c0cac9a8,...) at read+0x4f syscall(e9a38d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281a2b43, esp = 0xbfbfe9bc, ebp = 0xbfbfe9d8 --- Tracing command sh pid 1132 tid 100105 td 0xc74d66c0 sched_switch(c74d66c0,0,104,186,d4c674c6,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c74d66c0,0,c0bd3a0b,189,c74d27d4,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c74d27d4,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74d27d4,c74d285c,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c74d66c0,ffffffff,e9a32c74,2,0,...) at kern_wait+0xab6 wait4(c74d66c0,e9a32cf8,10,c0bd60b5,c0caca08,...) at wait4+0x3b syscall(e9a32d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfe98c, ebp = 0xbfbfe9a8 --- Tracing command tail pid 1131 tid 100059 td 0xc7339000 sched_switch(c7339000,0,104,186,2441318a,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,58,...) at mi_switch+0x200 sleepq_switch(c7339000,0,c0bd3a0b,189,c73c7300,...) at sleepq_switch+0x15f sleepq_catch_signals(3e9,c0838610,c7339000,1,100,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c73c7300,58,c0bcb8bc,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c73c7300,c73c7300,158,c0bcb8bc,3e9,...) at _sleep+0x30e kern_kevent(c7339000,4,0,1,e997bc58,...) at kern_kevent+0x357 kevent(c7339000,e997bcf8,18,c0b77b22,c0caeb68,...) at kevent+0x19b syscall(e997bd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (363, FreeBSD ELF32, kevent), eip = 0x281506ab, esp = 0xbfbfebbc, ebp = 0xbfbfec68 --- Tracing command sshd pid 1130 tid 100069 td 0xc73386c0 sched_switch(c73386c0,0,104,186,76408f34,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c73386c0,0,c0bd3a0b,189,c75c81a4,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c75c8190,0,c0bce135,c73386c0,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c75c81a4,0,e99a5a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c75c81a4,c75c8190,c0bd5b20,570,c75b8d58,...) at _cv_wait_sig+0x240 seltdwait(c72b342c,c0bd5b20,3d6,c73386c0,200246,...) at seltdwait+0xa2 kern_select(c73386c0,c,81030b8,81030dc,0,0,0,280f5818) at kern_select+0x4cf select(c73386c0,e99a5cf8,14,c0bbde30,c0cad218,...) at select+0x5e syscall(e99a5d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283b6ac3, esp = 0xbfbfdeac, ebp = 0xbfbfdef8 --- Tracing command sh pid 1129 tid 100094 td 0xc74a26c0 sched_switch(c74a26c0,0,104,186,b2bdce9a,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c74a26c0,0,c0bd3a0b,189,c74a07d4,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c74a07d4,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74a07d4,c74a085c,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c74a26c0,ffffffff,e9a11c74,2,0,...) at kern_wait+0xab6 wait4(c74a26c0,e9a11cf8,10,e9a11d38,c0caca08,...) at wait4+0x3b syscall(e9a11d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x28153aeb, esp = 0xbfbfeb8c, ebp = 0xbfbfeba8 --- Tracing command top pid 1128 tid 100130 td 0xc7314900 sched_switch(c7314900,0,104,186,360e440e,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c7314900,0,c0bd3a0b,189,c729fe64,...) at sleepq_switch+0x15f sleepq_catch_signals(e9aa7a4c,c07f48fa,c729fe50,0,c7314900,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c729fe64,0,e9aa7a7c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c729fe64,c729fe50,3e9,570,c731bce8,...) at _cv_timedwait_sig+0x250 seltdwait(e9aa7c2c,e9aa7c34,3d6,c7314900,353a3531,...) at seltdwait+0x8a kern_select(c7314900,2,bfbfec3c,0,0,e9aa7c70,1,0) at kern_select+0x4cf select(c7314900,e9aa7cf8,14,c0bbde30,c0cad218,...) at select+0x5e syscall(e9aa7d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281d7ac3, esp = 0xbfbfebdc, ebp = 0xbfbfed38 --- Tracing command sshd pid 1127 tid 100088 td 0xc74a3480 sched_switch(c74a3480,0,104,186,d7dc87c7,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c74a3480,0,c0bd3a0b,189,c729d024,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c729d010,0,c0bce135,c74a3480,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c729d024,0,e99ffa7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c729d024,c729d010,c0bd5b20,570,c78b3348,...) at _cv_wait_sig+0x240 seltdwait(c76b672c,c0bd5b20,3d6,c74a3480,200246,...) at seltdwait+0xa2 kern_select(c74a3480,c,81030b8,81030dc,0,0,0,280f5818) at kern_select+0x4cf select(c74a3480,e99ffcf8,14,c0bbde30,c0cad218,...) at select+0x5e syscall(e99ffd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283b6ac3, esp = 0xbfbfdeac, ebp = 0xbfbfdef8 --- Tracing command sshd pid 1126 tid 100108 td 0xc74d6000 sched_switch(c74d6000,0,104,186,3611ce0b,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c74d6000,0,c0bd3a0b,189,c75c8064,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c75c8050,0,c0bce135,c74d6000,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c75c8064,0,e9a3ba7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c75c8064,c75c8050,c0bd5b20,570,c739bc08,...) at _cv_wait_sig+0x240 seltdwait(c76b662c,c0bd5b20,3d6,c74d6000,200246,...) at seltdwait+0xa2 kern_select(c74d6000,a,81030b8,81030dc,0,0,0,280f5818) at kern_select+0x4cf select(c74d6000,e9a3bcf8,14,c0bbde30,c0cad218,...) at select+0x5e syscall(e9a3bd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283b6ac3, esp = 0xbfbfdeac, ebp = 0xbfbfdef8 --- Tracing command sshd pid 1122 tid 100093 td 0xc74a2900 Tracing command sshd pid 1121 tid 100089 td 0xc74a3240 Tracing command sshd pid 1120 tid 100092 td 0xc74a2b40 Tracing command bash pid 1116 tid 100110 td 0xc74d4b40 sched_switch(c74d4b40,0,104,186,4cd6f9da,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c74d4b40,0,c0bd3a0b,189,c74d1a70,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c74d1a70,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74d1a70,c74d1af8,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c74d4b40,ffffffff,e9a41c74,6,0,...) at kern_wait+0xab6 wait4(c74d4b40,e9a41cf8,10,c0bd5fb0,c0caca08,...) at wait4+0x3b syscall(e9a41d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282a6aeb, esp = 0xbfbfea4c, ebp = 0xbfbfea68 --- Tracing command sshd pid 1115 tid 100086 td 0xc74a3900 sched_switch(c74a3900,0,104,186,89087163,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c74a3900,0,c0bd3a0b,189,c72a0ba4,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c72a0b90,0,c0bce135,c74a3900,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c72a0ba4,0,e99f8a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c72a0ba4,c72a0b90,c0bd5b20,570,c731c1f8,...) at _cv_wait_sig+0x240 seltdwait(c76b632c,c0bd5b20,3d6,c74a3900,200246,...) at seltdwait+0xa2 kern_select(c74a3900,a,81030b8,81030dc,0,0,0,280f5818) at kern_select+0x4cf select(c74a3900,e99f8cf8,14,c0bbde30,c0cad218,...) at select+0x5e syscall(e99f8d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283b6ac3, esp = 0xbfbfdeac, ebp = 0xbfbfdef8 --- Tracing command sshd pid 1113 tid 100128 td 0xc7314d80 Tracing command getty pid 1109 tid 100109 td 0xc74d4d80 Tracing command getty pid 1108 tid 100057 td 0xc7318d80 Tracing command getty pid 1107 tid 100072 td 0xc7318b40 Tracing command getty pid 1106 tid 100073 td 0xc7318900 Tracing command getty pid 1105 tid 100074 td 0xc73186c0 Tracing command getty pid 1104 tid 100075 td 0xc7318480 Tracing command getty pid 1103 tid 100076 td 0xc7318240 Tracing command getty pid 1102 tid 100101 td 0xc73f26c0 Tracing command inetd pid 1076 tid 100085 td 0xc73336c0 Tracing command cron pid 1054 tid 100129 td 0xc7314b40 sched_switch(c7314b40,0,104,186,f8a52ed4,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c7314b40,0,c0bd3a0b,189,c0d1fcc4,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c0838610,c7314b40,2,100,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c0d1fcc4,5c,c0bd0acb,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0d1fcc4,0,15c,c0bd0acb,ea61,...) at _sleep+0x30e kern_nanosleep(c7314b40,e9aa3c64,e9aa3c6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c7314b40,e9aa3cf8,8,c0bd621a,c0cadfe0,...) at nanosleep+0x6f syscall(e9aa3d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x281694bf, esp = 0xbfbfecdc, ebp = 0xbfbfed08 --- Tracing command sendmail pid 1048 tid 100064 td 0xc6fcd480 sched_switch(c6fcd480,0,104,186,359871c6,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,68,...) at mi_switch+0x200 sleepq_switch(c6fcd480,0,c0bd3a0b,189,c72b12f4,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c72b12f4,68,c0b8bf3c,100,0,...) at sleepq_wait_sig+0x17 _sleep(c72b12f4,c72b1324,168,c0b8bf3c,0,...) at _sleep+0x344 kern_sigsuspend(c6fcd480,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c6fcd480,e998fcf8,4,c0bd5fb0,c0cae958,...) at sigsuspend+0x4d syscall(e998fd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x28324dbb, esp = 0xbfbfd00c, ebp = 0xbfbfd038 --- Tracing command sendmail pid 1044 tid 100098 td 0xc73f2d80 sched_switch(c73f2d80,0,104,186,67348b3a,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c73f2d80,0,c0bd3a0b,189,c72a00a4,...) at sleepq_switch+0x15f sleepq_catch_signals(e9a1da4c,c07f48fa,c72a0090,0,c73f2d80,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c72a00a4,0,e9a1da7c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c72a00a4,c72a0090,1389,570,c732f738,...) at _cv_timedwait_sig+0x250 seltdwait(e9a1dc2c,e9a1dc34,3d6,c73f2d80,c188ca38,...) at seltdwait+0x8a kern_select(c73f2d80,4,bfbfc590,0,0,e9a1dc70,5,0) at kern_select+0x4cf select(c73f2d80,e9a1dcf8,14,c0bd64f0,c0cad218,...) at select+0x5e syscall(e9a1dd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283c1ac3, esp = 0xbfbfc4fc, ebp = 0xbfbfd028 --- Tracing command sshd pid 1037 tid 100087 td 0xc74a36c0 Tracing command ntpd pid 1000 tid 100095 td 0xc74a2480 sched_switch(c74a2480,0,104,186,3e1ccfc0,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c74a2480,0,c0bd3a0b,189,c75c7464,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c75c7450,0,c0bce135,c74a2480,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c75c7464,0,e9a14a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c75c7464,c75c7450,c0bd5b20,570,c732f3b8,...) at _cv_wait_sig+0x240 seltdwait(c72b4d2c,c0bd5b20,3d6,c74a2480,0,...) at seltdwait+0xa2 kern_select(c74a2480,1c,bfbfed78,0,0,0,246,0) at kern_select+0x4cf select(c74a2480,e9a14cf8,14,16,c0cad218,...) at select+0x5e syscall(e9a14d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28342ac3, esp = 0xbfbfed4c, ebp = 0xbfbfee18 --- Tracing command nfsd pid 881 tid 100127 td 0xc74a3b40 sched_switch(c74a3b40,0,104,186,89524b7a,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c74a3b40,0,c0bd3a0b,189,c72a0110,...) at sleepq_switch+0x15f sleepq_catch_signals(e9a9cbfc,c07f48fa,c71a0900,0,c74a3b40,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c72a0110,0,e9a9cc2c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c72a0110,c71a0900,1388,488,e9a9ccdc,...) at _cv_timedwait_sig+0x250 svc_run_internal(e9a9cd24,c07e1748,c71a0900,e9a9cd38,c0bcbb55,...) at svc_run_internal+0x35e svc_thread_start(c71a0900,e9a9cd38,c0bcbb55,327,c733229c,...) at svc_thread_start+0x10 fork_exit(c09d75c0,c71a0900,e9a9cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 881 tid 100126 td 0xc74a3d80 kdb_enter(c0bcf865,c0bcf865,c0b73411,e9a99c14,3,...) at kdb_enter+0x3a panic(c0b73411,c735f800,c71a0900,c735f800,e9a99c40,...) at panic+0x136 xprt_unregister_locked(c71a0900,0,c0bf02ba,134,0,...) at xprt_unregister_locked+0xa9 xprt_unregister(c735f800,e9a99ca4,c82f3848,e9a99ce0,e9a99cdc,...) at xprt_unregister+0x35 svc_run_internal(e9a99d24,c07e1748,c71a0900,e9a99d38,c0bcbb55,...) at svc_run_internal+0x6da svc_thread_start(c71a0900,e9a99d38,c0bcbb55,327,c733229c,...) at svc_thread_start+0x10 fork_exit(c09d75c0,c71a0900,e9a99d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 881 tid 100125 td 0xc74d4000 cpustop_handler(4,e9a96bb0,c0b208bd,e9a96b40,c07f4564,...) at cpustop_handler+0x32 ipi_nmi_handler(e9a96b40,c07f4564,c0d1fd70,4,c733229c,...) at ipi_nmi_handler+0x2f trap(e9a96bbc) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc07f4b29, esp = 0xe9a96bfc, ebp = 0xe9a96c18 --- _mtx_lock_sleep(c71a0900,c74d4000,0,c0bf02ba,47e,...) at _mtx_lock_sleep+0x99 _mtx_lock_flags(c71a0900,0,c0bf02ba,47e,e9a96cdc,...) at _mtx_lock_flags+0xf7 svc_run_internal(e9a96d24,c07e1748,c71a0900,e9a96d38,c0bcbb55,...) at svc_run_internal+0x8bd svc_thread_start(c71a0900,e9a96d38,c0bcbb55,327,c733229c,...) at svc_thread_start+0x10 fork_exit(c09d75c0,c71a0900,e9a96d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 881 tid 100065 td 0xc7338d80 sched_switch(c7338d80,0,104,186,89523803,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c7338d80,0,c0bd3a0b,189,c75c7a10,...) at sleepq_switch+0x15f sleepq_catch_signals(e9993b00,c07f48fa,c71a0900,0,c7338d80,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c75c7a10,0,e9993b30,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c75c7a10,c71a0900,1388,488,e9993be0,...) at _cv_timedwait_sig+0x250 svc_run_internal(c7338eec,14,c0bf04c0,c0beedcd,0,...) at svc_run_internal+0x35e svc_run(c71a0900,0,c0bef0d1,202,e9993cf8,...) at svc_run+0x7f nfssvc_nfsd(c7338d80,122,c0bce135,ca,c7332324,...) at nfssvc_nfsd+0xad nfssvc(c7338d80,e9993cf8,8,e9993d38,c0cad7e8,...) at nfssvc+0x235 syscall(e9993d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280d199b, esp = 0xbfbfeb1c, ebp = 0xbfbfeb38 --- Tracing command nfsd pid 880 tid 100112 td 0xc74d46c0 sched_switch(c74d46c0,0,104,186,894e82ab,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c74d46c0,0,c0bd3a0b,189,c70c99a4,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c70c9990,0,c0bce135,c74d46c0,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c70c99a4,0,e9a47a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c70c99a4,c70c9990,c0bd5b20,570,c732fab8,...) at _cv_wait_sig+0x240 seltdwait(c73a192c,c0bd5b20,3d6,c74d46c0,c0bd13a2,...) at seltdwait+0xa2 kern_select(c74d46c0,5,bfbfed44,0,0,0,0,5) at kern_select+0x4cf select(c74d46c0,e9a47cf8,14,c0beec03,c0cad218,...) at select+0x5e syscall(e9a47d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2816dac3, esp = 0xbfbfeb3c, ebp = 0xbfbfee78 --- Tracing command mountd pid 878 tid 100091 td 0xc74a2d80 cpustop_handler(1,e9a08a2c,c0b208bd,0,c186b908,...) at cpustop_handler+0x32 ipi_nmi_handler(0,c186b908,0,e9a08a14,c74a1000,...) at ipi_nmi_handler+0x2f trap(e9a08a38) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc07f48a0, esp = 0xe9a08a78, ebp = 0xe9a08a94 --- _mtx_unlock_flags(c6d38240,0,c0bd5b20,4ec,c74e6670,...) at _mtx_unlock_flags+0x110 selfdfree(c74fe22c,0,c0bd5b20,38d,cf20b85c,...) at selfdfree+0xdb kern_select(c74a2d80,a,bfbfedec,0,0,0,49257aa0,281949e0) at kern_select+0x634 select(c74a2d80,e9a08cf8,14,c0bd621a,c0cad218,...) at select+0x5e syscall(e9a08d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28181ac3, esp = 0xbfbfedbc, ebp = 0xbfbfee88 --- Tracing command rpcbind pid 796 tid 100081 td 0xc7338000 sched_switch(c7338000,0,104,186,89468bee,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c7338000,0,c0bd3a0b,189,c7336724,...) at sleepq_switch+0x15f sleepq_catch_signals(e99dbaa8,c07f48fa,c7336710,0,c7338000,...) at sleepq_catch_signals+0xb4 sleepq_timedwait_sig(c7336724,0,e99dbad8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7336724,c7336710,7531,570,e99dbb8c,...) at _cv_timedwait_sig+0x250 seltdwait(e99dbc5c,e99dbc64,480,c7338000,e99dbb5c,...) at seltdwait+0x8a poll(c7338000,e99dbcf8,c,c0bd62a6,c0cadcf8,...) at poll+0x2f9 syscall(e99dbd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (209, FreeBSD ELF32, poll), eip = 0x281323ff, esp = 0xbfbfcc7c, ebp = 0xbfbfee48 --- Tracing command syslogd pid 767 tid 100100 td 0xc73f2900 sched_switch(c73f2900,0,104,186,88fcf507,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c73f2900,0,c0bd3a0b,189,c72a0da4,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c72a0d90,0,c0bce135,c73f2900,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c72a0da4,0,e9a23a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c72a0da4,c72a0d90,c0bd5b20,570,c731bc08,...) at _cv_wait_sig+0x240 seltdwait(c74ab22c,c0bd5b20,3d6,c73f2900,c73f2900,...) at seltdwait+0xa2 kern_select(c73f2900,9,81290ac,0,0,0,0,281949e0) at kern_select+0x4cf select(c73f2900,e9a23cf8,14,c0bd5fb0,c0cad218,...) at select+0x5e syscall(e9a23d38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28181ac3, esp = 0xbfbfdedc, ebp = 0xbfbfee88 --- Tracing command devd pid 572 tid 100113 td 0xc74d4480 sched_switch(c74d4480,0,104,186,dfdae436,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c74d4480,0,c0bd3a0b,189,c73360e4,...) at sleepq_switch+0x15f sleepq_catch_signals(c07f48fa,c73360d0,0,c0bce135,c74d4480,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c73360e4,0,e9a4aa7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c73360e4,c73360d0,c0bd5b20,570,c731b118,...) at _cv_wait_sig+0x240 seltdwait(c73a162c,c0bd5b20,3d6,c74d4480,c0bc8d5e,...) at seltdwait+0xa2 kern_select(c74d4480,5,bfbfe9f0,0,0,0,0,8) at kern_select+0x4cf select(c74d4480,e9a4acf8,14,c74d4480,c0cad218,...) at select+0x5e syscall(e9a4ad38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x80880df, esp = 0xbfbfe9bc, ebp = 0xbfbfee98 --- Tracing command softdepflush pid 27 tid 100056 td 0xc72b0240 sched_switch(c72b0240,0,104,186,82df03ef,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,44,...) at mi_switch+0x200 sleepq_switch(c72b0240,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e981a0,44,c0bf3334,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e981a0,c0e98144,44,c0bf3334,3e8,...) at _sleep+0x329 softdep_flush(0,e7784d38,c0bcbb55,327,c72b229c,...) at softdep_flush+0x2b0 fork_exit(c0a02a20,0,e7784d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7784d70, ebp = 0 --- Tracing command syncer pid 26 tid 100055 td 0xc72b0480 sched_switch(c72b0480,0,104,186,40fef60c,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c72b0480,0,c0bd3a0b,266,c72b0480,...) at sleepq_switch+0x15f sleepq_timedwait(c0e8c350,0,e7781c88,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0e8c350,c0e8c33c,3e8,6ac,4e20,...) at _cv_timedwait+0x250 sched_sync(0,e7781d38,c0bcbb55,327,c72b2538,...) at sched_sync+0x502 fork_exit(c088be70,0,e7781d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7781d70, ebp = 0 --- Tracing command vnlru pid 25 tid 100054 td 0xc72b06c0 sched_switch(c72b06c0,0,104,186,2afe7304,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,50,...) at mi_switch+0x200 sleepq_switch(c72b06c0,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c72b27d4,50,c0bdd3d2,0,0,...) at sleepq_timedwait+0x6b _sleep(c72b27d4,c0e8c310,250,c0bdd3d2,3e8,...) at _sleep+0x329 vnlru_proc(0,e777ed38,c0bcbb55,327,c72b27d4,...) at vnlru_proc+0x10b fork_exit(c088ca00,0,e777ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe777ed70, ebp = 0 --- Tracing command bufdaemon pid 24 tid 100053 td 0xc72b0900 sched_switch(c72b0900,0,104,186,8cdcc438,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,44,...) at mi_switch+0x200 sleepq_switch(c72b0900,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e8c0a4,44,c0bdad94,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e8c0a4,c0e8c0a8,44,c0bdad94,3e8,...) at _sleep+0x329 buf_daemon(0,e777bd38,c0bcbb55,327,c6eb77d4,...) at buf_daemon+0x21e fork_exit(c08774c0,0,e777bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe777bd70, ebp = 0 --- Tracing command pagezero pid 23 tid 100052 td 0xc72b0b40 sched_switch(c72b0b40,0,104,186,9e8f0d89,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c72b0b40,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e98d94,0,c0bf8385,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e98d94,c0e98950,0,c0bf8385,493e0,...) at _sleep+0x329 vm_pagezero(0,e7778d38,c0bcbb55,327,c6eb7a70,...) at vm_pagezero+0xdc fork_exit(c0a3cb30,0,e7778d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7778d70, ebp = 0 --- Tracing command vmdaemon pid 22 tid 100051 td 0xc72b0d80 sched_switch(c72b0d80,0,104,186,2157afab,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,68,...) at mi_switch+0x200 sleepq_switch(c72b0d80,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c0e989bc,68,c0bdad94,0,0,...) at sleepq_wait+0x63 _sleep(c0e989bc,c0e989c0,68,c0bdad94,0,...) at _sleep+0x35b vm_daemon(0,e7775d38,c0bcbb55,327,c6eb7d0c,...) at vm_daemon+0x59 fork_exit(c0a36fd0,0,e7775d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7775d70, ebp = 0 --- Tracing command pagedaemon pid 9 tid 100050 td 0xc6f4f6c0 sched_switch(c6f4f6c0,0,104,186,587eadc0,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,44,...) at mi_switch+0x200 sleepq_switch(c6f4f6c0,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e98984,44,c0bdad94,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e98984,c0e98950,44,c0bdad94,1388,...) at _sleep+0x329 vm_pageout(0,e7772d38,c0bcbb55,327,c6fcb000,...) at vm_pageout+0x2bb fork_exit(c0a37e70,0,e7772d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7772d70, ebp = 0 --- Tracing command sctp_iterator pid 8 tid 100049 td 0xc6f4f900 sched_switch(c6f4f900,0,104,186,98ee8af3,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6f4f900,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c0e8dd80,0,c0be6626,0,0,...) at sleepq_wait+0x63 _sleep(c0e8dd80,c0e8dc94,0,c0be6626,0,...) at _sleep+0x35b sctp_iterator_thread(0,e776fd38,c0bcbb55,327,c6fcb29c,...) at sctp_iterator_thread+0x60 fork_exit(c0900d90,0,e776fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe776fd70, ebp = 0 --- Tracing command fdc0 pid 7 tid 100046 td 0xc6fcc000 sched_switch(c6fcc000,0,104,186,5134b600,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,4c,...) at mi_switch+0x200 sleepq_switch(c6fcc000,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6fd823c,4c,c0bc6814,0,0,...) at sleepq_timedwait+0x6b _sleep(c6fd823c,c6fd82f0,4c,c0bc6814,3e8,...) at _sleep+0x329 fdc_thread(c6fd8200,e7763d38,c0bcbb55,327,c6fcb538,...) at fdc_thread+0x2be fork_exit(c0adf2d0,c6fd8200,e7763d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7763d70, ebp = 0 --- Tracing command fw0_probe pid 6 tid 100044 td 0xc6fcc480 sched_switch(c6fcc480,0,104,186,15febb5b,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6fcc480,0,c0bd3a0b,189,c6fd9000,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c6fd9000,5c,c0bc6814,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6fd9000,c6fdd488,15c,c0bc6814,0,...) at _sleep+0x344 fw_bus_probe_thread(c6fd9000,e7753d38,c0bcbb55,327,c6fcb7d4,...) at fw_bus_probe_thread+0x721 fork_exit(c06067d0,c6fd9000,e7753d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7753d70, ebp = 0 --- Tracing command usb5 pid 21 tid 100041 td 0xc6fccb40 sched_switch(c6fccb40,0,104,186,ea4bd1cb,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6fccb40,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6fa9a10,5c,c0bc2a5c,0,0,...) at sleepq_timedwait+0x6b _sleep(c6fa9a10,0,5c,c0bc2a5c,ea60,...) at _sleep+0x329 usb_event_thread(c6f8e8c0,e773dd38,c0bcbb55,327,c6fcba70,...) at usb_event_thread+0xca fork_exit(c0769f20,c6f8e8c0,e773dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe773dd70, ebp = 0 --- Tracing command usb4 pid 20 tid 100039 td 0xc6fcd000 sched_switch(c6fcd000,0,104,186,ea30639f,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6fcd000,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6fc6210,5c,c0bc2a5c,0,0,...) at sleepq_timedwait+0x6b _sleep(c6fc6210,0,5c,c0bc2a5c,ea60,...) at _sleep+0x329 usb_event_thread(c6fba880,e7727d38,c0bcbb55,327,c6fcbd0c,...) at usb_event_thread+0xca fork_exit(c0769f20,c6fba880,e7727d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7727d70, ebp = 0 --- Tracing command usb3 pid 19 tid 100038 td 0xc6f39d80 sched_switch(c6f39d80,0,104,186,ea49da8b,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6f39d80,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6fb2210,5c,c0bc2a5c,0,0,...) at sleepq_timedwait+0x6b _sleep(c6fb2210,0,5c,c0bc2a5c,ea60,...) at _sleep+0x329 usb_event_thread(c6fbb100,e7723d38,c0bcbb55,327,c6d6529c,...) at usb_event_thread+0xca fork_exit(c0769f20,c6fbb100,e7723d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7723d70, ebp = 0 --- Tracing command usb2 pid 18 tid 100037 td 0xc6f4e000 sched_switch(c6f4e000,0,104,186,ea6732bb,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6f4e000,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6fb1210,5c,c0bc2a5c,0,0,...) at sleepq_timedwait+0x6b _sleep(c6fb1210,0,5c,c0bc2a5c,ea60,...) at _sleep+0x329 usb_event_thread(c6f82340,e771fd38,c0bcbb55,327,c6d65538,...) at usb_event_thread+0xca fork_exit(c0769f20,c6f82340,e771fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe771fd70, ebp = 0 --- Tracing command usb1 pid 17 tid 100035 td 0xc6f4e480 sched_switch(c6f4e480,0,104,186,ea4bcd20,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6f4e480,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6f83210,5c,c0bc2a5c,0,0,...) at sleepq_timedwait+0x6b _sleep(c6f83210,0,5c,c0bc2a5c,ea60,...) at _sleep+0x329 usb_event_thread(c6f82a40,e7718d38,c0bcbb55,327,c6d657d4,...) at usb_event_thread+0xca fork_exit(c0769f20,c6f82a40,e7718d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7718d70, ebp = 0 --- Tracing command usbtask-dr pid 16 tid 100033 td 0xc6f4e900 sched_switch(c6f4e900,0,104,186,ec73aef4,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6f4e900,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c0d1d034,5c,c0bc2a4e,0,0,...) at sleepq_wait+0x63 _sleep(c0d1d034,0,5c,c0bc2a4e,0,...) at _sleep+0x35b usb_task_thread(c0d1d034,e7711d38,c0bcbb55,327,c6d65a70,...) at usb_task_thread+0x62 fork_exit(c0769de0,c0d1d034,e7711d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7711d70, ebp = 0 --- Tracing command usbtask-hc pid 15 tid 100032 td 0xc6f4eb40 sched_switch(c6f4eb40,0,104,186,ec739e78,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6f4eb40,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c0d1d020,5c,c0bc2a4e,0,0,...) at sleepq_wait+0x63 _sleep(c0d1d020,0,5c,c0bc2a4e,0,...) at _sleep+0x35b usb_task_thread(c0d1d020,e770ed38,c0bcbb55,327,c6d65d0c,...) at usb_task_thread+0x62 fork_exit(c0769de0,c0d1d020,e770ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe770ed70, ebp = 0 --- Tracing command usb0 pid 14 tid 100031 td 0xc6f4ed80 sched_switch(c6f4ed80,0,104,186,ea653161,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6f4ed80,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6f8a210,5c,c0bc2a5c,0,0,...) at sleepq_timedwait+0x6b _sleep(c6f8a210,0,5c,c0bc2a5c,ea60,...) at _sleep+0x329 usb_event_thread(c6f8d3c0,e770bd38,c0bcbb55,327,c6eb7000,...) at usb_event_thread+0xca fork_exit(c0769f20,c6f8d3c0,e770bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe770bd70, ebp = 0 --- Tracing command xpt_thrd pid 5 tid 100020 td 0xc6f396c0 sched_switch(c6f396c0,0,104,186,ec7372d7,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,4c,...) at mi_switch+0x200 sleepq_switch(c6f396c0,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c0cec154,4c,c0b737dd,0,0,...) at sleepq_wait+0x63 _sleep(c0cec154,c0cec16c,4c,c0b737dd,0,...) at _sleep+0x35b xpt_scanner_thread(0,c6ac7d38,c0bcbb55,327,c6eb729c,...) at xpt_scanner_thread+0x41 fork_exit(c0479b80,0,c6ac7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac7d70, ebp = 0 --- Tracing command yarrow pid 13 tid 100017 td 0xc6d68240 sched_switch(c6d68240,0,104,186,82f05b4c,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6d68240,0,c0bd3a0b,266,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0d1fb04,0,c0bc6814,2,0,...) at sleepq_timedwait+0x6b _sleep(c0d1fb04,0,0,c0bc6814,64,...) at _sleep+0x329 pause(c0bc6814,64,c0bb84f6,112,0,...) at pause+0x47 random_kthread(0,c6abed38,c0bcbb55,327,c6eb7538,...) at random_kthread+0x1d9 fork_exit(c06d80d0,0,c6abed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abed70, ebp = 0 --- Tracing command g_down pid 4 tid 100015 td 0xc6d686c0 sched_switch(c6d686c0,0,104,186,8d189c94,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,4c,...) at mi_switch+0x200 sleepq_switch(c6d686c0,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d1d764,4c,c0bc6814,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d1d764,c0d1d6c8,24c,c0bc6814,64,...) at _sleep+0x329 g_io_schedule_down(c6d686c0,0,c0bc7ee1,74,0,...) at g_io_schedule_down+0x6b g_down_procbody(0,c6ab8d38,c0bcbb55,327,c6d64000,...) at g_down_procbody+0x8d fork_exit(c07adc90,0,c6ab8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab8d70, ebp = 0 --- Tracing command g_up pid 3 tid 100014 td 0xc6d68900 sched_switch(c6d68900,0,104,186,8dcc9337,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,4c,...) at mi_switch+0x200 sleepq_switch(c6d68900,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d1d760,4c,c0bc6814,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d1d760,c0d1d6e8,24c,c0bc6814,64,...) at _sleep+0x329 g_io_schedule_up(c6d68900,0,c0bc7ee1,5d,0,...) at g_io_schedule_up+0x133 g_up_procbody(0,c6ab5d38,c0bcbb55,327,c6d6429c,...) at g_up_procbody+0x8d fork_exit(c07add20,0,c6ab5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab5d70, ebp = 0 --- Tracing command g_event pid 2 tid 100013 td 0xc6d68b40 sched_switch(c6d68b40,0,104,186,7faa7397,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,4c,...) at mi_switch+0x200 sleepq_switch(c6d68b40,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d1d758,4c,c0bc6814,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d1d758,0,4c,c0bc6814,64,...) at _sleep+0x329 g_event_procbody(0,c6ab2d38,c0bcbb55,327,c6d64538,...) at g_event_procbody+0xcb fork_exit(c07addb0,0,c6ab2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab2d70, ebp = 0 --- Tracing command intr pid 12 tid 100048 td 0xc6f4fb40 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100047 td 0xc6f4fd80 sched_switch(c6f4fd80,0,109,186,ec6fd27f,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6d61d70,...) at mi_switch+0x200 ithread_loop(c70c29b0,e7766d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c70c29b0,e7766d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7766d70, ebp = 0 --- Tracing command intr pid 12 tid 100045 td 0xc6fcc240 sched_switch(c6fcc240,0,109,186,276cb05e,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c70b5bf0,...) at mi_switch+0x200 ithread_loop(c70b8b80,e7756d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c70b8b80,e7756d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7756d70, ebp = 0 --- Tracing command intr pid 12 tid 100042 td 0xc6fcc900 sched_switch(c6fcc900,0,109,186,fd1ec81a,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6dab0f0,...) at mi_switch+0x200 ithread_loop(c6fca9e0,e7748d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6fca9e0,e7748d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7748d70, ebp = 0 --- Tracing command intr pid 12 tid 100040 td 0xc6fccd80 sched_switch(c6fccd80,0,109,186,89081fec,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daad70,...) at mi_switch+0x200 ithread_loop(c6fbc920,e772bd38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6fbc920,e772bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe772bd70, ebp = 0 --- Tracing command intr pid 12 tid 100036 td 0xc6f4e240 sched_switch(c6f4e240,0,109,186,ec69cebc,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daadf0,...) at mi_switch+0x200 ithread_loop(c6f8c3f0,e771cd38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6f8c3f0,e771cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe771cd70, ebp = 0 --- Tracing command intr pid 12 tid 100034 td 0xc6f4e6c0 sched_switch(c6f4e6c0,0,109,186,ec695883,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daae70,...) at mi_switch+0x200 ithread_loop(c6f8c790,e7715d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6f8c790,e7715d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7715d70, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc6f4f000 sched_switch(c6f4f000,0,109,186,ec68e15c,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daaef0,...) at mi_switch+0x200 ithread_loop(c6f8c050,e7708d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6f8c050,e7708d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7708d70, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc6f4f240 sched_switch(c6f4f240,0,109,186,9056a45c,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daabf0,...) at mi_switch+0x200 ithread_loop(c6f76430,e72e4d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6f76430,e72e4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe72e4d70, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc6f4f480 sched_switch(c6f4f480,0,109,186,ec70c392,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6d61970,...) at mi_switch+0x200 ithread_loop(c6f0c470,c6b05d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6f0c470,c6b05d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6b05d70, ebp = 0 --- Tracing command intr pid 12 tid 100027 td 0xc6dac6c0 sched_switch(c6dac6c0,0,109,186,147900a4,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6f28070,...) at mi_switch+0x200 ithread_loop(c6ed28c0,c6adcd38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6ed28c0,c6adcd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6adcd70, ebp = 0 --- Tracing command intr pid 12 tid 100025 td 0xc6dacb40 sched_switch(c6dacb40,0,109,186,89085a69,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6f280f0,...) at mi_switch+0x200 ithread_loop(c6ed28e0,c6ad6d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6ed28e0,c6ad6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad6d70, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc6f39480 sched_switch(c6f39480,0,109,186,faa9aeaf,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6f28170,...) at mi_switch+0x200 ithread_loop(c6ed2900,c6acad38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6ed2900,c6acad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6acad70, ebp = 0 --- Tracing command intr pid 12 tid 100018 td 0xc6f39b40 sched_switch(c6f39b40,0,109,186,539126d4,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6f28370,...) at mi_switch+0x200 ithread_loop(c6ed2990,c6ac1d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6ed2990,c6ac1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac1d70, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc6d68d80 sched_switch(c6d68d80,0,109,186,82f01428,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daa870,...) at mi_switch+0x200 ithread_loop(c6d630f0,c6aafd38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6d630f0,c6aafd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aafd70, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc6dac000 sched_switch(c6dac000,0,109,186,8cdca911,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daa8f0,...) at mi_switch+0x200 ithread_loop(c6d63100,c6aacd38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6d63100,c6aacd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aacd70, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc6dac240 sched_switch(c6dac240,0,109,186,8fca552a,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daa970,...) at mi_switch+0x200 ithread_loop(c6d63110,c6aa9d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6d63110,c6aa9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa9d70, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc6dac480 sched_switch(c6dac480,0,109,186,891b6fef,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daa9f0,...) at mi_switch+0x200 ithread_loop(c6d63120,c6aa6d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6d63120,c6aa6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa6d70, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc6d66000 sched_switch(c6d66000,0,109,186,8954c709,...) at sched_switch+0x437 mi_switch(109,0,c0bcbddb,4d7,c6daaa70,...) at mi_switch+0x200 ithread_loop(c6d63130,c6aa3d38,c0bcbb55,327,c6d647d4,...) at ithread_loop+0x1f6 fork_exit(c07e4560,c6d63130,c6aa3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa3d70, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc6d66240 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc6d66480 sched_switch(c6d66480,0,108,181,891e9c54,...) at sched_switch+0x437 mi_switch(108,0,c0bd13a2,3a5,c6d66480,...) at mi_switch+0x200 sched_idletd(0,c6a9cd38,c0bcbb55,327,c6d64a70,...) at sched_idletd+0x178 fork_exit(c0824630,0,c6a9cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a9cd70, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc6d666c0 cpustop_handler(2,c6a99c40,c0b208bd,63,3f8,...) at cpustop_handler+0x32 ipi_nmi_handler(63,3f8,0,c6fcf300,c6d64a70,...) at ipi_nmi_handler+0x2f trap(c6a99c4c) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0824625, esp = 0xc6a99c8c, ebp = 0xc6a99cb4 --- cpu_search_highest(c0d32250,c6a99cd8,c0bd13a2,3a7,c6d666c0,...) at cpu_search_highest+0x115 sched_idletd(0,c6a99d38,c0bcbb55,327,c6d64a70,...) at sched_idletd+0xcf fork_exit(c0824630,0,c6a99d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a99d70, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc6d66900 sched_switch(c6d66900,0,108,181,891be55a,...) at sched_switch+0x437 mi_switch(108,0,c0bd13a2,3a5,c6d66900,...) at mi_switch+0x200 sched_idletd(0,c6a96d38,c0bcbb55,327,c6d64a70,...) at sched_idletd+0x178 fork_exit(c0824630,0,c6a96d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a96d70, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc6d66b40 sched_switch(c6d66b40,0,108,181,8923413c,...) at sched_switch+0x437 mi_switch(108,0,c0bd13a2,3a5,c6d66b40,...) at mi_switch+0x200 sched_idletd(0,c6a93d38,c0bcbb55,327,c6d64a70,...) at sched_idletd+0x178 fork_exit(c0824630,0,c6a93d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a93d70, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc6d66d80 sched_switch(c6d66d80,0,104,186,45f8b61c,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,5c,...) at mi_switch+0x200 sleepq_switch(c6d66d80,0,c0bd3a0b,189,c6d64d0c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0bd3a0b,159,0,100,100,...) at sleepq_catch_signals+0xb4 sleepq_wait_sig(c6d64d0c,5c,c0bd60d6,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6d64d0c,c6d64d94,15c,c0bd60d6,0,...) at _sleep+0x344 kern_wait(c6d66d80,ffffffff,c6a8fc74,0,0,...) at kern_wait+0xab6 wait4(c6d66d80,c6a8fcf8,10,c0bd5ef1,c0caca08,...) at wait4+0x3b syscall(c6a8fd38) at syscall+0x2a3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x8054c2f, esp = 0xbfbfe94c, ebp = 0xbfbfe968 --- Tracing command audit pid 10 tid 100001 td 0xc6d68000 sched_switch(c6d68000,0,104,186,ec711954,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6d68000,0,c0bd3a0b,243,c6d68000,...) at sleepq_switch+0x15f sleepq_wait(c0e97b60,0,c6a8cc9c,1,0,...) at sleepq_wait+0x63 _cv_wait(c0e97b60,c0e97b44,c0bf124a,194,0,...) at _cv_wait+0x240 audit_worker(0,c6a8cd38,c0bcbb55,327,c6d65000,...) at audit_worker+0x84 fork_exit(c09e63c0,0,c6a8cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a8cd70, ebp = 0 --- Tracing command kernel pid 0 tid 100043 td 0xc6fcc6c0 sched_switch(c6fcc6c0,0,104,186,879b4a74,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6fcc6c0,0,c0bd3a0b,243,c6fcc6c0,...) at sleepq_switch+0x15f sleepq_wait(c6fbbe80,0,c0bd03ce,c0bc6814,0,...) at sleepq_wait+0x63 msleep_spin(c6fbbe80,c6fbbe9c,c0bc6814,0,c0bce135,...) at msleep_spin+0x21d taskqueue_thread_loop(c6fdd49c,e7750d38,c0bcbb55,327,c0d1d820,...) at taskqueue_thread_loop+0x8a fork_exit(c083ad40,c6fdd49c,e7750d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe7750d70, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc6dac900 sched_switch(c6dac900,0,104,186,faab807c,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6dac900,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c6f1e500,0,c0bc6814,0,0,...) at sleepq_wait+0x63 _sleep(c6f1e500,c6f1e51c,0,c0bc6814,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d32448,c6ad9d38,c0bcbb55,327,c0d1d820,...) at taskqueue_thread_loop+0xb4 fork_exit(c083ad40,c0d32448,c6ad9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad9d70, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc6dacd80 sched_switch(c6dacd80,0,104,186,eca1b683,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6dacd80,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c6f1e5c0,0,c0bc6814,0,0,...) at sleepq_wait+0x63 _sleep(c6f1e5c0,c6f1e5dc,0,c0bc6814,0,...) at _sleep+0x35b taskqueue_thread_loop(c0ceee04,c6ad3d38,c0bcbb55,327,c0d1d820,...) at taskqueue_thread_loop+0xb4 fork_exit(c083ad40,c0ceee04,c6ad3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad3d70, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc6f39000 sched_switch(c6f39000,0,104,186,eca1a577,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6f39000,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c6f1e5c0,0,c0bc6814,0,0,...) at sleepq_wait+0x63 _sleep(c6f1e5c0,c6f1e5dc,0,c0bc6814,0,...) at _sleep+0x35b taskqueue_thread_loop(c0ceee04,c6ad0d38,c0bcbb55,327,c0d1d820,...) at taskqueue_thread_loop+0xb4 fork_exit(c083ad40,c0ceee04,c6ad0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad0d70, ebp = 0 --- Tracing command kernel pid 0 tid 100022 td 0xc6f39240 sched_switch(c6f39240,0,104,186,eca192fc,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6f39240,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c6f1e5c0,0,c0bc6814,0,0,...) at sleepq_wait+0x63 _sleep(c6f1e5c0,c6f1e5dc,0,c0bc6814,0,...) at _sleep+0x35b taskqueue_thread_loop(c0ceee04,c6acdd38,c0bcbb55,327,c0d1d820,...) at taskqueue_thread_loop+0xb4 fork_exit(c083ad40,c0ceee04,c6acdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6acdd70, ebp = 0 --- Tracing command kernel pid 0 tid 100019 td 0xc6f39900 sched_switch(c6f39900,0,104,186,ec9ab725,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6f39900,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c6f1e700,0,c0bc6814,0,0,...) at sleepq_wait+0x63 _sleep(c6f1e700,c6f1e71c,0,c0bc6814,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d1e0c0,c6ac4d38,c0bcbb55,327,c0d1d820,...) at taskqueue_thread_loop+0xb4 fork_exit(c083ad40,c0d1e0c0,c6ac4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac4d70, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc6d68480 sched_switch(c6d68480,0,104,186,12558c80,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,0,...) at mi_switch+0x200 sleepq_switch(c6d68480,0,c0bd3a0b,243,0,...) at sleepq_switch+0x15f sleepq_wait(c6d4bd80,0,c0bc6814,0,0,...) at sleepq_wait+0x63 _sleep(c6d4bd80,c6d4bd9c,0,c0bc6814,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d30f80,c6abbd38,c0bcbb55,327,c0d1d820,...) at taskqueue_thread_loop+0xb4 fork_exit(c083ad40,c0d30f80,c6abbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abbd70, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc0d1dac0 sched_switch(c0d1dac0,0,104,186,17e630d6,...) at sched_switch+0x437 mi_switch(104,0,c0bd3a0b,1ce,44,...) at mi_switch+0x200 sleepq_switch(c0d1dac0,0,c0bd3a0b,266,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0d1d820,44,c0bd1c2a,0,0,...) at sleepq_timedwait+0x6b _sleep(c0d1d820,0,44,c0bd1c2a,2710,...) at _sleep+0x329 scheduler(0,141ec00,141ec00,141e000,1425000,...) at scheduler+0x23e mi_startup() at mi_startup+0x96 begin() at begin+0x2c db> call doadump Physical memory: 3439 MB Dumping 245 MB: 230 214 198 182 166 150 134 118 102 86 70 54 38 22 6 Dump complete = 0xf db> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 3 (kgdb) bt #0 doadump () at pcpu.h:246 #1 0xc04bc459 in db_fncall (dummy1=1, dummy2=0, dummy3=0, dummy4=0xe9a999ec "") at ../../../ddb/db_command.c:548 #2 0xc04bc851 in db_command (last_cmdp=0xc0ced0dc, cmd_table=0x0, dopager=1) at ../../../ddb/db_command.c:445 #3 0xc04bc9aa in db_command_loop () at ../../../ddb/db_command.c:498 #4 0xc04be80d in db_trap (type=3, code=0) at ../../../ddb/db_main.c:229 #5 0xc08302a6 in kdb_trap (type=3, code=0, tf=0xe9a99b94) at ../../../kern/subr_kdb.c:534 #6 0xc0b20e5b in trap (frame=0xe9a99b94) at ../../../i386/i386/trap.c:680 #7 0xc0b04c9b in calltrap () at ../../../i386/i386/exception.s:165 #8 0xc083042a in kdb_enter (why=0xc0bcf865 "panic", msg=0xc0bcf865 "panic") at cpufunc.h:71 #9 0xc0802d36 in panic (fmt=0xc0b73411 "Bad link elm %p prev->next != elm") at ../../../kern/kern_shutdown.c:559 #10 0xc09d51a9 in xprt_unregister_locked (xprt=0xc735f800) at ../../../rpc/svc.c:299 #11 0xc09d5285 in xprt_unregister (xprt=0xc735f800) at ../../../rpc/svc.c:309 #12 0xc09d708a in svc_run_internal (pool=0xc71a0900, ismaster=0) at ../../../rpc/svc.c:855 #13 0xc09d75d0 in svc_thread_start (arg=0xc71a0900) at ../../../rpc/svc.c:1188 #14 0xc07e1748 in fork_exit (callout=0xc09d75c0 , arg=0xc71a0900, frame=0xe9a99d38) at ../../../kern/kern_fork.c:815 #15 0xc0b04d10 in fork_trampoline () at ../../../i386/i386/exception.s:270 (kgdb) f 10 #10 0xc09d51a9 in xprt_unregister_locked (xprt=0xc735f800) at ../../../rpc/svc.c:299 299 TAILQ_REMOVE(&pool->sp_xlist, xprt, xp_link); (kgdb) l 294 295 if (xprt->xp_active) { 296 TAILQ_REMOVE(&pool->sp_active, xprt, xp_alink); 297 xprt->xp_active = FALSE; 298 } 299 TAILQ_REMOVE(&pool->sp_xlist, xprt, xp_link); 300 xprt->xp_registered = FALSE; 301 } 302 303 void (kgdb) info loc pool = (SVCPOOL *) 0xc71a0900 (kgdb) p *pool $1 = {sp_lock = {lock_object = {lo_name = 0xc0bf02d9 "sp_lock", lo_flags = 16973824, lo_data = 0, lo_witness = 0xc6d23480}, mtx_lock = 3343531392}, sp_name = 0xc0beedcd "nfsd", sp_state = SVCPOOL_ACTIVE, sp_proc = 0xc733229c, sp_xlist = {tqh_first = 0xc7480800, tqh_last = 0xc7480a1c}, sp_active = {tqh_first = 0x0, tqh_last = 0xc71a0928}, sp_callouts = {tqh_first = 0xc72a6ae0, tqh_last = 0xc72a6100}, sp_threads = {lh_first = 0xc73a2cc0}, sp_idlethreads = {lh_first = 0xc72a0100}, sp_minthreads = 4, sp_maxthreads = 4, sp_threadcount = 4, sp_lastcreatetime = 14, sp_lastidlecheck = 25717, sp_assign = 0xc09ae050 , sp_done = 0xc09adf70 , sp_space_low = 8738133, sp_space_high = 13107200, sp_space_used = 0, sp_space_used_highest = 753144, sp_space_throttled = 0, sp_space_throttle_count = 0, sp_rcache = 0xc70a8000, sp_sysctl = {tqh_first = 0xc70d5060, tqh_last = 0xc70d27f4}} (kgdb) -- Test scenario: crossmp2.sh