GENERIC HEAD from 2009-06-04 22:00:48 UTC, r193481M, vmcore.185 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2009 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 8.0-CURRENT #1 r193481M: Fri Jun 5 18:39:22 CEST 2009 pho@x4.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1799.99-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) avail memory = 3539730432 (3375 MB) : Trying to mount root from ufs:/dev/ad4s1a WARNING: / was not properly dismounted Entropy harvesting: interrupts ethernet point_to_point kickstart. /dev/ad4s1a: 17850 files, 764673 used, 248342 free (2390 frags, 30744 blocks, 0.2% fragmentation) /dev/ad4s1e: INCORRECT BLOCK COUNT I=12529773 (48 should be 4) (CORRECTED) /dev/ad4s1e: UNREF FILE I=12529682 OWNER=4294967294 MODE=100660 /dev/ad4s1e: SIZE=0 MTIME=Jun 5 21:09 2009 (CLEARED) /dev/ad4s1e: UNREF FILE I=12529698 OWNER=4294967294 MODE=100660 : /dev/ad4s1d: FREE BLK COUNT(S) WRONG IN SUPERBLK (SALVAGED) /dev/ad4s1d: SUMMARY INFORMATION BAD (SALVAGED) /dev/ad4s1d: BLK(S) MISSING IN BIT MAPS (SALVAGED) /dev/ad4s1d: 897908 files, 31767938 used, 19009137 free (105393 frags, 2362968 blocks, 0.2% fragmentation) re0: link state changed to DOWN Starting Network: lo0 re0. add net default: gateway 192.168.1.1 savecore: reboot after panic: vml_pageout_flush: ock order reversal:partially invali d page 0xc37f459 1st 0xe6d134e4 0 index 0/1 Junbufwait (bufwait 5 21:20:39 x4 ) @ kern/vfs_biosavecore: reboot.c:2558 after panic: vm 2nd 0xc735880_pageout_flush: 0 dirhash partially invali(dirhash) @ d page 0xc37f459ufs/uf0 index 0/1 s/ufs_dirhash.c:285 KDB: stack backtrace: db_trace_self_wrapper(c0c5a420,f62d6778,c08a9885,c089a50b,c0c5d335,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c089a50b,c0c5d335,c6d2cd58,c6d30020,f62d67d4,...) at kdb_backtrace+0x29 _witness_debugger(c0c5d335,c7358800,c0c7dcc4,c6d30020,c0c7d96a,...) at _witness_debugger+0x25 witness_checkorder(c7358800,9,c0c7d961,11d,0,...) at witness_checkorder+0x839 _sx_xlock(c7358800,0,c0c7d961,11d,e761daec,...) at _sx_xlock+0x85 ufsdirhash_acquire(c77b132c,e761daec,f62d689c,c0a9ac44,c77b7b54,...) at ufsdirhash_acquire+0x48 ufsdirhash_move(c77b7b54,e761daec,aec,ae4,f62d688c,...) at ufsdirhash_move+0xf ufs_direnter(c77b4158,c77bb000,f62d68e4,f62d6bd4,0,...) at ufs_direnter+0x644 ufs_makeinode(f62d6bd4,c0d5c240,f62d6acc,f62d6a2c,c0b994c5,...) at ufs_makeinode+0x55f ufs_create(f62d6acc,c0c92c65,0,0,f62d6ba8,...) at ufs_create+0x30 VOP_CREATE_APV(c0d5bd00,f62d6acc,f62d6bd4,f62d6a64,c0c51db4,...) at VOP_CREATE_APV+0xc5 vn_open_cred(f62d6ba8,f62d6c5c,180,c6d45300,c74230e0,...) at vn_open_cred+0x1d0 vn_open(f62d6ba8,f62d6c5c,180,c74230e0,c7420b40,...) at vn_open+0x33 kern_openat(c7420b40,ffffff9c,28300000,0,602,...) at kern_openat+0x108 kern_open(c7420b40,28300000,0,601,180,...) at kern_open+0x35 open(c7420b40,f62d6cf8,c,c0c77e47,c0d3b14c,...) at open+0x30 syscall(f62d6d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x2817a343, esp = 0xbfbfe2dc, ebp = 0xbfbfedf8 --- savecore: writing core to vmcore.184 Writing crash summary to /var/crash/core.txt.184. lock order reversal: 1st 0xc767522c filedesc structure (filedesc structure) @ kern/sys_generic.c:1211 2nd 0xc77bdb18 ufs (ufs) @ kern/vfs_vnops.c:863 KDB: stack backtrace: db_trace_self_wrapper(c0c5a420,f630894c,c08a9885,c089a50b,c0c5d335,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c089a50b,c0c5d335,c6d2cfc8,c6d2ffb8,f63089a8,...) at kdb_backtrace+0x29 _witness_debugger(c0c5d335,c77bdb18,c0c50040,c6d2ffb8,c0c65832,...) at _witness_debugger+0x25 witness_checkorder(c77bdb18,9,c0c65829,35f,c77bdb80,...) at witness_checkorder+0x839 __lockmgr_args(c77bdb18,80400,c77bdb80,0,0,...) at __lockmgr_args+0x7a7 ffs_lock(f6308ab4,c779fc08,f6308b5c,80400,c77bdac0,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0d5bd00,f6308ab4,7c8,c0d746c0,c77bdac0,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c77bdac0,80400,c0c65829,35f,0,...) at _vn_lock+0x78 vn_poll(c7424508,1,c6d45300,c765e000,f6308b5c,...) at vn_poll+0x77 poll(c765e000,f6308cf8,c,c0c6e07e,c0d3c79c,...) at poll+0x20c syscall(f6308d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (209, FreeBSD ELF32, poll), eip = 0x283b107f, esp = 0xbfbfe21c, ebp = 0xbfbfe248 --- Additional ABI support: linux. rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered rpc.umntall: 127.0.0.1: RPCPROG_MNT: RPC: Program not registered Configuring syscons: keymap blanktime. Local package initialization:lock order reversal: 1st 0xc7509868 ufs (ufs) @ kern/vfs_mount.c:1198 2nd 0xc7506308 devfs (devfs) @ ufs/ffs/ffs_softdep.c:1029 KDB: stack backtrace: db_trace_self_wrapper(c0c5a420,f63049f0,c08a9885,c089a50b,c0c5d335,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c089a50b,c0c5d335,c6d2ffb8,c6d2fe80,f6304a4c,...) at kdb_backtrace+0x29 _witness_debugger(c0c5d335,c7506308,c0c4c1f9,c6d2fe80,c0c7bd05,...) at _witness_debugger+0x25 witness_checkorder(c7506308,9,c0c7bcfc,405,c7506370,...) at witness_checkorder+0x839 __lockmgr_args(c7506308,80400,c7506370,0,0,...) at __lockmgr_args+0x7a7 vop_stdlock(f6304b54,4,c0c5591d,80400,c75062b0,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0d37d40,f6304b54,c0f27d64,c0d746c0,c75062b0,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c75062b0,80400,c0c7bcfc,405,c76405a0,...) at _vn_lock+0x78 softdep_flushworklist(c76405a0,f6304c00,c732b000,52c,c7509158,...) at softdep_flushworklist+0x47 ffs_sync(c76405a0,1,c0c63bff,4f3,80,...) at ffs_sync+0x2fd dounmount(c76405a0,8000000,c732b000,479,ff1ca3a5,...) at dounmount+0x44e unmount(c732b000,f6304cf8,8,f6304d38,c0d3b328,...) at unmount+0x2df syscall(f6304d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d916f, esp = 0xbfbfe69c, ebp = 0xbfbfe768 --- ** /dev/ad4s1e ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 16150 files, 475371 used, 50301704 free (1912 frags, 6287474 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Fri Jun 5 21:21:48 CEST 2009 FreeBSD/i386 (x4.osted.lan) (console) login: Jun 5 22:23:50 x4 su: pho to root on /dev/pts/1 lock order reversal: 1st 0xc7da0b18 ufs (ufs) @ kern/vfs_lookup.c:497 2nd 0xe6daf034 bufwait (bufwait) @ ufs/ffs/ffs_softdep.c:6150 3rd 0xc804ec70 ufs (ufs) @ kern/vfs_subr.c:2084 KDB: stack backtrace: db_trace_self_wrapper(c0c5a420,f63aa14c,c08a9885,c089a50b,c0c5d34e,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c089a50b,c0c5d34e,c6d2cd58,c6d2ffb8,f63aa1a8,...) at kdb_backtrace+0x29 _witness_debugger(c0c5d34e,c804ec70,c0c50040,c6d2ffb8,c0c643e5,...) at _witness_debugger+0x25 witness_checkorder(c804ec70,9,c0c643dc,824,0,...) at witness_checkorder+0x839 __lockmgr_args(c804ec70,80100,c804ecd8,0,0,...) at __lockmgr_args+0x7a7 ffs_lock(f63aa2b4,c08a962b,c0c638ca,80100,c804ec18,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0d5bd00,f63aa2b4,c74bce24,c0d746c0,c804ec18,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c804ec18,80100,c0c643dc,824,4,...) at _vn_lock+0x78 vget(c804ec18,80100,c74bcd80,50,0,...) at vget+0xbb vfs_hash_get(c76402d0,94c817,80000,c74bcd80,f63aa410,...) at vfs_hash_get+0xed ffs_vgetf(c76402d0,94c817,80000,f63aa410,1,...) at ffs_vgetf+0x49 softdep_sync_metadata(c7da0ac0,0,c0c7d5ad,131,0,...) at softdep_sync_metadata+0x5ba ffs_syncvnode(c7da0ac0,1,c74bcd80,f63aa4d0,246,...) at ffs_syncvnode+0x3e2 ffs_truncate(c7da0ac0,200,0,880,c6d45300,...) at ffs_truncate+0x696 ufs_direnter(c7da0ac0,c804ec18,f63aa78c,f63aaa8c,e6dbb42c,...) at ufs_direnter+0x946 ufs_mkdir(f63aaab0,c0c92a9f,f63aa9f8,f63aaa60,f63aaab0,...) at ufs_mkdir+0x981 VOP_MKDIR_APV(c0d5bd00,f63aaab0,f63aaadc,8,0,...) at VOP_MKDIR_APV+0xc5 nfsrv_mkdir(f63aabb0,0,f63aaba8,0,c7668000,...) at nfsrv_mkdir+0x5ec nfssvc_program(c7668000,c747e800,c0c79210,48f,f63aacdc,...) at nfssvc_program+0x395 svc_run_internal(f63aad24,c083f528,c70f8200,f63aad38,c0c52867,...) at svc_run_internal+0x942 svc_thread_start(c70f8200,f63aad38,c0c52867,335,c744c548,...) at svc_thread_start+0x10 fork_exit(c0a4c6b0,c70f8200,f63aad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Expensive timeout(9) function: 0xc071b3f0(0xc6f8f000) 0.019541509 s Expensive timeout(9) function: 0xc071b3f0(0xc6f8f000) 0.046607486 s Jun 6 08:29:19 x4 kernel: pid 43241 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:29:22 x4 kernel: pid 43240 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:29:23 x4 kernel: pid 43258 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:29:23 x4 kernel: pid 43239 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:29:48 x4 kernel: pid 43242 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:30:56 x4 kernel: pid 46431 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:13 x4 kernel: pid 62905 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:13 x4 kernel: pid 62917 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:15 x4 kernel: pid 62913 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:15 x4 kernel: pid 62912 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:24 x4 kernel: pid 62910 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:25 x4 kernel: pid 62909 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:26 x4 kernel: pid 62900 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:26 x4 kernel: pid 62918 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:28 x4 kernel: pid 62911 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:28 x4 kernel: pid 62890 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:28 x4 kernel: pid 62889 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:29 x4 kernel: pid 62908 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 08:56:30 x4 kernel: pid 62916 (lockf), uid 0, was killed: exceeded maximum CPU limit Expensive timeout(9) function: 0xc0897970(0) 0.124953116 s Jun 6 09:10:15 x4 kernel: pid 71761 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:10:39 x4 kernel: pid 72197 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:10:40 x4 kernel: pid 72198 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:10:41 x4 kernel: pid 72194 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:10:41 x4 kernel: pid 72236 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:10:49 x4 kernel: pid 71767 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:48 x4 kernel: pid 97871 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:50 x4 kernel: pid 97882 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:50 x4 kernel: pid 97868 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:52 x4 kernel: pid 97869 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:52 x4 kernel: pid 97867 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:53 x4 kernel: pid 97910 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:55 x4 kernel: pid 97875 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:56 x4 kernel: pid 97893 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:56 x4 kernel: pid 97874 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:56 x4 kernel: pid 97876 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:26:56 x4 kernel: pid 97892 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:42:54 x4 kernel: pid 5638 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:04 x4 kernel: pid 5663 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:09 x4 kernel: pid 5641 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:12 x4 kernel: pid 5667 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:14 x4 kernel: pid 5646 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:15 x4 kernel: pid 5647 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:15 x4 kernel: pid 5637 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:17 x4 kernel: pid 5666 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 6 09:43:20 x4 kernel: pid 5665 (lockf), uid 0, was killed: exceeded maximum CPU limit Expensive timeout(9) function: 0xc0b34d40(0xc0f28fc0) 0.135321983 s kernel trap 12 with interrupts disabled Fatal trap 12: page fault while in kernel mode cpuid = 1; apic id = 01 fault virtual address = 0x19c fault code = supervisor read, page not present instruction pointer = 0x20:0xc08703a7 stack pointer = 0x28:0xf6810a10 frame pointer = 0x28:0xf6810a48 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = resume, IOPL = 0 current process = 29367 (umount) [thread pid 29367 tid 100427 ] Stopped at _sx_xlock_hard+0x287: movl 0x19c(%eax),%eax db:0:kdb.enter.unknown> run pho db:1:pho> bt Tracing pid 29367 tid 100427 td 0xcfccc240 _sx_xlock_hard(c87e7bc8,cfccc240,0,c0c54490,795,...) at _sx_xlock_hard+0x287 _sx_xlock(c87e7bc8,0,c0c54490,795,c87e7bc8,...) at _sx_xlock+0xc0 lf_iteratelocks_vnode(c945b158,c0a372a0,0,f6810b14,f6810ad8,...) at lf_iteratelocks_vnode+0x4d nlm_reclaim(f6810b14,c0d59380,f6810acc,c0d59380,f6810b14,...) at nlm_reclaim+0x30 nfs_reclaim(f6810b14,c0c92843,c945b158,c945b158,cfccc240,...) at nfs_reclaim+0x4e VOP_RECLAIM_APV(c0d59380,f6810b14,c0c643dc,a0e,c945b218,...) at VOP_RECLAIM_APV+0xc5 vgonel(c945b218,0,c0c643dc,987,0,...) at vgonel+0x1c4 vflush(cd386b40,1,2,cfccc240,ce28b810,...) at vflush+0x337 nfs_unmount(cd386b40,8080000,f6810c44,4f3,80,...) at nfs_unmount+0x41 dounmount(cd386b40,8080000,cfccc240,479,3,...) at dounmount+0x46d unmount(cfccc240,f6810cf8,8,f6810d38,c0d3b328,...) at unmount+0x2df syscall(f6810d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d916f, esp = 0xbfbfe15c, ebp = 0xbfbfe228 --- db:1:bt> show allpcpu Current CPU: 1 cpuid = 0 curthread = 0xc6d7b900: pid 3 "g_up" curpcb = 0xc6ab5d90 fpcurthread = none idlethread = 0xc6d79480: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 curthread = 0xcfccc240: pid 29367 "umount" curpcb = 0xf6810d90 fpcurthread = none idlethread = 0xc6d796c0: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 curthread = 0xc74b6900: pid 20849 "lockf" curpcb = 0xf6386d90 fpcurthread = none idlethread = 0xc6d79900: pid 11 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 curthread = 0xcfcead80: pid 25118 "lockf" curpcb = 0xf66edd90 fpcurthread = none idlethread = 0xc6d79b40: pid 11 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:1:allpcpu> show alllocks Process 29367 (umount) thread 0xcfccc240 (100427) exclusive lockmgr nfs (nfs) r = 0 (0xc945b1b0) locked @ kern/vfs_subr.c:2405 exclusive lockmgr ufs (ufs) r = 0 (0xce28b868) locked @ kern/vfs_mount.c:1198 Process 25118 (lockf) thread 0xcfcead80 (100360) exclusive sleep mutex vnode interlock (vnode interlock) r = 0 (0xcb48c620) locked @ kern/kern_lockf.c:724 Process 1446 (sshd) thread 0xc741cb40 (100131) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc79e621c) locked @ kern/uipc_sockbuf.c:148 Process 1419 (sshd) thread 0xc765e900 (100174) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc7a0f86c) locked @ kern/uipc_sockbuf.c:148 Process 1418 (sshd) thread 0xc765e6c0 (100175) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc7a0fa00) locked @ kern/uipc_sockbuf.c:148 Process 1417 (sshd) thread 0xc7a0ad80 (100138) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc7a13544) locked @ kern/uipc_sockbuf.c:148 db:1:alllocks> show lockedvnods Locked vnodes 0xce28b810: tag ufs, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0xcd386b40 flags () v_object 0xc7eb14d8 ref 0 pages 0 lock type ufs: EXCL by thread 0xcfccc240 (pid 29367) #0 0xc0851f80 at __lockmgr_args+0xb80 #1 0xc0a92861 at ffs_lock+0xa1 #2 0xc0b984f5 at VOP_LOCK1_APV+0xb5 #3 0xc0902c08 at _vn_lock+0x78 #4 0xc08f003c at dounmount+0x8c #5 0xc08f098f at unmount+0x2df #6 0xc0b8a244 at syscall+0x2b4 #7 0xc0b6d4a0 at Xint0x80_syscall+0x20 ino 70657, on dev ad4s1a 0xc945b158: tag nfs, type VREG usecount 1, writecount 1, refcount 2 mountedhere 0 flags (VI_DOOMED) v_object 0xc87b9e88 ref 0 pages 0 lock type nfs: EXCL by thread 0xcfccc240 (pid 29367) with shared waiters pending #0 0xc0851f80 at __lockmgr_args+0xb80 #1 0xc08e5b85 at vop_stdlock+0x65 #2 0xc0b984f5 at VOP_LOCK1_APV+0xb5 #3 0xc0902c08 at _vn_lock+0x78 #4 0xc08f913c at vflush+0x13c #5 0xc0a1a451 at nfs_unmount+0x41 #6 0xc08f041d at dounmount+0x46d #7 0xc08f098f at unmount+0x2df #8 0xc0b8a244 at syscall+0x2b4 #9 0xc0b6d4a0 at Xint0x80_syscall+0x20 fileid -159316928 fsid 0xc0c77518 db:1:lockedvnods> show mount 0xc73c6b40 /dev/ad4s1a on / (ufs) 0xc73c7000 devfs on /dev (devfs) 0xc7647000 /dev/ad4s1f on /usr (ufs) 0xc7646b40 /dev/ad4s1d on /var (ufs) 0xc7646870 procfs on /proc (procfs) 0xc76402d0 /dev/ad4s1e on /tmp (ufs) 0xcd386b40 127.0.0.1:/tmp on /mnt (nfs) More info: show mount db:1:mount> ps pid ppid pgrp uid state wmesg wchan cmd 29367 19703 42616 0 R+ CPU 1 umount 29358 1429 1427 1001 S nanslp 0xc0da7fe4 sleep 25120 1 42616 0 R+ lockf 25119 1 42616 0 R+ lockf 25118 1 42616 0 R+ CPU 3 lockf 25104 1 42616 0 S+ nfs 0xc945b1b0 lockf 20870 1 42616 0 SL+ swread 0xc424075c lockf 20854 1 42616 0 SL+ swread 0xc208dca4 lockf 20853 1 42616 0 SL+ swread 0xc431f62c lockf 20850 1 42616 0 SL+ swread 0xc4006b1c lockf 20849 1 42616 0 R+ CPU 2 lockf 19819 0 0 0 SL - 0xc0f21d44 [nfsiod 1] 19703 42616 42616 0 S+ wait 0xcc0822a4 sh 10095 0 0 0 SL - 0xc0f21d40 [nfsiod 0] 42616 1460 42616 0 S+ wait 0xc79e1000 sh 1515 1 1515 0 Ss select 0xc7847624 mountd 1460 1454 1460 0 S+ wait 0xc744a7ec bash 1454 1453 1454 0 S+ pause 0xc741b058 csh 1453 1449 1453 1001 S+ wait 0xc7b90a90 su 1449 1448 1449 1001 Ss+ wait 0xc74b4000 bash 1448 1446 1446 1001 S select 0xc78473a4 sshd 1446 1065 1446 0 Ss sbwait 0xc79e6248 sshd 1430 1427 1427 1001 S piperd 0xc7749c00 awk 1429 1427 1427 1001 S wait 0xc74b4548 sh 1428 1423 1428 1001 Ss+ select 0xc7a0d024 top 1427 1425 1427 1001 Ss wait 0xc7658000 sh 1426 1424 1426 1001 Ss kqread 0xc73d5d80 tail 1425 1418 1418 1001 S select 0xc7a0c1e4 sshd 1424 1417 1417 1001 S select 0xc77e0ae4 sshd 1423 1419 1419 1001 S select 0xc7a0ce24 sshd 1419 1065 1419 0 Ss sbwait 0xc7a0f898 sshd 1418 1065 1418 0 Ss sbwait 0xc7a0fa2c sshd 1417 1065 1417 0 Ss sbwait 0xc7a13570 sshd 1159 1 1159 0 Ss+ ttyin 0xc70fa870 getty 1158 1 1158 0 Ss+ ttyin 0xc70fa470 getty 1157 1 1157 0 Ss+ ttyin 0xc70fa070 getty 1156 1 1156 0 Ss+ ttyin 0xc6fd0e70 getty 1155 1 1155 0 Ss+ ttyin 0xc6fd0c70 getty 1154 1 1154 0 Ss+ ttyin 0xc6fd0670 getty 1153 1 1153 0 Ss+ ttyin 0xc6fd0870 getty 1152 1 1152 0 Ss+ ttyin 0xc6fd0a70 getty 1151 1 1151 0 Ss+ ttyin 0xc6fd0270 getty 1129 1 1129 0 Ss select 0xc738eb64 inetd 1100 1 1100 0 Ss nanslp 0xc0da7fe4 watchdogd 1084 1 1084 0 Ss nanslp 0xc0da7fe4 cron 1078 1 1078 25 Ss pause 0xc7342ae8 sendmail 1072 1 1072 0 Ss select 0xc7a0d4e4 sendmail 1065 1 1065 0 Ss select 0xc77f5c64 sshd 1028 1 1028 0 Ss select 0xc78820a4 ntpd 913 1 913 0 Ss rpcsvc 0xc738ed50 NLM: master 907 1 907 0 Ss select 0xc77f6aa4 rpc.statd 901 900 900 0 S (threaded) nfsd 100125 S rpcsvc 0xc738ed90 nfsd: service 100124 S rpcsvc 0xc738edd0 nfsd: service 100123 S rpcsvc 0xc738ee10 nfsd: service 100079 S rpcsvc 0xc77f6190 nfsd: master 900 1 900 0 Ss select 0xc786bbe4 nfsd 812 1 812 0 Ss select 0xc7881a64 rpcbind 730 1 730 0 Ss select 0xc730e164 syslogd 598 1 598 0 Ss select 0xc77ab0a4 devd 44 0 0 0 SL flowclea 0xc0da7e44 [flowcleaner] 43 0 0 0 SL sdflush 0xc0f27dc0 [softdepflush] 42 0 0 0 SL vlruwt 0xc73412a4 [vnlru] 41 0 0 0 SL syncer 0xc0f146d4 [syncer] 40 0 0 0 SL psleep 0xc0f14408 [bufdaemon] 39 0 0 0 SL pgzero 0xc0f289f4 [pagezero] 38 0 0 0 SL psleep 0xc0f2861c [vmdaemon] 9 0 0 0 SL psleep 0xc0f285e4 [pagedaemon] 37 0 0 0 SL wmsg 0xc6fe0d0c [usbus5] 36 0 0 0 SL wmsg 0xc6fe0cdc [usbus5] 35 0 0 0 SL wmsg 0xc6fe0cac [usbus5] 34 0 0 0 SL wmsg 0xc6fe0c7c [usbus5] 33 0 0 0 SL wmsg 0xc6fdab5c [usbus4] 32 0 0 0 SL wmsg 0xc6fdab2c [usbus4] 31 0 0 0 SL wmsg 0xc6fdaafc [usbus4] 30 0 0 0 SL wmsg 0xc6fdaacc [usbus4] 29 0 0 0 SL wmsg 0xc6fd1b5c [usbus3] 28 0 0 0 SL wmsg 0xc6fd1b2c [usbus3] 27 0 0 0 SL wmsg 0xc6fd1afc [usbus3] 26 0 0 0 SL wmsg 0xc6fd1acc [usbus3] 25 0 0 0 SL wmsg 0xc6fcdb5c [usbus2] 24 0 0 0 SL wmsg 0xc6fcdb2c [usbus2] 23 0 0 0 SL wmsg 0xc6fcdafc [usbus2] 22 0 0 0 SL wmsg 0xc6fcdacc [usbus2] 21 0 0 0 SL wmsg 0xc6fc7b5c [usbus1] 20 0 0 0 SL wmsg 0xc6fc7b2c [usbus1] 19 0 0 0 SL wmsg 0xc6fc7afc [usbus1] 18 0 0 0 SL wmsg 0xc6fc7acc [usbus1] 17 0 0 0 SL wmsg 0xc6fc2b5c [usbus0] 16 0 0 0 SL wmsg 0xc6fc2b2c [usbus0] 15 0 0 0 SL wmsg 0xc6fc2afc [usbus0] 14 0 0 0 SL wmsg 0xc6fc2acc [usbus0] 8 0 0 0 SL waiting_ 0xc0f1e4bc [sctp_iterator] 7 0 0 0 SL - 0xc6fd043c [fdc0] 6 0 0 0 SL - 0xc6fff000 [fw0_probe] 5 0 0 0 SL ccb_scan 0xc0d74e54 [xpt_thrd] 13 0 0 0 SL - 0xc0da7e44 [yarrow] 4 0 0 0 SL - 0xc0da5be4 [g_down] 3 0 0 0 RL CPU 0 [g_up] 2 0 0 0 SL - 0xc0da5bd8 [g_event] 12 0 0 0 WL (threaded) intr 100042 I [irq12: psm0] 100041 I [irq1: atkbd0] 100039 I [swi0: uart] 100036 I [irq20: fwohci0] 100035 I [irq14: ata0] 100034 I [irq19: ehci0] 100033 I [irq18: ohci2 ohci4] 100032 I [irq17: ohci1 ohci3] 100031 I [irq16: hdac1 ohci0] 100030 I [irq22: atapci0] 100029 I [irq256: hdac0] 100028 I [irq9: acpi0] 100027 I [swi5: +] 100026 I [swi2: cambio] 100020 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100012 I [swi1: netisr 0] 100011 I [swi3: vm] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi4: clock] 100007 I [swi4: clock] 11 0 0 0 RL (threaded) idle 100006 CanRun [idle: cpu0] 100005 CanRun [idle: cpu1] 100004 CanRun [idle: cpu2] 100003 CanRun [idle: cpu3] 1 0 1 0 SLs wait 0xc6d77d34 [init] 10 0 0 0 SL audit_wo 0xc0f276e0 [audit] 0 0 0 0 SLs (threaded) kernel 100037 D - 0xc6fd7a40 [fw0_taskq] 100024 D - 0xc6f2a0c0 [kqueue taskq] 100023 D - 0xc6f2a100 [acpi_task_2] 100022 D - 0xc6f2a100 [acpi_task_1] 100021 D - 0xc6f2a100 [acpi_task_0] 100018 D - 0xc6f2a400 [thread taskq] 100016 D - 0xc6d5fb40 [firmware taskq] 100000 D sched 0xc0da5ca0 [swapper] db:1:ps> allt Tracing command umount pid 29367 tid 100427 td 0xcfccc240 _sx_xlock_hard(c87e7bc8,cfccc240,0,c0c54490,795,...) at _sx_xlock_hard+0x287 _sx_xlock(c87e7bc8,0,c0c54490,795,c87e7bc8,...) at _sx_xlock+0xc0 lf_iteratelocks_vnode(c945b158,c0a372a0,0,f6810b14,f6810ad8,...) at lf_iteratelocks_vnode+0x4d nlm_reclaim(f6810b14,c0d59380,f6810acc,c0d59380,f6810b14,...) at nlm_reclaim+0x30 nfs_reclaim(f6810b14,c0c92843,c945b158,c945b158,cfccc240,...) at nfs_reclaim+0x4e VOP_RECLAIM_APV(c0d59380,f6810b14,c0c643dc,a0e,c945b218,...) at VOP_RECLAIM_APV+0xc5 vgonel(c945b218,0,c0c643dc,987,0,...) at vgonel+0x1c4 vflush(cd386b40,1,2,cfccc240,ce28b810,...) at vflush+0x337 nfs_unmount(cd386b40,8080000,f6810c44,4f3,80,...) at nfs_unmount+0x41 dounmount(cd386b40,8080000,cfccc240,479,3,...) at dounmount+0x46d unmount(cfccc240,f6810cf8,8,f6810d38,c0d3b328,...) at unmount+0x2df syscall(f6810d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d916f, esp = 0xbfbfe15c, ebp = 0xbfbfe228 --- Tracing command sleep pid 29358 tid 100348 td 0xc8210000 sched_switch(c8210000,0,104,18c,6ec18f97,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c8210000,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(2711,c089ff50,c8210000,3,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0da7fe4,5c,c0c582ee,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0da7fe4,0,15c,c0c582ee,2711,...) at _sleep+0x30e kern_nanosleep(c8210000,f66c9c64,f66c9c6c,a,0,...) at kern_nanosleep+0xc1 nanosleep(c8210000,f66c9cf8,8,f66c9d38,c0d3cb00,...) at nanosleep+0x6f syscall(f66c9d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815687f, esp = 0xbfbfecbc, ebp = 0xbfbfecf8 --- Tracing command lockf pid 25120 tid 100478 td 0xcb1a2480 sched_switch(cb1a2480,0,207,187,b20f0368,...) at sched_switch+0x437 mi_switch(207,0,c0c5bc6d,d5,c92da7ec,...) at mi_switch+0x200 ast(f68c9d38) at ast+0x2b7 doreti_ast() at doreti_ast+0x17 Tracing command lockf pid 25119 tid 100378 td 0xcfe91d80 sched_switch(cfe91d80,0,104,18c,b20e4cff,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(cfe91d80,0,c0c5b48b,247,c0da66c0,...) at sleepq_switch+0x15f sleepq_wait(c0da66c0,0,c0c5445d,3,0,...) at sleepq_wait+0x63 _sx_xlock_hard(c0da66c0,cfe91d80,0,c0c54490,25b,...) at _sx_xlock_hard+0x496 _sx_xlock(c0da66c0,0,c0c54490,25b,ce2a7b18,...) at _sx_xlock+0xc0 lf_advlockasync(f6723b3c,ce2a7c14,4,0,c0d4c2a0,...) at lf_advlockasync+0x4f3 lf_advlock(f6723c00,ce2a7c14,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f6723c00,c0c9255f,ce2a7ac0,ce2a7ac0,cc1eab00,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d59380,f6723c00,c0c51dab,259,c08a962b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(cfe91d80,3,d,f6723c40,0,...) at kern_fcntl+0x90f fcntl(cfe91d80,f6723cf8,c,c0c5dd40,c0d3bad0,...) at fcntl+0x101 syscall(f6723d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 25118 tid 100360 td 0xcfcead80 cpustop_handler(8,f66ed9dc,c0b8a54d,f66ed96c,c0859064,...) at cpustop_handler+0x32 ipi_nmi_handler(f66ed96c,c0859064,c0da80c8,4,cfcd3548,...) at ipi_nmi_handler+0x2f trap(f66ed9e8) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b8e2e9, esp = 0xf66eda28, ebp = 0xf66eda44 --- DELAY(1,cfcead80,c0dba44c,0,f66eda84,...) at DELAY+0x89 _mtx_lock_spin(c0dba44c,cfcead80,0,c0c5b48b,ec,...) at _mtx_lock_spin+0x6c _mtx_lock_spin_flags(c0dba44c,0,c0c5b48b,ec,f66edabc,...) at _mtx_lock_spin_flags+0x146 sleepq_lock(cb02fb00,2d4,0,c8711600,c8711600,...) at sleepq_lock+0x39 wakeup(cb02fb00,0,c0c54490,2d4,cb48c5b8,...) at wakeup+0x17 lf_advlockasync(f66edb3c,cb48c6b4,4,0,c0d4c2a0,...) at lf_advlockasync+0xdc4 lf_advlock(f66edc00,cb48c6b4,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f66edc00,c0c9255f,cb48c560,cb48c560,c849ea00,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d59380,f66edc00,c0c51dab,259,c08a962b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(cfcead80,3,d,f66edc40,0,...) at kern_fcntl+0x90f fcntl(cfcead80,f66edcf8,c,c0c5dd40,c0d3bad0,...) at fcntl+0x101 syscall(f66edd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 25104 tid 100495 td 0xc83f46c0 sched_switch(c83f46c0,0,104,18c,b208b8cb,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,50,...) at mi_switch+0x200 sleepq_switch(c83f46c0,0,c0c5b48b,247,c945b1b0,...) at sleepq_switch+0x15f sleepq_wait(c945b1b0,50,c0c77da8,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c945b1b0,200000,c945b218,0,0,...) at __lockmgr_args+0x4a5 vop_stdlock(f6923b3c,c0c51db4,c0c4f717,200000,c945b158,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0d59380,f6923b3c,c08a962b,c0d746c0,c945b158,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c945b158,200000,c0c77c3e,c7e,f6923b90,...) at _vn_lock+0x78 nfs_advlock(f6923c00,c0c9255f,c945b158,c945b158,cc318b00,...) at nfs_advlock+0x2e VOP_ADVLOCK_APV(c0d59380,f6923c00,c0c51dab,259,c08a962b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(c83f46c0,3,d,f6923c40,0,...) at kern_fcntl+0x90f fcntl(c83f46c0,f6923cf8,c,c0c5dd40,c0d3bad0,...) at fcntl+0x101 syscall(f6923d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 20870 tid 100439 td 0xcb2ef6c0 sched_switch(cb2ef6c0,0,104,18c,b2061cf8,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,40,...) at mi_switch+0x200 sleepq_switch(cb2ef6c0,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c424075c,40,c0c7ec60,0,0,...) at sleepq_timedwait+0x6b _sleep(c424075c,cd8b45d0,40,c0c7ec60,4e20,...) at _sleep+0x329 swap_pager_getpages(cd8b45d0,f6834bb8,1,0,f6834c3c,...) at swap_pager_getpages+0x44d vm_fault(c6d7dab0,2819b000,1,0,2819bd64,...) at vm_fault+0xf7f trap_pfault(5,0,c0c8fe8e,c0da80c8,c8094548,...) at trap_pfault+0x118 trap(f6834d38) at trap+0x2a9 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bca60, esp = 0xbfbfe658, ebp = 0xbfbfe698 --- Tracing command lockf pid 20854 tid 100389 td 0xc93cad80 sched_switch(c93cad80,0,104,18c,b2077ba4,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,40,...) at mi_switch+0x200 sleepq_switch(c93cad80,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c208dca4,40,c0c7ec60,0,0,...) at sleepq_timedwait+0x6b _sleep(c208dca4,cb77126c,40,c0c7ec60,4e20,...) at _sleep+0x329 swap_pager_getpages(cb77126c,f679ebb8,1,0,f679ec3c,...) at swap_pager_getpages+0x44d vm_fault(ca319000,2819b000,1,0,2819bd64,...) at vm_fault+0xf7f trap_pfault(5,0,c0c8fe8e,c0c5bc76,cb1862a4,...) at trap_pfault+0x118 trap(f679ed38) at trap+0x2a9 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bca60, esp = 0xbfbfe658, ebp = 0xbfbfe698 --- Tracing command lockf pid 20853 tid 100390 td 0xc93cab40 sched_switch(c93cab40,0,104,18c,b201f614,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,40,...) at mi_switch+0x200 sleepq_switch(c93cab40,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c431f62c,40,c0c7ec60,0,0,...) at sleepq_timedwait+0x6b _sleep(c431f62c,cc107934,40,c0c7ec60,4e20,...) at _sleep+0x329 swap_pager_getpages(cc107934,f67a1bb8,1,0,f67a1c3c,...) at swap_pager_getpages+0x44d vm_fault(c92dbe40,2807e000,1,0,2807e348,...) at vm_fault+0xf7f trap_pfault(5,0,c0c8fe8e,2eb,cb186000,...) at trap_pfault+0x118 trap(f67a1d38) at trap+0x2a9 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280514fd, esp = 0xbfbfe60c, ebp = 0xbfbfe644 --- Tracing command lockf pid 20850 tid 100194 td 0xc7d82240 sched_switch(c7d82240,0,104,18c,b204dd40,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,40,...) at mi_switch+0x200 sleepq_switch(c7d82240,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c4006b1c,40,c0c7ec60,0,0,...) at sleepq_timedwait+0x6b _sleep(c4006b1c,cfcb5364,40,c0c7ec60,4e20,...) at _sleep+0x329 swap_pager_getpages(cfcb5364,f649cbb8,1,0,f649cc3c,...) at swap_pager_getpages+0x44d vm_fault(ca03fb94,2807e000,1,0,2807e348,...) at vm_fault+0xf7f trap_pfault(5,0,c0c8fe8e,2eb,c7d7b000,...) at trap_pfault+0x118 trap(f649cd38) at trap+0x2a9 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280514fd, esp = 0xbfbfe60c, ebp = 0xbfbfe644 --- Tracing command lockf pid 20849 tid 100115 td 0xc74b6900 cpustop_handler(4,f6386c3c,c0b8a54d,f6386cf8,f6386c18,...) at cpustop_handler+0x32 ipi_nmi_handler(f6386cf8,f6386c18,c0c5bc76,c6d29340,c74af7ec,...) at ipi_nmi_handler+0x2f trap(f6386c48) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b89fed, esp = 0xf6386c88, ebp = 0xf6386d2c --- syscall(f6386d38) at syscall+0x5d Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (190, FreeBSD ELF32, lstat), eip = 0x281782c3, esp = 0xbebed29c, ebp = 0xbebed358 --- Tracing command nfsiod 1 pid 19819 tid 100370 td 0xcfcd76c0 sched_switch(cfcd76c0,0,104,18c,128affbc,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(cfcd76c0,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(1d4c0,c089ff50,cfcd76c0,2,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0f21d44,5c,c0c4d0ea,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0f21d44,c0f21d10,15c,c0c4d0ea,1d4c0,...) at _sleep+0x30e nfssvc_iod(c0f216c4,f670bd38,c0c52867,335,cfde4000,...) at nfssvc_iod+0xcf fork_exit(c0a19160,c0f216c4,f670bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf670bd70, ebp = 0 --- Tracing command sh pid 19703 tid 100283 td 0xcc0846c0 sched_switch(cc0846c0,0,104,18c,79d5e6d7,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(cc0846c0,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(cc0822a4,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0822a4,cc08232c,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(cc0846c0,ffffffff,f6606c74,2,0,...) at kern_wait+0xac6 wait4(cc0846c0,f6606cf8,10,c0c5dcd3,c0d3b184,...) at wait4+0x3b syscall(f6606d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfe84c, ebp = 0xbfbfe868 --- Tracing command nfsiod 0 pid 10095 tid 100280 td 0xcb3d4480 sched_switch(cb3d4480,0,104,18c,75d437c7,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(cb3d4480,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(1d4c0,c089ff50,cb3d4480,1,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0f21d40,5c,c0c4d0ea,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0f21d40,c0f21d10,15c,c0c4d0ea,1d4c0,...) at _sleep+0x30e nfssvc_iod(c0f216c0,f65e1d38,c0c52867,335,c9930548,...) at nfssvc_iod+0xcf fork_exit(c0a19160,c0f216c0,f65e1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf65e1d70, ebp = 0 --- Tracing command sh pid 42616 tid 100142 td 0xc7a0a480 sched_switch(c7a0a480,0,104,18c,1a329fdb,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c7a0a480,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c79e1000,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(c79e1000,c79e1088,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(c7a0a480,ffffffff,f63e4c74,2,0,...) at kern_wait+0xac6 wait4(c7a0a480,f63e4cf8,10,c0c5dcd3,c0d3b184,...) at wait4+0x3b syscall(f63e4d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfe71c, ebp = 0xbfbfe738 --- Tracing command mountd pid 1515 tid 100157 td 0xc7b94240 Tracing command bash pid 1460 tid 100084 td 0xc744d6c0 sched_switch(c744d6c0,0,104,18c,4578758c,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c744d6c0,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c744a7ec,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(c744a7ec,c744a874,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(c744d6c0,ffffffff,f62bec74,6,0,...) at kern_wait+0xac6 wait4(c744d6c0,f62becf8,10,c0c5dbe3,c0d3b184,...) at wait4+0x3b syscall(f62bed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282ace7b, esp = 0xbfbfe7fc, ebp = 0xbfbfe818 --- Tracing command csh pid 1454 tid 100118 td 0xc741fb40 sched_switch(c741fb40,0,104,18c,6a93839d,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,68,...) at mi_switch+0x200 sleepq_switch(c741fb40,0,c0c5b48b,189,68,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c741b058,68,c0c0618c,100,0,...) at sleepq_wait_sig+0x17 _sleep(c741b058,c741b088,168,c0c0618c,0,...) at _sleep+0x344 kern_sigsuspend(c741fb40,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c741fb40,f6393cf8,4,c0c5dbe3,c0d3d60c,...) at sigsuspend+0x4d syscall(f6393d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x28178eeb, esp = 0xbfbfe60c, ebp = 0xbfbfe678 --- Tracing command su pid 1453 tid 100164 td 0xc79dc900 sched_switch(c79dc900,0,104,18c,13f114bb,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c79dc900,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7b90a90,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7b90a90,c7b90b18,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(c79dc900,5ae,f642ec74,2,0,...) at kern_wait+0xac6 wait4(c79dc900,f642ecf8,10,f642ed38,c0d3b184,...) at wait4+0x3b syscall(f642ed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2811be7b, esp = 0xbfbfe64c, ebp = 0xbfbfe668 --- Tracing command bash pid 1449 tid 100111 td 0xc74bc000 sched_switch(c74bc000,0,104,18c,d6fd601a,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c74bc000,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c74b4000,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74b4000,c74b4088,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(c74bc000,ffffffff,f635ec74,6,0,...) at kern_wait+0xac6 wait4(c74bc000,f635ecf8,10,c0c5dbe3,c0d3b184,...) at wait4+0x3b syscall(f635ed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282ace7b, esp = 0xbfbfe9dc, ebp = 0xbfbfe9f8 --- Tracing command sshd pid 1448 tid 100146 td 0xc741c6c0 sched_switch(c741c6c0,0,104,18c,714cb91d,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c741c6c0,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c08593fa,c7847390,0,c0c5591d,c741c6c0,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c78473a4,0,f63f0a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c78473a4,c7847390,c0c5d753,5c4,c74b3380,...) at _cv_wait_sig+0x240 seltdwait(c74b3380,58,c74b0b00,c741c6c0,c0c73e2f,...) at seltdwait+0xa2 kern_select(c741c6c0,a,286030b8,286030dc,0,0,66,280fc6f8) at kern_select+0x4c4 select(c741c6c0,f63f0cf8,14,c0c39395,c0d3baec,...) at select+0x5e syscall(f63f0d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283c4d03, esp = 0xbfbfde5c, ebp = 0xbfbfdea8 --- Tracing command sshd pid 1446 tid 100131 td 0xc741cb40 Tracing command awk pid 1430 tid 100158 td 0xc7b94000 sched_switch(c7b94000,0,104,18c,6e4902a4,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c7b94000,0,c0c5b48b,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7749c00,4c,c0c5d90b,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7749c00,c7749d68,14c,c0c5d90b,0,...) at _sleep+0x344 pipe_read(c79be118,f641cc58,c73c8100,0,c7b94000,...) at pipe_read+0x417 dofileread(f641cc58,ffffffff,ffffffff,0,c79be118,...) at dofileread+0x96 kern_readv(c7b94000,0,f641cc58,f641cc78,1,...) at kern_readv+0x58 read(c7b94000,f641ccf8,c,c7b94000,c0d3b114,...) at read+0x4f syscall(f641cd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281acd83, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command sh pid 1429 tid 100094 td 0xc74bc480 sched_switch(c74bc480,0,104,18c,6e596a4f,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c74bc480,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c74b4548,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74b4548,c74b45d0,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(c74bc480,ffffffff,f62efc74,2,0,...) at kern_wait+0xac6 wait4(c74bc480,f62efcf8,10,c0c5dcd3,c0d3b184,...) at wait4+0x3b syscall(f62efd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfe95c, ebp = 0xbfbfe978 --- Tracing command top pid 1428 tid 100165 td 0xc7b936c0 sched_switch(c7b936c0,0,104,18c,9490f5fd,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7b936c0,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6431a50,c08593fa,c7a0d010,0,c7b936c0,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c7a0d024,0,f6431a80,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7a0d024,c7a0d010,3e9,5c4,c7424188,...) at _cv_timedwait_sig+0x250 seltdwait(f6431c2c,f6431c34,c7c8f600,c7b936c0,2032323a,...) at seltdwait+0x8a kern_select(c7b936c0,2,bfbfebec,0,0,f6431c70,1,0) at kern_select+0x4c4 select(c7b936c0,f6431cf8,14,c0c39395,c0d3baec,...) at select+0x5e syscall(f6431d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281e1d03, esp = 0xbfbfeb8c, ebp = 0xbfbfece8 --- Tracing command sh pid 1427 tid 100110 td 0xc765c240 sched_switch(c765c240,0,104,18c,d1e5efd3,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c765c240,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7658000,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7658000,c7658088,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(c765c240,ffffffff,f633ac74,2,0,...) at kern_wait+0xac6 wait4(c765c240,f633acf8,10,f633ad38,c0d3b184,...) at wait4+0x3b syscall(f633ad38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfeb5c, ebp = 0xbfbfeb78 --- Tracing command tail pid 1426 tid 100162 td 0xc79dcb40 sched_switch(c79dcb40,0,104,18c,ab9f3cdf,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,58,...) at mi_switch+0x200 sleepq_switch(c79dcb40,0,c0c5b48b,189,58,...) at sleepq_switch+0x15f sleepq_catch_signals(3e9,c089ff50,c79dcb40,0,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c73d5d80,58,c0c52519,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c73d5d80,c73d5d80,158,c0c52519,3e9,...) at _sleep+0x30e kern_kevent(c79dcb40,4,0,1,f6428c58,...) at kern_kevent+0x357 kevent(c79dcb40,f6428cf8,18,c0bf1c1f,c0d3d874,...) at kevent+0x19b syscall(f6428d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (363, FreeBSD ELF32, kevent), eip = 0x2815aa6b, esp = 0xbfbfeb6c, ebp = 0xbfbfec18 --- Tracing command sshd pid 1425 tid 100178 td 0xc7b9f240 Tracing command sshd pid 1424 tid 100152 td 0xc7b94d80 Tracing command sshd pid 1423 tid 100149 td 0xc79dd240 sched_switch(c79dd240,0,104,18c,94942eff,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c79dd240,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c08593fa,c7a0ce10,0,c0c5591d,c79dd240,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7a0ce24,0,f6401a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7a0ce24,c7a0ce10,c0c5d753,5c4,c7423310,...) at _cv_wait_sig+0x240 seltdwait(c7423310,58,c7ba1e00,c79dd240,c0c73e2f,...) at seltdwait+0xa2 kern_select(c79dd240,a,286030b8,286030dc,0,0,66,280fc6f8) at kern_select+0x4c4 select(c79dd240,f6401cf8,14,c0c39395,c0d3baec,...) at select+0x5e syscall(f6401d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283c4d03, esp = 0xbfbfde5c, ebp = 0xbfbfdea8 --- Tracing command sshd pid 1419 tid 100174 td 0xc765e900 Tracing command sshd pid 1418 tid 100175 td 0xc765e6c0 Tracing command sshd pid 1417 tid 100138 td 0xc7a0ad80 Tracing command getty pid 1159 tid 100076 td 0xc7420000 Tracing command getty pid 1158 tid 100147 td 0xc79ddb40 Tracing command getty pid 1157 tid 100145 td 0xc79ddd80 Tracing command getty pid 1156 tid 100137 td 0xc7a0b000 Tracing command getty pid 1155 tid 100139 td 0xc7a0ab40 Tracing command getty pid 1154 tid 100096 td 0xc732b000 Tracing command getty pid 1153 tid 100088 td 0xc7420d80 Tracing command getty pid 1152 tid 100128 td 0xc74206c0 Tracing command getty pid 1151 tid 100099 td 0xc765db40 Tracing command inetd pid 1129 tid 100144 td 0xc7a0a000 Tracing command watchdogd pid 1100 tid 100143 td 0xc7a0a240 sched_switch(c7a0a240,0,104,18c,ef7f12f1,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c7a0a240,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c089ff50,c7a0a240,3,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0da7fe4,5c,c0c582ee,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0da7fe4,0,15c,c0c582ee,ea61,...) at _sleep+0x30e kern_nanosleep(c7a0a240,f63e7c64,f63e7c6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c7a0a240,f63e7cf8,8,c0c6e07e,c0d3cb00,...) at nanosleep+0x6f syscall(f63e7d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2818087f, esp = 0xbfbfecec, ebp = 0xbfbfed18 --- Tracing command cron pid 1084 tid 100132 td 0xc7a0b900 sched_switch(c7a0b900,0,104,18c,892b1efe,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c7a0b900,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c089ff50,c7a0b900,2,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0da7fe4,5c,c0c582ee,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0da7fe4,0,15c,c0c582ee,ea61,...) at _sleep+0x30e kern_nanosleep(c7a0b900,f63c5c64,f63c5c6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c7a0b900,f63c5cf8,8,c0c5de31,c0d3cb00,...) at nanosleep+0x6f syscall(f63c5d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2817387f, esp = 0xbfbfec8c, ebp = 0xbfbfecb8 --- Tracing command sendmail pid 1078 tid 100133 td 0xc741c900 sched_switch(c741c900,0,104,18c,aeb2af9a,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,68,...) at mi_switch+0x200 sleepq_switch(c741c900,0,c0c5b48b,189,68,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7342ae8,68,c0c0618c,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7342ae8,c7342b18,168,c0c0618c,0,...) at _sleep+0x344 kern_sigsuspend(c741c900,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c741c900,f63c8cf8,4,c0c5dbe3,c0d3d60c,...) at sigsuspend+0x4d syscall(f63c8d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x2832beeb, esp = 0xbfbfcf9c, ebp = 0xbfbfcfc8 --- Tracing command sendmail pid 1072 tid 100135 td 0xc7a0b480 sched_switch(c7a0b480,0,104,18c,8afae684,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7a0b480,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f63cfa50,c08593fa,c7a0d4d0,0,c7a0b480,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c7a0d4e4,0,f63cfa80,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7a0d4e4,c7a0d4d0,1389,5c4,c7748bd0,...) at _cv_timedwait_sig+0x250 seltdwait(f63cfc2c,f63cfc34,c7673c00,c7a0b480,c6c12000,...) at seltdwait+0x8a kern_select(c7a0b480,5,bfbfc520,0,0,f63cfc70,5,0) at kern_select+0x4c4 select(c7a0b480,f63cfcf8,14,c0c5e107,c0d3baec,...) at select+0x5e syscall(f63cfd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283cbd03, esp = 0xbfbfc48c, ebp = 0xbfbfcfb8 --- Tracing command sshd pid 1065 tid 100136 td 0xc7a0b240 Tracing command ntpd pid 1028 tid 100129 td 0xc7420480 sched_switch(c7420480,0,104,18c,9bafcc7e,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7420480,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c08593fa,c7882090,0,c0c5591d,c7420480,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c78820a4,0,f63bba80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c78820a4,c7882090,c0c5d753,5c4,c7424380,...) at _cv_wait_sig+0x240 seltdwait(c7424380,58,c6d45300,c7420480,0,...) at seltdwait+0xa2 kern_select(c7420480,1b,bfbfed28,0,0,0,f63bbc98,246) at kern_select+0x4c4 select(c7420480,f63bbcf8,14,c7420480,c0d3baec,...) at select+0x5e syscall(f63bbd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2834cd03, esp = 0xbfbfecfc, ebp = 0xbfbfedc8 --- Tracing command rpc.lockd pid 913 tid 100130 td 0xc7420240 sched_switch(c7420240,0,104,18c,b2033ca2,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7420240,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f63bea04,c08593fa,c74df680,0,c7420240,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c738ed50,0,f63bea34,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c738ed50,c74df680,1388,487,c79ed000,...) at _cv_timedwait_sig+0x250 svc_run_internal(c74203a4,14,c0c7940b,c0c78b99,f63beb8c,...) at svc_run_internal+0x356 svc_run(c74df680,0,4,c0a3e830,f63beb8c,...) at svc_run+0x7f nlm_syscall(c7420240,f63becf8,10,c0c5e52f,c0d3c198,...) at nlm_syscall+0x6a9 syscall(f63bed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (154, FreeBSD ELF32, nlm_syscall), eip = 0x280f9b2b, esp = 0xbfbfed5c, ebp = 0xbfbfee28 --- Tracing command rpc.statd pid 907 tid 100104 td 0xc765d000 sched_switch(c765d000,0,104,18c,3c9d3087,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c765d000,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6325a50,c08593fa,c77f6a90,0,c765d000,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c77f6aa4,0,f6325a80,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c77f6aa4,c77f6a90,7531,5c4,c74234d0,...) at _cv_timedwait_sig+0x250 seltdwait(f6325c2c,f6325c34,c6d45300,c765d000,c0aada8e,...) at seltdwait+0x8a kern_select(c765d000,8,bfbfed44,0,0,f6325c70,1e,0) at kern_select+0x4c4 select(c765d000,f6325cf8,14,c0c5de31,c0d3baec,...) at select+0x5e syscall(f6325d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28181d03, esp = 0xbfbfec9c, ebp = 0xbfbfedd8 --- Tracing command nfsd pid 901 tid 100125 td 0xc79dd480 sched_switch(c79dd480,0,104,18c,19382f2c,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c79dd480,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f63adbf8,c08593fa,c70f8200,0,c79dd480,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c738ed90,0,f63adc28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c738ed90,c70f8200,1388,487,f63adcdc,...) at _cv_timedwait_sig+0x250 svc_run_internal(f63add24,c083f528,c70f8200,f63add38,c0c52867,...) at svc_run_internal+0x356 svc_thread_start(c70f8200,f63add38,c0c52867,335,c744c548,...) at svc_thread_start+0x10 fork_exit(c0a4c6b0,c70f8200,f63add38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 901 tid 100124 td 0xc74bcd80 sched_switch(c74bcd80,0,104,18c,4c89d583,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c74bcd80,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f63aabf8,c08593fa,c70f8200,0,c74bcd80,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c738edd0,0,f63aac28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c738edd0,c70f8200,1388,487,f63aacdc,...) at _cv_timedwait_sig+0x250 svc_run_internal(f63aad24,c083f528,c70f8200,f63aad38,c0c52867,...) at svc_run_internal+0x356 svc_thread_start(c70f8200,f63aad38,c0c52867,335,c744c548,...) at svc_thread_start+0x10 fork_exit(c0a4c6b0,c70f8200,f63aad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 901 tid 100123 td 0xc765c000 sched_switch(c765c000,0,104,18c,7fa9cb5e,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c765c000,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f63a7bf8,c08593fa,c70f8200,0,c765c000,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c738ee10,0,f63a7c28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c738ee10,c70f8200,1388,487,f63a7cdc,...) at _cv_timedwait_sig+0x250 svc_run_internal(f63a7d24,c083f528,c70f8200,f63a7d38,c0c52867,...) at svc_run_internal+0x356 svc_thread_start(c70f8200,f63a7d38,c0c52867,335,c744c548,...) at svc_thread_start+0x10 fork_exit(c0a4c6b0,c70f8200,f63a7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 901 tid 100079 td 0xc7453240 sched_switch(c7453240,0,104,18c,ebdabaef,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7453240,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f62a9ae8,c08593fa,c70f8200,0,c7453240,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c77f6190,0,f62a9b18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c77f6190,c70f8200,1388,487,f62a9bcc,...) at _cv_timedwait_sig+0x250 svc_run_internal(c74533a4,14,c0c7940b,c0c78082,f62a9c3c,...) at svc_run_internal+0x356 svc_run(c70f8200,0,c0c78386,1fc,0,...) at svc_run+0x7f nfssvc_nfsd(bfbfe8c0,f62a9c3c,c,c6d45300,f62a9c50,...) at nfssvc_nfsd+0xad nfssvc_nfsserver(c7453240,f62a9cf8,bfbfe8c0,c7453240,c744c548,...) at nfssvc_nfsserver+0x21c nfssvc(c7453240,f62a9cf8,8,c0c5e107,c0d3c1b4,...) at nfssvc+0x77 syscall(f62a9d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280d9b0b, esp = 0xbfbfe87c, ebp = 0xbfbfeae8 --- Tracing command nfsd pid 900 tid 100097 td 0xc765e000 sched_switch(c765e000,0,104,18c,bfa0a5c9,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c765e000,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c08593fa,c786bbd0,0,c0c5591d,c765e000,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c786bbe4,0,f6308a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c786bbe4,c786bbd0,c0c5d753,5c4,c79be268,...) at _cv_wait_sig+0x240 seltdwait(c79be268,58,c6d45300,c765e000,f6308af0,...) at seltdwait+0xa2 kern_select(c765e000,5,bfbfecf0,0,0,0,66,5) at kern_select+0x4c4 select(c765e000,f6308cf8,14,c0c77e47,c0d3baec,...) at select+0x5e syscall(f6308d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28178d03, esp = 0xbfbfeaec, ebp = 0xbfbfee28 --- Tracing command rpcbind pid 812 tid 100098 td 0xc765dd80 sched_switch(c765dd80,0,104,18c,a4ceea06,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c765dd80,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f630daa8,c08593fa,c7881a50,0,c765dd80,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c7881a64,0,f630dad8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7881a64,c7881a50,7531,5c4,f630db8c,...) at _cv_timedwait_sig+0x250 seltdwait(f630dc5c,f630dc64,4d4,c765dd80,f630db5c,...) at seltdwait+0x8a poll(c765dd80,f630dcf8,c,c0c5de1e,c0d3c79c,...) at poll+0x2f9 syscall(f630dd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (209, FreeBSD ELF32, poll), eip = 0x2813a07f, esp = 0xbfbfcc0c, ebp = 0xbfbfedd8 --- Tracing command syslogd pid 730 tid 100105 td 0xc765cd80 sched_switch(c765cd80,0,104,18c,f39eddb3,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c765cd80,0,c0c5b48b,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c08593fa,c730e150,0,c0c5591d,c765cd80,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c730e164,0,f6328a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c730e164,c730e150,c0c5d753,5c4,c75037a8,...) at _cv_wait_sig+0x240 seltdwait(c75037a8,58,c6d45300,c765cd80,0,...) at seltdwait+0xa2 kern_select(c765cd80,9,282290ac,0,0,0,66,2819e9c0) at kern_select+0x4c4 select(c765cd80,f6328cf8,14,c0c5dce3,c0d3baec,...) at select+0x5e syscall(f6328d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2818bd03, esp = 0xbfbfde7c, ebp = 0xbfbfee28 --- Tracing command devd pid 598 tid 100083 td 0xc744d900 Tracing command flowcleaner pid 44 tid 100075 td 0xc732b480 sched_switch(c732b480,0,104,18c,62304dad,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732b480,0,c0c5b48b,26a,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0da7e44,0,c0c66154,2,0,...) at sleepq_timedwait+0x6b _sleep(c0da7e44,0,0,c0c66154,4e20,...) at _sleep+0x329 pause(c0c66154,4e20,c0c66089,41e,0,...) at pause+0x47 flowtable_cleaner(0,f40aed38,c0c52867,335,c7328d34,...) at flowtable_cleaner+0x160 fork_exit(c090cb30,0,f40aed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40aed70, ebp = 0 --- Tracing command softdepflush pid 43 tid 100074 td 0xc732b6c0 sched_switch(c732b6c0,0,104,18c,49c1d6cb,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,44,...) at mi_switch+0x200 sleepq_switch(c732b6c0,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f27dc0,44,c0c7caec,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f27dc0,c0f27d64,44,c0c7caec,3e8,...) at _sleep+0x329 softdep_flush(0,f40abd38,c0c52867,335,c7341000,...) at softdep_flush+0x244 fork_exit(c0a88b90,0,f40abd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40abd70, ebp = 0 --- Tracing command vnlru pid 42 tid 100073 td 0xc732b900 sched_switch(c732b900,0,104,18c,51442f2d,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,50,...) at mi_switch+0x200 sleepq_switch(c732b900,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c73412a4,50,c0c65360,0,0,...) at sleepq_timedwait+0x6b _sleep(c73412a4,c0f14694,250,c0c65360,3e8,...) at _sleep+0x329 vnlru_proc(0,f40a8d38,c0c52867,335,c73412a4,...) at vnlru_proc+0xe7 fork_exit(c08f7fc0,0,f40a8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40a8d70, ebp = 0 --- Tracing command syncer pid 41 tid 100072 td 0xc732bb40 sched_switch(c732bb40,0,104,18c,84e0fa22,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732bb40,0,c0c5b48b,26a,c732bb40,...) at sleepq_switch+0x15f sleepq_timedwait(c0f146d4,0,f40a5c88,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0f146d4,c0f146c0,3e8,6cd,4e20,...) at _cv_timedwait+0x250 sched_sync(0,f40a5d38,c0c52867,335,c7341548,...) at sched_sync+0x502 fork_exit(c08f73f0,0,f40a5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40a5d70, ebp = 0 --- Tracing command bufdaemon pid 40 tid 100071 td 0xc732bd80 sched_switch(c732bd80,0,104,18c,66ba0be3,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,44,...) at mi_switch+0x200 sleepq_switch(c732bd80,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f14408,44,c0c62921,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f14408,c0f1440c,44,c0c62921,3e8,...) at _sleep+0x329 buf_daemon(0,f40a2d38,c0c52867,335,c73417ec,...) at buf_daemon+0x138 fork_exit(c08e00d0,0,f40a2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40a2d70, ebp = 0 --- Tracing command pagezero pid 39 tid 100070 td 0xc732d000 sched_switch(c732d000,0,104,18c,97ab04d2,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732d000,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f289f4,0,c0c82162,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f289f4,c0f285b0,0,c0c82162,493e0,...) at _sleep+0x329 vm_pagezero(0,f409fd38,c0c52867,335,c7341a90,...) at vm_pagezero+0xdc fork_exit(c0ac5d70,0,f409fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf409fd70, ebp = 0 --- Tracing command vmdaemon pid 38 tid 100069 td 0xc732d240 sched_switch(c732d240,0,104,18c,1408639c,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,68,...) at mi_switch+0x200 sleepq_switch(c732d240,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0f2861c,68,c0c62921,0,0,...) at sleepq_wait+0x63 _sleep(c0f2861c,c0f28620,68,c0c62921,0,...) at _sleep+0x35b vm_daemon(0,f409cd38,c0c52867,335,c7341d34,...) at vm_daemon+0x59 fork_exit(c0ac0220,0,f409cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf409cd70, ebp = 0 --- Tracing command pagedaemon pid 9 tid 100068 td 0xc732d480 sched_switch(c732d480,0,104,18c,b5c91567,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,44,...) at mi_switch+0x200 sleepq_switch(c732d480,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f285e4,44,c0c62921,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f285e4,c0f285b0,44,c0c62921,1388,...) at _sleep+0x329 vm_pageout(0,f4099d38,c0c52867,335,c7342000,...) at vm_pageout+0x2bb fork_exit(c0ac10c0,0,f4099d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4099d70, ebp = 0 --- Tracing command usbus5 pid 37 tid 100067 td 0xc732d6c0 sched_switch(c732d6c0,0,104,18c,f5b4c9f1,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732d6c0,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fe0d0c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fe0d0c,c6fe0dac,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4036cf8,c07a006b,c6fe0d0c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fe0d0c,c6fe0dac,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fe0d04,f4036d38,c0c52867,335,c73422a4,...) at usb_process+0x18b fork_exit(c079fee0,c6fe0d04,f4036d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4036d70, ebp = 0 --- Tracing command usbus5 pid 36 tid 100066 td 0xc732d900 sched_switch(c732d900,0,104,18c,d435f2c9,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732d900,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fe0cdc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fe0cdc,c6fe0dac,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4033cf8,c07a006b,c6fe0cdc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fe0cdc,c6fe0dac,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fe0cd4,f4033d38,c0c52867,335,c7342548,...) at usb_process+0x18b fork_exit(c079fee0,c6fe0cd4,f4033d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4033d70, ebp = 0 --- Tracing command usbus5 pid 35 tid 100065 td 0xc732db40 sched_switch(c732db40,0,104,18c,f54cf2d7,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732db40,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fe0cac,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fe0cac,c6fe0dac,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4030cf8,c07a006b,c6fe0cac,...) at usb2_msleep+0x58 usb2_cv_wait(c6fe0cac,c6fe0dac,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fe0ca4,f4030d38,c0c52867,335,c7327000,...) at usb_process+0x18b fork_exit(c079fee0,c6fe0ca4,f4030d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4030d70, ebp = 0 --- Tracing command usbus5 pid 34 tid 100064 td 0xc732dd80 sched_switch(c732dd80,0,104,18c,f54ce383,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732dd80,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fe0c7c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fe0c7c,c6fe0dac,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f402dcf8,c07a006b,c6fe0c7c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fe0c7c,c6fe0dac,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fe0c74,f402dd38,c0c52867,335,c73272a4,...) at usb_process+0x18b fork_exit(c079fee0,c6fe0c74,f402dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf402dd70, ebp = 0 --- Tracing command usbus4 pid 33 tid 100063 td 0xc70ed240 sched_switch(c70ed240,0,104,18c,f54cd276,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ed240,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdab5c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fdab5c,c6fdabfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f402acf8,c07a006b,c6fdab5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdab5c,c6fdabfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fdab54,f402ad38,c0c52867,335,c7327548,...) at usb_process+0x18b fork_exit(c079fee0,c6fdab54,f402ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf402ad70, ebp = 0 --- Tracing command usbus4 pid 32 tid 100062 td 0xc70ed480 sched_switch(c70ed480,0,104,18c,d2494159,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ed480,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdab2c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fdab2c,c6fdabfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4027cf8,c07a006b,c6fdab2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdab2c,c6fdabfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fdab24,f4027d38,c0c52867,335,c73277ec,...) at usb_process+0x18b fork_exit(c079fee0,c6fdab24,f4027d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4027d70, ebp = 0 --- Tracing command usbus4 pid 31 tid 100061 td 0xc70ed6c0 sched_switch(c70ed6c0,0,104,18c,f4e918d5,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ed6c0,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdaafc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fdaafc,c6fdabfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4024cf8,c07a006b,c6fdaafc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdaafc,c6fdabfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fdaaf4,f4024d38,c0c52867,335,c7327a90,...) at usb_process+0x18b fork_exit(c079fee0,c6fdaaf4,f4024d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4024d70, ebp = 0 --- Tracing command usbus4 pid 30 tid 100060 td 0xc70ed900 sched_switch(c70ed900,0,104,18c,f4e9094c,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ed900,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdaacc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fdaacc,c6fdabfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4021cf8,c07a006b,c6fdaacc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdaacc,c6fdabfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fdaac4,f4021d38,c0c52867,335,c7327d34,...) at usb_process+0x18b fork_exit(c079fee0,c6fdaac4,f4021d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4021d70, ebp = 0 --- Tracing command usbus3 pid 29 tid 100059 td 0xc70edb40 sched_switch(c70edb40,0,104,18c,f4e8f9af,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70edb40,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd1b5c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fd1b5c,c6fd1bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f401ecf8,c07a006b,c6fd1b5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd1b5c,c6fd1bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fd1b54,f401ed38,c0c52867,335,c7328000,...) at usb_process+0x18b fork_exit(c079fee0,c6fd1b54,f401ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf401ed70, ebp = 0 --- Tracing command usbus3 pid 28 tid 100058 td 0xc70edd80 sched_switch(c70edd80,0,104,18c,d2488b20,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70edd80,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd1b2c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fd1b2c,c6fd1bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f401bcf8,c07a006b,c6fd1b2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd1b2c,c6fd1bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fd1b24,f401bd38,c0c52867,335,c73282a4,...) at usb_process+0x18b fork_exit(c079fee0,c6fd1b24,f401bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf401bd70, ebp = 0 --- Tracing command usbus3 pid 27 tid 100057 td 0xc7325000 sched_switch(c7325000,0,104,18c,f48477fd,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7325000,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd1afc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fd1afc,c6fd1bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4018cf8,c07a006b,c6fd1afc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd1afc,c6fd1bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fd1af4,f4018d38,c0c52867,335,c7328548,...) at usb_process+0x18b fork_exit(c079fee0,c6fd1af4,f4018d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4018d70, ebp = 0 --- Tracing command usbus3 pid 26 tid 100056 td 0xc7325240 sched_switch(c7325240,0,104,18c,f484676c,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7325240,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd1acc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fd1acc,c6fd1bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4015cf8,c07a006b,c6fd1acc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd1acc,c6fd1bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fd1ac4,f4015d38,c0c52867,335,c73287ec,...) at usb_process+0x18b fork_exit(c079fee0,c6fd1ac4,f4015d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4015d70, ebp = 0 --- Tracing command usbus2 pid 25 tid 100055 td 0xc7325480 sched_switch(c7325480,0,104,18c,f4845705,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7325480,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdb5c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdb5c,c6fcdbfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4012cf8,c07a006b,c6fcdb5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdb5c,c6fcdbfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fcdb54,f4012d38,c0c52867,335,c6e9a7ec,...) at usb_process+0x18b fork_exit(c079fee0,c6fcdb54,f4012d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4012d70, ebp = 0 --- Tracing command usbus2 pid 24 tid 100054 td 0xc73256c0 sched_switch(c73256c0,0,104,18c,d247dd09,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c73256c0,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdb2c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdb2c,c6fcdbfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f400fcf8,c07a006b,c6fcdb2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdb2c,c6fcdbfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fcdb24,f400fd38,c0c52867,335,c6e9aa90,...) at usb_process+0x18b fork_exit(c079fee0,c6fcdb24,f400fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf400fd70, ebp = 0 --- Tracing command usbus2 pid 23 tid 100053 td 0xc7325900 sched_switch(c7325900,0,104,18c,f4211fad,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7325900,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdafc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdafc,c6fcdbfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f400ccf8,c07a006b,c6fcdafc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdafc,c6fcdbfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fcdaf4,f400cd38,c0c52867,335,c6e9ad34,...) at usb_process+0x18b fork_exit(c079fee0,c6fcdaf4,f400cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf400cd70, ebp = 0 --- Tracing command usbus2 pid 22 tid 100052 td 0xc7325b40 sched_switch(c7325b40,0,104,18c,f4210f83,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7325b40,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdacc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdacc,c6fcdbfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4009cf8,c07a006b,c6fcdacc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdacc,c6fcdbfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fcdac4,f4009d38,c0c52867,335,c7313000,...) at usb_process+0x18b fork_exit(c079fee0,c6fcdac4,f4009d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4009d70, ebp = 0 --- Tracing command usbus1 pid 21 tid 100051 td 0xc7325d80 sched_switch(c7325d80,0,104,18c,f420fd81,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7325d80,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc7b5c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc7b5c,c6fc7bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4006cf8,c07a006b,c6fc7b5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc7b5c,c6fc7bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc7b54,f4006d38,c0c52867,335,c73132a4,...) at usb_process+0x18b fork_exit(c079fee0,c6fc7b54,f4006d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4006d70, ebp = 0 --- Tracing command usbus1 pid 20 tid 100050 td 0xc6f676c0 sched_switch(c6f676c0,0,104,18c,d2472139,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f676c0,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc7b2c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc7b2c,c6fc7bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4003cf8,c07a006b,c6fc7b2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc7b2c,c6fc7bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc7b24,f4003d38,c0c52867,335,c7313548,...) at usb_process+0x18b fork_exit(c079fee0,c6fc7b24,f4003d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4003d70, ebp = 0 --- Tracing command usbus1 pid 19 tid 100049 td 0xc6f67900 sched_switch(c6f67900,0,104,18c,f3b6ecd8,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f67900,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc7afc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc7afc,c6fc7bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f4000cf8,c07a006b,c6fc7afc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc7afc,c6fc7bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc7af4,f4000d38,c0c52867,335,c73137ec,...) at usb_process+0x18b fork_exit(c079fee0,c6fc7af4,f4000d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4000d70, ebp = 0 --- Tracing command usbus1 pid 18 tid 100048 td 0xc6f67b40 sched_switch(c6f67b40,0,104,18c,f3b6db7a,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f67b40,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc7acc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc7acc,c6fc7bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f3ffdcf8,c07a006b,c6fc7acc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc7acc,c6fc7bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc7ac4,f3ffdd38,c0c52867,335,c7313a90,...) at usb_process+0x18b fork_exit(c079fee0,c6fc7ac4,f3ffdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ffdd70, ebp = 0 --- Tracing command usbus0 pid 17 tid 100047 td 0xc6f67d80 sched_switch(c6f67d80,0,104,18c,f622fb7a,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f67d80,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2b5c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2b5c,c6fc2bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f3ffacf8,c07a006b,c6fc2b5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2b5c,c6fc2bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc2b54,f3ffad38,c0c52867,335,c7313d34,...) at usb_process+0x18b fork_exit(c079fee0,c6fc2b54,f3ffad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ffad70, ebp = 0 --- Tracing command usbus0 pid 16 tid 100046 td 0xc70ec000 sched_switch(c70ec000,0,104,18c,d249c0ee,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec000,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2b2c,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2b2c,c6fc2bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f3ff7cf8,c07a006b,c6fc2b2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2b2c,c6fc2bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc2b24,f3ff7d38,c0c52867,335,c6d782a4,...) at usb_process+0x18b fork_exit(c079fee0,c6fc2b24,f3ff7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff7d70, ebp = 0 --- Tracing command usbus0 pid 15 tid 100045 td 0xc70ec240 sched_switch(c70ec240,0,104,18c,f5bef7c2,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec240,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2afc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2afc,c6fc2bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f3ff4cf8,c07a006b,c6fc2afc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2afc,c6fc2bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc2af4,f3ff4d38,c0c52867,335,c6d78548,...) at usb_process+0x18b fork_exit(c079fee0,c6fc2af4,f3ff4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff4d70, ebp = 0 --- Tracing command usbus0 pid 14 tid 100044 td 0xc70ec480 sched_switch(c70ec480,0,104,18c,f5bee76e,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec480,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2acc,0,c0c4634c,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2acc,c6fc2bfc,0,c0c4634c,0,...) at _sleep+0x35b usb2_msleep(c0c4634c,0,f3ff1cf8,c07a006b,c6fc2acc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2acc,c6fc2bfc,c0c462ab,51,c0dabd80,...) at usb2_cv_wait+0x23 usb_process(c6fc2ac4,f3ff1d38,c0c52867,335,c6d787ec,...) at usb_process+0x18b fork_exit(c079fee0,c6fc2ac4,f3ff1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff1d70, ebp = 0 --- Tracing command sctp_iterator pid 8 tid 100043 td 0xc70ec6c0 sched_switch(c70ec6c0,0,104,18c,14621d2,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec6c0,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0f1e4bc,0,c0c6f79c,0,0,...) at sleepq_wait+0x63 _sleep(c0f1e4bc,c0f1e3d0,0,c0c6f79c,0,...) at _sleep+0x35b sctp_iterator_thread(0,f3feed38,c0c52867,335,c6d78a90,...) at sctp_iterator_thread+0x60 fork_exit(c09759a0,0,f3feed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3feed70, ebp = 0 --- Tracing command fdc0 pid 7 tid 100040 td 0xc70ecd80 sched_switch(c70ecd80,0,104,18c,7ea7b918,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c70ecd80,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6fd043c,4c,c0c4d0ea,0,0,...) at sleepq_timedwait+0x6b _sleep(c6fd043c,c6fd04f0,4c,c0c4d0ea,3e8,...) at _sleep+0x329 fdc_thread(c6fd0400,f3fe2d38,c0c52867,335,c6d78d34,...) at fdc_thread+0x2be fork_exit(c0b432e0,c6fd0400,f3fe2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fe2d70, ebp = 0 --- Tracing command fw0_probe pid 6 tid 100038 td 0xc6f4fd80 sched_switch(c6f4fd80,0,104,18c,19df87b6,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c6f4fd80,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6fff000,5c,c0c4d0ea,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6fff000,c7003488,15c,c0c4d0ea,0,...) at _sleep+0x344 fw_bus_probe_thread(c6fff000,f3fd2d38,c0c52867,335,c6e9a000,...) at fw_bus_probe_thread+0xa08 fork_exit(c0640610,c6fff000,f3fd2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fd2d70, ebp = 0 --- Tracing command xpt_thrd pid 5 tid 100025 td 0xc6dffb40 sched_switch(c6dffb40,0,104,18c,f5b9c903,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6dffb40,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0d74e54,4c,c0bed8f8,0,0,...) at sleepq_wait+0x63 _sleep(c0d74e54,c0d74e6c,4c,c0bed8f8,0,...) at _sleep+0x35b xpt_scanner_thread(0,c6ad6d38,c0c52867,335,c6e9a2a4,...) at xpt_scanner_thread+0x41 fork_exit(c047f130,0,c6ad6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad6d70, ebp = 0 --- Tracing command yarrow pid 13 tid 100017 td 0xc6d7b240 sched_switch(c6d7b240,0,104,18c,b0717b19,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d7b240,0,c0c5b48b,26a,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0da7e44,0,c0c4d0ea,2,0,...) at sleepq_timedwait+0x6b _sleep(c0da7e44,0,0,c0c4d0ea,64,...) at _sleep+0x329 pause(c0c4d0ea,64,c0c33ab8,111,0,...) at pause+0x47 random_kthread(0,c6abed38,c0c52867,335,c6e9a548,...) at random_kthread+0x1ef fork_exit(c0718e60,0,c6abed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abed70, ebp = 0 --- Tracing command g_down pid 4 tid 100015 td 0xc6d7b6c0 sched_switch(c6d7b6c0,0,104,18c,b20819f4,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d7b6c0,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da5be4,4c,c0c4d0ea,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da5be4,c0da5b48,24c,c0c4d0ea,64,...) at _sleep+0x329 g_io_schedule_down(c6d7b6c0,0,c0c4e7b2,74,0,...) at g_io_schedule_down+0x6b g_down_procbody(0,c6ab8d38,c0c52867,335,c6d77000,...) at g_down_procbody+0x8d fork_exit(c0806d80,0,c6ab8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab8d70, ebp = 0 --- Tracing command g_up pid 3 tid 100014 td 0xc6d7b900 cpustop_handler(1,c6ab5b90,c0b8a54d,c93cab40,2bc0afb,...) at cpustop_handler+0x32 ipi_nmi_handler(c93cab40,2bc0afb,1,c6d2d780,c6d772a4,...) at ipi_nmi_handler+0x2f trap(c6ab5b9c) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b7ce68, esp = 0xc6ab5bdc, ebp = 0xc6ab5bfc --- smp_tlb_shootdown(c6ab5c24,c0b818af,eef21000,eef22000,c6ab5c70,...) at smp_tlb_shootdown+0x98 smp_invlpg_range(eef21000,eef22000,c6ab5c70,c0c61fe6,eef22000,...) at smp_invlpg_range+0x1c pmap_invalidate_range(c0f79be0,eef21000,eef22000) at pmap_invalidate_range+0x4f pmap_qremove(eef21000,1,c0ee8108,c6d7b9a4,c0ee8108,...) at pmap_qremove+0x54 swp_pager_async_iodone(e6cdc430,c6d43498,c78337e0,c0aa5c90,c6ab5c98,...) at swp_pager_async_iodone+0x5c bufdone(e6cdc430,c6d43498,c6ab5cbc,c08db4a5,c78337e0,...) at bufdone+0x53 swapgeom_done(c78337e0,0,c0c61fdd,c17,c78337e0,...) at swapgeom_done+0x37 biodone(c78337e0,c0da5b68,24c,c0c4d0ea,64,...) at biodone+0xa5 g_io_schedule_up(c6d7b900,0,c0c4e7b2,5d,0,...) at g_io_schedule_up+0xc7 g_up_procbody(0,c6ab5d38,c0c52867,335,c6d772a4,...) at g_up_procbody+0x8d fork_exit(c0806e10,0,c6ab5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab5d70, ebp = 0 --- Tracing command g_event pid 2 tid 100013 td 0xc6d7bb40 sched_switch(c6d7bb40,0,104,18c,b063e28a,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d7bb40,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da5bd8,4c,c0c4d0ea,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da5bd8,0,4c,c0c4d0ea,64,...) at _sleep+0x329 g_event_procbody(0,c6ab2d38,c0c52867,335,c6d77548,...) at g_event_procbody+0xcb fork_exit(c0806ea0,0,c6ab2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab2d70, ebp = 0 --- Tracing command intr pid 12 tid 100042 td 0xc70ec900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100041 td 0xc70ecb40 sched_switch(c70ecb40,0,109,18c,f5b4ecdf,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6d75170,...) at mi_switch+0x200 ithread_loop(c70ee320,f3fe5d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c70ee320,f3fe5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fe5d70, ebp = 0 --- Tracing command intr pid 12 tid 100039 td 0xc70ed000 sched_switch(c70ed000,0,109,18c,c2751722,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6ffa570,...) at mi_switch+0x200 ithread_loop(c70e25f0,f3fd5d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c70e25f0,f3fd5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fd5d70, ebp = 0 --- Tracing command intr pid 12 tid 100036 td 0xc6f65240 sched_switch(c6f65240,0,109,18c,f391c928,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbe1f0,...) at mi_switch+0x200 ithread_loop(c6ffb900,f3fc9d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6ffb900,f3fc9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc9d70, ebp = 0 --- Tracing command intr pid 12 tid 100035 td 0xc6f65480 sched_switch(c6f65480,0,109,18c,b40e25,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbe4f0,...) at mi_switch+0x200 ithread_loop(c6ffb170,f3fc1d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6ffb170,f3fc1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc1d70, ebp = 0 --- Tracing command intr pid 12 tid 100034 td 0xc6f656c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100033 td 0xc6f65900 sched_switch(c6f65900,0,109,18c,1f9d9e67,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbe2f0,...) at mi_switch+0x200 ithread_loop(c6fa8430,f3fafd38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6fa8430,f3fafd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fafd70, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xc6f65b40 sched_switch(c6f65b40,0,109,18c,1f1afb7c,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbe370,...) at mi_switch+0x200 ithread_loop(c6fa86b0,f3fabd38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6fa86b0,f3fabd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fabd70, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xc6f65d80 sched_switch(c6f65d80,0,109,18c,fabcb975,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbe3f0,...) at mi_switch+0x200 ithread_loop(c6f92210,f3fa7d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6f92210,f3fa7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fa7d70, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc6f67000 sched_switch(c6f67000,0,109,18c,b37d0e78,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbe0f0,...) at mi_switch+0x200 ithread_loop(c6f92450,f3c84d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6f92450,f3c84d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3c84d70, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc6f67240 sched_switch(c6f67240,0,109,18c,d9d44e6,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6f7a670,...) at mi_switch+0x200 ithread_loop(c6f8c620,c6b12d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6f8c620,c6b12d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6b12d70, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc6f67480 sched_switch(c6f67480,0,109,18c,f5b60b8e,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6d74c70,...) at mi_switch+0x200 ithread_loop(c6f4e3f0,c6b05d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6f4e3f0,c6b05d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6b05d70, ebp = 0 --- Tracing command intr pid 12 tid 100027 td 0xc6dff6c0 sched_switch(c6dff6c0,0,109,18c,9f3b756b,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6f4d4f0,...) at mi_switch+0x200 ithread_loop(c6f4c870,c6adcd38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6f4c870,c6adcd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6adcd70, ebp = 0 --- Tracing command intr pid 12 tid 100026 td 0xc6dff900 sched_switch(c6dff900,0,109,18c,11328b85,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6f4d570,...) at mi_switch+0x200 ithread_loop(c6f4c880,c6ad9d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6f4c880,c6ad9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad9d70, ebp = 0 --- Tracing command intr pid 12 tid 100020 td 0xc6f4f6c0 sched_switch(c6f4f6c0,0,109,18c,dbf8f0c9,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6f4d770,...) at mi_switch+0x200 ithread_loop(c6f4c920,c6ac7d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6f4c920,c6ac7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac7d70, ebp = 0 --- Tracing command intr pid 12 tid 100019 td 0xc6f4f900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100012 td 0xc6d7bd80 sched_switch(c6d7bd80,0,109,18c,b202c233,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbdc70,...) at mi_switch+0x200 ithread_loop(c6d761c0,c6aafd38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6d761c0,c6aafd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aafd70, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc6dff000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100010 td 0xc6dff240 sched_switch(c6dff240,0,109,18c,8fec1648,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbdd70,...) at mi_switch+0x200 ithread_loop(c6d761e0,c6aa9d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6d761e0,c6aa9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa9d70, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc6dff480 sched_switch(c6dff480,0,109,18c,b0716eb1,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbddf0,...) at mi_switch+0x200 ithread_loop(c6d761f0,c6aa6d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6d761f0,c6aa6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa6d70, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc6d79000 sched_switch(c6d79000,0,109,18c,a59b5ba1,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbde70,...) at mi_switch+0x200 ithread_loop(c6d76200,c6aa3d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6d76200,c6aa3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa3d70, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc6d79240 sched_switch(c6d79240,0,109,18c,b200525f,...) at sched_switch+0x437 mi_switch(109,0,c0c52ae6,4e3,c6dbdef0,...) at mi_switch+0x200 ithread_loop(c6d76210,c6aa0d38,c0c52867,335,c6d777ec,...) at ithread_loop+0x1f6 fork_exit(c0842350,c6d76210,c6aa0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa0d70, ebp = 0 --- Tracing command idle pid 11 tid 100006 td 0xc6d79480 sched_switch(c6d79480,0,108,187,b20e63bd,...) at sched_switch+0x437 mi_switch(108,0,c0c58d0f,3ad,c6d79480,...) at mi_switch+0x200 sched_idletd(0,c6a9cd38,c0c52867,335,c6d77a90,...) at sched_idletd+0x16c fork_exit(c088af90,0,c6a9cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a9cd70, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc6d796c0 sched_switch(c6d796c0,0,108,187,9f02eee3,...) at sched_switch+0x437 mi_switch(108,0,c0c58d0f,3ad,c6d796c0,...) at mi_switch+0x200 sched_idletd(0,c6a99d38,c0c52867,335,c6d77a90,...) at sched_idletd+0x16c fork_exit(c088af90,0,c6a99d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a99d70, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc6d79900 sched_switch(c6d79900,0,108,187,b20ed1aa,...) at sched_switch+0x437 mi_switch(108,0,c0c58d0f,3ad,c6d79900,...) at mi_switch+0x200 sched_idletd(0,c6a96d38,c0c52867,335,c6d77a90,...) at sched_idletd+0x16c fork_exit(c088af90,0,c6a96d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a96d70, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc6d79b40 sched_switch(c6d79b40,0,108,187,b204f2af,...) at sched_switch+0x437 mi_switch(108,0,c0c58d0f,3ad,c6d79b40,...) at mi_switch+0x200 sched_idletd(0,c6a93d38,c0c52867,335,c6d77a90,...) at sched_idletd+0x16c fork_exit(c088af90,0,c6a93d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a93d70, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc6d79d80 sched_switch(c6d79d80,0,104,18c,cfc28908,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c6d79d80,0,c0c5b48b,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5b48b,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6d77d34,5c,c0c5dcf4,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6d77d34,c6d77dbc,15c,c0c5dcf4,0,...) at _sleep+0x344 kern_wait(c6d79d80,ffffffff,c6a8fc74,0,0,...) at kern_wait+0xac6 wait4(c6d79d80,c6a8fcf8,10,c0c5db24,c0d3b184,...) at wait4+0x3b syscall(c6a8fd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x8054c5f, esp = 0xbfbfe90c, ebp = 0xbfbfe928 --- Tracing command audit pid 10 tid 100001 td 0xc6d7b000 sched_switch(c6d7b000,0,104,18c,f5b7641b,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d7b000,0,c0c5b48b,247,c6d7b000,...) at sleepq_switch+0x15f sleepq_wait(c0f276e0,0,c6a8cc9c,1,0,...) at sleepq_wait+0x63 _cv_wait(c0f276e0,c0f276c4,c0c7a1b1,194,0,...) at _cv_wait+0x240 audit_worker(0,c6a8cd38,c0c52867,335,c6d78000,...) at audit_worker+0x84 fork_exit(c0a5c1c0,0,c6a8cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a8cd70, ebp = 0 --- Tracing command kernel pid 0 tid 100037 td 0xc6f65000 sched_switch(c6f65000,0,104,18c,b1ffea85,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f65000,0,c0c5b48b,247,c6f65000,...) at sleepq_switch+0x15f sleepq_wait(c6fd7a40,0,c0c57c8b,c0c4d0ea,0,...) at sleepq_wait+0x63 msleep_spin(c6fd7a40,c6fd7a5c,c0c4d0ea,0,c0c5591d,...) at msleep_spin+0x21d taskqueue_thread_loop(c700349c,f3fcfd38,c0c52867,335,c0da5ca0,...) at taskqueue_thread_loop+0x8e fork_exit(c08a2780,c700349c,f3fcfd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fcfd70, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc6dffd80 sched_switch(c6dffd80,0,104,18c,f62e566d,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6dffd80,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f2a0c0,0,c0c4d0ea,0,0,...) at sleepq_wait+0x63 _sleep(c6f2a0c0,c6f2a0dc,0,c0c4d0ea,0,...) at _sleep+0x35b taskqueue_thread_loop(c0da6564,c6ad3d38,c0c52867,335,c0da5ca0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a2780,c0da6564,c6ad3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad3d70, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc6f4f000 sched_switch(c6f4f000,0,104,18c,f62e4550,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4f000,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f2a100,0,c0c4d0ea,0,0,...) at sleepq_wait+0x63 _sleep(c6f2a100,c6f2a11c,0,c0c4d0ea,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d77b04,c6ad0d38,c0c52867,335,c0da5ca0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a2780,c0d77b04,c6ad0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad0d70, ebp = 0 --- Tracing command kernel pid 0 tid 100022 td 0xc6f4f240 sched_switch(c6f4f240,0,104,18c,f62e3557,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4f240,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f2a100,0,c0c4d0ea,0,0,...) at sleepq_wait+0x63 _sleep(c6f2a100,c6f2a11c,0,c0c4d0ea,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d77b04,c6acdd38,c0c52867,335,c0da5ca0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a2780,c0d77b04,c6acdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6acdd70, ebp = 0 --- Tracing command kernel pid 0 tid 100021 td 0xc6f4f480 sched_switch(c6f4f480,0,104,18c,f62e22f9,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4f480,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f2a100,0,c0c4d0ea,0,0,...) at sleepq_wait+0x63 _sleep(c6f2a100,c6f2a11c,0,c0c4d0ea,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d77b04,c6acad38,c0c52867,335,c0da5ca0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a2780,c0d77b04,c6acad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6acad70, ebp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xc6f4fb40 sched_switch(c6f4fb40,0,104,18c,1134a774,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4fb40,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f2a400,0,c0c4d0ea,0,0,...) at sleepq_wait+0x63 _sleep(c6f2a400,c6f2a41c,0,c0c4d0ea,0,...) at _sleep+0x35b taskqueue_thread_loop(c0dba768,c6ac1d38,c0c52867,335,c0da5ca0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a2780,c0dba768,c6ac1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac1d70, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc6d7b480 sched_switch(c6d7b480,0,104,18c,186ad277,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d7b480,0,c0c5b48b,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6d5fb40,0,c0c4d0ea,0,0,...) at sleepq_wait+0x63 _sleep(c6d5fb40,c6d5fb5c,0,c0c4d0ea,0,...) at _sleep+0x35b taskqueue_thread_loop(c0db92a0,c6abbd38,c0c52867,335,c0da5ca0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a2780,c0db92a0,c6abbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abbd70, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc0da5f50 sched_switch(c0da5f50,0,104,18c,f484ff6d,...) at sched_switch+0x437 mi_switch(104,0,c0c5b48b,1d2,44,...) at mi_switch+0x200 sleepq_switch(c0da5f50,0,c0c5b48b,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da5ca0,44,c0c59584,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da5ca0,0,44,c0c59584,2710,...) at _sleep+0x329 scheduler(0,141ec00,141ec00,141e000,1425000,...) at scheduler+0x23e mi_startup() at mi_startup+0x96 begin() at begin+0x2c db:1:allt> call doadump Physical memory: 3439 MB Dumping 188 MB: 173 157 141 125 109 93 77 61 45 29 13 Dump complete = 0xf db:1:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 1 (kgdb) bt #0 doadump () at pcpu.h:246 #1 0xc04c1949 in db_fncall (dummy1=0xc08703a7, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf681060c " \006\201ö\001") at ../../../ddb/db_command.c:548 #2 0xc04c1d7f in db_command (last_cmdp=0xc0d75ddc, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #3 0xc04c1e34 in db_command_script (command=0xc0d76d68 "call doadump") at ../../../ddb/db_command.c:516 #4 0xc04c5f30 in db_script_exec (scriptname=0xc0d76640 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc04c5fc1 in db_run_cmd (addr=0x0, have_addr=0x0, count=0x0, modif=0xf6810744 "X\a\201ö") at ../../../ddb/db_script.c:375 #6 0xc04c1d7f in db_command (last_cmdp=0xc0d75ddc, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #7 0xc04c1e34 in db_command_script (command=0xc0d76ca0 "run pho") at ../../../ddb/db_command.c:516 #8 0xc04c5f30 in db_script_exec (scriptname=0xf6810850 "kdb.enter.unknown", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #9 0xc04c6017 in db_script_kdbenter (eventname=0xc0c8c215 "unknown") at ../../../ddb/db_script.c:324 #10 0xc04c3cf8 in db_trap (type=0xc, code=0x0) at ../../../ddb/db_main.c:228 #11 0xc0896b06 in kdb_trap (type=0xc, code=0x0, tf=0xf68109d0) at ../../../kern/subr_kdb.c:534 #12 0xc0b89d2f in trap_fatal (frame=0xf68109d0, eva=0x19c) at ../../../i386/i386/trap.c:924 #13 0xc0b8a6a1 in trap (frame=0xf68109d0) at ../../../i386/i386/trap.c:325 #14 0xc0b6d43b in calltrap () at ../../../i386/i386/exception.s:165 #15 0xc08703a7 in _sx_xlock_hard (sx=0xc87e7bc8, tid=0xcfccc240, opts=0x0, file=0xc0c54490 "../../../kern/kern_lockf.c", line=0x795) at ../../../kern/kern_sx.c:575 #16 0xc08708e0 in _sx_xlock (sx=0xc87e7bc8, opts=0x0, file=0xc0c54490 "../../../kern/kern_lockf.c", line=0x795) at sx.h:155 #17 0xc0852c4d in lf_iteratelocks_vnode (vp=0xc945b158, fn=0xc0a372a0 , arg=0x0) at ../../../kern/kern_lockf.c:1941 #18 0xc0a35460 in nlm_reclaim (ap=0xf6810b14) at ../../../nlm/nlm_advlock.c:480 #19 0xc0a1568e in nfs_reclaim (ap=0xf6810b14) at ../../../nfsclient/nfs_node.c:234 #20 0xc0b97d55 in VOP_RECLAIM_APV (vop=0xc0d59380, a=0xf6810b14) at vnode_if.c:1926 #21 0xc08f6894 in vgonel (vp=0xc945b158) at vnode_if.h:830 #22 0xc08f9337 in vflush (mp=0xcd386b40, rootrefs=0x1, flags=0x2, td=0xcfccc240) at ../../../kern/vfs_subr.c:2450 #23 0xc0a1a451 in nfs_unmount (mp=0xcd386b40, mntflags=0x8080000) at ../../../nfsclient/nfs_vfsops.c:1320 #24 0xc08f041d in dounmount (mp=0xcd386b40, flags=0x8080000, td=0xcfccc240) at ../../../kern/vfs_mount.c:1288 #25 0xc08f098f in unmount (td=0xcfccc240, uap=0xf6810cf8) at ../../../kern/vfs_mount.c:1173 #26 0xc0b8a244 in syscall (frame=0xf6810d38) at ../../../i386/i386/trap.c:1073 #27 0xc0b6d4a0 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261 #28 0x00000033 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) f 17 #17 0xc0852c4d in lf_iteratelocks_vnode (vp=0xc945b158, fn=0xc0a372a0 , arg=0x0) at ../../../kern/kern_lockf.c:1941 1941 sx_xlock(&ls->ls_lock); (kgdb) l 1936 STAILQ_INIT(&locks); 1937 ls = vp->v_lockf; 1938 if (!ls) 1939 return (0); 1940 1941 sx_xlock(&ls->ls_lock); 1942 LIST_FOREACH(lf, &ls->ls_active, lf_link) { 1943 ldesc = malloc(sizeof(struct lockdesc), M_LOCKF, 1944 M_WAITOK); 1945 ldesc->vp = lf->lf_vnode; (kgdb) p *ls $1 = {ls_link = {le_next = 0xdeadc0de, le_prev = 0xdeadc0de}, ls_lock = {lock_object = {lo_name = 0xdeadc0de
, lo_flags = 0xdeadc0de, lo_data = 0xdeadc0de, lo_witness = 0xdeadc0de}, sx_lock = 0xdeadc0de}, ls_active = {lh_first = 0xdeadc0de}, ls_pending = {lh_first = 0xdeadc0de}, ls_threads = 0xdeadc0de} (kgdb) p *vp $2 = {v_type = VREG, v_tag = 0xc0c77da8 "nfs", v_op = 0xc0d59380, v_data = 0xc9534740, v_mount = 0xcd386b40, v_nmntvnodes = { tqe_next = 0xc8933a00, tqe_prev = 0xcd386b74}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0, vu_yield = 0x0}, v_hashlist = {le_next = 0x0, le_prev = 0xc713feb0}, v_hash = 0x9d7886e8, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0x0, tqh_last = 0xc945b188}, v_cache_dd = 0x0, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0, v_clen = 0x0, v_lock = {lock_object = { lo_name = 0xc0c77da8 "nfs", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc6d30770}, lk_lock = 0xcfccc242, lk_timo = 0x33, lk_pri = 0x50, lk_stack = {depth = 0xa, pcs = {0xc0851f80, 0xc08e5b85, 0xc0b984f5, 0xc0902c08, 0xc08f913c, 0xc0a1a451, 0xc08f041d, 0xc08f098f, 0xc0b8a244, 0xc0b6d4a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}}, v_interlock = {lock_object = { lo_name = 0xc0c5d584 "vnode interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d2a248}, mtx_lock = 0x4}, v_vnlock = 0xc945b1b0, v_holdcnt = 0x2, v_usecount = 0x1, v_iflag = 0x80, v_vflag = 0x0, v_writecount = 0x1, v_freelist = {tqe_next = 0x0, tqe_prev = 0x0}, v_bufobj = {bo_mtx = {lock_object = {lo_name = 0xc0c65335 "bufobj interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d2ea30}, mtx_lock = 0x4}, bo_clean = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc945b260}, bv_root = 0x0, bv_cnt = 0x0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc945b270}, bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x0, bo_ops = 0xc0d59548, bo_bsize = 0x8000, bo_object = 0xc87b9e88, bo_synclist = {le_next = 0xc7506e64, le_prev = 0xc70dc364}, bo_private = 0xc945b158, __bo_vnode = 0xc945b158}, v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0} (kgdb) $ svn diff /usr/src/sys Index: /usr/src/sys/rpc/svc.c =================================================================== --- /usr/src/sys/rpc/svc.c (revision 193481) +++ /usr/src/sys/rpc/svc.c (working copy) @@ -174,6 +174,7 @@ svc_unreg(pool, s->sc_prog, s->sc_vers); mtx_lock(&pool->sp_lock); } + mtx_unlock(&pool->sp_lock); TAILQ_FOREACH_SAFE(xprt, &cleanup, xp_link, nxprt) { SVC_RELEASE(xprt); @@ -292,6 +293,8 @@ { SVCPOOL *pool = xprt->xp_pool; + KASSERT(xprt->xp_registered == TRUE, + ("xprt_unregister_locked: not registered")); if (xprt->xp_active) { TAILQ_REMOVE(&pool->sp_active, xprt, xp_alink); xprt->xp_active = FALSE; @@ -306,6 +309,11 @@ SVCPOOL *pool = xprt->xp_pool; mtx_lock(&pool->sp_lock); + if (xprt->xp_registered == FALSE) { + /* Already unregistered by aother thread */ + mtx_unlock(&pool->sp_lock); + return; + } xprt_unregister_locked(xprt); mtx_unlock(&pool->sp_lock); -- Test scenario: mount -t nfs -o nfsv3,tcp,nolockd -o retrycnt=3 -o intr -o soft -o rw 127.0.0.1:/tmp $mntpoint # stress test for 10 minutes on $mntpoint umount $mntpoint while mount | grep -q $mntpoint; do umount -f $mntpoint done