GENERIC HEAD from 2009-06-23 19:05:02 UTC, r194752M, vmcore.206 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2009 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 8.0-CURRENT #0 r194752M: Tue Jun 23 21:15:36 CEST 2009 pho@x4.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1799.99-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) avail memory = 3539746816 (3375 MB) : Trying to mount root from ufs:/dev/ad4s1a Entropy harvesting: interrupts ethernet point_to_point kickstart. /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 245687 free (3575 frags, 30264 blocks, 0.4% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 50264196 free (2116 frags, 6282760 blocks, 0.0% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 2178260 free (243820 frags, 241805 blocks, 1.1% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 15855371 free (107995 frags, 1968422 blocks, 0.2% fragmentation) re0: link state changed to DOWN Starting Network: lo0 re0. add net default: gateway 192.168.1.1 Additional ABI support: linux. lock order reversal: 1st 0xe6d193d8 bufwait (bufwait) @ kern/vfs_bio.c:2558 2nd 0xc731b400 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:285 KDB: stack backtrace: db_trace_self_wrapper(c0c5c38a,f638887c,c08b5ee5,c08a6b5b,c0c5f29f,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08a6b5b,c0c5f29f,c6d2ad58,c6d2ddb0,f63888d8,...) at kdb_backtrace+0x29 _witness_debugger(c0c5f29f,c731b400,c0c7fd2c,c6d2ddb0,c0c7f9d2,...) at _witness_debugger+0x25 witness_checkorder(c731b400,9,c0c7f9c9,11d,0,...) at witness_checkorder+0x839 _sx_xlock(c731b400,0,c0c7f9c9,11d,c747e828,...) at _sx_xlock+0x85 ufsdirhash_acquire(e6d19378,f6388a20,1bc,e770aa5c,f63889a8,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(c747e828,f6388a20,1a5c,f6388994,f6388998,...) at ufsdirhash_add+0x13 ufs_direnter(c760c408,c7772408,f6388a20,f6388c04,e6d1984c,...) at ufs_direnter+0x779 ufs_mkdir(f6388c28,c0c94c2e,0,0,f6388b70,...) at ufs_mkdir+0x981 VOP_MKDIR_APV(c0d5e780,f6388c28,f6388c04,f6388b70,0,...) at VOP_MKDIR_APV+0xc5 kern_mkdirat(c7831900,ffffff9c,bfbfef5a,0,1ff,...) at kern_mkdirat+0x258 kern_mkdir(c7831900,bfbfef5a,0,1ff,f6388d2c,...) at kern_mkdir+0x2e mkdir(c7831900,f6388cf8,8,c0c5fb8d,c0d3e900,...) at mkdir+0x29 syscall(f6388d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (136, FreeBSD ELF32, mkdir), eip = 0x28165303, esp = 0xbfbfed6c, ebp = 0xbfbfee38 --- Configuring syscons: keymap blanktime. Local package initialization:lock order reversal: 1st 0xc7479b18 ufs (ufs) @ kern/vfs_mount.c:1197 2nd 0xc74ba460 devfs (devfs) @ ufs/ffs/ffs_softdep.c:1029 KDB: stack backtrace: db_trace_self_wrapper(c0c5c38a,f627c9f0,c08b5ee5,c08a6b5b,c0c5f29f,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08a6b5b,c0c5f29f,c6d2dd48,c6d2dc10,f627ca4c,...) at kdb_backtrace+0x29 _witness_debugger(c0c5f29f,c74ba460,c0c4e100,c6d2dc10,c0c7dd6d,...) at _witness_debugger+0x25 witness_checkorder(c74ba460,9,c0c7dd64,405,c74ba4c8,...) at witness_checkorder+0x839 __lockmgr_args(c74ba460,80400,c74ba4c8,0,0,...) at __lockmgr_args+0x7b7 vop_stdlock(f627cb54,4,c0c578b6,80400,c74ba408,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0d3a680,f627cb54,c0f2aa44,c0d77240,c74ba408,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c74ba408,80400,c0c7dd64,405,c75f25a0,...) at _vn_lock+0x78 softdep_flushworklist(c75f25a0,f627cc00,c72fc6c0,52b,c760c2b0,...) at softdep_flushworklist+0x47 ffs_sync(c75f25a0,1,c0c65bdd,4f2,80,...) at ffs_sync+0x2fd dounmount(c75f25a0,8000000,c72fc6c0,478,ff1ca3a5,...) at dounmount+0x44e unmount(c72fc6c0,f627ccf8,8,f627cd38,c0d3dc88,...) at unmount+0x2df syscall(f627cd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d916f, esp = 0xbfbfe69c, ebp = 0xbfbfe768 --- ** /dev/ad4s1e ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes Jun 23 21:27:04 x4 su: pho to root on /dev/pts/1 ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 24037 files, 512879 used, 50264196 free (2124 frags, 6282759 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** mount: /dev/ad4s1e : Operation not permitted fsck -y /tmpJun 23 21:27:45 x4 mountd[813]: can't change attributes for /mnt Jun 23 21:27:45 x4 mountd[813]: bad exports list line /mnt watchdogd. Tue Jun 23 21:27:46 CEST 2009 FreeBSD/i386 (x4.osted.lan) (console) login: lock order reversal: 1st 0xc7b51058 ufs (ufs) @ kern/vfs_lookup.c:495 2nd 0xe6f60d70 bufwait (bufwait) @ ufs/ffs/ffs_softdep.c:6150 3rd 0xc8c22dc8 ufs (ufs) @ kern/vfs_subr.c:2084 KDB: stack backtrace: db_trace_self_wrapper(c0c5c38a,f638803c,c08b5ee5,c08a6b5b,c0c5f2b8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08a6b5b,c0c5f2b8,c6d2ad58,c6d2dd48,f6388098,...) at kdb_backtrace+0x29 _witness_debugger(c0c5f2b8,c8c22dc8,c0c51f31,c6d2dd48,c0c663c3,...) at _witness_debugger+0x25 witness_checkorder(c8c22dc8,9,c0c663ba,824,0,...) at witness_checkorder+0x839 __lockmgr_args(c8c22dc8,80100,c8c22e30,0,0,...) at __lockmgr_args+0x7b7 ffs_lock(f63881a4,c08b5c8b,c0c658c2,80100,c8c22d70,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0d5e780,f63881a4,c78319a4,c0d77240,c8c22d70,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c8c22d70,80100,c0c663ba,824,4,...) at _vn_lock+0x78 vget(c8c22d70,80100,c7831900,50,0,...) at vget+0xbb vfs_hash_get(c7b15000,94c802,80000,c7831900,f6388300,...) at vfs_hash_get+0xed ffs_vgetf(c7b15000,94c802,80000,f6388300,1,...) at ffs_vgetf+0x49 softdep_sync_metadata(c7b51000,0,c0c7f615,131,0,...) at softdep_sync_metadata+0x5ba ffs_syncvnode(c7b51000,1,c7831900,f63883c0,246,...) at ffs_syncvnode+0x3e2 ffs_truncate(c7b51000,200,0,880,c6d71100,...) at ffs_truncate+0x696 ufs_direnter(c7b51000,c8c22d70,f638867c,f638897c,e6eab8f8,...) at ufs_direnter+0x946 ufs_mkdir(f63889a0,c0c94c2e,f63888e8,f6388950,f63889a0,...) at ufs_mkdir+0x981 VOP_MKDIR_APV(c0d5e780,f63889a0,f63889cc,b,0,...) at VOP_MKDIR_APV+0xc5 nfsrv_mkdir(f6388aa0,0,f6388a98,0,c7601800,...) at nfsrv_mkdir+0x5ec nfssvc_program(c7601800,c86c7000,c0c7b278,490,f6388bcc,...) at nfssvc_program+0x3a0 svc_run_internal(c7831a64,14,c0c7b473,c0c7a0d7,f6388c3c,...) at svc_run_internal+0x942 svc_run(c70bf480,0,c0c7a3db,200,0,...) at svc_run+0x7f nfssvc_nfsd(bfbfe8c0,f6388c3c,c,c6d71100,f6388c50,...) at nfssvc_nfsd+0xad nfssvc_nfsserver(c7831900,f6388cf8,bfbfe8c0,c7831900,c782a2a8,...) at nfssvc_nfsserver+0x24f nfssvc(c7831900,f6388cf8,8,c0c60075,c0d3eb14,...) at nfssvc+0x77 syscall(f6388d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280d9b0b, esp = 0xbfbfe87c, ebp = 0xbfbfeae8 --- Expensive timeout(9) function: 0xc08ac5b0(0xc6d77900) 0.103160890 s Jun 23 21:51:51 x4 kernel: pid 3678 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 21:51:58 x4 kernel: pid 3675 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 21:51:59 x4 kernel: pid 3825 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 21:52:01 x4 kernel: pid 3685 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 21:52:09 x4 kernel: pid 3688 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 21:53:15 x4 kernel: pid 3687 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 21:59:01 x4 kernel: pid 5273 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:08:08 x4 kernel: pid 8329 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:22 x4 kernel: pid 8016 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:23 x4 kernel: pid 8019 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:23 x4 kernel: pid 8040 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:23 x4 kernel: pid 8041 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:23 x4 kernel: pid 8043 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:24 x4 kernel: pid 8039 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:24 x4 kernel: pid 8045 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:25 x4 kernel: pid 8013 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:25 x4 kernel: pid 8017 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:14:31 x4 kernel: pid 8012 (mkdir), uid 0, was killed: exceeded maximum CPU limit Expensive timeout(9) function: 0xc08d0560(0) 0.283272526 s Jun 23 22:45:03 x4 kernel: pid 33258 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:45:09 x4 kernel: pid 33253 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:45:12 x4 kernel: pid 33248 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:06 x4 kernel: pid 32619 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:06 x4 kernel: pid 32623 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:06 x4 kernel: pid 32627 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:07 x4 kernel: pid 32625 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:08 x4 kernel: pid 32626 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:08 x4 kernel: pid 32628 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:08 x4 kernel: pid 32631 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:09 x4 kernel: pid 32629 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:09 x4 kernel: pid 32630 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 22:52:09 x4 kernel: pid 32632 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:03:12 x4 kernel: pid 40375 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:25 x4 kernel: pid 40140 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:26 x4 kernel: pid 40146 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:29 x4 kernel: pid 40134 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:31 x4 kernel: pid 40142 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:33 x4 kernel: pid 40149 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:43 x4 kernel: pid 40144 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:44 x4 kernel: pid 40150 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:45 x4 kernel: pid 40138 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:07:46 x4 kernel: pid 40148 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:21:47 x4 kernel: pid 45515 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:26:42 x4 kernel: pid 46421 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:26:55 x4 kernel: pid 46431 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:27:11 x4 kernel: pid 46449 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:31:42 x4 kernel: pid 46427 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:32:44 x4 kernel: pid 50019 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:32:44 x4 kernel: pid 50013 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:10 x4 kernel: pid 49700 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:21 x4 kernel: pid 49711 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:23 x4 kernel: pid 49714 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:24 x4 kernel: pid 49715 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:24 x4 kernel: pid 49718 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:25 x4 kernel: pid 49716 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:27 x4 kernel: pid 49702 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:27 x4 kernel: pid 49704 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:27 x4 kernel: pid 49710 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:27 x4 kernel: pid 49713 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:29 x4 kernel: pid 49708 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:30 x4 kernel: pid 49706 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:40:30 x4 kernel: pid 49712 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:43:41 x4 kernel: pid 46420 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:43:44 x4 kernel: pid 46428 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:43:45 x4 kernel: pid 46432 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:54:44 x4 kernel: pid 71294 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:54:55 x4 kernel: pid 71311 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:55:08 x4 kernel: pid 71313 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:55:13 x4 kernel: pid 71295 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:57:21 x4 kernel: pid 71120 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:57:39 x4 kernel: pid 71138 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:57:44 x4 kernel: pid 71121 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:57:44 x4 kernel: pid 71139 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 23 23:57:46 x4 kernel: pid 71140 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:15:34 x4 kernel: pid 79497 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:15:36 x4 kernel: pid 79504 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:15:36 x4 kernel: pid 79506 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:15:39 x4 kernel: pid 79500 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:15:39 x4 kernel: pid 79501 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:15:39 x4 kernel: pid 79537 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:15:45 x4 kernel: pid 79499 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:16:38 x4 kernel: pid 79295 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:16:41 x4 kernel: pid 79299 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:16:49 x4 kernel: pid 79293 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:16:49 x4 kernel: pid 79297 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:17:01 x4 kernel: pid 79301 (mkdir), uid 0, was killed: exceeded maximum CPU limit Expensive timeout(9) function: 0xc08ac5b0(0xc6d77240) 0.346903062 s Jun 24 00:29:49 x4 kernel: pid 86542 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:37:04 x4 kernel: pid 89100 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:37:07 x4 kernel: pid 89102 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:37:14 x4 kernel: pid 89101 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:38:56 x4 kernel: pid 88550 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:38:56 x4 kernel: pid 88556 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 00:38:56 x4 kernel: pid 88559 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:02:58 x4 kernel: pid 99916 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:02:59 x4 kernel: pid 99911 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:10 x4 kernel: pid 99906 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:17 x4 kernel: pid 99918 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:18 x4 kernel: pid 99905 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:18 x4 kernel: pid 99912 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:19 x4 kernel: pid 99909 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:19 x4 kernel: pid 99910 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:19 x4 kernel: pid 99914 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:20 x4 kernel: pid 99899 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:20 x4 kernel: pid 99901 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:20 x4 kernel: pid 99903 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:20 x4 kernel: pid 99904 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:20 x4 kernel: pid 99908 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:20 x4 kernel: pid 99917 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:21 x4 kernel: pid 99907 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:21 x4 kernel: pid 99913 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:03:21 x4 kernel: pid 99915 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:20:07 x4 kernel: pid 35676 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:22:57 x4 kernel: pid 35356 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:23:03 x4 kernel: pid 35355 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:23:05 x4 kernel: pid 35353 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:23:06 x4 kernel: pid 35354 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:23:07 x4 kernel: pid 35350 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:23:07 x4 kernel: pid 35352 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:23:08 x4 kernel: pid 35349 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:46:05 x4 kernel: pid 44916 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:46:12 x4 kernel: pid 44923 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:46:29 x4 kernel: pid 44934 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:47:04 x4 kernel: pid 44915 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:47:08 x4 kernel: pid 44920 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:47:08 x4 kernel: pid 44914 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:47:09 x4 kernel: pid 44912 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:48:40 x4 kernel: pid 45967 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:49:03 x4 kernel: pid 45990 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:49:11 x4 kernel: pid 45968 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:37 x4 kernel: pid 45699 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:38 x4 kernel: pid 45696 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:38 x4 kernel: pid 45711 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:38 x4 kernel: pid 45712 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:39 x4 kernel: pid 45698 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:41 x4 kernel: pid 45703 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:42 x4 kernel: pid 45705 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:43 x4 kernel: pid 45701 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:43 x4 kernel: pid 45704 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:44 x4 kernel: pid 45697 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:44 x4 kernel: pid 45700 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:46 x4 kernel: pid 45702 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:46 x4 kernel: pid 45710 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:46 x4 kernel: pid 45713 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:48 x4 kernel: pid 45706 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:48 x4 kernel: pid 45708 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 01:59:50 x4 kernel: pid 45707 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:03:15 x4 kernel: pid 45988 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:03:17 x4 kernel: pid 45970 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:03:19 x4 kernel: pid 45989 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:03:20 x4 kernel: pid 45991 (lockf), uid 0, was killed: exceeded maximum CPU limit Expensive timeout(9) function: 0xc08d0560(0) 0.542852042 s Jun 24 02:33:47 x4 kernel: pid 69655 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:33:48 x4 kernel: pid 70277 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:33:55 x4 kernel: pid 70276 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:34:15 x4 kernel: pid 69662 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:34:15 x4 kernel: pid 70278 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:34:15 x4 kernel: pid 70279 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:34:23 x4 kernel: pid 69659 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:34:23 x4 kernel: pid 70295 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:34:32 x4 kernel: pid 70294 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:46 x4 kernel: pid 70088 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:46 x4 kernel: pid 70090 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:46 x4 kernel: pid 70096 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:48 x4 kernel: pid 70083 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:48 x4 kernel: pid 70098 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:48 x4 kernel: pid 70100 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:48 x4 kernel: pid 70102 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:49 x4 kernel: pid 70084 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:49 x4 kernel: pid 70106 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:50 x4 kernel: pid 70081 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:50 x4 kernel: pid 70086 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:50 x4 kernel: pid 70092 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:50 x4 kernel: pid 70108 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:51 x4 kernel: pid 70094 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:37:51 x4 kernel: pid 70104 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:54:58 x4 kernel: pid 87123 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:55:13 x4 kernel: pid 87116 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:55:13 x4 kernel: pid 86635 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:55:13 x4 kernel: pid 86637 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 02:55:19 x4 kernel: pid 86633 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:08:12 x4 kernel: pid 97705 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:08:15 x4 kernel: pid 97756 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:30:04 x4 kernel: pid 7446 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:30:07 x4 kernel: pid 7451 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:30:08 x4 kernel: pid 7449 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:30:11 x4 kernel: pid 7450 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:33:37 x4 kernel: pid 7098 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:33:37 x4 kernel: pid 7102 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:33:38 x4 kernel: pid 7094 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:33:39 x4 kernel: pid 7082 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:33:39 x4 kernel: pid 7090 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:33:41 x4 kernel: pid 7086 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:27 x4 kernel: pid 12343 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:29 x4 kernel: pid 12309 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:35 x4 kernel: pid 12342 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:37 x4 kernel: pid 12306 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:41 x4 kernel: pid 12345 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:52 x4 kernel: pid 12344 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:52 x4 kernel: pid 12307 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:57 x4 kernel: pid 12361 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:56:59 x4 kernel: pid 12211 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:03 x4 kernel: pid 12185 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:04 x4 kernel: pid 12200 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:04 x4 kernel: pid 12202 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:04 x4 kernel: pid 12213 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:05 x4 kernel: pid 12197 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:05 x4 kernel: pid 12201 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:05 x4 kernel: pid 12204 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:06 x4 kernel: pid 12195 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:06 x4 kernel: pid 12203 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 03:57:09 x4 kernel: pid 12209 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:09:55 x4 kernel: pid 20583 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:10:01 x4 kernel: pid 20575 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:10:01 x4 kernel: pid 20576 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:10:09 x4 kernel: pid 20590 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:12:49 x4 kernel: pid 20087 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:12:52 x4 kernel: pid 20040 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:12:53 x4 kernel: pid 20049 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:12:53 x4 kernel: pid 20055 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:12:54 x4 kernel: pid 20053 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:12:54 x4 kernel: pid 20089 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:12:55 x4 kernel: pid 20085 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:22:54 x4 kernel: pid 27392 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:22:54 x4 kernel: pid 27398 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:23:34 x4 kernel: pid 27395 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:24:57 x4 kernel: pid 27086 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:24:57 x4 kernel: pid 27087 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:24:58 x4 kernel: pid 27118 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:41:59 x4 kernel: pid 88669 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:30 x4 kernel: pid 88420 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:30 x4 kernel: pid 88426 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:32 x4 kernel: pid 88429 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:33 x4 kernel: pid 88403 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:34 x4 kernel: pid 88411 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:34 x4 kernel: pid 88417 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:34 x4 kernel: pid 88423 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:35 x4 kernel: pid 88404 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:35 x4 kernel: pid 88414 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:35 x4 kernel: pid 88432 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:49:35 x4 kernel: pid 88435 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 04:59:03 x4 kernel: pid 95890 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:06:33 x4 kernel: pid 10307 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:07:42 x4 kernel: pid 10306 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:09:14 x4 kernel: pid 10199 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:09:18 x4 kernel: pid 10208 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:09:21 x4 kernel: pid 10210 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:09:22 x4 kernel: pid 10194 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:09:28 x4 kernel: pid 10202 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:54 x4 kernel: pid 26817 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:55 x4 kernel: pid 26814 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:55 x4 kernel: pid 26819 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:56 x4 kernel: pid 26812 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:57 x4 kernel: pid 26811 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:57 x4 kernel: pid 26815 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:57 x4 kernel: pid 26816 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:57 x4 kernel: pid 26820 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:58 x4 kernel: pid 26813 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:26:58 x4 kernel: pid 26818 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 05:27:21 x4 kernel: pid 26810 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:15:17 x4 kernel: pid 31247 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:17:56 x4 kernel: pid 30860 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:17:57 x4 kernel: pid 30857 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:17:57 x4 kernel: pid 30858 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:17:58 x4 kernel: pid 30855 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:17:58 x4 kernel: pid 30859 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:17:59 x4 kernel: pid 30856 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:32:26 x4 kernel: pid 37455 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:32:27 x4 kernel: pid 37463 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:32:27 x4 kernel: pid 37466 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:32:27 x4 kernel: pid 37457 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:32:27 x4 kernel: pid 37460 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:32:27 x4 kernel: pid 37469 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:40:38 x4 kernel: pid 42590 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:40:39 x4 kernel: pid 42626 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:51:27 x4 kernel: pid 50080 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:51:44 x4 kernel: pid 50079 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:51:46 x4 kernel: pid 50078 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:52:11 x4 kernel: pid 50076 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:54:24 x4 kernel: pid 49776 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:54:25 x4 kernel: pid 49771 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:54:27 x4 kernel: pid 49779 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:54:29 x4 kernel: pid 49780 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 06:54:33 x4 kernel: pid 49781 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:36:49 x4 kernel: pid 75597 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:36:52 x4 kernel: pid 75601 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:36:53 x4 kernel: pid 75594 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:36:53 x4 kernel: pid 75585 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:36:54 x4 kernel: pid 75596 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:49:58 x4 kernel: pid 96052 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:50:07 x4 kernel: pid 96064 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:50:07 x4 kernel: pid 96056 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:50:08 x4 kernel: pid 96047 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:50:10 x4 kernel: pid 96077 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:50:11 x4 kernel: pid 96059 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 07:50:12 x4 kernel: pid 96078 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:01:14 x4 kernel: pid 12423 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:01:14 x4 kernel: pid 12425 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:01:14 x4 kernel: pid 12428 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:01:14 x4 kernel: pid 12431 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:01:14 x4 kernel: pid 12434 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:15:44 x4 kernel: pid 17577 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:57:44 x4 kernel: pid 27087 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:57:50 x4 kernel: pid 27096 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:57:51 x4 kernel: pid 27097 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:58:07 x4 kernel: pid 27114 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:58:10 x4 kernel: pid 27093 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:58:12 x4 kernel: pid 27116 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 08:58:25 x4 kernel: pid 27115 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:00:32 x4 kernel: pid 26920 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:10:22 x4 kernel: pid 34797 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:43 x4 kernel: pid 38209 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:44 x4 kernel: pid 38208 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:45 x4 kernel: pid 38206 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:46 x4 kernel: pid 38203 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:46 x4 kernel: pid 38210 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:46 x4 kernel: pid 38212 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:47 x4 kernel: pid 38207 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:47 x4 kernel: pid 38211 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:38:47 x4 kernel: pid 38213 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:08 x4 kernel: pid 38476 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:11 x4 kernel: pid 38479 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:26 x4 kernel: pid 38467 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:31 x4 kernel: pid 38482 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:38 x4 kernel: pid 38478 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:39 x4 kernel: pid 38463 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:39 x4 kernel: pid 38468 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:40 x4 kernel: pid 38473 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 09:39:47 x4 kernel: pid 38475 (lockf), uid 0, was killed: exceeded maximum CPU limit lock order reversal: 1st 0xc738652c filedesc structure (filedesc structure) @ kern/kern_descrip.c:1088 2nd 0xc77651b0 ufs (ufs) @ kern/vfs_subr.c:4092 KDB: stack backtrace: db_trace_self_wrapper(c0c5c38a,f631ba30,c08b5ee5,c08a6b5b,c0c5f29f,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08a6b5b,c0c5f29f,c6d2afc8,c6d2dd48,f631ba8c,...) at kdb_backtrace+0x29 _witness_debugger(c0c5f29f,c77651b0,c0c51f31,c6d2dd48,c0c663c3,...) at _witness_debugger+0x25 witness_checkorder(c77651b0,9,c0c663ba,ffc,c7765218,...) at witness_checkorder+0x839 __lockmgr_args(c77651b0,80400,c7765218,0,0,...) at __lockmgr_args+0x7b7 ffs_lock(f631bb98,c,0,80400,c7765158,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0d5e780,f631bb98,f631bbb4,c0d77240,c7765158,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c7765158,80400,c0c663ba,ffc,f631bbf4,...) at _vn_lock+0x78 vfs_knllock(c7765158,0,c0c5415e,68a,c7989198,...) at vfs_knllock+0x29 knlist_remove_kq(0,f631bc14,c08fee79,c79be01c,c7989198,...) at knlist_remove_kq+0x85 knlist_remove(c79be01c,c7989198,0,f631bc40,c0844ce5,...) at knlist_remove+0x1b filt_vfsdetach(c7989198,0,c0c5415e,76b,3,...) at filt_vfsdetach+0x39 knote_fdclose(c761f900,3,c0c53ca9,440,c738652c,...) at knote_fdclose+0xf5 kern_close(c761f900,3,f631bd2c,c0b97f74,c761f900,...) at kern_close+0xd2 close(c761f900,f631bcf8,4,f631bd38,c0d3dac8,...) at close+0x1a syscall(f631bd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (6, FreeBSD ELF32, close), eip = 0x28178d43, esp = 0xbfbfeb0c, ebp = 0xbfbfeb18 --- Jun 24 10:23:10 x4 kernel: pid 83405 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:23:15 x4 kernel: pid 83427 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:50 x4 kernel: pid 83088 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:50 x4 kernel: pid 83090 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:51 x4 kernel: pid 83091 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:51 x4 kernel: pid 83093 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:52 x4 kernel: pid 83087 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:53 x4 kernel: pid 83084 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:53 x4 kernel: pid 83086 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:53 x4 kernel: pid 83089 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:53 x4 kernel: pid 83094 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:53 x4 kernel: pid 83095 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:54 x4 kernel: pid 83092 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:55 x4 kernel: pid 83083 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:28:55 x4 kernel: pid 83085 (mkdir), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:48:13 x4 kernel: pid 7018 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:48:16 x4 kernel: pid 7021 (lockf), uid 0, was killed: exceeded maximum CPU limit Jun 24 10:48:46 x4 kernel: pid 7016 (lockf), uid 0, was killed: exceeded maximum CPU limit kernel trap 12 with interrupts disabled Fatal trap 12: page fault while in kernel mode cpuid = 3; apic id = 03 fault virtual address = 0x1a0 fault code = supervisor read, page not present instruction pointer = 0x20:0xc087ce67 stack pointer = 0x28:0xf6d01a10 frame pointer = 0x28:0xf6d01a48 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = resume, IOPL = 0 current process = 13718 (umount) [thread pid 13718 tid 100739 ] Stopped at _sx_xlock_hard+0x287: movl 0x1a0(%eax),%eax db:0:kdb.enter.unknown> run pho db:1:pho> bt Tracing pid 13718 tid 100739 td 0xc94b4000 _sx_xlock_hard(c7660a88,c94b4000,0,c0c56412,798,...) at _sx_xlock_hard+0x287 _sx_xlock(c7660a88,0,c0c56412,798,c7660a88,...) at _sx_xlock+0xc0 lf_iteratelocks_vnode(c991e6b8,c0a44b10,0,f6d01b14,f6d01ad8,...) at lf_iteratelocks_vnode+0x4d nlm_reclaim(f6d01b14,c0d5be00,f6d01acc,c0d5be00,f6d01b14,...) at nlm_reclaim+0x30 nfs_reclaim(f6d01b14,c0c94a13,c991e6b8,c991e6b8,c94b4000,...) at nfs_reclaim+0x4e VOP_RECLAIM_APV(c0d5be00,f6d01b14,c0c663ba,a0e,c991e778,...) at VOP_RECLAIM_APV+0xc5 vgonel(c991e778,0,c0c663ba,987,0,...) at vgonel+0x1c4 vflush(c9ac6870,1,2,c94b4000,cd1fd000,...) at vflush+0x337 nfs_unmount(c9ac6870,8080000,f6d01c44,4f2,80,...) at nfs_unmount+0x41 dounmount(c9ac6870,8080000,c94b4000,478,3,...) at dounmount+0x46d unmount(c94b4000,f6d01cf8,8,0,c0d3dc88,...) at unmount+0x2df syscall(f6d01d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d916f, esp = 0xbfbfe15c, ebp = 0xbfbfe228 --- db:1:bt> show allpcpu Current CPU: 3 cpuid = 0 curthread = 0xc8931480: pid 8739 "lockf" curpcb = 0xf69b6d90 fpcurthread = none idlethread = 0xc6d75480: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 curthread = 0xc8954000: pid 8741 "lockf" curpcb = 0xf6901d90 fpcurthread = none idlethread = 0xc6d756c0: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 curthread = 0xc7ee8000: pid 8748 "lockf" curpcb = 0xf6483d90 fpcurthread = none idlethread = 0xc6d75900: pid 11 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 curthread = 0xc94b4000: pid 13718 "umount" curpcb = 0xf6d01d90 fpcurthread = none idlethread = 0xc6d75b40: pid 11 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:1:allpcpu> show alllocks Process 13718 (umount) thread 0xc94b4000 (100739) exclusive lockmgr nfs (nfs) r = 0 (0xc991e710) locked @ kern/vfs_subr.c:2405 exclusive lockmgr ufs (ufs) r = 0 (0xcd1fd058) locked @ kern/vfs_mount.c:1197 Process 8748 (lockf) thread 0xc7ee8000 (100201) exclusive sleep mutex vnode interlock (vnode interlock) r = 0 (0xca9b8a28) locked @ kern/kern_lockf.c:727 Process 8744 (lockf) thread 0xc7fc9480 (100277) exclusive lockmgr bufwait (bufwait) r = 0 (0xe6e8e9d8) locked @ kern/vfs_bio.c:2558 shared lockmgr nfs (nfs) r = 0 (0xcae65b18) locked @ kern/vfs_vnops.c:533 Process 8741 (lockf) thread 0xc8954000 (100503) exclusive sleep mutex vnode interlock (vnode interlock) r = 0 (0xc87dc0c0) locked @ kern/kern_lockf.c:727 Process 8739 (lockf) thread 0xc8931480 (100546) exclusive sleep mutex vnode interlock (vnode interlock) r = 0 (0xcb013620) locked @ kern/kern_lockf.c:727 Process 1585 (sshd) thread 0xc7ffa6c0 (100324) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc7871560) locked @ kern/uipc_sockbuf.c:148 Process 1051 (sshd) thread 0xc748c6c0 (100101) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc782d08c) locked @ kern/uipc_sockbuf.c:148 Process 1024 (sshd) thread 0xc7434b40 (100124) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc784c6fc) locked @ kern/uipc_sockbuf.c:148 Process 1023 (sshd) thread 0xc761fd80 (100111) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc7641560) locked @ kern/uipc_sockbuf.c:148 Process 1022 (sshd) thread 0xc74346c0 (100127) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc78743c4) locked @ kern/uipc_sockbuf.c:148 db:1:alllocks> show lockedvnods Locked vnodes 0xcd1fd000: tag ufs, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0xc9ac6870 flags () v_object 0xcd88707c ref 0 pages 0 lock type ufs: EXCL by thread 0xc94b4000 (pid 13718) #0 0xc085e6f0 at __lockmgr_args+0xb90 #1 0xc0aa02a1 at ffs_lock+0xa1 #2 0xc0ba6295 at VOP_LOCK1_APV+0xb5 #3 0xc090f9e8 at _vn_lock+0x78 #4 0xc08fcc3c at dounmount+0x8c #5 0xc08fd58f at unmount+0x2df #6 0xc0b97f74 at syscall+0x2b4 #7 0xc0b7b1b0 at Xint0x80_syscall+0x20 ino 70657, on dev ad4s1a 0xc991e6b8: tag nfs, type VREG usecount 1, writecount 1, refcount 2 mountedhere 0 flags (VI_DOOMED) v_object 0xcd49907c ref 0 pages 0 lock type nfs: EXCL by thread 0xc94b4000 (pid 13718) with shared waiters pending #0 0xc085e6f0 at __lockmgr_args+0xb90 #1 0xc08f2845 at vop_stdlock+0x65 #2 0xc0ba6295 at VOP_LOCK1_APV+0xb5 #3 0xc090f9e8 at _vn_lock+0x78 #4 0xc0905ebc at vflush+0x13c #5 0xc0a27b41 at nfs_unmount+0x41 #6 0xc08fd01d at dounmount+0x46d #7 0xc08fd58f at unmount+0x2df #8 0xc0b97f74 at syscall+0x2b4 #9 0xc0b7b1b0 at Xint0x80_syscall+0x20 fileid -154135488 fsid 0xc0c7956d 0xcae65ac0: tag nfs, type VREG usecount 1, writecount 1, refcount 3 mountedhere 0 flags () v_object 0xcc36d2e8 ref 0 pages 1 lock type nfs: SHARED (count 1) #0 0xc085e0e2 at __lockmgr_args+0x582 #1 0xc08f2845 at vop_stdlock+0x65 #2 0xc0ba6295 at VOP_LOCK1_APV+0xb5 #3 0xc090f9e8 at _vn_lock+0x78 #4 0xc091100b at vn_read+0x16b #5 0xc08b8726 at dofileread+0x96 #6 0xc08b8a58 at kern_readv+0x58 #7 0xc08b8b6f at read+0x4f #8 0xc0b97f74 at syscall+0x2b4 #9 0xc0b7b1b0 at Xint0x80_syscall+0x20 fileid -154135488 fsid 0xc0c7956d db:1:lockedvnods> show mount 0xc739bb40 /dev/ad4s1a on / (ufs) 0xc739c000 devfs on /dev (devfs) 0xc75f22d0 /dev/ad4s1f on /usr (ufs) 0xc75f2000 /dev/ad4s1d on /var (ufs) 0xc739cb40 procfs on /proc (procfs) 0xc7b15000 /dev/ad4s1e on /tmp (ufs) 0xc9ac6870 127.0.0.1:/tmp on /mnt (nfs) More info: show mount db:1:mount> ps pid ppid pgrp uid state wmesg wchan cmd 13718 6806 37933 0 R+ CPU 3 umount 13715 1034 1031 1001 S nanslp 0xc0daabe4 sleep 8748 1 37933 0 R+ CPU 2 lockf 8747 1 37933 0 R+ lockf 8746 1 37933 0 R+ lockf 8744 1 37933 0 R+ lockf 8742 1 37933 0 R+ lockf 8741 1 37933 0 R+ CPU 1 lockf 8740 1 37933 0 R+ lockf 8739 1 37933 0 R+ CPU 0 lockf 8736 1 37933 0 S+ nfs 0xc991e710 lockf 8731 1 37933 0 R+ lockf 8695 1 37933 0 R+ lockf 6806 37933 37933 0 S+ wait 0xc84357f8 sh 37933 1078 37933 0 S+ wait 0xca858aa0 sh 1653 1650 1653 1001 Ss+ ttyin 0xc8405870 bash 1650 1585 1585 1001 S select 0xc80e8764 sshd 1585 993 1585 0 Ss sbwait 0xc787158c sshd 1213 1 1213 0 Ss+ ttyin 0xc70da470 getty 1212 1 1212 0 Ss+ ttyin 0xc70da670 getty 1211 1 1211 0 Ss+ ttyin 0xc70da870 getty 1210 1 1210 0 Ss+ ttyin 0xc70daa70 getty 1209 1 1209 0 Ss+ ttyin 0xc6ead870 getty 1208 1 1208 0 Ss+ ttyin 0xc6eadc70 getty 1207 1 1207 0 Ss+ ttyin 0xc6fa7070 getty 1206 1 1206 0 Ss+ ttyin 0xc6fa7270 getty 1205 1 1205 0 Ss+ ttyin 0xc6fa7470 getty 1179 1 1179 0 Ss select 0xc74339a4 inetd 1150 1 1150 0 Ss nanslp 0xc0daabe4 watchdogd 1078 1059 1078 0 S+ wait 0xc79c4d48 bash 1059 1058 1059 0 S+ pause 0xc782c5a8 csh 1058 1054 1058 1001 S+ wait 0xc75f1d48 su 1054 1053 1054 1001 Ss+ wait 0xc782caa0 bash 1053 1051 1051 1001 S select 0xc7839aa4 sshd 1051 993 1051 0 Ss sbwait 0xc782d0b8 sshd 1035 1031 1031 1001 S piperd 0xc74b5498 awk 1034 1031 1031 1001 S wait 0xc75ef2a8 sh 1033 1028 1033 1001 Ss+ select 0xc783a0a4 top 1032 1029 1032 1001 Ss kqread 0xc73a6c80 tail 1031 1030 1031 1001 Ss wait 0xc75efaa0 sh 1030 1023 1023 1001 S select 0xc7362024 sshd 1029 1024 1024 1001 S select 0xc73630e4 sshd 1028 1022 1022 1001 S select 0xc783a8e4 sshd 1024 993 1024 0 Ss sbwait 0xc784c728 sshd 1023 993 1023 0 Ss sbwait 0xc764158c sshd 1022 993 1022 0 Ss sbwait 0xc78743f0 sshd 1010 1 1010 0 Ss nanslp 0xc0daabe4 cron 1004 1 1004 25 Ss pause 0xc75ef850 sendmail 1000 1 1000 0 Ss select 0xc74329e4 sendmail 993 1 993 0 Ss select 0xc7362ca4 sshd 956 1 956 0 Ss select 0xc7325e64 ntpd 834 1 834 0 Ss rpcsvc 0xc739f650 NLM: master 828 1 828 0 Ss select 0xc741a1a4 rpc.statd 822 821 821 0 S (threaded) nfsd 100143 S rpcsvc 0xc7433010 nfsd: service 100142 S rpcsvc 0xc7433050 nfsd: service 100141 S rpcsvc 0xc7433090 nfsd: service 100136 S rpcsvc 0xc7839a50 nfsd: master 821 1 821 0 Ss select 0xc6fc57a4 nfsd 813 1 813 0 Ss select 0xc7839be4 mountd 734 1 734 0 Ss select 0xc74194a4 rpcbind 705 1 705 0 Ss select 0xc7419364 syslogd 573 1 573 0 Ss select 0xc7362064 devd 43 0 0 0 SL flowclea 0xc0daaa44 [flowcleaner] 42 0 0 0 SL sdflush 0xc0f2aaa0 [softdepflush] 41 0 0 0 SL syncer 0xc0f172f4 [syncer] 40 0 0 0 SL vlruwt 0xc73137f8 [vnlru] 39 0 0 0 SL psleep 0xc0f17028 [bufdaemon] 38 0 0 0 SL pgzero 0xc0f2b6d4 [pagezero] 9 0 0 0 SL psleep 0xc0f2b2fc [vmdaemon] 8 0 0 0 SL psleep 0xc0f2b2c4 [pagedaemon] 37 0 0 0 SL wmsg 0xc6fabd0c [usbus5] 36 0 0 0 SL wmsg 0xc6fabcdc [usbus5] 35 0 0 0 SL wmsg 0xc6fabcac [usbus5] 34 0 0 0 SL wmsg 0xc6fabc7c [usbus5] 33 0 0 0 SL wmsg 0xc6fa1b5c [usbus4] 32 0 0 0 SL wmsg 0xc6fa1b2c [usbus4] 31 0 0 0 SL wmsg 0xc6fa1afc [usbus4] 30 0 0 0 SL wmsg 0xc6fa1acc [usbus4] 29 0 0 0 SL wmsg 0xc6f9eb5c [usbus3] 28 0 0 0 SL wmsg 0xc6f9eb2c [usbus3] 27 0 0 0 SL wmsg 0xc6f9eafc [usbus3] 26 0 0 0 SL wmsg 0xc6f9eacc [usbus3] 25 0 0 0 SL wmsg 0xc6f96b5c [usbus2] 24 0 0 0 SL wmsg 0xc6f96b2c [usbus2] 23 0 0 0 SL wmsg 0xc6f96afc [usbus2] 22 0 0 0 SL wmsg 0xc6f96acc [usbus2] 21 0 0 0 SL wmsg 0xc6f92b5c [usbus1] 20 0 0 0 SL wmsg 0xc6f92b2c [usbus1] 19 0 0 0 SL wmsg 0xc6f92afc [usbus1] 18 0 0 0 SL wmsg 0xc6f92acc [usbus1] 17 0 0 0 SL wmsg 0xc6f8bb5c [usbus0] 16 0 0 0 SL wmsg 0xc6f8bb2c [usbus0] 15 0 0 0 SL wmsg 0xc6f8bafc [usbus0] 14 0 0 0 SL wmsg 0xc6f8bacc [usbus0] 7 0 0 0 SL waiting_ 0xc0f2119c [sctp_iterator] 6 0 0 0 SL - 0xc6fc9000 [fw0_probe] 5 0 0 0 SL ccb_scan 0xc0d779d4 [xpt_thrd] 13 0 0 0 SL - 0xc0daaa44 [yarrow] 4 0 0 0 SL - 0xc0da87e4 [g_down] 3 0 0 0 SL - 0xc0da87e0 [g_up] 2 0 0 0 SL - 0xc0da87d8 [g_event] 12 0 0 0 WL (threaded) intr 100040 I [irq12: psm0] 100039 I [irq1: atkbd0] 100038 I [swi0: uart] 100035 I [irq20: fwohci0] 100034 I [irq14: ata0] 100033 I [irq19: ehci0] 100032 I [irq18: ohci2 ohci4] 100031 I [irq17: ohci1 ohci3] 100030 I [irq16: hdac1 ohci0] 100029 I [irq22: atapci0] 100028 I [irq256: hdac0] 100026 I [swi5: +] 100025 I [swi2: cambio] 100019 I [swi6: task queue] 100018 I [swi6: Giant taskq] 100012 I [swi1: netisr 0] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi4: clock] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) idle 100006 CanRun [idle: cpu0] 100005 CanRun [idle: cpu1] 100004 CanRun [idle: cpu2] 100003 CanRun [idle: cpu3] 1 0 1 0 SLs wait 0xc6d73d48 [init] 10 0 0 0 SL audit_wo 0xc0f2a3c0 [audit] 0 0 0 0 SLs (threaded) kernel 100036 D - 0xc6fc6340 [fw0_taskq] 100027 D - 0xc6f25dc0 [thread taskq] 100023 D - 0xc6f260c0 [acpi_task_2] 100022 D - 0xc6f260c0 [acpi_task_1] 100021 D - 0xc6f260c0 [acpi_task_0] 100020 D - 0xc6f26100 [kqueue taskq] 100016 D - 0xc6d5bb80 [firmware taskq] 100000 D sched 0xc0da88a0 [swapper] db:1:ps> allt Tracing command umount pid 13718 tid 100739 td 0xc94b4000 _sx_xlock_hard(c7660a88,c94b4000,0,c0c56412,798,...) at _sx_xlock_hard+0x287 _sx_xlock(c7660a88,0,c0c56412,798,c7660a88,...) at _sx_xlock+0xc0 lf_iteratelocks_vnode(c991e6b8,c0a44b10,0,f6d01b14,f6d01ad8,...) at lf_iteratelocks_vnode+0x4d nlm_reclaim(f6d01b14,c0d5be00,f6d01acc,c0d5be00,f6d01b14,...) at nlm_reclaim+0x30 nfs_reclaim(f6d01b14,c0c94a13,c991e6b8,c991e6b8,c94b4000,...) at nfs_reclaim+0x4e VOP_RECLAIM_APV(c0d5be00,f6d01b14,c0c663ba,a0e,c991e778,...) at VOP_RECLAIM_APV+0xc5 vgonel(c991e778,0,c0c663ba,987,0,...) at vgonel+0x1c4 vflush(c9ac6870,1,2,c94b4000,cd1fd000,...) at vflush+0x337 nfs_unmount(c9ac6870,8080000,f6d01c44,4f2,80,...) at nfs_unmount+0x41 dounmount(c9ac6870,8080000,c94b4000,478,3,...) at dounmount+0x46d unmount(c94b4000,f6d01cf8,8,0,c0d3dc88,...) at unmount+0x2df syscall(f6d01d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d916f, esp = 0xbfbfe15c, ebp = 0xbfbfe228 --- Tracing command sleep pid 13715 tid 100648 td 0xca5e9900 sched_switch(ca5e9900,0,104,18c,5e91730c,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(ca5e9900,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(2711,c08ac5b0,ca5e9900,0,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0daabe4,5c,c0c5a2a4,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0daabe4,0,15c,c0c5a2a4,2711,...) at _sleep+0x30e kern_nanosleep(ca5e9900,f6bf0c64,f6bf0c6c,a,0,...) at kern_nanosleep+0xc1 nanosleep(ca5e9900,f6bf0cf8,8,f6bf0d38,c0d3f460,...) at nanosleep+0x6f syscall(f6bf0d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815687f, esp = 0xbfbfecbc, ebp = 0xbfbfecf8 --- Tracing command lockf pid 8748 tid 100201 td 0xc7ee8000 cpustop_handler(4,f64839dc,c0b9827d,f648396c,c0865894,...) at cpustop_handler+0x32 ipi_nmi_handler(f648396c,c0865894,c0daac90,4,c7b0a000,...) at ipi_nmi_handler+0x2f trap(f64839e8) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b9c0a9, esp = 0xf6483a28, ebp = 0xf6483a44 --- DELAY(1,c7ee8000,c0dbc5d4,0,f6483a84,...) at DELAY+0x89 _mtx_lock_spin(c0dbc5d4,c7ee8000,0,c0c5d3f5,ec,...) at _mtx_lock_spin+0x6c _mtx_lock_spin_flags(c0dbc5d4,0,c0c5d3f5,ec,f6483abc,...) at _mtx_lock_spin_flags+0x146 sleepq_lock(c8118a40,2d7,0,c7449640,c7449640,...) at sleepq_lock+0x39 wakeup(c8118a40,0,c0c56412,2d7,ca9b89c0,...) at wakeup+0x17 lf_advlockasync(f6483b3c,ca9b8abc,4,0,c0d4ec20,...) at lf_advlockasync+0xdf4 lf_advlock(f6483c00,ca9b8abc,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f6483c00,c0c9472f,ca9b8968,ca9b8968,c7b0d000,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d5be00,f6483c00,c0c53ca9,259,c08b5c8b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(c7ee8000,3,d,f6483c40,0,...) at kern_fcntl+0x90f fcntl(c7ee8000,f6483cf8,c,c0c5fcae,c0d3e430,...) at fcntl+0x101 syscall(f6483d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 8747 tid 100559 td 0xc91a3b40 sched_switch(c91a3b40,0,103,187,ce2337aa,...) at sched_switch+0x437 mi_switch(103,0,c0c5dc9d,2e1,c91add00,...) at mi_switch+0x200 turnstile_wait(c91add00,ca8b76c0,0,1ac,c1894808,...) at turnstile_wait+0x48a _mtx_lock_sleep(c1894808,c91a3b40,0,c0c80f6f,7c8,...) at _mtx_lock_sleep+0x14d _mtx_lock_flags(c1894808,0,c0c80f6f,7c8,c188c700,...) at _mtx_lock_flags+0xf7 uma_zalloc_arg(c188ca80,0,102,2,0,...) at uma_zalloc_arg+0xd3 malloc(38,c0d43c38,102,c887a040,c887a040,...) at malloc+0xe6 lf_alloc_lock(c0da92d8,c0c56412,21c,1ea,cac17710,...) at lf_alloc_lock+0x2a lf_advlockasync(f69ddb3c,cac1780c,4,0,c0d4ec20,...) at lf_advlockasync+0x37f lf_advlock(f69ddc00,cac1780c,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f69ddc00,c0c9472f,cac176b8,cac176b8,cafade00,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d5be00,f69ddc00,c0c53ca9,259,2,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(c91a3b40,3,d,f69ddc40,0,...) at kern_fcntl+0x90f fcntl(c91a3b40,f69ddcf8,c,c0c5fcae,c0d3e430,...) at fcntl+0x101 syscall(f69ddd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 8746 tid 100194 td 0xc7efa480 sched_switch(c7efa480,0,103,187,ce2327cc,...) at sched_switch+0x437 mi_switch(103,0,c0c5dc9d,2e1,c91add00,...) at mi_switch+0x200 turnstile_wait(c91add00,ca8b76c0,0,1ac,c1894808,...) at turnstile_wait+0x48a _mtx_lock_sleep(c1894808,c7efa480,0,c0c80f6f,7c8,...) at _mtx_lock_sleep+0x14d _mtx_lock_flags(c1894808,0,c0c80f6f,7c8,f6467a94,...) at _mtx_lock_flags+0xf7 uma_zalloc_arg(c188ca80,0,102,2,0,...) at uma_zalloc_arg+0xd3 malloc(38,c0d43c38,102,c90e4a80,c90e4a80,...) at malloc+0xe6 lf_alloc_lock(c0da92d8,c0c56412,21c,1ea,c9b67710,...) at lf_alloc_lock+0x2a lf_advlockasync(f6467b3c,c9b6780c,4,0,c0d4ec20,...) at lf_advlockasync+0x37f lf_advlock(f6467c00,c9b6780c,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f6467c00,c0c9472f,c8328e2c,c9b676b8,c8328e00,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d5be00,f6467c00,243,196,c08b5c8b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(c7efa480,3,c,f6467c40,0,...) at kern_fcntl+0x952 fcntl(c7efa480,f6467cf8,c,c0c5facf,c0d3e430,...) at fcntl+0x101 syscall(f6467d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 8744 tid 100277 td 0xc7fc9480 sched_switch(c7fc9480,0,103,187,cc34457f,...) at sched_switch+0x437 mi_switch(103,0,c0c5dc9d,2e1,c6f58e00,...) at mi_switch+0x200 turnstile_wait(c6f58e00,c9dfad80,0,1ac,c0f1705c,...) at turnstile_wait+0x48a _mtx_lock_sleep(c0f1705c,c7fc9480,0,c0c63fd5,188,...) at _mtx_lock_sleep+0x14d _mtx_lock_flags(c0f1705c,0,c0c63fd5,188,4,...) at _mtx_lock_flags+0xf7 brelse(e6e8e978,4,f6591c58,0,0,...) at brelse+0x8b6 nfs_bioread(cae65ac0,f6591c58,7f0000,c73a8b80,f6591b98,...) at nfs_bioread+0x1057 nfs_read(f6591bc8,c0c9445d,c7fc9480,0,c7b5c1c0,...) at nfs_read+0x44 VOP_READ_APV(c0d5be00,f6591bc8,cae65ac0,215,c08412ae,...) at VOP_READ_APV+0xc5 vn_read(c7b5c1c0,f6591c58,c73a8b80,0,c7fc9480,...) at vn_read+0x208 dofileread(f6591c58,ffffffff,ffffffff,0,c7b5c1c0,...) at dofileread+0x96 kern_readv(c7fc9480,3,f6591c58,f6591c78,1,...) at kern_readv+0x58 read(c7fc9480,f6591cf8,c,c0c5fcae,c0d3da74,...) at read+0x4f syscall(f6591d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x28188d83, esp = 0xbfbfe69c, ebp = 0xbfbfe6c8 --- Tracing command lockf pid 8742 tid 100465 td 0xca8b76c0 sched_switch(ca8b76c0,0,60b,187,ce249a76,...) at sched_switch+0x437 mi_switch(60b,0,c0c5acc5,813,3,...) at mi_switch+0x200 sched_preempt(ca8b76c0,3,c0c515d1,f684ba20,c0b7b83e,...) at sched_preempt+0xb5 ipi_bitmap_handler(8,28,28,2e5,c0c56414,...) at ipi_bitmap_handler+0x34 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0913200, esp = 0xf684ba18, ebp = 0xf684ba20 --- strncmp(c0c56412,c0c515ce,3,ca8b76c0,f684ba94,...) at strncmp+0x30 fixup_filename(2e2,f684ba5c,c08b5c8b,c0c5641b,c0c515ce,...) at fixup_filename+0x2b witness_checkorder(c0da92c0,9,c0c56412,2e5,0,...) at witness_checkorder+0x68 _sx_xlock(c0da92c0,0,c0c56412,2e5,c8191dc8,...) at _sx_xlock+0x85 lf_advlockasync(f684bb3c,c8191ec4,4,0,c0d4ec20,...) at lf_advlockasync+0xe78 lf_advlock(f684bc00,c8191ec4,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f684bc00,c0c9472f,c808952c,c8191d70,c8089500,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d5be00,f684bc00,243,196,c08b5c8b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(ca8b76c0,3,c,f684bc40,0,...) at kern_fcntl+0x952 fcntl(ca8b76c0,f684bcf8,c,c0c5facf,c0d3e430,...) at fcntl+0x101 syscall(f684bd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 8741 tid 100503 td 0xc8954000 cpustop_handler(2,f6901a00,c0b9827d,c70a2200,8c,...) at cpustop_handler+0x32 ipi_nmi_handler(c70a2200,8c,c6d70e00,f69019ac,cc75c550,...) at ipi_nmi_handler+0x2f trap(f6901a0c) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0865d97, esp = 0xf6901a4c, ebp = 0xf6901a5c --- _mtx_lock_spin(c0dbc5d4,c8954000,0,c0c5d3f5,ec,...) at _mtx_lock_spin+0x77 _mtx_lock_spin_flags(c0dbc5d4,0,c0c5d3f5,ec,f6901abc,...) at _mtx_lock_spin_flags+0x146 sleepq_lock(c7660a40,2d7,0,cb6d60c0,cb6d60c0,...) at sleepq_lock+0x39 wakeup(c7660a40,0,c0c56412,2d7,c87dc058,...) at wakeup+0x17 lf_advlockasync(f6901b3c,c87dc154,4,0,c0d4ec20,...) at lf_advlockasync+0xdf4 lf_advlock(f6901c00,c87dc154,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f6901c00,c0c9472f,c87dc000,c87dc000,cc5e8100,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d5be00,f6901c00,c0c53ca9,259,c08b5c8b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(c8954000,3,d,f6901c40,0,...) at kern_fcntl+0x90f fcntl(c8954000,f6901cf8,c,c0c5fcae,c0d3e430,...) at fcntl+0x101 syscall(f6901d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 8740 tid 100755 td 0xca2a4480 sched_switch(ca2a4480,0,207,187,cdfda026,...) at sched_switch+0x437 mi_switch(207,0,c0c5dbd7,d4,c946cd48,...) at mi_switch+0x200 ast(f6d31d38) at ast+0x2b7 doreti_ast() at doreti_ast+0x17 Tracing command lockf pid 8739 tid 100546 td 0xc8931480 cpustop_handler(1,f69b69dc,c0b9827d,c0881da8,68e5e16b,...) at cpustop_handler+0x32 ipi_nmi_handler(c0881da8,68e5e16b,7ffec189,1234de,cc75cd48,...) at ipi_nmi_handler+0x2f trap(f69b69e8) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b9c0a7, esp = 0xf69b6a28, ebp = 0xf69b6a44 --- DELAY(1,c8931480,c0dbc5d4,0,f69b6a84,...) at DELAY+0x87 _mtx_lock_spin(c0dbc5d4,c8931480,0,c0c5d3f5,ec,...) at _mtx_lock_spin+0x6c _mtx_lock_spin_flags(c0dbc5d4,0,c0c5d3f5,ec,f69b6abc,...) at _mtx_lock_spin_flags+0x146 sleepq_lock(c8118a80,2d7,0,c9824b40,c9824b40,...) at sleepq_lock+0x39 wakeup(c8118a80,0,c0c56412,2d7,cb0135b8,...) at wakeup+0x17 lf_advlockasync(f69b6b3c,cb0136b4,4,0,c0d4ec20,...) at lf_advlockasync+0xdf4 lf_advlock(f69b6c00,cb0136b4,4,0,4,...) at lf_advlock+0x55 nfs_advlock(f69b6c00,c0c9472f,cb013560,cb013560,c7385600,...) at nfs_advlock+0x93 VOP_ADVLOCK_APV(c0d5be00,f69b6c00,c0c53ca9,259,c08b5c8b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(c8931480,3,d,f69b6c40,0,...) at kern_fcntl+0x90f fcntl(c8931480,f69b6cf8,c,c0c5fcae,c0d3e430,...) at fcntl+0x101 syscall(f69b6d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 8736 tid 100678 td 0xc7957d80 sched_switch(c7957d80,0,104,18c,ce24da31,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,50,...) at mi_switch+0x200 sleepq_switch(c7957d80,0,c0c5d3f5,247,c991e710,...) at sleepq_switch+0x15f sleepq_wait(c991e710,50,c0c79dfd,4,1,...) at sleepq_wait+0x63 __lockmgr_args(c991e710,200000,c991e778,0,0,...) at __lockmgr_args+0x4a5 vop_stdlock(f6c4ab3c,c0c53cb2,c0c515ce,200000,c991e6b8,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0d5be00,f6c4ab3c,c08b5c8b,c0d77240,c991e6b8,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c991e6b8,200000,c0c79c93,c9b,f6c4ab90,...) at _vn_lock+0x78 nfs_advlock(f6c4ac00,c0c9472f,c991e6b8,c991e6b8,c7e4bd00,...) at nfs_advlock+0x2e VOP_ADVLOCK_APV(c0d5be00,f6c4ac00,c0c53ca9,259,c08b5c8b,...) at VOP_ADVLOCK_APV+0xd6 kern_fcntl(c7957d80,3,d,f6c4ac40,0,...) at kern_fcntl+0x90f fcntl(c7957d80,f6c4acf8,c,c0c5fcae,c0d3e430,...) at fcntl+0x101 syscall(f6c4ad38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (92, FreeBSD ELF32, fcntl), eip = 0x2817583f, esp = 0xbfbfe65c, ebp = 0xbfbfe698 --- Tracing command lockf pid 8731 tid 100826 td 0xc9dfad80 sched_switch(c9dfad80,0,207,187,ce186bb4,...) at sched_switch+0x437 mi_switch(207,0,c0c5dbd7,d4,c9477aa0,...) at mi_switch+0x200 ast(f6e72d38) at ast+0x2b7 doreti_ast() at doreti_ast+0x17 Tracing command lockf pid 8695 tid 100481 td 0xc7409240 sched_switch(c7409240,0,207,187,ced6e69f,...) at sched_switch+0x437 mi_switch(207,0,c0c5dbd7,d4,c84abaa0,...) at mi_switch+0x200 ast(f687bd38) at ast+0x2b7 doreti_ast() at doreti_ast+0x17 Tracing command sh pid 6806 tid 100419 td 0xc92ebb40 sched_switch(c92ebb40,0,104,18c,b2697023,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c92ebb40,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c84357f8,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(c84357f8,c8435880,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c92ebb40,ffffffff,f6784c74,2,0,...) at kern_wait+0xb56 wait4(c92ebb40,f6784cf8,10,c0c5fc41,c0d3dae4,...) at wait4+0x3b syscall(f6784d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfe83c, ebp = 0xbfbfe858 --- Tracing command sh pid 37933 tid 100469 td 0xc88f1d80 sched_switch(c88f1d80,0,104,18c,7a5220ce,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c88f1d80,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(ca858aa0,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(ca858aa0,ca858b28,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c88f1d80,ffffffff,f6857c74,2,0,...) at kern_wait+0xb56 wait4(c88f1d80,f6857cf8,10,c0c5fc41,c0d3dae4,...) at wait4+0x3b syscall(f6857d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfe6fc, ebp = 0xbfbfe718 --- Tracing command bash pid 1653 tid 100366 td 0xc92f0480 Tracing command sshd pid 1650 tid 100261 td 0xc7fcc900 Tracing command sshd pid 1585 tid 100324 td 0xc7ffa6c0 Tracing command getty pid 1213 tid 100153 td 0xc79c76c0 Tracing command getty pid 1212 tid 100147 td 0xc73ee900 Tracing command getty pid 1211 tid 100180 td 0xc7a38b40 Tracing command getty pid 1210 tid 100179 td 0xc7a38d80 Tracing command getty pid 1209 tid 100178 td 0xc7a39000 Tracing command getty pid 1208 tid 100177 td 0xc7a39240 Tracing command getty pid 1207 tid 100176 td 0xc7a39480 Tracing command getty pid 1206 tid 100175 td 0xc7a396c0 Tracing command getty pid 1205 tid 100174 td 0xc7a39900 sched_switch(c7a39900,0,104,18c,1333739e,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7a39900,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0865c2a,c6fa7404,0,c0c578b6,c7a39900,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6fa7470,0,f6406b0c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c6fa7470,c6fa7404,c0c6133c,4d4,0,...) at _cv_wait_sig+0x240 tty_wait(c6fa7400,c6fa7470,f6406c58,1,0,...) at tty_wait+0x71 ttydisc_read(c6fa7400,f6406c58,0,9e,0,...) at ttydisc_read+0x22c ttydev_read(c6e7b400,f6406c58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c740db28,f6406c58,c6d71100,0,c7a39900,...) at devfs_read_f+0x7e dofileread(f6406c58,ffffffff,ffffffff,0,c740db28,...) at dofileread+0x96 kern_readv(c7a39900,0,f6406c58,f6406c78,1,...) at kern_readv+0x58 read(c7a39900,f6406cf8,c,c0c7009c,c0d3da74,...) at read+0x4f syscall(f6406d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x28188d83, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command inetd pid 1179 tid 100165 td 0xc7620d80 Tracing command watchdogd pid 1150 tid 100121 td 0xc761e900 sched_switch(c761e900,0,104,18c,784803ea,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c761e900,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c08ac5b0,c761e900,0,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0daabe4,5c,c0c5a2a4,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0daabe4,0,15c,c0c5a2a4,ea61,...) at _sleep+0x30e kern_nanosleep(c761e900,f6330c64,f6330c6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c761e900,f6330cf8,8,c0c7009c,c0d3f460,...) at nanosleep+0x6f syscall(f6330d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2818087f, esp = 0xbfbfecec, ebp = 0xbfbfed18 --- Tracing command bash pid 1078 tid 100154 td 0xc79c7480 sched_switch(c79c7480,0,104,18c,37975ff2,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c79c7480,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c79c4d48,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(c79c4d48,c79c4dd0,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c79c7480,ffffffff,f63cac74,6,0,...) at kern_wait+0xb56 wait4(c79c7480,f63cacf8,10,c0c5fb51,c0d3dae4,...) at wait4+0x3b syscall(f63cad38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282bae7b, esp = 0xbfbfe7cc, ebp = 0xbfbfe7e8 --- Tracing command csh pid 1059 tid 100163 td 0xc79c6000 sched_switch(c79c6000,0,104,18c,c846c0d1,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,68,...) at mi_switch+0x200 sleepq_switch(c79c6000,0,c0c5d3f5,189,68,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c782c5a8,68,c0c0e4cb,100,0,...) at sleepq_wait_sig+0x17 _sleep(c782c5a8,c782c5d8,168,c0c0e4cb,0,...) at _sleep+0x344 kern_sigsuspend(c79c6000,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c79c6000,f63e5cf8,4,c0c5fb51,c0d3ff6c,...) at sigsuspend+0x4d syscall(f63e5d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x28178eeb, esp = 0xbfbfe5ec, ebp = 0xbfbfe658 --- Tracing command su pid 1058 tid 100138 td 0xc7831480 sched_switch(c7831480,0,104,18c,3c6866c4,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c7831480,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c75f1d48,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(c75f1d48,c75f1dd0,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c7831480,423,f638ec74,2,0,...) at kern_wait+0xb56 wait4(c7831480,f638ecf8,10,f638ed38,c0d3dae4,...) at wait4+0x3b syscall(f638ed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2811be7b, esp = 0xbfbfe62c, ebp = 0xbfbfe648 --- Tracing command bash pid 1054 tid 100161 td 0xc79c6480 sched_switch(c79c6480,0,104,18c,1bc132e6,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c79c6480,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c782caa0,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(c782caa0,c782cb28,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c79c6480,ffffffff,f63dfc74,6,0,...) at kern_wait+0xb56 wait4(c79c6480,f63dfcf8,10,c0c5fb51,c0d3dae4,...) at wait4+0x3b syscall(f63dfd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282bae7b, esp = 0xbfbfe9cc, ebp = 0xbfbfe9e8 --- Tracing command sshd pid 1053 tid 100094 td 0xc73f1480 sched_switch(c73f1480,0,104,18c,a6e53536,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c73f1480,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0865c2a,c7839a90,0,c0c578b6,c73f1480,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7839aa4,0,f62b9a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7839aa4,c7839a90,c0c5f6c1,5c4,c740d508,...) at _cv_wait_sig+0x240 seltdwait(c740d508,58,c7876e00,c73f1480,c0c75e9d,...) at seltdwait+0xa2 kern_select(c73f1480,a,286030b8,286030dc,0,0,65,280fc6f8) at kern_select+0x4c4 select(c73f1480,f62b9cf8,14,c0c41b66,c0d3e44c,...) at select+0x5e syscall(f62b9d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283c4d03, esp = 0xbfbfde5c, ebp = 0xbfbfdea8 --- Tracing command sshd pid 1051 tid 100101 td 0xc748c6c0 Tracing command awk pid 1035 tid 100115 td 0xc761f6c0 sched_switch(c761f6c0,0,104,18c,5e12d29c,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c761f6c0,0,c0c5d3f5,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c74b5498,4c,c0c5f879,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74b5498,c74b5608,14c,c0c5f879,0,...) at _sleep+0x344 pipe_read(c73f6dc8,f631ec58,c7876d80,0,c761f6c0,...) at pipe_read+0x417 dofileread(f631ec58,ffffffff,ffffffff,0,c73f6dc8,...) at dofileread+0x96 kern_readv(c761f6c0,0,f631ec58,f631ec78,1,...) at kern_readv+0x58 read(c761f6c0,f631ecf8,c,c761f6c0,c0d3da74,...) at read+0x4f syscall(f631ed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x281acd83, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command sh pid 1034 tid 100122 td 0xc761e6c0 sched_switch(c761e6c0,0,104,18c,5e245b3e,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c761e6c0,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c75ef2a8,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(c75ef2a8,c75ef330,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c761e6c0,ffffffff,f6333c74,2,0,...) at kern_wait+0xb56 wait4(c761e6c0,f6333cf8,10,c0c5fc41,c0d3dae4,...) at wait4+0x3b syscall(f6333d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfe95c, ebp = 0xbfbfe978 --- Tracing command top pid 1033 tid 100130 td 0xc78326c0 sched_switch(c78326c0,0,104,18c,af47219c,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c78326c0,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6376a50,c0865c2a,c783a090,0,c78326c0,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c783a0a4,0,f6376a80,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c783a0a4,c783a090,3e9,5c4,c7767bd0,...) at _cv_timedwait_sig+0x250 seltdwait(f6376c2c,f6376c34,c7877200,c78326c0,35323b38,...) at seltdwait+0x8a kern_select(c78326c0,2,bfbfebec,0,0,f6376c70,1,0) at kern_select+0x4c4 select(c78326c0,f6376cf8,14,c0c41b66,c0d3e44c,...) at select+0x5e syscall(f6376d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281e1d03, esp = 0xbfbfeb8c, ebp = 0xbfbfece8 --- Tracing command tail pid 1032 tid 100114 td 0xc761f900 sched_switch(c761f900,0,104,18c,9d1a9548,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,58,...) at mi_switch+0x200 sleepq_switch(c761f900,0,c0c5d3f5,189,58,...) at sleepq_switch+0x15f sleepq_catch_signals(3e9,c08ac5b0,c761f900,2,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c73a6c80,58,c0c543d5,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c73a6c80,c73a6c80,158,c0c543d5,3e9,...) at _sleep+0x30e kern_kevent(c761f900,4,0,1,f631bc58,...) at kern_kevent+0x357 kevent(c761f900,f631bcf8,18,c0bf9e60,c0d401d4,...) at kevent+0x19b syscall(f631bd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (363, FreeBSD ELF32, kevent), eip = 0x2815aa6b, esp = 0xbfbfeb6c, ebp = 0xbfbfec18 --- Tracing command sh pid 1031 tid 100119 td 0xc761ed80 sched_switch(c761ed80,0,104,18c,5a51c6c0,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c761ed80,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c75efaa0,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(c75efaa0,c75efb28,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c761ed80,ffffffff,f632ac74,2,0,...) at kern_wait+0xb56 wait4(c761ed80,f632acf8,10,f632ad38,c0d3dae4,...) at wait4+0x3b syscall(f632ad38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfeb5c, ebp = 0xbfbfeb78 --- Tracing command sshd pid 1030 tid 100125 td 0xc73f1240 Tracing command sshd pid 1029 tid 100103 td 0xc748c240 sched_switch(c748c240,0,104,18c,127d56a4,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c748c240,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0865c2a,c73630d0,0,c0c578b6,c748c240,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c73630e4,0,f62dfa80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c73630e4,c73630d0,c0c5f6c1,5c4,c741c738,...) at _cv_wait_sig+0x240 seltdwait(c741c738,58,c73a6b00,c748c240,c0c75e9d,...) at seltdwait+0xa2 kern_select(c748c240,c,286030b8,286030dc,0,0,65,280fc6f8) at kern_select+0x4c4 select(c748c240,f62dfcf8,14,c0c41b66,c0d3e44c,...) at select+0x5e syscall(f62dfd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283c4d03, esp = 0xbfbfde5c, ebp = 0xbfbfdea8 --- Tracing command sshd pid 1028 tid 100109 td 0xc72fc000 sched_switch(c72fc000,0,104,18c,af4aef3c,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72fc000,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0865c2a,c783a8d0,0,c0c578b6,c72fc000,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c783a8e4,0,f6301a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c783a8e4,c783a8d0,c0c5f6c1,5c4,c7767b98,...) at _cv_wait_sig+0x240 seltdwait(c7767b98,58,c74bb100,c72fc000,c0c75e9d,...) at seltdwait+0xa2 kern_select(c72fc000,a,286030b8,286030dc,0,0,65,280fc6f8) at kern_select+0x4c4 select(c72fc000,f6301cf8,14,c0c41b66,c0d3e44c,...) at select+0x5e syscall(f6301d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283c4d03, esp = 0xbfbfde5c, ebp = 0xbfbfdea8 --- Tracing command sshd pid 1024 tid 100124 td 0xc7434b40 Tracing command sshd pid 1023 tid 100111 td 0xc761fd80 Tracing command sshd pid 1022 tid 100127 td 0xc74346c0 Tracing command cron pid 1010 tid 100117 td 0xc761f240 sched_switch(c761f240,0,104,18c,f3e059d6,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c761f240,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c08ac5b0,c761f240,2,100,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c0daabe4,5c,c0c5a2a4,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0daabe4,0,15c,c0c5a2a4,ea61,...) at _sleep+0x30e kern_nanosleep(c761f240,f6324c64,f6324c6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c761f240,f6324cf8,8,c0c5fd9f,c0d3f460,...) at nanosleep+0x6f syscall(f6324d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2817387f, esp = 0xbfbfec8c, ebp = 0xbfbfecb8 --- Tracing command sendmail pid 1004 tid 100120 td 0xc761eb40 sched_switch(c761eb40,0,104,18c,8fcbee50,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,68,...) at mi_switch+0x200 sleepq_switch(c761eb40,0,c0c5d3f5,189,68,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c75ef850,68,c0c0e4cb,100,0,...) at sleepq_wait_sig+0x17 _sleep(c75ef850,c75ef880,168,c0c0e4cb,0,...) at _sleep+0x344 kern_sigsuspend(c761eb40,0,0,0,0,...) at kern_sigsuspend+0xe4 sigsuspend(c761eb40,f632dcf8,4,c0c5fb51,c0d3ff6c,...) at sigsuspend+0x4d syscall(f632dd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x2832beeb, esp = 0xbfbfcf9c, ebp = 0xbfbfcfc8 --- Tracing command sendmail pid 1000 tid 100082 td 0xc748fb40 sched_switch(c748fb40,0,104,18c,9ec4ceaf,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c748fb40,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6280a50,c0865c2a,c74329d0,0,c748fb40,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c74329e4,0,f6280a80,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c74329e4,c74329d0,1389,5c4,c74d4bd0,...) at _cv_timedwait_sig+0x250 seltdwait(f6280c2c,f6280c34,c7876c80,c748fb40,c6c13000,...) at seltdwait+0x8a kern_select(c748fb40,5,bfbfc520,0,0,f6280c70,5,0) at kern_select+0x4c4 select(c748fb40,f6280cf8,14,c0c60075,c0d3e44c,...) at select+0x5e syscall(f6280d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283cbd03, esp = 0xbfbfc48c, ebp = 0xbfbfcfb8 --- Tracing command sshd pid 993 tid 100133 td 0xc7832000 Tracing command ntpd pid 956 tid 100113 td 0xc761fb40 sched_switch(c761fb40,0,104,18c,7edda507,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c761fb40,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0865c2a,c7325e50,0,c0c578b6,c761fb40,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7325e64,0,f6318a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7325e64,c7325e50,c0c5f6c1,5c4,c7767e00,...) at _cv_wait_sig+0x240 seltdwait(c7767e00,58,c6d71100,c761fb40,0,...) at seltdwait+0xa2 kern_select(c761fb40,1b,bfbfed28,0,0,0,f6318c98,246) at kern_select+0x4c4 select(c761fb40,f6318cf8,14,c761fb40,c0d3e44c,...) at select+0x5e syscall(f6318d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2834cd03, esp = 0xbfbfecfc, ebp = 0xbfbfedc8 --- Tracing command rpc.lockd pid 834 tid 100139 td 0xc7831240 sched_switch(c7831240,0,104,18c,ce1b744d,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7831240,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f63919fc,c0865c2a,c7323680,0,c7831240,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c739f650,0,f6391a2c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c739f650,c7323680,1388,488,c7263700,...) at _cv_timedwait_sig+0x250 svc_run_internal(c78313a4,14,c0c7b473,c0c7ab99,f6391b8c,...) at svc_run_internal+0x356 svc_run(c7323680,0,4,c0a4c0c0,f6391b8c,...) at svc_run+0x7f nlm_syscall(c7831240,f6391cf8,10,c0c6049d,c0d3eaf8,...) at nlm_syscall+0x77d syscall(f6391d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (154, FreeBSD ELF32, nlm_syscall), eip = 0x280f9b2b, esp = 0xbfbfed5c, ebp = 0xbfbfee28 --- Tracing command rpc.statd pid 828 tid 100096 td 0xc742f480 sched_switch(c742f480,0,104,18c,1c03f58f,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c742f480,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f62c1a50,c0865c2a,c741a190,0,c742f480,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c741a1a4,0,f62c1a80,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c741a1a4,c741a190,7531,5c4,c73f6b98,...) at _cv_timedwait_sig+0x250 seltdwait(f62c1c2c,f62c1c34,c6d71100,c742f480,c0abb4ae,...) at seltdwait+0x8a kern_select(c742f480,8,bfbfed44,0,0,f62c1c70,1e,0) at kern_select+0x4c4 select(c742f480,f62c1cf8,14,c0c5fd9f,c0d3e44c,...) at select+0x5e syscall(f62c1d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28181d03, esp = 0xbfbfec9c, ebp = 0xbfbfedd8 --- Tracing command nfsd pid 822 tid 100143 td 0xc748fd80 sched_switch(c748fd80,0,104,18c,10f0e223,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c748fd80,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f639dbf8,c0865c2a,c70bf480,0,c748fd80,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c7433010,0,f639dc28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7433010,c70bf480,1388,488,f639dcdc,...) at _cv_timedwait_sig+0x250 svc_run_internal(f639dd24,c084bad8,c70bf480,f639dd38,c0c5473a,...) at svc_run_internal+0x356 svc_thread_start(c70bf480,f639dd38,c0c5473a,334,c782a2a8,...) at svc_thread_start+0x10 fork_exit(c0a5a0b0,c70bf480,f639dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 822 tid 100142 td 0xc761e000 sched_switch(c761e000,0,104,18c,76affec9,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c761e000,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f639abf8,c0865c2a,c70bf480,0,c761e000,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c7433050,0,f639ac28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7433050,c70bf480,1388,488,f639acdc,...) at _cv_timedwait_sig+0x250 svc_run_internal(f639ad24,c084bad8,c70bf480,f639ad38,c0c5473a,...) at svc_run_internal+0x356 svc_thread_start(c70bf480,f639ad38,c0c5473a,334,c782a2a8,...) at svc_thread_start+0x10 fork_exit(c0a5a0b0,c70bf480,f639ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 822 tid 100141 td 0xc761e240 sched_switch(c761e240,0,104,18c,cd5514e3,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c761e240,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6397bf8,c0865c2a,c70bf480,0,c761e240,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c7433090,0,f6397c28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7433090,c70bf480,1388,488,f6397cdc,...) at _cv_timedwait_sig+0x250 svc_run_internal(f6397d24,c084bad8,c70bf480,f6397d38,c0c5473a,...) at svc_run_internal+0x356 svc_thread_start(c70bf480,f6397d38,c0c5473a,334,c782a2a8,...) at svc_thread_start+0x10 fork_exit(c0a5a0b0,c70bf480,f6397d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 822 tid 100136 td 0xc7831900 sched_switch(c7831900,0,104,18c,b4e4df29,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7831900,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6388ae8,c0865c2a,c70bf480,0,c7831900,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c7839a50,0,f6388b18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7839a50,c70bf480,1388,488,f6388bcc,...) at _cv_timedwait_sig+0x250 svc_run_internal(c7831a64,14,c0c7b473,c0c7a0d7,f6388c3c,...) at svc_run_internal+0x356 svc_run(c70bf480,0,c0c7a3db,200,0,...) at svc_run+0x7f nfssvc_nfsd(bfbfe8c0,f6388c3c,c,c6d71100,f6388c50,...) at nfssvc_nfsd+0xad nfssvc_nfsserver(c7831900,f6388cf8,bfbfe8c0,c7831900,c782a2a8,...) at nfssvc_nfsserver+0x24f nfssvc(c7831900,f6388cf8,8,c0c60075,c0d3eb14,...) at nfssvc+0x77 syscall(f6388d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280d9b0b, esp = 0xbfbfe87c, ebp = 0xbfbfeae8 --- Tracing command nfsd pid 821 tid 100140 td 0xc7831000 sched_switch(c7831000,0,104,18c,6e6b33e8,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7831000,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0865c2a,c6fc5790,0,c0c578b6,c7831000,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6fc57a4,0,f6394a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c6fc57a4,c6fc5790,c0c5f6c1,5c4,c73f67a8,...) at _cv_wait_sig+0x240 seltdwait(c73f67a8,58,c6d71100,c7831000,f6394af0,...) at seltdwait+0xa2 kern_select(c7831000,5,bfbfecf0,0,0,0,65,5) at kern_select+0x4c4 select(c7831000,f6394cf8,14,c0c79e9c,c0d3e44c,...) at select+0x5e syscall(f6394d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28178d03, esp = 0xbfbfeaec, ebp = 0xbfbfee28 --- Tracing command mountd pid 813 tid 100134 td 0xc7831d80 Tracing command rpcbind pid 734 tid 100129 td 0xc7832900 sched_switch(c7832900,0,104,18c,ce24569c,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7832900,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6373aa8,c0865c2a,c7419490,0,c7832900,...) at sleepq_catch_signals+0xb6 sleepq_timedwait_sig(c74194a4,0,f6373ad8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c74194a4,c7419490,7531,5c4,f6373b8c,...) at _cv_timedwait_sig+0x250 seltdwait(f6373c5c,f6373c64,4d4,c7832900,f6373b5c,...) at seltdwait+0x8a poll(c7832900,f6373cf8,c,f6373d38,c0d3f0fc,...) at poll+0x2f9 syscall(f6373d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (209, FreeBSD ELF32, poll), eip = 0x2813a07f, esp = 0xbfbfcc0c, ebp = 0xbfbfedd8 --- Tracing command syslogd pid 705 tid 100086 td 0xc748f480 sched_switch(c748f480,0,104,18c,20a1cb8b,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c748f480,0,c0c5d3f5,189,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0865c2a,c7419350,0,c0c578b6,c748f480,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7419364,0,f6291a80,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7419364,c7419350,c0c5f6c1,5c4,c74d44d0,...) at _cv_wait_sig+0x240 seltdwait(c74d44d0,58,c6d71100,c748f480,0,...) at seltdwait+0xa2 kern_select(c748f480,9,282290ac,0,0,0,65,2819e9c0) at kern_select+0x4c4 select(c748f480,f6291cf8,14,c0c5fc51,c0d3e44c,...) at select+0x5e syscall(f6291d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2818bd03, esp = 0xbfbfde7c, ebp = 0xbfbfee28 --- Tracing command devd pid 573 tid 100104 td 0xc748c000 Tracing command flowcleaner pid 43 tid 100073 td 0xc72fc900 sched_switch(c72fc900,0,104,18c,a885a1e8,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72fc900,0,c0c5d3f5,26a,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0daaa44,0,c0c68135,2,0,...) at sleepq_timedwait+0x6b _sleep(c0daaa44,0,0,c0c68135,4e20,...) at _sleep+0x329 pause(c0c68135,4e20,c0c68095,3d7,0,...) at pause+0x47 flowtable_cleaner(0,f4071d38,c0c5473a,334,c7313000,...) at flowtable_cleaner+0x160 fork_exit(c0919080,0,f4071d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4071d70, ebp = 0 --- Tracing command softdepflush pid 42 tid 100072 td 0xc72fcb40 sched_switch(c72fcb40,0,104,18c,9f312e51,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,44,...) at mi_switch+0x200 sleepq_switch(c72fcb40,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f2aaa0,44,c0c7eb54,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f2aaa0,c0f2aa44,44,c0c7eb54,3e8,...) at _sleep+0x329 softdep_flush(0,f406ed38,c0c5473a,334,c73132a8,...) at softdep_flush+0x244 fork_exit(c0a965d0,0,f406ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf406ed70, ebp = 0 --- Tracing command syncer pid 41 tid 100071 td 0xc72fcd80 sched_switch(c72fcd80,0,104,18c,c1427f6e,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72fcd80,0,c0c5d3f5,26a,c72fcd80,...) at sleepq_switch+0x15f sleepq_timedwait(c0f172f4,0,f406bc88,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0f172f4,c0f172e0,3e8,6cd,4e20,...) at _cv_timedwait+0x250 sched_sync(0,f406bd38,c0c5473a,334,c7313550,...) at sched_sync+0x502 fork_exit(c0904170,0,f406bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf406bd70, ebp = 0 --- Tracing command vnlru pid 40 tid 100070 td 0xc7300000 sched_switch(c7300000,0,104,18c,9c73eaf5,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,50,...) at mi_switch+0x200 sleepq_switch(c7300000,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c73137f8,50,c0c6736c,0,0,...) at sleepq_timedwait+0x6b _sleep(c73137f8,c0f172b4,250,c0c6736c,3e8,...) at _sleep+0x329 vnlru_proc(0,f4068d38,c0c5473a,334,c73137f8,...) at vnlru_proc+0xe7 fork_exit(c0904d40,0,f4068d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4068d70, ebp = 0 --- Tracing command bufdaemon pid 39 tid 100069 td 0xc7300240 sched_switch(c7300240,0,104,18c,89ce0698,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,44,...) at mi_switch+0x200 sleepq_switch(c7300240,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f17028,44,c0c64919,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f17028,c0f1702c,44,c0c64919,3e8,...) at _sleep+0x329 buf_daemon(0,f4065d38,c0c5473a,334,c7313aa0,...) at buf_daemon+0x138 fork_exit(c08ecd10,0,f4065d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4065d70, ebp = 0 --- Tracing command pagezero pid 38 tid 100068 td 0xc7300480 sched_switch(c7300480,0,104,18c,c011723d,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7300480,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f2b6d4,0,c0c842bf,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f2b6d4,c0f2b290,0,c0c842bf,493e0,...) at _sleep+0x329 vm_pagezero(0,f4062d38,c0c5473a,334,c7313d48,...) at vm_pagezero+0xdc fork_exit(c0ad3a20,0,f4062d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4062d70, ebp = 0 --- Tracing command vmdaemon pid 9 tid 100067 td 0xc73006c0 sched_switch(c73006c0,0,104,18c,a5202e4a,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,68,...) at mi_switch+0x200 sleepq_switch(c73006c0,0,c0c5d3f5,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0f2b2fc,68,c0c64919,0,0,...) at sleepq_wait+0x63 _sleep(c0f2b2fc,c0f2b300,68,c0c64919,0,...) at _sleep+0x35b vm_daemon(0,f405fd38,c0c5473a,334,c7314000,...) at vm_daemon+0x59 fork_exit(c0acde60,0,f405fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf405fd70, ebp = 0 --- Tracing command pagedaemon pid 8 tid 100066 td 0xc7300900 sched_switch(c7300900,0,104,18c,422a20ab,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,44,...) at mi_switch+0x200 sleepq_switch(c7300900,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f2b2c4,44,c0c64919,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f2b2c4,c0f2b290,44,c0c64919,1388,...) at _sleep+0x329 vm_pageout(0,f405cd38,c0c5473a,334,c73142a8,...) at vm_pageout+0x2bb fork_exit(c0aced00,0,f405cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf405cd70, ebp = 0 --- Tracing command usbus5 pid 37 tid 100065 td 0xc7300b40 sched_switch(c7300b40,0,104,18c,79df8575,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7300b40,0,c0c5d3f5,247,c7300b40,...) at sleepq_switch+0x15f sleepq_wait(c6fabd0c,0,f3ff9cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fabd0c,c6fabdac,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fabd04,f3ff9d38,c0c5473a,334,c7314550,...) at usb_process+0x193 fork_exit(c07aab10,c6fabd04,f3ff9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff9d70, ebp = 0 --- Tracing command usbus5 pid 36 tid 100064 td 0xc7300d80 sched_switch(c7300d80,0,104,18c,c87315c9,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7300d80,0,c0c5d3f5,247,c7300d80,...) at sleepq_switch+0x15f sleepq_wait(c6fabcdc,0,f3ff6cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fabcdc,c6fabdac,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fabcd4,f3ff6d38,c0c5473a,334,c72f8000,...) at usb_process+0x193 fork_exit(c07aab10,c6fabcd4,f3ff6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff6d70, ebp = 0 --- Tracing command usbus5 pid 35 tid 100063 td 0xc70d6240 sched_switch(c70d6240,0,104,18c,797996d2,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d6240,0,c0c5d3f5,247,c70d6240,...) at sleepq_switch+0x15f sleepq_wait(c6fabcac,0,f3ff3cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fabcac,c6fabdac,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fabca4,f3ff3d38,c0c5473a,334,c72f82a8,...) at usb_process+0x193 fork_exit(c07aab10,c6fabca4,f3ff3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff3d70, ebp = 0 --- Tracing command usbus5 pid 34 tid 100062 td 0xc70d6480 sched_switch(c70d6480,0,104,18c,79798956,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d6480,0,c0c5d3f5,247,c70d6480,...) at sleepq_switch+0x15f sleepq_wait(c6fabc7c,0,f3ff0cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fabc7c,c6fabdac,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fabc74,f3ff0d38,c0c5473a,334,c72f8550,...) at usb_process+0x193 fork_exit(c07aab10,c6fabc74,f3ff0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff0d70, ebp = 0 --- Tracing command usbus4 pid 33 tid 100061 td 0xc70d66c0 sched_switch(c70d66c0,0,104,18c,79797b75,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d66c0,0,c0c5d3f5,247,c70d66c0,...) at sleepq_switch+0x15f sleepq_wait(c6fa1b5c,0,f3fedcb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fa1b5c,c6fa1bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fa1b54,f3fedd38,c0c5473a,334,c72f87f8,...) at usb_process+0x193 fork_exit(c07aab10,c6fa1b54,f3fedd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fedd70, ebp = 0 --- Tracing command usbus4 pid 32 tid 100060 td 0xc70d6900 sched_switch(c70d6900,0,104,18c,c871235b,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d6900,0,c0c5d3f5,247,c70d6900,...) at sleepq_switch+0x15f sleepq_wait(c6fa1b2c,0,f3feacb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fa1b2c,c6fa1bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fa1b24,f3fead38,c0c5473a,334,c72f8aa0,...) at usb_process+0x193 fork_exit(c07aab10,c6fa1b24,f3fead38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fead70, ebp = 0 --- Tracing command usbus4 pid 31 tid 100059 td 0xc70d6b40 sched_switch(c70d6b40,0,104,18c,791633a0,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d6b40,0,c0c5d3f5,247,c70d6b40,...) at sleepq_switch+0x15f sleepq_wait(c6fa1afc,0,f3fe7cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fa1afc,c6fa1bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fa1af4,f3fe7d38,c0c5473a,334,c72f8d48,...) at usb_process+0x193 fork_exit(c07aab10,c6fa1af4,f3fe7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fe7d70, ebp = 0 --- Tracing command usbus4 pid 30 tid 100058 td 0xc70d6d80 sched_switch(c70d6d80,0,104,18c,7916267f,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d6d80,0,c0c5d3f5,247,c70d6d80,...) at sleepq_switch+0x15f sleepq_wait(c6fa1acc,0,f3fe4cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fa1acc,c6fa1bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6fa1ac4,f3fe4d38,c0c5473a,334,c72f9000,...) at usb_process+0x193 fork_exit(c07aab10,c6fa1ac4,f3fe4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fe4d70, ebp = 0 --- Tracing command usbus3 pid 29 tid 100057 td 0xc72f5000 sched_switch(c72f5000,0,104,18c,7916180d,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72f5000,0,c0c5d3f5,247,c72f5000,...) at sleepq_switch+0x15f sleepq_wait(c6f9eb5c,0,f3fe1cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f9eb5c,c6f9ebfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f9eb54,f3fe1d38,c0c5473a,334,c72f92a8,...) at usb_process+0x193 fork_exit(c07aab10,c6f9eb54,f3fe1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fe1d70, ebp = 0 --- Tracing command usbus3 pid 28 tid 100056 td 0xc72f5240 sched_switch(c72f5240,0,104,18c,c8708e03,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72f5240,0,c0c5d3f5,247,c72f5240,...) at sleepq_switch+0x15f sleepq_wait(c6f9eb2c,0,f3fdecb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f9eb2c,c6f9ebfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f9eb24,f3fded38,c0c5473a,334,c72f9550,...) at usb_process+0x193 fork_exit(c07aab10,c6f9eb24,f3fded38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fded70, ebp = 0 --- Tracing command usbus3 pid 27 tid 100055 td 0xc72f5480 sched_switch(c72f5480,0,104,18c,78b2de43,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72f5480,0,c0c5d3f5,247,c72f5480,...) at sleepq_switch+0x15f sleepq_wait(c6f9eafc,0,f3fdbcb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f9eafc,c6f9ebfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f9eaf4,f3fdbd38,c0c5473a,334,c72f97f8,...) at usb_process+0x193 fork_exit(c07aab10,c6f9eaf4,f3fdbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fdbd70, ebp = 0 --- Tracing command usbus3 pid 26 tid 100054 td 0xc72f56c0 sched_switch(c72f56c0,0,104,18c,78b2cef5,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72f56c0,0,c0c5d3f5,247,c72f56c0,...) at sleepq_switch+0x15f sleepq_wait(c6f9eacc,0,f3fd8cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f9eacc,c6f9ebfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f9eac4,f3fd8d38,c0c5473a,334,c6e9a7f8,...) at usb_process+0x193 fork_exit(c07aab10,c6f9eac4,f3fd8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fd8d70, ebp = 0 --- Tracing command usbus2 pid 25 tid 100053 td 0xc72f5900 sched_switch(c72f5900,0,104,18c,78b2c1b0,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72f5900,0,c0c5d3f5,247,c72f5900,...) at sleepq_switch+0x15f sleepq_wait(c6f96b5c,0,f3fd5cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f96b5c,c6f96bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f96b54,f3fd5d38,c0c5473a,334,c6e9aaa0,...) at usb_process+0x193 fork_exit(c07aab10,c6f96b54,f3fd5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fd5d70, ebp = 0 --- Tracing command usbus2 pid 24 tid 100052 td 0xc72f5b40 sched_switch(c72f5b40,0,104,18c,c86f9f96,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72f5b40,0,c0c5d3f5,247,c72f5b40,...) at sleepq_switch+0x15f sleepq_wait(c6f96b2c,0,f3fd2cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f96b2c,c6f96bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f96b24,f3fd2d38,c0c5473a,334,c6e9ad48,...) at usb_process+0x193 fork_exit(c07aab10,c6f96b24,f3fd2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fd2d70, ebp = 0 --- Tracing command usbus2 pid 23 tid 100051 td 0xc72f5d80 sched_switch(c72f5d80,0,104,18c,784fcd91,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c72f5d80,0,c0c5d3f5,247,c72f5d80,...) at sleepq_switch+0x15f sleepq_wait(c6f96afc,0,f3fcfcb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f96afc,c6f96bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f96af4,f3fcfd38,c0c5473a,334,c72dd000,...) at usb_process+0x193 fork_exit(c07aab10,c6f96af4,f3fcfd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fcfd70, ebp = 0 --- Tracing command usbus2 pid 22 tid 100050 td 0xc6f596c0 sched_switch(c6f596c0,0,104,18c,784fbf48,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f596c0,0,c0c5d3f5,247,c6f596c0,...) at sleepq_switch+0x15f sleepq_wait(c6f96acc,0,f3fcccb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f96acc,c6f96bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f96ac4,f3fccd38,c0c5473a,334,c72dd2a8,...) at usb_process+0x193 fork_exit(c07aab10,c6f96ac4,f3fccd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fccd70, ebp = 0 --- Tracing command usbus1 pid 21 tid 100049 td 0xc6f59900 sched_switch(c6f59900,0,104,18c,784fafaf,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f59900,0,c0c5d3f5,247,c6f59900,...) at sleepq_switch+0x15f sleepq_wait(c6f92b5c,0,f3fc9cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f92b5c,c6f92bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f92b54,f3fc9d38,c0c5473a,334,c72dd550,...) at usb_process+0x193 fork_exit(c07aab10,c6f92b54,f3fc9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc9d70, ebp = 0 --- Tracing command usbus1 pid 20 tid 100048 td 0xc6f59b40 sched_switch(c6f59b40,0,104,18c,c5532cbf,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f59b40,0,c0c5d3f5,247,c6f59b40,...) at sleepq_switch+0x15f sleepq_wait(c6f92b2c,0,f3fc6cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f92b2c,c6f92bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f92b24,f3fc6d38,c0c5473a,334,c72dd7f8,...) at usb_process+0x193 fork_exit(c07aab10,c6f92b24,f3fc6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc6d70, ebp = 0 --- Tracing command usbus1 pid 19 tid 100047 td 0xc6f59d80 sched_switch(c6f59d80,0,104,18c,77e658dd,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f59d80,0,c0c5d3f5,247,c6f59d80,...) at sleepq_switch+0x15f sleepq_wait(c6f92afc,0,f3fc3cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f92afc,c6f92bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f92af4,f3fc3d38,c0c5473a,334,c72ddaa0,...) at usb_process+0x193 fork_exit(c07aab10,c6f92af4,f3fc3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc3d70, ebp = 0 --- Tracing command usbus1 pid 18 tid 100046 td 0xc70d5000 sched_switch(c70d5000,0,104,18c,7a472d5b,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d5000,0,c0c5d3f5,247,c70d5000,...) at sleepq_switch+0x15f sleepq_wait(c6f92acc,0,f3fc0cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f92acc,c6f92bfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f92ac4,f3fc0d38,c0c5473a,334,c72ddd48,...) at usb_process+0x193 fork_exit(c07aab10,c6f92ac4,f3fc0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc0d70, ebp = 0 --- Tracing command usbus0 pid 17 tid 100045 td 0xc70d5240 sched_switch(c70d5240,0,104,18c,7a471efc,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d5240,0,c0c5d3f5,247,c70d5240,...) at sleepq_switch+0x15f sleepq_wait(c6f8bb5c,0,f3fbdcb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f8bb5c,c6f8bbfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f8bb54,f3fbdd38,c0c5473a,334,c6d742a8,...) at usb_process+0x193 fork_exit(c07aab10,c6f8bb54,f3fbdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fbdd70, ebp = 0 --- Tracing command usbus0 pid 16 tid 100044 td 0xc70d5480 sched_switch(c70d5480,0,104,18c,c8720f7f,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d5480,0,c0c5d3f5,247,c70d5480,...) at sleepq_switch+0x15f sleepq_wait(c6f8bb2c,0,f3fbacb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f8bb2c,c6f8bbfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f8bb24,f3fbad38,c0c5473a,334,c6d74550,...) at usb_process+0x193 fork_exit(c07aab10,c6f8bb24,f3fbad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fbad70, ebp = 0 --- Tracing command usbus0 pid 15 tid 100043 td 0xc70d56c0 sched_switch(c70d56c0,0,104,18c,79e3a0b6,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d56c0,0,c0c5d3f5,247,c70d56c0,...) at sleepq_switch+0x15f sleepq_wait(c6f8bafc,0,f3fb7cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f8bafc,c6f8bbfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f8baf4,f3fb7d38,c0c5473a,334,c6d747f8,...) at usb_process+0x193 fork_exit(c07aab10,c6f8baf4,f3fb7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fb7d70, ebp = 0 --- Tracing command usbus0 pid 14 tid 100042 td 0xc70d5900 sched_switch(c70d5900,0,104,18c,79e39234,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d5900,0,c0c5d3f5,247,c70d5900,...) at sleepq_switch+0x15f sleepq_wait(c6f8bacc,0,f3fb4cb8,1,0,...) at sleepq_wait+0x63 _cv_wait(c6f8bacc,c6f8bbfc,c0c4817d,67,c0dae980,...) at _cv_wait+0x240 usb_process(c6f8bac4,f3fb4d38,c0c5473a,334,c6d74aa0,...) at usb_process+0x193 fork_exit(c07aab10,c6f8bac4,f3fb4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fb4d70, ebp = 0 --- Tracing command sctp_iterator pid 7 tid 100041 td 0xc70d5b40 sched_switch(c70d5b40,0,104,18c,3599c26c,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70d5b40,0,c0c5d3f5,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0f2119c,0,c0c717ba,0,0,...) at sleepq_wait+0x63 _sleep(c0f2119c,c0f210b0,0,c0c717ba,0,...) at _sleep+0x35b sctp_iterator_thread(0,f3fb1d38,c0c5473a,334,c6d74d48,...) at sctp_iterator_thread+0x60 fork_exit(c0982b10,0,f3fb1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fb1d70, ebp = 0 --- Tracing command fw0_probe pid 6 tid 100037 td 0xc6f57000 sched_switch(c6f57000,0,104,18c,9f4dae18,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c6f57000,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6fc9000,5c,c0c4efa1,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6fc9000,c6fcd488,15c,c0c4efa1,0,...) at _sleep+0x344 fw_bus_probe_thread(c6fc9000,f3fa2d38,c0c5473a,334,c6e9a000,...) at fw_bus_probe_thread+0xa08 fork_exit(c064ab20,c6fc9000,f3fa2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fa2d70, ebp = 0 --- Tracing command xpt_thrd pid 5 tid 100024 td 0xc6dbad80 sched_switch(c6dbad80,0,104,18c,79e35c89,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6dbad80,0,c0c5d3f5,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0d779d4,4c,c0bf5b18,0,0,...) at sleepq_wait+0x63 _sleep(c0d779d4,c0d779ec,4c,c0bf5b18,0,...) at _sleep+0x35b xpt_scanner_thread(0,c6ad2d38,c0c5473a,334,c6e9a2a8,...) at xpt_scanner_thread+0x41 fork_exit(c047fdb0,0,c6ad2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad2d70, ebp = 0 --- Tracing command yarrow pid 13 tid 100017 td 0xc6d77240 sched_switch(c6d77240,0,104,18c,ca849ec0,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d77240,0,c0c5d3f5,26a,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0daaa44,0,c0c4efa1,2,0,...) at sleepq_timedwait+0x6b _sleep(c0daaa44,0,0,c0c4efa1,64,...) at _sleep+0x329 pause(c0c4efa1,64,c0c3c280,111,0,...) at pause+0x47 random_kthread(0,c6abdd38,c0c5473a,334,c6e9a550,...) at random_kthread+0x1ef fork_exit(c07234a0,0,c6abdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abdd70, ebp = 0 --- Tracing command g_down pid 4 tid 100015 td 0xc6d776c0 sched_switch(c6d776c0,0,104,18c,ce17536d,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d776c0,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da87e4,4c,c0c4efa1,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da87e4,c0da8748,24c,c0c4efa1,64,...) at _sleep+0x329 g_io_schedule_down(c6d776c0,0,c0c50669,74,0,...) at g_io_schedule_down+0x6b g_down_procbody(0,c6ab7d38,c0c5473a,334,c6d73000,...) at g_down_procbody+0x8d fork_exit(c0812ed0,0,c6ab7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab7d70, ebp = 0 --- Tracing command g_up pid 3 tid 100014 td 0xc6d77900 sched_switch(c6d77900,0,104,18c,ce22f254,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d77900,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da87e0,4c,c0c4efa1,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da87e0,c0da8768,24c,c0c4efa1,64,...) at _sleep+0x329 g_io_schedule_up(c6d77900,0,c0c50669,5d,0,...) at g_io_schedule_up+0x133 g_up_procbody(0,c6ab4d38,c0c5473a,334,c6d732a8,...) at g_up_procbody+0x8d fork_exit(c0812f60,0,c6ab4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab4d70, ebp = 0 --- Tracing command g_event pid 2 tid 100013 td 0xc6d77b40 sched_switch(c6d77b40,0,104,18c,c5661cd3,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d77b40,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da87d8,4c,c0c4efa1,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da87d8,0,4c,c0c4efa1,64,...) at _sleep+0x329 g_event_procbody(0,c6ab1d38,c0c5473a,334,c6d73550,...) at g_event_procbody+0xcb fork_exit(c0812ff0,0,c6ab1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab1d70, ebp = 0 --- Tracing command intr pid 12 tid 100040 td 0xc70d5d80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100039 td 0xc70d6000 sched_switch(c70d6000,0,109,18c,79dfa7fb,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6d710f0,...) at mi_switch+0x200 ithread_loop(c70cc5d0,f3fa8d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c70cc5d0,f3fa8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fa8d70, ebp = 0 --- Tracing command intr pid 12 tid 100038 td 0xc6f4bd80 sched_switch(c6f4bd80,0,109,18c,13335f2d,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c70bf970,...) at mi_switch+0x200 ithread_loop(c70cc140,f3fa5d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c70cc140,f3fa5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fa5d70, ebp = 0 --- Tracing command intr pid 12 tid 100035 td 0xc6f57480 sched_switch(c6f57480,0,109,18c,77c265e1,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db90f0,...) at mi_switch+0x200 ithread_loop(c6fad9b0,f3f99d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6fad9b0,f3f99d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3f99d70, ebp = 0 --- Tracing command intr pid 12 tid 100034 td 0xc6f576c0 sched_switch(c6f576c0,0,109,18c,849d76cd,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db93f0,...) at mi_switch+0x200 ithread_loop(c6fad170,f3f91d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6fad170,f3f91d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3f91d70, ebp = 0 --- Tracing command intr pid 12 tid 100033 td 0xc6f57900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100032 td 0xc6f57b40 sched_switch(c6f57b40,0,109,18c,a21fc47f,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db91f0,...) at mi_switch+0x200 ithread_loop(c6f82950,f3f7fd38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6f82950,f3f7fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3f7fd70, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xc6f57d80 sched_switch(c6f57d80,0,109,18c,a16c13f1,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db9270,...) at mi_switch+0x200 ithread_loop(c6f66ba0,f3f7bd38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6f66ba0,f3f7bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3f7bd70, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc6f59000 sched_switch(c6f59000,0,109,18c,7d7959ba,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db92f0,...) at mi_switch+0x200 ithread_loop(c6f66930,f3f77d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6f66930,f3f77d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3f77d70, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc6f59240 sched_switch(c6f59240,0,109,18c,ce21e550,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db8ef0,...) at mi_switch+0x200 ithread_loop(c6f52c20,c6be8d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6f52c20,c6be8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6be8d70, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc6f59480 sched_switch(c6f59480,0,109,18c,8a06dab1,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6f55770,...) at mi_switch+0x200 ithread_loop(c6f52190,c6ae2d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6f52190,c6ae2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ae2d70, ebp = 0 --- Tracing command intr pid 12 tid 100026 td 0xc6dba900 sched_switch(c6dba900,0,109,18c,b9fec03d,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6f4a670,...) at mi_switch+0x200 ithread_loop(c6d72360,c6ad8d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d72360,c6ad8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad8d70, ebp = 0 --- Tracing command intr pid 12 tid 100025 td 0xc6dbab40 sched_switch(c6dbab40,0,109,18c,956a0c47,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6f4a6f0,...) at mi_switch+0x200 ithread_loop(c6d72350,c6ad5d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d72350,c6ad5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad5d70, ebp = 0 --- Tracing command intr pid 12 tid 100019 td 0xc6f4b900 sched_switch(c6f4b900,0,109,18c,d139f32,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6f4a8f0,...) at mi_switch+0x200 ithread_loop(c6d24a30,c6ac3d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d24a30,c6ac3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac3d70, ebp = 0 --- Tracing command intr pid 12 tid 100018 td 0xc6f4bb40 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100012 td 0xc6d77d80 sched_switch(c6d77d80,0,109,18c,ce1b75bf,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db8b70,...) at mi_switch+0x200 ithread_loop(c6d721b0,c6aaed38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d721b0,c6aaed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aaed70, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc6dba000 sched_switch(c6dba000,0,109,18c,c1427177,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db8bf0,...) at mi_switch+0x200 ithread_loop(c6d721c0,c6aabd38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d721c0,c6aabd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aabd70, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc6dba240 sched_switch(c6dba240,0,109,18c,c56606bb,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db8c70,...) at mi_switch+0x200 ithread_loop(c6d721d0,c6aa8d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d721d0,c6aa8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa8d70, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc6dba480 sched_switch(c6dba480,0,109,18c,cd550524,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db8cf0,...) at mi_switch+0x200 ithread_loop(c6d721e0,c6aa5d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d721e0,c6aa5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa5d70, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc6d75000 sched_switch(c6d75000,0,109,18c,d3345753,...) at sched_switch+0x437 mi_switch(109,0,c0c549b9,4e3,c6db8d70,...) at mi_switch+0x200 ithread_loop(c6d721f0,c6aa2d38,c0c5473a,334,c6d737f8,...) at ithread_loop+0x1f6 fork_exit(c084e8e0,c6d721f0,c6aa2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa2d70, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc6d75240 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc6d75480 sched_switch(c6d75480,0,108,187,c3dc364c,...) at sched_switch+0x437 mi_switch(108,0,c0c5acc5,3ad,c6d75480,...) at mi_switch+0x200 sched_idletd(0,c6a9bd38,c0c5473a,334,c6d73aa0,...) at sched_idletd+0x16c fork_exit(c0897a50,0,c6a9bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a9bd70, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc6d756c0 sched_switch(c6d756c0,0,108,187,b50089fa,...) at sched_switch+0x437 mi_switch(108,0,c0c5acc5,3ad,c6d756c0,...) at mi_switch+0x200 sched_idletd(0,c6a98d38,c0c5473a,334,c6d73aa0,...) at sched_idletd+0x16c fork_exit(c0897a50,0,c6a98d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a9sched_idletd(0,c6a95d38,c0c5473a,334,c6d73aa0,...) at sched_idletd+0x16c fork_exit(c0897a50,0,c6a95d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a95d70, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc6d75b40 sched_switch(c6d75b40,0,108,187,cdfcf58a,...) at sched_switch+0x437 mi_switch(108,0,c0c5acc5,3ad,c6d75b40,...) at mi_switch+0x200 sched_idletd(0,c6a92d38,c0c5473a,334,c6d73aa0,...) at sched_idletd+0x16c fork_exit(c0897a50,0,c6a92d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a92d70, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc6d75d80 sched_switch(c6d75d80,0,104,18c,126a2b3b,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c6d75d80,0,c0c5d3f5,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5d3f5,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6d73d48,5c,c0c5fc62,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6d73d48,c6d73dd0,15c,c0c5fc62,0,...) at _sleep+0x344 kern_wait(c6d75d80,ffffffff,c6a8ec74,0,0,...) at kern_wait+0xb56 wait4(c6d75d80,c6a8ecf8,10,c0c5fa92,c0d3dae4,...) at wait4+0x3b syscall(c6a8ed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x8054c5f, esp = 0xbfbfe90c, ebp = 0xbfbfe928 --- Tracing command audit pid 10 tid 100001 td 0xc6d77000 sched_switch(c6d77000,0,104,18c,79e0fecc,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d77000,0,c0c5d3f5,247,c6d77000,...) at sleepq_switch+0x15f sleepq_wait(c0f2a3c0,0,c6a8bc9c,1,0,...) at sleepq_wait+0x63 _cv_wait(c0f2a3c0,c0f2a3a4,c0c7c219,194,0,...) at _cv_wait+0x240 audit_worker(0,c6a8bd38,c0c5473a,334,c6d74000,...) at audit_worker+0x84 fork_exit(c0a69c00,0,c6a8bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a8bd70, ebp = 0 --- Tracing command kernel pid 0 tid 100036 td 0xc6f57240 sched_switch(c6f57240,0,104,18c,ca838cb6,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f57240,0,c0c5d3f5,247,c6f57240,...) at sleepq_switch+0x15f sleepq_wait(c6fc6340,0,c0c59c3c,c0c4efa1,0,...) at sleepq_wait+0x63 msleep_spin(c6fc6340,c6fc635c,c0c4efa1,0,c0c578b6,...) at msleep_spin+0x21d taskqueue_thread_loop(c6fcd49c,f3f9fd38,c0c5473a,334,c0da88a0,...) at taskqueue_thread_loop+0x8e fork_exit(c08aede0,c6fcd49c,f3f9fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3f9fd70, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc6dba6c0 sched_switch(c6dba6c0,0,104,18c,956c6048,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6dba6c0,0,c0c5d3f5,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f25dc0,0,c0c4efa1,0,0,...) at sleepq_wait+0x63 _sleep(c6f25dc0,c6f25ddc,0,c0c4efa1,0,...) at _sleep+0x35b taskqueue_thread_loop(c0dbd388,c6adbd38,c0c5473a,334,c0da88a0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08aede0,c0dbd388,c6adbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6adbd70, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc6f4b000 sched_switch(c6f4b000,0,104,18c,7a47b198,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4b000,0,c0c5d3f5,247,c6f4b000,...) at sleepq_switch+0x15f sleepq_wait(c6f260c0,0,c0c59c3c,c0c4efa1,0,...) at sleepq_wait+0x63 msleep_spin(c6f260c0,c6f260dc,c0c4efa1,0,c0c578b6,...) at msleep_spin+0x21d taskqueue_thread_loop(c0d7a6e0,c6acfd38,c0c5473a,334,c0da88a0,...) at taskqueue_thread_loop+0x8e fork_exit(c08aede0,c0d7a6e0,c6acfd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6acfd70, ebp = 0 --- Tracing command kernel pid 0 tid 100022 td 0xc6f4b240 sched_switch(c6f4b240,0,104,18c,7a47a2a3,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4b240,0,c0c5d3f5,247,c6f4b240,...) at sleepq_switch+0x15f sleepq_wait(c6f260c0,0,c0c59c3c,c0c4efa1,0,...) at sleepq_wait+0x63 msleep_spin(c6f260c0,c6f260dc,c0c4efa1,0,c0c578b6,...) at msleep_spin+0x21d taskqueue_thread_loop(c0d7a6e0,c6accd38,c0c5473a,334,c0da88a0,...) at taskqueue_thread_loop+0x8e fork_exit(c08aede0,c0d7a6e0,c6accd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6accd70, ebp = 0 --- Tracing command kernel pid 0 tid 100021 td 0xc6f4b480 sched_switch(c6f4b480,0,104,18c,7a4793fd,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4b480,0,c0c5d3f5,247,c6f4b480,...) at sleepq_switch+0x15f sleepq_wait(c6f260c0,0,c0c59c3c,c0c4efa1,0,...) at sleepq_wait+0x63 msleep_spin(c6f260c0,c6f260dc,c0c4efa1,0,c0c578b6,...) at msleep_spin+0x21d taskqueue_thread_loop(c0d7a6e0,c6ac9d38,c0c5473a,334,c0da88a0,...) at taskqueue_thread_loop+0x8e fork_exit(c08aede0,c0d7a6e0,c6ac9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac9d70, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc6f4b6c0 sched_switch(c6f4b6c0,0,104,18c,7a4758f5,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4b6c0,0,c0c5d3f5,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f26100,0,c0c4efa1,0,0,...) at sleepq_wait+0x63 _sleep(c6f26100,c6f2611c,0,c0c4efa1,0,...) at _sleep+0x35b taskqueue_thread_loop(c0da9164,c6ac6d38,c0c5473a,334,c0da88a0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08aede0,c0da9164,c6ac6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac6d70, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc6d77480 sched_switch(c6d77480,0,104,18c,9b17b77a,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d77480,0,c0c5d3f5,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6d5bb80,0,c0c4efa1,0,0,...) at sleepq_wait+0x63 _sleep(c6d5bb80,c6d5bb9c,0,c0c4efa1,0,...) at _sleep+0x35b taskqueue_thread_loop(c0dbbea0,c6abad38,c0c5473a,334,c0da88a0,...) at taskqueue_thread_loop+0xb4 fork_exit(c08aede0,c0dbbea0,c6abad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abad70, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc0da8b50 sched_switch(c0da8b50,0,104,18c,83036fb3,...) at sched_switch+0x437 mi_switch(104,0,c0c5d3f5,1d2,44,...) at mi_switch+0x200 sleepq_switch(c0da8b50,0,c0c5d3f5,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da88a0,44,c0c5b53a,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da88a0,0,44,c0c5b53a,2710,...) at _sleep+0x329 scheduler(0,141ec00,141ec00,141e000,1425000,...) at scheduler+0x23e mi_startup() at mi_startup+0x96 begin() at begin+0x2c db:1:allt> call doadump Physical memory: 3439 MB Dumping 188 MB: 173 157 141 125 109 93 77 61 45 29 13 Dump complete = 0xf db:1:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 3 (kgdb) bt #0 doadump () at pcpu.h:246 #1 0xc04c4369 in db_fncall (dummy1=0xc087ce67, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf6d0160c " \026Ðö\001") at ../../../ddb/db_command.c:548 #2 0xc04c479f in db_command (last_cmdp=0xc0d789bc, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #3 0xc04c4854 in db_command_script (command=0xc0d79948 "call doadump") at ../../../ddb/db_command.c:516 #4 0xc04c8950 in db_script_exec (scriptname=0xc0d79220 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc04c89e1 in db_run_cmd (addr=0x0, have_addr=0x0, count=0x0, modif=0xf6d01744 "X\027Ðö") at ../../../ddb/db_script.c:375 #6 0xc04c479f in db_command (last_cmdp=0xc0d789bc, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #7 0xc04c4854 in db_command_script (command=0xc0d79880 "run pho") at ../../../ddb/db_command.c:516 #8 0xc04c8950 in db_script_exec (scriptname=0xf6d01850 "kdb.enter.unknown", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #9 0xc04c8a37 in db_script_kdbenter (eventname=0xc0c8e37f "unknown") at ../../../ddb/db_script.c:324 #10 0xc04c6718 in db_trap (type=0xc, code=0x0) at ../../../ddb/db_main.c:228 #11 0xc08a3156 in kdb_trap (type=0xc, code=0x0, tf=0xf6d019d0) at ../../../kern/subr_kdb.c:534 #12 0xc0b97a5f in trap_fatal (frame=0xf6d019d0, eva=0x1a0) at ../../../i386/i386/trap.c:924 #13 0xc0b983d1 in trap (frame=0xf6d019d0) at ../../../i386/i386/trap.c:325 #14 0xc0b7b14b in calltrap () at ../../../i386/i386/exception.s:165 #15 0xc087ce67 in _sx_xlock_hard (sx=0xc7660a88, tid=0xc94b4000, opts=0x0, file=0xc0c56412 "../../../kern/kern_lockf.c", line=0x798) at ../../../kern/kern_sx.c:575 #16 0xc087d3a0 in _sx_xlock (sx=0xc7660a88, opts=0x0, file=0xc0c56412 "../../../kern/kern_lockf.c", line=0x798) at sx.h:155 #17 0xc085f41d in lf_iteratelocks_vnode (vp=0xc991e6b8, fn=0xc0a44b10 , arg=0x0) at ../../../kern/kern_lockf.c:1944 #18 0xc0a42cd0 in nlm_reclaim (ap=0xf6d01b14) at ../../../nlm/nlm_advlock.c:480 #19 0xc0a22d7e in nfs_reclaim (ap=0xf6d01b14) at ../../../nfsclient/nfs_node.c:234 #20 0xc0ba5c45 in VOP_RECLAIM_APV (vop=0xc0d5be00, a=0xf6d01b14) at vnode_if.c:1926 #21 0xc0903614 in vgonel (vp=0xc991e6b8) at vnode_if.h:830 #22 0xc09060b7 in vflush (mp=0xc9ac6870, rootrefs=0x1, flags=0x2, td=0xc94b4000) at ../../../kern/vfs_subr.c:2450 #23 0xc0a27b41 in nfs_unmount (mp=0xc9ac6870, mntflags=0x8080000) at ../../../nfsclient/nfs_vfsops.c:1321 #24 0xc08fd01d in dounmount (mp=0xc9ac6870, flags=0x8080000, td=0xc94b4000) at ../../../kern/vfs_mount.c:1287 #25 0xc08fd58f in unmount (td=0xc94b4000, uap=0xf6d01cf8) at ../../../kern/vfs_mount.c:1172 #26 0xc0b97f74 in syscall (frame=0xf6d01d38) at ../../../i386/i386/trap.c:1073 #27 0xc0b7b1b0 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261 #28 0x00000033 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) f 17 #17 0xc085f41d in lf_iteratelocks_vnode (vp=0xc991e6b8, fn=0xc0a44b10 , arg=0x0) at ../../../kern/kern_lockf.c:1944 1944 sx_xlock(&ls->ls_lock); (kgdb) l 1939 STAILQ_INIT(&locks); 1940 ls = vp->v_lockf; 1941 if (!ls) 1942 return (0); 1943 1944 sx_xlock(&ls->ls_lock); 1945 LIST_FOREACH(lf, &ls->ls_active, lf_link) { 1946 ldesc = malloc(sizeof(struct lockdesc), M_LOCKF, 1947 M_WAITOK); 1948 ldesc->vp = lf->lf_vnode; (kgdb) info loc ls = (struct lockf *) 0xc7660a80 lf = Variable "lf" is not available. (kgdb) p *vp $1 = {v_type = VREG, v_tag = 0xc0c79dfd "nfs", v_op = 0xc0d5be00, v_data = 0xc7bda1d0, v_mount = 0xc9ac6870, v_nmntvnodes = {tqe_next = 0xc83fa600, tqe_prev = 0xc9ac68a4}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0, vu_yield = 0x0}, v_hashlist = {le_next = 0x0, le_prev = 0xc71118a0}, v_hash = 0xd1bbbb8, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0x0, tqh_last = 0xc991e6e8}, v_cache_dd = 0x0, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0, v_clen = 0x0, v_lock = {lock_object = {lo_name = 0xc0c79dfd "nfs", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc6d2e500}, lk_lock = 0xc94b4002, lk_timo = 0x33, lk_pri = 0x50, lk_stack = {depth = 0xa, pcs = {0xc085e6f0, 0xc08f2845, 0xc0ba6295, 0xc090f9e8, 0xc0905ebc, 0xc0a27b41, 0xc08fd01d, 0xc08fd58f, 0xc0b97f74, 0xc0b7b1b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}}, v_interlock = {lock_object = {lo_name = 0xc0c5f4ee "vnode interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d28248}, mtx_lock = 0x4}, v_vnlock = 0xc991e710, v_holdcnt = 0x2, v_usecount = 0x1, v_iflag = 0x80, v_vflag = 0x0, v_writecount = 0x1, v_freelist = {tqe_next = 0x0, tqe_prev = 0x0}, v_bufobj = {bo_mtx = {lock_object = {lo_name = 0xc0c67341 "bufobj interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d2c7c0}, mtx_lock = 0x4}, bo_clean = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc991e7c0}, bv_root = 0x0, bv_cnt = 0x0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc991e7d0}, bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x0, bo_ops = 0xc0d5bfc8, bo_bsize = 0x8000, bo_object = 0xcd49907c, bo_synclist = {le_next = 0xc747b0f4, le_prev = 0xc70a77cc}, bo_private = 0xc991e6b8, __bo_vnode = 0xc991e6b8}, v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0} $ svn diff /usr/src/sys Index: /usr/src/sys/rpc/clnt_rc.c =================================================================== --- /usr/src/sys/rpc/clnt_rc.c (revision 194752) +++ /usr/src/sys/rpc/clnt_rc.c (working copy) @@ -146,33 +146,33 @@ int error; int one = 1; struct ucred *oldcred; + CLIENT *newclient = NULL; mtx_lock(&rc->rc_lock); -again: + while (rc->rc_connecting) { + error = msleep(rc, &rc->rc_lock, + rc->rc_intr ? PCATCH : 0, "rpcrecon", 0); + if (error) { + mtx_unlock(&rc->rc_lock); + return (RPC_INTR); + } + } if (rc->rc_closed) { mtx_unlock(&rc->rc_lock); return (RPC_CANTSEND); } - if (rc->rc_connecting) { - while (!rc->rc_closed && !rc->rc_client && rc->rc_connecting) { - error = msleep(rc, &rc->rc_lock, - rc->rc_intr ? PCATCH : 0, "rpcrecon", 0); - if (error) { - mtx_unlock(&rc->rc_lock); - return (RPC_INTR); - } - } - /* - * If the other guy failed to connect, we might as - * well have another go. - */ - if (!rc->rc_client || rc->rc_closed) - goto again; + if (rc->rc_client) { mtx_unlock(&rc->rc_lock); return (RPC_SUCCESS); - } else { - rc->rc_connecting = TRUE; } + + /* + * My turn to attempt a connect. The rc_connecting variable + * serializes the following code sequence, so it is guaranteed + * that rc_client will still be NULL after it is re-locked below, + * since that is the only place it is set non-NULL. + */ + rc->rc_connecting = TRUE; mtx_unlock(&rc->rc_lock); so = __rpc_nconf2socket(rc->rc_nconf); @@ -188,43 +188,52 @@ bindresvport(so, NULL); if (rc->rc_nconf->nc_semantics == NC_TPI_CLTS) - rc->rc_client = clnt_dg_create(so, + newclient = clnt_dg_create(so, (struct sockaddr *) &rc->rc_addr, rc->rc_prog, rc->rc_vers, rc->rc_sendsz, rc->rc_recvsz); else - rc->rc_client = clnt_vc_create(so, + newclient = clnt_vc_create(so, (struct sockaddr *) &rc->rc_addr, rc->rc_prog, rc->rc_vers, rc->rc_sendsz, rc->rc_recvsz); td->td_ucred = oldcred; - if (!rc->rc_client) { + if (!newclient) { soclose(so); rc->rc_err = rpc_createerr.cf_error; stat = rpc_createerr.cf_stat; goto out; } - CLNT_CONTROL(rc->rc_client, CLSET_FD_CLOSE, 0); - CLNT_CONTROL(rc->rc_client, CLSET_CONNECT, &one); - CLNT_CONTROL(rc->rc_client, CLSET_TIMEOUT, &rc->rc_timeout); - CLNT_CONTROL(rc->rc_client, CLSET_RETRY_TIMEOUT, &rc->rc_retry); - CLNT_CONTROL(rc->rc_client, CLSET_WAITCHAN, rc->rc_waitchan); - CLNT_CONTROL(rc->rc_client, CLSET_INTERRUPTIBLE, &rc->rc_intr); + CLNT_CONTROL(newclient, CLSET_FD_CLOSE, 0); + CLNT_CONTROL(newclient, CLSET_CONNECT, &one); + CLNT_CONTROL(newclient, CLSET_TIMEOUT, &rc->rc_timeout); + CLNT_CONTROL(newclient, CLSET_RETRY_TIMEOUT, &rc->rc_retry); + CLNT_CONTROL(newclient, CLSET_WAITCHAN, rc->rc_waitchan); + CLNT_CONTROL(newclient, CLSET_INTERRUPTIBLE, &rc->rc_intr); stat = RPC_SUCCESS; out: mtx_lock(&rc->rc_lock); - if (rc->rc_closed) { - if (rc->rc_client) { - CLNT_CLOSE(rc->rc_client); - CLNT_RELEASE(rc->rc_client); - rc->rc_client = NULL; - } + KASSERT(rc->rc_client == NULL, ("rc_client not null")); + if (!rc->rc_closed) { + rc->rc_client = newclient; + newclient = NULL; } rc->rc_connecting = FALSE; wakeup(rc); mtx_unlock(&rc->rc_lock); + if (newclient) { + /* + * It has been closed, so discard the new client. + * nb: clnt_[dg|vc]_close()/clnt_[dg|vc]_destroy() cannot + * be called with the rc_lock mutex held, since they may + * msleep() while holding a different mutex. + */ + CLNT_CLOSE(newclient); + CLNT_RELEASE(newclient); + } + return (stat); } @@ -240,19 +249,24 @@ struct rc_data *rc = (struct rc_data *)cl->cl_private; CLIENT *client; enum clnt_stat stat; - int tries; + int tries, error; tries = 0; do { + mtx_lock(&rc->rc_lock); if (rc->rc_closed) { + mtx_unlock(&rc->rc_lock); return (RPC_CANTSEND); } if (!rc->rc_client) { + mtx_unlock(&rc->rc_lock); stat = clnt_reconnect_connect(cl); if (stat == RPC_SYSTEMERROR) { - (void) tsleep(&fake_wchan, 0, - "rpccon", hz); + error = tsleep(&fake_wchan, + rc->rc_intr ? PCATCH : 0, "rpccon", hz); + if (error == EINTR || error == ERESTART) + return (RPC_INTR); tries++; if (tries >= rc->rc_retries) return (stat); @@ -260,9 +274,9 @@ } if (stat != RPC_SUCCESS) return (stat); + mtx_lock(&rc->rc_lock); } - mtx_lock(&rc->rc_lock); if (!rc->rc_client) { mtx_unlock(&rc->rc_lock); stat = RPC_FAILED; @@ -279,7 +293,6 @@ CLNT_GETERR(client, &rc->rc_err); } - CLNT_RELEASE(client); if (stat == RPC_TIMEDOUT) { /* * Check for async send misfeature for NLM @@ -290,6 +303,7 @@ || (rc->rc_timeout.tv_sec == -1 && utimeout.tv_sec == 0 && utimeout.tv_usec == 0)) { + CLNT_RELEASE(client); break; } } @@ -297,8 +311,10 @@ if (stat == RPC_TIMEDOUT || stat == RPC_CANTSEND || stat == RPC_CANTRECV) { tries++; - if (tries >= rc->rc_retries) + if (tries >= rc->rc_retries) { + CLNT_RELEASE(client); break; + } if (ext && ext->rc_feedback) ext->rc_feedback(FEEDBACK_RECONNECT, proc, @@ -307,14 +323,22 @@ mtx_lock(&rc->rc_lock); /* * Make sure that someone else hasn't already - * reconnected. + * reconnected by checking if rc_client has changed. + * If not, we are done with the client and must + * do CLNT_RELEASE(client) twice to dispose of it, + * because there is both an initial refcnt and one + * acquired by CLNT_ACQUIRE() above. */ if (rc->rc_client == client) { - CLNT_RELEASE(rc->rc_client); rc->rc_client = NULL; + mtx_unlock(&rc->rc_lock); + CLNT_RELEASE(client); + } else { + mtx_unlock(&rc->rc_lock); } - mtx_unlock(&rc->rc_lock); + CLNT_RELEASE(client); } else { + CLNT_RELEASE(client); break; } } while (stat != RPC_SUCCESS); @@ -333,6 +357,10 @@ *errp = rc->rc_err; } +/* + * Since this function requires that rc_client be valid, it can + * only be called when that is guaranteed to be the case. + */ static bool_t clnt_reconnect_freeres(CLIENT *cl, xdrproc_t xdr_res, void *res_ptr) { @@ -347,6 +375,10 @@ { } +/* + * CLNT_CONTROL() on the client returned by clnt_reconnect_create() must + * always be called before CLNT_CALL_MBUF() by a single thread only. + */ static bool_t clnt_reconnect_control(CLIENT *cl, u_int request, void *info) { -- Test scenario: nfs7.sh