r235350M, vmcore.354 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r235350M: Sat May 12 21:56:27 CEST 2012 pho@x4.osted.lan:/var/tmp/vmcontention/sys/i386/compile/PHO i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.01-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) lock order reversal: 1st 0xc1bb505c system map (system map) @ vm/vm_map.c:3723 2nd 0xc130d000 vm object (kernel object) @ vm/vm_fault.c:299 3rd 0xc1bb50e8 system map (system map) @ vm/vm_map.c:1454 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,34313a63,a3435,c1000a39,c0dbde40,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3bd,c12d9048,5ae,c18207c4,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3bd,c1bb50e8,c100de2a,c9582110,c100e4a8,...) at _witness_debugger+0x25 witness_checkorder(c1bb50e8,9,c100e49f,5ae,0,...) at witness_checkorder+0x86f _mtx_lock_flags(c1bb50e8,0,c100e49f,5ae,c596e000,...) at _mtx_lock_flags+0xd9 _vm_map_lock(c1bb508c,c100e49f,5ae,0,a,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,c1820878,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd08e7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,c18208df,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d464,871,c182092c,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,11,c596ff44,c182098c,c0d00311,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,c1bb7600,101,827,1,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,c596f4e0,8,...) at uma_zalloc_arg+0x591 vm_radix_insert(c130d030,23480,0,c21fbf80,0,...) at vm_radix_insert+0x1d8 vm_page_insert(c21fbf80,c130d000,23480,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(c130d000,23480,0,40,c1820bbc,...) at vm_page_alloc+0x51c vm_fault_hold(c1bb5000,e337f000,0,1,0,...) at vm_fault_hold+0x60a vm_fault(c1bb5000,e337f000,0,1,c1bb3a68,...) at vm_fault+0x8b vm_fault_wire(c1bb5000,e3000000,e33c9000,0,0,...) at vm_fault_wire+0x3f vm_map_wire(c1bb5000,e3000000,e33c9000,0,e3000000,...) at vm_map_wire+0x2bc kmem_alloc(c1bb5000,3c9000,c955c770,0,0,...) at kmem_alloc+0x10d vm_ksubmap_init(c131e94c,0,1,1000,0,...) at vm_ksubmap_init+0x64 cpu_startup(0,181ec00,181ec00,181e000,1825000,...) at cpu_startup+0x24f mi_startup() at mi_startup+0xac begin() at begin+0x2c avail memory = 3538612224 (3374 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 621108 free (1348 frags, 77470 blocks, 0.1% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 123185 free (5313 frags, 14734 blocks, 0.5% fragmentation) /dev/label/tmp: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/label/tmp: clean, 80775250 free (458 frags, 10096849 blocks, 0.0% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 3408645 free (231101 frags, 397193 blocks, 2.3% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 8840217 free (74625 frags, 1095699 blocks, 0.3% fragmentation) Mounting local file systems:. Setting hostname: x4.osted.lan. re0: link state changed to DOWN re0: link state changed to UP Starting Network: lo0 re0 fwe0 fwip0. lo0: flags=8049 metric 0 mtu 16384 options=3 inet6 ::1 prefixlen 128 inet6 fe80::1%lo0 prefixlen 64 scopeid 0xa inet 127.0.0.1 netmask 0xff000000 nd6 options=21 re0: flags=8843 metric 0 mtu 1500 options=8209b ether 00:21:85:15:33:7d inet 192.168.1.101 netmask 0xffffff00 broadcast 192.168.1.255 inet6 fe80::221:85ff:fe15:337d%re0 prefixlen 64 scopeid 0x1 nd6 options=29 media: Ethernet autoselect (100baseTX ) status: active fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 Starting devd. Starting Network: fwe0. fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 Starting Network: fwip0. fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/kde4/lib /usr/local/lib/compat /usr/local/lib/gcc46 /usr/local/lib/mysql /usr/local/lib/qt4 /usr/local/lib/virtualbox a.out ldconfig path: /usr/lib/aout /usr/lib/compat/aout Creating and/or trimming log files. Starting syslogd. No core dumps found. Additional ABI support: linux. Starting rpcbind. NFS access cache time=60 Clearing /tmp (X related). Starting mountd. Starting nfsuserd. Starting nfsd. Recovering vi editor sessions:lock order reversal: 1st 0xcc719310 ufs (ufs) @ kern/vfs_lookup.c:542 2nd 0xe3128e00 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:260 3rd 0xcc726724 ufs (ufs) @ kern/vfs_subr.c:2159 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,3a632e73,a303632,b00,63,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3bd,c12d74b8,86f,f400833c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3bd,cc726724,c0fc2884,c95895a8,c0fe23fa,...) at _witness_debugger+0x25 witness_checkorder(cc726724,9,c0fe23f1,86f,0,...) at witness_checkorder+0x86f __lockmgr_args(cc726724,80100,cc726790,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f4008444,c0ad499c,c0fdf847,5e2,c1108c08,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11378c0,f4008444,cbee80b0,c1149a20,cc7266cc,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc7266cc,80100,c0fe23f1,86f,eb,...) at _vn_lock+0x78 vget(cc7266cc,80100,cbee8000,50,0,...) at vget+0xbb vfs_hash_get(cbfd1b70,15ef55,80000,cbee8000,f4008590,...) at vfs_hash_get+0xed ffs_vgetf(cbfd1b70,15ef55,80000,f4008590,1,...) at ffs_vgetf+0x49 softdep_sync_buf(cc7192b8,e3128da0,1,106,0,...) at softdep_sync_buf+0xac9 ffs_syncvnode(cc7192b8,1,0,cbee80b0,c1108c08,...) at ffs_syncvnode+0x24c ffs_truncate(cc7192b8,200,0,880,c95da800,...) at ffs_truncate+0x8b5 ufs_direnter(cc7192b8,cc7266cc,f4008904,f4008bb8,0,...) at ufs_direnter+0x980 ufs_makeinode(f4008bb8,c1137e80,f4008afc,f4008a58,c0e28b10,...) at ufs_makeinode+0x65d ufs_create(f4008afc,c102d54e,c1149ca0,cc7192b8,f4008b14,...) at ufs_create+0x30 VOP_CREATE_APV(c11378c0,f4008afc,f4008bb8,f4008a94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f4008b78,f4008c40,1b0,0,c95da800,...) at vn_open_cred+0x205 vn_open(f4008b78,f4008c40,1b0,cbff7af0,4fafc76d,...) at vn_open+0x3b kern_openat(cbee8000,ffffff9c,886cf60,0,a03,...) at kern_openat+0x1ec kern_open(cbee8000,886cf60,0,a02,1b0,...) at kern_open+0x35 sys_open(cbee8000,f4008cec,c1025d9c,c0fdb0af,c1027350,...) at sys_open+0x30 syscall(f4008d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x283d01c3, esp = 0xbfbfc4ec, ebp = 0xbfbfc578 --- . Updating motd:. Starting ntpd. Starting default moused. Configuring syscons: keymap blanktime. Starting sshd. Starting cron. Local package initialization: backuplock order reversal: 1st 0xcc3199dc ufs (ufs) @ kern/vfs_mount.c:1254 2nd 0xcbc3a1b4 devfs (devfs) @ ufs/ffs/ffs_softdep.c:1834 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,38313a63,a3433,cc5da724,8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d70e0,72a,f403a9a8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,cbc3a1b4,c0fcd7a0,c9589408,c1007eb5,...) at _witness_debugger+0x25 witness_checkorder(cbc3a1b4,9,c1007eac,72a,cbc3a220,...) at witness_checkorder+0x86f __lockmgr_args(cbc3a1b4,80400,cbc3a220,0,0,...) at __lockmgr_args+0x8c5 vop_stdlock(f403aaac,c130c880,4,c0fd08e7,eb,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c10f8080,f403aaac,c130c880,c1149a20,cbc3a15c,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cbc3a15c,80400,c1007eac,72a,0,...) at _vn_lock+0x78 softdep_flushworklist(cc31a2dc,f403ab64,cc07b8a0,612,cc37b4d8,...) at softdep_flushworklist+0x47 ffs_sync(cc31a2dc,1,f403abb8,51f,c1bae710,...) at ffs_sync+0x3bf dounmount(cc31a2dc,8080000,cc07b8a0,4a1,f403ac3c,...) at dounmount+0x3f8 sys_unmount(cc07b8a0,f403acec,c93dbc80,0,c1027350,...) at sys_unmount+0x421 syscall(f403ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280bdd6b, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 19723 files, 37538731 used, 80775250 free (458 frags, 10096849 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 19723 files, 37538731 used, 80775250 free (458 frags, 10096849 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Starting inetd. Sun May 13 16:43:48 CEST 2012 FreeBSD/i386 (x4.osted.lan) (console) login: Expensive timeout(9) function: 0xc08cddb0(0xc132dd60) 0.012442515 s lock order reversal: 1st 0xe3139880 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcad2be00 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:284 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,2e687361,38323a63,c1000a34,cc01b390,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d7470,11c,f40f87c8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,cad2be00,c100bdba,c9589610,c100ba4c,...) at _witness_debugger+0x25 witness_checkorder(cad2be00,9,c100ba43,11c,0,...) at witness_checkorder+0x86f _sx_xlock(cad2be00,0,c100ba43,11c,cc78f89c,...) at _sx_xlock+0x9a ufsdirhash_acquire(e3139820,cc78f89c,12c,e40262fc,f40f8898,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(cc78f89c,f40f8904,22fc,f40f8884,f40f8888,...) at ufsdirhash_add+0x13 ufs_direnter(cc93a828,cca43570,f40f8904,f40f8bb8,0,...) at ufs_direnter+0x799 ufs_makeinode(f40f8bb8,c1137e80,f40f8afc,f40f8a58,c0e28b10,...) at ufs_makeinode+0x65d ufs_create(f40f8afc,c102d54e,c1149ca0,cc93a828,f40f8b14,...) at ufs_create+0x30 VOP_CREATE_APV(c11378c0,f40f8afc,f40f8bb8,f40f8a94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f40f8b78,f40f8c40,180,0,cc010000,...) at vn_open_cred+0x205 vn_open(f40f8b78,f40f8c40,180,cbff6a10,0,...) at vn_open+0x3b kern_openat(cc01b2e0,ffffff9c,bfbfe7ab,0,a02,...) at kern_openat+0x1ec kern_open(cc01b2e0,bfbfe7ab,0,a01,180,...) at kern_open+0x35 sys_open(cc01b2e0,f40f8cec,c1025d9c,c0fdb0af,c1027350,...) at sys_open+0x30 syscall(f40f8d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x2829d1c3, esp = 0xbfbfe37c, ebp = 0xbfbfec18 --- May 13 16:50:08 x4 su: pho to root on /dev/pts/0 20120513 16:50:13 all: suj15.sh 20120513 17:20:20 all: unload2.sh 20120513 17:20:22 all: nullfs8.sh lock order reversal: 1st 0xd11181b4 newnfs (newnfs) @ kern/vfs_syscalls.c:4185 2nd 0xe335a500 bufwait (bufwait) @ kern/vfs_bio.c:1905 3rd 0xcf29946c newnfs (newnfs) @ kern/vfs_subr.c:2159 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,a3935,a3500,c12d6320,cc076950,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3bd,c12d6540,86f,f41b5414,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3bd,cf29946c,c0fbcecb,c9589e98,c0fe23fa,...) at _witness_debugger+0x25 witness_checkorder(cf29946c,9,c0fe23f1,86f,0,...) at witness_checkorder+0x86f __lockmgr_args(cf29946c,80100,cf2994d8,0,0,...) at __lockmgr_args+0x8c5 nfs_lock1(f41b5518,c0ad499c,246,c12d6544,c95827f8,...) at nfs_lock1+0xcd VOP_LOCK1_APV(c10f91a0,f41b5518,cc076950,c1149a20,cf299414,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cf299414,80100,c0fe23f1,86f,eb,...) at _vn_lock+0x78 vget(cf299414,80100,cc0768a0,50,0,...) at vget+0xbb vfs_hash_get(cbfd15b8,1808f280,80000,cc0768a0,f41b55e0,...) at vfs_hash_get+0xed nfscl_nget(cbfd15b8,d111815c,cf470840,f41b5794,cc0768a0,...) at nfscl_nget+0x8d nfsrpc_readdirplus(d111815c,f41b5980,f41b593c,d2ed9b80,cc0768a0,...) at nfsrpc_readdirplus+0x13ce ncl_readdirplusrpc(d111815c,f41b5980,d2ed9b80,cc0768a0,ce93f378,...) at ncl_readdirplusrpc+0xec ncl_doio(d111815c,e335a4a0,d2ed9b80,cc0768a0,0,...) at ncl_doio+0x28d ncl_bioread(d111815c,f41b5be4,0,d2ed9b80,c0e2219a,...) at ncl_bioread+0xc4d nfs_readdir(f41b5c04,c102c9c1,200400,c0fe3ac7,1059,...) at nfs_readdir+0x192 VOP_READDIR_APV(c10f91a0,f41b5c04,c0fe3ac7,1059,f41b5c2c,...) at VOP_READDIR_APV+0x100 kern_getdirentries(cc0768a0,5,8893000,1000,f41b5c58,...) at kern_getdirentries+0x230 sys_getdirentries(cc0768a0,f41b5cec,c1025d9c,c0fdb063,c119bcc0,...) at sys_getdirentries+0x31 syscall(f41b5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (196, FreeBSD ELF32, sys_getdirentries), eip = 0x280dcedf, esp = 0xbfbfe72c, ebp = 0xbfbfe758 --- 20120513 17:22:40 all: tmpfs8.sh 20120513 17:23:22 all: zfs5.sh ZFS NOTICE: Prefetch is disabled by default on i386 -- to enable, add "vfs.zfs.prefetch_disable=0" to /boot/loader.conf. ZFS WARNING: Recommended minimum kmem_size is 512MB; expect unstable behavior. Consider tuning vm.kmem_size and vm.kmem_size_max in /boot/loader.conf. ZFS filesystem version 5 ZFS storage pool version 28 lock order reversal: 1st 0xd4b4c058 zfs (zfs) @ /var/tmp/vmcontention/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c:1704 2nd 0xd7ce8310 ufs (ufs) @ kern/vfs_vnops.c:407 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,6f6d2f73,656c7564,667a2f73,2e2e2f73,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d6270,197,f4233114,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,d7ce8310,c0fc2884,c95895a8,c0fe3cd9,...) at _witness_debugger+0x25 witness_checkorder(d7ce8310,1,c0fe3cd0,197,d7ce837c,...) at witness_checkorder+0x86f __lockmgr_args(d7ce8310,200400,d7ce837c,0,0,...) at __lockmgr_args+0x267 ffs_lock(f423321c,f4233244,c0ad499c,1,c95827f8,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11378c0,f423321c,c95827f8,c1149a20,d7ce82b8,...) at VOP_LOCK1_APV+0xf3 _vn_lock(d7ce82b8,200400,c0fe3cd0,197,0,...) at _vn_lock+0x78 vn_rdwr(0,d7ce82b8,ce87d200,200,400400,...) at vn_rdwr+0xf0 vdev_file_io_start(d9b41578,d9b41578,0,2843db,0,...) at vdev_file_io_start+0x117 zio_vdev_io_start(d9b41578,8000,1f0000,d1701000,1f0000,...) at zio_vdev_io_start+0xcc zio_execute(d9b41578,d9a1f310,cbf39000,400,0,...) at zio_execute+0xa1 vdev_mirror_io_start(d9a1f2bc,10,d9a1f2bc,1,eb,...) at vdev_mirror_io_start+0x193 zio_vdev_io_start(d9a1f2bc,d1c088a0,6,d1701000,d1c088a0,...) at zio_vdev_io_start+0x2f4 zio_execute(d9a1f2bc,d1701000,d9af7840,ce87d200,200,...) at zio_execute+0xa1 arc_read_nolock(d9b07834,d1701000,d9af7840,d83e6db0,d9ae4cc0,...) at arc_read_nolock+0x780 arc_read(d9b07834,d1701000,d9af7840,d9ae6e88,d83e6db0,...) at arc_read+0xb0 dbuf_read(d9ae4cc0,0,a,0,f42335f8,...) at dbuf_read+0x497 dmu_buf_hold(cc0b0800,4,0,0,0,...) at dmu_buf_hold+0x105 zap_lockdir(cc0b0800,4,0,0,1,...) at zap_lockdir+0x52 zap_lookup_norm(cc0b0800,4,0,f4233804,8,...) at zap_lookup_norm+0x4f zap_lookup(cc0b0800,4,0,f4233804,8,...) at zap_lookup+0x69 zfs_dirent_lock(f4233788,d9ba8218,f4233804,f4233784,6,...) at zfs_dirent_lock+0x57c zfs_dirlook(d9ba8218,f4233804,f4233af8,0,0,...) at zfs_dirlook+0x6c zfs_lookup(f4233b0c,0,cbfdbe00,d1c088a0,0,...) at zfs_lookup+0x2ba zfs_freebsd_lookup(f4233964,c102cc73,c0b14c7a,d4b4c058,f4233954,...) at zfs_freebsd_lookup+0x79 VOP_CACHEDLOOKUP_APV(d84ffec0,f4233964,f4233b0c,0,0,...) at VOP_CACHEDLOOKUP_APV+0x100 vfs_cache_lookup(f4233a0c,c0fe3148,c0b29a3f,d84ffec0,f42339a0,...) at vfs_cache_lookup+0xe6 VOP_LOOKUP_APV(d84ffec0,f4233a0c,f4233b0c,343,c12d6270,...) at VOP_LOOKUP_APV+0x108 lookup(f4233acc,c0fe17cc,10e,dc,c12d6270,...) at lookup+0x6eb namei(f4233acc,c100d464,f4233ae0,c0a7472a,c1bb7508,...) at namei+0x6ff kern_statat_vnhook(d1c088a0,200,ffffff9c,bfbfda98,0,...) at kern_statat_vnhook+0xbf kern_statat(d1c088a0,200,ffffff9c,bfbfda98,0,...) at kern_statat+0x3c kern_lstat(d1c088a0,bfbfda98,0,f4233bfc,246,...) at kern_lstat+0x36 sys_lstat(d1c088a0,f4233cec,c1025d9c,c0e371f2,cfbf2f55,...) at sys_lstat+0x2f syscall(f4233d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (190, FreeBSD ELF32, sys_lstat), eip = 0x281e69f3, esp = 0xbfbfd5fc, ebp = 0xbfbfdfa8 --- lock order reversal: 1st 0xce076c94 syncer (syncer) @ kern/vfs_subr.c:1759 2nd 0xd50fe880 ufs (ufs) @ /var/tmp/vmcontention/sys/modules/zfs/../../cddl/compat/opensolaris/sys/vnode.h:258 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,6f6d2f73,656c7564,667a2f73,2e2e2f73,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d7ce8,102,efbe38a0,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,d50fe880,c0fc2884,c95895a8,d84f564a,...) at _witness_debugger+0x25 witness_checkorder(d50fe880,9,d84f564a,102,d50fe8ec,...) at witness_checkorder+0x86f __lockmgr_args(d50fe880,80400,d50fe8ec,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(efbe39a8,eb,0,c100d464,cbfd1b70,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11378c0,efbe39a8,42e,c1149a20,d50fe828,...) at VOP_LOCK1_APV+0xf3 _vn_lock(d50fe828,80400,d84f564a,102,ccc63578,...) at _vn_lock+0x78 vdev_file_io_start(cd324578,1e9,1df,0,ccc637f4,...) at vdev_file_io_start+0x288 zio_vdev_io_start(cd324578,d1701000,cac9d000,422,0,...) at zio_vdev_io_start+0xcc zio_execute(cd324578,cac9d000,0,40,0,...) at zio_execute+0xa1 zil_commit(d9bfac00,0,0,240000,cacf98a0,...) at zil_commit+0x90d zfs_sync(cbfd0000,3,2,e06,cbfd0000,...) at zfs_sync+0xd9 sync_fsync(efbe3c6c,c102ca04,80400,c0fe23f1,6df,...) at sync_fsync+0x156 VOP_FSYNC_APV(c11142c0,efbe3c6c,c0fe23f1,6df,cacf98a0,...) at VOP_FSYNC_APV+0x100 sync_vnode(c1304bb4,c1304ba0,3e8,783,4e20,...) at sync_vnode+0x16b sched_sync(0,efbe3d28,c0fcc8d0,3d8,cbedb598,...) at sched_sync+0x282 fork_exit(c0b2ad70,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- 20120513 17:34:51 all: kevent.sh 20120513 17:36:04 all: snap8.sh 20120513 17:36:04 all: malloc2.sh 20120513 17:38:38 all: pread.sh 20120513 17:39:13 all: nfs12.sh Expensive timeout(9) function: 0xc0c33850(0xcc78cac0) 0.814103646 s Expensive timeout(9) function: 0xc0c33850(0xcc78cac0) 0.894253179 s Expensive timeout(9) function: 0xc0c33850(0xcd6a02b0) 0.943074835 s 20120513 18:25:14 all: pageout.sh 20120513 18:38:43 all: nfs10.sh 20120513 18:40:41 all: kevent6.sh 20120513 18:41:16 all: suj8.sh May 13 18:41:17 x4 kernel: pid 16989 (dd), uid 0 inumber 4 on /mnt: filesystem full May 13 18:41:19 x4 kernel: pid 17010 (dd), uid 0 inumber 5 on /mnt: filesystem full 20120513 18:41:21 all: umountf5.sh fsync: giving up on dirty 0xccaec984: tag devfs, type VCHR usecount 1, writecount 0, refcount 924 mountedhere 0xccb2c800 flags (VI(0x200)) v_object 0xcd197afc ref 0 pages 7346 lock type devfs: EXCL by thread 0xcccfc000 (pid 17159, umount, tid 1300851) #0 0xc0a6c68d at __lockmgr_args+0xd6d #1 0xc0b14b95 at vop_stdlock+0x65 #2 0xc0e276d3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37e98 at _vn_lock+0x78 #4 0xc0cd7e07 at softdep_flushworklist+0x47 #5 0xc0cdce0f at ffs_sync+0x3bf #6 0xc0b1fb48 at dounmount+0x3f8 #7 0xc0b20231 at sys_unmount+0x421 #8 0xc0e0091e at syscall+0x2fe #9 0xc0deabe1 at Xint0x80_syscall+0x21 dev md5a fsync: giving up on dirty 0xccaec984: tag devfs, type VCHR usecount 1, writecount 0, refcount 924 mountedhere 0xccb2c800 flags (VI(0x200)) v_object 0xcd197afc ref 0 pages 7346 lock type devfs: EXCL by thread 0xcccfc000 (pid 17159, umount, tid 1300851) #0 0xc0a6c68d at __lockmgr_args+0xd6d #1 0xc0b14b95 at vop_stdlock+0x65 #2 0xc0e276d3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37e98 at _vn_lock+0x78 #4 0xc0cdceb7 at ffs_sync+0x467 #5 0xc0b1fb48 at dounmount+0x3f8 #6 0xc0b20231 at sys_unmount+0x421 #7 0xc0e0091e at syscall+0x2fe #8 0xc0deabe1 at Xint0x80_syscall+0x21 dev md5a 20120513 18:42:09 all: context.sh 20120513 18:43:24 all: ffs_syncvnode.sh May 13 18:46:49 x4 kernel: pid 17753 (rw), uid 1004 inumber 163136 on /mnt: filesystem full May 13 18:47:22 x4 kernel: pid 17748 (rw), uid 1004 inumber 153879 on /mnt: filesystem full May 13 18:47:22 x4 kernel: pid 17749 (creat), uid 1004 inumber 131083 on /mnt: filesystem full 20120513 19:13:47 all: jail4.sh 20120513 19:13:49 all: softupdate.sh May 13 19:50:13 x4 kernel: pid 19844 (rw), uid 0 inumber 17345 on /mnt: filesystem full 20120513 19:55:35 all: sendfile.sh 20120513 20:02:02 all: malloc.sh 20120513 20:04:39 all: context2.sh 20120513 20:07:09 all: truncate5.sh 20120513 20:07:13 all: overlap.sh 20120513 20:17:13 all: rename.sh 20120513 20:50:54 all: nfs11.sh 20120513 21:22:18 all: linger4.sh 20120513 22:00:16 all: snap2-1.sh lock order reversal: 1st 0xe32cec40 bufwait (bufwait) @ kern/vfs_bio.c:1905 2nd 0xcf75d61c snaplk (snaplk) @ ufs/ffs/ffs_snapshot.c:2297 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,632e746f,3932323a,a37,efbe3914,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d7cf8,8f9,efbe39c0,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,cf75d61c,c1007974,c958a240,c10078fe,...) at _witness_debugger+0x25 witness_checkorder(cf75d61c,9,c10078f5,8f9,0,...) at witness_checkorder+0x86f __lockmgr_args(cf75d61c,80900,cbc3a220,0,0,...) at __lockmgr_args+0x8c5 ffs_copyonwrite(cbc3a15c,e33407e0,e33407e0,4,0,...) at ffs_copyonwrite+0x2c2 ffs_geom_strategy(cbc3a254,e33407e0,3a0,e33407e0,4000,...) at ffs_geom_strategy+0xeb bufwrite(e33407e0,0,c100ad90,811,1) at bufwrite+0x159 ffs_bufwrite(e33407e0,cc9d1000,efbe3b84,c0cdc7f1,e33407e0,...) at ffs_bufwrite+0x290 bawrite(e33407e0,ea218000,4000,4000,0,...) at bawrite+0x5c ffs_sbupdate(cc72eb00,3,0,efbe3bc0,c0b24a8a,...) at ffs_sbupdate+0x171 ffs_sync(cbfd0894,3,2,e06,cbfd0894,...) at ffs_sync+0x192 sync_fsync(efbe3c6c,c102ca04,80400,c0fe23f1,6df,...) at sync_fsync+0x156 VOP_FSYNC_APV(c11142c0,efbe3c6c,c0fe23f1,6df,cacf98a0,...) at VOP_FSYNC_APV+0x100 sync_vnode(c1304bb4,c1304ba0,3e8,783,4e20,...) at sync_vnode+0x16b sched_sync(0,efbe3d28,c0fcc8d0,3d8,cbedb598,...) at sched_sync+0x282 fork_exit(c0b2ad70,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- lock order reversal: 1st 0xcf75d61c snaplk (snaplk) @ ufs/ufs/ufs_vnops.c:966 2nd 0xcc27c9dc ufs (ufs) @ ufs/ffs/ffs_snapshot.c:1627 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,a373236,a00,2,f458d680,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d50e8,65b,f458d718,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,cc27c9dc,c0fc2884,c95895a8,c10078fe,...) at _witness_debugger+0x25 witness_checkorder(cc27c9dc,9,c10078f5,65b,0,...) at witness_checkorder+0x86f __lockmgr_args(cc27c9dc,80000,0,0,0,...) at __lockmgr_args+0x8c5 ffs_snapremove(cc27c984,c1007833,f458d848,c11a949c,c119bcc0,...) at ffs_snapremove+0x11f ffs_truncate(cc27c984,0,0,c00,0,...) at ffs_truncate+0x647 ufs_inactive(f458dad4,c102cf1d,eb,c11378c0,f458daa8,...) at ufs_inactive+0x1f6 VOP_INACTIVE_APV(c11378c0,f458dad4,c0fe23f1,9bf,f458db08,...) at VOP_INACTIVE_APV+0x100 vinactive(cc27c984,ccfe1b80,c0fe23f1,8be,0,...) at vinactive+0x127 vputx(f458dc34,c0b2eab3,cc27c984,ffffffdf,cc27c984,...) at vputx+0x370 vput(cc27c984,ffffffdf,cc27c984,f458db8c,6dda780,...) at vput+0x10 kern_unlinkat(ccfe1b80,ffffff9c,bfbfea8e,0,0,...) at kern_unlinkat+0x2a3 kern_unlink(ccfe1b80,bfbfea8e,0,f458dd1c,c0e0091e,...) at kern_unlink+0x2f sys_unlink(ccfe1b80,f458dcec,c1025d9c,c0fdb0c3,c1027350,...) at sys_unlink+0x22 syscall(f458dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (10, FreeBSD ELF32, sys_unlink), eip = 0x28160e1f, esp = 0xbfbfe7bc, ebp = 0xbfbfe838 --- 20120513 22:04:12 all: msdos2.sh 20120513 22:04:13 all: trim4.sh 20120513 22:44:46 all: quota1.sh 20120513 22:55:01 all: truncate4.sh 20120513 22:55:02 all: md2.sh lock order reversal: 1st 0xcfc4841c ufs_extattr_sx (ufs_extattr_sx) @ ufs/ufs/ufs_extattr.c:114 2nd 0xcea0db38 ufs (ufs) @ ufs/ffs/ffs_vfsops.c:1690 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,66000a30,66752f73,78655f73,74746174,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d5eb8,69a,f46a3748,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,cea0db38,c0fc2884,c95895a8,c100ad99,...) at _witness_debugger+0x25 witness_checkorder(cea0db38,9,c100ad90,69a,0,...) at witness_checkorder+0x86f __lockmgr_args(cea0db38,80000,0,0,0,...) at __lockmgr_args+0x8c5 ffs_vgetf(d9e8f2dc,2,80000,f46a387c,0,...) at ffs_vgetf+0x189 ffs_vget(d9e8f2dc,2,80000,f46a387c,0,...) at ffs_vget+0x2e ufs_root(d9e8f2dc,80000,f46a38a4,cfc48400,c0fd08e7,...) at ufs_root+0x28 ufs_extattr_autostart(d9e8f2dc,cc2038a0,c100ad90,42c,0,...) at ufs_extattr_autostart+0x4d ffs_mount(d9e8f2dc,c1137480,cceb9780,cbfdbe00,964,...) at ffs_mount+0x2e77 vfs_donmount(cc2038a0,0,0,d1a76880,d1a76880,...) at vfs_donmount+0xc0a sys_nmount(cc2038a0,f46a3cec,c1025d9c,c0fdb0af,c1027350,...) at sys_nmount+0x7f syscall(f46a3d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (378, FreeBSD ELF32, sys_nmount), eip = 0x280cf98b, esp = 0xbfbfd84c, ebp = 0xbfbfdda8 --- lock order reversal: 1st 0xcc716ae0 so_snd_sx (so_snd_sx) @ kern/uipc_sockbuf.c:148 2nd 0xe30e53c0 bufwait (bufwait) @ vm/vm_pager.c:309 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,75000a39,3a632e66,a383431,f4714500,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12d09c0,135,f47145ec,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,e30e53c0,c0fe0370,c9585308,c10107eb,...) at _witness_debugger+0x25 witness_checkorder(e30e53c0,9,c10107e2,135,0,...) at witness_checkorder+0x86f __lockmgr_args(e30e53c0,80000,0,0,0,...) at __lockmgr_args+0x8c5 initpbuf(c130e36c,0,c10107e2,160,cded7d4c,...) at initpbuf+0xe0 getpbuf(c130ca50,0,c100ce8e,47f,f4714720,...) at getpbuf+0x12f swap_pager_getpages(cded7d4c,f4714750,1,0,f47148cc,...) at swap_pager_getpages+0x1bd vm_fault_hold(cc16d910,8847000,1,0,0,...) at vm_fault_hold+0xe63 vm_fault(cc16d910,8847000,1,0,88470f8,...) at vm_fault+0x8b trap_pfault(0,f47149a8,c0aae34c,c119c9f0,d3d9b2cc,...) at trap_pfault+0x1b9 trap(f4714a60) at trap+0x5f1 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0xc0dfe202, esp = 0xf4714aa0, ebp = 0xf4714adc --- copyin(0,f4714b1c,c0af1b86,cdbc0718,4,...) at copyin+0x32 uiomove(cdbc0718,4,f4714c3c,1,0,...) at uiomove+0x1b m_uiotombuf(f4714c3c,2,a800,0,0,...) at m_uiotombuf+0xf6 sosend_generic(cc7169c0,0,f4714c3c,0,0,...) at sosend_generic+0x432 sosend(cc7169c0,0,f4714c3c,0,0,...) at sosend+0x3f soo_write(ccae5428,f4714c3c,cbfdbe00,0,d9a03000,...) at soo_write+0x63 dofilewrite(f4714c3c,ffffffff,ffffffff,0,f4714c18,...) at dofilewrite+0x95 kern_writev(d9a03000,3,f4714c3c,f4714c5c,1,...) at kern_writev+0x68 sys_write(d9a03000,f4714cec,c1025d9c,c0fa5962,f4714cb4,...) at sys_write+0x4f syscall(f4714d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x281857a3, esp = 0xbfbfe74c, ebp = 0xbfbfe7a8 --- 20120513 23:17:01 all: gjournal.sh GEOM_JOURNAL: Journal 3879556884: md5 contains data. GEOM_JOURNAL: Journal 3879556884: md5 contains journal. GEOM_JOURNAL: Journal md5 clean. GEOM_JOURNAL: BIO_FLUSH not supported by md5. May 13 23:17:19 x4 kernel: pid 34210 (dd), uid 0 inumber 4 on /media: filesystem full 20120513 23:17:21 all: fpu.sh 20120513 23:19:58 all: suj2.sh 20120513 23:19:59 all: sendfile2.sh 20120513 23:20:29 all: newfs3.sh 20120514 07:32:24 all: crossmp2.sh 20120514 07:33:29 all: kevent5.sh lock order reversal: 1st 0xcf9d772c filedesc structure (filedesc structure) @ kern/kern_descrip.c:1201 2nd 0xd1521df0 ufs (ufs) @ kern/vfs_subr.c:4315 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,20296572,656b2040,6b2f6e72,5f6e7265,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12c0f28,10db,f51c9a70,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,d1521df0,c0fc2884,c95895a8,c0fe23fa,...) at _witness_debugger+0x25 witness_checkorder(d1521df0,9,c0fe23f1,10db,d1521e5c,...) at witness_checkorder+0x86f __lockmgr_args(d1521df0,80400,d1521e5c,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f51c9b78,1,c95823e8,f51c9b5c,c0b14c7a,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11378c0,f51c9b78,c0ad499c,c1149a20,d1521d98,...) at VOP_LOCK1_APV+0xf3 _vn_lock(d1521d98,80400,c0fe23f1,10db,f51c9bd4,...) at _vn_lock+0x78 vfs_knllock(d1521d98,0,c0fcc1e5,728,d5f8a048,...) at vfs_knllock+0x29 knlist_remove_kq(0,f51c9bf4,c0b24169,d5b0d094,d5f8a048,...) at knlist_remove_kq+0x85 knlist_remove(d5b0d094,d5f8a048,0,f51c9c20,c0a4f555,...) at knlist_remove+0x1b filt_vfsdetach(d5f8a048,0,c0fcc1e5,832,3,...) at filt_vfsdetach+0x39 knote_fdclose(d6054000,3,c0fcb92c,4b1,cf9d772c,...) at knote_fdclose+0xf5 kern_close(d6054000,3,f51c9d1c,c0e0091e,d6054000,...) at kern_close+0xd2 sys_close(d6054000,f51c9cec,c1025d9c,c0fdbb29,0,...) at sys_close+0x1a syscall(f51c9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (6, FreeBSD ELF32, sys_close), eip = 0x28171783, esp = 0xbfbfe7cc, ebp = 0xbfbfe828 --- 20120514 07:37:15 all: mac.sh 20120514 07:37:15 all: snap2.sh 20120514 07:41:17 all: msdos3.sh 20120514 07:41:18 all: fdescfs.sh May 14 07:41:19 x4 mountd[1009]: can't delete exports for /mnt14: Invalid argument May 14 07:41:19 x4 mountd[1009]: can't delete exports for /mnt15: Invalid argument May 14 07:41:19 x4 mountd[1009]: can't delete exports for /mnt10: Invalid argument May 14 07:41:19 x4 mountd[1009]: can't delete exports for /mnt9: Invalid argument May 14 07:41:19 x4 mountd[1009]: can't delete exports for /mnt6: Invalid argument May 14 07:41:19 x4 mountd[1009]: can't delete exports for /mnt3: Invalid argument May 14 07:41:19 x4 mountd[1009]: can't delete exports for /mnt12: Invalid argument May 14 07:41:20 x4 mountd[1009]: can't delete exports for /mnt3: Invalid argument May 14 07:41:20 x4 mountd[1009]: can't delete exports for /mnt6: Invalid argument : : May 14 15:06:08 x4 kernel: pid 37241 (fragments), uid 1004 inumber 7086 on /mnt: filesystem full May 14 15:06:09 x4 kernel: pid 37235 (fragments), uid 1004 inumber 7069 on /mnt: filesystem full May 14 15:06:10 x4 kernel: pid 37237 (fragments), uid 1004 inumber 7106 on /mnt: filesystem full May 14 15:06:11 x4 kernel: pid 37238 (fragments), uid 1004 inumber 7108 on /mnt: filesystem full May 14 15:06:12 x4 kernel: pid 37240 (fragments), uid 1004 inumber 7090 on /mnt: filesystem full May 14 15:06:13 x4 kernel: pid 37236 (fragments), uid 1004 inumber 7082 on /mnt: filesystem full May 14 15:06:14 x4 kernel: pid 37240 (fragments), uid 1004 inumber 7090 on /mnt: filesystem full May 14 15:06:15 x4 kernel: pid 37236 (fragments), uid 1004 inumber 7069 on /mnt: filesystem full May 14 15:06:17 x4 kernel: pid 37238 (fragments), uid 1004 inumber 7088 on /mnt: filesystem full 20120514 15:07:11 all: kevent4.sh 20120514 15:10:47 all: posix_fadvise.sh 20120514 15:10:53 all: devfd.sh 20120514 15:11:06 all: umountf2.sh 20120514 15:22:34 all: linger2.sh May 14 15:22:38 x4 kernel: pid 40515 (linger2), uid 1004 inumber 889 on /mnt: filesystem full May 14 15:22:38 x4 kernel: pid 40517 (linger2), uid 1004 inumber 892 on /mnt: filesystem full 20120514 15:24:22 all: ldt.sh 20120514 15:24:24 all: suj12.sh GEOM_NOP: Device md5.nop created. GEOM_NOP: Device md5.nop removed. 20120514 15:45:25 all: lookup_shared2.sh 20120514 15:56:07 all: jail4.sh 20120514 15:56:07 all: nullfs3.sh 20120514 15:56:09 all: statfs.sh 20120514 15:56:27 all: truncate5.sh 20120514 15:56:31 all: sendfile3.sh 20120514 15:57:52 all: suj20.sh 20120514 15:59:16 all: suj3.sh cryptosoft0: on motherboard GEOM_ELI: Device md5.eli created. GEOM_ELI: Encryption: AES-XTS 128 GEOM_ELI: Crypto: software GEOM_ELI: md5 has been killed. GEOM_ELI: Device md5.eli destroyed. 20120514 16:05:39 all: trim4.sh 20120514 16:45:34 all: extattrctl.sh May 14 16:45:35 x4 mountd[1009]: can't delete exports for /mnt: Invalid argument 20120514 16:45:35 all: nfs10.sh 20120514 16:47:33 all: rename9.sh 20120514 16:48:20 all: sysctl.sh 20120514 16:48:29 all: symlink2.sh 20120514 16:48:36 all: mmap2.sh 20120514 17:09:40 all: union.sh 20120514 17:21:41 all: suj16.sh 20120514 17:51:46 all: altbufferflushes.sh FreeBSD/i386 (x4.osted.lan) (console) login: ~KDB: enter: Break to debugger [ thread pid 3841 tid 100193 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> set $lines 0 db> ps pid ppid pgrp uid state wmesg wchan cmd 46968 1153 1153 25 RL sendmail 46967 1150 1150 0 RL sendmail 46966 1158 1158 0 RL cron 46965 1201 1201 0 RL CPU 3 watchdogd 46964 36057 36055 1001 RL sh 46946 46276 38014 0 S+ piperd 0xccad37d0 egrep 46945 46276 38014 0 R+ dd 46276 38014 38014 0 S+ wait 0xcc1fc598 sh 51708 0 0 0 DL crypto_r 0xd43f7ccc [crypto returns] 51707 0 0 0 DL crypto_w 0xd43f7ca8 [crypto] 38014 1299 38014 0 S+ wait 0xd637ab30 sh 16539 16538 16539 1001 Rs+ bash 16538 16536 16536 1001 S select 0xd0832764 sshd 16536 1147 16536 0 Ss select 0xcce18d24 sshd 36058 36055 36055 1001 S piperd 0xccdd6320 awk 36057 36055 36055 1001 RL CPU 1 sh 36056 36051 36056 1001 ?s+ top 36055 36053 36055 1001 SWs wait 0xcc9db598 sh 36054 36052 36054 1001 Ss kqread 0xcce49600 tail 36053 36045 36045 1001 S select 0xcc067524 sshd 36052 36046 36046 1001 S select 0xce2096a4 sshd 36051 36047 36047 1001 S select 0xccbb5ca4 sshd 36047 1147 36047 0 Ss select 0xcf0d3ea4 sshd 36046 1147 36046 0 Ss select 0xce08d7e4 sshd 36045 1147 36045 0 Ss select 0xcbfcf424 sshd 3841 0 0 0 RL (threaded) [zfskern] 100220 D l2arc_fe 0xd8510884 [l2arc_feed_thread] 100193 Run CPU 2 [arc_reclaim_thread] 1299 1296 1299 0 SW+ wait 0xcc0722cc bash 1296 1295 1296 0 SW+ pause 0xcc0ac5f0 csh 1295 1290 1295 1001 SW+ wait 0xcc950864 su 1290 1289 1290 1001 SWs+ wait 0xcc095864 bash 1289 1285 1285 1001 S select 0xcbfcf8a4 sshd 1285 1147 1285 0 Ss select 0xcc040de4 sshd 1251 1 1251 0 Ss+ ttyin 0xcae33870 getty 1250 1 1250 0 Ss+ ttyin 0xcae33a70 getty 1249 1 1249 0 Ss+ ttyin 0xc970d270 getty 1248 1 1248 0 Ss+ ttyin 0xc970d470 getty 1247 1 1247 0 Ss+ ttyin 0xc970d670 getty 1246 1 1246 0 Ss+ ttyin 0xc970d870 getty 1245 1 1245 0 Ss+ ttyin 0xc970da70 getty 1244 1 1244 0 Ss+ ttyin 0xc970dc70 getty 1243 1 1243 0 Ss+ ttyin 0xc970de70 getty 1226 1 1226 0 Ss select 0xcac861a4 inetd 1201 1 1201 0 DLs pfault 0xc130cf7c watchdogd 1158 1 1158 0 RLs CPU 0 cron 1153 1 1153 25 RLs sendmail 1150 1 1150 0 RLs sendmail 1147 1 1147 0 Ss select 0xcac7ada4 sshd 1115 1 1115 0 Ss select 0xcac86564 moused 1091 1 1091 0 Ss select 0xcc0416a4 ntpd 1035 1031 1031 0 S (threaded) nfsd 100125 S rpcsvc 0xcc008b90 nfsd: service 100124 S rpcsvc 0xcc008bd0 nfsd: service 100123 S rpcsvc 0xcc008c10 nfsd: service 100092 S rpcsvc 0xcaca9e90 nfsd: master 1031 1 1031 0 Ss select 0xcac865e4 nfsd 1029 1025 1025 0 S select 0xcbfcf3a4 nfsuserd 1028 1025 1025 0 S select 0xcc040064 nfsuserd 1027 1025 1025 0 S select 0xcbfcf3e4 nfsuserd 1026 1025 1025 0 S select 0xcc0400a4 nfsuserd 1025 1 1025 0 SWs pause 0xcc0ab5f0 nfsuserd 1009 1 1009 0 Ss select 0xcc011764 mountd 976 1 976 0 Ss select 0xc955f124 rpcbind 961 1 961 0 Ss select 0xcc040864 syslogd 838 1 838 0 Ss select 0xcbfcf024 devd 19 0 0 0 RL [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc1304bb4 [syncer] 16 0 0 0 DL psleep 0xc1304848 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e76c [pagezero] 8 0 0 0 DL psleep 0xc130e348 [vmdaemon] 7 0 0 0 DL zfs:lowm 0xd85016b4 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151354 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc13080d8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 D - 0xcacfed04 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d7e4 [yarrow] 13 0 0 0 DL (threaded) [geom] 100016 D - 0xc118a8a8 [g_down] 100015 D - 0xc118a8a4 [g_up] 100014 D - 0xc118a89c [g_event] 12 0 0 0 WL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 CanRun [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 CanRun [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130c000 [audit] 0 0 0 0 DLs (threaded) [kernel] 1818774 D - 0xd8396740 [aiod_bio taskq] 100219 D - 0xd8396840 [system_taskq_3] 100218 D - 0xd8396840 [system_taskq_2] 100217 D - 0xd8396840 [system_taskq_1] 100216 D - 0xd8396840 [system_taskq_0] 100069 D - 0xc118d7e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118aa60 [swapper] db> c root ~KDB: enter: Break to debugger [ thread pid 46965 tid 100126 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> ps pid ppid pgrp uid state wmesg wchan cmd 46968 1153 1153 25 RL sendmail 46967 1150 1150 0 RL sendmail 46966 1158 1158 0 RL cron 46965 1201 1201 0 RL CPU 2 watchdogd 46964 36057 36055 1001 RL sh 46946 46276 38014 0 S+ piperd 0xccad37d0 egrep 46945 46276 38014 0 R+ dd 46276 38014 38014 0 S+ wait 0xcc1fc598 sh 51708 0 0 0 DL crypto_r 0xd43f7ccc [crypto returns] 51707 0 0 0 DL crypto_w 0xd43f7ca8 [crypto] 38014 1299 38014 0 S+ wait 0xd637ab30 sh 16539 16538 16539 1001 Rs+ bash 16538 16536 16536 1001 S select 0xd0832764 sshd 16536 1147 16536 0 Ss select 0xcce18d24 sshd 36058 36055 36055 1001 S piperd 0xccdd6320 awk 36057 36055 36055 1001 RL sh 36056 36051 36056 1001 Rs+ top 36055 36053 36055 1001 SWs wait 0xcc9db598 sh 36054 36052 36054 1001 Ds ufs 0xd403a1b4 tail 36053 36045 36045 1001 S select 0xcc067524 sshd 36052 36046 36046 1001 S select 0xce2096a4 sshd 36051 36047 36047 1001 S select 0xccbb5ca4 sshd 36047 1147 36047 0 Ss select 0xcf0d3ea4 sshd 36046 1147 36046 0 Ss select 0xce08d7e4 sshd 36045 1147 36045 0 Ss select 0xcbfcf424 sshd 3841 0 0 0 DL (threaded) [zfskern] 100220 D l2arc_fe 0xd8510884 [l2arc_feed_thread] 100193 D arc_recl 0xd85016fc [arc_reclaim_thread] 1299 1296 1299 0 SW+ wait 0xcc0722cc bash 1296 1295 1296 0 SW+ pause 0xcc0ac5f0 csh 1295 1290 1295 1001 SW+ wait 0xcc950864 su 1290 1289 1290 1001 SWs+ wait 0xcc095864 bash 1289 1285 1285 1001 S select 0xcbfcf8a4 sshd 1285 1147 1285 0 Ss select 0xcc040de4 sshd 1251 1 1251 0 Ss+ ttyin 0xcae33870 getty 1250 1 1250 0 Ss+ ttyin 0xcae33a70 getty 1249 1 1249 0 Ss+ ttyin 0xc970d270 getty 1248 1 1248 0 Ss+ ttyin 0xc970d470 getty 1247 1 1247 0 Ss+ ttyin 0xc970d670 getty 1246 1 1246 0 Ss+ ttyin 0xc970d870 getty 1245 1 1245 0 Ss+ ttyin 0xc970da70 getty 1244 1 1244 0 Ss+ ttyin 0xc970dc70 getty 1243 1 1243 0 Rs+ getty 1226 1 1226 0 Ss select 0xcac861a4 inetd 1201 1 1201 0 RLs CPU 3 watchdogd 1158 1 1158 0 RLs cron 1153 1 1153 25 RLs sendmail 1150 1 1150 0 DLs pfault 0xc130cf7c sendmail 1147 1 1147 0 Ss select 0xcac7ada4 sshd 1115 1 1115 0 Ss select 0xcac86564 moused 1091 1 1091 0 Ss select 0xcc0416a4 ntpd 1035 1031 1031 0 S (threaded) nfsd 100125 S rpcsvc 0xcc008b90 nfsd: service 100124 S rpcsvc 0xcc008bd0 nfsd: service 100123 S rpcsvc 0xcc008c10 nfsd: service 100092 S rpcsvc 0xcaca9e90 nfsd: master 1031 1 1031 0 Ss select 0xcac865e4 nfsd 1029 1025 1025 0 S select 0xcbfcf3a4 nfsuserd 1028 1025 1025 0 S select 0xcc040064 nfsuserd 1027 1025 1025 0 S select 0xcbfcf3e4 nfsuserd 1026 1025 1025 0 S select 0xcc0400a4 nfsuserd 1025 1 1025 0 SWs pause 0xcc0ab5f0 nfsuserd 1009 1 1009 0 Ss select 0xcc011764 mountd 976 1 976 0 Ss select 0xc955f124 rpcbind 961 1 961 0 Ls *page loc 0xd7c55e80 syslogd 838 1 838 0 Ss select 0xcbfcf024 devd 19 0 0 0 RL CPU 1 [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc1304bb4 [syncer] 16 0 0 0 DL psleep 0xc1304848 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e76c [pagezero] 8 0 0 0 DL psleep 0xc130e348 [vmdaemon] 7 0 0 0 RL [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151354 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc13080d8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 D - 0xcacfed04 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d7e4 [yarrow] 13 0 0 0 DL (threaded) [geom] 100016 D - 0xc118a8a8 [g_down] 100015 D - 0xc118a8a4 [g_up] 100014 D - 0xc118a89c [g_event] 12 0 0 0 RL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 Run CPU 0 [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 CanRun [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 CanRun [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130c000 [audit] 0 0 0 0 DLs (threaded) [kernel] 1818774 D - 0xd8396740 [aiod_bio taskq] 100219 D - 0xd8396840 [system_taskq_3] 100218 D - 0xd8396840 [system_taskq_2] 100217 D - 0xd8396840 [system_taskq_1] 100216 D - 0xd8396840 [system_taskq_0] 100069 D - 0xc118d7e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118aa60 [swapper] db> c ~KDB: enter: Break to debugger [ thread pid 46964 tid 1815941 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 46964 tid 1815941 td 0xccd688a0 kdb_enter(c0e46ce9,c0fd687e,f4add7e4,c08fb51d,c95a7200,...) at kdb_enter+0x3a kdb_break(c95a7200,c1bb7600,162,1000,f4add7f8,...) at kdb_break+0x28 uart_intr(c95a7200,c1bb7608,c1bb7608,f4add824,0,...) at uart_intr+0x12d intr_event_handle(c95aab80,f4add850,101,cc3ad2e0,23f,...) at intr_event_handle+0x76 intr_execute_handlers(c96210d0,f4add850,2,f4add8c4,c0deaf45,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,f4add850) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0a749b0, esp = 0xf4add890, ebp = 0xf4add8c4 --- _mtx_lock_spin(c11aab94,ccd688a0,0,c0fd8cb0,222,...) at _mtx_lock_spin+0x130 _mtx_lock_spin_flags(c11aab94,0,c0fd8cb0,222,c130d200,...) at _mtx_lock_spin_flags+0x165 turnstile_trywait(c130d200,c12c12bc,c9582178,ccd68950,0,...) at turnstile_trywait+0x3f _mtx_lock_sleep(c130d200,ccd688a0,0,c100f927,75c,...) at _mtx_lock_sleep+0x218 _mtx_lock_flags(c130d200,0,c100f927,75c,c2aefa40,...) at _mtx_lock_flags+0x10f vm_page_free_toq(c2aefa40,f4add9e8,c0d17413,c2aefa40,0,...) at vm_page_free_toq+0x17e vm_page_free(c2aefa40,0,c100f927,4fe,0,...) at vm_page_free+0x15 vm_page_alloc(d7a85784,1e,0,0,f4addb64,...) at vm_page_alloc+0x583 vm_fault_hold(cc494cb0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc494cb0,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(c0dffe2f,0,ccd688a0,d5d22b30,d5d22b30,...) at trap_pfault+0x1b9 trap(f4addd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- db:0:bt> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x6d4a00 curthread = 0xd17e92e0: pid 46945 "dd" curpcb = 0xf45f1d80 fpcurthread = none idlethread = 0xc95df8a0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x828aa00 curthread = 0xcc097b80: pid 1150 "sendmail" curpcb = 0xf402ad80 fpcurthread = none idlethread = 0xc95df5c0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x828da00 curthread = 0xccd688a0: pid 46964 "sh" curpcb = 0xf4addd80 fpcurthread = none idlethread = 0xc95df2e0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x8290a00 curthread = 0xcc01b5c0: pid 46965 "watchdogd" curpcb = 0xf40f5d80 fpcurthread = none idlethread = 0xc95df000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 46968 (sendmail) thread 0xcd9c6b80 (1816147) exclusive sleep mutex vm object (standard object) r = 0 (0xd7e5a6f0) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xd5d78cf8) locked @ vm/vm_map.c:3723 Process 46967 (sendmail) thread 0xcc9b02e0 (1816058) exclusive sleep mutex vm object (standard object) r = 0 (0xd53f1534) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xc95e5c10) locked @ vm/vm_map.c:3723 Process 46965 (watchdogd) thread 0xcc01b5c0 (100126) exclusive sleep mutex vm object (standard object) r = 0 (0xd0779cb8) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xd5b2c048) locked @ vm/vm_map.c:3723 Process 46964 (sh) thread 0xccd688a0 (1815941) exclusive sleep mutex page lock (page lock) r = 0 (0xc130df00) locked @ vm/vm_page.c:1278 exclusive sleep mutex vm object (standard object) r = 0 (0xd7a85784) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc494cf8) locked @ vm/vm_map.c:3723 Process 46945 (dd) thread 0xd17e92e0 (1301061) exclusive sleep mutex vm object (standard object) r = 0 (0xce00aa68) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe334fda0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3394b60) locked @ kern/vfs_bio.c:2652 exclusive lockmgr ufs (ufs) r = 0 (0xcd28cc94) locked @ kern/vfs_vnops.c:645 Process 16539 (bash) thread 0xcde99b80 (1816159) exclusive lockmgr bufwait (bufwait) r = 0 (0xe3390da0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr ufs (ufs) r = 0 (0xcd7d6724) locked @ ufs/ffs/ffs_vfsops.c:1690 Process 36056 (top) thread 0xccc6c8a0 (1301064) exclusive sleep mutex vm object (standard object) r = 0 (0xd76da128) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe31c5e40) locked @ kern/vfs_bio.c:1905 shared lockmgr ufs (ufs) r = 0 (0xcf34edf0) locked @ kern/vfs_vnops.c:562 Process 36054 (tail) thread 0xcc07c000 (100105) shared lockmgr ufs (ufs) r = 0 (0xcc311880) locked @ kern/vfs_subr.c:2159 Process 1201 (watchdogd) thread 0xcc3ad2e0 (100130) exclusive sleep mutex vm object (standard object) r = 0 (0xd05fa4a0) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc6fcb28) locked @ vm/vm_map.c:3723 Process 1153 (sendmail) thread 0xcae268a0 (100118) exclusive sleep mutex vm object (standard object) r = 0 (0xd0eaecb8) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc046218) locked @ vm/vm_map.c:3723 Process 1150 (sendmail) thread 0xcc097b80 (100103) exclusive sleep mutex vm object (standard object) r = 0 (0xd7585cb8) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc4ef5b8) locked @ vm/vm_map.c:3723 Process 961 (syslogd) thread 0xcbee68a0 (100113) exclusive lockmgr bufwait (bufwait) r = 0 (0xe3300f00) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3183fa0) locked @ kern/vfs_bio.c:2652 exclusive lockmgr snaplk (snaplk) r = 0 (0xcf75d61c) locked @ ufs/ffs/ffs_snapshot.c:2297 exclusive lockmgr ufs (ufs) r = 0 (0xd403a1b4) locked @ kern/vfs_syscalls.c:3587 Process 19 (softdepflush) thread 0xcbee88a0 (100078) exclusive sleep mutex vm object (standard object) r = 0 (0xcc312de0) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe31e9080) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3348560) locked @ kern/vfs_bio.c:1905 Process 7 (pagedaemon) thread 0xcae245c0 (100072) exclusive sleep mutex vm page queue free mutex (vm page queue free mutex) r = 0 (0xc130d200) locked @ vm/vm_pageout.c:1564 db:0:alllocks> show lockedvnods Locked vnodes 0xcf34ed98: tag ufs, type VREG usecount 1, writecount 0, refcount 2 mountedhere 0 flags (VI(0x200)) v_object 0xd76da128 ref 0 pages 0 lock type ufs: SHARED (count 1) #0 0xc0a6bf82 at __lockmgr_args+0x662 #1 0xc0ce2b51 at ffs_lock+0xa1 #2 0xc0e276d3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37e98 at _vn_lock+0x78 #4 0xc0b39396 at vn_read+0x456 #5 0xc0ad6efe at dofileread+0x9e #6 0xc0ad70d8 at kern_readv+0x68 #7 0xc0ad71ef at sys_read+0x4f #8 0xc0e0091e at syscall+0x2fe #9 0xc0deabe1 at Xint0x80_syscall+0x21 ino 23968, on dev ada0s1a 0xcd7d66cc: tag ufs, type VDIR usecount 1, writecount 0, refcount 2 mountedhere 0 flags (VI(0x200)) v_object 0xd597fb90 ref 0 pages 0 lock type ufs: EXCL by thread 0xcde99b80 (pid 16539, bash, tid 1816159) #0 0xc0a6c68d at __lockmgr_args+0xd6d #1 0xc0cdbea9 at ffs_vgetf+0x189 #2 0xc0cdc30e at ffs_vget+0x2e #3 0xc0ceafb4 at ufs_lookup_ino+0xbd4 #4 0xc0ceb05a at ufs_lookup+0x2a #5 0xc0e24e60 at VOP_CACHEDLOOKUP_APV+0x100 #6 0xc0b12546 at vfs_cache_lookup+0xe6 #7 0xc0e28d18 at VOP_LOOKUP_APV+0x108 #8 0xc0b1a50b at lookup+0x6eb #9 0xc0b1b56f at namei+0x6ff #10 0xc0b2e38f at kern_statat_vnhook+0xbf #11 0xc0b2e58c at kern_statat+0x3c #12 0xc0b2e6d6 at kern_stat+0x36 #13 0xc0b2e77f at sys_stat+0x2f #14 0xc0e0091e at syscall+0x2fe #15 0xc0deabe1 at Xint0x80_syscall+0x21 ino 212895, on dev ada0s1f 0xcc311828: tag ufs, type VDIR usecount 1, writecount 0, refcount 2 mountedhere 0 flags (VI(0x200)) v_object 0xcc0d86f0 ref 0 pages 0 lock type ufs: SHARED (count 1) #0 0xc0a6bf82 at __lockmgr_args+0x662 #1 0xc0ce2b51 at ffs_lock+0xa1 #2 0xc0e276d3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37e98 at _vn_lock+0x78 #4 0xc0b2aa2b at vget+0xbb #5 0xc0b12335 at cache_lookup+0x895 #6 0xc0b1251d at vfs_cache_lookup+0xbd #7 0xc0e28d18 at VOP_LOOKUP_APV+0x108 #8 0xc0b1a50b at lookup+0x6eb #9 0xc0b1b56f at namei+0x6ff #10 0xc0b2e38f at kern_statat_vnhook+0xbf #11 0xc0b2e58c at kern_statat+0x3c #12 0xc0b2e6d6 at kern_stat+0x36 #13 0xc0b2e77f at sys_stat+0x2f #14 0xc0e0091e at syscall+0x2fe #15 0xc0deabe1 at Xint0x80_syscall+0x21 ino 6052864, on dev ada0s1e 0xd403a15c: tag ufs, type VREG usecount 2, writecount 1, refcount 5 mountedhere 0 flags (VI(0x200)) v_object 0xcf0481bc ref 0 pages 2 lock type ufs: EXCL by thread 0xcbee68a0 (pid 961, syslogd, tid 100113) with shared waiters pending #0 0xc0a6c68d at __lockmgr_args+0xd6d #1 0xc0ce2b51 at ffs_lock+0xa1 #2 0xc0e276d3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37e98 at _vn_lock+0x78 #4 0xc0b36036 at sys_fsync+0x116 #5 0xc0e0091e at syscall+0x2fe #6 0xc0deabe1 at Xint0x80_syscall+0x21 ino 6052880, on dev ada0s1e 0xccbbf984: tag ufs, type VREG usecount 1, writecount 0, refcount 48 mountedhere 0 flags (VV_SYSTEM|VI(0x200)) v_object 0xd72b1378 ref 0 pages 1856 lock type snaplk: EXCL by thread 0xcbee68a0 (pid 961, syslogd, tid 100113) #0 0xc0a6c68d at __lockmgr_args+0xd6d #1 0xc0cbf452 at ffs_copyonwrite+0x2c2 #2 0xc0cdc47b at ffs_geom_strategy+0xeb #3 0xc0cf11b3 at ufs_strategy+0xd3 #4 0xc0e273a9 at VOP_STRATEGY_APV+0x119 #5 0xc0b09321 at bufstrategy+0x61 #6 0xc0b0cdb9 at bufwrite+0x159 #7 0xc0b0d128 at vfs_bio_awrite+0x328 #8 0xc0ce2fb5 at ffs_syncvnode+0x3a5 #9 0xc0ce320f at ffs_fsync+0x2f #10 0xc0e23e10 at VOP_FSYNC_APV+0x100 #11 0xc0b360ff at sys_fsync+0x1df #12 0xc0e0091e at syscall+0x2fe #13 0xc0deabe1 at Xint0x80_syscall+0x21 ino 23646, on dev ada0s1e 0xcd28cc3c: tag ufs, type VREG usecount 1, writecount 1, refcount 7018 mountedhere 0 flags (VI(0x200)) v_object 0xce00aa68 ref 0 pages 181984 lock type ufs: EXCL by thread 0xd17e92e0 (pid 46945, dd, tid 1301061) #0 0xc0a6c68d at __lockmgr_args+0xd6d #1 0xc0ce2b51 at ffs_lock+0xa1 #2 0xc0e276d3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37e98 at _vn_lock+0x78 #4 0xc0b38ce6 at vn_write+0x176 #5 0xc0ad6bf5 at dofilewrite+0x95 #6 0xc0ad7268 at kern_writev+0x68 #7 0xc0ad737f at sys_write+0x4f #8 0xc0e0091e at syscall+0x2fe #9 0xc0deabe1 at Xint0x80_syscall+0x21 ino 6194266, on dev ada0s1e db:0:lockedvnods> show mount 0xcbfd0b70 /dev/ad4s1a on / (ufs) 0xcbfd1000 devfs on /dev (devfs) 0xcc31a5b8 /dev/ad4s1f on /home (ufs) 0xcc31a000 /dev/ad4s1d on /usr (ufs) 0xcbfd1b70 /dev/ad4s1e on /var (ufs) 0xcbfd1894 procfs on /proc (procfs) 0xcbfd0894 /dev/label/tmp on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 46968 1153 1153 25 RL sendmail 46967 1150 1150 0 RL sendmail 46966 1158 1158 0 RL cron 46965 1201 1201 0 RL CPU 3 watchdogd 46964 36057 36055 1001 RL CPU 2 sh 46946 46276 38014 0 S+ piperd 0xccad37d0 egrep 46945 46276 38014 0 R+ CPU 0 dd 46276 38014 38014 0 S+ wait 0xcc1fc598 sh 51708 0 0 0 DL crypto_r 0xd43f7ccc [crypto returns] 51707 0 0 0 DL crypto_w 0xd43f7ca8 [crypto] 38014 1299 38014 0 S+ wait 0xd637ab30 sh 16539 16538 16539 1001 Rs+ bash 16538 16536 16536 1001 S select 0xd0832764 sshd 16536 1147 16536 0 Ss select 0xcce18d24 sshd 36058 36055 36055 1001 S piperd 0xccdd6320 awk 36057 36055 36055 1001 RL sh 36056 36051 36056 1001 Rs+ top 36055 36053 36055 1001 SWs wait 0xcc9db598 sh 36054 36052 36054 1001 Ds ufs 0xd403a1b4 tail 36053 36045 36045 1001 S select 0xcc067524 sshd 36052 36046 36046 1001 S select 0xce2096a4 sshd 36051 36047 36047 1001 S select 0xccbb5ca4 sshd 36047 1147 36047 0 Ss select 0xcf0d3ea4 sshd 36046 1147 36046 0 Ss select 0xce08d7e4 sshd 36045 1147 36045 0 Ss select 0xcbfcf424 sshd 3841 0 0 0 DL (threaded) [zfskern] 100220 D l2arc_fe 0xd8510884 [l2arc_feed_thread] 100193 D arc_recl 0xd85016fc [arc_reclaim_thread] 1299 1296 1299 0 SW+ wait 0xcc0722cc bash 1296 1295 1296 0 SW+ pause 0xcc0ac5f0 csh 1295 1290 1295 1001 SW+ wait 0xcc950864 su 1290 1289 1290 1001 SWs+ wait 0xcc095864 bash 1289 1285 1285 1001 S select 0xcbfcf8a4 sshd 1285 1147 1285 0 Ss select 0xcc040de4 sshd 1251 1 1251 0 Ss+ ttyin 0xcae33870 getty 1250 1 1250 0 Ss+ ttyin 0xcae33a70 getty 1249 1 1249 0 Ss+ ttyin 0xc970d270 getty 1248 1 1248 0 Ss+ ttyin 0xc970d470 getty 1247 1 1247 0 Ss+ ttyin 0xc970d670 getty 1246 1 1246 0 Ss+ ttyin 0xc970d870 getty 1245 1 1245 0 Ss+ ttyin 0xc970da70 getty 1244 1 1244 0 Ss+ ttyin 0xc970dc70 getty 1243 1 1243 0 Rs+ getty 1226 1 1226 0 Ss select 0xcac861a4 inetd 1201 1 1201 0 LLs *vm page 0xcc695380 watchdogd 1158 1 1158 0 RLs cron 1153 1 1153 25 RLs sendmail 1150 1 1150 0 RLs CPU 1 sendmail 1147 1 1147 0 Ss select 0xcac7ada4 sshd 1115 1 1115 0 Ss select 0xcac86564 moused 1091 1 1091 0 Ss select 0xcc0416a4 ntpd 1035 1031 1031 0 S (threaded) nfsd 100125 S rpcsvc 0xcc008b90 nfsd: service 100124 S rpcsvc 0xcc008bd0 nfsd: service 100123 S rpcsvc 0xcc008c10 nfsd: service 100092 S rpcsvc 0xcaca9e90 nfsd: master 1031 1 1031 0 Ss select 0xcac865e4 nfsd 1029 1025 1025 0 S select 0xcbfcf3a4 nfsuserd 1028 1025 1025 0 S select 0xcc040064 nfsuserd 1027 1025 1025 0 S select 0xcbfcf3e4 nfsuserd 1026 1025 1025 0 S select 0xcc0400a4 nfsuserd 1025 1 1025 0 SWs pause 0xcc0ab5f0 nfsuserd 1009 1 1009 0 Ss select 0xcc011764 mountd 976 1 976 0 Ss select 0xc955f124 rpcbind 961 1 961 0 Rs syslogd 838 1 838 0 Ss select 0xcbfcf024 devd 19 0 0 0 RL [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc1304bb4 [syncer] 16 0 0 0 DL psleep 0xc1304848 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e76c [pagezero] 8 0 0 0 DL psleep 0xc130e348 [vmdaemon] 7 0 0 0 RL [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151354 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc13080d8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 D - 0xcacfed04 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d7e4 [yarrow] 13 0 0 0 DL (threaded) [geom] 100016 D - 0xc118a8a8 [g_down] 100015 D - 0xc118a8a4 [g_up] 100014 D - 0xc118a89c [g_event] 12 0 0 0 WL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 CanRun [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 CanRun [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130c000 [audit] 0 0 0 0 DLs (threaded) [kernel] 1818774 D - 0xd8396740 [aiod_bio taskq] 100219 D - 0xd8396840 [system_taskq_3] 100218 D - 0xd8396840 [system_taskq_2] 100217 D - 0xd8396840 [system_taskq_1] 100216 D - 0xd8396840 [system_taskq_0] 100069 D - 0xc118d7e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118aa60 [swapper] db:0:ps> allt Tracing command sendmail pid 46968 tid 1816147 td 0xcd9c6b80 sched_switch(cd9c6b80,0,103,19e,936fd071,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,7f8,...) at mi_switch+0x1ff turnstile_wait(cc3aba80,cc01b5c0,0,1c5,0,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d580,cd9c6b80,0,c100f927,4fe,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d580,0,c100f927,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(d7e5a6f0,28,0,0,f4de5b64,...) at vm_page_alloc+0x565 vm_fault_hold(d5d78cb0,811c000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(d5d78cb0,811c000,2,0,811c750,...) at vm_fault+0x8b trap_pfault(c0dffe2f,0,cd9c6b80,d5e03598,d5e03598,...) at trap_pfault+0x1b9 trap(f4de5d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a20ec, esp = 0xbfbfcf10, ebp = 0xbfbfcf78 --- Tracing command sendmail pid 46967 tid 1816058 td 0xcc9b02e0 sched_switch(cc9b02e0,0,103,19e,93cfcb21,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,8d0,...) at mi_switch+0x1ff turnstile_wait(cd915b80,cbee68a0,0,1c5,0,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130de00,cc9b02e0,0,c100f927,4fe,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130de00,0,c100f927,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(d53f1534,28,0,0,f4c8eb64,...) at vm_page_alloc+0x565 vm_fault_hold(c95e5bc8,811c000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(c95e5bc8,811c000,2,0,811c750,...) at vm_fault+0x8b trap_pfault(c0dffe2f,0,cc9b02e0,d5d62864,d5d62864,...) at trap_pfault+0x1b9 trap(f4c8ed28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a20ec, esp = 0xbfbfc3c0, ebp = 0xbfbfc428 --- Tracing command cron pid 46966 tid 100394 td 0xccb27b80 sched_switch(ccb27b80,0,104,1a3,93d07920,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(ccb27b80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cf7c,78,c100fbed,0,0,...) at sleepq_wait+0x63 _sleep(c130cf7c,c130d200,278,c100fbed,0,...) at _sleep+0x39c vm_waitpfault(cd0b5094,1e,0,0,f441fb64,...) at vm_waitpfault+0x75 vm_fault_hold(d60a6e80,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(d60a6e80,bfbfe000,2,0,bfbfec10,...) at vm_fault+0x8b trap_pfault(c0dffe2f,0,ccb27b80,cd2da598,cd2da598,...) at trap_pfault+0x1b9 trap(f441fd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c0e7, esp = 0xbfbfec10, ebp = 0xbfbfec48 --- Tracing command watchdogd pid 46965 tid 100126 td 0xcc01b5c0 cpustop_handler(3,f40f58f4,c0e00e13,c93e1c80,f40f5844,...) at cpustop_handler+0x26 ipi_nmi_handler(c93e1c80,f40f5844,c0a74363,c93e1c80,cc950000,...) at ipi_nmi_handler+0x35 trap(f40f5900) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a74c87, esp = 0xf40f5940, ebp = 0xf40f5994 --- _mtx_lock_sleep(c130df00,cc01b5c0,0,c100f927,4fe,...) at _mtx_lock_sleep+0x207 _mtx_lock_flags(c130df00,0,c100f927,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(d0779cb8,1e,0,0,f40f5b64,...) at vm_page_alloc+0x565 vm_fault_hold(d5b2c000,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(d5b2c000,bfbfe000,2,0,bfbfec38,...) at vm_fault+0x8b trap_pfault(c0dffe2f,0,cc01b5c0,cc950000,cc950000,...) at trap_pfault+0x1b9 trap(f40f5d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bbe76, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command sh pid 46964 tid 1815941 td 0xccd688a0 kdb_enter(c0e46ce9,c0fd687e,f4add7e4,c08fb51d,c95a7200,...) at kdb_enter+0x3a kdb_break(c95a7200,c1bb7600,162,1000,f4add7f8,...) at kdb_break+0x28 uart_intr(c95a7200,c1bb7608,c1bb7608,f4add824,0,...) at uart_intr+0x12d intr_event_handle(c95aab80,f4add850,101,cc3ad2e0,23f,...) at intr_event_handle+0x76 intr_execute_handlers(c96210d0,f4add850,2,f4add8c4,c0deaf45,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,f4add850) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0a749b0, esp = 0xf4add890, ebp = 0xf4add8c4 --- _mtx_lock_spin(c11aab94,ccd688a0,0,c0fd8cb0,222,...) at _mtx_lock_spin+0x130 _mtx_lock_spin_flags(c11aab94,0,c0fd8cb0,222,c130d200,...) at _mtx_lock_spin_flags+0x165 turnstile_trywait(c130d200,c12c12bc,c9582178,ccd68950,0,...) at turnstile_trywait+0x3f _mtx_lock_sleep(c130d200,ccd688a0,0,c100f927,75c,...) at _mtx_lock_sleep+0x218 _mtx_lock_flags(c130d200,0,c100f927,75c,c2aefa40,...) at _mtx_lock_flags+0x10f vm_page_free_toq(c2aefa40,f4add9e8,c0d17413,c2aefa40,0,...) at vm_page_free_toq+0x17e vm_page_free(c2aefa40,0,c100f927,4fe,0,...) at vm_page_free+0x15 vm_page_alloc(d7a85784,1e,0,0,f4addb64,...) at vm_page_alloc+0x583 vm_fault_hold(cc494cb0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc494cb0,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(c0dffe2f,0,ccd688a0,d5d22b30,d5d22b30,...) at trap_pfault+0x1b9 trap(f4addd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- Tracing command egrep pid 46946 tid 1815013 td 0xd7c605c0 sched_switch(d7c605c0,0,104,1a3,f2f6f5bc,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(d7c605c0,0,c0fd82c0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ccad37d0,5c,c0fdac47,100,0,...) at sleepq_wait_sig+0x17 _sleep(ccad37d0,ccad3948,15c,c0fdac47,0,...) at _sleep+0x385 pipe_read(ccadb268,f4348c3c,cbfdbe00,0,d7c605c0,...) at pipe_read+0x427 dofileread(f4348c3c,ffffffff,ffffffff,0,f4348c18,...) at dofileread+0x9e kern_readv(d7c605c0,0,f4348c3c,f4348c5c,1,...) at kern_readv+0x68 sys_read(d7c605c0,f4348cec,c1025d9c,c0fdb13a,c1027350,...) at sys_read+0x4f syscall(f4348d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281b87c3, esp = 0xbfbfe81c, ebp = 0xbfbfe8a8 --- Tracing command dd pid 46945 tid 1301061 td 0xd17e92e0 cpustop_handler(0,f45f16b8,c0e00e13,c1825c80,f45f1608,...) at cpustop_handler+0x26 ipi_nmi_handler(c1825c80,f45f1608,c0a74363,c1825c80,cc169598,...) at ipi_nmi_handler+0x35 trap(f45f16c4) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a74c62, esp = 0xf45f1704, ebp = 0xf45f1758 --- _mtx_lock_sleep(c130df00,d17e92e0,0,c100f927,4fe,...) at _mtx_lock_sleep+0x1e2 _mtx_lock_flags(c130df00,0,c100f927,4fe,20,...) at _mtx_lock_flags+0x10f vm_page_alloc(ce00aa68,2c680,0,40222,40222,...) at vm_page_alloc+0x565 vm_page_grab(ce00aa68,2c680,0,412a2,c12c1f80,...) at vm_page_grab+0x152 allocbuf(e334fd40,4000,c0fdf847,b00,ffffffe0,...) at allocbuf+0x672 getblk(cd28cc3c,b1a0,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(cd28cc3c,2c680000,0,4000,cbfdbe00,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f45f1ba4,c102d483,cd28cc3c,cd28cc3c,f45f1b5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c11378c0,f45f1ba4,cd28cc3c,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cc988c40,f45f1c3c,cbfdbe00,0,d17e92e0,...) at vn_write+0x2bc dofilewrite(f45f1c3c,ffffffff,ffffffff,0,f45f1c18,...) at dofilewrite+0x95 kern_writev(d17e92e0,4,f45f1c3c,f45f1c5c,1,...) at kern_writev+0x68 sys_write(d17e92e0,f45f1cec,c1025d9c,c0e3242f,c119bcc0,...) at sys_write+0x4f syscall(f45f1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x281757a3, esp = 0xbfbfe9bc, ebp = 0xbfbfe9e8 --- Tracing command sh pid 46276 tid 1301180 td 0xd7c09b80 sched_switch(d7c09b80,0,104,1a3,ef7c0b6b,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(d7c09b80,0,c0fd82c0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc1fc598,6c,c0fdb1f8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc1fc598,cc1fc620,16c,c0fdb1f8,0,...) at _sleep+0x385 kern_wait(d7c09b80,ffffffff,f4036c58,2,0,...) at kern_wait+0x597 sys_wait4(d7c09b80,f4036cec,c1025d9c,c0fdb1d7,c119b640,...) at sys_wait4+0x3b syscall(f4036d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2814f6eb, esp = 0xbfbfe78c, ebp = 0xbfbfe7a8 --- Tracing command crypto returns pid 51708 tid 1816116 td 0xcd4d75c0 sched_switch(cd4d75c0,0,104,1a3,5845bb07,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cd4d75c0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(d43f7ccc,6c,d43f6721,0,0,...) at sleepq_wait+0x63 _sleep(d43f7ccc,d43f7cdc,6c,d43f6721,0,...) at _sleep+0x39c crypto_ret_proc(0,f4d69d28,c0fcc8d0,3d8,d5e17864,...) at crypto_ret_proc+0x18e fork_exit(d43d9a00,0,f4d69d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4d69d60, ebp = 0 --- Tracing command crypto pid 51707 tid 1300973 td 0xccea78a0 sched_switch(ccea78a0,0,104,1a3,584513b7,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(ccea78a0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(d43f7ca8,6c,d43f6970,0,0,...) at sleepq_wait+0x63 _sleep(d43f7ca8,d43f7cb8,6c,d43f6970,0,...) at _sleep+0x39c crypto_proc(0,f4478d28,c0fcc8d0,3d8,d094c864,...) at crypto_proc+0x1b0 fork_exit(d43dadc0,0,f4478d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4478d60, ebp = 0 --- Tracing command sh pid 38014 tid 1816853 td 0xd63bc000 sched_switch(d63bc000,0,104,1a3,b147c91,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(d63bc000,0,c0fd82c0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(d637ab30,6c,c0fdb1f8,100,0,...) at sleepq_wait_sig+0x17 _sleep(d637ab30,d637abb8,16c,c0fdb1f8,0,...) at _sleep+0x385 kern_wait(d63bc000,ffffffff,f594ac58,2,0,...) at kern_wait+0x597 sys_wait4(d63bc000,f594acec,c1025d9c,c0fdb1d7,c119b640,...) at sys_wait4+0x3b syscall(f594ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2814f6eb, esp = 0xbfbfe70c, ebp = 0xbfbfe728 --- Tracing command bash pid 16539 tid 1816159 td 0xcde99b80 sched_switch(cde99b80,0,104,1a3,93a5d221,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cde99b80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cf7c,54,c100fbfb,0,0,...) at sleepq_wait+0x63 _sleep(c130cf7c,c130d200,254,c100fbfb,0,...) at _sleep+0x39c vm_wait(d597fb90,0,c100f927,8f5,10222,...) at vm_wait+0xbe vm_page_grab(d597fb90,0,0,112a2,c12ce048,...) at vm_page_grab+0x180 allocbuf(e3390d40,800,c0fdf847,b00,d597fb90,...) at allocbuf+0x672 getblk(cd7d66cc,0,0,800,0,...) at getblk+0x5e3 breadn_flags(cd7d66cc,0,0,800,0,...) at breadn_flags+0x43 ffs_blkatoff(cd7d66cc,0,0,0,f4e158ec,...) at ffs_blkatoff+0xf1 ufs_lookup_ino(cd7d66cc,f4e15af8,f4e15b0c,0,f4e1593c,...) at ufs_lookup_ino+0x398 ufs_lookup(f4e15964,c102cc73,c0b14c7a,cd7d6724,f4e15954,...) at ufs_lookup+0x2a VOP_CACHEDLOOKUP_APV(c11378c0,f4e15964,f4e15b0c,0,0,...) at VOP_CACHEDLOOKUP_APV+0x100 vfs_cache_lookup(f4e15a0c,c0fe3148,c0b29a3f,c11378c0,f4e159a0,...) at vfs_cache_lookup+0xe6 VOP_LOOKUP_APV(c11378c0,f4e15a0c,f4e15b0c,343,c12ce028,...) at VOP_LOOKUP_APV+0x108 lookup(f4e15acc,c0fe17cc,10e,dc,cc93a8ec,...) at lookup+0x6eb namei(f4e15acc,885f000,0,2,cde99b80,...) at namei+0x6ff kern_statat_vnhook(cde99b80,0,ffffff9c,885b880,0,...) at kern_statat_vnhook+0xbf kern_statat(cde99b80,0,ffffff9c,885b880,0,...) at kern_statat+0x3c kern_stat(cde99b80,885b880,0,f4e15bfc,1,...) at kern_stat+0x36 sys_stat(cde99b80,f4e15cec,c1025d9c,c0fdb0af,c1027350,...) at sys_stat+0x2f syscall(f4e15d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (188, FreeBSD ELF32, sys_stat), eip = 0x2833aa13, esp = 0xbfbfe7dc, ebp = 0xbfbfe858 --- Tracing command sshd pid 16538 tid 1300996 td 0xccf60000 sched_switch(ccf60000,0,104,1a3,81b4379,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccf60000,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,d0832750,0,c0fd08e7,ccf60000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(d0832764,0,f44fda50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(d0832764,d0832750,c0fdaa18,6ae,ccae3000,...) at _cv_wait_sig+0x260 seltdwait(ccae3000,58,cc03ed00,ccf60000,ccf600b0,...) at seltdwait+0xa2 kern_select(ccf60000,c,8810238,8810220,0,0,20,cef1e864,0) at kern_select+0x655 sys_select(ccf60000,f44fdcec,c1025d9c,c0fdb345,c119c340,...) at sys_select+0x66 syscall(f44fdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284d8743, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 16536 tid 1300900 td 0xcccf1b80 sched_switch(cccf1b80,0,104,1a3,e93f57c9,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cccf1b80,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cce18d10,0,c0fd08e7,cccf1b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cce18d24,0,f431bab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cce18d24,cce18d10,c0fdaa18,6ae,f431bb3c,...) at _cv_wait_sig+0x260 seltdwait(cc5c172c,c0fdaa18,5ab,cccf1b80,696,...) at seltdwait+0xa2 sys_poll(cccf1b80,f431bcec,c1025d9c,c0fbccd1,cc01e8a0,...) at sys_poll+0x321 syscall(f431bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284700eb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command awk pid 36058 tid 1301068 td 0xccc842e0 sched_switch(ccc842e0,0,104,1a3,5eb1195d,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(ccc842e0,0,c0fd82c0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ccdd6320,5c,c0fdac47,100,0,...) at sleepq_wait_sig+0x17 _sleep(ccdd6320,ccdd6498,15c,c0fdac47,0,...) at _sleep+0x385 pipe_read(ccae8a80,f460dc3c,cc03e480,0,ccc842e0,...) at pipe_read+0x427 dofileread(f460dc3c,ffffffff,ffffffff,0,f460dc18,...) at dofileread+0x9e kern_readv(ccc842e0,0,f460dc3c,f460dc5c,1,...) at kern_readv+0x68 sys_read(ccc842e0,f460dcec,c1025d9c,c0e3242f,c1027350,...) at sys_read+0x4f syscall(f460dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281aa7c3, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command sh pid 36057 tid 1301065 td 0xccc6cb80 sched_switch(ccc6cb80,0,104,1a3,93cbf6b9,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(ccc6cb80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cf7c,78,c100fbed,0,0,...) at sleepq_wait+0x63 _sleep(c130cf7c,c130d200,278,c100fbed,0,...) at _sleep+0x39c vm_waitpfault(d687dcb8,1e,0,0,f4601b64,...) at vm_waitpfault+0x75 vm_fault_hold(d01ca0e8,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(d01ca0e8,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(cc168864,f4601c64,c0acc488,ccc6cb80,cc168864,...) at trap_pfault+0x1b9 trap(f4601d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- Tracing command top pid 36056 tid 1301064 td 0xccc6c8a0 sched_switch(ccc6c8a0,0,103,19e,93aaed7c,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,900,...) at mi_switch+0x1ff turnstile_wait(d99f9400,ccb27b80,0,1c5,0,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130e000,ccc6c8a0,0,c100f927,4fe,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130e000,0,c100f927,4fe,20,...) at _mtx_lock_flags+0x10f vm_page_alloc(d76da128,0,0,40222,40222,...) at vm_page_alloc+0x565 vm_page_grab(d76da128,0,0,412a2,c12cf7a8,...) at vm_page_grab+0x152 allocbuf(e31c5de0,4000,c0fdf847,b00,d9d4,...) at allocbuf+0x672 getblk(cf34ed98,0,0,4000,0,...) at getblk+0x5e3 cluster_read(cf34ed98,a000,0,0,0,...) at cluster_read+0xfa ffs_read(f45fdba0,c102ca87,200400,c0fe3cd0,232,...) at ffs_read+0x369 VOP_READ_APV(c11378c0,f45fdba0,cf34ed98,232,ccfed080,...) at VOP_READ_APV+0x100 vn_read(ccae8cb0,f45fdc3c,ccfed080,0,ccc6c8a0,...) at vn_read+0x256 dofileread(f45fdc3c,ffffffff,ffffffff,0,f45fdc18,...) at dofileread+0x9e kern_readv(ccc6c8a0,3,f45fdc3c,f45fdc5c,1,...) at kern_readv+0x68 sys_read(ccc6c8a0,f45fdcec,c1025d9c,c0fdb13a,f45fdcb4,...) at sys_read+0x4f syscall(f45fdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281df7c3, esp = 0xbfbfe5cc, ebp = 0xbfbfe6e8 --- Tracing command sh pid 36055 tid 100136 td 0xcc3acb80 Tracing command tail pid 36054 tid 100105 td 0xcc07c000 sched_switch(cc07c000,0,104,1a3,bdae6b02,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cc07c000,0,c0fd82c0,268,60,...) at sleepq_switch+0x162 sleepq_wait(d403a1b4,60,c0fc2884,4,1,...) at sleepq_wait+0x63 __lockmgr_args(d403a1b4,200000,d403a220,0,0,...) at __lockmgr_args+0x593 ffs_lock(f40328a4,c0ad499c,c13048fc,c110a3c0,2a1,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11378c0,f40328a4,cc07c0b0,c1149a20,d403a15c,...) at VOP_LOCK1_APV+0xf3 _vn_lock(d403a15c,200100,c0fe23f1,86f,29d,...) at _vn_lock+0x78 vget(d403a15c,200100,cc07c000,29d,cc311828,...) at vget+0xbb cache_lookup(cc311828,f4032af8,f4032b0c,0,0,...) at cache_lookup+0x895 vfs_cache_lookup(f4032a0c,c0fe3148,c0b29a3f,c11378c0,f40329a0,...) at vfs_cache_lookup+0xbd VOP_LOOKUP_APV(c11378c0,f4032a0c,f4032b0c,343,c12d5458,...) at VOP_LOOKUP_APV+0x108 lookup(f4032acc,c0fe17cc,10e,dc,c12d5458,...) at lookup+0x6eb namei(f4032acc,0,d6077288,f4032ad8,c0a4e2c4,...) at namei+0x6ff kern_statat_vnhook(cc07c000,0,ffffff9c,8809020,0,...) at kern_statat_vnhook+0xbf kern_statat(cc07c000,0,ffffff9c,8809020,0,...) at kern_statat+0x3c kern_stat(cc07c000,8809020,0,f4032bfc,d5,...) at kern_stat+0x36 sys_stat(cc07c000,f4032cec,c1025d9c,c0fdbb29,14,...) at sys_stat+0x2f syscall(f4032d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (188, FreeBSD ELF32, sys_stat), eip = 0x28164a13, esp = 0xbfbfeb0c, ebp = 0xbfbfebc8 --- Tracing command sshd pid 36053 tid 1301054 td 0xccd855c0 sched_switch(ccd855c0,0,104,1a3,1a673f09,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccd855c0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc067510,0,c0fd08e7,ccd855c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc067524,0,f45d5a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc067524,cc067510,c0fdaa18,6ae,cc08f0e0,...) at _cv_wait_sig+0x260 seltdwait(cc08f0e0,58,cc03ee80,ccd855c0,ccd85670,...) at seltdwait+0xa2 kern_select(ccd855c0,c,8810238,8810220,0,0,20,cc16ab30,0) at kern_select+0x655 sys_select(ccd855c0,f45d5cec,c1025d9c,c0fdb345,c119b640,...) at sys_select+0x66 syscall(f45d5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284d8743, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 36052 tid 1301026 td 0xcd0ccb80 sched_switch(cd0ccb80,0,104,1a3,52d6db3,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd0ccb80,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,ce209690,0,c0fd08e7,cd0ccb80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ce2096a4,0,f4565a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(ce2096a4,ce209690,c0fdaa18,6ae,cc02d5e8,...) at _cv_wait_sig+0x260 seltdwait(cc02d5e8,58,ce5e4380,cd0ccb80,cd0ccc30,...) at seltdwait+0xa2 kern_select(cd0ccb80,c,8810238,8810220,0,0,20,cc162864,0) at kern_select+0x655 sys_select(cd0ccb80,f4565cec,c1025d9c,c0fdb345,c1027350,...) at sys_select+0x66 syscall(f4565d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284d8743, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 36051 tid 1300851 td 0xcccfc000 sched_switch(cccfc000,0,104,1a3,ed10dbfa,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cccfc000,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,ccbb5c90,0,c0fd08e7,cccfc000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ccbb5ca4,0,f425ca50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(ccbb5ca4,ccbb5c90,c0fdaa18,6ae,cc9880a8,...) at _cv_wait_sig+0x260 seltdwait(cc9880a8,58,da2e8900,cccfc000,cccfc0b0,...) at seltdwait+0xa2 kern_select(cccfc000,a,8810238,8810220,0,0,20,cf600598,0) at kern_select+0x655 sys_select(cccfc000,f425ccec,c1025d9c,c0fdb345,f425ccb4,...) at sys_select+0x66 syscall(f425cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284d8743, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 36047 tid 1301052 td 0xd99f4000 sched_switch(d99f4000,0,104,1a3,482f82f7,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d99f4000,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cf0d3e90,0,c0fd08e7,d99f4000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cf0d3ea4,0,f45cdab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cf0d3ea4,cf0d3e90,c0fdaa18,6ae,f45cdb3c,...) at _cv_wait_sig+0x260 seltdwait(cfd0ee2c,c0fdaa18,5ab,d99f4000,0,...) at seltdwait+0xa2 sys_poll(d99f4000,f45cdcec,c1025d9c,c0e3242f,c1027350,...) at sys_poll+0x321 syscall(f45cdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284700eb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 36046 tid 1300997 td 0xccf608a0 sched_switch(ccf608a0,0,104,1a3,2552e894,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccf608a0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,ce08d7d0,0,c0fd08e7,ccf608a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ce08d7e4,0,f4500ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(ce08d7e4,ce08d7d0,c0fdaa18,6ae,f4500b3c,...) at _cv_wait_sig+0x260 seltdwait(cfd48d2c,c0fdaa18,5ab,ccf608a0,0,...) at seltdwait+0xa2 sys_poll(ccf608a0,f4500cec,c1025d9c,c0e3242f,c1027350,...) at sys_poll+0x321 syscall(f4500d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284700eb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 36045 tid 1300861 td 0xccce92e0 sched_switch(ccce92e0,0,104,1a3,255b9dc8,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccce92e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cbfcf410,0,c0fd08e7,ccce92e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfcf424,0,f427eab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfcf424,cbfcf410,c0fdaa18,6ae,f427eb3c,...) at _cv_wait_sig+0x260 seltdwait(cfdbc52c,c0fdaa18,5ab,ccce92e0,0,...) at seltdwait+0xa2 sys_poll(ccce92e0,f427ecec,c1025d9c,c0e3242f,c119c340,...) at sys_poll+0x321 syscall(f427ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284700eb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command zfskern pid 3841 tid 100220 td 0xd1c07000 sched_switch(d1c07000,0,104,1a3,5171047c,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d1c07000,0,c0fd82c0,28b,d1c07000,...) at sleepq_switch+0x162 sleepq_timedwait(d8510884,0,f424bbb4,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(d8510884,d8510870,3e8,0,0,...) at _cv_timedwait+0x270 l2arc_feed_thread(0,f424bd28,c0fcc8d0,3d8,d16ff864,...) at l2arc_feed_thread+0x204 fork_exit(d83e28e0,0,f424bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf424bd60, ebp = 0 --- Tracing command zfskern pid 3841 tid 100193 td 0xd18d22e0 sched_switch(d18d22e0,0,104,1a3,93ff6a77,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d18d22e0,0,c0fd82c0,28b,d18d22e0,...) at sleepq_switch+0x162 sleepq_timedwait(d85016fc,0,f41fac68,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(d85016fc,d85016e8,3e8,0,b21d7596,...) at _cv_timedwait+0x270 arc_reclaim_thread(0,f41fad28,c0fcc8d0,3d8,d16ff864,...) at arc_reclaim_thread+0x28e fork_exit(d83e2480,0,f41fad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf41fad60, ebp = 0 --- Tracing command bash pid 1299 tid 100108 td 0xcc07b5c0 Tracing command csh pid 1296 tid 100095 td 0xcbee8000 Tracing command su pid 1295 tid 100150 td 0xcc3aa000 Tracing command bash pid 1290 tid 100094 td 0xcc09b000 Tracing command sshd pid 1289 tid 100111 td 0xcbee6b80 sched_switch(cbee6b80,0,104,1a3,1b238ecb,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee6b80,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cbfcf890,0,c0fd08e7,cbee6b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfcf8a4,0,f404da50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfcf8a4,cbfcf890,c0fdaa18,6ae,cc02d150,...) at _cv_wait_sig+0x260 seltdwait(cc02d150,58,cbfda200,cbee6b80,cbee6c30,...) at seltdwait+0xa2 kern_select(cbee6b80,c,8810238,8810220,0,0,20,cc0ac2cc,0) at kern_select+0x655 sys_select(cbee6b80,f404dcec,c1025d9c,c0fdb345,f404dcb4,...) at sys_select+0x66 syscall(f404dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284d8743, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1285 tid 100098 td 0xcc09ab80 sched_switch(cc09ab80,0,104,1a3,b6677a0b,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc09ab80,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc040dd0,0,c0fd08e7,cc09ab80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc040de4,0,f4016ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc040de4,cc040dd0,c0fdaa18,6ae,f4016b3c,...) at _cv_wait_sig+0x260 seltdwait(cc018a2c,c0fdaa18,5ab,cc09ab80,696,...) at seltdwait+0xa2 sys_poll(cc09ab80,f4016cec,c1025d9c,c0fbccd1,c1027350,...) at sys_poll+0x321 syscall(f4016d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284700eb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command getty pid 1251 tid 100101 td 0xcc09a2e0 sched_switch(cc09a2e0,0,104,1a3,8931b567,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc09a2e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc09a2e0,cc09a2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33870,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33870,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33800,cae33870,f4022c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33800,f4022c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27100,f4022c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02dab8,f4022c3c,c95d9b80,0,cc09a2e0,...) at devfs_read_f+0x7e dofileread(f4022c3c,ffffffff,ffffffff,0,f4022c18,...) at dofileread+0x9e kern_readv(cc09a2e0,0,f4022c3c,f4022c5c,1,...) at kern_readv+0x68 sys_read(cc09a2e0,f4022cec,c1025d9c,c0e371f2,cc01f2e0,...) at sys_read+0x4f syscall(f4022d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1250 tid 100132 td 0xcc98b8a0 sched_switch(cc98b8a0,0,104,1a3,891539b5,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc98b8a0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc98b8a0,cc98b8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33a70,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33a70,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33a00,cae33a70,f4107c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33a00,f4107c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27000,f4107c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08fbd0,f4107c3c,c95d9b80,0,cc98b8a0,...) at devfs_read_f+0x7e dofileread(f4107c3c,ffffffff,ffffffff,0,f4107c18,...) at dofileread+0x9e kern_readv(cc98b8a0,0,f4107c3c,f4107c5c,1,...) at kern_readv+0x68 sys_read(cc98b8a0,f4107cec,c1025d9c,c0e371f2,c1027350,...) at sys_read+0x4f syscall(f4107d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1249 tid 100133 td 0xcc98b5c0 sched_switch(cc98b5c0,0,104,1a3,89313b33,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc98b5c0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc98b5c0,cc98b5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d270,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d270,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d200,c970d270,f410ac3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d200,f410ac3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0ce00,f410ac3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc034658,f410ac3c,c95d9b80,0,cc98b5c0,...) at devfs_read_f+0x7e dofileread(f410ac3c,ffffffff,ffffffff,0,f410ac18,...) at dofileread+0x9e kern_readv(cc98b5c0,0,f410ac3c,f410ac5c,1,...) at kern_readv+0x68 sys_read(cc98b5c0,f410acec,c1025d9c,c0e371f2,cc3aab80,...) at sys_read+0x4f syscall(f410ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1248 tid 100083 td 0xcc01f2e0 sched_switch(cc01f2e0,0,104,1a3,892087af,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01f2e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc01f2e0,cc01f2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d470,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d470,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d400,c970d470,f3fd5c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d400,f3fd5c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cd00,f3fd5c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cbff7118,f3fd5c3c,c95d9b80,0,cc01f2e0,...) at devfs_read_f+0x7e dofileread(f3fd5c3c,ffffffff,ffffffff,0,f3fd5c18,...) at dofileread+0x9e kern_readv(cc01f2e0,0,f3fd5c3c,f3fd5c5c,1,...) at kern_readv+0x68 sys_read(cc01f2e0,f3fd5cec,c1025d9c,c0e371f2,c1027350,...) at sys_read+0x4f syscall(f3fd5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1247 tid 100134 td 0xcc98b2e0 sched_switch(cc98b2e0,0,104,1a3,8934b02d,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc98b2e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc98b2e0,cc98b2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d670,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d670,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d600,c970d670,f410dc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d600,f410dc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cc00,f410dc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02d070,f410dc3c,c95d9b80,0,cc98b2e0,...) at devfs_read_f+0x7e dofileread(f410dc3c,ffffffff,ffffffff,0,f410dc18,...) at dofileread+0x9e kern_readv(cc98b2e0,0,f410dc3c,f410dc5c,1,...) at kern_readv+0x68 sys_read(cc98b2e0,f410dcec,c1025d9c,c0e371f2,cc98ab80,...) at sys_read+0x4f syscall(f410dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1246 tid 100135 td 0xcc98b000 sched_switch(cc98b000,0,104,1a3,8924200b,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc98b000,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc98b000,cc98b000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d870,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d870,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d800,c970d870,f4110c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d800,f4110c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c900,f4110c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cbff69a0,f4110c3c,c95d9b80,0,cc98b000,...) at devfs_read_f+0x7e dofileread(f4110c3c,ffffffff,ffffffff,0,f4110c18,...) at dofileread+0x9e kern_readv(cc98b000,0,f4110c3c,f4110c5c,1,...) at kern_readv+0x68 sys_read(cc98b000,f4110cec,c1025d9c,c0e371f2,c1027350,...) at sys_read+0x4f syscall(f4110d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1245 tid 100138 td 0xcc98ab80 sched_switch(cc98ab80,0,104,1a3,89382b46,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc98ab80,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc98ab80,cc98ab80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970da70,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970da70,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970da00,c970da70,f4119c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970da00,f4119c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c800,f4119c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc988118,f4119c3c,c95d9b80,0,cc98ab80,...) at devfs_read_f+0x7e dofileread(f4119c3c,ffffffff,ffffffff,0,f4119c18,...) at dofileread+0x9e kern_readv(cc98ab80,0,f4119c3c,f4119c5c,1,...) at kern_readv+0x68 sys_read(cc98ab80,f4119cec,c1025d9c,c0e371f2,cc98b5c0,...) at sys_read+0x4f syscall(f4119d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1244 tid 100139 td 0xcc98a8a0 sched_switch(cc98a8a0,0,104,1a3,892b4a66,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc98a8a0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,cc98a8a0,cc98a8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970dc70,0,c0fdd080,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970dc70,c118c710,c0fdca58,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970dc00,c970dc70,f411cc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970dc00,f411cc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c700,f411cc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02d460,f411cc3c,c95d9b80,0,cc98a8a0,...) at devfs_read_f+0x7e dofileread(f411cc3c,ffffffff,ffffffff,0,f411cc18,...) at dofileread+0x9e kern_readv(cc98a8a0,0,f411cc3c,f411cc5c,1,...) at kern_readv+0x68 sys_read(cc98a8a0,f411ccec,c1025d9c,c0e371f2,c1027350,...) at sys_read+0x4f syscall(f411cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281877c3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1243 tid 100144 td 0xcc3aab80 sched_switch(cc3aab80,0,103,19e,937cdec2,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,7b0,...) at mi_switch+0x1ff turnstile_wait(cd58ca00,cae268a0,0,1c5,cc3aac30,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d200,cc3aab80,0,c100f927,67b,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d200,0,c100f927,67b,cf07940c,...) at _mtx_lock_flags+0x10f vm_waitpfault(cf07940c,0,0,40,f412b9e4,...) at vm_waitpfault+0x2a vm_fault_hold(c1bb5000,ec4e0000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(c1bb5000,ec4e0000,2,0,ec4e0000,...) at vm_fault+0x8b trap_pfault(ccb22318,0,c1bb6928,f412bb30,cc9da598,...) at trap_pfault+0x227 trap(f412bba8) at trap+0x5f1 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0xc0dfe3f8, esp = 0xf412bbe8, ebp = 0xf412bc1c --- copyinstr(f412bc38,8823030,0,bfbfed60,804fa60,...) at copyinstr+0x38 sys_execve(cc3aab80,f412bcec,c1025d9c,c0fdb40f,f412bcb4,...) at sys_execve+0x31 syscall(f412bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (59, FreeBSD ELF32, sys_execve), eip = 0x280e96ab, esp = 0xbfbfed4c, ebp = 0xbfbfeda8 --- Tracing command inetd pid 1226 tid 100145 td 0xcc98a2e0 sched_switch(cc98a2e0,0,104,1a3,79535970,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc98a2e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac86190,0,c0fd08e7,cc98a2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac861a4,0,f412ea50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac861a4,cac86190,c0fdaa18,6ae,cc08f230,...) at _cv_wait_sig+0x260 seltdwait(cc08f230,58,c95d9b80,cc98a2e0,202,...) at seltdwait+0xa2 kern_select(cc98a2e0,8,bfbfdf90,0,0,0,20,cc0acb30,4) at kern_select+0x655 sys_select(cc98a2e0,f412ecec,c1025d9c,c0fdb244,c1027350,...) at sys_select+0x66 syscall(f412ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28199743, esp = 0xbfbfdf4c, ebp = 0xbfbfedd8 --- Tracing command watchdogd pid 1201 tid 100130 td 0xcc3ad2e0 sched_switch(cc3ad2e0,0,103,19e,94049ca4,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,7b0,...) at mi_switch+0x1ff turnstile_wait(cc695380,cae245c0,0,1c5,0,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d200,cc3ad2e0,0,c100f927,479,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d200,0,c100f927,479,1e,...) at _mtx_lock_flags+0x10f vm_page_alloc(d05fa4a0,1e,0,0,f4101b64,...) at vm_page_alloc+0x99 vm_fault_hold(cc6fcae0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc6fcae0,bfbfe000,2,0,bfbfec3c,...) at vm_fault+0x8b trap_pfault(cc9dbb30,f4101c64,c0acc488,cc3ad2e0,cc9dbb30,...) at trap_pfault+0x1b9 trap(f4101d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bbdd0, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command cron pid 1158 tid 100104 td 0xcc07c2e0 sched_switch(cc07c2e0,0,103,19e,93a2fc11,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,7b0,...) at mi_switch+0x1ff turnstile_wait(d7c55e80,cc097b80,0,1c5,cc07c390,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d200,cc07c2e0,0,c100f927,67b,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d200,0,c100f927,67b,ceb72940,...) at _mtx_lock_flags+0x10f vm_waitpfault(ceb72940,1e,0,0,f402eb64,...) at vm_waitpfault+0x2a vm_fault_hold(cc493e80,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc493e80,bfbfe000,2,0,bfbfec50,...) at vm_fault+0x8b trap_pfault(f402ec58,c0a8e100,c119c9f0,cc07c2e0,cc073000,...) at trap_pfault+0x1b9 trap(f402ed28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c20c, esp = 0xbfbfec50, ebp = 0xbfbfec68 --- Tracing command sendmail pid 1153 tid 100118 td 0xcae268a0 sched_switch(cae268a0,0,103,19e,93ad8d75,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,8d0,...) at mi_switch+0x1ff turnstile_wait(cd915b80,cbee68a0,0,1c5,0,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130de00,cae268a0,0,c100f927,4fe,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130de00,0,c100f927,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(d0eaecb8,1c,0,0,f407cb64,...) at vm_page_alloc+0x565 vm_fault_hold(cc0461d0,bfbfc000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc0461d0,bfbfc000,2,0,bfbfcf10,...) at vm_fault+0x8b trap_pfault(cc0ab000,f407cc64,c0acc488,cae268a0,cc0ab000,...) at trap_pfault+0x1b9 trap(f407cd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a23a1, esp = 0xbfbfcf10, ebp = 0xbfbfcf78 --- Tracing command sendmail pid 1150 tid 100103 td 0xcc097b80 cpustop_handler(1,f402a8f4,c0e00e13,c93dbc80,f402a844,...) at cpustop_handler+0x26 ipi_nmi_handler(c93dbc80,f402a844,c0a74363,c93dbc80,cc094598,...) at ipi_nmi_handler+0x35 trap(f402a900) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a74c87, esp = 0xf402a940, ebp = 0xf402a994 --- _mtx_lock_sleep(c130df00,cc097b80,0,c100f927,4fe,...) at _mtx_lock_sleep+0x207 _mtx_lock_flags(c130df00,0,c100f927,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(d7585cb8,1c,0,0,f402ab64,...) at vm_page_alloc+0x565 vm_fault_hold(cc4ef570,bfbfc000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc4ef570,bfbfc000,2,0,bfbfc3c0,...) at vm_fault+0x8b trap_pfault(f402ac58,c0a8e100,c119c370,cbee88a0,cc094598,...) at trap_pfault+0x1b9 trap(f402ad28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a23a1, esp = 0xbfbfc3c0, ebp = 0xbfbfc428 --- Tracing command sshd pid 1147 tid 100086 td 0xcc07d000 sched_switch(cc07d000,0,104,1a3,d89c9a44,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc07d000,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac7ad90,0,c0fd08e7,cc07d000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac7ada4,0,f3fe1a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac7ada4,cac7ad90,c0fdaa18,6ae,cc034038,...) at _cv_wait_sig+0x260 seltdwait(cc034038,58,cc03ec00,cc07d000,f3fe1b14,...) at seltdwait+0xa2 kern_select(cc07d000,9,8810168,0,0,0,20,cc073b30,a) at kern_select+0x655 sys_select(cc07d000,f3fe1cec,c1025d9c,c0fbccd1,c1027350,...) at sys_select+0x66 syscall(f3fe1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284d8743, esp = 0xbfbfdedc, ebp = 0xbfbfede8 --- Tracing command moused pid 1115 tid 100084 td 0xcc01f000 sched_switch(cc01f000,0,104,1a3,bb5a2f0e,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01f000,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac86550,0,c0fd08e7,cc01f000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac86564,0,f3fd9a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac86564,cac86550,c0fdaa18,6ae,cc034c78,...) at _cv_wait_sig+0x260 seltdwait(cc034c78,58,c95d9b80,cc01f000,f6,...) at seltdwait+0xa2 kern_select(cc01f000,400,bfbfe9f4,0,0,0,20,cc01a000,1) at kern_select+0x655 sys_select(cc01f000,f3fd9cec,c1025d9c,c0e371f2,c1027350,...) at sys_select+0x66 syscall(f3fd9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281a6743, esp = 0xbfbfe98c, ebp = 0xbfbfeb18 --- Tracing command ntpd pid 1091 tid 100089 td 0xcbee82e0 sched_switch(cbee82e0,0,104,1a3,8faa6084,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee82e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc041690,0,c0fd08e7,cbee82e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0416a4,0,f3feda50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0416a4,cc041690,c0fdaa18,6ae,cbff6c08,...) at _cv_wait_sig+0x260 seltdwait(cbff6c08,58,c95d9b80,cbee82e0,0,...) at seltdwait+0xa2 kern_select(cbee82e0,1c,bfbfecd8,0,0,0,20,cac0b864,0) at kern_select+0x655 sys_select(cbee82e0,f3fedcec,c1025d9c,c0fdba20,e,...) at sys_select+0x66 syscall(f3fedd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28356743, esp = 0xbfbfecac, ebp = 0xbfbfed78 --- Tracing command nfsd pid 1035 tid 100125 td 0xcc01b8a0 sched_switch(cc01b8a0,0,104,1a3,7adf82dc,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01b8a0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40f2bdc,c0a747fa,cbc36e80,0,cc01b8a0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc008b90,0,f40f2c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc008b90,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40f2d14,c0a56ed8,cbc36e80,f40f2d28,c0fcc8d0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40f2d28,c0fcc8d0,3d8,cc096000,...) at svc_thread_start+0x10 fork_exit(c0c84f70,cbc36e80,f40f2d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1035 tid 100124 td 0xcc01bb80 sched_switch(cc01bb80,0,104,1a3,7ada8722,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01bb80,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40efbdc,c0a747fa,cbc36e80,0,cc01bb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc008bd0,0,f40efc18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc008bd0,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40efd14,c0a56ed8,cbc36e80,f40efd28,c0fcc8d0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40efd28,c0fcc8d0,3d8,cc096000,...) at svc_thread_start+0x10 fork_exit(c0c84f70,cbc36e80,f40efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1035 tid 100123 td 0xcc01e000 sched_switch(cc01e000,0,104,1a3,7ada738f,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01e000,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40ecbdc,c0a747fa,cbc36e80,0,cc01e000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc008c10,0,f40ecc18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc008c10,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40ecd14,c0a56ed8,cbc36e80,f40ecd28,c0fcc8d0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40ecd28,c0fcc8d0,3d8,cc096000,...) at svc_thread_start+0x10 fork_exit(c0c84f70,cbc36e80,f40ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1035 tid 100092 td 0xcc09b5c0 sched_switch(cc09b5c0,0,104,1a3,7ad9b527,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc09b5c0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3ffa48c,c0a747fa,cbc36e80,0,cc09b5c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca9e90,0,f3ffa4c8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca9e90,cbc36e80,1388,3b1,f3ffa510,...) at _cv_timedwait_sig+0x270 svc_run_internal(cc09b770,14,c1002afa,c0fbe1db,f3ffa808,...) at svc_run_internal+0x356 svc_run(cbc36e80,0,c0fbe1b6,19f,c12d7920,...) at svc_run+0x7b nfsrvd_nfsd(cc09b5c0,f3ffa808,c,f7b,c100e49f,...) at nfsrvd_nfsd+0x1c3 nfssvc_nfsd(cc09b5c0,f3ffacec,0,cc09b5c0,f3ffad1c,...) at nfssvc_nfsd+0x177 sys_nfssvc(cc09b5c0,f3ffacec,c1025d9c,c0fdb5d4,c1027350,...) at sys_nfssvc+0xeb syscall(f3ffad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (155, FreeBSD ELF32, sys_nfssvc), eip = 0x280be86b, esp = 0xbfbfe81c, ebp = 0xbfbfea88 --- Tracing command nfsd pid 1031 tid 100119 td 0xcae265c0 sched_switch(cae265c0,0,104,1a3,a1995f58,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae265c0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac865d0,0,c0fd08e7,cae265c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac865e4,0,f4080a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac865e4,cac865d0,c0fdaa18,6ae,cbff64d0,...) at _cv_wait_sig+0x260 seltdwait(cbff64d0,58,c95d9b80,cae265c0,58,...) at seltdwait+0xa2 kern_select(cae265c0,7,bfbfeca4,0,0,0,20,cc096b30,7) at kern_select+0x655 sys_select(cae265c0,f4080cec,c1025d9c,c0fbccd1,c1027350,...) at sys_select+0x66 syscall(f4080d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28174743, esp = 0xbfbfea8c, ebp = 0xbfbfedd8 --- Tracing command nfsuserd pid 1029 tid 100100 td 0xcc09a5c0 sched_switch(cc09a5c0,0,104,1a3,31a51a8f,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc09a5c0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f401ea14,c0a747fa,cbfcf390,0,cc09a5c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbfcf3a4,0,f401ea50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbfcf3a4,cbfcf390,7531,6ae,cc02d310,...) at _cv_timedwait_sig+0x270 seltdwait(f401ec08,f401ec10,c95d9b80,cc09a5c0,0,...) at seltdwait+0x8a kern_select(cc09a5c0,4,bfbfebb4,0,0,f401ec54,20,1e,0) at kern_select+0x655 sys_select(cc09a5c0,f401ecec,c1025d9c,c0fdb32e,f401ecb4,...) at sys_select+0x66 syscall(f401ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28173743, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1028 tid 100120 td 0xcae262e0 sched_switch(cae262e0,0,104,1a3,13cd6295,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae262e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4084a14,c0a747fa,cc040050,0,cae262e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc040064,0,f4084a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc040064,cc040050,7531,6ae,cc02d310,...) at _cv_timedwait_sig+0x270 seltdwait(f4084c08,f4084c10,c95d9b80,cae262e0,0,...) at seltdwait+0x8a kern_select(cae262e0,4,bfbfebb4,0,0,f4084c54,20,1e,0) at kern_select+0x655 sys_select(cae262e0,f4084cec,c1025d9c,c0fdb32e,f4084cb4,...) at sys_select+0x66 syscall(f4084d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28173743, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1027 tid 100085 td 0xcc01eb80 sched_switch(cc01eb80,0,104,1a3,49bb4a76,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01eb80,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fdda14,c0a747fa,cbfcf3d0,0,cc01eb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbfcf3e4,0,f3fdda50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbfcf3e4,cbfcf3d0,7531,6ae,cc02d310,...) at _cv_timedwait_sig+0x270 seltdwait(f3fddc08,f3fddc10,c95d9b80,cc01eb80,0,...) at seltdwait+0x8a kern_select(cc01eb80,4,bfbfebb4,0,0,f3fddc54,20,1e,0) at kern_select+0x655 sys_select(cc01eb80,f3fddcec,c1025d9c,c0fdb32e,f3fddcb4,...) at sys_select+0x66 syscall(f3fddd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28173743, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1026 tid 100090 td 0xcc01e5c0 sched_switch(cc01e5c0,0,104,1a3,318927cd,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01e5c0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3ff2a14,c0a747fa,cc040090,0,cc01e5c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc0400a4,0,f3ff2a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc0400a4,cc040090,7531,6ae,cc02d310,...) at _cv_timedwait_sig+0x270 seltdwait(f3ff2c08,f3ff2c10,c95d9b80,cc01e5c0,0,...) at seltdwait+0x8a kern_select(cc01e5c0,4,bfbfebb4,0,0,f3ff2c54,20,1e,0) at kern_select+0x655 sys_select(cc01e5c0,f3ff2cec,c1025d9c,c0fdb32e,f3ff2cb4,...) at sys_select+0x66 syscall(f3ff2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28173743, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1025 tid 100116 td 0xcbee6000 Tracing command mountd pid 1009 tid 100097 td 0xcc07c5c0 sched_switch(cc07c5c0,0,104,1a3,340efe7,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc07c5c0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc011750,0,c0fd08e7,cc07c5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc011764,0,f4011a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc011764,cc011750,c0fdaa18,6ae,cc02d540,...) at _cv_wait_sig+0x260 seltdwait(cc02d540,58,c95d9b80,cc07c5c0,f4011ac0,...) at seltdwait+0xa2 kern_select(cc07c5c0,9,bfbfecf4,0,0,0,20,cc0732cc,bfbfecf4) at kern_select+0x655 sys_select(cc07c5c0,f4011cec,c1025d9c,c0fbbf37,1,...) at sys_select+0x66 syscall(f4011d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818b743, esp = 0xbfbfec8c, ebp = 0xbfbfedd8 --- Tracing command rpcbind pid 976 tid 100115 td 0xcbee62e0 sched_switch(cbee62e0,0,104,1a3,b453549b,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee62e0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4062a78,c0a747fa,c955f110,0,cbee62e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c955f124,0,f4062ab4,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c955f124,c955f110,7531,6ae,f4062b6c,...) at _cv_timedwait_sig+0x270 seltdwait(f4062c3c,f4062c44,5ab,cbee62e0,cbee62e0,...) at seltdwait+0x8a sys_poll(cbee62e0,f4062cec,c1025d9c,c0fdb32e,f4062cb4,...) at sys_poll+0x321 syscall(f4062d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2812a0eb, esp = 0xbfbfcb9c, ebp = 0xbfbfed68 --- Tracing command syslogd pid 961 tid 100113 td 0xcbee68a0 sched_switch(cbee68a0,0,104,1a3,93de7da1,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cbee68a0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cf7c,54,c100fbfb,0,0,...) at sleepq_wait+0x63 _sleep(c130cf7c,c130d200,254,c100fbfb,0,...) at _sleep+0x39c vm_wait(d72b1378,0,c100f927,8f5,40222,...) at vm_wait+0xbe vm_page_grab(d72b1378,b88204,0,412a2,c12d7960,...) at vm_page_grab+0x180 allocbuf(e3300ea0,4000,c0fdf847,b00,ffffffe0,...) at allocbuf+0x672 getblk(ccbbf984,2e2081,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(ccbbf984,88204000,b,4000,c9561b80,...) at ffs_balloc_ufs2+0x1e6c ffs_copyonwrite(cbff5c3c,e313ee40,0,e313ee40,d403a15c,...) at ffs_copyonwrite+0x509 ffs_geom_strategy(cbff5d34,e313ee40,c1108c08,c12d7920,d403a15c,...) at ffs_geom_strategy+0xeb ufs_strategy(f405aad0,c102d24b,c0deabe1,e313eea0,cbee68a0,...) at ufs_strategy+0xd3 VOP_STRATEGY_APV(c11378c0,f405aad0,c0fdf847,c1149c00,d403a15c,...) at VOP_STRATEGY_APV+0x119 bufstrategy(d403a254,e313ee40,3a0,2000,0,...) at bufstrategy+0x61 bufwrite(e313ee40,0,c0fdf847,6d6,0,...) at bufwrite+0x159 vfs_bio_awrite(e313ee40,0,c100b511,106,0,...) at vfs_bio_awrite+0x328 ffs_syncvnode(d403a15c,1,0,c11378c0,d403a254,...) at ffs_syncvnode+0x3a5 ffs_fsync(f405ac40,c102ca04,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c11378c0,f405ac40,c0fe3ac7,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cbee68a0,f405acec,c1025d9c,c0fdc0af,e,...) at sys_fsync+0x1df syscall(f405ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x28188637, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- Tracing command devd pid 838 tid 100088 td 0xcc01e8a0 sched_switch(cc01e8a0,0,104,1a3,dbb70f3e,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01e8a0,0,c0fd82c0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cbfcf010,0,c0fd08e7,cc01e8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfcf024,0,f3fe9a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfcf024,cbfcf010,c0fdaa18,6ae,cc02d700,...) at _cv_wait_sig+0x260 seltdwait(cc02d700,58,c95d9b80,cc01e8a0,c1108c08,...) at seltdwait+0xa2 kern_select(cc01e8a0,5,bfbfe950,0,0,0,20,cbedc864,46) at kern_select+0x655 sys_select(cc01e8a0,f3fe9cec,c1025d9c,c0e3242f,c1027350,...) at sys_select+0x66 syscall(f3fe9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x808936b, esp = 0xbfbfe91c, ebp = 0xbfbfedf8 --- Tracing command softdepflush pid 19 tid 100078 td 0xcbee88a0 sched_switch(cbee88a0,0,103,19e,938ccdfc,...) at sched_switch+0x413 mi_switch(103,0,c0fd8cb0,2e0,b10,...) at mi_switch+0x1ff turnstile_wait(ce3fa480,cae268a0,0,1c5,848,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c1bb7608,cbee88a0,0,c100d464,827,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c1bb7608,0,c100d464,827,a0fa5166,...) at _mtx_lock_flags+0x10f uma_zalloc_arg(c1bb69a0,0,101,c62ddea0,1,...) at uma_zalloc_arg+0x1cb vm_radix_insert(cc312e10,41c1f4,0,c4e652c0,0,...) at vm_radix_insert+0x1d8 vm_page_insert(c4e652c0,cc312de0,41c1f4,0,20,...) at vm_page_insert+0x1b5 vm_page_alloc(cc312de0,41c1f4,0,40222,40222,...) at vm_page_alloc+0x51c vm_page_grab(cc312de0,41c1f4,0,412a2,c12d7d60,...) at vm_page_grab+0x152 allocbuf(e31e9020,4000,c0fdf847,b00,efbe99fc,...) at allocbuf+0x672 getblk(cbff5c3c,20e0fa0,0,4000,0,...) at getblk+0x5e3 breadn_flags(cbff5c3c,20e0fa0,0,4000,0,...) at breadn_flags+0x43 indir_trunc(fffce7f4,ffffffff,c1007eac,1e0e,0,...) at indir_trunc+0x114 indir_trunc(fffff7f3,ffffffff,c1007eac,1cd0,c12d7d40,...) at indir_trunc+0x4ba handle_workitem_freeblocks(0,efbe9c70,2,6ba,efbe9c74,...) at handle_workitem_freeblocks+0x24f process_worklist_item(c130c880,0,c1007eac,5e9,cc021300,...) at process_worklist_item+0x2c2 softdep_process_worklist(cbfd1b70,0,c1007eac,562,3e8,...) at softdep_process_worklist+0x84 softdep_flush(0,efbe9d28,c0fcc8d0,3d8,cbedb000,...) at softdep_flush+0x2a0 fork_exit(c0cd7fd0,0,efbe9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe9d60, ebp = 0 --- Tracing command vnlru pid 18 tid 100077 td 0xcbee8b80 sched_switch(cbee8b80,0,104,1a3,51703d8a,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cbee8b80,0,c0fd82c0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cbedb2cc,60,c0fe3415,0,0,...) at sleepq_timedwait+0x6b _sleep(cbedb2cc,c1304b74,260,c0fe3415,3e8,...) at _sleep+0x36a vnlru_proc(0,efbe6d28,c0fcc8d0,3d8,cbedb2cc,...) at vnlru_proc+0xe7 fork_exit(c0b2ba40,0,efbe6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe6d60, ebp = 0 --- Tracing command syncer pid 17 tid 100076 td 0xcacf98a0 sched_switch(cacf98a0,0,104,1a3,518ca4f5,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf98a0,0,c0fd82c0,28b,cacf98a0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304bb4,0,efbe3c78,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c1304bb4,c1304ba0,3e8,783,4e20,...) at _cv_timedwait+0x270 sched_sync(0,efbe3d28,c0fcc8d0,3d8,cbedb598,...) at sched_sync+0x595 fork_exit(c0b2ad70,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xcacf9b80 sched_switch(cacf9b80,0,104,1a3,5181676e,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cacf9b80,0,c0fd82c0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304848,54,c0fe018e,0,0,...) at sleepq_timedwait+0x6b _sleep(c1304848,c1304850,54,c0fe018e,3e8,...) at _sleep+0x36a buf_daemon(0,efbe0d28,c0fcc8d0,3d8,cbedb864,...) at buf_daemon+0x175 fork_exit(c0b0da90,0,efbe0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe0d60, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xcae24000 sched_switch(cae24000,0,104,1a3,98f524fb,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24000,0,c0fd82c0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e76c,0,c10115cd,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e76c,c130d200,0,c10115cd,493e0,...) at _sleep+0x36a vm_pagezero(0,efbddd28,c0fcc8d0,3d8,cbedbb30,...) at vm_pagezero+0xdc fork_exit(c0d1fd40,0,efbddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbddd60, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xcae242e0 sched_switch(cae242e0,0,104,1a3,e32a80ca,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cae242e0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130e348,74,c0fe018e,0,0,...) at sleepq_wait+0x63 _sleep(c130e348,c130e34c,74,c0fe018e,0,...) at _sleep+0x39c vm_daemon(0,efbdad28,c0fcc8d0,3d8,cbedc000,...) at vm_daemon+0x59 fork_exit(c0d184d0,0,efbdad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbdad60, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xcae245c0 sched_switch(cae245c0,0,602,19e,9404658d,...) at sched_switch+0x413 mi_switch(602,0,c0fd2f7f,cd,cae245c0,...) at mi_switch+0x1ff critical_exit(c11a9bec,efbd7be8,c0a74363,c11a9bec,4,...) at critical_exit+0xb5 spinlock_exit(c11a9bec,4,c0fd08e7,119,c0ac8a69,c119b640) at spinlock_exit+0x10 _mtx_unlock_spin_flags(c11a9bec,0,c0fd82c0,10d,efbd7c20,...) at _mtx_unlock_spin_flags+0x133 sleepq_release(c130cf7c,0,0,0,c39cfe80,...) at sleepq_release+0x39 wakeup(c130cf7c,0,c10103a7,61c,1388,...) at wakeup+0x41 vm_pageout(0,efbd7d28,c0fcc8d0,3d8,c95de2cc,...) at vm_pageout+0x227 fork_exit(c0d19550,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xcae248a0 sched_switch(cae248a0,0,104,1a3,288bcbc8,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae248a0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1151354,5c,c0e2cc1e,0,0,...) at sleepq_wait+0x63 _sleep(c1151354,c1151398,5c,c0e2cc1e,0,...) at _sleep+0x39c xpt_scanner_thread(0,efa54d28,c0fcc8d0,3d8,c95de598,...) at xpt_scanner_thread+0x47 fork_exit(c04a72f0,0,efa54d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa54d60, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xcae24b80 sched_switch(cae24b80,0,104,1a3,ca53493d,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24b80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c13080d8,0,c0ff78df,0,0,...) at sleepq_wait+0x63 _sleep(c13080d8,c13080a0,0,c0ff78df,0,...) at _sleep+0x39c sctp_iterator_thread(0,efa51d28,c0fcc8d0,3d8,c95de864,...) at sctp_iterator_thread+0x5c fork_exit(c0bc5c00,0,efa51d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa51d60, ebp = 0 --- Tracing command fdc0 pid 4 tid 100065 td 0xcae25b80 sched_switch(cae25b80,0,104,1a3,5182c894,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae25b80,0,c0fd82c0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cac7223c,5c,c0fbec52,0,0,...) at sleepq_timedwait+0x6b _sleep(cac7223c,cac722f0,5c,c0fbec52,3e8,...) at _sleep+0x36a fdc_thread(cac72200,efa42d28,c0fcc8d0,3d8,c95deb30,...) at fdc_thread+0x27d fork_exit(c0da2080,cac72200,efa42d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa42d60, ebp = 0 --- Tracing command fw0_probe pid 3 tid 100063 td 0xcace42e0 sched_switch(cace42e0,0,104,1a3,288c0862,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cace42e0,0,c0fd82c0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cad2c000,6c,c0fbec52,100,0,...) at sleepq_wait_sig+0x17 _sleep(cad2c000,cad30488,16c,c0fbec52,0,...) at _sleep+0x385 fw_bus_probe_thread(cad2c000,efa32d28,c0fcc8d0,3d8,cac0b000,...) at fw_bus_probe_thread+0xa08 fork_exit(c0701880,cad2c000,efa32d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa32d60, ebp = 0 --- Tracing command usb pid 15 tid 100059 td 0xcacf7000 sched_switch(cacf7000,0,104,1a3,163c552a,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf7000,0,c0fd82c0,268,cacf7000,...) at sleepq_switch+0x162 sleepq_wait(cacfed34,0,efa1bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed34,cacfee4c,c0fb02d1,74,cacfed3c,...) at _cv_wait+0x260 usb_process(cacfed2c,efa1bd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfed2c,efa1bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa1bd60, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xcacf72e0 sched_switch(cacf72e0,0,104,1a3,e6472a69,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf72e0,0,c0fd82c0,268,cacf72e0,...) at sleepq_switch+0x162 sleepq_wait(cacfed04,0,efa18cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed04,cacfee4c,c0fb02d1,74,cacfed0c,...) at _cv_wait+0x260 usb_process(cacfecfc,efa18d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfecfc,efa18d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa18d60, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xcacf75c0 sched_switch(cacf75c0,0,104,1a3,15d5fada,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf75c0,0,c0fd82c0,268,cacf75c0,...) at sleepq_switch+0x162 sleepq_wait(cacfecd4,0,efa15cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfecd4,cacfee4c,c0fb02d1,74,cacfecdc,...) at _cv_wait+0x260 usb_process(cacfeccc,efa15d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfeccc,efa15d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa15d60, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xcacf78a0 sched_switch(cacf78a0,0,104,1a3,15d5e9c5,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf78a0,0,c0fd82c0,268,cacf78a0,...) at sleepq_switch+0x162 sleepq_wait(cacfeca4,0,efa12cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfeca4,cacfee4c,c0fb02d1,74,cacfecac,...) at _cv_wait+0x260 usb_process(cacfec9c,efa12d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfec9c,efa12d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa12d60, ebp = 0 --- Tracing command usb pid 15 tid 100054 td 0xcacf9000 sched_switch(cacf9000,0,104,1a3,15d5d3ad,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf9000,0,c0fd82c0,268,cacf9000,...) at sleepq_switch+0x162 sleepq_wait(cacefb5c,0,efa09cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb5c,cacefc74,c0fb02d1,74,cacefb64,...) at _cv_wait+0x260 usb_process(cacefb54,efa09d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefb54,efa09d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa09d60, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xcacf92e0 sched_switch(cacf92e0,0,104,1a3,e645eb90,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf92e0,0,c0fd82c0,268,cacf92e0,...) at sleepq_switch+0x162 sleepq_wait(cacefb2c,0,efa06cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb2c,cacefc74,c0fb02d1,74,cacefb34,...) at _cv_wait+0x260 usb_process(cacefb24,efa06d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefb24,efa06d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa06d60, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xcacf95c0 sched_switch(cacf95c0,0,104,1a3,157167b9,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf95c0,0,c0fd82c0,268,cacf95c0,...) at sleepq_switch+0x162 sleepq_wait(cacefafc,0,efa03cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefafc,cacefc74,c0fb02d1,74,cacefb04,...) at _cv_wait+0x260 usb_process(cacefaf4,efa03d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefaf4,efa03d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa03d60, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xcace2000 sched_switch(cace2000,0,104,1a3,157153ae,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2000,0,c0fd82c0,268,cace2000,...) at sleepq_switch+0x162 sleepq_wait(cacefacc,0,efa00cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefacc,cacefc74,c0fb02d1,74,cacefad4,...) at _cv_wait+0x260 usb_process(cacefac4,efa00d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefac4,efa00d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa00d60, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xcace22e0 sched_switch(cace22e0,0,104,1a3,15713af0,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace22e0,0,c0fd82c0,268,cace22e0,...) at sleepq_switch+0x162 sleepq_wait(caceab5c,0,ef9fccac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab5c,caceac74,c0fb02d1,74,caceab64,...) at _cv_wait+0x260 usb_process(caceab54,ef9fcd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceab54,ef9fcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9fcd60, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xcace25c0 sched_switch(cace25c0,0,104,1a3,e6446733,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace25c0,0,c0fd82c0,268,cace25c0,...) at sleepq_switch+0x162 sleepq_wait(caceab2c,0,ef9f9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab2c,caceac74,c0fb02d1,74,caceab34,...) at _cv_wait+0x260 usb_process(caceab24,ef9f9d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceab24,ef9f9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f9d60, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xcace28a0 sched_switch(cace28a0,0,104,1a3,fb16c70,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace28a0,0,c0fd82c0,268,cace28a0,...) at sleepq_switch+0x162 sleepq_wait(caceaafc,0,ef9f6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaafc,caceac74,c0fb02d1,74,caceab04,...) at _cv_wait+0x260 usb_process(caceaaf4,ef9f6d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceaaf4,ef9f6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f6d60, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xcace2b80 sched_switch(cace2b80,0,104,1a3,fb1592a,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2b80,0,c0fd82c0,268,cace2b80,...) at sleepq_switch+0x162 sleepq_wait(caceaacc,0,ef9f3cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaacc,caceac74,c0fb02d1,74,caceaad4,...) at _cv_wait+0x260 usb_process(caceaac4,ef9f3d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceaac4,ef9f3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f3d60, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xcace3000 sched_switch(cace3000,0,104,1a3,fb1422c,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace3000,0,c0fd82c0,268,cace3000,...) at sleepq_switch+0x162 sleepq_wait(cace5b5c,0,ef9efcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b5c,cace5c74,c0fb02d1,74,cace5b64,...) at _cv_wait+0x260 usb_process(cace5b54,ef9efd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5b54,ef9efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9efd60, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xcace32e0 sched_switch(cace32e0,0,104,1a3,e642cd90,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace32e0,0,c0fd82c0,268,cace32e0,...) at sleepq_switch+0x162 sleepq_wait(cace5b2c,0,ef9eccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b2c,cace5c74,c0fb02d1,74,cace5b34,...) at _cv_wait+0x260 usb_process(cace5b24,ef9ecd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5b24,ef9ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ecd60, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xcace35c0 sched_switch(cace35c0,0,104,1a3,f4d01aa,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace35c0,0,c0fd82c0,268,cace35c0,...) at sleepq_switch+0x162 sleepq_wait(cace5afc,0,ef9e9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5afc,cace5c74,c0fb02d1,74,cace5b04,...) at _cv_wait+0x260 usb_process(cace5af4,ef9e9d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5af4,ef9e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e9d60, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xcace38a0 sched_switch(cace38a0,0,104,1a3,f4cf05b,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace38a0,0,c0fd82c0,268,cace38a0,...) at sleepq_switch+0x162 sleepq_wait(cace5acc,0,ef9e6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5acc,cace5c74,c0fb02d1,74,cace5ad4,...) at _cv_wait+0x260 usb_process(cace5ac4,ef9e6d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5ac4,ef9e6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e6d60, ebp = 0 --- Tracing command usb pid 15 tid 100041 td 0xcace4000 sched_switch(cace4000,0,104,1a3,f4cdaa6,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace4000,0,c0fd82c0,268,cace4000,...) at sleepq_switch+0x162 sleepq_wait(cacddb5c,0,ef9dfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb5c,cacddc74,c0fb02d1,74,cacddb64,...) at _cv_wait+0x260 usb_process(cacddb54,ef9dfd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddb54,ef9dfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dfd60, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xcacbf000 sched_switch(cacbf000,0,104,1a3,e6421d75,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf000,0,c0fd82c0,268,cacbf000,...) at sleepq_switch+0x162 sleepq_wait(cacddb2c,0,ef9dccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb2c,cacddc74,c0fb02d1,74,cacddb34,...) at _cv_wait+0x260 usb_process(cacddb24,ef9dcd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddb24,ef9dcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dcd60, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xcacbf2e0 sched_switch(cacbf2e0,0,104,1a3,ee8dc80,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf2e0,0,c0fd82c0,268,cacbf2e0,...) at sleepq_switch+0x162 sleepq_wait(cacddafc,0,ef9d9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddafc,cacddc74,c0fb02d1,74,cacddb04,...) at _cv_wait+0x260 usb_process(cacddaf4,ef9d9d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddaf4,ef9d9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d9d60, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xcacbf5c0 sched_switch(cacbf5c0,0,104,1a3,ee8caa7,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf5c0,0,c0fd82c0,268,cacbf5c0,...) at sleepq_switch+0x162 sleepq_wait(cacddacc,0,ef9d6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddacc,cacddc74,c0fb02d1,74,cacddad4,...) at _cv_wait+0x260 usb_process(cacddac4,ef9d6d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddac4,ef9d6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d6d60, ebp = 0 --- Tracing command usb pid 15 tid 100036 td 0xcacbfb80 sched_switch(cacbfb80,0,104,1a3,ee8b42e,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbfb80,0,c0fd82c0,268,cacbfb80,...) at sleepq_switch+0x162 sleepq_wait(cacd5b5c,0,ef9cfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b5c,cacd5c74,c0fb02d1,74,cacd5b64,...) at _cv_wait+0x260 usb_process(cacd5b54,ef9cfd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5b54,ef9cfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9cfd60, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xcacc1000 sched_switch(cacc1000,0,104,1a3,e64118ac,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc1000,0,c0fd82c0,268,cacc1000,...) at sleepq_switch+0x162 sleepq_wait(cacd5b2c,0,ef9cccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b2c,cacd5c74,c0fb02d1,74,cacd5b34,...) at _cv_wait+0x260 usb_process(cacd5b24,ef9ccd28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5b24,ef9ccd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ccd60, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xcacc12e0 sched_switch(cacc12e0,0,104,1a3,e848e15,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc12e0,0,c0fd82c0,268,cacc12e0,...) at sleepq_switch+0x162 sleepq_wait(cacd5afc,0,ef9c9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5afc,cacd5c74,c0fb02d1,74,cacd5b04,...) at _cv_wait+0x260 usb_process(cacd5af4,ef9c9d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5af4,ef9c9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c9d60, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xcacc15c0 sched_switch(cacc15c0,0,104,1a3,e847959,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc15c0,0,c0fd82c0,268,cacc15c0,...) at sleepq_switch+0x162 sleepq_wait(cacd5acc,0,ef9c6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5acc,cacd5c74,c0fb02d1,74,cacd5ad4,...) at _cv_wait+0x260 usb_process(cacd5ac4,ef9c6d28,c0fcc8d0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5ac4,ef9c6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c6d60, ebp = 0 --- Tracing command ctl_thrd pid 2 tid 100028 td 0xc96eb8a0 sched_switch(c96eb8a0,0,104,1a3,e845cd2,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb8a0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c97c4000,5c,c0e37690,0,0,...) at sleepq_wait+0x63 _sleep(c97c4000,c97c4000,5c,c0e37690,0,...) at _sleep+0x39c ctl_work_thread(c97c4000,ef733d28,c0fcc8d0,3d8,cac0b598,...) at ctl_work_thread+0x22f6 fork_exit(c04d08d0,c97c4000,ef733d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef733d60, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xc96ea5c0 sched_switch(c96ea5c0,0,104,1a3,91ce9333,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ea5c0,0,c0fd82c0,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d7e4,0,c0fbec52,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d7e4,0,0,c0fbec52,64) at _sleep+0x36a pause(c0fbec52,64,c0f8c330,111,0,...) at pause+0xa0 random_kthread(0,e33e0d28,c0fcc8d0,3d8,c95dd000,...) at random_kthread+0x1ef fork_exit(c082d0c0,0,e33e0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e0d60, ebp = 0 --- Tracing command geom pid 13 tid 100016 td 0xc96ea8a0 sched_switch(c96ea8a0,0,104,1a3,e51171d3,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96ea8a0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a8a8,5c,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(c118a8a8,c118a808,25c,c0fbec52,0,...) at _sleep+0x39c g_io_schedule_down(c96ea8a0,0,c0fc07a1,6c,e33ddd14,...) at g_io_schedule_down+0x5b g_down_procbody(0,e33ddd28,c0fcc8d0,3d8,c95dd2cc,...) at g_down_procbody+0x9d fork_exit(c09e6b60,0,e33ddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ddd60, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xc96eab80 sched_switch(c96eab80,0,104,1a3,e51b894a,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eab80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a8a4,5c,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(c118a8a4,c118a828,25c,c0fbec52,0,...) at _sleep+0x39c g_io_schedule_up(c96eab80,0,c0fc07a1,5f,e33dad14,...) at g_io_schedule_up+0x134 g_up_procbody(0,e33dad28,c0fcc8d0,3d8,c95dd2cc,...) at g_up_procbody+0x9d fork_exit(c09e6c00,0,e33dad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33dad60, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xc96eb000 sched_switch(c96eb000,0,104,1a3,3881ee8,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb000,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a89c,5c,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(c118a89c,c118a7cc,25c,c0fbec52,0,...) at _sleep+0x39c g_run_events(c119b640,0,c0fc07a1,79,e33d7d14,...) at g_run_events+0x54b g_event_procbody(0,e33d7d28,c0fcc8d0,3d8,c95dd2cc,...) at g_event_procbody+0x90 fork_exit(c09e6ca0,0,e33d7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d7d60, ebp = 0 --- Tracing command intr pid 12 tid 100067 td 0xcae255c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100066 td 0xcae258a0 sched_switch(cae258a0,0,109,1a3,e7f7552,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cae258a0,...) at mi_switch+0x1ff ithread_loop(cae1e990,efa45d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cae1e990,efa45d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa45d60, ebp = 0 --- Tracing command intr pid 12 tid 100064 td 0xcae26000 sched_switch(cae26000,0,109,1a3,3181c760,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cae26000,...) at mi_switch+0x1ff ithread_loop(cae18a40,efa35d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cae18a40,efa35d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa35d60, ebp = 0 --- Tracing command intr pid 12 tid 100061 td 0xcace48a0 sched_switch(cace48a0,0,109,1a3,fed1413d,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cace48a0,...) at mi_switch+0x1ff ithread_loop(cad15880,efa29d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cad15880,efa29d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa29d60, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xcace4b80 sched_switch(cace4b80,0,109,1a3,3c4ad02,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cace4b80,...) at mi_switch+0x1ff ithread_loop(cad15010,efa21d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cad15010,efa21d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa21d60, ebp = 0 --- Tracing command intr pid 12 tid 100055 td 0xcacf7b80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100042 td 0xcace3b80 sched_switch(cace3b80,0,109,1a3,3383825d,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cace3b80,...) at mi_switch+0x1ff ithread_loop(cacc5720,ef9e3d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cacc5720,ef9e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xcacbf8a0 sched_switch(cacbf8a0,0,109,1a3,2e0a6eeb,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cacbf8a0,...) at mi_switch+0x1ff ithread_loop(cacad590,ef9d3d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cacad590,ef9d3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d3d60, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xcacc18a0 sched_switch(cacc18a0,0,109,1a3,73b4bf16,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cacc18a0,...) at mi_switch+0x1ff ithread_loop(cacad830,ef9c3d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cacad830,ef9c3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c3d60, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xcacc1b80 sched_switch(cacc1b80,0,109,1a3,e51908b3,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,cacc1b80,...) at mi_switch+0x1ff ithread_loop(cac97ae0,ef86dd28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cac97ae0,ef86dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef86dd60, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc96eb2e0 sched_switch(c96eb2e0,0,109,1a3,73a4bbab,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c96eb2e0,...) at mi_switch+0x1ff ithread_loop(cac97c80,ef869d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cac97c80,ef869d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef869d60, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc96eb5c0 sched_switch(c96eb5c0,0,109,1a3,a9f188,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c96eb5c0,...) at mi_switch+0x1ff ithread_loop(cac97500,ef763d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cac97500,ef763d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef763d60, ebp = 0 --- Tracing command intr pid 12 tid 100022 td 0xc97c1b80 sched_switch(c97c1b80,0,109,1a3,a0e36bac,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c97c1b80,...) at mi_switch+0x1ff ithread_loop(c95b3450,e33efd28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3450,e33efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33efd60, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc96228a0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100019 td 0xc96ea000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100018 td 0xc96ea2e0 sched_switch(c96ea2e0,0,109,1a3,aabb449f,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c96ea2e0,...) at mi_switch+0x1ff ithread_loop(c95b3490,e33e3d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3490,e33e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc95e05c0 sched_switch(c95e05c0,0,109,1a3,719f27c5,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c95e05c0,...) at mi_switch+0x1ff ithread_loop(c95b36f0,e33d1d28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b36f0,e33d1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d1d60, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc95e08a0 sched_switch(c95e08a0,0,109,1a3,87140869,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c95e08a0,...) at mi_switch+0x1ff ithread_loop(c95b3700,e33ced28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3700,e33ced28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ced60, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc95e0b80 sched_switch(c95e0b80,0,109,1a3,91cea168,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c95e0b80,...) at mi_switch+0x1ff ithread_loop(c95b3710,e33cbd28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3710,e33cbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33cbd60, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc9622000 sched_switch(c9622000,0,109,1a3,91d0f140,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c9622000,...) at mi_switch+0x1ff ithread_loop(c95b3720,c93fed28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3720,c93fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fed60, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc96222e0 sched_switch(c96222e0,0,109,1a3,bb20dd03,...) at sched_switch+0x413 mi_switch(109,0,c0fccc01,557,c96222e0,...) at mi_switch+0x1ff ithread_loop(c95b3730,c93fbd28,c0fcc8d0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3730,c93fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fbd60, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc96225c0 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc95df000 sched_switch(c95df000,0,108,19e,93939816,...) at sched_switch+0x413 mi_switch(108,0,c0fd494c,3ce,c95df000,...) at mi_switch+0x1ff sched_idletd(0,c93f4d28,c0fcc8d0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93f4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f4d60, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc95df2e0 sched_switch(c95df2e0,0,108,19e,9310af47,...) at sched_switch+0x413 mi_switch(108,0,c0fd494c,3ce,c95df2e0,...) at mi_switch+0x1ff sched_idletd(0,c93f1d28,c0fcc8d0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93f1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f1d60, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc95df5c0 sched_switch(c95df5c0,0,108,19e,93a3050f,...) at sched_switch+0x413 mi_switch(108,0,c0fd494c,3ce,c95df5c0,...) at mi_switch+0x1ff sched_idletd(0,c93eed28,c0fcc8d0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93eed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93eed60, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc95df8a0 sched_switch(c95df8a0,0,108,19e,93cfda12,...) at sched_switch+0x413 mi_switch(108,0,c0fd494c,3ce,c95df8a0,...) at mi_switch+0x1ff sched_idletd(0,c93ebd28,c0fcc8d0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93ebd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93ebd60, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc95dfb80 sched_switch(c95dfb80,0,104,1a3,dbb2ed27,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c95dfb80,0,c0fd82c0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd82c0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c95ddb30,6c,c0fdb1f8,100,0,...) at sleepq_wait_sig+0x17 _sleep(c95ddb30,c95ddbb8,16c,c0fdb1f8,0,...) at _sleep+0x385 kern_wait(c95dfb80,ffffffff,c93e7c58,0,0,...) at kern_wait+0x597 sys_wait4(c95dfb80,c93e7cec,c1025d9c,c0fdb046,d61b98a0,...) at sys_wait4+0x3b syscall(c93e7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x806070f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8c8 --- Tracing command audit pid 10 tid 100001 td 0xc95e0000 sched_switch(c95e0000,0,104,1a3,e821e1f,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e0000,0,c0fd82c0,268,c95e0000,...) at sleepq_switch+0x162 sleepq_wait(c130c000,0,c93e4c8c,1,0,...) at sleepq_wait+0x63 _cv_wait(c130c000,c130bfe4,c1004577,194,c93e4cc4,...) at _cv_wait+0x260 audit_worker(0,c93e4d28,c0fcc8d0,3d8,c95de000,...) at audit_worker+0x84 fork_exit(c0c952c0,0,c93e4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93e4d60, ebp = 0 --- Tracing command kernel pid 0 tid 1818774 td 0xd604a000 sched_switch(d604a000,0,104,1a3,19ef977,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d604a000,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(d8396740,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(d8396740,d8396758,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(d7cbe900,f3ff6d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,d7cbe900,f3ff6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff6d60, ebp = 0 --- Tracing command kernel pid 0 tid 100219 td 0xd1c072e0 sched_switch(d1c072e0,0,104,1a3,116d0245,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d1c072e0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(d8396840,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(d8396840,d8396858,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c955c1f0,f4248d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c955c1f0,f4248d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4248d60, ebp = 0 --- Tracing command kernel pid 0 tid 100218 td 0xd1c075c0 sched_switch(d1c075c0,0,104,1a3,116d5347,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d1c075c0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(d8396840,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(d8396840,d8396858,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c955c1f0,f4245d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c955c1f0,f4245d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4245d60, ebp = 0 --- Tracing command kernel pid 0 tid 100217 td 0xd1c078a0 sched_switch(d1c078a0,0,104,1a3,116d3cba,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d1c078a0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(d8396840,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(d8396840,d8396858,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c955c1f0,f4242d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c955c1f0,f4242d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4242d60, ebp = 0 --- Tracing command kernel pid 0 tid 100216 td 0xd1c07b80 sched_switch(d1c07b80,0,104,1a3,116d60ce,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(d1c07b80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(d8396840,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(d8396840,d8396858,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c955c1f0,f423fd28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c955c1f0,f423fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf423fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xcae25000 sched_switch(cae25000,0,104,1a3,7adc5297,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae25000,0,c0fd82c0,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d7e4,0,c0fbec52,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d7e4,0,0,c0fbec52,bb8) at _sleep+0x36a pause(c0fbec52,bb8,126,124,c118aae8,...) at pause+0xa0 deadlkres(0,efa4ed28,c0fcc8d0,3d8,c118aa60,...) at deadlkres+0x324 fork_exit(c0a39910,0,efa4ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4ed60, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xcae252e0 sched_switch(cae252e0,0,104,1a3,c0ee2d24,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae252e0,0,c0fd82c0,268,cae252e0,...) at sleepq_switch+0x162 sleepq_wait(c971ddc0,0,c0fd34a1,c0fbec52,0,...) at sleepq_wait+0x63 msleep_spin(c971ddc0,c971ddd8,c0fbec52,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c136d740,efa4bd28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c136d740,efa4bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4bd60, ebp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xcace45c0 sched_switch(cace45c0,0,104,1a3,91d03215,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace45c0,0,c0fd82c0,268,cace45c0,...) at sleepq_switch+0x162 sleepq_wait(c971de00,0,c0fd34a1,c0fbec52,0,...) at sleepq_wait+0x63 msleep_spin(c971de00,c971de18,c0fbec52,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(cad3049c,efa2fd28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,cad3049c,efa2fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa2fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc96ebb80 sched_switch(c96ebb80,0,104,1a3,28905e88,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ebb80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de40,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(c971de40,c971de58,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c118b3f8,e33fed28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c118b3f8,e33fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fed60, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc97c1000 sched_switch(c97c1000,0,104,1a3,3d1ec2e7,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c1000,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de80,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(c971de80,c971de98,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c130c5e4,e33fbd28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c130c5e4,e33fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fbd60, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc97c12e0 sched_switch(c97c12e0,0,104,1a3,28903937,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c12e0,0,c0fd82c0,268,c97c12e0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd34a1,c0fbec52,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbec52,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154598,e33f8d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c1154598,e33f8d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f8d60, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc97c15c0 sched_switch(c97c15c0,0,104,1a3,2890288e,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c15c0,0,c0fd82c0,268,c97c15c0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd34a1,c0fbec52,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbec52,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154598,e33f5d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c1154598,e33f5d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f5d60, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc97c18a0 sched_switch(c97c18a0,0,104,1a3,289019bc,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c18a0,0,c0fd82c0,268,c97c18a0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd34a1,c0fbec52,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbec52,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154598,e33f2d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c1154598,e33f2d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f2d60, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc9622b80 sched_switch(c9622b80,0,104,1a3,19ccd2d9,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c9622b80,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971e0c0,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(c971e0c0,c971e0d8,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c11aa3ac,e33e9d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c11aa3ac,e33e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e9d60, ebp = 0 --- Tracing command kernel pid 0 tid 100013 td 0xc95e02e0 sched_switch(c95e02e0,0,104,1a3,220ed45c,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e02e0,0,c0fd82c0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c96e9000,0,c0fbec52,0,0,...) at sleepq_wait+0x63 _sleep(c96e9000,c96e9018,0,c0fbec52,0,...) at _sleep+0x39c taskqueue_thread_loop(c11a8dc0,e33d4d28,c0fcc8d0,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c11a8dc0,e33d4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc118ad30 sched_switch(c118ad30,0,104,1a3,626953ee,...) at sched_switch+0x413 mi_switch(104,0,c0fd82c0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c118ad30,0,c0fd82c0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c118aa60,54,c1004f58,0,0,...) at sleepq_timedwait+0x6b _sleep(c118aa60,0,54,c1004f58,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x240 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 420 MB:lock order reversal: 1st 0xc130df00 page lock (page lock) @ vm/vm_page.c:1278 2nd 0xcac11804 AHCI channel lock (AHCI channel lock) @ cam/cam_periph.h:191 KDB: stack backtrace: db_trace_self_wrapper(c0fd68d7,0,74,0,750000,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda3a4,c12c12b8,bf,f4adcd44,...) at kdb_backtrace+0x2a _witness_debugger(c0fda3a4,cac11804,c0e5533d,c9586f10,c0e2e29b,...) at _witness_debugger+0x25 witness_checkorder(cac11804,9,c0e2e292,bf,0,...) at witness_checkorder+0x86f _mtx_lock_flags(cac11804,0,c0e2e292,bf,0,...) at _mtx_lock_flags+0xd9 adadump(c95ba400,c1326460,0,65bcca00,4,...) at adadump+0x67 dump_write(c118d500,c1326460,0,65bcca00,4,...) at dump_write+0xbf minidumpsys(c118d500,c051c7e3,c15b87cc,f4add334,f4add334,...) at minidumpsys+0x356 dumpsys(c118d500,f4add3e4,c051b3a9,0,0,...) at dumpsys+0x2b doadump(0,0,0,0,0,...) at doadump+0x6c db_fncall(c0abd9ba,0,ffffffff,f4add420,0,...) at db_fncall+0x149 db_command(c1153668,f4add510,c051fcd0,c1153668,0,...) at db_command+0x3bf db_command_script(c1153668,0,c1152fc0,c1153667,c1152fc0,...) at db_command_script+0x34 db_script_exec(f4add754,f4add5e0,c051b7a1,1,0,...) at db_script_exec+0x130 db_run_cmd(1,0,c132dd60,f4add558,1,...) at db_run_cmd+0x41 db_command(c0e469be,f4add640,c051d8ed,c0e46ce9,c9775000,...) at db_command+0x381 db_command_loop(c0e46ce9,c9775000,0,f4add618,0,...) at db_command_loop+0x5a db_trap(3,0,1,46,2,...) at db_trap+0xdd kdb_trap(3,0,f4add754,f4add688,d5d22b30,...) at kdb_trap+0xb6 trap(f4add754) at trap+0x79b calltrap() at calltrap+0x6 --- trap 0x3, eip = 0xc0abd9ba, esp = 0xf4add794, ebp = 0xf4add79c --- kdb_enter(c0e46ce9,c0fd687e,f4add7e4,c08fb51d,c95a7200,...) at kdb_enter+0x3a kdb_break(c95a7200,c1bb7600,162,1000,f4add7f8,...) at kdb_break+0x28 uart_intr(c95a7200,c1bb7608,c1bb7608,f4add824,0,...) at uart_intr+0x12d intr_event_handle(c95aab80,f4add850,101,cc3ad2e0,23f,...) at intr_event_handle+0x76 intr_execute_handlers(c96210d0,f4add850,2,f4add8c4,c0deaf45,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,f4add850) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0a749b0, esp = 0xf4add890, ebp = 0xf4add8c4 --- _mtx_lock_spin(c11aab94,ccd688a0,0,c0fd8cb0,222,...) at _mtx_lock_spin+0x130 _mtx_lock_spin_flags(c11aab94,0,c0fd8cb0,222,c130d200,...) at _mtx_lock_spin_flags+0x165 turnstile_trywait(c130d200,c12c12bc,c9582178,ccd68950,0,...) at turnstile_trywait+0x3f _mtx_lock_sleep(c130d200,ccd688a0,0,c100f927,75c,...) at _mtx_lock_sleep+0x218 _mtx_lock_flags(c130d200,0,c100f927,75c,c2aefa40,...) at _mtx_lock_flags+0x10f vm_page_free_toq(c2aefa40,f4add9e8,c0d17413,c2aefa40,0,...) at vm_page_free_toq+0x17e vm_page_free(c2aefa40,0,c100f927,4fe,0,...) at vm_page_free+0x15 vm_page_alloc(d7a85784,1e,0,0,f4addb64,...) at vm_page_alloc+0x583 vm_fault_hold(cc494cb0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc494cb0,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(c0dffe2f,0,ccd688a0,d5d22b30,d5d22b30,...) at trap_pfault+0x1b9 trap(f4addd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- 405 389 373 357 341 325 309 293 277 261 245 229 213 197 181 165 149 133 117 101 85 69 53 37 21 5 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 2 $ svn diff -x -p /var/tmp/vmcontention/sys Index: /var/tmp/vmcontention/sys/vm/vm_radix.c =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.c (revision 235350) +++ /var/tmp/vmcontention/sys/vm/vm_radix.c (working copy) @@ -97,6 +97,7 @@ struct vm_radix_node { CTASSERT(sizeof(struct vm_radix_node) < PAGE_SIZE); static uma_zone_t vm_radix_node_zone; +static u_long allocfail; #ifndef UMA_MD_SMALL_ALLOC static vm_map_t rnode_map; @@ -192,6 +193,10 @@ static __inline struct vm_radix_node * vm_radix_node_get(void) { + if (++allocfail == 100000000) { + allocfail = 0; + return (NULL); + } return (uma_zalloc(vm_radix_node_zone, M_NOWAIT | M_ZERO)); } @@ -280,11 +285,11 @@ vm_radix_setroot(struct vm_radix *rtree, struct vm rtree->rt_root = root; } -static inline void -vm_radix_unwind_heightup(struct vm_radix *rtree, struct vm_radix_node *root, - struct vm_radix_node *iroot, int ilevel) +static void +vm_radix_unwind_heightup(struct vm_radix *rtree, volatile struct vm_radix_node *root, + volatile struct vm_radix_node *iroot, int ilevel) { - struct vm_radix_node *rnode; + volatile struct vm_radix_node *rnode; CTR4(KTR_VM, "unwind: tree %p, root %p, iroot %p, ilevel %d", rtree, root, iroot, ilevel); @@ -293,9 +298,9 @@ vm_radix_setroot(struct vm_radix *rtree, struct vm MPASS(rnode->rn_count == 0 || rnode->rn_count == 1); rnode->rn_count = 0; root = rnode->rn_child[0]; - vm_radix_node_put(rnode); + vm_radix_node_put(__DEVOLATILE(struct vm_radix_node *, rnode)); } - vm_radix_setroot(rtree, iroot, ilevel); + vm_radix_setroot(rtree, __DEVOLATILE(struct vm_radix_node *, iroot), ilevel); } static inline void * @@ -349,15 +354,15 @@ vm_radix_reclaim_allnodes_internal(struct vm_radix int vm_radix_insert(struct vm_radix *rtree, vm_pindex_t index, void *val) { - struct vm_radix_node *iroot, *rnode, *root; - u_int allocmsk; - int clev, ilevel, level, slot; + volatile struct vm_radix_node *iroot, *rnode, *root; + volatile u_int allocmsk; + volatile int clev, ilevel, level, slot; CTR3(KTR_VM, "insert: tree %p, index %ju, val %p", rtree, (uintmax_t)index, val); if (index == -1) panic("vm_radix_insert: -1 is not a valid index.\n"); - level = vm_radix_height(rtree, &root); + level = vm_radix_height(rtree, __DEVOLATILE(struct vm_radix_node **, &root)); /* * Increase the height by adding nodes at the root until * there is sufficient space. @@ -395,7 +400,7 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ } root = rnode; } - vm_radix_setroot(rtree, root, level); + vm_radix_setroot(rtree, __DEVOLATILE(struct vm_radix_node *, root), level); } /* Now that the tree is tall enough, fill in the path to the index. */ -- Test scenario: altbufferflushes.sh