GENERIC HEAD r236123M, vmcore.376 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r236123M: Sun May 27 01:16:21 CEST 2012 pho@x4.osted.lan:/var/tmp/vmcontention/sys/i386/compile/PHO i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.02-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) lock order reversal: 1st 0xc1bb505c system map (system map) @ vm/vm_map.c:3723 2nd 0xc130cd00 vm object (kernel object) @ vm/vm_fault.c:299 3rd 0xc1bb50e8 system map (system map) @ vm/vm_map.c:1454 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,34313a63,a3435,c0000a39,c1183500,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda07d,c12d8d48,5ae,c18207c8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda07d,c1bb50e8,c100daea,c9582110,c100e168,...) at _witness_debugger+0x25 witness_checkorder(c1bb50e8,9,c100e15f,5ae,0,...) at witness_checkorder+0x86f _mtx_lock_flags(c1bb50e8,0,c100e15f,5ae,c596e000,...) at _mtx_lock_flags+0xd9 _vm_map_lock(c1bb508c,c100e15f,5ae,a,0,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,c182087c,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd05a7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,c18208e3,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d124,871,c1820930,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,11,c596ff44,c1820990,c0d0015f,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,c1bb7600,101,827,c9582110,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,30cd30,5,...) at uma_zalloc_arg+0x59f vm_radix_insert(c130cd30,23480,0,c21fbf80,0,...) at vm_radix_insert+0x1e4 vm_page_insert(c21fbf80,c130cd00,23480,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(c130cd00,23480,0,40,c1820bbc,...) at vm_page_alloc+0x51c vm_fault_hold(c1bb5000,e337f000,0,1,0,...) at vm_fault_hold+0x60a vm_fault(c1bb5000,e337f000,0,1,c1bb3a68,...) at vm_fault+0x8b vm_fault_wire(c1bb5000,e3000000,e33c9000,0,0,...) at vm_fault_wire+0x3f vm_map_wire(c1bb5000,e3000000,e33c9000,0,e3000000,...) at vm_map_wire+0x2bc kmem_alloc(c1bb5000,3c9000,c955c770,0,0,...) at kmem_alloc+0x10d vm_ksubmap_init(c131e64c,0,1,1000,0,...) at vm_ksubmap_init+0x64 cpu_startup(0,181ec00,181ec00,181e000,1825000,...) at cpu_startup+0x24f mi_startup() at mi_startup+0xac begin() at begin+0x2c avail memory = 3538612224 (3374 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... WARNING: / was not properly dismounted Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: 15267 files, 398388 used, 614627 free (1307 frags, 76665 blocks, 0.1% fragmentation) /dev/ad4s1f: 20912 files, 889992 used, 123023 free (5295 frags, 14716 blocks, 0.5% fragmentation) /dev/label/tmp: 5929 files, 37539076 used, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) /dev/ad4s1d: 755816 files, 7692293 used, 2461906 free (205066 frags, 282105 blocks, 2.0% fragmentation) /dev/ad4s1e: 812932 files, 15341797 used, 10046720 free (77728 frags, 1246124 blocks, 0.3% fragmentation) Mounting local file systems:. Setting hostname: x4.osted.lan. re0: link state changed to DOWN re0: link state changed to UP Starting Network: lo0 re0 fwe0 fwip0. lo0: flags=8049 metric 0 mtu 16384 options=3 inet6 ::1 prefixlen 128 inet6 fe80::1%lo0 prefixlen 64 scopeid 0xa inet 127.0.0.1 netmask 0xff000000 nd6 options=21 re0: flags=8843 metric 0 mtu 1500 options=8209b ether 00:21:85:15:33:7d inet 192.168.1.101 netmask 0xffffff00 broadcast 192.168.1.255 inet6 fe80::221:85ff:fe15:337d%re0 prefixlen 64 scopeid 0x1 nd6 options=29 media: Ethernet autoselect (100baseTX ) status: active fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 Starting devd. Starting Network: fwe0. fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 Starting Network: fwip0. fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/kde4/lib /usr/local/lib/compat /usr/local/lib/gcc46 /usr/local/lib/mysql /usr/local/lib/nss /usr/local/lib/qt4 /usr/local/lib/virtualbox a.out ldconfig path: /usr/lib/aout /usr/lib/compat/aout Creating and/or trimming log files. Starting syslogd. savecore: reboot May 27 08:23:47 x4 savecore: reboot lock order reversal: 1st 0xe3117ea0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcad2b800 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:284 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,2e687361,38323a63,c1000a34,cc090390,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d7850,11c,f403e7c8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,cad2b800,c100ba7a,c9589610,c100b70c,...) at _witness_debugger+0x25 witness_checkorder(cad2b800,9,c100b703,11c,0,...) at witness_checkorder+0x86f _sx_xlock(cad2b800,0,c100b703,11c,cc5c00e8,...) at _sx_xlock+0x9a ufsdirhash_acquire(e3117e40,cc5c00e8,2c,e3af90b0,f403e898,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(cc5c00e8,f403e904,10b0,f403e884,f403e888,...) at ufsdirhash_add+0x13 ufs_direnter(cc5cbd98,cc491000,f403e904,f403ebb8,0,...) at ufs_direnter+0x799 ufs_makeinode(f403ebb8,c1137b40,f403eafc,f403ea58,c0e287e0,...) at ufs_makeinode+0x65d ufs_create(f403eafc,c102d20e,c1149960,cc5cbd98,f403eb14,...) at ufs_create+0x30 VOP_CREATE_APV(c1137580,f403eafc,f403ebb8,f403ea94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f403eb78,f403ec40,180,0,c95d9b80,...) at vn_open_cred+0x205 vn_open(f403eb78,f403ec40,180,cc02a968,c1baa0f0,...) at vn_open+0x3b kern_openat(cc0902e0,ffffff9c,28807000,0,602,...) at kern_openat+0x1ec kern_open(cc0902e0,28807000,0,601,180,...) at kern_open+0x35 sys_open(cc0902e0,f403ecec,c1025a5c,c0fbc991,c1027010,...) at sys_open+0x30 syscall(f403ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x281828d3, esp = 0xbfbfe28c, ebp = 0xbfbfeda8 --- savecore: writing core to vmcore.375 Writing crash summary to /var/crash/core.txt.375. Additional ABI support: linux. Starting rpcbind. NFS access cache time=60 rpc.umntall: 127.0.0.1: MOUNTPROG: RPC: Program not registered Clearing /tmp (X related). Starting mountd. Starting nfsuserd. Starting nfsd. Recovering vi editor sessions:. Updating motd:. Starting ntpd. Starting default moused. Expensive timeout(9) function: 0xc08cdd10(0xc132da60) 0.012300725 s Configuring syscons: keymap blanktime. Starting sshd. Starting cron. Local package initialization: backuplock order reversal: 1st 0xcc0f1310 ufs (ufs) @ kern/vfs_subr.c:2159 2nd 0xe3368bc0 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:260 3rd 0xcc8ab310 ufs (ufs) @ kern/vfs_subr.c:2159 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,3a632e73,a303632,0,3,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda07d,c12d6da8,86f,f404a83c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda07d,cc8ab310,c0fc2544,c95895a8,c0fe20be,...) at _witness_debugger+0x25 witness_checkorder(cc8ab310,9,c0fe20b5,86f,0,...) at witness_checkorder+0x86f __lockmgr_args(cc8ab310,80100,cc8ab37c,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f404a944,c0ad492c,0,4,cc05ce90,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c1137580,f404a944,cc06dc30,c11496e0,cc8ab2b8,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc8ab2b8,80100,c0fe20b5,86f,eb,...) at _vn_lock+0x78 vget(cc8ab2b8,80100,cc06db80,50,0,...) at vget+0xbb vfs_hash_get(cc31a2dc,32ac00,80000,cc06db80,f404aa90,...) at vfs_hash_get+0xed ffs_vgetf(cc31a2dc,32ac00,80000,f404aa90,1,...) at ffs_vgetf+0x49 softdep_sync_buf(cc0f12b8,e3368b60,1,106,0,...) at softdep_sync_buf+0x4a3 ffs_syncvnode(cc0f12b8,1,0,5fe,cc0f137c,...) at ffs_syncvnode+0x24c ffs_sync(cc31a2dc,1,f404abb8,51f,c1bae710,...) at ffs_sync+0x379 dounmount(cc31a2dc,8080000,cc06db80,4a1,f404ac3c,...) at dounmount+0x3f8 sys_unmount(cc06db80,f404acec,c1025a5c,c0fdad32,c1027010,...) at sys_unmount+0x421 syscall(f404ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280beebb, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- lock order reversal: 1st 0xcc05c880 ufs (ufs) @ kern/vfs_mount.c:1254 2nd 0xcc05cdf0 devfs (devfs) @ ufs/ffs/ffs_softdep.c:1834 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,38313a63,a3433,cc5b8310,8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d6d88,72a,f404a9a8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,cc05cdf0,c0fcd460,c9589408,c1007b75,...) at _witness_debugger+0x25 witness_checkorder(cc05cdf0,9,c1007b6c,72a,cc05ce5c,...) at witness_checkorder+0x86f __lockmgr_args(cc05cdf0,80400,cc05ce5c,0,0,...) at __lockmgr_args+0x8c5 vop_stdlock(f404aaac,c130c580,4,c0fd05a7,eb,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c10f7d40,f404aaac,c130c580,c11496e0,cc05cd98,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc05cd98,80400,c1007b6c,72a,0,...) at _vn_lock+0x78 softdep_flushworklist(cc31a2dc,f404ab64,cc06db80,612,cc0f137c,...) at softdep_flushworklist+0x47 ffs_sync(cc31a2dc,1,f404abb8,51f,c1bae710,...) at ffs_sync+0x3bf dounmount(cc31a2dc,8080000,cc06db80,4a1,f404ac3c,...) at dounmount+0x3f8 sys_unmount(cc06db80,f404acec,c1025a5c,c0fdad32,c1027010,...) at sys_unmount+0x421 syscall(f404ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280beebb, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups May 27 08:26:16 x4 su: pho to root on /dev/pts/0 5929 files, 37539076 used, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5929 files, 37539076 used, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Starting inetd. Sun May 27 08:28:24 CEST 2012 FreeBSD/i386 (x4.osted.lan) (console) login: lock order reversal: 1st 0xe30f64c0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xce22f11c snaplk (snaplk) @ ufs/ffs/ffs_snapshot.c:2297 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,632e746f,3932323a,a37,ffffffdd,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d7b00,8f9,efbe39c0,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,ce22f11c,c1007634,c9589c90,c10075be,...) at _witness_debugger+0x25 witness_checkorder(ce22f11c,9,c10075b5,8f9,0,...) at witness_checkorder+0x86f __lockmgr_args(ce22f11c,80900,cbfe0e5c,0,0,...) at __lockmgr_args+0x8c5 ffs_copyonwrite(cbfe0d98,e33b5cc0,e33b5cc0,4,0,...) at ffs_copyonwrite+0x2c2 ffs_geom_strategy(cbfe0e90,e33b5cc0,3a0,e33b5cc0,1800,...) at ffs_geom_strategy+0xeb bufwrite(e33b5cc0,0,c100aa50,811,40f) at bufwrite+0x159 ffs_bufwrite(e33b5cc0,cb711000,efbe3b84,c0cdc631,e33b5cc0,...) at ffs_bufwrite+0x290 bawrite(e33b5cc0,ea208000,1800,1800,0,...) at bawrite+0x5c ffs_sbupdate(cc0bc000,3,0,efbe3bc0,c0b249ba,...) at ffs_sbupdate+0x171 ffs_sync(cbfb8b70,3,2,e06,cbfb8b70,...) at ffs_sync+0x192 sync_fsync(efbe3c6c,c102c6c4,80400,c0fe20b5,6df,...) at sync_fsync+0x156 VOP_FSYNC_APV(c1113f80,efbe3c6c,c0fe20b5,6df,cacf98a0,...) at VOP_FSYNC_APV+0x100 sync_vnode(c13048b4,c13048a0,3e8,783,4e20,...) at sync_vnode+0x16b sched_sync(0,efbe3d28,c0fcc590,3d8,cbedb598,...) at sched_sync+0x282 fork_exit(c0b2aca0,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- lock order reversal: 1st 0xcc5cddf0 ufs (ufs) @ kern/vfs_syscalls.c:3587 2nd 0xce22f11c snaplk (snaplk) @ ufs/ffs/ffs_snapshot.c:2297 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,632e746f,3932323a,c0000a37,0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d7620,8f9,f406292c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,ce22f11c,c1007634,c9589c90,c10075be,...) at _witness_debugger+0x25 witness_checkorder(ce22f11c,9,c10075b5,8f9,0,...) at witness_checkorder+0x86f __lockmgr_args(ce22f11c,80900,cbfe0e5c,0,0,...) at __lockmgr_args+0x8c5 ffs_copyonwrite(cbfe0d98,e321ac60,0,e321ac60,cc5cdd98,...) at ffs_copyonwrite+0x2c2 ffs_geom_strategy(cbfe0e90,e321ac60,c11088c8,c12d7620,cc5cdd98,...) at ffs_geom_strategy+0xeb ufs_strategy(f4062ad0,c102cf0b,c0dea8b1,e321acc0,cbee6000,...) at ufs_strategy+0xd3 VOP_STRATEGY_APV(c1137580,f4062ad0,c0fdf50b,c11498c0,cc5cdd98,...) at VOP_STRATEGY_APV+0x119 bufstrategy(cc5cde90,e321ac60,3a0,2800,0,...) at bufstrategy+0x61 bufwrite(e321ac60,0,c0fdf50b,6d6,0,...) at bufwrite+0x159 vfs_bio_awrite(e321ac60,0,c100b1d1,106,0,...) at vfs_bio_awrite+0x328 ffs_syncvnode(cc5cdd98,1,0,c1137580,cc5cde90,...) at ffs_syncvnode+0x3a5 ffs_fsync(f4062c40,c102c6c4,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c1137580,f4062c40,c0fe3787,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cbee6000,f4062cec,c1025a5c,c0fdbd6f,e,...) at sys_fsync+0x1df syscall(f4062d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x2818ad47, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- Slab at 0xc7997f44, freei 6 = 0. panic: Duplicate free of item 0xc7997240 from zone 0xc1bb69a0(RADIX NODE) cpuid = 1 KDB: enter: panic [ thread pid 2127 tid 100176 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 2127 tid 100176 td 0xcf0d85c0 kdb_enter(c0fd23c4,c0fd23c4,c100d906,f419d614,1,...) at kdb_enter+0x3a panic(c100d906,c7997240,c1bb69a0,c1010bc5,c100d124,...) at panic+0x18c uma_dbg_free(c1bb69a0,0,c7997240,a1f,c0fd05a7,...) at uma_dbg_free+0x17b uma_zfree_arg(c1bb69a0,c7997240,0,1,1,...) at uma_zfree_arg+0x94 vm_radix_sweep(1,0,101,2d65f8,10,...) at vm_radix_sweep+0x1b8 vm_radix_insert(ceb82314,45c10,0,c385e440,0,...) at vm_radix_insert+0x20c vm_page_insert(c385e440,ceb822e4,45c10,0,20,...) at vm_page_insert+0x1b5 vm_page_alloc(ceb822e4,45c10,0,40222,40222,...) at vm_page_alloc+0x51c vm_page_grab(ceb822e4,45c10,0,412a2,c12d6628,...) at vm_page_grab+0x152 allocbuf(e3313b40,4000,c0fdf50b,b00,ffffffe0,...) at allocbuf+0x672 getblk(ce285984,11704,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(ce285984,45c10000,0,4000,c95dab00,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f419dba4,c102d143,ce285984,ce285984,f419db5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c1137580,f419dba4,ce285984,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cc31c1c0,f419dc3c,c95dab00,0,cf0d85c0,...) at vn_write+0x2bc dofilewrite(f419dc3c,ffffffff,ffffffff,0,f419dc18,...) at dofilewrite+0x95 kern_writev(cf0d85c0,4,f419dc3c,f419dc5c,1,...) at kern_writev+0x68 sys_write(cf0d85c0,f419dcec,c1025a5c,c0e320ef,c119b340,...) at sys_write+0x4f syscall(f419dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28177eb3, esp = 0xbfbfe9ac, ebp = 0xbfbfe9d8 --- db:0:bt> show allpcpu Current CPU: 1 cpuid = 0 dynamic pcpu = 0x6d4d00 curthread = 0xc95df8a0: pid 11 "idle: cpu0" curpcb = 0xc93ebd80 fpcurthread = none idlethread = 0xc95df8a0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x828ad00 curthread = 0xcf0d85c0: pid 2127 "dd" curpcb = 0xf419dd80 fpcurthread = none idlethread = 0xc95df5c0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x828dd00 curthread = 0xc95df2e0: pid 11 "idle: cpu2" curpcb = 0xc93f1d80 fpcurthread = none idlethread = 0xc95df2e0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x8290d00 curthread = 0xcbee88a0: pid 19 "softdepflush" curpcb = 0xefbe9d80 fpcurthread = none idlethread = 0xc95df000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 2127 (dd) thread 0xcf0d85c0 (100176) exclusive sleep mutex RADIX NODE (UMA zone) r = 0 (0xc1bb7608) locked @ vm/uma_core.c:2591 exclusive sleep mutex vm object (standard object) r = 0 (0xceb822e4) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3313ba0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3362a40) locked @ kern/vfs_bio.c:2652 exclusive lockmgr ufs (ufs) r = 0 (0xce2859dc) locked @ kern/vfs_vnops.c:645 Process 19 (softdepflush) thread 0xcbee88a0 (100078) exclusive lockmgr bufwait (bufwait) r = 0 (0xe3307be0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe332aae0) locked @ kern/vfs_bio.c:1905 db:0:alllocks> show lockedvnods Locked vnodes 0xce285984: tag ufs, type VREG usecount 1, writecount 1, refcount 7107 mountedhere 0 flags (VI(0x200)) v_object 0xceb822e4 ref 0 pages 285856 lock type ufs: EXCL by thread 0xcf0d85c0 (pid 2127, dd, tid 100176) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273a3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b38c16 at vn_write+0x176 #5 0xc0ad6b85 at dofilewrite+0x95 #6 0xc0ad71f8 at kern_writev+0x68 #7 0xc0ad730f at sys_write+0x4f #8 0xc0e005ee at syscall+0x2fe #9 0xc0dea8b1 at Xint0x80_syscall+0x21 ino 6194268, on dev ada0s1e db:0:lockedvnods> show mount 0xcbfb7b70 /dev/ad4s1a on / (ufs) 0xcbfb8000 devfs on /dev (devfs) 0xcc31a5b8 /dev/ad4s1f on /home (ufs) 0xcc31a000 /dev/ad4s1d on /usr (ufs) 0xcbfb8b70 /dev/ad4s1e on /var (ufs) 0xcbfb8894 procfs on /proc (procfs) 0xcd66d000 /dev/label/tmp on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 2160 1439 1437 1001 S nanslp 0xc118e6e4 sleep 2128 1419 1419 0 S+ piperd 0xcc02d000 egrep 2127 1419 1419 0 R+ CPU 1 dd 1440 1437 1437 1001 S piperd 0xce2a2af0 awk 1439 1437 1437 1001 S wait 0xcc0852cc sh 1438 1434 1438 1001 Ss+ select 0xcc026de4 top 1437 1435 1437 1001 Ss wait 0xcac0b864 sh 1436 1433 1436 1001 Ss kqread 0xcbfb2800 tail 1435 1427 1427 1001 S select 0xcc025664 sshd 1434 1428 1428 1001 S select 0xcbffb924 sshd 1433 1429 1429 1001 S select 0xcac86564 sshd 1429 1217 1429 0 Ss select 0xcac7ada4 sshd 1428 1217 1428 0 Ss select 0xcbfb6d24 sshd 1427 1217 1427 0 Ss select 0xcac862e4 sshd 1419 1307 1419 0 S+ wait 0xcc084598 sh 1383 1 1383 0 Ss+ ttyin 0xcae33870 getty 1382 1 1382 0 Ss+ ttyin 0xcae33a70 getty 1381 1 1381 0 Ss+ ttyin 0xc970d270 getty 1380 1 1380 0 Ss+ ttyin 0xc970d470 getty 1379 1 1379 0 Ss+ ttyin 0xc970d670 getty 1378 1 1378 0 Ss+ ttyin 0xc970d870 getty 1377 1 1377 0 Ss+ ttyin 0xc970da70 getty 1376 1 1376 0 Ss+ ttyin 0xc970dc70 getty 1375 1 1375 0 Ss+ ttyin 0xc970de70 getty 1358 1 1358 0 Ss select 0xcbfb6ce4 inetd 1333 1 1333 0 Ss nanslp 0xc118e6e4 watchdogd 1307 1304 1307 0 S+ wait 0xcc08f864 bash 1304 1303 1304 0 S+ pause 0xcc08fb88 csh 1303 1294 1303 1001 S+ wait 0xcc085000 su 1294 1293 1294 1001 Ss+ wait 0xcc064598 bash 1293 1291 1291 1001 S select 0xcc026724 sshd 1291 1217 1291 0 Ss select 0xcbffbda4 sshd 1228 1 1228 0 Ss nanslp 0xc118e6e4 cron 1223 1 1223 25 Ss pause 0xcc0858bc sendmail 1220 1 1220 0 Ss select 0xcc0255a4 sendmail 1217 1 1217 0 Ss select 0xcac86ce4 sshd 1185 1 1185 0 Ss select 0xcc0269e4 moused 1160 1 1160 0 Ss select 0xcbffbc64 ntpd 1103 1100 1100 0 S (threaded) nfsd 100125 S rpcsvc 0xcbff3ad0 nfsd: service 100124 S rpcsvc 0xcbff3b10 nfsd: service 100123 S rpcsvc 0xcbff3b50 nfsd: service 100117 S rpcsvc 0xcaca8b90 nfsd: master 1100 1 1100 0 Ss select 0xcc0265e4 nfsd 1098 1094 1094 0 S select 0xcbffba64 nfsuserd 1097 1094 1094 0 S select 0xcbfb6264 nfsuserd 1096 1094 1094 0 S select 0xcbffba24 nfsuserd 1095 1094 1094 0 S select 0xcbfb62a4 nfsuserd 1094 1 1094 0 Ss pause 0xcc08e5f0 nfsuserd 1078 1 1078 0 Ss select 0xcac86aa4 mountd 1044 1 1044 0 Ss select 0xcac866a4 rpcbind 968 1 968 0 Ss select 0xcc025324 syslogd 845 1 845 0 Ss select 0xcbffb9e4 devd 19 0 0 0 RL CPU 3 [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc13048b4 [syncer] 16 0 0 0 DL psleep 0xc1304548 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e468 [pagezero] 8 0 0 0 DL psleep 0xc130e048 [vmdaemon] 7 0 0 0 DL psleep 0xc130e010 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151054 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc1307dd8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 D - 0xcacfed04 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d4e4 [yarrow] 13 0 0 0 DL (threaded) [geom] 100016 D - 0xc118a5a8 [g_down] 100015 D - 0xc118a5a4 [g_up] 100014 D - 0xc118a59c [g_event] 12 0 0 0 WL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 Run CPU 2 [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 Run CPU 0 [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130bd00 [audit] 0 0 0 0 DLs (threaded) [kernel] 100069 D - 0xc118d4e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118a760 [swapper] db:0:ps> allt Tracing command sleep pid 2160 tid 100127 td 0xcc3188a0 sched_switch(cc3188a0,0,104,1a3,6e1e4f84,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc3188a0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(2711,c0ac8bc0,cc3188a0,3,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c118e6e4,6c,c0fd394b,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c118e6e4,0,16c,c0fd394b,2711,...) at _sleep+0x34f kern_nanosleep(cc3188a0,f40e6c48,f40e6c50,a,0,...) at kern_nanosleep+0xc1 sys_nanosleep(cc3188a0,f40e6cec,c1025a5c,c0fdb6ce,c1027010,...) at sys_nanosleep+0x6f syscall(f40e6d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28152483, esp = 0xbfbfec4c, ebp = 0xbfbfec98 --- Tracing command egrep pid 2128 tid 100118 td 0xcae265c0 sched_switch(cae265c0,0,104,1a3,dca1a014,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae265c0,0,c0fd7f80,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc02d000,5c,c0fda907,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc02d000,cc02d178,15c,c0fda907,0,...) at _sleep+0x385 pipe_read(cc02f620,f406ec3c,c95dab00,0,cae265c0,...) at pipe_read+0x427 dofileread(f406ec3c,ffffffff,ffffffff,0,f406ec18,...) at dofileread+0x9e kern_readv(cae265c0,0,f406ec3c,f406ec5c,1,...) at kern_readv+0x68 sys_read(cae265c0,f406ecec,c1025a5c,c0fdadfa,c1027010,...) at sys_read+0x4f syscall(f406ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281baed3, esp = 0xbfbfe80c, ebp = 0xbfbfe898 --- Tracing command dd pid 2127 tid 100176 td 0xcf0d85c0 kdb_enter(c0fd23c4,c0fd23c4,c100d906,f419d614,1,...) at kdb_enter+0x3a panic(c100d906,c7997240,c1bb69a0,c1010bc5,c100d124,...) at panic+0x18c uma_dbg_free(c1bb69a0,0,c7997240,a1f,c0fd05a7,...) at uma_dbg_free+0x17b uma_zfree_arg(c1bb69a0,c7997240,0,1,1,...) at uma_zfree_arg+0x94 vm_radix_sweep(1,0,101,2d65f8,10,...) at vm_radix_sweep+0x1b8 vm_radix_insert(ceb82314,45c10,0,c385e440,0,...) at vm_radix_insert+0x20c vm_page_insert(c385e440,ceb822e4,45c10,0,20,...) at vm_page_insert+0x1b5 vm_page_alloc(ceb822e4,45c10,0,40222,40222,...) at vm_page_alloc+0x51c vm_page_grab(ceb822e4,45c10,0,412a2,c12d6628,...) at vm_page_grab+0x152 allocbuf(e3313b40,4000,c0fdf50b,b00,ffffffe0,...) at allocbuf+0x672 getblk(ce285984,11704,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(ce285984,45c10000,0,4000,c95dab00,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f419dba4,c102d143,ce285984,ce285984,f419db5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c1137580,f419dba4,ce285984,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cc31c1c0,f419dc3c,c95dab00,0,cf0d85c0,...) at vn_write+0x2bc dofilewrite(f419dc3c,ffffffff,ffffffff,0,f419dc18,...) at dofilewrite+0x95 kern_writev(cf0d85c0,4,f419dc3c,f419dc5c,1,...) at kern_writev+0x68 sys_write(cf0d85c0,f419dcec,c1025a5c,c0e320ef,c119b340,...) at sys_write+0x4f syscall(f419dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28177eb3, esp = 0xbfbfe9ac, ebp = 0xbfbfe9d8 --- Tracing command awk pid 1440 tid 100087 td 0xcc0098a0 sched_switch(cc0098a0,0,104,1a3,6d10b228,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cc0098a0,0,c0fd7f80,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ce2a2af0,5c,c0fda907,100,0,...) at sleepq_wait_sig+0x17 _sleep(ce2a2af0,ce2a2c68,15c,c0fda907,0,...) at _sleep+0x385 pipe_read(cc31c0e0,f3fe6c3c,cbffa300,0,cc0098a0,...) at pipe_read+0x427 dofileread(f3fe6c3c,ffffffff,ffffffff,0,f3fe6c18,...) at dofileread+0x9e kern_readv(cc0098a0,0,f3fe6c3c,f3fe6c5c,1,...) at kern_readv+0x68 sys_read(cc0098a0,f3fe6cec,c1025a5c,c0e320ef,c1027010,...) at sys_read+0x4f syscall(f3fe6d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281aced3, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command sh pid 1439 tid 100108 td 0xcc0902e0 sched_switch(cc0902e0,0,104,1a3,6e06cd29,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0902e0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0852cc,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0852cc,cc085354,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cc0902e0,ffffffff,f403ec58,2,0,...) at kern_wait+0x597 sys_wait4(cc0902e0,f403ecec,c1025a5c,c0fdae97,c119b340,...) at sys_wait4+0x3b syscall(f403ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command top pid 1438 tid 100146 td 0xcd74fb80 sched_switch(cd74fb80,0,104,1a3,9675486a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd74fb80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4125a14,c0a7478a,cc026dd0,0,cd74fb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc026de4,0,f4125a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc026de4,cc026dd0,3e9,6ae,cc31c2d8,...) at _cv_timedwait_sig+0x270 seltdwait(f4125c08,f4125c10,cc022100,cd74fb80,335b1b0a,...) at seltdwait+0x8a kern_select(cd74fb80,2,bfbfeb9c,0,0,f4125c54,20,1,0) at kern_select+0x655 sys_select(cd74fb80,f4125cec,c1025a5c,c0fa5622,c1027010,...) at sys_select+0x66 syscall(f4125d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281e1e53, esp = 0xbfbfeb3c, ebp = 0xbfbfec98 --- Tracing command sh pid 1437 tid 100086 td 0xcbee82e0 sched_switch(cbee82e0,0,104,1a3,ea5a19a7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cbee82e0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac0b864,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cac0b864,cac0b8ec,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cbee82e0,ffffffff,f3fe1c58,2,0,...) at kern_wait+0x597 sys_wait4(cbee82e0,f3fe1cec,c1025a5c,c0fdae97,c1027010,...) at sys_wait4+0x3b syscall(f3fe1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfeb0c, ebp = 0xbfbfeb28 --- Tracing command tail pid 1436 tid 100152 td 0xcd74e8a0 sched_switch(cd74e8a0,0,104,1a3,45a6cdd4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,68,...) at mi_switch+0x1ff sleepq_switch(cd74e8a0,0,c0fd7f80,1a7,68,...) at sleepq_switch+0x162 sleepq_catch_signals(3e9,c0ac8bc0,cd74e8a0,3,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbfb2800,68,c0fcc142,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(cbfb2800,cbfb2800,168,c0fcc142,3e9,...) at _sleep+0x34f kern_kevent(cd74e8a0,4,0,1,f4137c3c,...) at kern_kevent+0x374 sys_kevent(cd74e8a0,f4137cec,c1025a5c,c0e320ef,c119c6c0,...) at sys_kevent+0x19e syscall(f4137d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2815639b, esp = 0xbfbfeb0c, ebp = 0xbfbfebc8 --- Tracing command sshd pid 1435 tid 100140 td 0xcc005b80 sched_switch(cc005b80,0,104,1a3,fcc65727,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc005b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc025650,0,c0fd05a7,cc005b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc025664,0,f410da50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc025664,cc025650,c0fda6d8,6ae,cc038d90,...) at _cv_wait_sig+0x260 seltdwait(cc038d90,58,cbffaa00,cc005b80,cc005c30,...) at seltdwait+0xa2 kern_select(cc005b80,c,28c0c170,28c0c1b0,0,0,20,cce74864,0) at kern_select+0x655 sys_select(cc005b80,f410dcec,c1025a5c,c0fdb005,c1027010,...) at sys_select+0x66 syscall(f410dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1434 tid 100147 td 0xcd74f8a0 sched_switch(cd74f8a0,0,104,1a3,9678c71c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd74f8a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbffb910,0,c0fd05a7,cd74f8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbffb924,0,f4128a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbffb924,cbffb910,c0fda6d8,6ae,cbfe2818,...) at _cv_wait_sig+0x260 seltdwait(cbfe2818,58,cbfc3080,cd74f8a0,cd74f950,...) at seltdwait+0xa2 kern_select(cd74f8a0,a,28c0c170,28c0c1b0,0,0,20,cce732cc,0) at kern_select+0x655 sys_select(cd74f8a0,f4128cec,c1025a5c,c0fdb005,cacc1b80,...) at sys_select+0x66 syscall(f4128d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1433 tid 100134 td 0xcc0f22e0 sched_switch(cc0f22e0,0,104,1a3,26f9e8d2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0f22e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86550,0,c0fd05a7,cc0f22e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac86564,0,f40fba50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac86564,cac86550,c0fda6d8,6ae,cc02f8f8,...) at _cv_wait_sig+0x260 seltdwait(cc02f8f8,58,c95da780,cc0f22e0,cc0f2390,...) at seltdwait+0xa2 kern_select(cc0f22e0,c,28c0c170,28c0c1b0,0,0,20,ccb57598,0) at kern_select+0x655 sys_select(cc0f22e0,f40fbcec,c1025a5c,c0fdb005,f40fbcb4,...) at sys_select+0x66 syscall(f40fbd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1429 tid 100150 td 0xcd74f000 sched_switch(cd74f000,0,104,1a3,de6512b7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd74f000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac7ad90,0,c0fd05a7,cd74f000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac7ada4,0,f4131ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac7ada4,cac7ad90,c0fda6d8,6ae,f4131b3c,...) at _cv_wait_sig+0x260 seltdwait(cc49782c,c0fda6d8,5ab,cd74f000,0,...) at seltdwait+0xa2 sys_poll(cd74f000,f4131cec,c1025a5c,c0e320ef,f4131cb4,...) at sys_poll+0x321 syscall(f4131d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1428 tid 100149 td 0xcd74f2e0 sched_switch(cd74f2e0,0,104,1a3,df53cff2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd74f2e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfb6d10,0,c0fd05a7,cd74f2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfb6d24,0,f412eab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfb6d24,cbfb6d10,c0fda6d8,6ae,f412eb3c,...) at _cv_wait_sig+0x260 seltdwait(cbe1742c,c0fda6d8,5ab,cd74f2e0,0,...) at seltdwait+0xa2 sys_poll(cd74f2e0,f412ecec,c1025a5c,c0e320ef,c119c040,...) at sys_poll+0x321 syscall(f412ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1427 tid 100148 td 0xcd74f5c0 sched_switch(cd74f5c0,0,104,1a3,df0a3200,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd74f5c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac862d0,0,c0fd05a7,cd74f5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac862e4,0,f412bab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac862e4,cac862d0,c0fda6d8,6ae,f412bb3c,...) at _cv_wait_sig+0x260 seltdwait(cc94032c,c0fda6d8,5ab,cd74f5c0,0,...) at seltdwait+0xa2 sys_poll(cd74f5c0,f412bcec,c1025a5c,c0e320ef,c119c040,...) at sys_poll+0x321 syscall(f412bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sh pid 1419 tid 100113 td 0xcbee65c0 sched_switch(cbee65c0,0,104,1a3,cec6f923,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cbee65c0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc084598,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc084598,cc084620,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cbee65c0,ffffffff,f4053c58,2,0,...) at kern_wait+0x597 sys_wait4(cbee65c0,f4053cec,c1025a5c,c0fdae97,c1027010,...) at sys_wait4+0x3b syscall(f4053d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe76c, ebp = 0xbfbfe788 --- Tracing command getty pid 1383 tid 100160 td 0xcd76f000 sched_switch(cd76f000,0,104,1a3,adb7c72a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd76f000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cd76f000,cd76f000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33870,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33870,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33800,cae33870,f414fc3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33800,f414fc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27100,f414fc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02aab8,f414fc3c,c95d9b80,0,cd76f000,...) at devfs_read_f+0x7e dofileread(f414fc3c,ffffffff,ffffffff,0,f414fc18,...) at dofileread+0x9e kern_readv(cd76f000,0,f414fc3c,f414fc5c,1,...) at kern_readv+0x68 sys_read(cd76f000,f414fcec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f414fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1382 tid 100159 td 0xcd76f2e0 sched_switch(cd76f2e0,0,104,1a3,ae1697d8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd76f2e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cd76f2e0,cd76f2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33a70,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33a70,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33a00,cae33a70,f414cc3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33a00,f414cc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27000,f414cc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02a968,f414cc3c,c95d9b80,0,cd76f2e0,...) at devfs_read_f+0x7e dofileread(f414cc3c,ffffffff,ffffffff,0,f414cc18,...) at dofileread+0x9e kern_readv(cd76f2e0,0,f414cc3c,f414cc5c,1,...) at kern_readv+0x68 sys_read(cd76f2e0,f414ccec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f414cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1381 tid 100158 td 0xcc067000 sched_switch(cc067000,0,104,1a3,b027478c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc067000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc067000,cc067000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d270,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d270,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d200,c970d270,f4149c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d200,f4149c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0ce00,f4149c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cbfe1070,f4149c3c,c95d9b80,0,cc067000,...) at devfs_read_f+0x7e dofileread(f4149c3c,ffffffff,ffffffff,0,f4149c18,...) at dofileread+0x9e kern_readv(cc067000,0,f4149c3c,f4149c5c,1,...) at kern_readv+0x68 sys_read(cc067000,f4149cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f4149d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1380 tid 100157 td 0xcc0672e0 sched_switch(cc0672e0,0,104,1a3,ada1a4e5,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0672e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc0672e0,cc0672e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d470,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d470,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d400,c970d470,f4146c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d400,f4146c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cd00,f4146c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cbfe1310,f4146c3c,c95d9b80,0,cc0672e0,...) at devfs_read_f+0x7e dofileread(f4146c3c,ffffffff,ffffffff,0,f4146c18,...) at dofileread+0x9e kern_readv(cc0672e0,0,f4146c3c,f4146c5c,1,...) at kern_readv+0x68 sys_read(cc0672e0,f4146cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f4146d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1379 tid 100156 td 0xcc0675c0 sched_switch(cc0675c0,0,104,1a3,ae046af2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0675c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc0675c0,cc0675c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d670,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d670,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d600,c970d670,f4143c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d600,f4143c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cc00,f4143c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02a6c8,f4143c3c,c95d9b80,0,cc0675c0,...) at devfs_read_f+0x7e dofileread(f4143c3c,ffffffff,ffffffff,0,f4143c18,...) at dofileread+0x9e kern_readv(cc0675c0,0,f4143c3c,f4143c5c,1,...) at kern_readv+0x68 sys_read(cc0675c0,f4143cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f4143d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1378 tid 100155 td 0xcc0678a0 sched_switch(cc0678a0,0,104,1a3,ae470869,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0678a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc0678a0,cc0678a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d870,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d870,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d800,c970d870,f4140c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d800,f4140c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c900,f4140c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02fc40,f4140c3c,c95d9b80,0,cc0678a0,...) at devfs_read_f+0x7e dofileread(f4140c3c,ffffffff,ffffffff,0,f4140c18,...) at dofileread+0x9e kern_readv(cc0678a0,0,f4140c3c,f4140c5c,1,...) at kern_readv+0x68 sys_read(cc0678a0,f4140cec,c1025a5c,c0e36eb2,cc06d000,...) at sys_read+0x4f syscall(f4140d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1377 tid 100154 td 0xcc067b80 sched_switch(cc067b80,0,104,1a3,ae622e3d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc067b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc067b80,cc067b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970da70,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970da70,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970da00,c970da70,f413dc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970da00,f413dc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c800,f413dc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cbfe2038,f413dc3c,c95d9b80,0,cc067b80,...) at devfs_read_f+0x7e dofileread(f413dc3c,ffffffff,ffffffff,0,f413dc18,...) at dofileread+0x9e kern_readv(cc067b80,0,f413dc3c,f413dc5c,1,...) at kern_readv+0x68 sys_read(cc067b80,f413dcec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f413dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1376 tid 100153 td 0xcc06d000 sched_switch(cc06d000,0,104,1a3,ae31e6a5,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc06d000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc06d000,cc06d000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970dc70,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970dc70,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970dc00,c970dc70,f413ac3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970dc00,f413ac3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c700,f413ac3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02f1f8,f413ac3c,c95d9b80,0,cc06d000,...) at devfs_read_f+0x7e dofileread(f413ac3c,ffffffff,ffffffff,0,f413ac18,...) at dofileread+0x9e kern_readv(cc06d000,0,f413ac3c,f413ac5c,1,...) at kern_readv+0x68 sys_read(cc06d000,f413acec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f413ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1375 tid 100095 td 0xcbee6b80 sched_switch(cbee6b80,0,104,1a3,aeb869d6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee6b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,c970de04,0,c0fd05a7,cbee6b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970de70,0,f4007ae8,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970de70,c970de04,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970de00,c970de70,f4007c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970de00,f4007c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(c95c6100,f4007c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc038658,f4007c3c,c95d9b80,0,cbee6b80,...) at devfs_read_f+0x7e dofileread(f4007c3c,ffffffff,ffffffff,0,f4007c18,...) at dofileread+0x9e kern_readv(cbee6b80,0,f4007c3c,f4007c5c,1,...) at kern_readv+0x68 sys_read(cbee6b80,f4007cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f4007d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command inetd pid 1358 tid 100101 td 0xcc0948a0 sched_switch(cc0948a0,0,104,1a3,83e9f203,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0948a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfb6cd0,0,c0fd05a7,cc0948a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfb6ce4,0,f4021a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfb6ce4,cbfb6cd0,c0fda6d8,6ae,cbfe25e8,...) at _cv_wait_sig+0x260 seltdwait(cbfe25e8,58,c95d9b80,cc0948a0,202,...) at seltdwait+0xa2 kern_select(cc0948a0,8,bfbfdf90,0,0,0,20,cc08e864,4) at kern_select+0x655 sys_select(cc0948a0,f4021cec,c1025a5c,c0fdaf04,c1027010,...) at sys_select+0x66 syscall(f4021d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2819be53, esp = 0xbfbfdf4c, ebp = 0xbfbfedd8 --- Tracing command watchdogd pid 1333 tid 100111 td 0xcc06db80 sched_switch(cc06db80,0,104,1a3,ecf054ef,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc06db80,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(ea61,c0ac8bc0,cc06db80,0,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c118e6e4,6c,c0fd394b,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c118e6e4,0,16c,c0fd394b,ea61,...) at _sleep+0x34f kern_nanosleep(cc06db80,f404ac48,f404ac50,3c,0,...) at kern_nanosleep+0xc1 sys_nanosleep(cc06db80,f404acec,c1025a5c,c0e36eb2,c1027010,...) at sys_nanosleep+0x6f syscall(f404ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28181483, esp = 0xbfbfecac, ebp = 0xbfbfecd8 --- Tracing command bash pid 1307 tid 100138 td 0xccd8fb80 sched_switch(ccd8fb80,0,104,1a3,7ed2ef3e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(ccd8fb80,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08f864,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc08f864,cc08f8ec,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(ccd8fb80,ffffffff,f4107c58,6,0,...) at kern_wait+0x597 sys_wait4(ccd8fb80,f4107cec,c1025a5c,c0fdadb7,c1027010,...) at sys_wait4+0x3b syscall(f4107d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe78c, ebp = 0xbfbfe7a8 --- Tracing command csh pid 1304 tid 100137 td 0xcc0958a0 sched_switch(cc0958a0,0,104,1a3,4d8ac9f4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0958a0,0,c0fd7f80,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08fb88,74,c0e58cce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc08fb88,cc08fbb8,174,c0e58cce,0,...) at _sleep+0x385 kern_sigsuspend(cc0958a0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0958a0,f4104cec,c1025a5c,c0fdadb7,c1027010,...) at sys_sigsuspend+0x4d syscall(f4104d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x2816fd7b, esp = 0xbfbfe07c, ebp = 0xbfbfe0e8 --- Tracing command su pid 1303 tid 100090 td 0xcbee8000 sched_switch(cbee8000,0,104,1a3,f83e4c44,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cbee8000,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc085000,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc085000,cc085088,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cbee8000,518,f3ff3c58,2,0,...) at kern_wait+0x597 sys_wait4(cbee8000,f3ff3cec,c1025a5c,c0fdb6ce,c1027010,...) at sys_wait4+0x3b syscall(f3ff3d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2810b78b, esp = 0xbfbfe5cc, ebp = 0xbfbfe5e8 --- Tracing command bash pid 1294 tid 100112 td 0xcc06d8a0 sched_switch(cc06d8a0,0,104,1a3,fd6eac56,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc06d8a0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc064598,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc064598,cc064620,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cc06d8a0,ffffffff,f404ec58,6,0,...) at kern_wait+0x597 sys_wait4(cc06d8a0,f404ecec,c1025a5c,c0fdadb7,c1027010,...) at sys_wait4+0x3b syscall(f404ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command sshd pid 1293 tid 100132 td 0xcc0f28a0 sched_switch(cc0f28a0,0,104,1a3,cea7639b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0f28a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc026710,0,c0fd05a7,cc0f28a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc026724,0,f40f5a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc026724,cc026710,c0fda6d8,6ae,cc02f540,...) at _cv_wait_sig+0x260 seltdwait(cc02f540,58,cbffa000,cc0f28a0,cc0f2950,...) at seltdwait+0xa2 kern_select(cc0f28a0,c,28c0c170,28c0c1b0,0,0,20,ccb57b30,0) at kern_select+0x655 sys_select(cc0f28a0,f40f5cec,c1025a5c,c0fdb005,c1027010,...) at sys_select+0x66 syscall(f40f5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1291 tid 100131 td 0xcc0f2b80 sched_switch(cc0f2b80,0,104,1a3,79f491f0,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0f2b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbffbd90,0,c0fd05a7,cc0f2b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbffbda4,0,f40f2ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbffbda4,cbffbd90,c0fda6d8,6ae,f40f2b3c,...) at _cv_wait_sig+0x260 seltdwait(cc384a2c,c0fda6d8,5ab,cc0f2b80,696,...) at seltdwait+0xa2 sys_poll(cc0f2b80,f40f2cec,c1025a5c,c0fbc991,c1027010,...) at sys_poll+0x321 syscall(f40f2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command cron pid 1228 tid 100099 td 0xcc095000 sched_switch(cc095000,0,104,1a3,5cd44d2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc095000,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(e679,c0ac8bc0,cc095000,0,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c118e6e4,6c,c0fd394b,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c118e6e4,0,16c,c0fd394b,e679,...) at _sleep+0x34f kern_nanosleep(cc095000,f4019c48,f4019c50,3b,0,...) at kern_nanosleep+0xc1 sys_nanosleep(cc095000,f4019cec,c1825c80,0,c1027010,...) at sys_nanosleep+0x6f syscall(f4019d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28174483, esp = 0xbfbfec3c, ebp = 0xbfbfec68 --- Tracing command sendmail pid 1223 tid 100106 td 0xcc0908a0 sched_switch(cc0908a0,0,104,1a3,fc1c71a3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0908a0,0,c0fd7f80,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0858bc,74,c0e58cce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0858bc,cc0858ec,174,c0e58cce,0,...) at _sleep+0x385 kern_sigsuspend(cc0908a0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0908a0,f4036cec,c1025a5c,c0fdadb7,c119c040,...) at sys_sigsuspend+0x4d syscall(f4036d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28322d7b, esp = 0xbfbfcf8c, ebp = 0xbfbfcfb8 --- Tracing command sendmail pid 1220 tid 100089 td 0xcc06eb80 sched_switch(cc06eb80,0,104,1a3,c9bd78bc,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc06eb80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3feea14,c0a7478a,cc025590,0,cc06eb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc0255a4,0,f3feea50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc0255a4,cc025590,1389,6ae,cc02f188,...) at _cv_timedwait_sig+0x270 seltdwait(f3feec08,f3feec10,c95da900,cc06eb80,40,...) at seltdwait+0x8a kern_select(cc06eb80,5,bfbfc500,0,0,f3feec54,20,5,0) at kern_select+0x655 sys_select(cc06eb80,f3feecec,c1025a5c,c0fdb294,cacc1b80,...) at sys_select+0x66 syscall(f3feed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x283dae53, esp = 0xbfbfc46c, ebp = 0xbfbfcf98 --- Tracing command sshd pid 1217 tid 100091 td 0xcc06e8a0 sched_switch(cc06e8a0,0,104,1a3,de523863,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc06e8a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86cd0,0,c0fd05a7,cc06e8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac86ce4,0,f3ff7a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac86ce4,cac86cd0,c0fda6d8,6ae,cbfe1b28,...) at _cv_wait_sig+0x260 seltdwait(cbfe1b28,58,cc022a80,cc06e8a0,f3ff7b14,...) at seltdwait+0xa2 kern_select(cc06e8a0,9,28c0c140,0,0,0,20,cc066598,a) at kern_select+0x655 sys_select(cc06e8a0,f3ff7cec,c1025a5c,c0fbc991,14,...) at sys_select+0x66 syscall(f3ff7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfdedc, ebp = 0xbfbfede8 --- Tracing command moused pid 1185 tid 100094 td 0xcc0092e0 sched_switch(cc0092e0,0,104,1a3,cabf37b5,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0092e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc0269d0,0,c0fd05a7,cc0092e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0269e4,0,f4003a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0269e4,cc0269d0,c0fda6d8,6ae,cbfe22d8,...) at _cv_wait_sig+0x260 seltdwait(cbfe22d8,58,c95d9b80,cc0092e0,f6,...) at seltdwait+0xa2 kern_select(cc0092e0,400,bfbfe9f4,0,0,0,20,cbedc2cc,1) at kern_select+0x655 sys_select(cc0092e0,f4003cec,c1025a5c,c0e36eb2,c1027010,...) at sys_select+0x66 syscall(f4003d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281a8e53, esp = 0xbfbfe98c, ebp = 0xbfbfeb18 --- Tracing command ntpd pid 1160 tid 100109 td 0xcc06e2e0 sched_switch(cc06e2e0,0,104,1a3,985e9b28,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc06e2e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbffbc50,0,c0fd05a7,cc06e2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbffbc64,0,f4042a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbffbc64,cbffbc50,c0fda6d8,6ae,cc02adc8,...) at _cv_wait_sig+0x260 seltdwait(cc02adc8,58,c95d9b80,cc06e2e0,0,...) at seltdwait+0xa2 kern_select(cc06e2e0,1c,bfbfecd8,0,0,0,20,cc066000,0) at kern_select+0x655 sys_select(cc06e2e0,f4042cec,c1025a5c,c0fdb6e0,e,...) at sys_select+0x66 syscall(f4042d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28358e53, esp = 0xbfbfecac, ebp = 0xbfbfed78 --- Tracing command nfsd pid 1103 tid 100125 td 0xcc319000 sched_switch(cc319000,0,104,1a3,6e85c2e2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc319000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40e0bdc,c0a7478a,cbc36e80,0,cc319000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbff3ad0,0,f40e0c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbff3ad0,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40e0d14,c0a56e68,cbc36e80,f40e0d28,c0fcc590,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40e0d28,c0fcc590,3d8,cc081864,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40e0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1103 tid 100124 td 0xcc3192e0 sched_switch(cc3192e0,0,104,1a3,6e180dae,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3192e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40ddbdc,c0a7478a,cbc36e80,0,cc3192e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbff3b10,0,f40ddc18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbff3b10,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40ddd14,c0a56e68,cbc36e80,f40ddd28,c0fcc590,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40ddd28,c0fcc590,3d8,cc081864,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40ddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1103 tid 100123 td 0xcc3195c0 sched_switch(cc3195c0,0,104,1a3,6e17fa06,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3195c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40dabdc,c0a7478a,cbc36e80,0,cc3195c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbff3b50,0,f40dac18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbff3b50,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40dad14,c0a56e68,cbc36e80,f40dad28,c0fcc590,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40dad28,c0fcc590,3d8,cc081864,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40dad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1103 tid 100117 td 0xcae268a0 sched_switch(cae268a0,0,104,1a3,6e17ea1d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae268a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f406a48c,c0a7478a,cbc36e80,0,cae268a0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca8b90,0,f406a4c8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca8b90,cbc36e80,1388,3b1,f406a510,...) at _cv_timedwait_sig+0x270 svc_run_internal(cae26a50,14,c10027ba,c0fbde9b,f406a808,...) at svc_run_internal+0x356 svc_run(cbc36e80,0,c0fbde76,19f,c95859f0,...) at svc_run+0x7b nfsrvd_nfsd(cae268a0,f406a808,c,ab,c0fd05a7,...) at nfsrvd_nfsd+0x1c3 nfssvc_nfsd(cae268a0,f406acec,0,cae268a0,f406ad1c,...) at nfssvc_nfsd+0x177 sys_nfssvc(cae268a0,f406acec,c1025a5c,c0fdb294,c1027010,...) at sys_nfssvc+0xeb syscall(f406ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (155, FreeBSD ELF32, sys_nfssvc), eip = 0x280bf9bb, esp = 0xbfbfe81c, ebp = 0xbfbfea88 --- Tracing command nfsd pid 1100 tid 100103 td 0xcc0942e0 sched_switch(cc0942e0,0,104,1a3,89437b1a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0942e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc0265d0,0,c0fd05a7,cc0942e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0265e4,0,f4029a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0265e4,cc0265d0,c0fda6d8,6ae,cc02f0e0,...) at _cv_wait_sig+0x260 seltdwait(cc02f0e0,58,c95d9b80,cc0942e0,0,...) at seltdwait+0xa2 kern_select(cc0942e0,7,bfbfeca4,0,0,0,20,cc08e2cc,1) at kern_select+0x655 sys_select(cc0942e0,f4029cec,c1025a5c,c0fdb294,c1027010,...) at sys_select+0x66 syscall(f4029d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28176e53, esp = 0xbfbfea8c, ebp = 0xbfbfedd8 --- Tracing command nfsuserd pid 1098 tid 100116 td 0xcae26b80 sched_switch(cae26b80,0,104,1a3,6c60ff94,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae26b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4066a14,c0a7478a,cbffba50,0,cae26b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbffba64,0,f4066a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbffba64,cbffba50,7531,6ae,cc02a230,...) at _cv_timedwait_sig+0x270 seltdwait(f4066c08,f4066c10,c95d9b80,cae26b80,0,...) at seltdwait+0x8a kern_select(cae26b80,4,bfbfebb4,0,0,f4066c54,20,1e,0) at kern_select+0x655 sys_select(cae26b80,f4066cec,c1025a5c,c0fdafee,c1027010,...) at sys_select+0x66 syscall(f4066d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1097 tid 100080 td 0xcc00ab80 sched_switch(cc00ab80,0,104,1a3,6c459770,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc00ab80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fc9a14,c0a7478a,cbfb6250,0,cc00ab80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbfb6264,0,f3fc9a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbfb6264,cbfb6250,7531,6ae,cc02a230,...) at _cv_timedwait_sig+0x270 seltdwait(f3fc9c08,f3fc9c10,c95d9b80,cc00ab80,0,...) at seltdwait+0x8a kern_select(cc00ab80,4,bfbfebb4,0,0,f3fc9c54,20,1e,0) at kern_select+0x655 sys_select(cc00ab80,f3fc9cec,c1025a5c,c0fdafee,c1027010,...) at sys_select+0x66 syscall(f3fc9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1096 tid 100097 td 0xcc0955c0 sched_switch(cc0955c0,0,104,1a3,6c4601c0,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0955c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4010a14,c0a7478a,cbffba10,0,cc0955c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbffba24,0,f4010a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbffba24,cbffba10,7531,6ae,cc02a230,...) at _cv_timedwait_sig+0x270 seltdwait(f4010c08,f4010c10,c95d9b80,cc0955c0,0,...) at seltdwait+0x8a kern_select(cc0955c0,4,bfbfebb4,0,0,f4010c54,20,1e,0) at kern_select+0x655 sys_select(cc0955c0,f4010cec,c1025a5c,c0fdafee,c1027010,...) at sys_select+0x66 syscall(f4010d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1095 tid 100085 td 0xcc009b80 sched_switch(cc009b80,0,104,1a3,6c45ac18,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc009b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fdda14,c0a7478a,cbfb6290,0,cc009b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbfb62a4,0,f3fdda50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbfb62a4,cbfb6290,7531,6ae,cc02a230,...) at _cv_timedwait_sig+0x270 seltdwait(f3fddc08,f3fddc10,c95d9b80,cc009b80,0,...) at seltdwait+0x8a kern_select(cc009b80,4,bfbfebb4,0,0,f3fddc54,20,1e,0) at kern_select+0x655 sys_select(cc009b80,f3fddcec,c1025a5c,c0fdafee,cc06d5c0,...) at sys_select+0x66 syscall(f3fddd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1094 tid 100102 td 0xcc0945c0 sched_switch(cc0945c0,0,104,1a3,7fb3a950,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0945c0,0,c0fd7f80,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08e5f0,74,c0e58cce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc08e5f0,cc08e620,174,c0e58cce,0,...) at _sleep+0x385 kern_sigsuspend(cc0945c0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0945c0,f4025cec,c1025a5c,c0fdb294,c119b340,...) at sys_sigsuspend+0x4d syscall(f4025d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x280bdd7b, esp = 0xbfbfec4c, ebp = 0xbfbfede8 --- Tracing command mountd pid 1078 tid 100092 td 0xcc06e5c0 sched_switch(cc06e5c0,0,104,1a3,35f2a8c1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc06e5c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86a90,0,c0fd05a7,cc06e5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac86aa4,0,f3ffba50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac86aa4,cac86a90,c0fda6d8,6ae,cc02a348,...) at _cv_wait_sig+0x260 seltdwait(cc02a348,58,c95d9b80,cc06e5c0,f3ffbac0,...) at seltdwait+0xa2 kern_select(cc06e5c0,9,bfbfecf4,0,0,0,20,cc0662cc,bfbfecf4) at kern_select+0x655 sys_select(cc06e5c0,f3ffbcec,c1025a5c,c0fbbbf7,aa,...) at sys_select+0x66 syscall(f3ffbd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de53, esp = 0xbfbfec8c, ebp = 0xbfbfedd8 --- Tracing command rpcbind pid 1044 tid 100096 td 0xcbee68a0 sched_switch(cbee68a0,0,104,1a3,75d25878,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee68a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f400ba78,c0a7478a,cac86690,0,cbee68a0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac866a4,0,f400bab4,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac866a4,cac86690,7531,6ae,f400bb6c,...) at _cv_timedwait_sig+0x270 seltdwait(f400bc3c,f400bc44,5ab,cbee68a0,cc3a4bec,...) at seltdwait+0x8a sys_poll(cbee68a0,f400bcec,c1025a5c,c0fdafee,c1027010,...) at sys_poll+0x321 syscall(f400bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2812c3ab, esp = 0xbfbfcb9c, ebp = 0xbfbfed68 --- Tracing command syslogd pid 968 tid 100115 td 0xcbee6000 sched_switch(cbee6000,0,104,1a3,2e1bf09d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee6000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc025310,0,c0fd05a7,cbee6000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc025324,0,f4062a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc025324,cc025310,c0fda6d8,6ae,cbfe1770,...) at _cv_wait_sig+0x260 seltdwait(cbfe1770,58,c95d9b80,cbee6000,0,...) at seltdwait+0xa2 kern_select(cbee6000,9,28849100,0,0,0,20,cc084000,0) at kern_select+0x655 sys_select(cbee6000,f4062cec,c1025a5c,c0fdaea7,e,...) at sys_select+0x66 syscall(f4062d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de53, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- Tracing command devd pid 845 tid 100107 td 0xcc0905c0 sched_switch(cc0905c0,0,104,1a3,df0e97cb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0905c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbffb9d0,0,c0fd05a7,cc0905c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbffb9e4,0,f403aa50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbffb9e4,cbffb9d0,c0fda6d8,6ae,cbfe2498,...) at _cv_wait_sig+0x260 seltdwait(cbfe2498,58,c95d9b80,cc0905c0,c11088c8,...) at seltdwait+0xa2 kern_select(cc0905c0,5,bfbfe950,0,0,0,20,cc085598,34) at kern_select+0x655 sys_select(cc0905c0,f403acec,c93e1c80,0,c1027010,...) at sys_select+0x66 syscall(f403ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x80893bb, esp = 0xbfbfe91c, ebp = 0xbfbfedf8 --- Tracing command softdepflush pid 19 tid 100078 td 0xcbee88a0 cpustop_handler(3,efbe9860,c0e00ae3,0,c0fdf50b,...) at cpustop_handler+0x26 ipi_nmi_handler(0,c0fdf50b,a5c,efbe97c0,cbedb000,...) at ipi_nmi_handler+0x35 trap(efbe986c) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0ad24b4, esp = 0xefbe98ac, ebp = 0xefbe98c4 --- witness_lock(cbfe0e90,8,c0fdf50b,a4e,0,...) at witness_lock+0x34 _mtx_lock_flags(cbfe0e90,0,c0fdf50b,a4e,1,...) at _mtx_lock_flags+0x166 getblk(cbfe0d98,4e3e800,0,4000,0,...) at getblk+0xe3 breadn_flags(cbfe0d98,4e3e800,0,4000,0,...) at breadn_flags+0x43 ffs_blkfree_cg(13973c0,0,4000,5e8458,efbe9b08,...) at ffs_blkfree_cg+0x2c9 ffs_blkfree(cc0bc000,cb711000,cbfe0d98,13973c0,0,...) at ffs_blkfree+0x9e indir_trunc(fffc7ff4,ffffffff,c1007b6c,1e0e,0,...) at indir_trunc+0x5b7 indir_trunc(fffff7f3,ffffffff,c1007b6c,1cd0,c12d7a40,...) at indir_trunc+0x4ba handle_workitem_freeblocks(0,efbe9c70,2,6ba,efbe9c74,...) at handle_workitem_freeblocks+0x24f process_worklist_item(c130c580,0,c1007b6c,5e9,cc0bc000,...) at process_worklist_item+0x2c2 softdep_process_worklist(cbfb8b70,0,c1007b6c,562,3e8,...) at softdep_process_worklist+0x84 softdep_flush(0,efbe9d28,c0fcc590,3d8,cbedb000,...) at softdep_flush+0x2a0 fork_exit(c0cd7e10,0,efbe9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe9d60, ebp = 0 --- Tracing command vnlru pid 18 tid 100077 td 0xcbee8b80 sched_switch(cbee8b80,0,104,1a3,48c1b9b8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cbee8b80,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cbedb2cc,60,c0fe30d9,0,0,...) at sleepq_timedwait+0x6b _sleep(cbedb2cc,c1304874,260,c0fe30d9,3e8,...) at _sleep+0x36a vnlru_proc(0,efbe6d28,c0fcc590,3d8,cbedb2cc,...) at vnlru_proc+0xe7 fork_exit(c0b2b970,0,efbe6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe6d60, ebp = 0 --- Tracing command syncer pid 17 tid 100076 td 0xcacf98a0 sched_switch(cacf98a0,0,104,1a3,6d8eab39,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf98a0,0,c0fd7f80,28b,cacf98a0,...) at sleepq_switch+0x162 sleepq_timedwait(c13048b4,0,efbe3c78,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c13048b4,c13048a0,3e8,783,4e20,...) at _cv_timedwait+0x270 sched_sync(0,efbe3d28,c0fcc590,3d8,cbedb598,...) at sched_sync+0x595 fork_exit(c0b2aca0,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xcacf9b80 sched_switch(cacf9b80,0,104,1a3,9111c2c1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cacf9b80,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304548,54,c0fdfe52,0,0,...) at sleepq_timedwait+0x6b _sleep(c1304548,c1304550,54,c0fdfe52,3e8,...) at _sleep+0x36a buf_daemon(0,efbe0d28,c0fcc590,3d8,cbedb864,...) at buf_daemon+0x175 fork_exit(c0b0d9c0,0,efbe0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe0d60, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xcae24000 sched_switch(cae24000,0,104,1a3,6aec95c2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24000,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e468,0,c101128c,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e468,c130cf00,0,c101128c,493e0,...) at _sleep+0x36a vm_pagezero(0,efbddd28,c0fcc590,3d8,cbedbb30,...) at vm_pagezero+0xdc fork_exit(c0d1fa20,0,efbddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbddd60, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xcae242e0 sched_switch(cae242e0,0,104,1a3,58225cf7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cae242e0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130e048,74,c0fdfe52,0,0,...) at sleepq_wait+0x63 _sleep(c130e048,c130e04c,74,c0fdfe52,0,...) at _sleep+0x39c vm_daemon(0,efbdad28,c0fcc590,3d8,cbedc000,...) at vm_daemon+0x59 fork_exit(c0d18320,0,efbdad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbdad60, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xcae245c0 sched_switch(cae245c0,0,104,1a3,a20f0e27,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cae245c0,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e010,54,c0fdfe52,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e010,c130cf00,54,c0fdfe52,1388,...) at _sleep+0x36a vm_pageout(0,efbd7d28,c0fcc590,3d8,c95de2cc,...) at vm_pageout+0x2c7 fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xcae248a0 sched_switch(cae248a0,0,104,1a3,cf1bba71,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae248a0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1151054,5c,c0e2c8de,0,0,...) at sleepq_wait+0x63 _sleep(c1151054,c1151098,5c,c0e2c8de,0,...) at _sleep+0x39c xpt_scanner_thread(0,efa54d28,c0fcc590,3d8,c95de598,...) at xpt_scanner_thread+0x47 fork_exit(c04a72f0,0,efa54d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa54d60, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xcae24b80 sched_switch(cae24b80,0,104,1a3,63f435bb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24b80,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1307dd8,0,c0ff759f,0,0,...) at sleepq_wait+0x63 _sleep(c1307dd8,c1307da0,0,c0ff759f,0,...) at _sleep+0x39c sctp_iterator_thread(0,efa51d28,c0fcc590,3d8,c95de864,...) at sctp_iterator_thread+0x5c fork_exit(c0bc5a50,0,efa51d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa51d60, ebp = 0 --- Tracing command fdc0 pid 4 tid 100065 td 0xcae25b80 sched_switch(cae25b80,0,104,1a3,88981634,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae25b80,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cac7223c,5c,c0fbe912,0,0,...) at sleepq_timedwait+0x6b _sleep(cac7223c,cac722f0,5c,c0fbe912,3e8,...) at _sleep+0x36a fdc_thread(cac72200,efa42d28,c0fcc590,3d8,c95deb30,...) at fdc_thread+0x27d fork_exit(c0da1d50,cac72200,efa42d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa42d60, ebp = 0 --- Tracing command fw0_probe pid 3 tid 100063 td 0xcace42e0 sched_switch(cace42e0,0,104,1a3,cf1bd85d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cace42e0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cad2c000,6c,c0fbe912,100,0,...) at sleepq_wait_sig+0x17 _sleep(cad2c000,cad30488,16c,c0fbe912,0,...) at _sleep+0x385 fw_bus_probe_thread(cad2c000,efa32d28,c0fcc590,3d8,cac0b000,...) at fw_bus_probe_thread+0xa08 fork_exit(c07017d0,cad2c000,efa32d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa32d60, ebp = 0 --- Tracing command usb pid 15 tid 100059 td 0xcacf7000 sched_switch(cacf7000,0,104,1a3,bdf6d83a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf7000,0,c0fd7f80,268,cacf7000,...) at sleepq_switch+0x162 sleepq_wait(cacfed34,0,efa1bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed34,cacfee4c,c0faff91,74,cacfed3c,...) at _cv_wait+0x260 usb_process(cacfed2c,efa1bd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfed2c,efa1bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa1bd60, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xcacf72e0 sched_switch(cacf72e0,0,104,1a3,faf98592,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf72e0,0,c0fd7f80,268,cacf72e0,...) at sleepq_switch+0x162 sleepq_wait(cacfed04,0,efa18cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed04,cacfee4c,c0faff91,74,cacfed0c,...) at _cv_wait+0x260 usb_process(cacfecfc,efa18d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfecfc,efa18d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa18d60, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xcacf75c0 sched_switch(cacf75c0,0,104,1a3,bd904169,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf75c0,0,c0fd7f80,268,cacf75c0,...) at sleepq_switch+0x162 sleepq_wait(cacfecd4,0,efa15cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfecd4,cacfee4c,c0faff91,74,cacfecdc,...) at _cv_wait+0x260 usb_process(cacfeccc,efa15d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfeccc,efa15d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa15d60, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xcacf78a0 sched_switch(cacf78a0,0,104,1a3,bd903117,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf78a0,0,c0fd7f80,268,cacf78a0,...) at sleepq_switch+0x162 sleepq_wait(cacfeca4,0,efa12cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfeca4,cacfee4c,c0faff91,74,cacfecac,...) at _cv_wait+0x260 usb_process(cacfec9c,efa12d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfec9c,efa12d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa12d60, ebp = 0 --- Tracing command usb pid 15 tid 100054 td 0xcacf9000 sched_switch(cacf9000,0,104,1a3,bd901ac2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf9000,0,c0fd7f80,268,cacf9000,...) at sleepq_switch+0x162 sleepq_wait(cacefb5c,0,efa09cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb5c,cacefc74,c0faff91,74,cacefb64,...) at _cv_wait+0x260 usb_process(cacefb54,efa09d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb54,efa09d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa09d60, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xcacf92e0 sched_switch(cacf92e0,0,104,1a3,d310e7fb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf92e0,0,c0fd7f80,268,cacf92e0,...) at sleepq_switch+0x162 sleepq_wait(cacefb2c,0,efa06cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb2c,cacefc74,c0faff91,74,cacefb34,...) at _cv_wait+0x260 usb_process(cacefb24,efa06d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb24,efa06d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa06d60, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xcacf95c0 sched_switch(cacf95c0,0,104,1a3,b67dd605,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf95c0,0,c0fd7f80,268,cacf95c0,...) at sleepq_switch+0x162 sleepq_wait(cacefafc,0,efa03cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefafc,cacefc74,c0faff91,74,cacefb04,...) at _cv_wait+0x260 usb_process(cacefaf4,efa03d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefaf4,efa03d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa03d60, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xcace2000 sched_switch(cace2000,0,104,1a3,b67dc407,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2000,0,c0fd7f80,268,cace2000,...) at sleepq_switch+0x162 sleepq_wait(cacefacc,0,efa00cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefacc,cacefc74,c0faff91,74,cacefad4,...) at _cv_wait+0x260 usb_process(cacefac4,efa00d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefac4,efa00d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa00d60, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xcace22e0 sched_switch(cace22e0,0,104,1a3,b67daf9e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace22e0,0,c0fd7f80,268,cace22e0,...) at sleepq_switch+0x162 sleepq_wait(caceab5c,0,ef9fccac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab5c,caceac74,c0faff91,74,caceab64,...) at _cv_wait+0x260 usb_process(caceab54,ef9fcd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab54,ef9fcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9fcd60, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xcace25c0 sched_switch(cace25c0,0,104,1a3,d310b223,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace25c0,0,c0fd7f80,268,cace25c0,...) at sleepq_switch+0x162 sleepq_wait(caceab2c,0,ef9f9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab2c,caceac74,c0faff91,74,caceab34,...) at _cv_wait+0x260 usb_process(caceab24,ef9f9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab24,ef9f9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f9d60, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xcace28a0 sched_switch(cace28a0,0,104,1a3,b61a216e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace28a0,0,c0fd7f80,268,cace28a0,...) at sleepq_switch+0x162 sleepq_wait(caceaafc,0,ef9f6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaafc,caceac74,c0faff91,74,caceab04,...) at _cv_wait+0x260 usb_process(caceaaf4,ef9f6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaaf4,ef9f6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f6d60, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xcace2b80 sched_switch(cace2b80,0,104,1a3,b61a0f6e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2b80,0,c0fd7f80,268,cace2b80,...) at sleepq_switch+0x162 sleepq_wait(caceaacc,0,ef9f3cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaacc,caceac74,c0faff91,74,caceaad4,...) at _cv_wait+0x260 usb_process(caceaac4,ef9f3d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaac4,ef9f3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f3d60, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xcace3000 sched_switch(cace3000,0,104,1a3,b619fb03,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace3000,0,c0fd7f80,268,cace3000,...) at sleepq_switch+0x162 sleepq_wait(cace5b5c,0,ef9efcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b5c,cace5c74,c0faff91,74,cace5b64,...) at _cv_wait+0x260 usb_process(cace5b54,ef9efd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b54,ef9efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9efd60, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xcace32e0 sched_switch(cace32e0,0,104,1a3,d3106fc1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace32e0,0,c0fd7f80,268,cace32e0,...) at sleepq_switch+0x162 sleepq_wait(cace5b2c,0,ef9eccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b2c,cace5c74,c0faff91,74,cace5b34,...) at _cv_wait+0x260 usb_process(cace5b24,ef9ecd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b24,ef9ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ecd60, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xcace35c0 sched_switch(cace35c0,0,104,1a3,b5b597f0,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace35c0,0,c0fd7f80,268,cace35c0,...) at sleepq_switch+0x162 sleepq_wait(cace5afc,0,ef9e9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5afc,cace5c74,c0faff91,74,cace5b04,...) at _cv_wait+0x260 usb_process(cace5af4,ef9e9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5af4,ef9e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e9d60, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xcace38a0 sched_switch(cace38a0,0,104,1a3,b5b58736,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace38a0,0,c0fd7f80,268,cace38a0,...) at sleepq_switch+0x162 sleepq_wait(cace5acc,0,ef9e6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5acc,cace5c74,c0faff91,74,cace5ad4,...) at _cv_wait+0x260 usb_process(cace5ac4,ef9e6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5ac4,ef9e6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e6d60, ebp = 0 --- Tracing command usb pid 15 tid 100041 td 0xcace4000 sched_switch(cace4000,0,104,1a3,b5b571d4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace4000,0,c0fd7f80,268,cace4000,...) at sleepq_switch+0x162 sleepq_wait(cacddb5c,0,ef9dfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb5c,cacddc74,c0faff91,74,cacddb64,...) at _cv_wait+0x260 usb_process(cacddb54,ef9dfd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb54,ef9dfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dfd60, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xcacbf000 sched_switch(cacbf000,0,104,1a3,d3103f30,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf000,0,c0fd7f80,268,cacbf000,...) at sleepq_switch+0x162 sleepq_wait(cacddb2c,0,ef9dccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb2c,cacddc74,c0faff91,74,cacddb34,...) at _cv_wait+0x260 usb_process(cacddb24,ef9dcd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb24,ef9dcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dcd60, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xcacbf2e0 sched_switch(cacbf2e0,0,104,1a3,b5521d87,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf2e0,0,c0fd7f80,268,cacbf2e0,...) at sleepq_switch+0x162 sleepq_wait(cacddafc,0,ef9d9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddafc,cacddc74,c0faff91,74,cacddb04,...) at _cv_wait+0x260 usb_process(cacddaf4,ef9d9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddaf4,ef9d9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d9d60, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xcacbf5c0 sched_switch(cacbf5c0,0,104,1a3,b5520bfd,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf5c0,0,c0fd7f80,268,cacbf5c0,...) at sleepq_switch+0x162 sleepq_wait(cacddacc,0,ef9d6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddacc,cacddc74,c0faff91,74,cacddad4,...) at _cv_wait+0x260 usb_process(cacddac4,ef9d6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddac4,ef9d6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d6d60, ebp = 0 --- Tracing command usb pid 15 tid 100036 td 0xcacbfb80 sched_switch(cacbfb80,0,104,1a3,b551f690,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbfb80,0,c0fd7f80,268,cacbfb80,...) at sleepq_switch+0x162 sleepq_wait(cacd5b5c,0,ef9cfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b5c,cacd5c74,c0faff91,74,cacd5b64,...) at _cv_wait+0x260 usb_process(cacd5b54,ef9cfd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b54,ef9cfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9cfd60, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xcacc1000 sched_switch(cacc1000,0,104,1a3,d3100560,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc1000,0,c0fd7f80,268,cacc1000,...) at sleepq_switch+0x162 sleepq_wait(cacd5b2c,0,ef9cccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b2c,cacd5c74,c0faff91,74,cacd5b34,...) at _cv_wait+0x260 usb_process(cacd5b24,ef9ccd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b24,ef9ccd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ccd60, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xcacc12e0 sched_switch(cacc12e0,0,104,1a3,b4ee5818,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc12e0,0,c0fd7f80,268,cacc12e0,...) at sleepq_switch+0x162 sleepq_wait(cacd5afc,0,ef9c9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5afc,cacd5c74,c0faff91,74,cacd5b04,...) at _cv_wait+0x260 usb_process(cacd5af4,ef9c9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5af4,ef9c9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c9d60, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xcacc15c0 sched_switch(cacc15c0,0,104,1a3,b4ee442d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc15c0,0,c0fd7f80,268,cacc15c0,...) at sleepq_switch+0x162 sleepq_wait(cacd5acc,0,ef9c6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5acc,cacd5c74,c0faff91,74,cacd5ad4,...) at _cv_wait+0x260 usb_process(cacd5ac4,ef9c6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5ac4,ef9c6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c6d60, ebp = 0 --- Tracing command ctl_thrd pid 2 tid 100028 td 0xc96eb8a0 sched_switch(c96eb8a0,0,104,1a3,b4ee28a9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb8a0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c97c4000,5c,c0e37350,0,0,...) at sleepq_wait+0x63 _sleep(c97c4000,c97c4000,5c,c0e37350,0,...) at _sleep+0x39c ctl_work_thread(c97c4000,ef733d28,c0fcc590,3d8,cac0b598,...) at ctl_work_thread+0x22f6 fork_exit(c04d08d0,c97c4000,ef733d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef733d60, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xc96ea5c0 sched_switch(c96ea5c0,0,104,1a3,a19c22fd,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ea5c0,0,c0fd7f80,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe912,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe912,64) at _sleep+0x36a pause(c0fbe912,64,c0f8bff0,111,0,...) at pause+0xa0 random_kthread(0,e33e0d28,c0fcc590,3d8,c95dd000,...) at random_kthread+0x1ef fork_exit(c082d020,0,e33e0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e0d60, ebp = 0 --- Tracing command geom pid 13 tid 100016 td 0xc96ea8a0 sched_switch(c96ea8a0,0,104,1a3,a1c2586f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96ea8a0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a5a8,5c,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c118a5a8,c118a508,25c,c0fbe912,0,...) at _sleep+0x39c g_io_schedule_down(c96ea8a0,0,c0fc0461,6c,e33ddd14,...) at g_io_schedule_down+0x5b g_down_procbody(0,e33ddd28,c0fcc590,3d8,c95dd2cc,...) at g_down_procbody+0x9d fork_exit(c09e6af0,0,e33ddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ddd60, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xc96eab80 sched_switch(c96eab80,0,104,1a3,a1e20ee1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eab80,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a5a4,5c,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c118a5a4,c118a528,25c,c0fbe912,0,...) at _sleep+0x39c g_io_schedule_up(c96eab80,0,c0fc0461,5f,e33dad14,...) at g_io_schedule_up+0x134 g_up_procbody(0,e33dad28,c0fcc590,3d8,c95dd2cc,...) at g_up_procbody+0x9d fork_exit(c09e6b90,0,e33dad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33dad60, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xc96eb000 sched_switch(c96eb000,0,104,1a3,8cdb102e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb000,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a59c,5c,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c118a59c,c118a4cc,25c,c0fbe912,0,...) at _sleep+0x39c g_run_events(c119b340,0,c0fc0461,79,e33d7d14,...) at g_run_events+0x54b g_event_procbody(0,e33d7d28,c0fcc590,3d8,c95dd2cc,...) at g_event_procbody+0x90 fork_exit(c09e6c30,0,e33d7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d7d60, ebp = 0 --- Tracing command intr pid 12 tid 100067 td 0xcae255c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100066 td 0xcae258a0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100064 td 0xcae26000 sched_switch(cae26000,0,109,1a3,aeb83189,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cae26000,...) at mi_switch+0x1ff ithread_loop(cae18a40,efa35d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cae18a40,efa35d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa35d60, ebp = 0 --- Tracing command intr pid 12 tid 100061 td 0xcace48a0 sched_switch(cace48a0,0,109,1a3,a4dae33f,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cace48a0,...) at mi_switch+0x1ff ithread_loop(cad15880,efa29d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15880,efa29d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa29d60, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xcace4b80 sched_switch(cace4b80,0,109,1a3,90d9aa47,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cace4b80,...) at mi_switch+0x1ff ithread_loop(cad15010,efa21d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15010,efa21d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa21d60, ebp = 0 --- Tracing command intr pid 12 tid 100055 td 0xcacf7b80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100042 td 0xcace3b80 sched_switch(cace3b80,0,109,1a3,d61e02a6,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cace3b80,...) at mi_switch+0x1ff ithread_loop(cacc5720,ef9e3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacc5720,ef9e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xcacbf8a0 sched_switch(cacbf8a0,0,109,1a3,d4d49de1,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cacbf8a0,...) at mi_switch+0x1ff ithread_loop(cacad590,ef9d3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad590,ef9d3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d3d60, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xcacc18a0 sched_switch(cacc18a0,0,109,1a3,769fd86f,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cacc18a0,...) at mi_switch+0x1ff ithread_loop(cacad830,ef9c3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad830,ef9c3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c3d60, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xcacc1b80 sched_switch(cacc1b80,0,109,1a3,a1d1f1d6,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cacc1b80,...) at mi_switch+0x1ff ithread_loop(cac97ae0,ef86dd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97ae0,ef86dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef86dd60, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc96eb2e0 sched_switch(c96eb2e0,0,109,1a3,a133743e,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96eb2e0,...) at mi_switch+0x1ff ithread_loop(cac97c80,ef869d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97c80,ef869d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef869d60, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc96eb5c0 sched_switch(c96eb5c0,0,109,1a3,a6ba96e8,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96eb5c0,...) at mi_switch+0x1ff ithread_loop(cac97500,ef763d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97500,ef763d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef763d60, ebp = 0 --- Tracing command intr pid 12 tid 100022 td 0xc97c1b80 sched_switch(c97c1b80,0,109,1a3,3a8ff80d,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c97c1b80,...) at mi_switch+0x1ff ithread_loop(c95b3450,e33efd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3450,e33efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33efd60, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc96228a0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100019 td 0xc96ea000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100018 td 0xc96ea2e0 sched_switch(c96ea2e0,0,109,1a3,50e0621e,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96ea2e0,...) at mi_switch+0x1ff ithread_loop(c95b3490,e33e3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3490,e33e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc95e05c0 sched_switch(c95e05c0,0,109,1a3,889805c8,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c95e05c0,...) at mi_switch+0x1ff ithread_loop(c95b36f0,e33d1d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b36f0,e33d1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d1d60, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc95e08a0 sched_switch(c95e08a0,0,109,1a3,7021e4e3,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c95e08a0,...) at mi_switch+0x1ff ithread_loop(c95b3700,e33ced28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3700,e33ced28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ced60, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc95e0b80 sched_switch(c95e0b80,0,109,1a3,96dcdb74,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c95e0b80,...) at mi_switch+0x1ff ithread_loop(c95b3710,e33cbd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3710,e33cbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33cbd60, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc9622000 sched_switch(c9622000,0,109,1a3,a2734218,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c9622000,...) at mi_switch+0x1ff ithread_loop(c95b3720,c93fed28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3720,c93fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fed60, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc96222e0 sched_switch(c96222e0,0,109,1a3,57d4532c,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96222e0,...) at mi_switch+0x1ff ithread_loop(c95b3730,c93fbd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3730,c93fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fbd60, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc96225c0 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc95df000 sched_switch(c95df000,0,108,19e,a1c1c8dd,...) at sched_switch+0x413 mi_switch(108,0,c0fd460c,a1d,c95df000,...) at mi_switch+0x1ff sched_idletd(0,c93f4d28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x28f fork_exit(c0aaf180,0,c93f4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f4d60, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc95df2e0 cpustop_handler(2,c93f1c30,c0e00ae3,c93dec80,c93f1b80,...) at cpustop_handler+0x26 ipi_nmi_handler(c93dec80,c93f1b80,c0a742f3,c93dec80,c95dd864,...) at ipi_nmi_handler+0x35 trap(c93f1c3c) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0df064d, esp = 0xc93f1c7c, ebp = 0xc93f1c80 --- cpu_idle_mwait(1,2710,c119c040,2,c93f1ce8,...) at cpu_idle_mwait+0x4d cpu_idle(1,c93f1cc0,c0fd460c,a1f,c95df2e0,...) at cpu_idle+0x30 sched_idletd(0,c93f1d28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x235 fork_exit(c0aaf180,0,c93f1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f1d60, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc95df5c0 sched_switch(c95df5c0,0,108,19e,a20f066e,...) at sched_switch+0x413 mi_switch(108,0,c0fd460c,a1d,c95df5c0,...) at mi_switch+0x1ff sched_idletd(0,c93eed28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x28f fork_exit(c0aaf180,0,c93eed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93eed60, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc95df8a0 cpustop_handler(0,c93ebc30,c0e00ae3,c1825c80,c93ebb80,...) at cpustop_handler+0x26 ipi_nmi_handler(c1825c80,c93ebb80,c0a742f3,c1825c80,c95dd864,...) at ipi_nmi_handler+0x35 trap(c93ebc3c) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0df064d, esp = 0xc93ebc7c, ebp = 0xc93ebc80 --- cpu_idle_mwait(1,2710,c119b340,2,c93ebce8,...) at cpu_idle_mwait+0x4d cpu_idle(1,c93ebcc0,c0fd460c,a1f,c95df8a0,...) at cpu_idle+0x30 sched_idletd(0,c93ebd28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x235 fork_exit(c0aaf180,0,c93ebd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93ebd60, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc95dfb80 sched_switch(c95dfb80,0,104,1a3,604453c4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c95dfb80,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c95ddb30,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(c95ddb30,c95ddbb8,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(c95dfb80,ffffffff,c93e7c58,0,0,...) at kern_wait+0x597 sys_wait4(c95dfb80,c93e7cec,c1025a5c,c0fdad06,c1027010,...) at sys_wait4+0x3b syscall(c93e7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x806070f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8c8 --- Tracing command audit pid 10 tid 100001 td 0xc95e0000 sched_switch(c95e0000,0,104,1a3,b4ebf52c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e0000,0,c0fd7f80,268,c95e0000,...) at sleepq_switch+0x162 sleepq_wait(c130bd00,0,c93e4c8c,1,0,...) at sleepq_wait+0x63 _cv_wait(c130bd00,c130bce4,c1004237,194,c93e4cc4,...) at _cv_wait+0x260 audit_worker(0,c93e4d28,c0fcc590,3d8,c95de000,...) at audit_worker+0x84 fork_exit(c0c95110,0,c93e4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93e4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xcae25000 sched_switch(cae25000,0,104,1a3,7023f759,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae25000,0,c0fd7f80,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe912,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe912,bb8) at _sleep+0x36a pause(c0fbe912,bb8,126,124,c118a7e8,...) at pause+0xa0 deadlkres(0,efa4ed28,c0fcc590,3d8,c118a760,...) at deadlkres+0x324 fork_exit(c0a398a0,0,efa4ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4ed60, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xcae252e0 sched_switch(cae252e0,0,104,1a3,a4dca1aa,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae252e0,0,c0fd7f80,268,cae252e0,...) at sleepq_switch+0x162 sleepq_wait(c971ddc0,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971ddc0,c971ddd8,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c136d440,efa4bd28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c136d440,efa4bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4bd60, ebp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xcace45c0 sched_switch(cace45c0,0,104,1a3,9c8cccfa,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace45c0,0,c0fd7f80,268,cace45c0,...) at sleepq_switch+0x162 sleepq_wait(c971de00,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971de00,c971de18,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(cad3049c,efa2fd28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,cad3049c,efa2fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa2fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc96ebb80 sched_switch(c96ebb80,0,104,1a3,cf203008,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ebb80,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de40,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c971de40,c971de58,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c118b0f8,e33fed28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c118b0f8,e33fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fed60, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc97c1000 sched_switch(c97c1000,0,104,1a3,cf201f80,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c1000,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de80,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c971de80,c971de98,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c130c2e4,e33fbd28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c130c2e4,e33fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fbd60, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc97c12e0 sched_switch(c97c12e0,0,104,1a3,cf200f2b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c12e0,0,c0fd7f80,268,c97c12e0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f8d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f8d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f8d60, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc97c15c0 sched_switch(c97c15c0,0,104,1a3,cf20002a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c15c0,0,c0fd7f80,268,c97c15c0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f5d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f5d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f5d60, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc97c18a0 sched_switch(c97c18a0,0,104,1a3,cf1ff06c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c18a0,0,c0fd7f80,268,c97c18a0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f2d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f2d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f2d60, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc9622b80 sched_switch(c9622b80,0,104,1a3,7ff4a786,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c9622b80,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971e0c0,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c971e0c0,c971e0d8,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c11aa0ac,e33e9d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11aa0ac,e33e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e9d60, ebp = 0 --- Tracing command kernel pid 0 tid 100013 td 0xc95e02e0 sched_switch(c95e02e0,0,104,1a3,c71e2656,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e02e0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c96e9000,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c96e9000,c96e9018,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c11a8ac0,e33d4d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11a8ac0,e33d4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc118aa30 sched_switch(c118aa30,0,104,1a3,8733994d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c118aa30,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c118a760,54,c1004c18,0,0,...) at sleepq_timedwait+0x6b _sleep(c118a760,0,54,c1004c18,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x240 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 330 MB: 315 299 283 267 251 235 219 203 187 171 155 139 123 107 91 75 59 43 27 11 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 1 (kgdb) bt #0 doadump (textdump=0x0) at pcpu.h:244 #1 0xc051b3a9 in db_fncall (dummy1=0xc0abd94a, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf419d260 "tÒ\031ô") at ../../../ddb/db_command.c:573 #2 0xc051b7df in db_command (last_cmdp=0xc115245c, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:449 #3 0xc051b894 in db_command_script (command=0xc1153368 "call doadump") at ../../../ddb/db_command.c:520 #4 0xc051fcd0 in db_script_exec (scriptname=0xc1152cc0 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc051fd61 in db_run_cmd (addr=0x1, have_addr=0x0, count=0xc132da60, modif=0xf419d398 "") at ../../../ddb/db_script.c:375 #6 0xc051b7a1 in db_command (last_cmdp=0xc115245c, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:449 #7 0xc051b8fa in db_command_loop () at ../../../ddb/db_command.c:502 #8 0xc051d8ed in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:231 #9 0xc0abde66 in kdb_trap (type=0x3, code=0x0, tf=0xf419d594) at ../../../kern/subr_kdb.c:654 #10 0xc0e0124b in trap (frame=0xf419d594) at ../../../i386/i386/trap.c:710 #11 0xc0dea81c in calltrap () at ../../../i386/i386/exception.s:169 #12 0xc0abd94a in kdb_enter (why=0xc0fd23c4 "panic", msg=0xc0fd23c4 "panic") at cpufunc.h:71 #13 0xc0a86e3c in panic (fmt=0xc100d906 "Duplicate free of item %p from zone %p(%s)\n") at ../../../kern/kern_shutdown.c:633 #14 0xc0d0095b in uma_dbg_free (zone=0xc1bb69a0, slab=0xc7997f44, item=0xc7997240) at ../../../vm/uma_dbg.c:302 #15 0xc0cff7e4 in uma_zfree_arg (zone=0xc1bb69a0, item=0xc7997240, udata=0x0) at ../../../vm/uma_core.c:2595 #16 0xc0d1d1f8 in vm_radix_sweep (rtree=0xceb82314, index=0x45c10, color=0x1, hard=0x0) at uma.h:337 #17 0xc0d1dd8c in vm_radix_insert (rtree=0xceb82314, index=0x45c10, val=0xc385e440) at ../../../vm/vm_radix.c:528 #18 0xc0d14f95 in vm_page_insert (m=0xc385e440, object=0xceb822e4, pindex=0x45c10) at ../../../vm/vm_page.c:811 #19 0xc0d171fc in vm_page_alloc (object=0xceb822e4, pindex=0x45c10, req=0x40222) at ../../../vm/vm_page.c:1273 #20 0xc0d17722 in vm_page_grab (object=0xceb822e4, pindex=0x45c10, allocflags=0x412a2) at ../../../vm/vm_page.c:2290 #21 0xc0b0ad92 in allocbuf (bp=0xe3313b40, size=0x4000) at ../../../kern/vfs_bio.c:3067 #22 0xc0b0eb53 in getblk (vp=0xce285984, blkno=0x11704, size=0x4000, slpflag=0x0, slptimeo=0x0, flags=0x0) at ../../../kern/vfs_bio.c:2837 #23 0xc0cb6d5c in ffs_balloc_ufs2 (vp=0xce285984, startoffset=Variable "startoffset" is not available. ) at ../../../ufs/ffs/ffs_balloc.c:926 #24 0xc0ce1f0d in ffs_write (ap=0xf419dba4) at ../../../ufs/ffs/ffs_vnops.c:721 #25 0xc0e281e4 in VOP_WRITE_APV (vop=0xc1137580, a=0xf419dba4) at vnode_if.c:951 #26 0xc0b38d5c in vn_write (fp=0xcc31c1c0, uio=0xf419dc3c, active_cred=0xc95dab00, flags=0x0, td=0xcf0d85c0) at vnode_if.h:413 #27 0xc0ad6b85 in dofilewrite (td=0xcf0d85c0, fd=0x4, fp=0xcc31c1c0, auio=0xf419dc3c, offset=0xffffffffffffffff, flags=0x0) at file.h:271 #28 0xc0ad71f8 in kern_writev (td=0xcf0d85c0, fd=0x4, auio=0xf419dc3c) at ../../../kern/sys_generic.c:459 #29 0xc0ad730f in sys_write (td=0xcf0d85c0, uap=0xf419dcec) at ../../../kern/sys_generic.c:375 #30 0xc0e005ee in syscall (frame=0xf419dd28) at subr_syscall.c:135 #31 0xc0dea8b1 in Xint0x80_syscall () at ../../../i386/i386/exception.s:267 #32 0x00000033 in ?? () (kgdb) f 17 #17 0xc0d1dd8c in vm_radix_insert (rtree=0xceb82314, index=0x45c10, val=0xc385e440) at ../../../vm/vm_radix.c:528 528 vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); (kgdb) l 523 level, slot, rnode); 524 CTR4(KTR_VM, 525 "insert: tree %p, rnode %p, child %p, count %u ENOMEM", 526 rtree, rnode, rnode->rn_child[slot], 527 rnode->rn_count); 528 vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); 529 vm_radix_unwind_heightup(rtree, root, iroot, 530 ilevel); 531 return (ENOMEM); 532 } (kgdb) info loc rnode = (struct vm_radix_node *) 0xc7997240 root = (struct vm_radix_node *) 0xc7997240 level = 0xf slot = 0x0 (kgdb) p *rtree $1 = {rt_root = 0xc7997250} (kgdb) $ svn diff -x -p /var/tmp/vmcontention/sys Index: /var/tmp/vmcontention/sys/vm/vm_radix.c =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.c (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.c (working copy) @@ -99,14 +99,12 @@ #define KSPLT64H(x) ((u_long)(((x) >> 32) & 0xFFFFFFFF)) #endif -CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); -CTASSERT((sizeof(u_int) * NBBY) >= VM_RADIX_LIMIT); - struct vm_radix_node { void *rn_child[VM_RADIX_COUNT]; /* Child nodes. */ volatile uint32_t rn_count; /* Valid children. */ }; +CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); CTASSERT(sizeof(struct vm_radix_node) < PAGE_SIZE); static uma_zone_t vm_radix_node_zone; @@ -356,6 +354,102 @@ vm_radix_reclaim_allnodes_internal(struct vm_radix } /* + * Remove the specified index from the tree. If possible the height of the + * tree is adjusted after deletion. May be used to cleanup intermediate + * nodes of the path if the key is not entirely present. + */ +static void +vm_radix_sweep(struct vm_radix *rtree, vm_pindex_t index, int color, int hard) +{ + struct vm_radix_node *stack[VM_RADIX_LIMIT]; + struct vm_radix_node *rnode, *root; + int level; + int slot; + + level = vm_radix_height(rtree, &root); + KASSERT(index <= VM_RADIX_MAX(level), + ("vm_radix_sweep: %p index out of range %jd.", rtree, + VM_RADIX_MAX(level))); + rnode = root; + level--; + /* + * Find the node and record the path in stack. + */ + while (level && rnode) { + stack[level] = rnode; + slot = vm_radix_slot(index, level); + CTR6(KTR_VM, + "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", + rtree, rnode, rnode->rn_child[slot], rnode->rn_count); + rnode = rnode->rn_child[slot]; + level--; + } + if (hard) { + if (rnode == NULL) + panic("vm_radix_sweep: index not present.\n"); + slot = vm_radix_slot(index, 0); + if (vm_radix_match(rnode->rn_child[slot], color) == NULL) + panic("vm_radix_sweep: index not present as leaf.\n"); + } else { + + /* + * Almost certainly it got an half-constructed tree it was + * expected. + */ + KASSERT(level < (VM_RADIX_LIMIT - 1), + ("vm_radix_sweep: level %d not consistent.\n", level)); + ++level; + rnode = stack[level]; + slot = vm_radix_slot(index, level); + } + + for (;;) { + CTR6(KTR_VM, +"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, + "remove: resetting tree %p, rnode %p, child %p, count %u", + rtree, rnode, + (rnode != NULL) ? rnode->rn_child[slot] : NULL, + (rnode != NULL) ? rnode->rn_count : 0); + rnode->rn_child[slot] = NULL; + /* + * Use atomics for the last level since red and black + * will both adjust it. + * Use a write memory barrier here in order to avoid + * rn_count reaching 0 before to fetch the actual pointer. + * Concurrent black removal, infact, may want to reclaim + * the radix node itself before to read it. + */ + if (rnode->rn_count != 0) { + if (level == 0) + atomic_add_rel_32(&rnode->rn_count, -1); + else + rnode->rn_count--; + } else if (hard) + panic("vm_radix_sweep: empty node midway.\n"); + + /* + * Only allow black removes to prune the tree. + */ + if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) + break; + vm_radix_node_put(rnode); + if (rnode == root) { + vm_radix_setroot(rtree, NULL, 0); + break; + } + rnode = stack[++level]; + slot = vm_radix_slot(index, level); + + } +} + +/* * Inserts the key-value pair in to the radix tree. Returns errno. * Panics if the key already exists. */ @@ -363,8 +457,7 @@ int vm_radix_insert(struct vm_radix *rtree, vm_pindex_t index, void *val) { struct vm_radix_node *iroot, *rnode, *root; - u_int allocmsk; - int clev, ilevel, level, slot; + int ilevel, level, slot; CTR4(KTR_VM, "insert: tree %p, " KFRMT64(index) ", val %p", rtree, @@ -417,8 +510,6 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ } /* Now that the tree is tall enough, fill in the path to the index. */ - allocmsk = 0; - clev = level; rnode = root; for (level = level - 1; level > 0; level--) { slot = vm_radix_slot(index, level); @@ -434,35 +525,12 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ "insert: tree %p, rnode %p, child %p, count %u ENOMEM", rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - MPASS(level != clev || allocmsk == 0); - while (allocmsk != 0) { - rnode = root; - level = clev; - level--; - slot = vm_radix_slot(index, level); - CTR4(KTR_VM, - "insert: unwind root %p, level %d, slot %d, allocmsk: 0x%x", - root, level, slot, allocmsk); - MPASS(level >= (ffs(allocmsk) - 1)); - while (level > (ffs(allocmsk) - 1)) { - MPASS(level > 0); - slot = vm_radix_slot(index, - level); - rnode = rnode->rn_child[slot]; - level--; - } - MPASS((allocmsk & (1 << level)) != 0); - allocmsk &= ~(1 << level); - rnode->rn_count--; - vm_radix_node_put(rnode->rn_child[slot]); - rnode->rn_child[slot] = NULL; - } + vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); vm_radix_unwind_heightup(rtree, root, iroot, ilevel); return (ENOMEM); } rnode->rn_count++; - allocmsk |= (1 << level); } CTR6(KTR_VM, "insert: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", @@ -812,86 +880,13 @@ restart: } /* - * Remove the specified index from the tree. If possible the height of the - * tree is adjusted after deletion. The value stored at index is returned - * panics if the key is not present. + * Remove an entry from the tree. Expects the entry to be already present. */ -void * +void vm_radix_remove(struct vm_radix *rtree, vm_pindex_t index, int color) { - struct vm_radix_node *stack[VM_RADIX_LIMIT]; - struct vm_radix_node *rnode, *root; - void *val; - int level; - int slot; - level = vm_radix_height(rtree, &root); - KASSERT(index <= VM_RADIX_MAX(level), - ("vm_radix_remove: %p index out of range %jd.", rtree, - VM_RADIX_MAX(level))); - rnode = root; - val = NULL; - level--; - /* - * Find the node and record the path in stack. - */ - while (level && rnode) { - stack[level] = rnode; - slot = vm_radix_slot(index, level); - CTR6(KTR_VM, - "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", - rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - rnode = rnode->rn_child[slot]; - level--; - } - KASSERT(rnode != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - slot = vm_radix_slot(index, 0); - val = vm_radix_match(rnode->rn_child[slot], color); - KASSERT(val != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - - for (;;) { - CTR6(KTR_VM, -"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, - "remove: resetting tree %p, rnode %p, child %p, count %u", - rtree, rnode, - (rnode != NULL) ? rnode->rn_child[slot] : NULL, - (rnode != NULL) ? rnode->rn_count : 0); - rnode->rn_child[slot] = NULL; - /* - * Use atomics for the last level since red and black - * will both adjust it. - * Use a write memory barrier here in order to avoid - * rn_count reaching 0 before to fetch the actual pointer. - * Concurrent black removal, infact, may want to reclaim - * the radix node itself before to read it. - */ - if (level == 0) - atomic_add_rel_32(&rnode->rn_count, -1); - else - rnode->rn_count--; - /* - * Only allow black removes to prune the tree. - */ - if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) - break; - vm_radix_node_put(rnode); - if (rnode == root) { - vm_radix_setroot(rtree, NULL, 0); - break; - } - rnode = stack[++level]; - slot = vm_radix_slot(index, level); - - } - return (val); + vm_radix_sweep(rtree, index, color, 1); } /* Index: /var/tmp/vmcontention/sys/vm/vm_radix.h =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.h (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.h (working copy) @@ -63,7 +63,7 @@ int vm_radix_lookupn(struct vm_radix *, vm_pindex_ void **, int, vm_pindex_t *); void *vm_radix_lookup_le(struct vm_radix *, vm_pindex_t, int); void vm_radix_reclaim_allnodes(struct vm_radix *); -void *vm_radix_remove(struct vm_radix *, vm_pindex_t, int); +void vm_radix_remove(struct vm_radix *, vm_pindex_t, int); /* * Look up any entry at a position greater or equal to index. -- Test scenario: altbufferflushes.sh