GENERIC HEAD from 2012-05-12 19:54:57 UTC, r236123M, vmcore.378 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r236123M: Sun May 27 21:25:28 CEST 2012 pho@x4.osted.lan:/var/tmp/vmcontention/sys/i386/compile/PHO i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.02-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) lock order reversal: 1st 0xc1bb505c system map (system map) @ vm/vm_map.c:3723 2nd 0xc130cd00 vm object (kernel object) @ vm/vm_fault.c:299 3rd 0xc1bb50e8 system map (system map) @ vm/vm_map.c:1454 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,34313a63,a3435,c0000a39,c1183500,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda07d,c12d8d48,5ae,c18207c8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda07d,c1bb50e8,c100daea,c9582110,c100e168,...) at _witness_debugger+0x25 witness_checkorder(c1bb50e8,9,c100e15f,5ae,0,...) at witness_checkorder+0x86f _mtx_lock_flags(c1bb50e8,0,c100e15f,5ae,c596e000,...) at _mtx_lock_flags+0xd9 _vm_map_lock(c1bb508c,c100e15f,5ae,a,0,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,c182087c,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd05a7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,c18208e3,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d124,871,c1820930,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,11,c596ff44,c1820990,c0d0015f,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,c1bb7600,101,827,c9582110,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,30cd30,5,...) at uma_zalloc_arg+0x59f vm_radix_insert(c130cd30,23480,0,c21fbf80,0,...) at vm_radix_insert+0x1e4 vm_page_insert(c21fbf80,c130cd00,23480,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(c130cd00,23480,0,40,c1820bbc,...) at vm_page_alloc+0x51c vm_fault_hold(c1bb5000,e337f000,0,1,0,...) at vm_fault_hold+0x60a vm_fault(c1bb5000,e337f000,0,1,c1bb3a68,...) at vm_fault+0x8b vm_fault_wire(c1bb5000,e3000000,e33c9000,0,0,...) at vm_fault_wire+0x3f vm_map_wire(c1bb5000,e3000000,e33c9000,0,e3000000,...) at vm_map_wire+0x2bc kmem_alloc(c1bb5000,3c9000,c955c770,0,0,...) at kmem_alloc+0x10d vm_ksubmap_init(c131e64c,0,1,1000,0,...) at vm_ksubmap_init+0x64 cpu_startup(0,181ec00,181ec00,181e000,1825000,...) at cpu_startup+0x24f mi_startup() at mi_startup+0xac begin() at begin+0x2c avail memory = 3538612224 (3374 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 614627 free (1299 frags, 76666 blocks, 0.1% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 123015 free (5287 frags, 14716 blocks, 0.5% fragmentation) /dev/label/tmp: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/label/tmp: clean, 80774904 free (2576 frags, 10096541 blocks, 0.0% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 2956327 free (219183 frags, 342143 blocks, 2.2% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 7558456 free (83024 frags, 934429 blocks, 0.3% fragmentation) Mounting local file systems:. Setting hostname: x4.osted.lan. re0: link state changed to DOWN re0: link state changed to UP Starting Network: lo0 re0 fwe0 fwip0. lo0: flags=8049 metric 0 mtu 16384 options=3 inet6 ::1 prefixlen 128 inet6 fe80::1%lo0 prefixlen 64 scopeid 0xa inet 127.0.0.1 netmask 0xff000000 nd6 options=21 re0: flags=8843 metric 0 mtu 1500 options=8209b ether 00:21:85:15:33:7d inet 192.168.1.101 netmask 0xffffff00 broadcast 192.168.1.255 inet6 fe80::221:85ff:fe15:337d%re0 prefixlen 64 scopeid 0x1 nd6 options=29 media: Ethernet autoselect (100baseTX ) status: active fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 Starting devd. Starting Network: fwe0. fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 Starting Network: fwip0. fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/kde4/lib /usr/local/lib/compat /usr/local/lib/gcc46 /usr/local/lib/mysql /usr/local/lib/nss /usr/local/lib/qt4 /usr/local/lib/virtualbox a.out ldconfig path: /usr/lib/aout /usr/lib/compat/aout Creating and/or trimming log files. Starting syslogd. lock order reversal: 1st 0xe310c560 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcbfd461c snaplk (snaplk) @ ufs/ffs/ffs_snapshot.c:2297 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,632e746f,3932323a,c0000a37,1,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d75d8,8f9,f400c990,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,cbfd461c,c1007634,c95896e0,c10075be,...) at _witness_debugger+0x25 witness_checkorder(cbfd461c,9,c10075b5,8f9,0,...) at witness_checkorder+0x86f __lockmgr_args(cbfd461c,80900,cc14d8ec,0,0,...) at __lockmgr_args+0x8c5 ffs_copyonwrite(cc14d828,e310c500,e310c500,0,0,...) at ffs_copyonwrite+0x2c2 ffs_geom_strategy(cc14d920,e310c500,cc5d44fc,e310c500,170f788,...) at ffs_geom_strategy+0xeb bufwrite(e310c500,0,c100aa50,811,0) at bufwrite+0x159 ffs_bufwrite(e310c500,cc5cc100,100,4000,0,...) at ffs_bufwrite+0x290 ffs_update(cc47a984,1,c100b1d1,151,0,...) at ffs_update+0x3a5 ffs_syncvnode(cc47a984,1,0,c1137580,cc47aa7c,...) at ffs_syncvnode+0x591 ffs_fsync(f400cc40,c102c6c4,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c1137580,f400cc40,c0fe3787,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cc0c98a0,f400ccec,c1025a5c,c0fdbd6f,c1027010,...) at sys_fsync+0x1df syscall(f400cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x2818ad47, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- No core dumps found. Additional ABI support: linux. Starting rpcbind. NFS access cache time=60 rpc.umntall: 127.0.0.1: MOUNTPROG: RPC: Program not registered Clearing /tmp (X related). Starting mountd. Starting nfsuserd. Starting nfsd. Recovering vi editor sessions:lock order reversal: 1st 0xcc732058 ufs (ufs) @ kern/vfs_lookup.c:542 2nd 0xe3129b00 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:260 3rd 0xcc7315c8 ufs (ufs) @ kern/vfs_subr.c:2159 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,3a632e73,a303632,0,0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda07d,c12d71b8,86f,f403a33c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda07d,cc7315c8,c0fc2544,c95895a8,c0fe20be,...) at _witness_debugger+0x25 witness_checkorder(cc7315c8,9,c0fe20b5,86f,0,...) at witness_checkorder+0x86f __lockmgr_args(cc7315c8,80100,cc731634,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f403a444,c0ad492c,c0fdf50b,5e2,c11088c8,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c1137580,f403a444,cc0c9670,c11496e0,cc731570,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc731570,80100,c0fe20b5,86f,eb,...) at _vn_lock+0x78 vget(cc731570,80100,cc0c95c0,50,0,...) at vget+0xbb vfs_hash_get(cbfc02dc,15ef55,80000,cc0c95c0,f403a590,...) at vfs_hash_get+0xed ffs_vgetf(cbfc02dc,15ef55,80000,f403a590,1,...) at ffs_vgetf+0x49 softdep_sync_buf(cc732000,e3129aa0,1,106,0,...) at softdep_sync_buf+0xac9 ffs_syncvnode(cc732000,1,0,cc0c9670,c11088c8,...) at ffs_syncvnode+0x24c ffs_truncate(cc732000,200,0,880,cc095400,...) at ffs_truncate+0x8b5 ufs_direnter(cc732000,cc731570,f403a904,f403abb8,0,...) at ufs_direnter+0x980 ufs_makeinode(f403abb8,c1137b40,f403aafc,f403aa58,c0e287e0,...) at ufs_makeinode+0x65d ufs_create(f403aafc,c102d20e,c1149960,cc732000,f403ab14,...) at ufs_create+0x30 VOP_CREATE_APV(c1137580,f403aafc,f403abb8,f403aa94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f403ab78,f403ac40,1b0,0,cc095400,...) at vn_open_cred+0x205 vn_open(f403ab78,f403ac40,1b0,cc08b3f0,4fc284de,...) at vn_open+0x3b kern_openat(cc0c95c0,ffffff9c,28c72f60,0,a03,...) at kern_openat+0x1ec kern_open(cc0c95c0,28c72f60,0,a02,1b0,...) at kern_open+0x35 sys_open(cc0c95c0,f403acec,c1025a5c,c0fdad6f,c1027010,...) at sys_open+0x30 syscall(f403ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x283d28d3, esp = 0xbfbfc4ec, ebp = 0xbfbfc578 --- . Updating motd:. Starting ntpd. Starting default moused. Configuring syscons: keymap blanktime. Starting sshd. Starting cron. Local package initialization: backuplock order reversal: 1st 0xcc14d46c ufs (ufs) @ kern/vfs_mount.c:1254 2nd 0xcc0e3724 devfs (devfs) @ ufs/ffs/ffs_softdep.c:1834 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,38313a63,a3433,8,28,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d6d30,72a,f40bf9a8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,cc0e3724,c0fcd460,c9589408,c1007b75,...) at _witness_debugger+0x25 witness_checkorder(cc0e3724,9,c1007b6c,72a,cc0e3790,...) at witness_checkorder+0x86f __lockmgr_args(cc0e3724,80400,cc0e3790,0,0,...) at __lockmgr_args+0x8c5 vop_stdlock(f40bfaac,c130c580,4,c0fd05a7,eb,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c10f7d40,f40bfaac,c130c580,c11496e0,cc0e36cc,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc0e36cc,80400,c1007b6c,72a,0,...) at _vn_lock+0x78 softdep_flushworklist(cbfc0894,f40bfb64,cc0498a0,612,cc0e3e5c,...) at softdep_flushworklist+0x47 ffs_sync(cbfc0894,1,f40bfbb8,51f,c1bae710,...) at ffs_sync+0x3bf dounmount(cbfc0894,8080000,cc0498a0,4a1,f40bfc3c,...) at dounmount+0x3f8 sys_unmount(cc0498a0,f40bfcec,c1025a5c,c0fdad32,c1027010,...) at sys_unmount+0x421 syscall(f40bfd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280beebb, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5930 files, 37539077 used, 80774904 free (2576 frags, 10096541 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5930 files, 37539077 used, 80774904 free (2576 frags, 10096541 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Starting inetd. Sun May 27 21:53:02 CEST 2012 FreeBSD/i386 (x4.osted.lan) (console) login: lock order reversal: 1st 0xe313ada0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcc113400 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:284 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,2e687361,38323a63,c1000a34,cc974390,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d7170,11c,f414c7c8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,cc113400,c100ba7a,c9589610,c100b70c,...) at _witness_debugger+0x25 witness_checkorder(cc113400,9,c100b703,11c,0,...) at witness_checkorder+0x86f _sx_xlock(cc113400,0,c100b703,11c,cc74832c,...) at _sx_xlock+0x9a ufsdirhash_acquire(e313ad40,cc74832c,70,e405a3b4,f414c898,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(cc74832c,f414c904,23b4,f414c884,f414c888,...) at ufsdirhash_add+0x13 ufs_direnter(cc75115c,cbc39d98,f414c904,f414cbb8,0,...) at ufs_direnter+0x799 ufs_makeinode(f414cbb8,c1137b40,f414cafc,f414ca58,c0e287e0,...) at ufs_makeinode+0x65d ufs_create(f414cafc,c102d20e,c1149960,cc75115c,f414cb14,...) at ufs_create+0x30 VOP_CREATE_APV(c1137580,f414cafc,f414cbb8,f414ca94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f414cb78,f414cc40,180,0,c95da700,...) at vn_open_cred+0x205 vn_open(f414cb78,f414cc40,180,cc07b7a8,0,...) at vn_open+0x3b kern_openat(cc9742e0,ffffff9c,bfbfe7ab,0,a02,...) at kern_openat+0x1ec kern_open(cc9742e0,bfbfe7ab,0,a01,180,...) at kern_open+0x35 sys_open(cc9742e0,f414ccec,c1025a5c,c0fdad6f,c1027010,...) at sys_open+0x30 syscall(f414cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x2829f8d3, esp = 0xbfbfe37c, ebp = 0xbfbfec18 --- May 27 22:09:21 x4 su: pho to root on /dev/pts/0 lock order reversal: 1st 0xcbfd461c snaplk (snaplk) @ ufs/ufs/ufs_vnops.c:966 2nd 0xcc156df0 ufs (ufs) @ ufs/ffs/ffs_snapshot.c:1627 KDB: stack backtrace: db_trace_self_wrapper(c0fd6597,a373236,a00,2,f411c680,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda064,c12d69c0,65b,f411c718,...) at kdb_backtrace+0x2a _witness_debugger(c0fda064,cc156df0,c0fc2544,c95895a8,c10075be,...) at _witness_debugger+0x25 witness_checkorder(cc156df0,9,c10075b5,65b,0,...) at witness_checkorder+0x86f __lockmgr_args(cc156df0,80000,0,0,0,...) at __lockmgr_args+0x8c5 ffs_snapremove(cc156d98,c10074f3,f411c848,c11a99c4,c119c6c0,...) at ffs_snapremove+0x11f ffs_truncate(cc156d98,0,0,c00,0,...) at ffs_truncate+0x647 ufs_inactive(f411cad4,c102cbdd,eb,c1137580,f411caa8,...) at ufs_inactive+0x1f6 VOP_INACTIVE_APV(c1137580,f411cad4,c0fe20b5,9bf,f411cb08,...) at VOP_INACTIVE_APV+0x100 vinactive(cc156d98,cc969b80,c0fe20b5,8be,0,...) at vinactive+0x127 vputx(f411cc34,c0b2e9e3,cc156d98,ffffffdf,cc156d98,...) at vputx+0x370 vput(cc156d98,ffffffdf,cc156d98,f411cb8c,1be40,...) at vput+0x10 kern_unlinkat(cc969b80,ffffff9c,bfbfea8e,0,0,...) at kern_unlinkat+0x2a3 kern_unlink(cc969b80,bfbfea8e,0,f411cd1c,c0e005ee,...) at kern_unlink+0x2f sys_unlink(cc969b80,f411ccec,c1025a5c,c0fdad83,c1027010,...) at sys_unlink+0x22 syscall(f411cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (10, FreeBSD ELF32, sys_unlink), eip = 0x2816352f, esp = 0xbfbfe7ac, ebp = 0xbfbfe828 --- 20120527 22:09:42 all: altbufferflushes.sh Slab at 0xc7643f44, freei 21 = 0. panic: Duplicate free of item 0xc76437e0 from zone 0xc1bb69a0(RADIX NODE) cpuid = 2 KDB: enter: panic [ thread pid 2200 tid 100105 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 2200 tid 100105 td 0xcc0c95c0 kdb_enter(c0fd23c4,c0fd23c4,c100d906,f403a614,2,...) at kdb_enter+0x3a panic(c100d906,c76437e0,c1bb69a0,c1010bc5,c100d124,...) at panic+0x18c uma_dbg_free(c1bb69a0,0,c76437e0,a1f,c0fd05a7,...) at uma_dbg_free+0x17b uma_zfree_arg(c1bb69a0,c76437e0,0,2,1,...) at uma_zfree_arg+0x94 vm_radix_sweep(1,0,101,2d6548,10,...) at vm_radix_sweep+0x1b8 vm_radix_insert(cdaea5f8,7b9e0,0,c48dc600,0,...) at vm_radix_insert+0x20c vm_page_insert(c48dc600,cdaea5c8,7b9e0,0,20,...) at vm_page_insert+0x1b5 vm_page_alloc(cdaea5c8,7b9e0,0,40222,40222,...) at vm_page_alloc+0x51c vm_page_grab(cdaea5c8,7b9e0,0,412a2,c12d6578,...) at vm_page_grab+0x152 allocbuf(e31d06e0,4000,c0fdf50b,b00,ffffffe0,...) at allocbuf+0x672 getblk(cca27d98,1ee78,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(cca27d98,7b9e0000,0,4000,cc095d80,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f403aba4,c102d143,cca27d98,cca27d98,f403ab5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c1137580,f403aba4,cca27d98,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cc05ab98,f403ac3c,cc095d80,0,cc0c95c0,...) at vn_write+0x2bc dofilewrite(f403ac3c,ffffffff,ffffffff,0,f403ac18,...) at dofilewrite+0x95 kern_writev(cc0c95c0,4,f403ac3c,f403ac5c,1,...) at kern_writev+0x68 sys_write(cc0c95c0,f403acec,c1025a5c,c0e320ef,c119b340,...) at sys_write+0x4f syscall(f403ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28177eb3, esp = 0xbfbfe9bc, ebp = 0xbfbfe9e8 --- db:0:bt> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x6d4d00 curthread = 0xc9622000: pid 12 "swi4: clock" curpcb = 0xc93fed80 fpcurthread = none idlethread = 0xc95df8a0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x828ad00 curthread = 0xc96eab80: pid 13 "g_up" curpcb = 0xe33dad80 fpcurthread = none idlethread = 0xc95df5c0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x828dd00 curthread = 0xcc0c95c0: pid 2200 "dd" curpcb = 0xf403ad80 fpcurthread = none idlethread = 0xc95df2e0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x8290d00 curthread = 0xc95df000: pid 11 "idle: cpu3" curpcb = 0xc93f4d80 fpcurthread = none idlethread = 0xc95df000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 2200 (dd) thread 0xcc0c95c0 (100105) exclusive sleep mutex RADIX NODE (UMA zone) r = 0 (0xc1bb7608) locked @ vm/uma_core.c:2591 exclusive sleep mutex vm object (standard object) r = 0 (0xcdaea5c8) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe31d0740) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe32834e0) locked @ kern/vfs_bio.c:2652 exclusive lockmgr ufs (ufs) r = 0 (0xcca27df0) locked @ kern/vfs_vnops.c:645 Process 19 (softdepflush) thread 0xcbee88a0 (100078) exclusive lockmgr bufwait (bufwait) r = 0 (0xe3277860) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3184b00) locked @ kern/vfs_bio.c:1905 Process 12 (intr) thread 0xc9622000 (100009) exclusive sleep mutex ttymtx (ttymtx) r = 0 (0xc970d004) locked @ dev/dcons/dcons_os.c:216 db:0:alllocks> show lockedvnods Locked vnodes 0xcca27d98: tag ufs, type VREG usecount 1, writecount 1, refcount 7106 mountedhere 0 flags (VI(0x200)) v_object 0xcdaea5c8 ref 0 pages 506588 lock type ufs: EXCL by thread 0xcc0c95c0 (pid 2200, dd, tid 100105) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273a3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b38c16 at vn_write+0x176 #5 0xc0ad6b85 at dofilewrite+0x95 #6 0xc0ad71f8 at kern_writev+0x68 #7 0xc0ad730f at sys_write+0x4f #8 0xc0e005ee at syscall+0x2fe #9 0xc0dea8b1 at Xint0x80_syscall+0x21 ino 6194269, on dev ada0s1e db:0:lockedvnods> show mount 0xcbfc0b70 /dev/ad4s1a on / (ufs) 0xcbfc1000 devfs on /dev (devfs) 0xcc1555b8 /dev/ad4s1f on /home (ufs) 0xcbfc05b8 /dev/ad4s1d on /usr (ufs) 0xcbfc02dc /dev/ad4s1e on /var (ufs) 0xcbfc0000 procfs on /proc (procfs) 0xcc1552dc /dev/label/tmp on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 2248 1404 1393 1001 S nanslp 0xc118e6e4 sleep 2201 1479 1344 0 S+ piperd 0xcc0de190 egrep 2200 1479 1344 0 R+ CPU 2 dd 1479 1344 1344 0 S+ wait 0xcc9f6b30 sh 1405 1393 1393 1001 S piperd 0xcc9ef190 awk 1404 1393 1393 1001 S wait 0xcc0c5864 sh 1401 1398 1401 1001 Ss kqread 0xcca10d80 tail 1398 1380 1380 1001 S select 0xcc098624 sshd 1397 1396 1397 1001 Ss+ select 0xcc098464 top 1396 1379 1379 1001 S select 0xcc0975e4 sshd 1393 1392 1393 1001 Ss wait 0xcc0e92cc sh 1392 1378 1378 1001 S select 0xcbfbf1e4 sshd 1380 1149 1380 0 Ss select 0xcbfbfca4 sshd 1379 1149 1379 0 Ss select 0xcc097224 sshd 1378 1149 1378 0 Ss select 0xcc0976a4 sshd 1344 1340 1344 0 S+ wait 0xcc0ea864 sh 1340 1337 1340 0 S+ wait 0xcc70e2cc bash 1337 1336 1337 0 S+ pause 0xcbedc5f0 csh 1336 1331 1336 1001 S+ wait 0xcc9f5000 su 1331 1330 1331 1001 Ss+ wait 0xcc914864 bash 1330 1328 1328 1001 S select 0xcc08ad24 sshd 1328 1149 1328 0 Ss select 0xcbfbfc24 sshd 1255 1 1255 0 Ss+ ttyin 0xcae33870 getty 1254 1 1254 0 Ss+ ttyin 0xcae33a70 getty 1253 1 1253 0 Ss+ ttyin 0xc970d270 getty 1252 1 1252 0 Ss+ ttyin 0xc970d470 getty 1251 1 1251 0 Ss+ ttyin 0xc970d670 getty 1250 1 1250 0 Ss+ ttyin 0xc970d870 getty 1249 1 1249 0 Ss+ ttyin 0xc970da70 getty 1248 1 1248 0 Ss+ ttyin 0xc970dc70 getty 1247 1 1247 0 Ss+ ttyin 0xc970de70 getty 1230 1 1230 0 Ss select 0xcac86864 inetd 1205 1 1205 0 Ss nanslp 0xc118e6e4 watchdogd 1160 1 1160 0 Ss nanslp 0xc118e6e4 cron 1155 1 1155 25 Ss pause 0xcc0c5058 sendmail 1152 1 1152 0 Ss select 0xcc097864 sendmail 1149 1 1149 0 Ss select 0xcc743624 sshd 1117 1 1117 0 Ss select 0xcc097324 moused 1092 1 1092 0 Ss select 0xcc0977a4 ntpd 1038 1032 1032 0 S (threaded) nfsd 100115 S rpcsvc 0xcaca9a90 nfsd: service 100114 S rpcsvc 0xcaca9ad0 nfsd: service 100113 S rpcsvc 0xcc723150 nfsd: service 100106 S rpcsvc 0xcc724250 nfsd: master 1032 1 1032 0 Ss select 0xcac7ad64 nfsd 1030 1026 1026 0 S select 0xc955f1a4 nfsuserd 1029 1026 1026 0 S select 0xcc08aa24 nfsuserd 1028 1026 1026 0 S select 0xc955f364 nfsuserd 1027 1026 1026 0 S select 0xcc08aa64 nfsuserd 1026 1 1026 0 Ss pause 0xcc0ea5f0 nfsuserd 1010 1 1010 0 Ss select 0xcac791a4 mountd 976 1 976 0 Ss select 0xcac79164 rpcbind 961 1 961 0 Ss select 0xcac85964 syslogd 838 1 838 0 Ss select 0xc95601e4 devd 19 0 0 0 DL biord 0xe3277800 [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc13048b4 [syncer] 16 0 0 0 DL psleep 0xc1304548 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e468 [pagezero] 8 0 0 0 DL psleep 0xc130e048 [vmdaemon] 7 0 0 0 DL psleep 0xc130e010 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151054 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc1307dd8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 D - 0xcacfed04 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d4e4 [yarrow] 13 0 0 0 RL (threaded) [geom] 100016 D - 0xc118a5a8 [g_down] 100015 Run CPU 1 [g_up] 100014 D - 0xc118a59c [g_event] 12 0 0 0 RL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 Run CPU 0 [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 Run CPU 3 [idle: cpu3] 100005 CanRun [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 CanRun [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130bd00 [audit] 0 0 0 0 DLs (threaded) [kernel] 100069 D - 0xc118d4e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118a760 [swapper] db:0:ps> allt Tracing command sleep pid 2248 tid 100151 td 0xcc9688a0 sched_switch(cc9688a0,0,104,1a3,7445d454,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc9688a0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(2711,c0ac8bc0,cc9688a0,1,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c118e6e4,6c,c0fd394b,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c118e6e4,0,16c,c0fd394b,2711,...) at _sleep+0x34f kern_nanosleep(cc9688a0,f412ec48,f412ec50,a,0,...) at kern_nanosleep+0xc1 sys_nanosleep(cc9688a0,f412ecec,c1025a5c,c0fdb6ce,c1027010,...) at sys_nanosleep+0x6f syscall(f412ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28152483, esp = 0xbfbfec4c, ebp = 0xbfbfec98 --- Tracing command egrep pid 2201 tid 100124 td 0xcc7e3b80 sched_switch(cc7e3b80,0,104,1a3,5656a722,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cc7e3b80,0,c0fd7f80,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0de190,5c,c0fda907,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0de190,cc0de308,15c,c0fda907,0,...) at _sleep+0x385 pipe_read(cc05a7e0,f40ddc3c,cc095d80,0,cc7e3b80,...) at pipe_read+0x427 dofileread(f40ddc3c,ffffffff,ffffffff,0,f40ddc18,...) at dofileread+0x9e kern_readv(cc7e3b80,0,f40ddc3c,f40ddc5c,1,...) at kern_readv+0x68 sys_read(cc7e3b80,f40ddcec,c1025a5c,c0fdadfa,c1027010,...) at sys_read+0x4f syscall(f40ddd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281baed3, esp = 0xbfbfe81c, ebp = 0xbfbfe8a8 --- Tracing command dd pid 2200 tid 100105 td 0xcc0c95c0 kdb_enter(c0fd23c4,c0fd23c4,c100d906,f403a614,2,...) at kdb_enter+0x3a panic(c100d906,c76437e0,c1bb69a0,c1010bc5,c100d124,...) at panic+0x18c uma_dbg_free(c1bb69a0,0,c76437e0,a1f,c0fd05a7,...) at uma_dbg_free+0x17b uma_zfree_arg(c1bb69a0,c76437e0,0,2,1,...) at uma_zfree_arg+0x94 vm_radix_sweep(1,0,101,2d6548,10,...) at vm_radix_sweep+0x1b8 vm_radix_insert(cdaea5f8,7b9e0,0,c48dc600,0,...) at vm_radix_insert+0x20c vm_page_insert(c48dc600,cdaea5c8,7b9e0,0,20,...) at vm_page_insert+0x1b5 vm_page_alloc(cdaea5c8,7b9e0,0,40222,40222,...) at vm_page_alloc+0x51c vm_page_grab(cdaea5c8,7b9e0,0,412a2,c12d6578,...) at vm_page_grab+0x152 allocbuf(e31d06e0,4000,c0fdf50b,b00,ffffffe0,...) at allocbuf+0x672 getblk(cca27d98,1ee78,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(cca27d98,7b9e0000,0,4000,cc095d80,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f403aba4,c102d143,cca27d98,cca27d98,f403ab5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c1137580,f403aba4,cca27d98,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cc05ab98,f403ac3c,cc095d80,0,cc0c95c0,...) at vn_write+0x2bc dofilewrite(f403ac3c,ffffffff,ffffffff,0,f403ac18,...) at dofilewrite+0x95 kern_writev(cc0c95c0,4,f403ac3c,f403ac5c,1,...) at kern_writev+0x68 sys_write(cc0c95c0,f403acec,c1025a5c,c0e320ef,c119b340,...) at sys_write+0x4f syscall(f403ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28177eb3, esp = 0xbfbfe9bc, ebp = 0xbfbfe9e8 --- Tracing command sh pid 1479 tid 100157 td 0xcc976000 sched_switch(cc976000,0,104,1a3,4f372cdf,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc976000,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc9f6b30,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc9f6b30,cc9f6bb8,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cc976000,ffffffff,f4140c58,2,0,...) at kern_wait+0x597 sys_wait4(cc976000,f4140cec,c1025a5c,c0fdae97,c1027010,...) at sys_wait4+0x3b syscall(f4140d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe78c, ebp = 0xbfbfe7a8 --- Tracing command awk pid 1405 tid 100133 td 0xcae268a0 sched_switch(cae268a0,0,104,1a3,735fd703,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae268a0,0,c0fd7f80,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc9ef190,5c,c0fda907,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc9ef190,cc9ef308,15c,c0fda907,0,...) at _sleep+0x385 pipe_read(cc09a700,f40f8c3c,c95da480,0,cae268a0,...) at pipe_read+0x427 dofileread(f40f8c3c,ffffffff,ffffffff,0,f40f8c18,...) at dofileread+0x9e kern_readv(cae268a0,0,f40f8c3c,f40f8c5c,1,...) at kern_readv+0x68 sys_read(cae268a0,f40f8cec,c1025a5c,c0e320ef,c1027010,...) at sys_read+0x4f syscall(f40f8d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281aced3, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command sh pid 1404 tid 100083 td 0xcc0cab80 sched_switch(cc0cab80,0,104,1a3,742ed46a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0cab80,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0c5864,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0c5864,cc0c58ec,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cc0cab80,ffffffff,f3fdac58,2,0,...) at kern_wait+0x597 sys_wait4(cc0cab80,f3fdacec,c1025a5c,c0fdae97,c119b340,...) at sys_wait4+0x3b syscall(f3fdad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command tail pid 1401 tid 100107 td 0xcc0c92e0 sched_switch(cc0c92e0,0,104,1a3,d0cdd75d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,68,...) at mi_switch+0x1ff sleepq_switch(cc0c92e0,0,c0fd7f80,1a7,68,...) at sleepq_switch+0x162 sleepq_catch_signals(3e9,c0ac8bc0,cc0c92e0,1,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cca10d80,68,c0fcc142,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(cca10d80,cca10d80,168,c0fcc142,3e9,...) at _sleep+0x34f kern_kevent(cc0c92e0,4,0,1,f4042c3c,...) at kern_kevent+0x374 sys_kevent(cc0c92e0,f4042cec,c1025a5c,c0e320ef,cacc1b80,...) at sys_kevent+0x19e syscall(f4042d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2815639b, esp = 0xbfbfeb0c, ebp = 0xbfbfebc8 --- Tracing command sshd pid 1398 tid 100139 td 0xcae262e0 sched_switch(cae262e0,0,104,1a3,da13755f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae262e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc098610,0,c0fd05a7,cae262e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc098624,0,f410aa50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc098624,cc098610,c0fda6d8,6ae,cc05b268,...) at _cv_wait_sig+0x260 seltdwait(cc05b268,58,cc096400,cae262e0,cae26390,...) at seltdwait+0xa2 kern_select(cae262e0,c,28c0c170,28c0c1b0,0,0,20,cc9132cc,0) at kern_select+0x655 sys_select(cae262e0,f410acec,c1025a5c,c0fdb005,cc049000,...) at sys_select+0x66 syscall(f410ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command top pid 1397 tid 100080 td 0xcbee85c0 sched_switch(cbee85c0,0,104,1a3,d505c4eb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee85c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fc9a14,c0a7478a,cc098450,0,cbee85c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc098464,0,f3fc9a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc098464,cc098450,3e9,6ae,cc08b1f8,...) at _cv_timedwait_sig+0x270 seltdwait(f3fc9c08,f3fc9c10,cc095c00,cbee85c0,73734830,...) at seltdwait+0x8a kern_select(cbee85c0,2,bfbfeb9c,0,0,f3fc9c54,20,1,0) at kern_select+0x655 sys_select(cbee85c0,f3fc9cec,c1025a5c,c0fa5622,c1027010,...) at sys_select+0x66 syscall(f3fc9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281e1e53, esp = 0xbfbfeb3c, ebp = 0xbfbfec98 --- Tracing command sshd pid 1396 tid 100136 td 0xcc0cb5c0 sched_switch(cc0cb5c0,0,104,1a3,d509d240,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0cb5c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc0975d0,0,c0fd05a7,cc0cb5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0975e4,0,f4101a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0975e4,cc0975d0,c0fda6d8,6ae,cc08b038,...) at _cv_wait_sig+0x260 seltdwait(cc08b038,58,cbfcc680,cc0cb5c0,cc0cb670,...) at seltdwait+0xa2 kern_select(cc0cb5c0,a,28c0c170,28c0c1b0,0,0,20,cc951864,0) at kern_select+0x655 sys_select(cc0cb5c0,f4101cec,c1025a5c,c0fdb005,cacc1b80,...) at sys_select+0x66 syscall(f4101d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sh pid 1393 tid 100103 td 0xcc0ee8a0 sched_switch(cc0ee8a0,0,104,1a3,580f6c95,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0ee8a0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0e92cc,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0e92cc,cc0e9354,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cc0ee8a0,ffffffff,f4031c58,2,0,...) at kern_wait+0x597 sys_wait4(cc0ee8a0,f4031cec,c1025a5c,c0fdae97,c1027010,...) at sys_wait4+0x3b syscall(f4031d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfeb0c, ebp = 0xbfbfeb28 --- Tracing command sshd pid 1392 tid 100089 td 0xcc04d000 sched_switch(cc04d000,0,104,1a3,1943f2fa,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc04d000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfbf1d0,0,c0fd05a7,cc04d000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfbf1e4,0,f3ff2a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfbf1e4,cbfbf1d0,c0fda6d8,6ae,cc05b850,...) at _cv_wait_sig+0x260 seltdwait(cc05b850,58,cc096000,cc04d000,cc04d0b0,...) at seltdwait+0xa2 kern_select(cc04d000,c,28c0c170,28c0c1b0,0,0,20,cc046000,0) at kern_select+0x655 sys_select(cc04d000,f3ff2cec,c1025a5c,c0fdb005,c1027010,...) at sys_select+0x66 syscall(f3ff2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1380 tid 100119 td 0xcc049000 sched_switch(cc049000,0,104,1a3,50a9e5af,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc049000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfbfc90,0,c0fd05a7,cc049000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfbfca4,0,f40caab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfbfca4,cbfbfc90,c0fda6d8,6ae,f40cab3c,...) at _cv_wait_sig+0x260 seltdwait(cc5b192c,c0fda6d8,5ab,cc049000,0,...) at seltdwait+0xa2 sys_poll(cc049000,f40cacec,c1025a5c,c0e320ef,cc0492e0,...) at sys_poll+0x321 syscall(f40cad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1379 tid 100091 td 0xcc0ca000 sched_switch(cc0ca000,0,104,1a3,4e17c4e7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0ca000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc097210,0,c0fd05a7,cc0ca000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc097224,0,f3ffaab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc097224,cc097210,c0fda6d8,6ae,f3ffab3c,...) at _cv_wait_sig+0x260 seltdwait(cc3ae12c,c0fda6d8,5ab,cc0ca000,0,...) at seltdwait+0xa2 sys_poll(cc0ca000,f3ffacec,c1025a5c,c0e320ef,c1027010,...) at sys_poll+0x321 syscall(f3ffad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1378 tid 100094 td 0xcc0f42e0 sched_switch(cc0f42e0,0,104,1a3,4a2d4925,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0f42e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc097690,0,c0fd05a7,cc0f42e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0976a4,0,f4008ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0976a4,cc097690,c0fda6d8,6ae,f4008b3c,...) at _cv_wait_sig+0x260 seltdwait(cc106c2c,c0fda6d8,5ab,cc0f42e0,0,...) at seltdwait+0xa2 sys_poll(cc0f42e0,f4008cec,c1025a5c,c0e320ef,cc0c98a0,...) at sys_poll+0x321 syscall(f4008d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sh pid 1344 tid 100096 td 0xcc0f4000 sched_switch(cc0f4000,0,104,1a3,e2d61134,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0f4000,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0ea864,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0ea864,cc0ea8ec,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cc0f4000,ffffffff,f4010c58,2,0,...) at kern_wait+0x597 sys_wait4(cc0f4000,f4010cec,c1025a5c,c0fdae97,c1027010,...) at sys_wait4+0x3b syscall(f4010d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe6ec, ebp = 0xbfbfe708 --- Tracing command bash pid 1340 tid 100120 td 0xcc7e48a0 sched_switch(cc7e48a0,0,104,1a3,c1ccac78,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc7e48a0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc70e2cc,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc70e2cc,cc70e354,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cc7e48a0,ffffffff,f40cec58,6,0,...) at kern_wait+0x597 sys_wait4(cc7e48a0,f40cecec,c1025a5c,c0fdadb7,c119c040,...) at sys_wait4+0x3b syscall(f40ced28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe76c, ebp = 0xbfbfe788 --- Tracing command csh pid 1337 tid 100109 td 0xcc04c5c0 sched_switch(cc04c5c0,0,104,1a3,8406a183,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc04c5c0,0,c0fd7f80,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbedc5f0,74,c0e58cce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cbedc5f0,cbedc620,174,c0e58cce,0,...) at _sleep+0x385 kern_sigsuspend(cc04c5c0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc04c5c0,f404fcec,c1025a5c,c0fdadb7,c1027010,...) at sys_sigsuspend+0x4d syscall(f404fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x2816fd7b, esp = 0xbfbfe05c, ebp = 0xbfbfe0c8 --- Tracing command su pid 1336 tid 100166 td 0xcca128a0 sched_switch(cca128a0,0,104,1a3,2010d59f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cca128a0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc9f5000,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc9f5000,cc9f5088,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cca128a0,539,f415bc58,2,0,...) at kern_wait+0x597 sys_wait4(cca128a0,f415bcec,c1025a5c,c0fdb6ce,c1027010,...) at sys_wait4+0x3b syscall(f415bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2810b78b, esp = 0xbfbfe5ac, ebp = 0xbfbfe5c8 --- Tracing command bash pid 1331 tid 100128 td 0xcbee68a0 sched_switch(cbee68a0,0,104,1a3,2d588d20,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cbee68a0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc914864,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc914864,cc9148ec,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(cbee68a0,ffffffff,f40e9c58,6,0,...) at kern_wait+0x597 sys_wait4(cbee68a0,f40e9cec,c1025a5c,c0fdadb7,c1027010,...) at sys_wait4+0x3b syscall(f40e9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command sshd pid 1330 tid 100160 td 0xcc9745c0 sched_switch(cc9745c0,0,104,1a3,4f19b29f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9745c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc08ad10,0,c0fd05a7,cc9745c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08ad24,0,f4149a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc08ad24,cc08ad10,c0fda6d8,6ae,cc08b508,...) at _cv_wait_sig+0x260 seltdwait(cc08b508,58,cbfcc980,cc9745c0,cc974670,...) at seltdwait+0xa2 kern_select(cc9745c0,c,28c0c170,28c0c1b0,0,0,20,cc9f62cc,0) at kern_select+0x655 sys_select(cc9745c0,f4149cec,c1025a5c,c0fdb005,c1027010,...) at sys_select+0x66 syscall(f4149d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1328 tid 100158 td 0xcc974b80 sched_switch(cc974b80,0,104,1a3,85ce4a23,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc974b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfbfc10,0,c0fd05a7,cc974b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfbfc24,0,f4143ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfbfc24,cbfbfc10,c0fda6d8,6ae,f4143b3c,...) at _cv_wait_sig+0x260 seltdwait(cc10522c,c0fda6d8,5ab,cc974b80,696,...) at seltdwait+0xa2 sys_poll(cc974b80,f4143cec,c1025a5c,c0fbc991,c1027010,...) at sys_poll+0x321 syscall(f4143d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command getty pid 1255 tid 100153 td 0xcc976b80 sched_switch(cc976b80,0,104,1a3,da7878b7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc976b80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc976b80,cc976b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33870,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33870,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33800,cae33870,f4134c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33800,f4134c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27100,f4134c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc07bd58,f4134c3c,c95d9b80,0,cc976b80,...) at devfs_read_f+0x7e dofileread(f4134c3c,ffffffff,ffffffff,0,f4134c18,...) at dofileread+0x9e kern_readv(cc976b80,0,f4134c3c,f4134c5c,1,...) at kern_readv+0x68 sys_read(cc976b80,f4134cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f4134d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1254 tid 100152 td 0xcc977000 sched_switch(cc977000,0,104,1a3,dbc171fb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc977000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc977000,cc977000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33a70,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33a70,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33a00,cae33a70,f4131c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33a00,f4131c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27000,f4131c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08b3f0,f4131c3c,c95d9b80,0,cc977000,...) at devfs_read_f+0x7e dofileread(f4131c3c,ffffffff,ffffffff,0,f4131c18,...) at dofileread+0x9e kern_readv(cc977000,0,f4131c3c,f4131c5c,1,...) at kern_readv+0x68 sys_read(cc977000,f4131cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f4131d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1253 tid 100141 td 0xcc9778a0 sched_switch(cc9778a0,0,104,1a3,d9b2c42f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9778a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc9778a0,cc9778a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d270,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d270,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d200,c970d270,f4110c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d200,f4110c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0ce00,f4110c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08b230,f4110c3c,c95d9b80,0,cc9778a0,...) at devfs_read_f+0x7e dofileread(f4110c3c,ffffffff,ffffffff,0,f4110c18,...) at dofileread+0x9e kern_readv(cc9778a0,0,f4110c3c,f4110c5c,1,...) at kern_readv+0x68 sys_read(cc9778a0,f4110cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f4110d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1252 tid 100134 td 0xcae265c0 sched_switch(cae265c0,0,104,1a3,daeaaa10,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae265c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cae265c0,cae265c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d470,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d470,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d400,c970d470,f40fbc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d400,f40fbc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cd00,f40fbc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc05b070,f40fbc3c,c95d9b80,0,cae265c0,...) at devfs_read_f+0x7e dofileread(f40fbc3c,ffffffff,ffffffff,0,f40fbc18,...) at dofileread+0x9e kern_readv(cae265c0,0,f40fbc3c,f40fbc5c,1,...) at kern_readv+0x68 sys_read(cae265c0,f40fbcec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f40fbd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1251 tid 100146 td 0xcc9698a0 sched_switch(cc9698a0,0,104,1a3,d8749766,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9698a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc9698a0,cc9698a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d670,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d670,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d600,c970d670,f411fc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d600,f411fc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cc00,f411fc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc07bab8,f411fc3c,c95d9b80,0,cc9698a0,...) at devfs_read_f+0x7e dofileread(f411fc3c,ffffffff,ffffffff,0,f411fc18,...) at dofileread+0x9e kern_readv(cc9698a0,0,f411fc3c,f411fc5c,1,...) at kern_readv+0x68 sys_read(cc9698a0,f411fcec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f411fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1250 tid 100121 td 0xcc7e45c0 sched_switch(cc7e45c0,0,104,1a3,daff7eab,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc7e45c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc7e45c0,cc7e45c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d870,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d870,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d800,c970d870,f40d2c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d800,f40d2c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c900,f40d2c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08b5e8,f40d2c3c,c95d9b80,0,cc7e45c0,...) at devfs_read_f+0x7e dofileread(f40d2c3c,ffffffff,ffffffff,0,f40d2c18,...) at dofileread+0x9e kern_readv(cc7e45c0,0,f40d2c3c,f40d2c5c,1,...) at kern_readv+0x68 sys_read(cc7e45c0,f40d2cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f40d2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1249 tid 100102 td 0xcc0eeb80 sched_switch(cc0eeb80,0,104,1a3,da3f4a17,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0eeb80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cc0eeb80,cc0eeb80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970da70,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970da70,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970da00,c970da70,f402dc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970da00,f402dc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c800,f402dc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc09a1c0,f402dc3c,c95d9b80,0,cc0eeb80,...) at devfs_read_f+0x7e dofileread(f402dc3c,ffffffff,ffffffff,0,f402dc18,...) at dofileread+0x9e kern_readv(cc0eeb80,0,f402dc3c,f402dc5c,1,...) at kern_readv+0x68 sys_read(cc0eeb80,f402dcec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f402dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1248 tid 100131 td 0xcbee6000 sched_switch(cbee6000,0,104,1a3,da67e910,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee6000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,cbee6000,cbee6000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970dc70,0,c0fdcd40,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970dc70,c118c410,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970dc00,c970dc70,f40f2c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970dc00,f40f2c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c700,f40f2c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc07b150,f40f2c3c,c95d9b80,0,cbee6000,...) at devfs_read_f+0x7e dofileread(f40f2c3c,ffffffff,ffffffff,0,f40f2c18,...) at dofileread+0x9e kern_readv(cbee6000,0,f40f2c3c,f40f2c5c,1,...) at kern_readv+0x68 sys_read(cbee6000,f40f2cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f40f2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1247 tid 100079 td 0xcc04db80 sched_switch(cc04db80,0,104,1a3,d86fc0e2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc04db80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,c970de04,0,c0fd05a7,cc04db80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970de70,0,f3fc5ae8,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970de70,c970de04,c0fdc718,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970de00,c970de70,f3fc5c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970de00,f3fc5c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(c95c6100,f3fc5c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc05aa48,f3fc5c3c,c95d9b80,0,cc04db80,...) at devfs_read_f+0x7e dofileread(f3fc5c3c,ffffffff,ffffffff,0,f3fc5c18,...) at dofileread+0x9e kern_readv(cc04db80,0,f3fc5c3c,f3fc5c5c,1,...) at kern_readv+0x68 sys_read(cc04db80,f3fc5cec,c1025a5c,c0e36eb2,c1027010,...) at sys_read+0x4f syscall(f3fc5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command inetd pid 1230 tid 100130 td 0xcbee62e0 sched_switch(cbee62e0,0,104,1a3,b1bc3982,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee62e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86850,0,c0fd05a7,cbee62e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac86864,0,f40efa50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac86864,cac86850,c0fda6d8,6ae,cc05ae00,...) at _cv_wait_sig+0x260 seltdwait(cc05ae00,58,c95d9b80,cbee62e0,202,...) at seltdwait+0xa2 kern_select(cbee62e0,8,bfbfdf90,0,0,0,20,cc9142cc,4) at kern_select+0x655 sys_select(cbee62e0,f40efcec,c1025a5c,c0fdaf04,c1027010,...) at sys_select+0x66 syscall(f40efd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2819be53, esp = 0xbfbfdf4c, ebp = 0xbfbfedd8 --- Tracing command watchdogd pid 1205 tid 100144 td 0xcc9772e0 sched_switch(cc9772e0,0,104,1a3,6ad05071,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc9772e0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(ea61,c0ac8bc0,cc9772e0,2,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c118e6e4,6c,c0fd394b,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c118e6e4,0,16c,c0fd394b,ea61,...) at _sleep+0x34f kern_nanosleep(cc9772e0,f4119c48,f4119c50,3c,0,...) at kern_nanosleep+0xc1 sys_nanosleep(cc9772e0,f4119cec,c1025a5c,c0e36eb2,c119b340,...) at sys_nanosleep+0x6f syscall(f4119d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28181483, esp = 0xbfbfecac, ebp = 0xbfbfecd8 --- Tracing command cron pid 1160 tid 100087 td 0xcc0ca5c0 sched_switch(cc0ca5c0,0,104,1a3,47fcf4c3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0ca5c0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(ea61,c0ac8bc0,cc0ca5c0,1,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c118e6e4,6c,c0fd394b,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c118e6e4,0,16c,c0fd394b,ea61,...) at _sleep+0x34f kern_nanosleep(cc0ca5c0,f3feac48,f3feac50,3c,0,...) at kern_nanosleep+0xc1 sys_nanosleep(cc0ca5c0,f3feacec,c1025a5c,c0fdb30e,14,...) at sys_nanosleep+0x6f syscall(f3fead28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28174483, esp = 0xbfbfec3c, ebp = 0xbfbfec68 --- Tracing command sendmail pid 1155 tid 100090 td 0xcc0ca2e0 sched_switch(cc0ca2e0,0,104,1a3,fec91e77,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0ca2e0,0,c0fd7f80,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0c5058,74,c0e58cce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0c5058,cc0c5088,174,c0e58cce,0,...) at _sleep+0x385 kern_sigsuspend(cc0ca2e0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0ca2e0,f3ff6cec,c1025a5c,c0fdadb7,c119c6c0,...) at sys_sigsuspend+0x4d syscall(f3ff6d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28322d7b, esp = 0xbfbfcf8c, ebp = 0xbfbfcfb8 --- Tracing command sendmail pid 1152 tid 100098 td 0xcc0f38a0 sched_switch(cc0f38a0,0,104,1a3,9cfd872e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0f38a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4018a14,c0a7478a,cc097850,0,cc0f38a0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc097864,0,f4018a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc097864,cc097850,1389,6ae,cc08ba10,...) at _cv_timedwait_sig+0x270 seltdwait(f4018c08,f4018c10,cbfccc80,cc0f38a0,40,...) at seltdwait+0x8a kern_select(cc0f38a0,5,bfbfc500,0,0,f4018c54,20,5,0) at kern_select+0x655 sys_select(cc0f38a0,f4018cec,c1025a5c,c0fdb294,c1027010,...) at sys_select+0x66 syscall(f4018d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x283dae53, esp = 0xbfbfc46c, ebp = 0xbfbfcf98 --- Tracing command sshd pid 1149 tid 100118 td 0xcc0492e0 sched_switch(cc0492e0,0,104,1a3,4fe2b571,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0492e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc743610,0,c0fd05a7,cc0492e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc743624,0,f40c6a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc743624,cc743610,c0fda6d8,6ae,cc07b2a0,...) at _cv_wait_sig+0x260 seltdwait(cc07b2a0,58,cbfccb00,cc0492e0,f40c6b14,...) at seltdwait+0xa2 kern_select(cc0492e0,9,28c0c140,0,0,0,20,cc70e864,a) at kern_select+0x655 sys_select(cc0492e0,f40c6cec,c1025a5c,c0fbc991,c1027010,...) at sys_select+0x66 syscall(f40c6d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfdedc, ebp = 0xbfbfede8 --- Tracing command moused pid 1117 tid 100104 td 0xcc0ee5c0 sched_switch(cc0ee5c0,0,104,1a3,f5db8bd7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0ee5c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc097310,0,c0fd05a7,cc0ee5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc097324,0,f4035a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc097324,cc097310,c0fda6d8,6ae,cc07b620,...) at _cv_wait_sig+0x260 seltdwait(cc07b620,58,c95d9b80,cc0ee5c0,f6,...) at seltdwait+0xa2 kern_select(cc0ee5c0,400,bfbfe9f4,0,0,0,20,cc0e9000,1) at kern_select+0x655 sys_select(cc0ee5c0,f4035cec,c1025a5c,c0e36eb2,c1027010,...) at sys_select+0x66 syscall(f4035d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281a8e53, esp = 0xbfbfe98c, ebp = 0xbfbfeb18 --- Tracing command ntpd pid 1092 tid 100081 td 0xcbee82e0 sched_switch(cbee82e0,0,104,1a3,ec7c03a9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee82e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc097790,0,c0fd05a7,cbee82e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0977a4,0,f3fcda50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0977a4,cc097790,c0fda6d8,6ae,cc08bab8,...) at _cv_wait_sig+0x260 seltdwait(cc08bab8,58,c95d9b80,cbee82e0,0,...) at seltdwait+0xa2 kern_select(cbee82e0,1c,bfbfecd8,0,0,0,20,cac0b864,0) at kern_select+0x655 sys_select(cbee82e0,f3fcdcec,c1025a5c,c0fdb6e0,e,...) at sys_select+0x66 syscall(f3fcdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28358e53, esp = 0xbfbfecac, ebp = 0xbfbfed78 --- Tracing command nfsd pid 1038 tid 100115 td 0xcc0cbb80 sched_switch(cc0cbb80,0,104,1a3,2cdd23d6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0cbb80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40bcbdc,c0a7478a,cbc36e80,0,cc0cbb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca9a90,0,f40bcc18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca9a90,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40bcd14,c0a56e68,cbc36e80,f40bcd28,c0fcc590,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40bcd28,c0fcc590,3d8,cbedcb30,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40bcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1038 tid 100114 td 0xcc0ee000 sched_switch(cc0ee000,0,104,1a3,2cdd3d3e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0ee000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40b9bdc,c0a7478a,cbc36e80,0,cc0ee000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca9ad0,0,f40b9c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca9ad0,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40b9d14,c0a56e68,cbc36e80,f40b9d28,c0fcc590,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40b9d28,c0fcc590,3d8,cbedcb30,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40b9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1038 tid 100113 td 0xcc0ee2e0 sched_switch(cc0ee2e0,0,104,1a3,2cdd0ae0,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0ee2e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40b6bdc,c0a7478a,cbc36e80,0,cc0ee2e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc723150,0,f40b6c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc723150,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40b6d14,c0a56e68,cbc36e80,f40b6d28,c0fcc590,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40b6d28,c0fcc590,3d8,cbedcb30,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40b6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1038 tid 100106 td 0xcc04cb80 sched_switch(cc04cb80,0,104,1a3,2cdceee2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc04cb80,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f403e48c,c0a7478a,cbc36e80,0,cc04cb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc724250,0,f403e4c8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc724250,cbc36e80,1388,3b1,f403e510,...) at _cv_timedwait_sig+0x270 svc_run_internal(cc04cd30,14,c10027ba,c0fbde9b,f403e808,...) at svc_run_internal+0x356 svc_run(cbc36e80,0,c0fbde76,19f,cc04cc30,...) at svc_run+0x7b nfsrvd_nfsd(cc04cb80,f403e808,c,ab,c0fd05a7,...) at nfsrvd_nfsd+0x1c3 nfssvc_nfsd(cc04cb80,f403ecec,0,cc04cb80,f403ed1c,...) at nfssvc_nfsd+0x177 sys_nfssvc(cc04cb80,f403ecec,c1025a5c,c0fdb294,c1027010,...) at sys_nfssvc+0xeb syscall(f403ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (155, FreeBSD ELF32, sys_nfssvc), eip = 0x280bf9bb, esp = 0xbfbfe81c, ebp = 0xbfbfea88 --- Tracing command nfsd pid 1032 tid 100108 td 0xcc04c8a0 sched_switch(cc04c8a0,0,104,1a3,9ee88539,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc04c8a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac7ad50,0,c0fd05a7,cc04c8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac7ad64,0,f4046a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac7ad64,cac7ad50,c0fda6d8,6ae,cc07bd20,...) at _cv_wait_sig+0x260 seltdwait(cc07bd20,58,c95d9b80,cc04c8a0,0,...) at seltdwait+0xa2 kern_select(cc04c8a0,7,bfbfeca4,0,0,0,20,cbedc864,1) at kern_select+0x655 sys_select(cc04c8a0,f4046cec,c1025a5c,c0fdb294,c1027010,...) at sys_select+0x66 syscall(f4046d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28176e53, esp = 0xbfbfea8c, ebp = 0xbfbfedd8 --- Tracing command nfsuserd pid 1030 tid 100111 td 0xcc04c000 sched_switch(cc04c000,0,104,1a3,b0911a85,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc04c000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40afa14,c0a7478a,c955f190,0,cc04c000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c955f1a4,0,f40afa50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c955f1a4,c955f190,7531,6ae,cc05a658,...) at _cv_timedwait_sig+0x270 seltdwait(f40afc08,f40afc10,c95d9b80,cc04c000,0,...) at seltdwait+0x8a kern_select(cc04c000,4,bfbfebb4,0,0,f40afc54,20,1e,0) at kern_select+0x655 sys_select(cc04c000,f40afcec,c1025a5c,c0fdafee,0,...) at sys_select+0x66 syscall(f40afd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1029 tid 100082 td 0xcc0cb000 sched_switch(cc0cb000,0,104,1a3,b091ede4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0cb000,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fd4a14,c0a7478a,cc08aa10,0,cc0cb000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc08aa24,0,f3fd4a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc08aa24,cc08aa10,7531,6ae,cc05a658,...) at _cv_timedwait_sig+0x270 seltdwait(f3fd4c08,f3fd4c10,c95d9b80,cc0cb000,0,...) at seltdwait+0x8a kern_select(cc0cb000,4,bfbfebb4,0,0,f3fd4c54,20,1e,0) at kern_select+0x655 sys_select(cc0cb000,f3fd4cec,c1025a5c,c0fdafee,c1027010,...) at sys_select+0x66 syscall(f3fd4d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1028 tid 100110 td 0xcc04c2e0 sched_switch(cc04c2e0,0,104,1a3,b0917d79,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc04c2e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4059a14,c0a7478a,c955f350,0,cc04c2e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c955f364,0,f4059a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c955f364,c955f350,7531,6ae,cc05a658,...) at _cv_timedwait_sig+0x270 seltdwait(f4059c08,f4059c10,c95d9b80,cc04c2e0,0,...) at seltdwait+0x8a kern_select(cc04c2e0,4,bfbfebb4,0,0,f4059c54,20,1e,0) at kern_select+0x655 sys_select(cc04c2e0,f4059cec,c1025a5c,c0fdafee,c1027010,...) at sys_select+0x66 syscall(f4059d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1027 tid 100100 td 0xcc0f32e0 sched_switch(cc0f32e0,0,104,1a3,b09179ba,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0f32e0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4020a14,c0a7478a,cc08aa50,0,cc0f32e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc08aa64,0,f4020a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc08aa64,cc08aa50,7531,6ae,cc05a658,...) at _cv_timedwait_sig+0x270 seltdwait(f4020c08,f4020c10,c95d9b80,cc0f32e0,0,...) at seltdwait+0x8a kern_select(cc0f32e0,4,bfbfebb4,0,0,f4020c54,20,1e,0) at kern_select+0x655 sys_select(cc0f32e0,f4020cec,c1025a5c,c0fdafee,c1027010,...) at sys_select+0x66 syscall(f4020d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1026 tid 100097 td 0xcc0f3b80 sched_switch(cc0f3b80,0,104,1a3,92a166f2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0f3b80,0,c0fd7f80,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0ea5f0,74,c0e58cce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0ea5f0,cc0ea620,174,c0e58cce,0,...) at _sleep+0x385 kern_sigsuspend(cc0f3b80,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0f3b80,f4014cec,c1025a5c,c0fdb294,c1027010,...) at sys_sigsuspend+0x4d syscall(f4014d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x280bdd7b, esp = 0xbfbfec4c, ebp = 0xbfbfede8 --- Tracing command mountd pid 1010 tid 100086 td 0xcc0ca8a0 sched_switch(cc0ca8a0,0,104,1a3,65696a6e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0ca8a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac79190,0,c0fd05a7,cc0ca8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac791a4,0,f3fe6a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac791a4,cac79190,c0fda6d8,6ae,cc08b000,...) at _cv_wait_sig+0x260 seltdwait(cc08b000,58,c95d9b80,cc0ca8a0,f3fe6ac0,...) at seltdwait+0xa2 kern_select(cc0ca8a0,9,bfbfecf4,0,0,0,20,cc0c5598,bfbfecf4) at kern_select+0x655 sys_select(cc0ca8a0,f3fe6cec,c1025a5c,c0fbbbf7,1,...) at sys_select+0x66 syscall(f3fe6d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de53, esp = 0xbfbfec8c, ebp = 0xbfbfedd8 --- Tracing command rpcbind pid 976 tid 100099 td 0xcc0f35c0 sched_switch(cc0f35c0,0,104,1a3,bccc7bbe,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0f35c0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f401ca78,c0a7478a,cac79150,0,cc0f35c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac79164,0,f401cab4,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac79164,cac79150,7531,6ae,f401cb6c,...) at _cv_timedwait_sig+0x270 seltdwait(f401cc3c,f401cc44,5ab,cc0f35c0,cc72d70c,...) at seltdwait+0x8a sys_poll(cc0f35c0,f401ccec,c1025a5c,c0fdafee,c1027010,...) at sys_poll+0x321 syscall(f401cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2812c3ab, esp = 0xbfbfcb9c, ebp = 0xbfbfed68 --- Tracing command syslogd pid 961 tid 100095 td 0xcc0c98a0 sched_switch(cc0c98a0,0,104,1a3,a47e04ff,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0c98a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac85950,0,c0fd05a7,cc0c98a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac85964,0,f400ca50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac85964,cac85950,c0fda6d8,6ae,cc05be70,...) at _cv_wait_sig+0x260 seltdwait(cc05be70,58,c95d9b80,cc0c98a0,0,...) at seltdwait+0xa2 kern_select(cc0c98a0,9,28849100,0,0,0,20,cc0c4598,0) at kern_select+0x655 sys_select(cc0c98a0,f400ccec,c1025a5c,c0fdaea7,e,...) at sys_select+0x66 syscall(f400cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de53, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- Tracing command devd pid 838 tid 100084 td 0xcc04d8a0 sched_switch(cc04d8a0,0,104,1a3,4c7d3178,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc04d8a0,0,c0fd7f80,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,c95601d0,0,c0fd05a7,cc04d8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c95601e4,0,f3fdea50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c95601e4,c95601d0,c0fda6d8,6ae,cc08b268,...) at _cv_wait_sig+0x260 seltdwait(cc08b268,58,c95d9b80,cc04d8a0,c11088c8,...) at seltdwait+0xa2 kern_select(cc04d8a0,5,bfbfe950,0,0,0,20,cc046864,34) at kern_select+0x655 sys_select(cc04d8a0,f3fdecec,c1025a5c,c0e320ef,c1027010,...) at sys_select+0x66 syscall(f3fded28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x80893bb, esp = 0xbfbfe91c, ebp = 0xbfbfedf8 --- Tracing command softdepflush pid 19 tid 100078 td 0xcbee88a0 sched_switch(cbee88a0,0,104,1a3,ffeb83e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cbee88a0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(e3277800,5c,c0fdf8b5,0,0,...) at sleepq_wait+0x63 _sleep(e3277800,c959c5ec,5c,c0fdf8b5,0,...) at _sleep+0x39c bwait(e3277800,5c,c0fdf8b5,e3277800,efbe9a48,...) at bwait+0x6f bufwait(e3277800,0,0,0,0,...) at bufwait+0x28 breadn_flags(cc14d828,49381a0,0,4000,0,...) at breadn_flags+0x139 indir_trunc(fffc2ff4,ffffffff,c1007b6c,1e0e,0,...) at indir_trunc+0x114 indir_trunc(fffff7f3,ffffffff,c1007b6c,1cd0,c12d79e8,...) at indir_trunc+0x4ba handle_workitem_freeblocks(0,efbe9c70,2,6ba,efbe9c74,...) at handle_workitem_freeblocks+0x24f process_worklist_item(c130c580,0,c1007b6c,5e9,cbfd4700,...) at process_worklist_item+0x2c2 softdep_process_worklist(cbfc02dc,0,c1007b6c,562,3e8,...) at softdep_process_worklist+0x84 softdep_flush(0,efbe9d28,c0fcc590,3d8,cbedb000,...) at softdep_flush+0x2a0 fork_exit(c0cd7e10,0,efbe9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe9d60, ebp = 0 --- Tracing command vnlru pid 18 tid 100077 td 0xcbee8b80 sched_switch(cbee8b80,0,104,1a3,1937caf1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cbee8b80,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cbedb2cc,60,c0fe30d9,0,0,...) at sleepq_timedwait+0x6b _sleep(cbedb2cc,c1304874,260,c0fe30d9,3e8,...) at _sleep+0x36a vnlru_proc(0,efbe6d28,c0fcc590,3d8,cbedb2cc,...) at vnlru_proc+0xe7 fork_exit(c0b2b970,0,efbe6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe6d60, ebp = 0 --- Tracing command syncer pid 17 tid 100076 td 0xcacf98a0 sched_switch(cacf98a0,0,104,1a3,f8112d89,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf98a0,0,c0fd7f80,28b,cacf98a0,...) at sleepq_switch+0x162 sleepq_timedwait(c13048b4,0,efbe3c78,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c13048b4,c13048a0,3e8,783,4e20,...) at _cv_timedwait+0x270 sched_sync(0,efbe3d28,c0fcc590,3d8,cbedb598,...) at sched_sync+0x595 fork_exit(c0b2aca0,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xcacf9b80 sched_switch(cacf9b80,0,104,1a3,c02ab965,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cacf9b80,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304548,54,c0fdfe52,0,0,...) at sleepq_timedwait+0x6b _sleep(c1304548,c1304550,54,c0fdfe52,3e8,...) at _sleep+0x36a buf_daemon(0,efbe0d28,c0fcc590,3d8,cbedb864,...) at buf_daemon+0x175 fork_exit(c0b0d9c0,0,efbe0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe0d60, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xcae24000 sched_switch(cae24000,0,104,1a3,35fab098,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24000,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e468,0,c101128c,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e468,c130cf00,0,c101128c,493e0,...) at _sleep+0x36a vm_pagezero(0,efbddd28,c0fcc590,3d8,cbedbb30,...) at vm_pagezero+0xdc fork_exit(c0d1fa20,0,efbddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbddd60, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xcae242e0 sched_switch(cae242e0,0,104,1a3,66836a7a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cae242e0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130e048,74,c0fdfe52,0,0,...) at sleepq_wait+0x63 _sleep(c130e048,c130e04c,74,c0fdfe52,0,...) at _sleep+0x39c vm_daemon(0,efbdad28,c0fcc590,3d8,cbedc000,...) at vm_daemon+0x59 fork_exit(c0d18320,0,efbdad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbdad60, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xcae245c0 sched_switch(cae245c0,0,104,1a3,29727a73,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cae245c0,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e010,54,c0fdfe52,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e010,c130cf00,54,c0fdfe52,1388,...) at _sleep+0x36a vm_pageout(0,efbd7d28,c0fcc590,3d8,c95de2cc,...) at vm_pageout+0x2c7 fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xcae248a0 sched_switch(cae248a0,0,104,1a3,dda4407e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae248a0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1151054,5c,c0e2c8de,0,0,...) at sleepq_wait+0x63 _sleep(c1151054,c1151098,5c,c0e2c8de,0,...) at _sleep+0x39c xpt_scanner_thread(0,efa54d28,c0fcc590,3d8,c95de598,...) at xpt_scanner_thread+0x47 fork_exit(c04a72f0,0,efa54d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa54d60, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xcae24b80 sched_switch(cae24b80,0,104,1a3,2ff86c11,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24b80,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1307dd8,0,c0ff759f,0,0,...) at sleepq_wait+0x63 _sleep(c1307dd8,c1307da0,0,c0ff759f,0,...) at _sleep+0x39c sctp_iterator_thread(0,efa51d28,c0fcc590,3d8,c95de864,...) at sctp_iterator_thread+0x5c fork_exit(c0bc5a50,0,efa51d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa51d60, ebp = 0 --- Tracing command fdc0 pid 4 tid 100065 td 0xcae25b80 sched_switch(cae25b80,0,104,1a3,1a9cedf6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae25b80,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cac7223c,5c,c0fbe912,0,0,...) at sleepq_timedwait+0x6b _sleep(cac7223c,cac722f0,5c,c0fbe912,3e8,...) at _sleep+0x36a fdc_thread(cac72200,efa42d28,c0fcc590,3d8,c95deb30,...) at fdc_thread+0x27d fork_exit(c0da1d50,cac72200,efa42d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa42d60, ebp = 0 --- Tracing command fw0_probe pid 3 tid 100063 td 0xcace42e0 sched_switch(cace42e0,0,104,1a3,dda45d7c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cace42e0,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cad2c000,6c,c0fbe912,100,0,...) at sleepq_wait_sig+0x17 _sleep(cad2c000,cad30488,16c,c0fbe912,0,...) at _sleep+0x385 fw_bus_probe_thread(cad2c000,efa32d28,c0fcc590,3d8,cac0b000,...) at fw_bus_probe_thread+0xa08 fork_exit(c07017d0,cad2c000,efa32d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa32d60, ebp = 0 --- Tracing command usb pid 15 tid 100059 td 0xcacf7000 sched_switch(cacf7000,0,104,1a3,cb3d9385,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf7000,0,c0fd7f80,268,cacf7000,...) at sleepq_switch+0x162 sleepq_wait(cacfed34,0,efa1bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed34,cacfee4c,c0faff91,74,cacfed3c,...) at _cv_wait+0x260 usb_process(cacfed2c,efa1bd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfed2c,efa1bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa1bd60, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xcacf72e0 sched_switch(cacf72e0,0,104,1a3,98ccf2bb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf72e0,0,c0fd7f80,268,cacf72e0,...) at sleepq_switch+0x162 sleepq_wait(cacfed04,0,efa18cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed04,cacfee4c,c0faff91,74,cacfed0c,...) at _cv_wait+0x260 usb_process(cacfecfc,efa18d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfecfc,efa18d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa18d60, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xcacf75c0 sched_switch(cacf75c0,0,104,1a3,cad75887,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf75c0,0,c0fd7f80,268,cacf75c0,...) at sleepq_switch+0x162 sleepq_wait(cacfecd4,0,efa15cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfecd4,cacfee4c,c0faff91,74,cacfecdc,...) at _cv_wait+0x260 usb_process(cacfeccc,efa15d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfeccc,efa15d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa15d60, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xcacf78a0 sched_switch(cacf78a0,0,104,1a3,cad7479f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf78a0,0,c0fd7f80,268,cacf78a0,...) at sleepq_switch+0x162 sleepq_wait(cacfeca4,0,efa12cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfeca4,cacfee4c,c0faff91,74,cacfecac,...) at _cv_wait+0x260 usb_process(cacfec9c,efa12d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfec9c,efa12d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa12d60, ebp = 0 --- Tracing command usb pid 15 tid 100054 td 0xcacf9000 sched_switch(cacf9000,0,104,1a3,cad73282,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf9000,0,c0fd7f80,268,cacf9000,...) at sleepq_switch+0x162 sleepq_wait(cacefb5c,0,efa09cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb5c,cacefc74,c0faff91,74,cacefb64,...) at _cv_wait+0x260 usb_process(cacefb54,efa09d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb54,efa09d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa09d60, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xcacf92e0 sched_switch(cacf92e0,0,104,1a3,1e42e624,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf92e0,0,c0fd7f80,268,cacf92e0,...) at sleepq_switch+0x162 sleepq_wait(cacefb2c,0,efa06cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb2c,cacefc74,c0faff91,74,cacefb34,...) at _cv_wait+0x260 usb_process(cacefb24,efa06d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb24,efa06d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa06d60, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xcacf95c0 sched_switch(cacf95c0,0,104,1a3,ca733996,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf95c0,0,c0fd7f80,268,cacf95c0,...) at sleepq_switch+0x162 sleepq_wait(cacefafc,0,efa03cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefafc,cacefc74,c0faff91,74,cacefb04,...) at _cv_wait+0x260 usb_process(cacefaf4,efa03d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefaf4,efa03d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa03d60, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xcace2000 sched_switch(cace2000,0,104,1a3,ca73266c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2000,0,c0fd7f80,268,cace2000,...) at sleepq_switch+0x162 sleepq_wait(cacefacc,0,efa00cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefacc,cacefc74,c0faff91,74,cacefad4,...) at _cv_wait+0x260 usb_process(cacefac4,efa00d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefac4,efa00d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa00d60, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xcace22e0 sched_switch(cace22e0,0,104,1a3,ca731191,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace22e0,0,c0fd7f80,268,cace22e0,...) at sleepq_switch+0x162 sleepq_wait(caceab5c,0,ef9fccac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab5c,caceac74,c0faff91,74,caceab64,...) at _cv_wait+0x260 usb_process(caceab54,ef9fcd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab54,ef9fcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9fcd60, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xcace25c0 sched_switch(cace25c0,0,104,1a3,1b41e628,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace25c0,0,c0fd7f80,268,cace25c0,...) at sleepq_switch+0x162 sleepq_wait(caceab2c,0,ef9f9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab2c,caceac74,c0faff91,74,caceab34,...) at _cv_wait+0x260 usb_process(caceab24,ef9f9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab24,ef9f9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f9d60, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xcace28a0 sched_switch(cace28a0,0,104,1a3,c4a21d49,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace28a0,0,c0fd7f80,268,cace28a0,...) at sleepq_switch+0x162 sleepq_wait(caceaafc,0,ef9f6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaafc,caceac74,c0faff91,74,caceab04,...) at _cv_wait+0x260 usb_process(caceaaf4,ef9f6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaaf4,ef9f6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f6d60, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xcace2b80 sched_switch(cace2b80,0,104,1a3,c4a20b9f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2b80,0,c0fd7f80,268,cace2b80,...) at sleepq_switch+0x162 sleepq_wait(caceaacc,0,ef9f3cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaacc,caceac74,c0faff91,74,caceaad4,...) at _cv_wait+0x260 usb_process(caceaac4,ef9f3d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaac4,ef9f3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f3d60, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xcace3000 sched_switch(cace3000,0,104,1a3,c4a1f892,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace3000,0,c0fd7f80,268,cace3000,...) at sleepq_switch+0x162 sleepq_wait(cace5b5c,0,ef9efcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b5c,cace5c74,c0faff91,74,cace5b64,...) at _cv_wait+0x260 usb_process(cace5b54,ef9efd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b54,ef9efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9efd60, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xcace32e0 sched_switch(cace32e0,0,104,1a3,1a9d1fc0,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace32e0,0,c0fd7f80,268,cace32e0,...) at sleepq_switch+0x162 sleepq_wait(cace5b2c,0,ef9eccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b2c,cace5c74,c0faff91,74,cace5b34,...) at _cv_wait+0x260 usb_process(cace5b24,ef9ecd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b24,ef9ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ecd60, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xcace35c0 sched_switch(cace35c0,0,104,1a3,c43e30d9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace35c0,0,c0fd7f80,268,cace35c0,...) at sleepq_switch+0x162 sleepq_wait(cace5afc,0,ef9e9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5afc,cace5c74,c0faff91,74,cace5b04,...) at _cv_wait+0x260 usb_process(cace5af4,ef9e9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5af4,ef9e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e9d60, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xcace38a0 sched_switch(cace38a0,0,104,1a3,c43e2069,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace38a0,0,c0fd7f80,268,cace38a0,...) at sleepq_switch+0x162 sleepq_wait(cace5acc,0,ef9e6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5acc,cace5c74,c0faff91,74,cace5ad4,...) at _cv_wait+0x260 usb_process(cace5ac4,ef9e6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5ac4,ef9e6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e6d60, ebp = 0 --- Tracing command usb pid 15 tid 100041 td 0xcace4000 sched_switch(cace4000,0,104,1a3,c43e0c5a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace4000,0,c0fd7f80,268,cace4000,...) at sleepq_switch+0x162 sleepq_wait(cacddb5c,0,ef9dfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb5c,cacddc74,c0faff91,74,cacddb64,...) at _cv_wait+0x260 usb_process(cacddb54,ef9dfd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb54,ef9dfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dfd60, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xcacbf000 sched_switch(cacbf000,0,104,1a3,1953ed69,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf000,0,c0fd7f80,268,cacbf000,...) at sleepq_switch+0x162 sleepq_wait(cacddb2c,0,ef9dccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb2c,cacddc74,c0faff91,74,cacddb34,...) at _cv_wait+0x260 usb_process(cacddb24,ef9dcd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb24,ef9dcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dcd60, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xcacbf2e0 sched_switch(cacbf2e0,0,104,1a3,c3da5b0d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf2e0,0,c0fd7f80,268,cacbf2e0,...) at sleepq_switch+0x162 sleepq_wait(cacddafc,0,ef9d9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddafc,cacddc74,c0faff91,74,cacddb04,...) at _cv_wait+0x260 usb_process(cacddaf4,ef9d9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddaf4,ef9d9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d9d60, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xcacbf5c0 sched_switch(cacbf5c0,0,104,1a3,c3da492f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf5c0,0,c0fd7f80,268,cacbf5c0,...) at sleepq_switch+0x162 sleepq_wait(cacddacc,0,ef9d6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddacc,cacddc74,c0faff91,74,cacddad4,...) at _cv_wait+0x260 usb_process(cacddac4,ef9d6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddac4,ef9d6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d6d60, ebp = 0 --- Tracing command usb pid 15 tid 100036 td 0xcacbfb80 sched_switch(cacbfb80,0,104,1a3,c3da3572,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbfb80,0,c0fd7f80,268,cacbfb80,...) at sleepq_switch+0x162 sleepq_wait(cacd5b5c,0,ef9cfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b5c,cacd5c74,c0faff91,74,cacd5b64,...) at _cv_wait+0x260 usb_process(cacd5b54,ef9cfd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b54,ef9cfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9cfd60, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xcacc1000 sched_switch(cacc1000,0,104,1a3,1953ba30,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc1000,0,c0fd7f80,268,cacc1000,...) at sleepq_switch+0x162 sleepq_wait(cacd5b2c,0,ef9cccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b2c,cacd5c74,c0faff91,74,cacd5b34,...) at _cv_wait+0x260 usb_process(cacd5b24,ef9ccd28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b24,ef9ccd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ccd60, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xcacc12e0 sched_switch(cacc12e0,0,104,1a3,c3764cce,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc12e0,0,c0fd7f80,268,cacc12e0,...) at sleepq_switch+0x162 sleepq_wait(cacd5afc,0,ef9c9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5afc,cacd5c74,c0faff91,74,cacd5b04,...) at _cv_wait+0x260 usb_process(cacd5af4,ef9c9d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5af4,ef9c9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c9d60, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xcacc15c0 sched_switch(cacc15c0,0,104,1a3,c37638c3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc15c0,0,c0fd7f80,268,cacc15c0,...) at sleepq_switch+0x162 sleepq_wait(cacd5acc,0,ef9c6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5acc,cacd5c74,c0faff91,74,cacd5ad4,...) at _cv_wait+0x260 usb_process(cacd5ac4,ef9c6d28,c0fcc590,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5ac4,ef9c6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c6d60, ebp = 0 --- Tracing command ctl_thrd pid 2 tid 100028 td 0xc96eb8a0 sched_switch(c96eb8a0,0,104,1a3,c3761c9c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb8a0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c97c4000,5c,c0e37350,0,0,...) at sleepq_wait+0x63 _sleep(c97c4000,c97c4000,5c,c0e37350,0,...) at _sleep+0x39c ctl_work_thread(c97c4000,ef733d28,c0fcc590,3d8,cac0b598,...) at ctl_work_thread+0x22f6 fork_exit(c04d08d0,c97c4000,ef733d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef733d60, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xc96ea5c0 sched_switch(c96ea5c0,0,104,1a3,204ff631,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ea5c0,0,c0fd7f80,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe912,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe912,64) at _sleep+0x36a pause(c0fbe912,64,c0f8bff0,111,0,...) at pause+0xa0 random_kthread(0,e33e0d28,c0fcc590,3d8,c95dd000,...) at random_kthread+0x1ef fork_exit(c082d020,0,e33e0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e0d60, ebp = 0 --- Tracing command geom pid 13 tid 100016 td 0xc96ea8a0 sched_switch(c96ea8a0,0,104,1a3,27e5b56d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96ea8a0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a5a8,5c,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c118a5a8,c118a508,25c,c0fbe912,0,...) at _sleep+0x39c g_io_schedule_down(c96ea8a0,0,c0fc0461,6c,e33ddd14,...) at g_io_schedule_down+0x5b g_down_procbody(0,e33ddd28,c0fcc590,3d8,c95dd2cc,...) at g_down_procbody+0x9d fork_exit(c09e6af0,0,e33ddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ddd60, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xc96eab80 cpustop_handler(1,e33daad4,c0e00ae3,c93dbc80,e33daa24,...) at cpustop_handler+0x26 ipi_nmi_handler(c93dbc80,e33daa24,c0a742f3,c93dbc80,c95dd2cc,...) at ipi_nmi_handler+0x35 trap(e33daae0) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a74bf0, esp = 0xe33dab20, ebp = 0xe33dab74 --- _mtx_lock_sleep(cdaea5c8,c96eab80,0,c0fdf50b,d2a,...) at _mtx_lock_sleep+0x1e0 _mtx_lock_flags(cdaea5c8,0,c0fdf50b,d2a,c0fdf50b,...) at _mtx_lock_flags+0x10f bufdone_finish(e31febe0,e30e9e20,e336fec0,0,e33dac3c,...) at bufdone_finish+0x11a bufdone(e31febe0,20,e30e9e20,cca27e90,ccfef980,...) at bufdone+0x6a cluster_callback(e30e9e20,cbfd8040,cbfd8040,e30e9e20,e33dac80,...) at cluster_callback+0x89 bufdone(e30e9e20,0,c0fc13d7,9d,cca27e5c,c95b6cc0) at bufdone+0x53 g_vfs_done(ccfef980,0,c0fdf50b,c52,c96eab80,...) at g_vfs_done+0x1d0 biodone(ccfef980,c118a528,25c,c0fbe912,0,...) at biodone+0xa5 g_io_schedule_up(c96eab80,0,c0fc0461,5f,e33dad14,...) at g_io_schedule_up+0xde g_up_procbody(0,e33dad28,c0fcc590,3d8,c95dd2cc,...) at g_up_procbody+0x9d fork_exit(c09e6b90,0,e33dad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33dad60, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xc96eb000 sched_switch(c96eb000,0,104,1a3,cf6e13e0,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb000,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a59c,5c,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c118a59c,c118a4cc,25c,c0fbe912,0,...) at _sleep+0x39c g_run_events(c119b340,0,c0fc0461,79,e33d7d14,...) at g_run_events+0x54b g_event_procbody(0,e33d7d28,c0fcc590,3d8,c95dd2cc,...) at g_event_procbody+0x90 fork_exit(c09e6c30,0,e33d7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d7d60, ebp = 0 --- Tracing command intr pid 12 tid 100067 td 0xcae255c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100066 td 0xcae258a0 sched_switch(cae258a0,0,109,1a3,c3726abc,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cae258a0,...) at mi_switch+0x1ff ithread_loop(cae1e990,efa45d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cae1e990,efa45d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa45d60, ebp = 0 --- Tracing command intr pid 12 tid 100064 td 0xcae26000 sched_switch(cae26000,0,109,1a3,d86f8de4,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cae26000,...) at mi_switch+0x1ff ithread_loop(cae18a40,efa35d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cae18a40,efa35d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa35d60, ebp = 0 --- Tracing command intr pid 12 tid 100061 td 0xcace48a0 sched_switch(cace48a0,0,109,1a3,b3570e96,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cace48a0,...) at mi_switch+0x1ff ithread_loop(cad15880,efa29d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15880,efa29d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa29d60, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xcace4b80 sched_switch(cace4b80,0,109,1a3,9e30c313,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cace4b80,...) at mi_switch+0x1ff ithread_loop(cad15010,efa21d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15010,efa21d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa21d60, ebp = 0 --- Tracing command intr pid 12 tid 100055 td 0xcacf7b80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100042 td 0xcace3b80 sched_switch(cace3b80,0,109,1a3,e878220f,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cace3b80,...) at mi_switch+0x1ff ithread_loop(cacc5720,ef9e3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacc5720,ef9e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xcacbf8a0 sched_switch(cacbf8a0,0,109,1a3,e2fec1df,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cacbf8a0,...) at mi_switch+0x1ff ithread_loop(cacad590,ef9d3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad590,ef9d3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d3d60, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xcacc18a0 sched_switch(cacc18a0,0,109,1a3,a69498bf,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cacc18a0,...) at mi_switch+0x1ff ithread_loop(cacad830,ef9c3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad830,ef9c3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c3d60, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xcacc1b80 sched_switch(cacc1b80,0,109,1a3,2979d94e,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,cacc1b80,...) at mi_switch+0x1ff ithread_loop(cac97ae0,ef86dd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97ae0,ef86dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef86dd60, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc96eb2e0 sched_switch(c96eb2e0,0,109,1a3,dfbea2cc,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96eb2e0,...) at mi_switch+0x1ff ithread_loop(cac97c80,ef869d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97c80,ef869d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef869d60, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc96eb5c0 sched_switch(c96eb5c0,0,109,1a3,b5353393,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96eb5c0,...) at mi_switch+0x1ff ithread_loop(cac97500,ef763d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97500,ef763d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef763d60, ebp = 0 --- Tracing command intr pid 12 tid 100022 td 0xc97c1b80 sched_switch(c97c1b80,0,109,1a3,68b3891,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c97c1b80,...) at mi_switch+0x1ff ithread_loop(c95b3450,e33efd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3450,e33efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33efd60, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc96228a0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100019 td 0xc96ea000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100018 td 0xc96ea2e0 sched_switch(c96ea2e0,0,109,1a3,5f421135,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96ea2e0,...) at mi_switch+0x1ff ithread_loop(c95b3490,e33e3d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3490,e33e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc95e05c0 sched_switch(c95e05c0,0,109,1a3,1937b0b4,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c95e05c0,...) at mi_switch+0x1ff ithread_loop(c95b36f0,e33d1d28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b36f0,e33d1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d1d60, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc95e08a0 sched_switch(c95e08a0,0,109,1a3,1a9cbfd1,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c95e08a0,...) at mi_switch+0x1ff ithread_loop(c95b3700,e33ced28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3700,e33ced28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ced60, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc95e0b80 sched_switch(c95e0b80,0,109,1a3,1591c596,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c95e0b80,...) at mi_switch+0x1ff ithread_loop(c95b3710,e33cbd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3710,e33cbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33cbd60, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc9622000 cpustop_handler(0,c93feb80,c0e00ae3,1,cace45c0,...) at cpustop_handler+0x26 ipi_nmi_handler(1,cace45c0,c93fead8,6c1,c95dd598,...) at ipi_nmi_handler+0x35 trap(c93feb8c) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0aedba0, esp = 0xc93febcc, ebp = 0xc93febf0 --- ttydisc_rint_done(c970ce00,0,c0fd3af2,1e7,c0aae1bf,...) at ttydisc_rint_done softclock(c118e720,c119b340,4,c0fd05a7,c95aa0b8,...) at softclock+0x297 intr_event_execute_handlers(c95dd598,c95aa080,c0fcc8c1,561,c9622000,...) at intr_event_execute_handlers+0x125 ithread_loop(c95b3720,c93fed28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0xa9 fork_exit(c0a5a9c0,c95b3720,c93fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fed60, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc96222e0 sched_switch(c96222e0,0,109,1a3,f1bea180,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8c1,557,c96222e0,...) at mi_switch+0x1ff ithread_loop(c95b3730,c93fbd28,c0fcc590,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3730,c93fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fbd60, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc96225c0 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc95df000 cpustop_handler(3,c93f4bfc,c0e00ae3,0,6858d74f,...) at cpustop_handler+0x26 ipi_nmi_handler(0,6858d74f,987368ad,c136d380,c95dd864,...) at ipi_nmi_handler+0x35 trap(c93f4c08) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0de6185, esp = 0xc93f4c48, ebp = 0xc93f4c48 --- acpi_cpu_c1(c93f4c64,c1151630,3,317,6c1,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c93f4c98,c0df33e9,0,3,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,3,0,2,c93f4ce8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c93f4cc0,c0fd460c,a1f,c95df000,...) at cpu_idle+0x89 sched_idletd(0,c93f4d28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x235 fork_exit(c0aaf180,0,c93f4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f4d60, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc95df2e0 sched_switch(c95df2e0,0,108,19e,293b922a,...) at sched_switch+0x413 mi_switch(108,0,c0fd460c,a1d,c95df2e0,...) at mi_switch+0x1ff sched_idletd(0,c93f1d28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x28f fork_exit(c0aaf180,0,c93f1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f1d60, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc95df5c0 sched_switch(c95df5c0,0,608,19e,292ad0f5,...) at sched_switch+0x413 mi_switch(608,0,c0fd2c3f,cd,0,...) at mi_switch+0x1ff critical_exit(0,2710,c119b9c0,2,c93eece8,...) at critical_exit+0xb5 cpu_idle(0,c93eecc0,c0fd460c,a1f,c95df5c0,...) at cpu_idle+0x97 sched_idletd(0,c93eed28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x235 fork_exit(c0aaf180,0,c93eed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93eed60, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc95df8a0 sched_switch(c95df8a0,0,608,19e,2a46c549,...) at sched_switch+0x413 mi_switch(608,0,c0fd2c3f,cd,0,...) at mi_switch+0x1ff critical_exit(0,2710,c119b340,2,c93ebce8,...) at critical_exit+0xb5 cpu_idle(0,c93ebcc0,c0fd460c,a1f,c95df8a0,...) at cpu_idle+0x97 sched_idletd(0,c93ebd28,c0fcc590,3d8,c95dd864,...) at sched_idletd+0x235 fork_exit(c0aaf180,0,c93ebd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93ebd60, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc95dfb80 sched_switch(c95dfb80,0,104,1a3,3f59d753,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c95dfb80,0,c0fd7f80,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7f80,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c95ddb30,6c,c0fdaeb8,100,0,...) at sleepq_wait_sig+0x17 _sleep(c95ddb30,c95ddbb8,16c,c0fdaeb8,0,...) at _sleep+0x385 kern_wait(c95dfb80,ffffffff,c93e7c58,0,0,...) at kern_wait+0x597 sys_wait4(c95dfb80,c93e7cec,c1025a5c,c0fdad06,c1027010,...) at sys_wait4+0x3b syscall(c93e7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x806070f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8c8 --- Tracing command audit pid 10 tid 100001 td 0xc95e0000 sched_switch(c95e0000,0,104,1a3,c373e4ef,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e0000,0,c0fd7f80,268,c95e0000,...) at sleepq_switch+0x162 sleepq_wait(c130bd00,0,c93e4c8c,1,0,...) at sleepq_wait+0x63 _cv_wait(c130bd00,c130bce4,c1004237,194,c93e4cc4,...) at _cv_wait+0x260 audit_worker(0,c93e4d28,c0fcc590,3d8,c95de000,...) at audit_worker+0x84 fork_exit(c0c95110,0,c93e4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93e4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xcae25000 sched_switch(cae25000,0,104,1a3,38dafaf,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae25000,0,c0fd7f80,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe912,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe912,bb8) at _sleep+0x36a pause(c0fbe912,bb8,126,124,c118a7e8,...) at pause+0xa0 deadlkres(0,efa4ed28,c0fcc590,3d8,c118a760,...) at deadlkres+0x324 fork_exit(c0a398a0,0,efa4ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4ed60, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xcae252e0 sched_switch(cae252e0,0,104,1a3,c373c83d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae252e0,0,c0fd7f80,268,cae252e0,...) at sleepq_switch+0x162 sleepq_wait(c971ddc0,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971ddc0,c971ddd8,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c136d440,efa4bd28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c136d440,efa4bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4bd60, ebp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xcace45c0 sched_switch(cace45c0,0,104,1a3,257304f6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace45c0,0,c0fd7f80,268,cace45c0,...) at sleepq_switch+0x162 sleepq_wait(c971de00,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971de00,c971de18,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(cad3049c,efa2fd28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,cad3049c,efa2fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa2fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc96ebb80 sched_switch(c96ebb80,0,104,1a3,dda8b640,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ebb80,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de40,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c971de40,c971de58,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c118b0f8,e33fed28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c118b0f8,e33fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fed60, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc97c1000 sched_switch(c97c1000,0,104,1a3,dda8a60b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c1000,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de80,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c971de80,c971de98,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c130c2e4,e33fbd28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c130c2e4,e33fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fbd60, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc97c12e0 sched_switch(c97c12e0,0,104,1a3,dda896c6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c12e0,0,c0fd7f80,268,c97c12e0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f8d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f8d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f8d60, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc97c15c0 sched_switch(c97c15c0,0,104,1a3,dda888a3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c15c0,0,c0fd7f80,268,c97c15c0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f5d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f5d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f5d60, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc97c18a0 sched_switch(c97c18a0,0,104,1a3,dda8788d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c18a0,0,c0fd7f80,268,c97c18a0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3161,c0fbe912,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe912,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f2d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f2d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f2d60, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc9622b80 sched_switch(c9622b80,0,104,1a3,8e4fde58,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c9622b80,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971e0c0,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c971e0c0,c971e0d8,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c11aa0ac,e33e9d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11aa0ac,e33e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e9d60, ebp = 0 --- Tracing command kernel pid 0 tid 100013 td 0xc95e02e0 sched_switch(c95e02e0,0,104,1a3,cfff9e8e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e02e0,0,c0fd7f80,268,0,...) at sleepq_switch+0x162 sleepq_wait(c96e9000,0,c0fbe912,0,0,...) at sleepq_wait+0x63 _sleep(c96e9000,c96e9018,0,c0fbe912,0,...) at _sleep+0x39c taskqueue_thread_loop(c11a8ac0,e33d4d28,c0fcc590,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11a8ac0,e33d4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc118aa30 sched_switch(c118aa30,0,104,1a3,6c2b534e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7f80,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c118aa30,0,c0fd7f80,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c118a760,54,c1004c18,0,0,...) at sleepq_timedwait+0x6b _sleep(c118a760,0,54,c1004c18,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x240 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 310 MB: 295 279 263 247 231 215 199 183 167 151 135 119 103 87 71 55 39 23 7 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 2 (kgdb) bt #0 doadump (textdump=0x0) at pcpu.h:244 #1 0xc051b3a9 in db_fncall (dummy1=0xc0abd94a, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf403a260 "t¢\003ô") at ../../../ddb/db_command.c:573 #2 0xc051b7df in db_command (last_cmdp=0xc115245c, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:449 #3 0xc051b894 in db_command_script (command=0xc1153368 "call doadump") at ../../../ddb/db_command.c:520 #4 0xc051fcd0 in db_script_exec (scriptname=0xc1152cc0 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc051fd61 in db_run_cmd (addr=0x1, have_addr=0x0, count=0xc132da60, modif=0xf403a398 "") at ../../../ddb/db_script.c:375 #6 0xc051b7a1 in db_command (last_cmdp=0xc115245c, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:449 #7 0xc051b8fa in db_command_loop () at ../../../ddb/db_command.c:502 #8 0xc051d8ed in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:231 #9 0xc0abde66 in kdb_trap (type=0x3, code=0x0, tf=0xf403a594) at ../../../kern/subr_kdb.c:654 #10 0xc0e0124b in trap (frame=0xf403a594) at ../../../i386/i386/trap.c:710 #11 0xc0dea81c in calltrap () at ../../../i386/i386/exception.s:169 #12 0xc0abd94a in kdb_enter (why=0xc0fd23c4 "panic", msg=0xc0fd23c4 "panic") at cpufunc.h:71 #13 0xc0a86e3c in panic (fmt=0xc100d906 "Duplicate free of item %p from zone %p(%s)\n") at ../../../kern/kern_shutdown.c:633 #14 0xc0d0095b in uma_dbg_free (zone=0xc1bb69a0, slab=0xc7643f44, item=0xc76437e0) at ../../../vm/uma_dbg.c:302 #15 0xc0cff7e4 in uma_zfree_arg (zone=0xc1bb69a0, item=0xc76437e0, udata=0x0) at ../../../vm/uma_core.c:2595 #16 0xc0d1d1f8 in vm_radix_sweep (rtree=0xcdaea5f8, index=0x7b9e0, color=0x1, hard=0x0) at uma.h:337 #17 0xc0d1dd8c in vm_radix_insert (rtree=0xcdaea5f8, index=0x7b9e0, val=0xc48dc600) at ../../../vm/vm_radix.c:529 #18 0xc0d14f95 in vm_page_insert (m=0xc48dc600, object=0xcdaea5c8, pindex=0x7b9e0) at ../../../vm/vm_page.c:811 #19 0xc0d171fc in vm_page_alloc (object=0xcdaea5c8, pindex=0x7b9e0, req=0x40222) at ../../../vm/vm_page.c:1273 #20 0xc0d17722 in vm_page_grab (object=0xcdaea5c8, pindex=0x7b9e0, allocflags=0x412a2) at ../../../vm/vm_page.c:2290 #21 0xc0b0ad92 in allocbuf (bp=0xe31d06e0, size=0x4000) at ../../../kern/vfs_bio.c:3067 #22 0xc0b0eb53 in getblk (vp=0xcca27d98, blkno=0x1ee78, size=0x4000, slpflag=0x0, slptimeo=0x0, flags=0x0) at ../../../kern/vfs_bio.c:2837 #23 0xc0cb6d5c in ffs_balloc_ufs2 (vp=0xcca27d98, startoffset=Variable "startoffset" is not available. ) at ../../../ufs/ffs/ffs_balloc.c:926 #24 0xc0ce1f0d in ffs_write (ap=0xf403aba4) at ../../../ufs/ffs/ffs_vnops.c:721 #25 0xc0e281e4 in VOP_WRITE_APV (vop=0xc1137580, a=0xf403aba4) at vnode_if.c:951 #26 0xc0b38d5c in vn_write (fp=0xcc05ab98, uio=0xf403ac3c, active_cred=0xcc095d80, flags=0x0, td=0xcc0c95c0) at vnode_if.h:413 #27 0xc0ad6b85 in dofilewrite (td=0xcc0c95c0, fd=0x4, fp=0xcc05ab98, auio=0xf403ac3c, offset=0xffffffffffffffff, flags=0x0) at file.h:271 #28 0xc0ad71f8 in kern_writev (td=0xcc0c95c0, fd=0x4, auio=0xf403ac3c) at ../../../kern/sys_generic.c:459 #29 0xc0ad730f in sys_write (td=0xcc0c95c0, uap=0xf403acec) at ../../../kern/sys_generic.c:375 #30 0xc0e005ee in syscall (frame=0xf403ad28) at subr_syscall.c:135 #31 0xc0dea8b1 in Xint0x80_syscall () at ../../../i386/i386/exception.s:267 #32 0x00000033 in ?? () (kgdb) f 17 #17 0xc0d1dd8c in vm_radix_insert (rtree=0xcdaea5f8, index=0x7b9e0, val=0xc48dc600) at ../../../vm/vm_radix.c:529 529 vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); (kgdb) l 524 level, slot, rnode); 525 CTR4(KTR_VM, 526 "insert: tree %p, rnode %p, child %p, count %u ENOMEM", 527 rtree, rnode, rnode->rn_child[slot], 528 rnode->rn_count); 529 vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); 530 vm_radix_unwind_heightup(rtree, root, iroot, 531 ilevel); 532 return (ENOMEM); 533 } (kgdb) info loc rnode = (struct vm_radix_node *) 0xc76437e0 root = (struct vm_radix_node *) 0xc76437e0 level = 0xf slot = 0x0 (kgdb) (kgdb) bt #0 doadump (textdump=0x0) at pcpu.h:244 #1 0xc051b3a9 in db_fncall (dummy1=0xc0abd94a, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf403a260 "t¢\003ô") at ../../../ddb/db_command.c:573 #2 0xc051b7df in db_command (last_cmdp=0xc115245c, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:449 #3 0xc051b894 in db_command_script (command=0xc1153368 "call doadump") at ../../../ddb/db_command.c:520 #4 0xc051fcd0 in db_script_exec (scriptname=0xc1152cc0 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc051fd61 in db_run_cmd (addr=0x1, have_addr=0x0, count=0xc132da60, modif=0xf403a398 "") at ../../../ddb/db_script.c:375 #6 0xc051b7a1 in db_command (last_cmdp=0xc115245c, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:449 #7 0xc051b8fa in db_command_loop () at ../../../ddb/db_command.c:502 #8 0xc051d8ed in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:231 #9 0xc0abde66 in kdb_trap (type=0x3, code=0x0, tf=0xf403a594) at ../../../kern/subr_kdb.c:654 #10 0xc0e0124b in trap (frame=0xf403a594) at ../../../i386/i386/trap.c:710 #11 0xc0dea81c in calltrap () at ../../../i386/i386/exception.s:169 #12 0xc0abd94a in kdb_enter (why=0xc0fd23c4 "panic", msg=0xc0fd23c4 "panic") at cpufunc.h:71 #13 0xc0a86e3c in panic (fmt=0xc100d906 "Duplicate free of item %p from zone %p(%s)\n") at ../../../kern/kern_shutdown.c:633 #14 0xc0d0095b in uma_dbg_free (zone=0xc1bb69a0, slab=0xc7643f44, item=0xc76437e0) at ../../../vm/uma_dbg.c:302 #15 0xc0cff7e4 in uma_zfree_arg (zone=0xc1bb69a0, item=0xc76437e0, udata=0x0) at ../../../vm/uma_core.c:2595 #16 0xc0d1d1f8 in vm_radix_sweep (rtree=0xcdaea5f8, index=0x7b9e0, color=0x1, hard=0x0) at uma.h:337 #17 0xc0d1dd8c in vm_radix_insert (rtree=0xcdaea5f8, index=0x7b9e0, val=0xc48dc600) at ../../../vm/vm_radix.c:529 #18 0xc0d14f95 in vm_page_insert (m=0xc48dc600, object=0xcdaea5c8, pindex=0x7b9e0) at ../../../vm/vm_page.c:811 #19 0xc0d171fc in vm_page_alloc (object=0xcdaea5c8, pindex=0x7b9e0, req=0x40222) at ../../../vm/vm_page.c:1273 #20 0xc0d17722 in vm_page_grab (object=0xcdaea5c8, pindex=0x7b9e0, allocflags=0x412a2) at ../../../vm/vm_page.c:2290 #21 0xc0b0ad92 in allocbuf (bp=0xe31d06e0, size=0x4000) at ../../../kern/vfs_bio.c:3067 #22 0xc0b0eb53 in getblk (vp=0xcca27d98, blkno=0x1ee78, size=0x4000, slpflag=0x0, slptimeo=0x0, flags=0x0) at ../../../kern/vfs_bio.c:2837 #23 0xc0cb6d5c in ffs_balloc_ufs2 (vp=0xcca27d98, startoffset=Variable "startoffset" is not available. ) at ../../../ufs/ffs/ffs_balloc.c:926 #24 0xc0ce1f0d in ffs_write (ap=0xf403aba4) at ../../../ufs/ffs/ffs_vnops.c:721 #25 0xc0e281e4 in VOP_WRITE_APV (vop=0xc1137580, a=0xf403aba4) at vnode_if.c:951 #26 0xc0b38d5c in vn_write (fp=0xcc05ab98, uio=0xf403ac3c, active_cred=0xcc095d80, flags=0x0, td=0xcc0c95c0) at vnode_if.h:413 #27 0xc0ad6b85 in dofilewrite (td=0xcc0c95c0, fd=0x4, fp=0xcc05ab98, auio=0xf403ac3c, offset=0xffffffffffffffff, flags=0x0) at file.h:271 #28 0xc0ad71f8 in kern_writev (td=0xcc0c95c0, fd=0x4, auio=0xf403ac3c) at ../../../kern/sys_generic.c:459 #29 0xc0ad730f in sys_write (td=0xcc0c95c0, uap=0xf403acec) at ../../../kern/sys_generic.c:375 #30 0xc0e005ee in syscall (frame=0xf403ad28) at subr_syscall.c:135 #31 0xc0dea8b1 in Xint0x80_syscall () at ../../../i386/i386/exception.s:267 #32 0x00000033 in ?? () (kgdb) f 17 #17 0xc0d1dd8c in vm_radix_insert (rtree=0xcdaea5f8, index=0x7b9e0, val=0xc48dc600) at ../../../vm/vm_radix.c:529 529 vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); (kgdb) l 524 level, slot, rnode); 525 CTR4(KTR_VM, 526 "insert: tree %p, rnode %p, child %p, count %u ENOMEM", 527 rtree, rnode, rnode->rn_child[slot], 528 rnode->rn_count); 529 vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); 530 vm_radix_unwind_heightup(rtree, root, iroot, 531 ilevel); 532 return (ENOMEM); 533 } (kgdb) info loc rnode = (struct vm_radix_node *) 0xc76437e0 root = (struct vm_radix_node *) 0xc76437e0 level = 0xf slot = 0x0 (kgdb) $ svn diff -x -p /var/tmp/vmcontention/sys Index: /var/tmp/vmcontention/sys/vm/vm_radix.c =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.c (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.c (working copy) @@ -99,14 +99,12 @@ #define KSPLT64H(x) ((u_long)(((x) >> 32) & 0xFFFFFFFF)) #endif -CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); -CTASSERT((sizeof(u_int) * NBBY) >= VM_RADIX_LIMIT); - struct vm_radix_node { void *rn_child[VM_RADIX_COUNT]; /* Child nodes. */ volatile uint32_t rn_count; /* Valid children. */ }; +CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); CTASSERT(sizeof(struct vm_radix_node) < PAGE_SIZE); static uma_zone_t vm_radix_node_zone; @@ -356,6 +354,103 @@ vm_radix_reclaim_allnodes_internal(struct vm_radix } /* + * Remove the specified index from the tree. If possible the height of the + * tree is adjusted after deletion. May be used to cleanup intermediate + * nodes of the path if the key is not entirely present. + */ +static void +vm_radix_sweep(struct vm_radix *rtree, vm_pindex_t index, int color, int hard) +{ + struct vm_radix_node *stack[VM_RADIX_LIMIT]; + struct vm_radix_node *rnode, *root; + int level; + int slot; + + level = vm_radix_height(rtree, &root); + KASSERT(index <= VM_RADIX_MAX(level), + ("vm_radix_sweep: %p index out of range %jd.", rtree, + VM_RADIX_MAX(level))); + rnode = root; + level--; + /* + * Find the node and record the path in stack. + */ + while (level && rnode) { + stack[level] = rnode; + slot = vm_radix_slot(index, level); + CTR6(KTR_VM, + "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", + rtree, rnode, rnode->rn_child[slot], rnode->rn_count); + rnode = rnode->rn_child[slot]; + level--; + } + if (rnode == NULL) { + if (hard) + panic("vm_radix_sweep: index not present.\n"); + + /* + * Almost certainly it got an half-constructed tree it was + * expected. + */ + KASSERT(level < (VM_RADIX_LIMIT - 1), + ("vm_radix_sweep: level %d not consistent.\n", level)); + ++level; + rnode = stack[level]; + slot = vm_radix_slot(index, level); + } else { + slot = vm_radix_slot(index, 0); + if (hard && + vm_radix_match(rnode->rn_child[slot], color) == NULL) + panic("vm_radix_sweep: index not present as leaf.\n"); + } + + for (;;) { + CTR6(KTR_VM, +"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, + "remove: resetting tree %p, rnode %p, child %p, count %u", + rtree, rnode, + (rnode != NULL) ? rnode->rn_child[slot] : NULL, + (rnode != NULL) ? rnode->rn_count : 0); + rnode->rn_child[slot] = NULL; + /* + * Use atomics for the last level since red and black + * will both adjust it. + * Use a write memory barrier here in order to avoid + * rn_count reaching 0 before to fetch the actual pointer. + * Concurrent black removal, infact, may want to reclaim + * the radix node itself before to read it. + */ + if (rnode->rn_count != 0) { + if (level == 0) + atomic_add_rel_32(&rnode->rn_count, -1); + else + rnode->rn_count--; + } else if (hard) + panic("vm_radix_sweep: empty node midway.\n"); + + /* + * Only allow black removes to prune the tree. + */ + if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) + break; + vm_radix_node_put(rnode); + if (rnode == root) { + vm_radix_setroot(rtree, NULL, 0); + break; + } + rnode = stack[++level]; + slot = vm_radix_slot(index, level); + + } +} + +/* * Inserts the key-value pair in to the radix tree. Returns errno. * Panics if the key already exists. */ @@ -363,8 +458,7 @@ int vm_radix_insert(struct vm_radix *rtree, vm_pindex_t index, void *val) { struct vm_radix_node *iroot, *rnode, *root; - u_int allocmsk; - int clev, ilevel, level, slot; + int ilevel, level, slot; CTR4(KTR_VM, "insert: tree %p, " KFRMT64(index) ", val %p", rtree, @@ -417,8 +511,6 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ } /* Now that the tree is tall enough, fill in the path to the index. */ - allocmsk = 0; - clev = level; rnode = root; for (level = level - 1; level > 0; level--) { slot = vm_radix_slot(index, level); @@ -434,35 +526,12 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ "insert: tree %p, rnode %p, child %p, count %u ENOMEM", rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - MPASS(level != clev || allocmsk == 0); - while (allocmsk != 0) { - rnode = root; - level = clev; - level--; - slot = vm_radix_slot(index, level); - CTR4(KTR_VM, - "insert: unwind root %p, level %d, slot %d, allocmsk: 0x%x", - root, level, slot, allocmsk); - MPASS(level >= (ffs(allocmsk) - 1)); - while (level > (ffs(allocmsk) - 1)) { - MPASS(level > 0); - slot = vm_radix_slot(index, - level); - rnode = rnode->rn_child[slot]; - level--; - } - MPASS((allocmsk & (1 << level)) != 0); - allocmsk &= ~(1 << level); - rnode->rn_count--; - vm_radix_node_put(rnode->rn_child[slot]); - rnode->rn_child[slot] = NULL; - } + vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); vm_radix_unwind_heightup(rtree, root, iroot, ilevel); return (ENOMEM); } rnode->rn_count++; - allocmsk |= (1 << level); } CTR6(KTR_VM, "insert: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", @@ -812,86 +881,13 @@ restart: } /* - * Remove the specified index from the tree. If possible the height of the - * tree is adjusted after deletion. The value stored at index is returned - * panics if the key is not present. + * Remove an entry from the tree. Expects the entry to be already present. */ -void * +void vm_radix_remove(struct vm_radix *rtree, vm_pindex_t index, int color) { - struct vm_radix_node *stack[VM_RADIX_LIMIT]; - struct vm_radix_node *rnode, *root; - void *val; - int level; - int slot; - level = vm_radix_height(rtree, &root); - KASSERT(index <= VM_RADIX_MAX(level), - ("vm_radix_remove: %p index out of range %jd.", rtree, - VM_RADIX_MAX(level))); - rnode = root; - val = NULL; - level--; - /* - * Find the node and record the path in stack. - */ - while (level && rnode) { - stack[level] = rnode; - slot = vm_radix_slot(index, level); - CTR6(KTR_VM, - "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", - rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - rnode = rnode->rn_child[slot]; - level--; - } - KASSERT(rnode != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - slot = vm_radix_slot(index, 0); - val = vm_radix_match(rnode->rn_child[slot], color); - KASSERT(val != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - - for (;;) { - CTR6(KTR_VM, -"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, - "remove: resetting tree %p, rnode %p, child %p, count %u", - rtree, rnode, - (rnode != NULL) ? rnode->rn_child[slot] : NULL, - (rnode != NULL) ? rnode->rn_count : 0); - rnode->rn_child[slot] = NULL; - /* - * Use atomics for the last level since red and black - * will both adjust it. - * Use a write memory barrier here in order to avoid - * rn_count reaching 0 before to fetch the actual pointer. - * Concurrent black removal, infact, may want to reclaim - * the radix node itself before to read it. - */ - if (level == 0) - atomic_add_rel_32(&rnode->rn_count, -1); - else - rnode->rn_count--; - /* - * Only allow black removes to prune the tree. - */ - if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) - break; - vm_radix_node_put(rnode); - if (rnode == root) { - vm_radix_setroot(rtree, NULL, 0); - break; - } - rnode = stack[++level]; - slot = vm_radix_slot(index, level); - - } - return (val); + vm_radix_sweep(rtree, index, color, 1); } /* Index: /var/tmp/vmcontention/sys/vm/vm_radix.h =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.h (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.h (working copy) @@ -63,7 +63,7 @@ int vm_radix_lookupn(struct vm_radix *, vm_pindex_ void **, int, vm_pindex_t *); void *vm_radix_lookup_le(struct vm_radix *, vm_pindex_t, int); void vm_radix_reclaim_allnodes(struct vm_radix *); -void *vm_radix_remove(struct vm_radix *, vm_pindex_t, int); +void vm_radix_remove(struct vm_radix *, vm_pindex_t, int); /* * Look up any entry at a position greater or equal to index. -- Test scenario: altbufferflushes.sh