GENERIC HEAD from 2012-05-12 19:54:57 UTC, r235728M, vmcore.357 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r235728M: Mon May 21 20:47:53 CEST 2012 pho@x4.osted.lan:/var/tmp/vmcontention/sys/i386/compile/PHO_KTR i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.01-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) lock order reversal: 1st 0xc1bb505c system map (system map) @ vm/vm_map.c:3723 2nd 0xc130d000 vm object (kernel object) @ vm/vm_fault.c:299 3rd 0xc1bb50e8 system map (system map) @ vm/vm_map.c:1454 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,34313a63,a3435,c1000a39,c0dbde10,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda37d,c12d9048,5ae,c18207c4,...) at kdb_backtrace+0x2a _witness_debugger(c0fda37d,c1bb50e8,c100ddea,c9582110,c100e468,...) at _witness_debugger+0x25 witness_checkorder(c1bb50e8,9,c100e45f,5ae,0,...) at witness_checkorder+0x86f _mtx_lock_flags(c1bb50e8,0,c100e45f,5ae,c596e000,...) at _mtx_lock_flags+0xd9 _vm_map_lock(c1bb508c,c100e45f,5ae,0,a,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,c1820878,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd08a7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,c18208df,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d424,871,c182092c,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,11,c596ff44,c182098c,c0d00311,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,c1bb7600,101,827,1,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,c596f4e0,8,...) at uma_zalloc_arg+0x591 vm_radix_insert(c130d030,23480,0,c21fbf80,0,...) at vm_radix_insert+0x1d8 vm_page_insert(c21fbf80,c130d000,23480,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(c130d000,23480,0,40,c1820bbc,...) at vm_page_alloc+0x51c vm_fault_hold(c1bb5000,e337f000,0,1,0,...) at vm_fault_hold+0x60a vm_fault(c1bb5000,e337f000,0,1,c1bb3a68,...) at vm_fault+0x8b vm_fault_wire(c1bb5000,e3000000,e33c9000,0,0,...) at vm_fault_wire+0x3f vm_map_wire(c1bb5000,e3000000,e33c9000,0,e3000000,...) at vm_map_wire+0x2bc kmem_alloc(c1bb5000,3c9000,c955c770,0,0,...) at kmem_alloc+0x10d vm_ksubmap_init(c131e94c,0,1,1000,0,...) at vm_ksubmap_init+0x64 cpu_startup(0,181ec00,181ec00,181e000,1825000,...) at cpu_startup+0x24f mi_startup() at mi_startup+0xac begin() at begin+0x2c avail memory = 3538612224 (3374 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... WARNING: / was not properly dismounted Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: 15262 files, 392020 used, 620995 free (1363 frags, 77454 blocks, 0.1% fragmentation) /dev/ad4s1f: 20903 files, 889839 used, 123176 free (5304 frags, 14734 blocks, 0.5% fragmentation) /dev/label/tmp: 5930 files, 37539079 used, 80774902 free (2574 frags, 10096541 blocks, 0.0% fragmentation) /dev/ad4s1d: 705719 files, 6834675 used, 3319524 free (229660 frags, 386233 blocks, 2.3% fragmentation) /dev/ad4s1e: INCORRECT BLOCK COUNT I=6 (60864 should be 61024) (CORRECTED) /dev/ad4s1e: INCORRECT BLOCK COUNT I=6194267 (8392736 should be 8327552) (CORRECTED) /dev/ad4s1e: LINK COUNT FILE I=1742872 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=2048 MTIME=May 21 20:44 2012 COUNT 2 SHOULD BE 1 (ADJUSTED) /dev/ad4s1e: LINK COUNT FILE I=1742893 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=2048 MTIME=May 21 20:11 2012 COUNT 2 SHOULD BE 1 (ADJUSTED) /dev/ad4s1e: UNREF FILE I=1742936 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=0 MTIME=May 21 21:22 2012 (CLEARED) /dev/ad4s1e: LINK COUNT FILE I=1742937 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=2048 MTIME=May 21 20:22 2012 COUNT 2 SHOULD BE 1 (ADJUSTED) /dev/ad4s1e: LINK COUNT FILE I=1742944 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=2048 MTIME=May 21 20:33 2012 COUNT 2 SHOULD BE 1 (ADJUSTED) /dev/ad4s1e: LINK COUNT FILE I=1742962 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=2048 MTIME=May 21 21:00 2012 COUNT 2 SHOULD BE 1 (ADJUSTED) /dev/ad4s1e: LINK COUNT FILE I=1742981 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=2048 MTIME=May 21 20:55 2012 COUNT 2 SHOULD BE 1 (ADJUSTED) /dev/ad4s1e: LINK COUNT FILE I=1742982 OWNER=operator MODE=100400 /dev/ad4s1e: SIZE=2048 MTIME=May 21 20:00 2012 COUNT 2 SHOULD BE 1 (ADJUSTED) /dev/ad4s1e: UNREF FILE I=6194263 OWNER=root MODE=100644 /dev/ad4s1e: SIZE=0 MTIME=May 21 21:22 2012 (CLEARED) /dev/ad4s1e: UNREF FILE I=6194264 OWNER=root MODE=100644 /dev/ad4s1e: SIZE=0 MTIME=May 21 21:22 2012 (CLEARED) /dev/ad4s1e: UNREF FILE I=6194265 OWNER=root MODE=100644 /dev/ad4s1e: SIZE=0 MTIME=May 21 21:22 2012 (CLEARED) /dev/ad4s1e: UNREF FILE I=6194266 OWNER=root MODE=100644 /dev/ad4s1e: SIZE=0 MTIME=May 21 21:22 2012 (CLEARED) /dev/ad4s1e: UNREF FILE I=6194268 OWNER=root MODE=100644 /dev/ad4s1e: SIZE=0 MTIME=May 21 21:23 2012 (CLEARED) /dev/ad4s1e: FREE BLK COUNT(S) WRONG IN SUPERBLK (SALVAGED) /dev/ad4s1e: SUMMARY INFORMATION BAD (SALVAGED) /dev/ad4s1e: BLK(S) MISSING IN BIT MAPS (SALVAGED) /dev/ad4s1e: 799255 files, 15292677 used, 10095840 free (81072 frags, 1251846 blocks, 0.3% fragmentation) Mounting local file systems:. Setting hostname: x4.osted.lan. re0: link state changed to DOWN re0: link state changed to UP Starting Network: lo0 re0 fwe0 fwip0. lo0: flags=8049 metric 0 mtu 16384 options=3 inet6 ::1 prefixlen 128 inet6 fe80::1%lo0 prefixlen 64 scopeid 0xa inet 127.0.0.1 netmask 0xff000000 nd6 options=21 re0: flags=8843 metric 0 mtu 1500 options=8209b ether 00:21:85:15:33:7d inet 192.168.1.101 netmask 0xffffff00 broadcast 192.168.1.255 inet6 fe80::221:85ff:fe15:337d%re0 prefixlen 64 scopeid 0x1 nd6 options=29 media: Ethernet autoselect (100baseTX ) status: active fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 Starting devd. Starting Network: fwe0. fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 Starting Network: fwip0. fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/kde4/lib /usr/local/lib/compat /usr/local/lib/gcc46 /usr/local/lib/mysql /usr/local/lib/nss /usr/local/lib/qt4 /usr/local/lib/virtualbox a.out ldconfig path: /usr/lib/aout /usr/lib/compat/aout Creating and/or trimming log files. Starting syslogd. lock order reversal: 1st 0xe3106da0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcc16061c snaplk (snaplk) @ ufs/ffs/ffs_snapshot.c:2297 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,632e746f,3932323a,ff000a37,0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda364,c12d77d0,8f9,f3ff9990,...) at kdb_backtrace+0x2a _witness_debugger(c0fda364,cc16061c,c1007934,c958aa60,c10078be,...) at _witness_debugger+0x25 witness_checkorder(cc16061c,9,c10078b5,8f9,0,...) at witness_checkorder+0x86f __lockmgr_args(cc16061c,80900,cc1474d8,0,0,...) at __lockmgr_args+0x8c5 ffs_copyonwrite(cc147414,e3106d40,e3106d40,0,0,...) at ffs_copyonwrite+0x2c2 ffs_geom_strategy(cc14750c,e3106d40,cc596910,e3106d40,170f788,...) at ffs_geom_strategy+0xeb bufwrite(e3106d40,0,c100ad50,811,0) at bufwrite+0x159 ffs_bufwrite(e3106d40,cc3bfe00,100,4000,0,...) at ffs_bufwrite+0x290 ffs_update(cc398984,1,c100b4d1,151,0,...) at ffs_update+0x3a5 ffs_syncvnode(cc398984,1,0,c1137880,cc398a7c,...) at ffs_syncvnode+0x591 ffs_fsync(f3ff9c40,c102c9c4,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c1137880,f3ff9c40,c0fe3a87,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cbee82e0,f3ff9cec,c1025d5c,c0fdc06f,c1027310,...) at sys_fsync+0x1df syscall(f3ff9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x2818ad87, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- savecore: reboot after panic: vm_radix_node_put: Freeing a node with 16 children May 21 21:37:18 x4 savecore: reboot after panic: vm_radix_node_put: Freeing a node with 16 chilock order reversal: ldren 1st 0xe31181e0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcad2b600 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:284 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,2e687361,38323a63,c1000a34,cc05d670,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda364,c12d7998,11c,f40097c8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda364,cad2b600,c100bd7a,c958a990,c100ba0c,...) at _witness_debugger+0x25 witness_checkorder(cad2b600,9,c100ba03,11c,0,...) at witness_checkorder+0x86f _sx_xlock(cad2b600,0,c100ba03,11c,cc596570,...) at _sx_xlock+0x9a ufsdirhash_acquire(e3118180,cc596570,2c,e3b00948,f4009898,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(cc596570,f4009904,948,f4009884,f4009888,...) at ufsdirhash_add+0x13 ufs_direnter(cc599414,cc581c3c,f4009904,f4009bb8,0,...) at ufs_direnter+0x799 ufs_makeinode(f4009bb8,c1137e40,f4009afc,f4009a58,c0e28ae0,...) at ufs_makeinode+0x65d ufs_create(f4009afc,c102d50e,c1149c60,cc599414,f4009b14,...) at ufs_create+0x30 VOP_CREATE_APV(c1137880,f4009afc,f4009bb8,f4009a94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f4009b78,f4009c40,180,0,c95d9b80,...) at vn_open_cred+0x205 vn_open(f4009b78,f4009c40,180,cc06ecb0,c1baa0f0,...) at vn_open+0x3b kern_openat(cc05d5c0,ffffff9c,28807000,0,602,...) at kern_openat+0x1ec kern_open(cc05d5c0,28807000,0,601,180,...) at kern_open+0x35 sys_open(cc05d5c0,f4009cec,c1025d5c,c0fbcc91,c1027310,...) at sys_open+0x30 syscall(f4009d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x28182913, esp = 0xbfbfe28c, ebp = 0xbfbfeda8 --- savecore: writing core to vmcore.356 Writing crash summary to /var/crash/core.txt.356. Additional ABI support: linux. Starting rpcbind. NFS access cache time=60 Clearing /tmp (X related). Starting mountd. Starting nfsuserd. Starting nfsd. Recovering vi editor sessions:. Updating motd:. Starting ntpd. Starting default moused. Configuring syscons: keymap blanktime. Starting sshd. Starting cron. Local package initialization: backuplock order reversal: 1st 0xcc15d058 ufs (ufs) @ kern/vfs_subr.c:2159 2nd 0xe30eaec0 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:260 3rd 0xcc59c310 ufs (ufs) @ kern/vfs_subr.c:2159 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,3a632e73,a303632,0,c9582c70,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda37d,c12d6ef0,86f,f3ffd83c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda37d,cc59c310,c0fc2844,c958a928,c0fe23ba,...) at _witness_debugger+0x25 witness_checkorder(cc59c310,9,c0fe23b1,86f,0,...) at witness_checkorder+0x86f __lockmgr_args(cc59c310,80100,cc59c37c,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f3ffd944,c0ad499c,0,4,cc147a7c,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c1137880,f3ffd944,cc0c1950,c11499e0,cc59c2b8,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc59c2b8,80100,c0fe23b1,86f,eb,...) at _vn_lock+0x78 vget(cc59c2b8,80100,cc0c18a0,50,0,...) at vget+0xbb vfs_hash_get(cbf90b70,c66000,80000,cc0c18a0,f3ffda90,...) at vfs_hash_get+0xed ffs_vgetf(cbf90b70,c66000,80000,f3ffda90,1,...) at ffs_vgetf+0x49 softdep_sync_buf(cc15d000,e30eae60,1,106,0,...) at softdep_sync_buf+0x4a3 ffs_syncvnode(cc15d000,1,0,5fe,cc15d0c4,...) at ffs_syncvnode+0x24c ffs_sync(cbf90b70,1,f3ffdbb8,51f,c1bae710,...) at ffs_sync+0x379 dounmount(cbf90b70,8080000,cc0c18a0,4a1,f3ffdc3c,...) at dounmount+0x3f8 sys_unmount(cc0c18a0,f3ffdcec,c1025d5c,c0fdb032,c119b640,...) at sys_unmount+0x421 syscall(f3ffdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280bef0b, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- lock order reversal: 1st 0xcc0b2880 ufs (ufs) @ kern/vfs_mount.c:1254 2nd 0xcc1479dc devfs (devfs) @ ufs/ffs/ffs_softdep.c:1834 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,38313a63,a3433,cc59db38,8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda364,c12d6ed0,72a,f3ffd9a8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda364,cc1479dc,c0fcd760,c958a788,c1007e75,...) at _witness_debugger+0x25 witness_checkorder(cc1479dc,9,c1007e6c,72a,cc147a48,...) at witness_checkorder+0x86f __lockmgr_args(cc1479dc,80400,cc147a48,0,0,...) at __lockmgr_args+0x8c5 vop_stdlock(f3ffdaac,c130c880,4,c0fd08a7,eb,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c10f8040,f3ffdaac,c130c880,c11499e0,cc147984,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc147984,80400,c1007e6c,72a,0,...) at _vn_lock+0x78 softdep_flushworklist(cbf90b70,f3ffdb64,cc0c18a0,612,cc15d0c4,...) at softdep_flushworklist+0x47 ffs_sync(cbf90b70,1,f3ffdbb8,51f,c1bae710,...) at ffs_sync+0x3bf dounmount(cbf90b70,8080000,cc0c18a0,4a1,f3ffdc3c,...) at dounmount+0x3f8 sys_unmount(cc0c18a0,f3ffdcec,c1025d5c,c0fdb032,c119b640,...) at sys_unmount+0x421 syscall(f3ffdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280bef0b, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5930 files, 37539079 used, 80774902 free (2574 frags, 10096541 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5930 files, 37539079 used, 80774902 free (2574 frags, 10096541 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Starting inetd. Mon May 21 21:41:36 CEST 2012 FreeBSD/i386 (x4.osted.lan) (console) login: Expensive timeout(9) function: 0xc08cddb0(0xc132dd60) 0.013998541 s May 21 21:45:14 x4 su: pho to root on /dev/pts/0 May 21 22:06:03 x4 su: pho to root on /dev/pts/2 lock order reversal: 1st 0xcc16061c snaplk (snaplk) @ ufs/ufs/ufs_vnops.c:966 2nd 0xcc15c9dc ufs (ufs) @ ufs/ffs/ffs_snapshot.c:1627 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,a373236,a00,3,f41ce680,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda364,c12d65e0,65b,f41ce718,...) at kdb_backtrace+0x2a _witness_debugger(c0fda364,cc15c9dc,c0fc2844,c958a928,c10078be,...) at _witness_debugger+0x25 witness_checkorder(cc15c9dc,9,c10078b5,65b,0,...) at witness_checkorder+0x86f __lockmgr_args(cc15c9dc,80000,0,0,0,...) at __lockmgr_args+0x8c5 ffs_snapremove(cc15c984,c10077f3,f41ce848,c11aa024,c119b640,...) at ffs_snapremove+0x11f ffs_truncate(cc15c984,0,0,c00,0,...) at ffs_truncate+0x647 ufs_inactive(f41cead4,c102cedd,eb,c1137880,f41ceaa8,...) at ufs_inactive+0x1f6 VOP_INACTIVE_APV(c1137880,f41cead4,c0fe23b1,9bf,f41ceb08,...) at VOP_INACTIVE_APV+0x100 vinactive(cc15c984,ce1702e0,c0fe23b1,8be,0,...) at vinactive+0x127 vputx(f41cec34,c0b2eab3,cc15c984,ffffffdf,cc15c984,...) at vputx+0x370 vput(cc15c984,ffffffdf,cc15c984,f41ceb8c,1be38,...) at vput+0x10 kern_unlinkat(ce1702e0,ffffff9c,bfbfec02,0,0,...) at kern_unlinkat+0x2a3 kern_unlink(ce1702e0,bfbfec02,0,f41ced1c,c0e008ee,...) at kern_unlink+0x2f sys_unlink(ce1702e0,f41cecec,c1025d5c,c0fdb083,c1027310,...) at sys_unlink+0x22 syscall(f41ced28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (10, FreeBSD ELF32, sys_unlink), eip = 0x2816356f, esp = 0xbfbfe97c, ebp = 0xbfbfe9f8 --- ~KDB: enter: Break to debugger [ thread pid 4182 tid 100175 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> lock order reversal: 1st 0xc1bb7388 128 Bucket (UMA zone) @ vm/uma_core.c:2770 2nd 0xcad30474 firewire (firewire) @ dev/firewire/fwohci.c:2118 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,20,f00,f,c132ddd0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda364,c12d7008,846,f418f4e0,...) at kdb_backtrace+0x2a _witness_debugger(c0fda364,cad30474,c0f6151a,c95885d0,c0f60c02,...) at _witness_debugger+0x25 witness_checkorder(cad30474,9,c0f60bf9,846,0,...) at witness_checkorder+0x86f _mtx_lock_flags(cad30474,0,c0f60bf9,846,cac93720,...) at _mtx_lock_flags+0xd9 fwohci_poll(cad2c000,ffffffff,ffffffff,f418f568,c069068e,...) at fwohci_poll+0x35 dcons_crom_poll(cac72e00,0,c0f57768,b1,c115d300,...) at dcons_crom_poll+0x25 dcons_os_checkc(f418f580,c0a3ea1a,c1093a00,78,f418f588,...) at dcons_os_checkc+0xae dcons_cngetc(c1093a00,78,f418f588,c0a3ea56,f418f5a8,...) at dcons_cngetc+0xe cncheckc(f418f5a8,c051cdf5,c0e4697e,c051e0f0,f418f5a4,...) at cncheckc+0x3a cngetc(c0e4697e,c051e0f0,f418f5a4,f418f5e0,c1045680,...) at cngetc+0x16 db_readline(c1153040,78,f418f5c4,c051b8e6,c0e4697e,...) at db_readline+0x75 db_read_line(c0e4697e,f418f618,c051d8ed,c0e46ca9,c9775000,...) at db_read_line+0x1a db_command_loop(c0e46ca9,c9775000,0,f418f5f0,0,...) at db_command_loop+0x46 db_trap(3,0,1,46,2,...) at db_trap+0xdd kdb_trap(3,0,f418f72c,c133ada0,cde72864,...) at kdb_trap+0xb6 trap(f418f72c) at trap+0x79b calltrap() at calltrap+0x6 --- trap 0x3, eip = 0xc0abd9ba, esp = 0xf418f76c, ebp = 0xf418f774 --- kdb_enter(c0e46ca9,c0fd683e,f418f7bc,c08fb51d,c95a7200,...) at kdb_enter+0x3a kdb_break(c95a7200,c596e000,7,7,f418f7b4,...) at kdb_break+0x28 uart_intr(c95a7200,c0fda75e,f418f824,c0ad499c,0,...) at uart_intr+0x12d intr_event_handle(c95aab80,f418f828,c11ab01c,2,ccbc1fc8,...) at intr_event_handle+0x76 intr_execute_handlers(c96210d0,f418f828,2,f418f894,c0deaf15,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,f418f828) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0d00a23, esp = 0xf418f868, ebp = 0xf418f894 --- uma_dbg_free(c1bb6540,ccbc1fc8,ccbc1418,ad2,ccbc1418,...) at uma_dbg_free+0x93 zone_free_item(0,2) at zone_free_item+0x343 bucket_free(c1bb69a0,0,101,969,2,...) at bucket_free+0x2f uma_zalloc_arg(c1bb69a0,0,101,c0d1f53c,ce240128,...) at uma_zalloc_arg+0x773 vm_radix_insert(ce240158,1e,0,c3f89f40,c100f8e7,...) at vm_radix_insert+0xbd vm_page_insert(c3f89f40,ce240128,1e,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(ce240128,1e,0,0,f418fb64,...) at vm_page_alloc+0x51c vm_fault_hold(ce1c5ae0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(ce1c5ae0,bfbfe000,2,0,bfbfec38,...) at vm_fault+0x8b trap_pfault(c0dffdff,0,cde8e000,cde72864,cde72864,...) at trap_pfault+0x1b9 trap(f418fd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bcf06, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- db> run pho db:0:pho> bt Tracing pid 4182 tid 100175 td 0xcde8e000 kdb_enter(c0e46ca9,c0fd683e,f418f7bc,c08fb51d,c95a7200,...) at kdb_enter+0x3a kdb_break(c95a7200,c596e000,7,7,f418f7b4,...) at kdb_break+0x28 uart_intr(c95a7200,c0fda75e,f418f824,c0ad499c,0,...) at uart_intr+0x12d intr_event_handle(c95aab80,f418f828,c11ab01c,2,ccbc1fc8,...) at intr_event_handle+0x76 intr_execute_handlers(c96210d0,f418f828,2,f418f894,c0deaf15,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,f418f828) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0d00a23, esp = 0xf418f868, ebp = 0xf418f894 --- uma_dbg_free(c1bb6540,ccbc1fc8,ccbc1418,ad2,ccbc1418,...) at uma_dbg_free+0x93 zone_free_item(0,2) at zone_free_item+0x343 bucket_free(c1bb69a0,0,101,969,2,...) at bucket_free+0x2f uma_zalloc_arg(c1bb69a0,0,101,c0d1f53c,ce240128,...) at uma_zalloc_arg+0x773 vm_radix_insert(ce240158,1e,0,c3f89f40,c100f8e7,...) at vm_radix_insert+0xbd vm_page_insert(c3f89f40,ce240128,1e,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(ce240128,1e,0,0,f418fb64,...) at vm_page_alloc+0x51c vm_fault_hold(ce1c5ae0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(ce1c5ae0,bfbfe000,2,0,bfbfec38,...) at vm_fault+0x8b trap_pfault(c0dffdff,0,cde8e000,cde72864,cde72864,...) at trap_pfault+0x1b9 trap(f418fd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bcf06, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- db:0:bt> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x6d4a00 curthread = 0xcc01cb80: pid 1263 "watchdogd" curpcb = 0xf403ed80 fpcurthread = none idlethread = 0xc95df8a0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x828aa00 curthread = 0xcc0585c0: pid 4184 "sshd" curpcb = 0xf4183d80 fpcurthread = none idlethread = 0xc95df5c0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x828da00 curthread = 0xcde8e000: pid 4182 "watchdogd" curpcb = 0xf418fd80 fpcurthread = none idlethread = 0xc95df2e0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x8290a00 curthread = 0xcae245c0: pid 7 "pagedaemon" curpcb = 0xefbd7d80 fpcurthread = none idlethread = 0xc95df000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 4184 (sshd) thread 0xcc0585c0 (100171) exclusive sleep mutex vm object (standard object) r = 0 (0xce2d8afc) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc084a40) locked @ vm/vm_map.c:3723 Process 4183 (cron) thread 0xce1ae2e0 (100214) exclusive sleep mutex vm object (standard object) r = 0 (0xcedda9d4) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc764870) locked @ vm/vm_map.c:3723 Process 4182 (watchdogd) thread 0xcde8e000 (100175) exclusive sleep mutex 128 Bucket (UMA zone) r = 0 (0xc1bb7388) locked @ vm/uma_core.c:2770 exclusive sleep mutex RADIX NODE (UMA zone) r = 0 (0xc1bb7608) locked @ vm/uma_core.c:857 exclusive sleep mutex vm object (standard object) r = 0 (0xce240128) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xce1c5b28) locked @ vm/vm_map.c:3723 Process 4181 (sh) thread 0xcde8c2e0 (100184) exclusive sleep mutex vm object (standard object) r = 0 (0xcde5fd4c) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xc95e4de0) locked @ vm/vm_map.c:3723 Process 4145 (dd) thread 0xccb2b5c0 (100158) exclusive sleep mutex vm object (standard object) r = 0 (0xce23f5c8) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3276ea0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe30fc160) locked @ kern/vfs_bio.c:2652 exclusive lockmgr ufs (ufs) r = 0 (0xcdf61c94) locked @ kern/vfs_vnops.c:645 Process 1263 (watchdogd) thread 0xcc01cb80 (100107) exclusive sleep mutex vm object (standard object) r = 0 (0xcd6c5a68) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc7644d0) locked @ vm/vm_map.c:3723 Process 1218 (cron) thread 0xcc0192e0 (100130) exclusive sleep mutex page lock (page lock) r = 0 (0xc130d400) locked @ vm/vm_page.c:1278 exclusive sleep mutex vm object (standard object) r = 0 (0xce1f1784) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xc95e55b8) locked @ vm/vm_map.c:3723 Process 19 (softdepflush) thread 0xcbee88a0 (100078) exclusive lockmgr bufwait (bufwait) r = 0 (0xe3168a80) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe33c33a0) locked @ kern/vfs_bio.c:1905 Process 7 (pagedaemon) thread 0xcae245c0 (100072) exclusive sleep mutex UMA lock (UMA lock) r = 0 (0xc130cc6c) locked @ vm/uma_core.c:1596 db:0:alllocks> show lockedvnods Locked vnodes 0xcdf61c3c: tag ufs, type VREG usecount 1, writecount 1, refcount 7081 mountedhere 0 flags (VI(0x200)) v_object 0xce23f5c8 ref 0 pages 196984 lock type ufs: EXCL by thread 0xccb2b5c0 (pid 4145, dd, tid 100158) #0 0xc0a6c68d at __lockmgr_args+0xd6d #1 0xc0ce2b51 at ffs_lock+0xa1 #2 0xc0e276a3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37e98 at _vn_lock+0x78 #4 0xc0b38ce6 at vn_write+0x176 #5 0xc0ad6bf5 at dofilewrite+0x95 #6 0xc0ad7268 at kern_writev+0x68 #7 0xc0ad737f at sys_write+0x4f #8 0xc0e008ee at syscall+0x2fe #9 0xc0deabb1 at Xint0x80_syscall+0x21 ino 6194269, on dev ada0s1e db:0:lockedvnods> show mount 0xcbfa25b8 /dev/ad4s1a on / (ufs) 0xcbf91000 devfs on /dev (devfs) 0xcc15b000 /dev/ad4s1f on /home (ufs) 0xcbf90894 /dev/ad4s1d on /usr (ufs) 0xcbf905b8 /dev/ad4s1e on /var (ufs) 0xcbf902dc procfs on /proc (procfs) 0xcbfa22dc /dev/label/tmp on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 4184 1207 1207 0 RL CPU 1 sshd 4183 1218 1218 0 RL cron 4182 1263 1263 0 RL CPU 2 watchdogd 4181 3157 3153 1001 RL sh 4146 3317 3317 0 S+ piperd 0xce2d24b0 egrep 4145 3317 3317 0 R+ dd 3317 3298 3317 0 S+ wait 0xccc862cc sh 3298 3295 3298 0 S+ wait 0xce18f2cc bash 3295 3294 3295 0 S+ pause 0xcde71324 csh 3294 3288 3294 1001 S+ wait 0xcc0ba598 su 3288 3287 3288 1001 Ss+ wait 0xce155864 bash 3287 3285 3285 1001 S select 0xcc08c464 sshd 3285 1207 3285 0 Ss select 0xcc93e064 sshd 3158 3153 3153 1001 S piperd 0xcc0db640 awk 3157 3153 3153 1001 RL sh 3156 3155 3156 1001 Ss kqread 0xcc05ce00 tail 3155 3149 3149 1001 S select 0xcbfa4ce4 sshd 3154 3152 3154 1001 Ss+ select 0xcc08de24 top 3153 3151 3153 1001 Ss wait 0xcbedc598 sh 3152 3146 3146 1001 S select 0xcbfa49e4 sshd 3151 3145 3145 1001 S select 0xcac60864 sshd 3149 1207 3149 0 Ss select 0xcc04f8a4 sshd 3146 1207 3146 0 Ss select 0xcac864a4 sshd 3145 1207 3145 0 Ss select 0xcc04fa24 sshd 1313 1 1313 0 Ss+ ttyin 0xcae33870 getty 1312 1 1312 0 Ss+ ttyin 0xcae33a70 getty 1311 1 1311 0 Ss+ ttyin 0xc970d270 getty 1310 1 1310 0 Ss+ ttyin 0xc970d470 getty 1309 1 1309 0 Ss+ ttyin 0xc970d670 getty 1308 1 1308 0 Ss+ ttyin 0xc970d870 getty 1307 1 1307 0 Ss+ ttyin 0xc970da70 getty 1306 1 1306 0 Ss+ ttyin 0xc970dc70 getty 1305 1 1305 0 Ss+ ttyin 0xc970de70 getty 1288 1 1288 0 Ss select 0xcac86424 inetd 1263 1 1263 0 RLs CPU 0 watchdogd 1218 1 1218 0 RLs cron 1213 1 1213 25 Ss pause 0xccb108bc sendmail 1210 1 1210 0 Ss select 0xcc08d764 sendmail 1207 1 1207 0 DLs pfault 0xc130cf7c sshd 1175 1 1175 0 Ss select 0xcbfa4c64 moused 1151 1 1151 0 Ss select 0xcbfa4324 ntpd 1099 1093 1093 0 S (threaded) nfsd 100125 S rpcsvc 0xcaca8a90 nfsd: service 100124 S rpcsvc 0xcaca8ad0 nfsd: service 100123 S rpcsvc 0xcaca8b10 nfsd: service 100112 S rpcsvc 0xcc4b6190 nfsd: master 1093 1 1093 0 Ss select 0xcac867a4 nfsd 1091 1087 1087 0 S select 0xcac86b64 nfsuserd 1090 1087 1087 0 S select 0xcac864e4 nfsuserd 1089 1087 1087 0 S select 0xcac86ba4 nfsuserd 1088 1087 1087 0 S select 0xc955f364 nfsuserd 1087 1 1087 0 Ss pause 0xcc0bb058 nfsuserd 1071 1 1071 0 Ss select 0xcc08d5e4 mountd 1038 1 1038 0 Ss select 0xcac60924 rpcbind 968 1 968 0 Ss select 0xcc08c664 syslogd 845 1 845 0 Ss select 0xcc08d424 devd 19 0 0 0 RL [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc1304bb4 [syncer] 16 0 0 0 DL psleep 0xc1304848 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e76c [pagezero] 8 0 0 0 DL psleep 0xc130e348 [vmdaemon] 7 0 0 0 RL CPU 3 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151354 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc13080d8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 D - 0xcacfed04 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d7e4 [yarrow] 13 0 0 0 DL (threaded) [geom] 100016 D - 0xc118a8a8 [g_down] 100015 D - 0xc118a8a4 [g_up] 100014 D - 0xc118a89c [g_event] 12 0 0 0 WL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 CanRun [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 CanRun [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130c000 [audit] 0 0 0 0 DLs (threaded) [kernel] 100069 D - 0xc118d7e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118aa60 [swapper] db:0:ps> allt Tracing command sshd pid 4184 tid 100171 td 0xcc0585c0 cpustop_handler(1,f4183808,c0e00de3,c93dbc80,f4183758,...) at cpustop_handler+0x26 ipi_nmi_handler(c93dbc80,f4183758,c0a74363,c93dbc80,cde73598,...) at ipi_nmi_handler+0x35 trap(f4183814) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a74c60, esp = 0xf4183854, ebp = 0xf41838a8 --- _mtx_lock_sleep(c1bb7608,cc0585c0,0,c100d424,827,...) at _mtx_lock_sleep+0x1e0 _mtx_lock_flags(c1bb7608,0,c100d424,827,9f84695a,...) at _mtx_lock_flags+0x10f uma_zalloc_arg(c1bb69a0,0,101,c0d1f53c,ce2d8afc,...) at uma_zalloc_arg+0x1cb vm_radix_insert(ce2d8b2c,1d,0,c3276480,c100f8e7,...) at vm_radix_insert+0xbd vm_page_insert(c3276480,ce2d8afc,1d,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(ce2d8afc,1d,0,0,f4183b64,...) at vm_page_alloc+0x51c vm_fault_hold(cc0849f8,bfbfd000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc0849f8,bfbfd000,2,0,bfbfdedc,...) at vm_fault+0x8b trap_pfault(c0dffdff,0,cc0585c0,cde73598,cde73598,...) at trap_pfault+0x1b9 trap(f4183d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80547c4, esp = 0xbfbfdee0, ebp = 0xbfbfede8 --- Tracing command cron pid 4183 tid 100214 td 0xce1ae2e0 sched_switch(ce1ae2e0,0,602,19e,e9d2f365,...) at sched_switch+0x413 mi_switch(602,0,c0fd2f3f,cd,c,...) at mi_switch+0x1ff critical_exit(ce1ae2e0,0,3,cc01cb80,726,...) at critical_exit+0xb5 ipi_bitmap_handler(8,28,28,c11aab94,726,...) at ipi_bitmap_handler+0x82 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2f --- interrupt, eip = 0xc0a749e7, esp = 0xf42158b8, ebp = 0xf42158ec --- _mtx_lock_spin(c11aab94,ce1ae2e0,0,c0fd8c70,222,...) at _mtx_lock_spin+0x167 _mtx_lock_spin_flags(c11aab94,0,c0fd8c70,222,c130d200,...) at _mtx_lock_spin_flags+0x165 turnstile_trywait(c130d200,c9583b10,4,c12d6530,0,...) at turnstile_trywait+0x3f _mtx_lock_sleep(c130d200,ce1ae2e0,0,c100f8e7,479,...) at _mtx_lock_sleep+0x218 _mtx_lock_flags(c130d200,0,c100f8e7,479,1e,...) at _mtx_lock_flags+0x10f vm_page_alloc(cedda9d4,1e,0,0,f4215b64,...) at vm_page_alloc+0x99 vm_fault_hold(cc764828,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc764828,bfbfe000,2,0,bfbfec10,...) at vm_fault+0x8b trap_pfault(c0dffdff,0,ce1ae2e0,ce157864,ce157864,...) at trap_pfault+0x1b9 trap(f4215d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c0e7, esp = 0xbfbfec10, ebp = 0xbfbfec48 --- Tracing command watchdogd pid 4182 tid 100175 td 0xcde8e000 kdb_enter(c0e46ca9,c0fd683e,f418f7bc,c08fb51d,c95a7200,...) at kdb_enter+0x3a kdb_break(c95a7200,c596e000,7,7,f418f7b4,...) at kdb_break+0x28 uart_intr(c95a7200,c0fda75e,f418f824,c0ad499c,0,...) at uart_intr+0x12d intr_event_handle(c95aab80,f418f828,c11ab01c,2,ccbc1fc8,...) at intr_event_handle+0x76 intr_execute_handlers(c96210d0,f418f828,2,f418f894,c0deaf15,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,f418f828) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0d00a23, esp = 0xf418f868, ebp = 0xf418f894 --- uma_dbg_free(c1bb6540,ccbc1fc8,ccbc1418,ad2,ccbc1418,...) at uma_dbg_free+0x93 zone_free_item(0,2) at zone_free_item+0x343 bucket_free(c1bb69a0,0,101,969,2,...) at bucket_free+0x2f uma_zalloc_arg(c1bb69a0,0,101,c0d1f53c,ce240128,...) at uma_zalloc_arg+0x773 vm_radix_insert(ce240158,1e,0,c3f89f40,c100f8e7,...) at vm_radix_insert+0xbd vm_page_insert(c3f89f40,ce240128,1e,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(ce240128,1e,0,0,f418fb64,...) at vm_page_alloc+0x51c vm_fault_hold(ce1c5ae0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(ce1c5ae0,bfbfe000,2,0,bfbfec38,...) at vm_fault+0x8b trap_pfault(c0dffdff,0,cde8e000,cde72864,cde72864,...) at trap_pfault+0x1b9 trap(f418fd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bcf06, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command sh pid 4181 tid 100184 td 0xcde8c2e0 sched_switch(cde8c2e0,0,602,19e,e9d751f9,...) at sched_switch+0x413 mi_switch(602,0,c0fd2f3f,cd,cde8c2e0,...) at mi_switch+0x1ff critical_exit(c11aac6c,f41aa95c,c0a74363,c11aac6c,4,...) at critical_exit+0xb5 spinlock_exit(c11aac6c,4,c0fd08a7,119,0,c0fd8c70) at spinlock_exit+0x10 _mtx_unlock_spin_flags(c11aac6c,0,c0fd8c70,260,f41aa994,...) at _mtx_unlock_spin_flags+0x133 turnstile_chain_unlock(c130db00,0,c12d6e20,c130db00,c130db00,...) at turnstile_chain_unlock+0x39 _mtx_unlock_sleep(c130db00,0,c100f8e7,502,76a,...) at _mtx_unlock_sleep+0xb0 _mtx_unlock_flags(c130db00,0,c100f8e7,502,0,...) at _mtx_unlock_flags+0x157 vm_page_alloc(cde5fd4c,1e,0,0,f41aab64,...) at vm_page_alloc+0x5b3 vm_fault_hold(c95e4d98,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(c95e4d98,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(c0dffdff,0,cde8c2e0,ce1572cc,ce1572cc,...) at trap_pfault+0x1b9 trap(f41aad28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- Tracing command egrep pid 4146 tid 100215 td 0xce1ae000 sched_switch(ce1ae000,0,104,1a3,68fdd86a,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(ce1ae000,0,c0fd8280,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ce2d24b0,5c,c0fdac07,100,0,...) at sleepq_wait_sig+0x17 _sleep(ce2d24b0,ce2d2628,15c,c0fdac07,0,...) at _sleep+0x385 pipe_read(cc950a48,f4218c3c,cc04e700,0,ce1ae000,...) at pipe_read+0x427 dofileread(f4218c3c,ffffffff,ffffffff,0,f4218c18,...) at dofileread+0x9e kern_readv(ce1ae000,0,f4218c3c,f4218c5c,1,...) at kern_readv+0x68 sys_read(ce1ae000,f4218cec,c1025d5c,c0fdb0fa,c1027310,...) at sys_read+0x4f syscall(f4218d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281baf13, esp = 0xbfbfe80c, ebp = 0xbfbfe898 --- Tracing command dd pid 4145 tid 100158 td 0xccb2b5c0 sched_switch(ccb2b5c0,0,103,19e,e9d917d6,...) at sched_switch+0x413 mi_switch(103,0,c0fd8c70,2e0,7b0,...) at mi_switch+0x1ff turnstile_wait(cc0be600,cc0192e0,0,1c5,0,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d200,ccb2b5c0,0,c100f8e7,479,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d200,0,c100f8e7,479,30110,...) at _mtx_lock_flags+0x10f vm_page_alloc(ce23f5c8,30110,0,40222,40222,...) at vm_page_alloc+0x99 vm_page_grab(ce23f5c8,30110,0,412a2,c12d6df8,...) at vm_page_grab+0x152 allocbuf(e3276e40,4000,c0fdf807,b00,ffffffe0,...) at allocbuf+0x672 getblk(cdf61c3c,c044,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(cdf61c3c,30110000,0,4000,cc04e700,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f4142ba4,c102d443,cdf61c3c,cdf61c3c,f4142b5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c1137880,f4142ba4,cdf61c3c,285,0,...) at VOP_WRITE_APV+0x174 vn_write(ce2d70a8,f4142c3c,cc04e700,0,ccb2b5c0,...) at vn_write+0x2bc dofilewrite(f4142c3c,ffffffff,ffffffff,0,f4142c18,...) at dofilewrite+0x95 kern_writev(ccb2b5c0,4,f4142c3c,f4142c5c,1,...) at kern_writev+0x68 sys_write(ccb2b5c0,f4142cec,c1025d5c,c0e323ef,c119c9c0,...) at sys_write+0x4f syscall(f4142d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28177ef3, esp = 0xbfbfe9ac, ebp = 0xbfbfe9d8 --- Tracing command sh pid 3317 tid 100166 td 0xcd7252e0 sched_switch(cd7252e0,0,104,1a3,598c8eec,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cd7252e0,0,c0fd8280,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ccc862cc,6c,c0fdb1b8,100,0,...) at sleepq_wait_sig+0x17 _sleep(ccc862cc,ccc86354,16c,c0fdb1b8,0,...) at _sleep+0x385 kern_wait(cd7252e0,ffffffff,f415ac58,2,0,...) at kern_wait+0x597 sys_wait4(cd7252e0,f415acec,c1025d5c,c0fdb197,c1027310,...) at sys_wait4+0x3b syscall(f415ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x281507db, esp = 0xbfbfe76c, ebp = 0xbfbfe788 --- Tracing command bash pid 3298 tid 100210 td 0xce1aeb80 sched_switch(ce1aeb80,0,104,1a3,2de0910a,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(ce1aeb80,0,c0fd8280,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ce18f2cc,6c,c0fdb1b8,100,0,...) at sleepq_wait_sig+0x17 _sleep(ce18f2cc,ce18f354,16c,c0fdb1b8,0,...) at _sleep+0x385 kern_wait(ce1aeb80,ffffffff,f41f8c58,6,0,...) at kern_wait+0x597 sys_wait4(ce1aeb80,f41f8cec,c1025d5c,c0fdb0b7,c1027310,...) at sys_wait4+0x3b syscall(f41f8d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad7db, esp = 0xbfbfe78c, ebp = 0xbfbfe7a8 --- Tracing command csh pid 3295 tid 100182 td 0xcde8c8a0 sched_switch(cde8c8a0,0,104,1a3,1a041864,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cde8c8a0,0,c0fd8280,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cde71324,74,c0e58fce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cde71324,cde71354,174,c0e58fce,0,...) at _sleep+0x385 kern_sigsuspend(cde8c8a0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cde8c8a0,f41a4cec,c1025d5c,c0fdb0b7,c96eb2e0,...) at sys_sigsuspend+0x4d syscall(f41a4d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x2816fdcb, esp = 0xbfbfe07c, ebp = 0xbfbfe0e8 --- Tracing command su pid 3294 tid 100090 td 0xcc0c1b80 sched_switch(cc0c1b80,0,104,1a3,bb894e7a,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0c1b80,0,c0fd8280,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0ba598,6c,c0fdb1b8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0ba598,cc0ba620,16c,c0fdb1b8,0,...) at _sleep+0x385 kern_wait(cc0c1b80,cdf,f3ff1c58,2,0,...) at kern_wait+0x597 sys_wait4(cc0c1b80,f3ff1cec,c1025d5c,c0fdb9ce,9ac1a99f,...) at sys_wait4+0x3b syscall(f3ff1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2810b7db, esp = 0xbfbfe5cc, ebp = 0xbfbfe5e8 --- Tracing command bash pid 3288 tid 100192 td 0xce171000 sched_switch(ce171000,0,104,1a3,94e21480,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(ce171000,0,c0fd8280,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ce155864,6c,c0fdb1b8,100,0,...) at sleepq_wait_sig+0x17 _sleep(ce155864,ce1558ec,16c,c0fdb1b8,0,...) at _sleep+0x385 kern_wait(ce171000,ffffffff,f41c2c58,6,0,...) at kern_wait+0x597 sys_wait4(ce171000,f41c2cec,c1025d5c,c0fdb0b7,c1027310,...) at sys_wait4+0x3b syscall(f41c2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad7db, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command sshd pid 3287 tid 100117 td 0xcae26b80 sched_switch(cae26b80,0,104,1a3,4a71a019,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae26b80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc08c450,0,c0fd08a7,cae26b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08c464,0,f4081a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc08c464,cc08c450,c0fda9d8,6ae,cc07e118,...) at _cv_wait_sig+0x260 seltdwait(cc07e118,58,cc04ec80,cae26b80,cae26c30,...) at seltdwait+0xa2 kern_select(cae26b80,c,28c0c170,28c0c1b0,0,0,20,cc3a12cc,0) at kern_select+0x655 sys_select(cae26b80,f4081cec,c1025d5c,c0fdb305,f4081cb4,...) at sys_select+0x66 syscall(f4081d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae93, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 3285 tid 100160 td 0xccb2b000 sched_switch(ccb2b000,0,104,1a3,3a028777,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccb2b000,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc93e050,0,c0fd08a7,ccb2b000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc93e064,0,f4148ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc93e064,cc93e050,c0fda9d8,6ae,f4148b3c,...) at _cv_wait_sig+0x260 seltdwait(cbe1792c,c0fda9d8,5ab,ccb2b000,696,...) at seltdwait+0xa2 sys_poll(ccb2b000,f4148cec,c1025d5c,c0fbcc91,c1027310,...) at sys_poll+0x321 syscall(f4148d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723fb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command awk pid 3158 tid 100133 td 0xccb2d8a0 sched_switch(ccb2d8a0,0,104,1a3,394f183a,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(ccb2d8a0,0,c0fd8280,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0db640,5c,c0fdac07,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0db640,cc0db7b8,15c,c0fdac07,0,...) at _sleep+0x385 pipe_read(cc950b98,f40f5c3c,cc752100,0,ccb2d8a0,...) at pipe_read+0x427 dofileread(f40f5c3c,ffffffff,ffffffff,0,f40f5c18,...) at dofileread+0x9e kern_readv(ccb2d8a0,0,f40f5c3c,f40f5c5c,1,...) at kern_readv+0x68 sys_read(ccb2d8a0,f40f5cec,c1025d5c,c0e323ef,c1027310,...) at sys_read+0x4f syscall(f40f5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281acf13, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command sh pid 3157 tid 100131 td 0xcc019000 sched_switch(cc019000,0,104,1a3,e9d78c47,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cc019000,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cf7c,78,c100fbad,0,0,...) at sleepq_wait+0x63 _sleep(c130cf7c,c130d200,278,c100fbad,0,...) at _sleep+0x39c vm_waitpfault(d01d5d4c,1e,0,0,f40efb64,...) at vm_waitpfault+0x75 vm_fault_hold(cc084658,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc084658,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(ccb10000,f40efc64,c0acc488,cc019000,ccb10000,...) at trap_pfault+0x1b9 trap(f40efd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- Tracing command tail pid 3156 tid 100176 td 0xcde8db80 sched_switch(cde8db80,0,104,1a3,96700177,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,68,...) at mi_switch+0x1ff sleepq_switch(cde8db80,0,c0fd8280,1a7,68,...) at sleepq_switch+0x162 sleepq_catch_signals(3e9,c0ac8c30,cde8db80,2,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc05ce00,68,c0fcc442,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(cc05ce00,cc05ce00,168,c0fcc442,3e9,...) at _sleep+0x34f kern_kevent(cde8db80,4,0,1,f4192c3c,...) at kern_kevent+0x374 sys_kevent(cde8db80,f4192cec,c1025d5c,c0e323ef,f4192cb4,...) at sys_kevent+0x19e syscall(f4192d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x281563db, esp = 0xbfbfeb0c, ebp = 0xbfbfebc8 --- Tracing command sshd pid 3155 tid 100170 td 0xcc0588a0 sched_switch(cc0588a0,0,104,1a3,68969c74,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0588a0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cbfa4cd0,0,c0fd08a7,cc0588a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa4ce4,0,f4180a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa4ce4,cbfa4cd0,c0fda9d8,6ae,cc9509a0,...) at _cv_wait_sig+0x260 seltdwait(cc9509a0,58,cbfa3080,cc0588a0,cc058950,...) at seltdwait+0xa2 kern_select(cc0588a0,c,28c0c170,28c0c1b0,0,0,20,cde73864,0) at kern_select+0x655 sys_select(cc0588a0,f4180cec,c1025d5c,c0fdb305,c1027310,...) at sys_select+0x66 syscall(f4180d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae93, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command top pid 3154 tid 100181 td 0xcde8cb80 sched_switch(cde8cb80,0,104,1a3,9edcf574,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cde8cb80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f41a1a14,c0a747fa,cc08de10,0,cde8cb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc08de24,0,f41a1a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc08de24,cc08de10,3e9,6ae,cc07e578,...) at _cv_timedwait_sig+0x270 seltdwait(f41a1c08,f41a1c10,cc04e000,cde8cb80,31323148,...) at seltdwait+0x8a kern_select(cde8cb80,2,bfbfeb9c,0,0,f41a1c54,20,1,0) at kern_select+0x655 sys_select(cde8cb80,f41a1cec,c1025d5c,c0fa5922,f41a1cb4,...) at sys_select+0x66 syscall(f41a1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281e1e93, esp = 0xbfbfeb3c, ebp = 0xbfbfec98 --- Tracing command sh pid 3153 tid 100109 td 0xcc01c5c0 sched_switch(cc01c5c0,0,104,1a3,a71e3934,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc01c5c0,0,c0fd8280,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbedc598,6c,c0fdb1b8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cbedc598,cbedc620,16c,c0fdb1b8,0,...) at _sleep+0x385 kern_wait(cc01c5c0,ffffffff,f4047c58,2,0,...) at kern_wait+0x597 sys_wait4(cc01c5c0,f4047cec,c1025d5c,c0fdb197,c1027310,...) at sys_wait4+0x3b syscall(f4047d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x281507db, esp = 0xbfbfeb0c, ebp = 0xbfbfeb28 --- Tracing command sshd pid 3152 tid 100159 td 0xccb2b2e0 sched_switch(ccb2b2e0,0,104,1a3,9ee2df82,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccb2b2e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cbfa49d0,0,c0fd08a7,ccb2b2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa49e4,0,f4145a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa49e4,cbfa49d0,c0fda9d8,6ae,cc06ee38,...) at _cv_wait_sig+0x260 seltdwait(cc06ee38,58,cbf97400,ccb2b2e0,ccb2b390,...) at seltdwait+0xa2 kern_select(ccb2b2e0,a,28c0c170,28c0c1b0,0,0,20,ccc87864,0) at kern_select+0x655 sys_select(ccb2b2e0,f4145cec,c1025d5c,c0fdb305,f4145cb4,...) at sys_select+0x66 syscall(f4145d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae93, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 3151 tid 100156 td 0xccb2bb80 sched_switch(ccb2bb80,0,104,1a3,8c8b8667,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccb2bb80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac60850,0,c0fd08a7,ccb2bb80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac60864,0,f413ca50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac60864,cac60850,c0fda9d8,6ae,cc02dd90,...) at _cv_wait_sig+0x260 seltdwait(cc02dd90,58,cbf98880,ccb2bb80,ccb2bc30,...) at seltdwait+0xa2 kern_select(ccb2bb80,c,28c0c170,28c0c1b0,0,0,20,ccc8a2cc,0) at kern_select+0x655 sys_select(ccb2bb80,f413ccec,c1025d5c,c0fdb305,c1027310,...) at sys_select+0x66 syscall(f413cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae93, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 3149 tid 100167 td 0xcd725000 sched_switch(cd725000,0,104,1a3,a6c46a2f,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cd725000,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc04f890,0,c0fd08a7,cd725000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc04f8a4,0,f415dab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc04f8a4,cc04f890,c0fda9d8,6ae,f415db3c,...) at _cv_wait_sig+0x260 seltdwait(cd88d42c,c0fda9d8,5ab,cd725000,0,...) at seltdwait+0xa2 sys_poll(cd725000,f415dcec,c1025d5c,c0e323ef,c1027310,...) at sys_poll+0x321 syscall(f415dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723fb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 3146 tid 100121 td 0xcc3ab8a0 sched_switch(cc3ab8a0,0,104,1a3,9d0f943b,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3ab8a0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac86490,0,c0fd08a7,cc3ab8a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac864a4,0,f408dab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac864a4,cac86490,c0fda9d8,6ae,f408db3c,...) at _cv_wait_sig+0x260 seltdwait(cc0d2c2c,c0fda9d8,5ab,cc3ab8a0,0,...) at seltdwait+0xa2 sys_poll(cc3ab8a0,f408dcec,c1025d5c,c0e323ef,c119c340,...) at sys_poll+0x321 syscall(f408dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723fb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 3145 tid 100153 td 0xccb2c5c0 sched_switch(ccb2c5c0,0,104,1a3,9cc36f98,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccb2c5c0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc04fa10,0,c0fd08a7,ccb2c5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc04fa24,0,f4132ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc04fa24,cc04fa10,c0fda9d8,6ae,f4132b3c,...) at _cv_wait_sig+0x260 seltdwait(cc3cb12c,c0fda9d8,5ab,ccb2c5c0,0,...) at seltdwait+0xa2 sys_poll(ccb2c5c0,f4132cec,c1025d5c,c0e323ef,cbee82e0,...) at sys_poll+0x321 syscall(f4132d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723fb, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command getty pid 1313 tid 100145 td 0xccbb52e0 sched_switch(ccbb52e0,0,104,1a3,3b112bee,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb52e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,ccbb52e0,ccbb52e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33870,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33870,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33800,cae33870,f4119c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33800,f4119c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27100,f4119c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc07eb98,f4119c3c,c95d9b80,0,ccbb52e0,...) at devfs_read_f+0x7e dofileread(f4119c3c,ffffffff,ffffffff,0,f4119c18,...) at dofileread+0x9e kern_readv(ccbb52e0,0,f4119c3c,f4119c5c,1,...) at kern_readv+0x68 sys_read(ccbb52e0,f4119cec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f4119d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1312 tid 100079 td 0xcc01db80 sched_switch(cc01db80,0,104,1a3,3a9e09fa,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc01db80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,cc01db80,cc01db80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33a70,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33a70,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33a00,cae33a70,f3fc5c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33a00,f3fc5c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27000,f3fc5c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc07e498,f3fc5c3c,c95d9b80,0,cc01db80,...) at devfs_read_f+0x7e dofileread(f3fc5c3c,ffffffff,ffffffff,0,f3fc5c18,...) at dofileread+0x9e kern_readv(cc01db80,0,f3fc5c3c,f3fc5c5c,1,...) at kern_readv+0x68 sys_read(cc01db80,f3fc5cec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f3fc5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1311 tid 100152 td 0xccbb3b80 sched_switch(ccbb3b80,0,104,1a3,3e885f9f,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb3b80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,ccbb3b80,ccbb3b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d270,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d270,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d200,c970d270,f412ec3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d200,f412ec3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0ce00,f412ec3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02d3f0,f412ec3c,c95d9b80,0,ccbb3b80,...) at devfs_read_f+0x7e dofileread(f412ec3c,ffffffff,ffffffff,0,f412ec18,...) at dofileread+0x9e kern_readv(ccbb3b80,0,f412ec3c,f412ec5c,1,...) at kern_readv+0x68 sys_read(ccbb3b80,f412ecec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f412ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1310 tid 100151 td 0xccbb4000 sched_switch(ccbb4000,0,104,1a3,3ae6c854,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb4000,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,ccbb4000,ccbb4000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d470,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d470,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d400,c970d470,f412bc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d400,f412bc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cd00,f412bc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02e738,f412bc3c,c95d9b80,0,ccbb4000,...) at devfs_read_f+0x7e dofileread(f412bc3c,ffffffff,ffffffff,0,f412bc18,...) at dofileread+0x9e kern_readv(ccbb4000,0,f412bc3c,f412bc5c,1,...) at kern_readv+0x68 sys_read(ccbb4000,f412bcec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f412bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1309 tid 100150 td 0xccbb42e0 sched_switch(ccbb42e0,0,104,1a3,3a741e80,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb42e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,ccbb42e0,ccbb42e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d670,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d670,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d600,c970d670,f4128c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d600,f4128c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cc00,f4128c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc06e4d0,f4128c3c,c95d9b80,0,ccbb42e0,...) at devfs_read_f+0x7e dofileread(f4128c3c,ffffffff,ffffffff,0,f4128c18,...) at dofileread+0x9e kern_readv(ccbb42e0,0,f4128c3c,f4128c5c,1,...) at kern_readv+0x68 sys_read(ccbb42e0,f4128cec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f4128d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1308 tid 100149 td 0xccbb45c0 sched_switch(ccbb45c0,0,104,1a3,3ed2118f,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb45c0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,ccbb45c0,ccbb45c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d870,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d870,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d800,c970d870,f4125c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d800,f4125c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c900,f4125c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc07ed58,f4125c3c,c95d9b80,0,ccbb45c0,...) at devfs_read_f+0x7e dofileread(f4125c3c,ffffffff,ffffffff,0,f4125c18,...) at dofileread+0x9e kern_readv(ccbb45c0,0,f4125c3c,f4125c5c,1,...) at kern_readv+0x68 sys_read(ccbb45c0,f4125cec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f4125d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1307 tid 100148 td 0xccbb48a0 sched_switch(ccbb48a0,0,104,1a3,3b01334c,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb48a0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,ccbb48a0,ccbb48a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970da70,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970da70,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970da00,c970da70,f4122c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970da00,f4122c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c800,f4122c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc06e498,f4122c3c,c95d9b80,0,ccbb48a0,...) at devfs_read_f+0x7e dofileread(f4122c3c,ffffffff,ffffffff,0,f4122c18,...) at dofileread+0x9e kern_readv(ccbb48a0,0,f4122c3c,f4122c5c,1,...) at kern_readv+0x68 sys_read(ccbb48a0,f4122cec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f4122d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1306 tid 100147 td 0xccbb4b80 sched_switch(ccbb4b80,0,104,1a3,3acf0a4a,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb4b80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,ccbb4b80,ccbb4b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970dc70,0,c0fdd040,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970dc70,c118c710,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970dc00,c970dc70,f411fc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970dc00,f411fc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c700,f411fc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02e2a0,f411fc3c,c95d9b80,0,ccbb4b80,...) at devfs_read_f+0x7e dofileread(f411fc3c,ffffffff,ffffffff,0,f411fc18,...) at dofileread+0x9e kern_readv(ccbb4b80,0,f411fc3c,f411fc5c,1,...) at kern_readv+0x68 sys_read(ccbb4b80,f411fcec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f411fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1305 tid 100146 td 0xccbb5000 sched_switch(ccbb5000,0,104,1a3,bc1ff8bd,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccbb5000,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,c970de04,0,c0fd08a7,ccbb5000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970de70,0,f411cae8,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970de70,c970de04,c0fdca18,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970de00,c970de70,f411cc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970de00,f411cc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(c95c6100,f411cc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc02e968,f411cc3c,c95d9b80,0,ccbb5000,...) at devfs_read_f+0x7e dofileread(f411cc3c,ffffffff,ffffffff,0,f411cc18,...) at dofileread+0x9e kern_readv(ccbb5000,0,f411cc3c,f411cc5c,1,...) at kern_readv+0x68 sys_read(ccbb5000,f411ccec,c1025d5c,c0e371b2,c1027310,...) at sys_read+0x4f syscall(f411cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189f13, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command inetd pid 1288 tid 100132 td 0xccb2db80 sched_switch(ccb2db80,0,104,1a3,1bbcbe18,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccb2db80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac86410,0,c0fd08a7,ccb2db80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac86424,0,f40f2a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac86424,cac86410,c0fda9d8,6ae,cc950038,...) at _cv_wait_sig+0x260 seltdwait(cc950038,58,c95d9b80,ccb2db80,0,...) at seltdwait+0xa2 kern_select(ccb2db80,8,bfbfdf90,0,0,0,20,ccb0fb30,4) at kern_select+0x655 sys_select(ccb2db80,f40f2cec,c1025d5c,c0fdb204,c1027310,...) at sys_select+0x66 syscall(f40f2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2819be93, esp = 0xbfbfdf4c, ebp = 0xbfbfedd8 --- Tracing command watchdogd pid 1263 tid 100107 td 0xcc01cb80 cpustop_handler(0,f403e808,c0e00de3,c1825c80,f403e758,...) at cpustop_handler+0x26 ipi_nmi_handler(c1825c80,f403e758,c0a74363,c1825c80,cbedcb30,...) at ipi_nmi_handler+0x35 trap(f403e814) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a74c87, esp = 0xf403e854, ebp = 0xf403e8a8 --- _mtx_lock_sleep(c1bb7608,cc01cb80,0,c100d424,827,...) at _mtx_lock_sleep+0x207 _mtx_lock_flags(c1bb7608,0,c100d424,827,6067f644,...) at _mtx_lock_flags+0x10f uma_zalloc_arg(c1bb69a0,0,101,c0d1f53c,cd6c5a68,...) at uma_zalloc_arg+0x1cb vm_radix_insert(cd6c5a98,1e,0,c3f1dec0,c100f8e7,...) at vm_radix_insert+0xbd vm_page_insert(c3f1dec0,cd6c5a68,1e,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(cd6c5a68,1e,0,0,f403eb64,...) at vm_page_alloc+0x51c vm_fault_hold(cc764488,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc764488,bfbfe000,2,0,bfbfec3c,...) at vm_fault+0x8b trap_pfault(cbedcb30,f403ec64,c0acc488,cc01cb80,cbedcb30,...) at trap_pfault+0x1b9 trap(f403ed28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bce60, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command cron pid 1218 tid 100130 td 0xcc0192e0 sched_switch(cc0192e0,0,602,19e,e9d96bab,...) at sched_switch+0x413 mi_switch(602,0,c0fd2f3f,cd,cc0192e0,...) at mi_switch+0x1ff critical_exit(c11aab94,f40ec934,c0a74363,c11aab94,4,...) at critical_exit+0xb5 spinlock_exit(c11aab94,4,c0fd08a7,119,0,c0fd8c70) at spinlock_exit+0x10 _mtx_unlock_spin_flags(c11aab94,0,c0fd8c70,260,f40ec96c,...) at _mtx_unlock_spin_flags+0x133 turnstile_chain_unlock(c130d200,0,c12d7088,c130d200,c130d200,...) at turnstile_chain_unlock+0x39 _mtx_unlock_sleep(c130d200,0,c100f8e7,76a,0,...) at _mtx_unlock_sleep+0xb0 _mtx_unlock_flags(c130d200,0,c100f8e7,76a,c4f99dc0,...) at _mtx_unlock_flags+0x157 vm_page_free_toq(c4f99dc0,f40ec9e8,c0d17413,c4f99dc0,0,...) at vm_page_free_toq+0x26c vm_page_free(c4f99dc0,0,c100f8e7,4fe,0,...) at vm_page_free+0x15 vm_page_alloc(ce1f1784,1e,0,0,f40ecb64,...) at vm_page_alloc+0x583 vm_fault_hold(c95e5570,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(c95e5570,bfbfe000,2,0,bfbfec50,...) at vm_fault+0x8b trap_pfault(f40ecc58,c0a8e100,c119bcf0,cc0192e0,ccb102cc,...) at trap_pfault+0x1b9 trap(f40ecd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c20c, esp = 0xbfbfec50, ebp = 0xbfbfec68 --- Tracing command sendmail pid 1213 tid 100128 td 0xcc0198a0 sched_switch(cc0198a0,0,104,1a3,1fcd6b1b,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0198a0,0,c0fd8280,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(ccb108bc,74,c0e58fce,100,0,...) at sleepq_wait_sig+0x17 _sleep(ccb108bc,ccb108ec,174,c0e58fce,0,...) at _sleep+0x385 kern_sigsuspend(cc0198a0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0198a0,f40e6cec,c1025d5c,c0fdb0b7,f40e6c9c,...) at sys_sigsuspend+0x4d syscall(f40e6d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28322dcb, esp = 0xbfbfcf8c, ebp = 0xbfbfcfb8 --- Tracing command sendmail pid 1210 tid 100099 td 0xcc05d2e0 sched_switch(cc05d2e0,0,104,1a3,a83a62b5,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc05d2e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4015a14,c0a747fa,cc08d750,0,cc05d2e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc08d764,0,f4015a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc08d764,cc08d750,1389,6ae,cc07eaf0,...) at _cv_timedwait_sig+0x270 seltdwait(f4015c08,f4015c10,cc089a00,cc05d2e0,40,...) at seltdwait+0x8a kern_select(cc05d2e0,5,bfbfc500,0,0,f4015c54,20,5,0) at kern_select+0x655 sys_select(cc05d2e0,f4015cec,c1025d5c,c0fdb594,f4015cb4,...) at sys_select+0x66 syscall(f4015d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x283dae93, esp = 0xbfbfc46c, ebp = 0xbfbfcf98 --- Tracing command sshd pid 1207 tid 100126 td 0xcc01c000 sched_switch(cc01c000,0,104,1a3,e9d9402c,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cc01c000,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cf7c,78,c100fbad,0,0,...) at sleepq_wait+0x63 _sleep(c130cf7c,c130d200,278,c100fbad,0,...) at _sleep+0x39c vm_waitpfault(ce13f818,1d,0,0,f40e0b64,...) at vm_waitpfault+0x75 vm_fault_hold(cc072828,bfbfd000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc072828,bfbfd000,2,0,bfbfdee0,...) at vm_fault+0x8b trap_pfault(ccb11000,f40e0c64,c0acc488,cc01c000,ccb11000,...) at trap_pfault+0x1b9 trap(f40e0d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80544c8, esp = 0xbfbfdee0, ebp = 0xbfbfede8 --- Tracing command moused pid 1175 tid 100096 td 0xcc05d5c0 sched_switch(cc05d5c0,0,104,1a3,c9a79964,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc05d5c0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cbfa4c50,0,c0fd08a7,cc05d5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa4c64,0,f4009a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa4c64,cbfa4c50,c0fda9d8,6ae,cc08f620,...) at _cv_wait_sig+0x260 seltdwait(cc08f620,58,c95d9b80,cc05d5c0,f6,...) at seltdwait+0xa2 kern_select(cc05d5c0,400,bfbfe9f4,0,0,0,20,cc0532cc,1) at kern_select+0x655 sys_select(cc05d5c0,f4009cec,c1025d5c,c0e371b2,c1027310,...) at sys_select+0x66 syscall(f4009d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281a8e93, esp = 0xbfbfe98c, ebp = 0xbfbfeb18 --- Tracing command ntpd pid 1151 tid 100080 td 0xcc05f000 sched_switch(cc05f000,0,104,1a3,c73bde58,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc05f000,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cbfa4310,0,c0fd08a7,cc05f000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa4324,0,f3fc9a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa4324,cbfa4310,c0fda9d8,6ae,cc02e0e0,...) at _cv_wait_sig+0x260 seltdwait(cc02e0e0,58,c95d9b80,cc05f000,0,...) at seltdwait+0xa2 kern_select(cc05f000,1c,bfbfecd8,0,0,0,20,cc056b30,0) at kern_select+0x655 sys_select(cc05f000,f3fc9cec,c1025d5c,c0fdb9e0,e,...) at sys_select+0x66 syscall(f3fc9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28358e93, esp = 0xbfbfecac, ebp = 0xbfbfed78 --- Tracing command nfsd pid 1099 tid 100125 td 0xcc05fb80 sched_switch(cc05fb80,0,104,1a3,76df5d64,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc05fb80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40d1bdc,c0a747fa,cbc36e80,0,cc05fb80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca8a90,0,f40d1c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca8a90,cbc36e80,1388,3ec,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40d1d14,c0a56ed8,cbc36e80,f40d1d28,c0fcc890,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40d1d28,c0fcc890,3d8,cc3a22cc,...) at svc_thread_start+0x10 fork_exit(c0c84f70,cbc36e80,f40d1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1099 tid 100124 td 0xcc0bc000 sched_switch(cc0bc000,0,104,1a3,76e38351,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0bc000,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40cebdc,c0a747fa,cbc36e80,0,cc0bc000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca8ad0,0,f40cec18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca8ad0,cbc36e80,1388,3ec,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40ced14,c0a56ed8,cbc36e80,f40ced28,c0fcc890,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40ced28,c0fcc890,3d8,cc3a22cc,...) at svc_thread_start+0x10 fork_exit(c0c84f70,cbc36e80,f40ced28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1099 tid 100123 td 0xcc0bc2e0 sched_switch(cc0bc2e0,0,104,1a3,76fa7bcd,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0bc2e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40cbbdc,c0a747fa,cbc36e80,0,cc0bc2e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca8b10,0,f40cbc18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca8b10,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40cbd14,c0a56ed8,cbc36e80,f40cbd28,c0fcc890,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40cbd28,c0fcc890,3d8,cc3a22cc,...) at svc_thread_start+0x10 fork_exit(c0c84f70,cbc36e80,f40cbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1099 tid 100112 td 0xcbee6b80 sched_switch(cbee6b80,0,104,1a3,56b729b4,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee6b80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f406a48c,c0a747fa,cbc36e80,0,cbee6b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc4b6190,0,f406a4c8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc4b6190,cbc36e80,1388,3b1,f406a510,...) at _cv_timedwait_sig+0x270 svc_run_internal(cbee6d30,14,c1002aba,c0fbe19b,f406a808,...) at svc_run_internal+0x356 svc_run(cbc36e80,0,c0fbe176,19f,1,...) at svc_run+0x7b nfsrvd_nfsd(cbee6b80,f406a808,c,f7b,e45f,...) at nfsrvd_nfsd+0x1c3 nfssvc_nfsd(cbee6b80,f406acec,0,cbee6b80,f406ad1c,...) at nfssvc_nfsd+0x177 sys_nfssvc(cbee6b80,f406acec,c1025d5c,c0fdb594,c1027310,...) at sys_nfssvc+0xeb syscall(f406ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (155, FreeBSD ELF32, sys_nfssvc), eip = 0x280bfa0b, esp = 0xbfbfe81c, ebp = 0xbfbfea88 --- Tracing command nfsd pid 1093 tid 100115 td 0xcbee62e0 sched_switch(cbee62e0,0,104,1a3,7323c4e9,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee62e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cac86790,0,c0fd08a7,cbee62e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac867a4,0,f4079a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac867a4,cac86790,c0fda9d8,6ae,cc06e380,...) at _cv_wait_sig+0x260 seltdwait(cc06e380,58,c95d9b80,cbee62e0,0,...) at seltdwait+0xa2 kern_select(cbee62e0,7,bfbfeca4,0,0,0,20,cc3a1864,1) at kern_select+0x655 sys_select(cbee62e0,f4079cec,c1025d5c,c0fdb594,c1027310,...) at sys_select+0x66 syscall(f4079d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28176e93, esp = 0xbfbfea8c, ebp = 0xbfbfedd8 --- Tracing command nfsuserd pid 1091 tid 100119 td 0xcae265c0 sched_switch(cae265c0,0,104,1a3,590ca8e1,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae265c0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4087a14,c0a747fa,cac86b50,0,cae265c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac86b64,0,f4087a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac86b64,cac86b50,7531,6ae,cc06ee00,...) at _cv_timedwait_sig+0x270 seltdwait(f4087c08,f4087c10,c95d9b80,cae265c0,0,...) at seltdwait+0x8a kern_select(cae265c0,4,bfbfebb4,0,0,f4087c54,20,1e,0) at kern_select+0x655 sys_select(cae265c0,f4087cec,c1025d5c,c0fdb2ee,f4087cb4,...) at sys_select+0x66 syscall(f4087d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e93, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1090 tid 100097 td 0xcc0c12e0 sched_switch(cc0c12e0,0,104,1a3,590bed1f,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0c12e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f400da14,c0a747fa,cac864d0,0,cc0c12e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac864e4,0,f400da50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac864e4,cac864d0,7531,6ae,cc06ee00,...) at _cv_timedwait_sig+0x270 seltdwait(f400dc08,f400dc10,c95d9b80,cc0c12e0,0,...) at seltdwait+0x8a kern_select(cc0c12e0,4,bfbfebb4,0,0,f400dc54,20,1e,0) at kern_select+0x655 sys_select(cc0c12e0,f400dcec,c1025d5c,c0fdb2ee,f400dcb4,...) at sys_select+0x66 syscall(f400dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e93, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1089 tid 100091 td 0xcbee85c0 sched_switch(cbee85c0,0,104,1a3,590cd006,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee85c0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3ff5a14,c0a747fa,cac86b90,0,cbee85c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac86ba4,0,f3ff5a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac86ba4,cac86b90,7531,6ae,cc06ee00,...) at _cv_timedwait_sig+0x270 seltdwait(f3ff5c08,f3ff5c10,c95d9b80,cbee85c0,0,...) at seltdwait+0x8a kern_select(cbee85c0,4,bfbfebb4,0,0,f3ff5c54,20,1e,0) at kern_select+0x655 sys_select(cbee85c0,f3ff5cec,c1025d5c,c0fdb2ee,f3ff5cb4,...) at sys_select+0x66 syscall(f3ff5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e93, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1088 tid 100087 td 0xcc0c22e0 sched_switch(cc0c22e0,0,104,1a3,590d978d,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0c22e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fe5a14,c0a747fa,c955f350,0,cc0c22e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(c955f364,0,f3fe5a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c955f364,c955f350,7531,6ae,cc06ee00,...) at _cv_timedwait_sig+0x270 seltdwait(f3fe5c08,f3fe5c10,c95d9b80,cc0c22e0,0,...) at seltdwait+0x8a kern_select(cc0c22e0,4,bfbfebb4,0,0,f3fe5c54,20,1e,0) at kern_select+0x655 sys_select(cc0c22e0,f3fe5cec,c1025d5c,c0fdb2ee,f3fe5cb4,...) at sys_select+0x66 syscall(f3fe5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e93, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1087 tid 100086 td 0xcc0c25c0 sched_switch(cc0c25c0,0,104,1a3,688aa59a,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0c25c0,0,c0fd8280,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0bb058,74,c0e58fce,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0bb058,cc0bb088,174,c0e58fce,0,...) at _sleep+0x385 kern_sigsuspend(cc0c25c0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0c25c0,f3fe1cec,c1025d5c,c0fdb594,c1027310,...) at sys_sigsuspend+0x4d syscall(f3fe1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x280bddcb, esp = 0xbfbfec4c, ebp = 0xbfbfede8 --- Tracing command mountd pid 1071 tid 100102 td 0xcc0bcb80 sched_switch(cc0bcb80,0,104,1a3,d20f36ae,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0bcb80,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc08d5d0,0,c0fd08a7,cc0bcb80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08d5e4,0,f4022a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc08d5e4,cc08d5d0,c0fda9d8,6ae,cc02ed90,...) at _cv_wait_sig+0x260 seltdwait(cc02ed90,58,c95d9b80,cc0bcb80,f4022ac0,...) at seltdwait+0xa2 kern_select(cc0bcb80,9,bfbfecf4,0,0,0,20,cc0b7598,bfbfecf4) at kern_select+0x655 sys_select(cc0bcb80,f4022cec,c1025d5c,c0fbbef7,c1027310,...) at sys_select+0x66 syscall(f4022d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de93, esp = 0xbfbfec8c, ebp = 0xbfbfedd8 --- Tracing command rpcbind pid 1038 tid 100114 td 0xcbee65c0 sched_switch(cbee65c0,0,104,1a3,63ad5368,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee65c0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4075a78,c0a747fa,cac60910,0,cbee65c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac60924,0,f4075ab4,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac60924,cac60910,7531,6ae,f4075b6c,...) at _cv_timedwait_sig+0x270 seltdwait(f4075c3c,f4075c44,5ab,cbee65c0,c1108be4,...) at seltdwait+0x8a sys_poll(cbee65c0,f4075cec,c1025d5c,c0fdb2ee,c1027310,...) at sys_poll+0x321 syscall(f4075d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2812c3fb, esp = 0xbfbfcb9c, ebp = 0xbfbfed68 --- Tracing command syslogd pid 968 tid 100092 td 0xcbee82e0 sched_switch(cbee82e0,0,104,1a3,82c6a9ee,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee82e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc08c650,0,c0fd08a7,cbee82e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08c664,0,f3ff9a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc08c664,cc08c650,c0fda9d8,6ae,cc02d118,...) at _cv_wait_sig+0x260 seltdwait(cc02d118,58,c95d9b80,cbee82e0,0,...) at seltdwait+0xa2 kern_select(cbee82e0,9,28839100,0,0,0,20,cac0b864,0) at kern_select+0x655 sys_select(cbee82e0,f3ff9cec,c1025d5c,c0fdb1a7,e,...) at sys_select+0x66 syscall(f3ff9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de93, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- Tracing command devd pid 845 tid 100084 td 0xcc05e2e0 sched_switch(cc05e2e0,0,104,1a3,909d5ff4,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc05e2e0,0,c0fd8280,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a747fa,cc08d410,0,c0fd08a7,cc05e2e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08d424,0,f3fd9a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc08d424,cc08d410,c0fda9d8,6ae,cc02ebd0,...) at _cv_wait_sig+0x260 seltdwait(cc02ebd0,58,c95d9b80,cc05e2e0,c1108bc8,...) at seltdwait+0xa2 kern_select(cc05e2e0,5,bfbfe950,0,0,0,20,cc056000,35) at kern_select+0x655 sys_select(cc05e2e0,f3fd9cec,c1025d5c,c0e323ef,c119b640,...) at sys_select+0x66 syscall(f3fd9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x808940b, esp = 0xbfbfe91c, ebp = 0xbfbfedf8 --- Tracing command softdepflush pid 19 tid 100078 td 0xcbee88a0 sched_switch(cbee88a0,0,602,19e,e9d7d6d4,...) at sched_switch+0x413 mi_switch(602,0,c0fd2f3f,cd,8,...) at mi_switch+0x1ff critical_exit(cbee88a0,0,2,c130d200,c130d200,...) at critical_exit+0xb5 ipi_bitmap_handler(8,28,28,c130d200,c130d200,...) at ipi_bitmap_handler+0x82 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2f --- interrupt, eip = 0xc0a74c62, esp = 0xefbe989c, ebp = 0xefbe98f0 --- _mtx_lock_sleep(c130d200,cbee88a0,0,c100f8e7,65e,...) at _mtx_lock_sleep+0x1e2 _mtx_lock_flags(c130d200,0,c100f8e7,65e,1000,...) at _mtx_lock_flags+0x10f vm_wait(cad176f0,0,c100f8e7,8f5,40222,...) at vm_wait+0x2a vm_page_grab(cad176f0,982e54,0,412a2,c12d7ba8,...) at vm_page_grab+0x180 allocbuf(e3168a20,4000,c0fdf807,b00,efbe99fc,...) at allocbuf+0x672 getblk(cc147414,4c172a0,0,4000,0,...) at getblk+0x5e3 breadn_flags(cc147414,4c172a0,0,4000,0,...) at breadn_flags+0x43 indir_trunc(fffc7ff4,ffffffff,c1007e6c,1e0e,0,...) at indir_trunc+0x114 indir_trunc(fffff7f3,ffffffff,c1007e6c,1cd0,c12d7b88,...) at indir_trunc+0x4ba handle_workitem_freeblocks(0,efbe9c70,2,6ba,efbe9c74,...) at handle_workitem_freeblocks+0x24f process_worklist_item(c130c880,0,c1007e6c,5e9,cc160700,...) at process_worklist_item+0x2c2 softdep_process_worklist(cbf905b8,0,c1007e6c,562,3e8,...) at softdep_process_worklist+0x84 softdep_flush(0,efbe9d28,c0fcc890,3d8,cbedb000,...) at softdep_flush+0x2a0 fork_exit(c0cd7fd0,0,efbe9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe9d60, ebp = 0 --- Tracing command vnlru pid 18 tid 100077 td 0xcbee8b80 sched_switch(cbee8b80,0,104,1a3,e01e4edf,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cbee8b80,0,c0fd8280,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cbedb2cc,60,c0fe33d5,0,0,...) at sleepq_timedwait+0x6b _sleep(cbedb2cc,c1304b74,260,c0fe33d5,3e8,...) at _sleep+0x36a vnlru_proc(0,efbe6d28,c0fcc890,3d8,cbedb2cc,...) at vnlru_proc+0xe7 fork_exit(c0b2ba40,0,efbe6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe6d60, ebp = 0 --- Tracing command syncer pid 17 tid 100076 td 0xcacf98a0 sched_switch(cacf98a0,0,104,1a3,9f87242b,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf98a0,0,c0fd8280,28b,cacf98a0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304bb4,0,efbe3c78,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c1304bb4,c1304ba0,3e8,783,4e20,...) at _cv_timedwait+0x270 sched_sync(0,efbe3d28,c0fcc890,3d8,cbedb598,...) at sched_sync+0x595 fork_exit(c0b2ad70,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xcacf9b80 sched_switch(cacf9b80,0,104,1a3,9a978cb2,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cacf9b80,0,c0fd8280,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304848,54,c0fe014e,0,0,...) at sleepq_timedwait+0x6b _sleep(c1304848,c1304850,54,c0fe014e,3e8,...) at _sleep+0x36a buf_daemon(0,efbe0d28,c0fcc890,3d8,cbedb864,...) at buf_daemon+0x175 fork_exit(c0b0da90,0,efbe0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe0d60, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xcae24000 sched_switch(cae24000,0,104,1a3,e4f79e1d,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24000,0,c0fd8280,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e76c,0,c1011585,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e76c,c130d200,0,c1011585,493e0,...) at _sleep+0x36a vm_pagezero(0,efbddd28,c0fcc890,3d8,cbedbb30,...) at vm_pagezero+0xdc fork_exit(c0d1fd10,0,efbddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbddd60, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xcae242e0 sched_switch(cae242e0,0,104,1a3,b195b714,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cae242e0,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130e348,74,c0fe014e,0,0,...) at sleepq_wait+0x63 _sleep(c130e348,c130e34c,74,c0fe014e,0,...) at _sleep+0x39c vm_daemon(0,efbdad28,c0fcc890,3d8,cbedc000,...) at vm_daemon+0x59 fork_exit(c0d184d0,0,efbdad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbdad60, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xcae245c0 cpustop_handler(3,efbd7af4,c0e00de3,c93e1c80,efbd7a44,...) at cpustop_handler+0x26 ipi_nmi_handler(c93e1c80,efbd7a44,c0a74363,c93e1c80,c95de2cc,...) at ipi_nmi_handler+0x35 trap(efbd7b00) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a74c62, esp = 0xefbd7b40, ebp = 0xefbd7b94 --- _mtx_lock_sleep(c1bb7608,cae245c0,0,c100d424,300,...) at _mtx_lock_sleep+0x1e2 _mtx_lock_flags(c1bb7608,0,c100d424,300,0,...) at _mtx_lock_flags+0x10f zone_drain_wait(efbd7c10,c0cfb855,c1bb69a0,0,c100d424,...) at zone_drain_wait+0x2f zone_drain(c1bb69a0,0,c100d424,63c,0,...) at zone_drain+0x10 zone_foreach(c95b46c0,0,efbd7ce8,c0d19d5c,c95b46cc,...) at zone_foreach+0x45 uma_reclaim(c95b46cc,0,c1010367,2ee,1388,...) at uma_reclaim+0x16 vm_pageout(0,efbd7d28,c0fcc890,3d8,c95de2cc,...) at vm_pageout+0x80c fork_exit(c0d19550,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xcae248a0 sched_switch(cae248a0,0,104,1a3,29d26294,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae248a0,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1151354,5c,c0e2cbde,0,0,...) at sleepq_wait+0x63 _sleep(c1151354,c1151398,5c,c0e2cbde,0,...) at _sleep+0x39c xpt_scanner_thread(0,efa54d28,c0fcc890,3d8,c95de598,...) at xpt_scanner_thread+0x47 fork_exit(c04a72f0,0,efa54d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa54d60, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xcae24b80 sched_switch(cae24b80,0,104,1a3,36c64a48,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24b80,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c13080d8,0,c0ff789f,0,0,...) at sleepq_wait+0x63 _sleep(c13080d8,c13080a0,0,c0ff789f,0,...) at _sleep+0x39c sctp_iterator_thread(0,efa51d28,c0fcc890,3d8,c95de864,...) at sctp_iterator_thread+0x5c fork_exit(c0bc5c00,0,efa51d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa51d60, ebp = 0 --- Tracing command fdc0 pid 4 tid 100065 td 0xcae25b80 sched_switch(cae25b80,0,104,1a3,9deabae5,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae25b80,0,c0fd8280,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cac7223c,5c,c0fbec12,0,0,...) at sleepq_timedwait+0x6b _sleep(cac7223c,cac722f0,5c,c0fbec12,3e8,...) at _sleep+0x36a fdc_thread(cac72200,efa42d28,c0fcc890,3d8,c95deb30,...) at fdc_thread+0x27d fork_exit(c0da2050,cac72200,efa42d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa42d60, ebp = 0 --- Tracing command fw0_probe pid 3 tid 100063 td 0xcace42e0 sched_switch(cace42e0,0,104,1a3,29d2891d,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cace42e0,0,c0fd8280,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cad2c000,6c,c0fbec12,100,0,...) at sleepq_wait_sig+0x17 _sleep(cad2c000,cad30488,16c,c0fbec12,0,...) at _sleep+0x385 fw_bus_probe_thread(cad2c000,efa32d28,c0fcc890,3d8,cac0b000,...) at fw_bus_probe_thread+0xa08 fork_exit(c0701880,cad2c000,efa32d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa32d60, ebp = 0 --- Tracing command usb pid 15 tid 100059 td 0xcacf7000 sched_switch(cacf7000,0,104,1a3,172feef2,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf7000,0,c0fd8280,268,cacf7000,...) at sleepq_switch+0x162 sleepq_wait(cacfed34,0,efa1bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed34,cacfee4c,c0fb0291,74,cacfed3c,...) at _cv_wait+0x260 usb_process(cacfed2c,efa1bd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfed2c,efa1bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa1bd60, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xcacf72e0 sched_switch(cacf72e0,0,104,1a3,5bf0a5ac,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf72e0,0,c0fd8280,268,cacf72e0,...) at sleepq_switch+0x162 sleepq_wait(cacfed04,0,efa18cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed04,cacfee4c,c0fb0291,74,cacfed0c,...) at _cv_wait+0x260 usb_process(cacfecfc,efa18d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfecfc,efa18d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa18d60, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xcacf75c0 sched_switch(cacf75c0,0,104,1a3,16c8fff6,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf75c0,0,c0fd8280,268,cacf75c0,...) at sleepq_switch+0x162 sleepq_wait(cacfecd4,0,efa15cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfecd4,cacfee4c,c0fb0291,74,cacfecdc,...) at _cv_wait+0x260 usb_process(cacfeccc,efa15d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfeccc,efa15d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa15d60, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xcacf78a0 sched_switch(cacf78a0,0,104,1a3,16c8ec56,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf78a0,0,c0fd8280,268,cacf78a0,...) at sleepq_switch+0x162 sleepq_wait(cacfeca4,0,efa12cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfeca4,cacfee4c,c0fb0291,74,cacfecac,...) at _cv_wait+0x260 usb_process(cacfec9c,efa12d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacfec9c,efa12d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa12d60, ebp = 0 --- Tracing command usb pid 15 tid 100054 td 0xcacf9000 sched_switch(cacf9000,0,104,1a3,16c8cfa5,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf9000,0,c0fd8280,268,cacf9000,...) at sleepq_switch+0x162 sleepq_wait(cacefb5c,0,efa09cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb5c,cacefc74,c0fb0291,74,cacefb64,...) at _cv_wait+0x260 usb_process(cacefb54,efa09d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefb54,efa09d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa09d60, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xcacf92e0 sched_switch(cacf92e0,0,104,1a3,353501f9,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf92e0,0,c0fd8280,268,cacf92e0,...) at sleepq_switch+0x162 sleepq_wait(cacefb2c,0,efa06cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb2c,cacefc74,c0fb0291,74,cacefb34,...) at _cv_wait+0x260 usb_process(cacefb24,efa06d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefb24,efa06d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa06d60, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xcacf95c0 sched_switch(cacf95c0,0,104,1a3,1663a927,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf95c0,0,c0fd8280,268,cacf95c0,...) at sleepq_switch+0x162 sleepq_wait(cacefafc,0,efa03cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefafc,cacefc74,c0fb0291,74,cacefb04,...) at _cv_wait+0x260 usb_process(cacefaf4,efa03d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefaf4,efa03d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa03d60, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xcace2000 sched_switch(cace2000,0,104,1a3,1663939c,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2000,0,c0fd8280,268,cace2000,...) at sleepq_switch+0x162 sleepq_wait(cacefacc,0,efa00cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefacc,cacefc74,c0fb0291,74,cacefad4,...) at _cv_wait+0x260 usb_process(cacefac4,efa00d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacefac4,efa00d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa00d60, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xcace22e0 sched_switch(cace22e0,0,104,1a3,16637b69,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace22e0,0,c0fd8280,268,cace22e0,...) at sleepq_switch+0x162 sleepq_wait(caceab5c,0,ef9fccac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab5c,caceac74,c0fb0291,74,caceab64,...) at _cv_wait+0x260 usb_process(caceab54,ef9fcd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceab54,ef9fcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9fcd60, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xcace25c0 sched_switch(cace25c0,0,104,1a3,32181dd4,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace25c0,0,c0fd8280,268,cace25c0,...) at sleepq_switch+0x162 sleepq_wait(caceab2c,0,ef9f9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab2c,caceac74,c0fb0291,74,caceab34,...) at _cv_wait+0x260 usb_process(caceab24,ef9f9d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceab24,ef9f9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f9d60, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xcace28a0 sched_switch(cace28a0,0,104,1a3,1097a301,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace28a0,0,c0fd8280,268,cace28a0,...) at sleepq_switch+0x162 sleepq_wait(caceaafc,0,ef9f6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaafc,caceac74,c0fb0291,74,caceab04,...) at _cv_wait+0x260 usb_process(caceaaf4,ef9f6d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceaaf4,ef9f6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f6d60, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xcace2b80 sched_switch(cace2b80,0,104,1a3,10978ad9,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2b80,0,c0fd8280,268,cace2b80,...) at sleepq_switch+0x162 sleepq_wait(caceaacc,0,ef9f3cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaacc,caceac74,c0fb0291,74,caceaad4,...) at _cv_wait+0x260 usb_process(caceaac4,ef9f3d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,caceaac4,ef9f3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f3d60, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xcace3000 sched_switch(cace3000,0,104,1a3,10977339,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace3000,0,c0fd8280,268,cace3000,...) at sleepq_switch+0x162 sleepq_wait(cace5b5c,0,ef9efcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b5c,cace5c74,c0fb0291,74,cace5b64,...) at _cv_wait+0x260 usb_process(cace5b54,ef9efd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5b54,ef9efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9efd60, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xcace32e0 sched_switch(cace32e0,0,104,1a3,30e94b1b,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace32e0,0,c0fd8280,268,cace32e0,...) at sleepq_switch+0x162 sleepq_wait(cace5b2c,0,ef9eccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b2c,cace5c74,c0fb0291,74,cace5b34,...) at _cv_wait+0x260 usb_process(cace5b24,ef9ecd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5b24,ef9ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ecd60, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xcace35c0 sched_switch(cace35c0,0,104,1a3,10330f57,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace35c0,0,c0fd8280,268,cace35c0,...) at sleepq_switch+0x162 sleepq_wait(cace5afc,0,ef9e9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5afc,cace5c74,c0fb0291,74,cace5b04,...) at _cv_wait+0x260 usb_process(cace5af4,ef9e9d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5af4,ef9e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e9d60, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xcace38a0 sched_switch(cace38a0,0,104,1a3,1032fa90,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace38a0,0,c0fd8280,268,cace38a0,...) at sleepq_switch+0x162 sleepq_wait(cace5acc,0,ef9e6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5acc,cace5c74,c0fb0291,74,cace5ad4,...) at _cv_wait+0x260 usb_process(cace5ac4,ef9e6d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cace5ac4,ef9e6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e6d60, ebp = 0 --- Tracing command usb pid 15 tid 100041 td 0xcace4000 sched_switch(cace4000,0,104,1a3,1032e08d,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace4000,0,c0fd8280,268,cace4000,...) at sleepq_switch+0x162 sleepq_wait(cacddb5c,0,ef9dfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb5c,cacddc74,c0fb0291,74,cacddb64,...) at _cv_wait+0x260 usb_process(cacddb54,ef9dfd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddb54,ef9dfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dfd60, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xcacbf000 sched_switch(cacbf000,0,104,1a3,2fbb5fe8,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf000,0,c0fd8280,268,cacbf000,...) at sleepq_switch+0x162 sleepq_wait(cacddb2c,0,ef9dccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb2c,cacddc74,c0fb0291,74,cacddb34,...) at _cv_wait+0x260 usb_process(cacddb24,ef9dcd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddb24,ef9dcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dcd60, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xcacbf2e0 sched_switch(cacbf2e0,0,104,1a3,fce2fab,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf2e0,0,c0fd8280,268,cacbf2e0,...) at sleepq_switch+0x162 sleepq_wait(cacddafc,0,ef9d9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddafc,cacddc74,c0fb0291,74,cacddb04,...) at _cv_wait+0x260 usb_process(cacddaf4,ef9d9d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddaf4,ef9d9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d9d60, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xcacbf5c0 sched_switch(cacbf5c0,0,104,1a3,fce1838,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf5c0,0,c0fd8280,268,cacbf5c0,...) at sleepq_switch+0x162 sleepq_wait(cacddacc,0,ef9d6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddacc,cacddc74,c0fb0291,74,cacddad4,...) at _cv_wait+0x260 usb_process(cacddac4,ef9d6d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacddac4,ef9d6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d6d60, ebp = 0 --- Tracing command usb pid 15 tid 100036 td 0xcacbfb80 sched_switch(cacbfb80,0,104,1a3,fce0156,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbfb80,0,c0fd8280,268,cacbfb80,...) at sleepq_switch+0x162 sleepq_wait(cacd5b5c,0,ef9cfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b5c,cacd5c74,c0fb0291,74,cacd5b64,...) at _cv_wait+0x260 usb_process(cacd5b54,ef9cfd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5b54,ef9cfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9cfd60, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xcacc1000 sched_switch(cacc1000,0,104,1a3,2f4df1ae,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc1000,0,c0fd8280,268,cacc1000,...) at sleepq_switch+0x162 sleepq_wait(cacd5b2c,0,ef9cccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b2c,cacd5c74,c0fb0291,74,cacd5b34,...) at _cv_wait+0x260 usb_process(cacd5b24,ef9ccd28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5b24,ef9ccd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ccd60, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xcacc12e0 sched_switch(cacc12e0,0,104,1a3,f697f76,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc12e0,0,c0fd8280,268,cacc12e0,...) at sleepq_switch+0x162 sleepq_wait(cacd5afc,0,ef9c9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5afc,cacd5c74,c0fb0291,74,cacd5b04,...) at _cv_wait+0x260 usb_process(cacd5af4,ef9c9d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5af4,ef9c9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c9d60, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xcacc15c0 sched_switch(cacc15c0,0,104,1a3,f696819,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc15c0,0,c0fd8280,268,cacc15c0,...) at sleepq_switch+0x162 sleepq_wait(cacd5acc,0,ef9c6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5acc,cacd5c74,c0fb0291,74,cacd5ad4,...) at _cv_wait+0x260 usb_process(cacd5ac4,ef9c6d28,c0fcc890,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f860,cacd5ac4,ef9c6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c6d60, ebp = 0 --- Tracing command ctl_thrd pid 2 tid 100028 td 0xc96eb8a0 sched_switch(c96eb8a0,0,104,1a3,f694a7b,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb8a0,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c97c4000,5c,c0e37650,0,0,...) at sleepq_wait+0x63 _sleep(c97c4000,c97c4000,5c,c0e37650,0,...) at _sleep+0x39c ctl_work_thread(c97c4000,ef733d28,c0fcc890,3d8,cac0b598,...) at ctl_work_thread+0x22f6 fork_exit(c04d08d0,c97c4000,ef733d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef733d60, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xc96ea5c0 sched_switch(c96ea5c0,0,104,1a3,e4162f4a,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ea5c0,0,c0fd8280,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d7e4,0,c0fbec12,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d7e4,0,0,c0fbec12,64) at _sleep+0x36a pause(c0fbec12,64,c0f8c2f0,111,0,...) at pause+0xa0 random_kthread(0,e33e0d28,c0fcc890,3d8,c95dd000,...) at random_kthread+0x1ef fork_exit(c082d0c0,0,e33e0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e0d60, ebp = 0 --- Tracing command geom pid 13 tid 100016 td 0xc96ea8a0 sched_switch(c96ea8a0,0,104,1a3,26d21ccf,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96ea8a0,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a8a8,5c,c0fbec12,0,0,...) at sleepq_wait+0x63 _sleep(c118a8a8,c118a808,25c,c0fbec12,0,...) at _sleep+0x39c g_io_schedule_down(c96ea8a0,0,c0fc0761,6c,e33ddd14,...) at g_io_schedule_down+0x5b g_down_procbody(0,e33ddd28,c0fcc890,3d8,c95dd2cc,...) at g_down_procbody+0x9d fork_exit(c09e6b60,0,e33ddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ddd60, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xc96eab80 sched_switch(c96eab80,0,104,1a3,27197667,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eab80,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a8a4,5c,c0fbec12,0,0,...) at sleepq_wait+0x63 _sleep(c118a8a4,c118a828,25c,c0fbec12,0,...) at _sleep+0x39c g_io_schedule_up(c96eab80,0,c0fc0761,5f,e33dad14,...) at g_io_schedule_up+0x134 g_up_procbody(0,e33dad28,c0fcc890,3d8,c95dd2cc,...) at g_up_procbody+0x9d fork_exit(c09e6c00,0,e33dad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33dad60, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xc96eb000 sched_switch(c96eb000,0,104,1a3,236e556b,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb000,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a89c,5c,c0fbec12,0,0,...) at sleepq_wait+0x63 _sleep(c118a89c,c118a7cc,25c,c0fbec12,0,...) at _sleep+0x39c g_run_events(c119b640,0,c0fc0761,79,e33d7d14,...) at g_run_events+0x54b g_event_procbody(0,e33d7d28,c0fcc890,3d8,c95dd2cc,...) at g_event_procbody+0x90 fork_exit(c09e6ca0,0,e33d7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d7d60, ebp = 0 --- Tracing command intr pid 12 tid 100067 td 0xcae255c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100066 td 0xcae258a0 sched_switch(cae258a0,0,109,1a3,f6434c5,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cae258a0,...) at mi_switch+0x1ff ithread_loop(cae1e990,efa45d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cae1e990,efa45d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa45d60, ebp = 0 --- Tracing command intr pid 12 tid 100064 td 0xcae26000 sched_switch(cae26000,0,109,1a3,bc1fb57c,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cae26000,...) at mi_switch+0x1ff ithread_loop(cae18a40,efa35d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cae18a40,efa35d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa35d60, ebp = 0 --- Tracing command intr pid 12 tid 100061 td 0xcace48a0 sched_switch(cace48a0,0,109,1a3,fe58fe9f,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cace48a0,...) at mi_switch+0x1ff ithread_loop(cad15880,efa29d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cad15880,efa29d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa29d60, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xcace4b80 sched_switch(cace4b80,0,109,1a3,f104a9ae,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cace4b80,...) at mi_switch+0x1ff ithread_loop(cad15010,efa21d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cad15010,efa21d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa21d60, ebp = 0 --- Tracing command intr pid 12 tid 100055 td 0xcacf7b80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100042 td 0xcace3b80 sched_switch(cace3b80,0,109,1a3,346eb72e,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cace3b80,...) at mi_switch+0x1ff ithread_loop(cacc5720,ef9e3d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cacc5720,ef9e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xcacbf8a0 sched_switch(cacbf8a0,0,109,1a3,2ef66df3,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cacbf8a0,...) at mi_switch+0x1ff ithread_loop(cacad590,ef9d3d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cacad590,ef9d3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d3d60, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xcacc18a0 sched_switch(cacc18a0,0,109,1a3,fad77bf,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cacc18a0,...) at mi_switch+0x1ff ithread_loop(cacad830,ef9c3d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cacad830,ef9c3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c3d60, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xcacc1b80 sched_switch(cacc1b80,0,109,1a3,26db5738,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,cacc1b80,...) at mi_switch+0x1ff ithread_loop(cac97ae0,ef86dd28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cac97ae0,ef86dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef86dd60, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc96eb2e0 sched_switch(c96eb2e0,0,109,1a3,a98cac6e,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c96eb2e0,...) at mi_switch+0x1ff ithread_loop(cac97c80,ef869d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cac97c80,ef869d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef869d60, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc96eb5c0 sched_switch(c96eb5c0,0,109,1a3,3d48cc,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c96eb5c0,...) at mi_switch+0x1ff ithread_loop(cac97500,ef763d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,cac97500,ef763d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef763d60, ebp = 0 --- Tracing command intr pid 12 tid 100022 td 0xc97c1b80 sched_switch(c97c1b80,0,109,1a3,d4d78aa,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c97c1b80,...) at mi_switch+0x1ff ithread_loop(c95b3450,e33efd28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3450,e33efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33efd60, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc96228a0 sched_switch(c96228a0,0,109,1a3,909aa0b3,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c96228a0,...) at mi_switch+0x1ff ithread_loop(c95b3460,e33ecd28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3460,e33ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ecd60, ebp = 0 --- Tracing command intr pid 12 tid 100019 td 0xc96ea000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100018 td 0xc96ea2e0 sched_switch(c96ea2e0,0,109,1a3,aa4171d6,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c96ea2e0,...) at mi_switch+0x1ff ithread_loop(c95b3490,e33e3d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3490,e33e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc95e05c0 sched_switch(c95e05c0,0,109,1a3,e4163c0f,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c95e05c0,...) at mi_switch+0x1ff ithread_loop(c95b36f0,e33d1d28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b36f0,e33d1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d1d60, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc95e08a0 sched_switch(c95e08a0,0,109,1a3,d95b4cfe,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c95e08a0,...) at mi_switch+0x1ff ithread_loop(c95b3700,e33ced28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3700,e33ced28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ced60, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc95e0b80 sched_switch(c95e0b80,0,109,1a3,ae70cc27,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c95e0b80,...) at mi_switch+0x1ff ithread_loop(c95b3710,e33cbd28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3710,e33cbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33cbd60, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc9622000 sched_switch(c9622000,0,109,1a3,e9fcdb04,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c9622000,...) at mi_switch+0x1ff ithread_loop(c95b3720,c93fed28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3720,c93fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fed60, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc96222e0 sched_switch(c96222e0,0,109,1a3,475dd0f,...) at sched_switch+0x413 mi_switch(109,0,c0fccbc1,557,c96222e0,...) at mi_switch+0x1ff ithread_loop(c95b3730,c93fbd28,c0fcc890,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5aa30,c95b3730,c93fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fbd60, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc96225c0 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc95df000 sched_switch(c95df000,0,108,19e,e9cd338a,...) at sched_switch+0x413 mi_switch(108,0,c0fd490c,3ce,c95df000,...) at mi_switch+0x1ff sched_idletd(0,c93f4d28,c0fcc890,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93f4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f4d60, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc95df2e0 sched_switch(c95df2e0,0,108,19e,e9cfaf87,...) at sched_switch+0x413 mi_switch(108,0,c0fd490c,3ce,c95df2e0,...) at mi_switch+0x1ff sched_idletd(0,c93f1d28,c0fcc890,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93f1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f1d60, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc95df5c0 sched_switch(c95df5c0,0,108,19e,e9d956ff,...) at sched_switch+0x413 mi_switch(108,0,c0fd490c,3ce,c95df5c0,...) at mi_switch+0x1ff sched_idletd(0,c93eed28,c0fcc890,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93eed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93eed60, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc95df8a0 sched_switch(c95df8a0,0,108,19e,e9d66482,...) at sched_switch+0x413 mi_switch(108,0,c0fd490c,3ce,c95df8a0,...) at mi_switch+0x1ff sched_idletd(0,c93ebd28,c0fcc890,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf1f0,0,c93ebd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93ebd60, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc95dfb80 sched_switch(c95dfb80,0,104,1a3,914892a2,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c95dfb80,0,c0fd8280,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd8280,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c95ddb30,6c,c0fdb1b8,100,0,...) at sleepq_wait_sig+0x17 _sleep(c95ddb30,c95ddbb8,16c,c0fdb1b8,0,...) at _sleep+0x385 kern_wait(c95dfb80,ffffffff,c93e7c58,0,0,...) at kern_wait+0x597 sys_wait4(c95dfb80,c93e7cec,c1025d5c,c0fdb006,c1027310,...) at sys_wait4+0x3b syscall(c93e7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x806070f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8c8 --- Tracing command audit pid 10 tid 100001 td 0xc95e0000 sched_switch(c95e0000,0,104,1a3,f66ed84,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e0000,0,c0fd8280,268,c95e0000,...) at sleepq_switch+0x162 sleepq_wait(c130c000,0,c93e4c8c,1,0,...) at sleepq_wait+0x63 _cv_wait(c130c000,c130bfe4,c1004537,194,c93e4cc4,...) at _cv_wait+0x260 audit_worker(0,c93e4d28,c0fcc890,3d8,c95de000,...) at audit_worker+0x84 fork_exit(c0c952c0,0,c93e4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93e4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xcae25000 sched_switch(cae25000,0,104,1a3,835261b4,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae25000,0,c0fd8280,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d7e4,0,c0fbec12,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d7e4,0,0,c0fbec12,bb8) at _sleep+0x36a pause(c0fbec12,bb8,126,124,c118aae8,...) at pause+0xa0 deadlkres(0,efa4ed28,c0fcc890,3d8,c118aa60,...) at deadlkres+0x324 fork_exit(c0a39910,0,efa4ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4ed60, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xcae252e0 sched_switch(cae252e0,0,104,1a3,f66ceb3,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae252e0,0,c0fd8280,268,cae252e0,...) at sleepq_switch+0x162 sleepq_wait(c971ddc0,0,c0fd3461,c0fbec12,0,...) at sleepq_wait+0x63 msleep_spin(c971ddc0,c971ddd8,c0fbec12,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c136d740,efa4bd28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c136d740,efa4bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4bd60, ebp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xcace45c0 sched_switch(cace45c0,0,104,1a3,e0f9f197,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace45c0,0,c0fd8280,268,cace45c0,...) at sleepq_switch+0x162 sleepq_wait(c971de00,0,c0fd3461,c0fbec12,0,...) at sleepq_wait+0x63 msleep_spin(c971de00,c971de18,c0fbec12,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(cad3049c,efa2fd28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,cad3049c,efa2fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa2fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc96ebb80 sched_switch(c96ebb80,0,104,1a3,29d7e353,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ebb80,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de40,0,c0fbec12,0,0,...) at sleepq_wait+0x63 _sleep(c971de40,c971de58,0,c0fbec12,0,...) at _sleep+0x39c taskqueue_thread_loop(c118b3f8,e33fed28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c118b3f8,e33fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fed60, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc97c1000 sched_switch(c97c1000,0,104,1a3,29d7d00d,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c1000,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de80,0,c0fbec12,0,0,...) at sleepq_wait+0x63 _sleep(c971de80,c971de98,0,c0fbec12,0,...) at _sleep+0x39c taskqueue_thread_loop(c130c5e4,e33fbd28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c130c5e4,e33fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fbd60, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc97c12e0 sched_switch(c97c12e0,0,104,1a3,29d7b9a6,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c12e0,0,c0fd8280,268,c97c12e0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3461,c0fbec12,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbec12,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154598,e33f8d28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c1154598,e33f8d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f8d60, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc97c15c0 sched_switch(c97c15c0,0,104,1a3,29d7a77c,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c15c0,0,c0fd8280,268,c97c15c0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3461,c0fbec12,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbec12,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154598,e33f5d28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c1154598,e33f5d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f5d60, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc97c18a0 sched_switch(c97c18a0,0,104,1a3,29d793de,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c18a0,0,c0fd8280,268,c97c18a0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3461,c0fbec12,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbec12,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154598,e33f2d28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc2b0,c1154598,e33f2d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f2d60, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc9622b80 sched_switch(c9622b80,0,104,1a3,b01e8314,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c9622b80,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971e0c0,0,c0fbec12,0,0,...) at sleepq_wait+0x63 _sleep(c971e0c0,c971e0d8,0,c0fbec12,0,...) at _sleep+0x39c taskqueue_thread_loop(c11aa3ac,e33e9d28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c11aa3ac,e33e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e9d60, ebp = 0 --- Tracing command kernel pid 0 tid 100013 td 0xc95e02e0 sched_switch(c95e02e0,0,104,1a3,28231e60,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e02e0,0,c0fd8280,268,0,...) at sleepq_switch+0x162 sleepq_wait(c96e9000,0,c0fbec12,0,0,...) at sleepq_wait+0x63 _sleep(c96e9000,c96e9018,0,c0fbec12,0,...) at _sleep+0x39c taskqueue_thread_loop(c11a8dc0,e33d4d28,c0fcc890,3d8,c118aa60,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc2b0,c11a8dc0,e33d4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc118ad30 sched_switch(c118ad30,0,104,1a3,ab901340,...) at sched_switch+0x413 mi_switch(104,0,c0fd8280,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c118ad30,0,c0fd8280,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c118aa60,54,c1004f18,0,0,...) at sleepq_timedwait+0x6b _sleep(c118aa60,0,54,c1004f18,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x240 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 318 MB:lock order reversal: 1st 0xc1bb7388 128 Bucket (UMA zone) @ vm/uma_core.c:2770 2nd 0xcac11804 AHCI channel lock (AHCI channel lock) @ cam/cam_periph.h:191 KDB: stack backtrace: db_trace_self_wrapper(c0fd6897,0,0,0,0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac240b,c0fda364,c12d7008,bf,f418ed1c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda364,cac11804,c0e552fd,c9588228,c0e2e25b,...) at _witness_debugger+0x25 witness_checkorder(cac11804,9,c0e2e252,bf,0,...) at witness_checkorder+0x86f _mtx_lock_flags(cac11804,0,c0e2e252,bf,0,...) at _mtx_lock_flags+0xd9 adadump(c95ba400,c1326460,0,6c14ca00,4,...) at adadump+0x67 dump_write(c118d500,c1326460,0,6c14ca00,4,...) at dump_write+0xbf minidumpsys(c118d500,c051c7e3,c15b87cc,f418f30c,f418f30c,...) at minidumpsys+0x356 dumpsys(c118d500,f418f3bc,c051b3a9,0,0,...) at dumpsys+0x2b doadump(0,0,0,0,0,...) at doadump+0x6c db_fncall(c0abd9ba,0,ffffffff,f418f3f8,0,...) at db_fncall+0x149 db_command(c1153668,f418f4e8,c051fcd0,c1153668,0,...) at db_command+0x3bf db_command_script(c1153668,0,c1152fc0,c1153667,c1152fc0,...) at db_command_script+0x34 db_script_exec(f418f72c,f418f5b8,c051b7a1,1,0,...) at db_script_exec+0x130 db_run_cmd(1,0,c132dd60,f418f530,1,...) at db_run_cmd+0x41 db_command(c0e4697e,f418f618,c051d8ed,c0e46ca9,c9775000,...) at db_command+0x381 db_command_loop(c0e46ca9,c9775000,0,f418f5f0,0,...) at db_command_loop+0x5a db_trap(3,0,1,46,2,...) at db_trap+0xdd kdb_trap(3,0,f418f72c,c133ada0,cde72864,...) at kdb_trap+0xb6 trap(f418f72c) at trap+0x79b calltrap() at calltrap+0x6 --- trap 0x3, eip = 0xc0abd9ba, esp = 0xf418f76c, ebp = 0xf418f774 --- kdb_enter(c0e46ca9,c0fd683e,f418f7bc,c08fb51d,c95a7200,...) at kdb_enter+0x3a kdb_break(c95a7200,c596e000,7,7,f418f7b4,...) at kdb_break+0x28 uart_intr(c95a7200,c0fda75e,f418f824,c0ad499c,0,...) at uart_intr+0x12d intr_event_handle(c95aab80,f418f828,c11ab01c,2,ccbc1fc8,...) at intr_event_handle+0x76 intr_execute_handlers(c96210d0,f418f828,2,f418f894,c0deaf15,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,f418f828) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0d00a23, esp = 0xf418f868, ebp = 0xf418f894 --- uma_dbg_free(c1bb6540,ccbc1fc8,ccbc1418,ad2,ccbc1418,...) at uma_dbg_free+0x93 zone_free_item(0,2) at zone_free_item+0x343 bucket_free(c1bb69a0,0,101,969,2,...) at bucket_free+0x2f uma_zalloc_arg(c1bb69a0,0,101,c0d1f53c,ce240128,...) at uma_zalloc_arg+0x773 vm_radix_insert(ce240158,1e,0,c3f89f40,c100f8e7,...) at vm_radix_insert+0xbd vm_page_insert(c3f89f40,ce240128,1e,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(ce240128,1e,0,0,f418fb64,...) at vm_page_alloc+0x51c vm_fault_hold(ce1c5ae0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(ce1c5ae0,bfbfe000,2,0,bfbfec38,...) at vm_fault+0x8b trap_pfault(c0dffdff,0,cde8e000,cde72864,cde72864,...) at trap_pfault+0x1b9 trap(f418fd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bcf06, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- 303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 2 $ svn diff -x -p /var/tmp/vmcontention/sys Index: /var/tmp/vmcontention/sys/vm/vm_radix.c =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.c (revision 235728) +++ /var/tmp/vmcontention/sys/vm/vm_radix.c (working copy) @@ -110,6 +110,7 @@ struct vm_radix_node { CTASSERT(sizeof(struct vm_radix_node) < PAGE_SIZE); static uma_zone_t vm_radix_node_zone; +static u_long allocfail; #ifndef UMA_MD_SMALL_ALLOC static vm_map_t rnode_map; @@ -205,6 +206,10 @@ static __inline struct vm_radix_node * vm_radix_node_get(void) { + if (++allocfail == 100000000) { + allocfail = 0; + return (NULL); + } return (uma_zalloc(vm_radix_node_zone, M_NOWAIT | M_ZERO)); } @@ -293,11 +298,11 @@ vm_radix_setroot(struct vm_radix *rtree, struct vm rtree->rt_root = root; } -static inline void -vm_radix_unwind_heightup(struct vm_radix *rtree, struct vm_radix_node *root, - struct vm_radix_node *iroot, int ilevel) +static void +vm_radix_unwind_heightup(struct vm_radix *rtree, volatile struct vm_radix_node *root, + volatile struct vm_radix_node *iroot, int ilevel) { - struct vm_radix_node *rnode; + volatile struct vm_radix_node *rnode; CTR4(KTR_VM, "unwind: tree %p, root %p, iroot %p, ilevel %d", rtree, root, iroot, ilevel); @@ -306,9 +311,9 @@ vm_radix_setroot(struct vm_radix *rtree, struct vm MPASS(rnode->rn_count == 0 || rnode->rn_count == 1); rnode->rn_count = 0; root = rnode->rn_child[0]; - vm_radix_node_put(rnode); + vm_radix_node_put(__DEVOLATILE(struct vm_radix_node *, rnode)); } - vm_radix_setroot(rtree, iroot, ilevel); + vm_radix_setroot(rtree, __DEVOLATILE(struct vm_radix_node *, iroot), ilevel); } static inline void * @@ -362,16 +367,16 @@ vm_radix_reclaim_allnodes_internal(struct vm_radix int vm_radix_insert(struct vm_radix *rtree, vm_pindex_t index, void *val) { - struct vm_radix_node *iroot, *rnode, *root; - u_int allocmsk; - int clev, ilevel, level, slot; + volatile struct vm_radix_node *iroot, *rnode, *root; + volatile u_int allocmsk; + volatile int clev, ilevel, level, slot; CTR4(KTR_VM, "insert: tree %p, " KFRMT64(index) ", val %p", rtree, KSPLT64L(index), KSPLT64H(index), val); if (index == -1) panic("vm_radix_insert: -1 is not a valid index.\n"); - level = vm_radix_height(rtree, &root); + level = vm_radix_height(rtree, __DEVOLATILE(struct vm_radix_node **, &root)); /* * Increase the height by adding nodes at the root until * there is sufficient space. @@ -413,7 +418,7 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ } root = rnode; } - vm_radix_setroot(rtree, root, level); + vm_radix_setroot(rtree, __DEVOLATILE(struct vm_radix_node *, root), level); } /* Now that the tree is tall enough, fill in the path to the index. */ -- Test scenario: altbufferflushes.sh