GENERIC HEAD from 2012-05-12 19:54:57 UTC, r236123M, vmcore.382 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r236123M: Tue May 29 07:34:57 CEST 2012 pho@x4.osted.lan:/var/tmp/vmcontention/sys/i386/compile/PHO i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.02-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) lock order reversal: 1st 0xc1bb505c system map (system map) @ vm/vm_map.c:3723 2nd 0xc130cd00 vm object (kernel object) @ vm/vm_fault.c:299 3rd 0xc1bb50e8 system map (system map) @ vm/vm_map.c:1454 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,34313a63,a3435,ff000a39,c1820740,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda09d,c12d8d48,5ae,c18207d0,...) at kdb_backtrace+0x2a _witness_debugger(c0fda09d,c1bb50e8,c100db0a,c9582110,c100e188,...) at _witness_debugger+0x25 witness_checkorder(c1bb50e8,9,c100e17f,5ae,0,...) at witness_checkorder+0x86f _mtx_lock_flags(c1bb50e8,0,c100e17f,5ae,c596e000,...) at _mtx_lock_flags+0xd9 _vm_map_lock(c1bb508c,c100e17f,5ae,0,c182090b,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,c1820884,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd05c7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,c18208eb,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d144,871,c1820938,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,11,c596ff44,c1820998,c0d0015f,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,c1bb7600,101,827,c1bb1ba0,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,0,5,...) at uma_zalloc_arg+0x59f vm_radix_insert(c130cd30,23480,0,c21fbf80,0,...) at vm_radix_insert+0x1e9 vm_page_insert(c21fbf80,c130cd00,23480,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(c130cd00,23480,0,40,c1820bbc,...) at vm_page_alloc+0x51c vm_fault_hold(c1bb5000,e337f000,0,1,0,...) at vm_fault_hold+0x60a vm_fault(c1bb5000,e337f000,0,1,c1bb3a68,...) at vm_fault+0x8b vm_fault_wire(c1bb5000,e3000000,e33c9000,0,0,...) at vm_fault_wire+0x3f vm_map_wire(c1bb5000,e3000000,e33c9000,0,e3000000,...) at vm_map_wire+0x2bc kmem_alloc(c1bb5000,3c9000,c955c770,0,0,...) at kmem_alloc+0x10d vm_ksubmap_init(c131e64c,0,1,1000,0,...) at vm_ksubmap_init+0x64 cpu_startup(0,181ec00,181ec00,181e000,1825000,...) at cpu_startup+0x24f mi_startup() at mi_startup+0xac begin() at begin+0x2c avail memory = 3538612224 (3374 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 614586 free (1298 frags, 76661 blocks, 0.1% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 122960 free (5272 frags, 14711 blocks, 0.5% fragmentation) /dev/label/tmp: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/label/tmp: clean, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 2956327 free (219183 frags, 342143 blocks, 2.2% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 7587913 free (78721 frags, 938649 blocks, 0.3% fragmentation) Mounting local file systems:. Setting hostname: x4.osted.lan. re0: link state changed to DOWN re0: link state changed to UP Starting Network: lo0 re0 fwe0 fwip0. lo0: flags=8049 metric 0 mtu 16384 options=3 inet6 ::1 prefixlen 128 inet6 fe80::1%lo0 prefixlen 64 scopeid 0xa inet 127.0.0.1 netmask 0xff000000 nd6 options=21 re0: flags=8843 metric 0 mtu 1500 options=8209b ether 00:21:85:15:33:7d inet 192.168.1.101 netmask 0xffffff00 broadcast 192.168.1.255 inet6 fe80::221:85ff:fe15:337d%re0 prefixlen 64 scopeid 0x1 nd6 options=29 media: Ethernet autoselect (100baseTX ) status: active fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 Starting devd. Starting Network: fwe0. fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 Starting Network: fwip0. fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/kde4/lib /usr/local/lib/compat /usr/local/lib/gcc46 /usr/local/lib/mysql /usr/local/lib/nss /usr/local/lib/qt4 /usr/local/lib/virtualbox a.out ldconfig path: /usr/lib/aout /usr/lib/compat/aout Creating and/or trimming log files. Starting syslogd. lock order reversal: 1st 0xe310c560 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcbfba61c snaplk (snaplk) @ ufs/ffs/ffs_snapshot.c:2297 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,632e746f,3932323a,c0000a37,1,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d75d8,8f9,f404f990,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cbfba61c,c1007654,c95896e0,c10075de,...) at _witness_debugger+0x25 witness_checkorder(cbfba61c,9,c10075d5,8f9,0,...) at witness_checkorder+0x86f __lockmgr_args(cbfba61c,80900,cc3704d8,0,0,...) at __lockmgr_args+0x8c5 ffs_copyonwrite(cc370414,e310c500,e310c500,0,0,...) at ffs_copyonwrite+0x2c2 ffs_geom_strategy(cc37050c,e310c500,cc064a6c,e310c500,170f788,...) at ffs_geom_strategy+0xeb bufwrite(e310c500,0,c100aa70,811,0) at bufwrite+0x159 ffs_bufwrite(e310c500,cc4f9a00,100,4000,0,...) at ffs_bufwrite+0x290 ffs_update(cc6476cc,1,c100b1f1,151,0,...) at ffs_update+0x3a5 ffs_syncvnode(cc6476cc,1,0,c11375a0,cc6477c4,...) at ffs_syncvnode+0x591 ffs_fsync(f404fc40,c102c6e4,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c11375a0,f404fc40,c0fe37a7,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cc0e15c0,f404fcec,c1025a7c,c0fdbd8f,c1027030,...) at sys_fsync+0x1df syscall(f404fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x2818ad47, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- No core dumps found. Additional ABI support: linux. Starting rpcbind. NFS access cache time=60 Clearing /tmp (X related). Starting mountd. Starting nfsuserd. Starting nfsd. Recovering vi editor sessions:lock order reversal: 1st 0xcc7ab310 ufs (ufs) @ kern/vfs_lookup.c:542 2nd 0xe31297c0 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:260 3rd 0xcc7aa880 ufs (ufs) @ kern/vfs_subr.c:2159 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,3a632e73,a303632,0,c7b87284,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda09d,c12d71b8,86f,f403233c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda09d,cc7aa880,c0fc2564,c95895a8,c0fe20de,...) at _witness_debugger+0x25 witness_checkorder(cc7aa880,9,c0fe20d5,86f,0,...) at witness_checkorder+0x86f __lockmgr_args(cc7aa880,80100,cc7aa8ec,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f4032444,c0ad492c,c0fdf52b,5e2,c11088e8,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11375a0,f4032444,cc0e4670,c1149700,cc7aa828,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc7aa828,80100,c0fe20d5,86f,eb,...) at _vn_lock+0x78 vget(cc7aa828,80100,cc0e45c0,50,0,...) at vget+0xbb vfs_hash_get(cbfa62dc,15ef55,80000,cc0e45c0,f4032590,...) at vfs_hash_get+0xed ffs_vgetf(cbfa62dc,15ef55,80000,f4032590,1,...) at ffs_vgetf+0x49 softdep_sync_buf(cc7ab2b8,e3129760,1,106,0,...) at softdep_sync_buf+0xac9 ffs_syncvnode(cc7ab2b8,1,0,cc0e4670,c11088e8,...) at ffs_syncvnode+0x24c ffs_truncate(cc7ab2b8,200,0,880,c95dab00,...) at ffs_truncate+0x8b5 ufs_direnter(cc7ab2b8,cc7aa828,f4032904,f4032bb8,0,...) at ufs_direnter+0x980 ufs_makeinode(f4032bb8,c1137b60,f4032afc,f4032a58,c0e28800,...) at ufs_makeinode+0x65d ufs_create(f4032afc,c102d22e,c1149980,cc7ab2b8,f4032b14,...) at ufs_create+0x30 VOP_CREATE_APV(c11375a0,f4032afc,f4032bb8,f4032a94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f4032b78,f4032c40,1b0,0,c95dab00,...) at vn_open_cred+0x205 vn_open(f4032b78,f4032c40,1b0,cc0537a8,4fc46536,...) at vn_open+0x3b kern_openat(cc0e45c0,ffffff9c,28c72f60,0,a03,...) at kern_openat+0x1ec kern_open(cc0e45c0,28c72f60,0,a02,1b0,...) at kern_open+0x35 sys_open(cc0e45c0,f4032cec,c1025a7c,c0fdad8f,c1027030,...) at sys_open+0x30 syscall(f4032d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x283d28d3, esp = 0xbfbfc4ec, ebp = 0xbfbfc578 --- . Updating motd:. Starting ntpd. Starting default moused. Configuring syscons: keymap blanktime. Starting sshd. Starting cron. Local package initialization: backuplock order reversal: 1st 0xcc370058 ufs (ufs) @ kern/vfs_mount.c:1254 2nd 0xcbfaa46c devfs (devfs) @ ufs/ffs/ffs_softdep.c:1834 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,38313a63,a3433,cc64f1b4,8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d6d88,72a,f3fe99a8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cbfaa46c,c0fcd480,c9589408,c1007b95,...) at _witness_debugger+0x25 witness_checkorder(cbfaa46c,9,c1007b8c,72a,cbfaa4d8,...) at witness_checkorder+0x86f __lockmgr_args(cbfaa46c,80400,cbfaa4d8,0,0,...) at __lockmgr_args+0x8c5 vop_stdlock(f3fe9aac,c130c580,4,c0fd05c7,eb,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c10f7d60,f3fe9aac,c130c580,c1149700,cbfaa414,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cbfaa414,80400,c1007b8c,72a,0,...) at _vn_lock+0x78 softdep_flushworklist(cbfa6894,f3fe9b64,cc0458a0,612,cc3e5a48,...) at softdep_flushworklist+0x47 ffs_sync(cbfa6894,1,f3fe9bb8,51f,c1bae710,...) at ffs_sync+0x3bf dounmount(cbfa6894,8080000,cc0458a0,4a1,f3fe9c3c,...) at dounmount+0x3f8 sys_unmount(cc0458a0,f3fe9cec,c1025a7c,c0fdad52,c1027030,...) at sys_unmount+0x421 syscall(f3fe9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280beebb, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes lock order reversal: 1st 0xe313ada0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcc7b2200 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:284 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,2e687361,38323a63,a34,f406a724,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d6828,11c,f406a7d4,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cc7b2200,c100ba9a,c9589610,c100b72c,...) at _witness_debugger+0x25 witness_checkorder(cc7b2200,9,c100b723,11c,0,...) at witness_checkorder+0x86f _sx_xlock(cc7b2200,0,c100b723,11c,2400,...) at _sx_xlock+0x9a ufsdirhash_acquire(e4058000,f406a898,c0ceb2a0,cc78ee0c,2400,...) at ufsdirhash_acquire+0x48 ufsdirhash_newblk(cc78ee0c,2400,18,200,cbfb3080,...) at ufsdirhash_newblk+0xe ufs_direnter(cc7a62b8,cca02ae0,f406a904,f406abb8,0,...) at ufs_direnter+0x230 ufs_makeinode(f406abb8,c1137b60,f406aafc,f406aa58,c0e28800,...) at ufs_makeinode+0x65d ufs_create(f406aafc,c102d22e,c1149980,cc7a62b8,f406ab14,...) at ufs_create+0x30 VOP_CREATE_APV(c11375a0,f406aafc,f406abb8,f406aa94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f406ab78,f406ac40,180,0,cbfb3080,...) at vn_open_cred+0x205 vn_open(f406ab78,f406ac40,180,cc08d310,0,...) at vn_open+0x3b kern_openat(cae26b80,ffffff9c,bfbfe7ab,0,a02,...) at kern_openat+0x1ec kern_open(cae26b80,bfbfe7ab,0,a01,180,...) at kern_open+0x35 sys_open(cae26b80,f406acec,c1025a7c,c0fdad8f,c1027030,...) at sys_open+0x30 syscall(f406ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x2829f8d3, esp = 0xbfbfe37c, ebp = 0xbfbfec18 --- May 29 07:58:51 x4 su: pho to root on /dev/pts/1 ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5925 files, 37539076 used, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5925 files, 37539076 used, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Starting inetd. Tue May 29 08:02:26 CEST 2012 FreeBSD/i386 (x4.osted.lan) (console) login: lock order reversal: 1st 0xcbfba61c snaplk (snaplk) @ ufs/ufs/ufs_vnops.c:966 2nd 0xcc3e546c ufs (ufs) @ ufs/ffs/ffs_snapshot.c:1627 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,a373236,a00,0,f3ffb680,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d64f0,65b,f3ffb718,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cc3e546c,c0fc2564,c95895a8,c10075de,...) at _witness_debugger+0x25 witness_checkorder(cc3e546c,9,c10075d5,65b,0,...) at witness_checkorder+0x86f __lockmgr_args(cc3e546c,80000,0,0,0,...) at __lockmgr_args+0x8c5 ffs_snapremove(cc3e5414,c1007513,f3ffb848,c11a99c4,c119b9c0,...) at ffs_snapremove+0x11f ffs_truncate(cc3e5414,0,0,c00,0,...) at ffs_truncate+0x647 ufs_inactive(f3ffbad4,c102cbfd,eb,c11375a0,f3ffbaa8,...) at ufs_inactive+0x1f6 VOP_INACTIVE_APV(c11375a0,f3ffbad4,c0fe20d5,9bf,f3ffbb08,...) at VOP_INACTIVE_APV+0x100 vinactive(cc3e5414,cc0452e0,c0fe20d5,8be,0,...) at vinactive+0x127 vputx(f3ffbc34,c0b2e9e3,cc3e5414,ffffffdf,cc3e5414,...) at vputx+0x370 vput(cc3e5414,ffffffdf,cc3e5414,f3ffbb8c,1d2f8,...) at vput+0x10 kern_unlinkat(cc0452e0,ffffff9c,bfbfea8e,0,0,...) at kern_unlinkat+0x2a3 kern_unlink(cc0452e0,bfbfea8e,0,f3ffbd1c,c0e0060e,...) at kern_unlink+0x2f sys_unlink(cc0452e0,f3ffbcec,c1025a7c,c0fdada3,c1027030,...) at sys_unlink+0x22 syscall(f3ffbd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (10, FreeBSD ELF32, sys_unlink), eip = 0x2816352f, esp = 0xbfbfe7ac, ebp = 0xbfbfe828 --- 20120529 08:02:43 all: altbufferflushes.sh interrupt total irq1: atkbd0 1 irq4: uart0 547 irq6: fdc0 1 irq9: acpi0 1 irq14: ata0 42 irq16: hdac1 ohci0 80 irq17: ohci1 ohci3 3 irq18: ohci2 ohci4 3 irq20: fwohci0 1 irq22: ahci0 1056263 cpu0:timer 5323624 irq256: hdac0 6 irq257: re0 16848 cpu1:timer 5172967 cpu2:timer 5162164 cpu3:timer 5166490 Total 21899074 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,417f44,efbd79d4,c0e09bbe,9024aac0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0fca24e,14e2742,0,14e2742,0,...) at kdb_backtrace+0x2a watchdog_fire(1,0,c0fca11b,21b,c95de2cc,...) at watchdog_fire+0xa9 hardclock_cnt(1,0,efbd7ab0,c0ab0c65,16c6d1c3,...) at hardclock_cnt+0x2bc handleevents(efbd7ae0,c9582790,4,c12d7ba0,0,c95827f8,140d,16c6d1c3,4a1d95da,828dd00) at handleevents+0x17d timercb(c136d380,0,2,cbdc0a08,30a,...) at timercb+0x292 lapic_handle_timer(efbd7b24) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0ad3eb0, esp = 0xefbd7b64, ebp = 0xefbd7b94 --- witness_unlock(cbdc0a08,8,c100d144,30a,0,...) at witness_unlock+0x50 Expensive timeout(9) function: 0xc0917300(0xcace5aa0) 0.259431876 s _mtx_unlock_flags(cbdc0a08,0,c100d144,30a,4,...) at _mtx_unlock_flags+0xca zone_drain_wait(efbd7c10,c0cfb695,cbe0d380,0,c100d144,...) at zone_drain_wait+0xf1 zone_drain(cbe0d380,0,c100d144,63c,0,...) at zone_drain+0x10 zone_foreach(c95b46c0,0,efbd7ce8,c0d19bac,c95b46cc,...) at zone_foreach+0x45 uma_reclaim(c95b46cc,0,c1010087,2ee,1388,...) at uma_reclaim+0x16 vm_pageout(0,efbd7d28,c0fcc5b0,3d8,c95de2cc,...) at vm_pageout+0x80c fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- KDB: enter: watchdog timeout [ thread pid 7 tid 100072 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> lock order reversal: 1st 0xcbdc0a08 sctp_readq (UMA zone) @ vm/uma_core.c:768 2nd 0xcad30474 firewire (firewire) @ dev/firewire/fwohci.c:2118 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,20,f00,f,c132dad0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d7bb0,846,efbd7770,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cad30474,c0f6123a,c95872b8,c0f60922,...) at _witness_debugger+0x25 witness_checkorder(cad30474,9,c0f60919,846,0,...) at witness_checkorder+0x86f _mtx_lock_flags(cad30474,0,c0f60919,846,cac93720,...) at _mtx_lock_flags+0xd9 fwohci_poll(cad2c000,ffffffff,ffffffff,efbd77f8,c06905de,...) at fwohci_poll+0x35 dcons_crom_poll(cac72e00,0,c0f57488,b1,c115d000,...) at dcons_crom_poll+0x25 dcons_os_checkc(efbd7810,c0a3e9aa,c1093720,78,efbd7818,...) at dcons_os_checkc+0xae dcons_cngetc(c1093720,78,efbd7818,c0a3e9e6,efbd7838,...) at dcons_cngetc+0xe cncheckc(efbd7838,c051cdf5,c0e4669e,c051e0f0,efbd7834,...) at cncheckc+0x3a cngetc(c0e4669e,c051e0f0,efbd7834,efbd7870,c10453a0,...) at cngetc+0x16 db_readline(c1152d40,78,efbd7854,c051b8e6,c0e4669e,...) at db_readline+0x75 db_read_line(c0e4669e,efbd78a8,c051d8ed,c0e655d6,c9775000,...) at db_read_line+0x1a db_command_loop(c0e655d6,c9775000,0,efbd7880,0,...) at db_command_loop+0x46 db_trap(3,0,1,46,2,...) at db_trap+0xdd kdb_trap(3,0,efbd79bc,efbd7974,c95de2cc,...) at kdb_trap+0xb6 trap(efbd79bc) at trap+0x79b calltrap() at calltrap+0x6 --- trap 0x3, eip = 0xc0abd94a, esp = 0xefbd79fc, ebp = 0xefbd7a04 --- kdb_enter(c0e655d6,c0f57edc,0,14e2742,0) at kdb_enter+0x3a watchdog_fire(1,0,c0fca11b,21b,c95de2cc,...) at watchdog_fire+0xbd hardclock_cnt(1,0,efbd7ab0,c0ab0c65,16c6d1c3,...) at hardclock_cnt+0x2bc handleevents(efbd7ae0,c9582790,4,c12d7ba0,0,c95827f8,140d,16c6d1c3,4a1d95da,828dd00) at handleevents+0x17d timercb(c136d380,0,2,cbdc0a08,30a,...) at timercb+0x292 lapic_handle_timer(efbd7b24) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0ad3eb0, esp = 0xefbd7b64, ebp = 0xefbd7b94 --- witness_unlock(cbdc0a08,8,c100d144,30a,0,...) at witness_unlock+0x50 _mtx_unlock_flags(cbdc0a08,0,c100d144,30a,4,...) at _mtx_unlock_flags+0xca zone_drain_wait(efbd7c10,c0cfb695,cbe0d380,0,c100d144,...) at zone_drain_wait+0xf1 zone_drain(cbe0d380,0,c100d144,63c,0,...) at zone_drain+0x10 zone_foreach(c95b46c0,0,efbd7ce8,c0d19bac,c95b46cc,...) at zone_foreach+0x45 uma_reclaim(c95b46cc,0,c1010087,2ee,1388,...) at uma_reclaim+0x16 vm_pageout(0,efbd7d28,c0fcc5b0,3d8,c95de2cc,...) at vm_pageout+0x80c fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- db> run pho db:0:pho> bt Tracing pid 7 tid 100072 td 0xcae245c0 kdb_enter(c0e655d6,c0f57edc,0,14e2742,0) at kdb_enter+0x3a watchdog_fire(1,0,c0fca11b,21b,c95de2cc,...) at watchdog_fire+0xbd hardclock_cnt(1,0,efbd7ab0,c0ab0c65,16c6d1c3,...) at hardclock_cnt+0x2bc handleevents(efbd7ae0,c9582790,4,c12d7ba0,0,c95827f8,140d,16c6d1c3,4a1d95da,828dd00) at handleevents+0x17d timercb(c136d380,0,2,cbdc0a08,30a,...) at timercb+0x292 lapic_handle_timer(efbd7b24) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0ad3eb0, esp = 0xefbd7b64, ebp = 0xefbd7b94 --- witness_unlock(cbdc0a08,8,c100d144,30a,0,...) at witness_unlock+0x50 _mtx_unlock_flags(cbdc0a08,0,c100d144,30a,4,...) at _mtx_unlock_flags+0xca zone_drain_wait(efbd7c10,c0cfb695,cbe0d380,0,c100d144,...) at zone_drain_wait+0xf1 zone_drain(cbe0d380,0,c100d144,63c,0,...) at zone_drain+0x10 zone_foreach(c95b46c0,0,efbd7ce8,c0d19bac,c95b46cc,...) at zone_foreach+0x45 uma_reclaim(c95b46cc,0,c1010087,2ee,1388,...) at uma_reclaim+0x16 vm_pageout(0,efbd7d28,c0fcc5b0,3d8,c95de2cc,...) at vm_pageout+0x80c fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- db:0:bt> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x6d4d00 curthread = 0xcca098a0: pid 2439 "bash" curpcb = 0xf4135d80 fpcurthread = none idlethread = 0xc95df8a0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x828ad00 curthread = 0xcca092e0: pid 3207 "sh" curpcb = 0xf414dd80 fpcurthread = none idlethread = 0xc95df5c0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x828dd00 curthread = 0xcae245c0: pid 7 "pagedaemon" curpcb = 0xefbd7d80 fpcurthread = none idlethread = 0xc95df2e0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x8290d00 curthread = 0xcc0e15c0: pid 961 "syslogd" curpcb = 0xf404fd80 fpcurthread = none idlethread = 0xc95df000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 3210 (cron) thread 0xcc0938a0 (100169) exclusive sleep mutex vm object (standard object) r = 0 (0xcca0d1bc) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc742130) locked @ vm/vm_map.c:3723 Process 3207 (sh) thread 0xcca092e0 (100153) exclusive sleep mutex vm object (standard object) r = 0 (0xceb7c094) locked @ vm/vm_object.c:1730 exclusive sleep mutex vm object (standard object) r = 0 (0xd01b5b90) locked @ vm/vm_object.c:545 Process 3205 (vmstat) thread 0xcca2bb80 (100162) exclusive sleep mutex vm object (standard object) r = 0 (0xcc12a40c) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcd6dd048) locked @ vm/vm_map.c:3723 Process 3156 (dd) thread 0xcca44b80 (100204) exclusive sleep mutex page lock (page lock) r = 0 (0xc130dd00) locked @ vm/vm_page.c:1278 exclusive sleep mutex vm object (standard object) r = 0 (0xccae5534) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe32d7180) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe31410c0) locked @ kern/vfs_bio.c:2652 exclusive lockmgr ufs (ufs) r = 0 (0xccb299dc) locked @ kern/vfs_vnops.c:645 Process 2439 (bash) thread 0xcca098a0 (100145) exclusive sleep mutex vm page queue free mutex (vm page queue free mutex) r = 0 (0xc130cf00) locked @ vm/vm_page.c:1884 exclusive sleep mutex page lock (page lock) r = 0 (0xc130d200) locked @ vm/vm_page.c:1278 exclusive sleep mutex vm object (standard object) r = 0 (0xcc9fed4c) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc742cf8) locked @ vm/vm_map.c:3723 Process 1766 (watchdogd) thread 0xcc9c22e0 (100156) exclusive sleep mutex vm object (standard object) r = 0 (0xceb7b818) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc4223e8) locked @ vm/vm_map.c:3723 Process 1151 (sendmail) thread 0xcc0e48a0 (100100) exclusive sleep mutex vm object (standard object) r = 0 (0xcc7d94a0) locked @ vm/vm_object.c:1730 exclusive sleep mutex vm object (standard object) r = 0 (0xcc7d9de0) locked @ vm/vm_map.c:2961 exclusive sx user map (user map) r = 0 (0xcc0d45b8) locked @ vm/vm_map.c:3105 exclusive sx user map (user map) r = 0 (0xcc4f1788) locked @ vm/vm_map.c:3101 Process 1091 (ntpd) thread 0xcc0e4000 (100104) exclusive sleep mutex vm object (standard object) r = 0 (0xd01b5de0) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe31b4520) locked @ kern/vfs_bio.c:1905 exclusive lockmgr ufs (ufs) r = 0 (0xccae4058) locked @ kern/vfs_vnops.c:645 Process 961 (syslogd) thread 0xcc0e15c0 (100108) exclusive lockmgr bufwait (bufwait) r = 0 (0xe31b8e40) locked @ kern/vfs_bio.c:1905 exclusive lockmgr snaplk (snaplk) r = 0 (0xcbfba61c) locked @ ufs/ffs/ffs_snapshot.c:2297 exclusive lockmgr ufs (ufs) r = 0 (0xcc9905c8) locked @ kern/vfs_syscalls.c:3587 Process 19 (softdepflush) thread 0xcbee88a0 (100078) exclusive sleep mutex page lock (page lock) r = 0 (0xc130d900) locked @ vm/vm_page.c:1278 exclusive sleep mutex vm object (standard object) r = 0 (0xcc072094) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe32957c0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe32e0d80) locked @ kern/vfs_bio.c:1905 Process 7 (pagedaemon) thread 0xcae245c0 (100072) exclusive sleep mutex sctp_readq (UMA zone) r = 0 (0xcbdc0a08) locked @ vm/uma_core.c:768 exclusive sleep mutex UMA lock (UMA lock) r = 0 (0xc130c96c) locked @ vm/uma_core.c:1596 Process 12 (intr) thread 0xc9622000 (100009) exclusive sleep mutex ehci0 (ehci0) r = 0 (0xcacfee4c) locked @ dev/usb/usb_hub.c:1910 db:0:alllocks> show lockedvnods Locked vnodes 0xcc990570: tag ufs, type VREG usecount 2, writecount 1, refcount 4 mountedhere 0 flags (VI(0x200)) v_object 0xcca0cd4c ref 0 pages 1 lock type ufs: EXCL by thread 0xcc0e15c0 (pid 961, syslogd, tid 100108) with shared waiters pending #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273c3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b35f66 at sys_fsync+0x116 #5 0xc0e0060e at syscall+0x2fe #6 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 6052908, on dev ada0s1e 0xccae4570: tag ufs, type VREG usecount 1, writecount 0, refcount 69 mountedhere 0 flags (VV_SYSTEM|VI(0x200)) v_object 0xcca4b4a0 ref 0 pages 2232 lock type snaplk: EXCL by thread 0xcc0e15c0 (pid 961, syslogd, tid 100108) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0cbf2a2 at ffs_copyonwrite+0x2c2 #2 0xc0cdc2bb at ffs_geom_strategy+0xeb #3 0xc0cf0ff3 at ufs_strategy+0xd3 #4 0xc0e27099 at VOP_STRATEGY_APV+0x119 #5 0xc0b09251 at bufstrategy+0x61 #6 0xc0b0cce9 at bufwrite+0x159 #7 0xc0b0d058 at vfs_bio_awrite+0x328 #8 0xc0ce2df5 at ffs_syncvnode+0x3a5 #9 0xc0ce304f at ffs_fsync+0x2f #10 0xc0e23b00 at VOP_FSYNC_APV+0x100 #11 0xc0b3602f at sys_fsync+0x1df #12 0xc0e0060e at syscall+0x2fe #13 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 6, on dev ada0s1e 0xccb29984: tag ufs, type VREG usecount 1, writecount 1, refcount 7046 mountedhere 0 flags (VI(0x200)) v_object 0xccae5534 ref 0 pages 460008 lock type ufs: EXCL by thread 0xcca44b80 (pid 3156, dd, tid 100204) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273c3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b38c16 at vn_write+0x176 #5 0xc0ad6b85 at dofilewrite+0x95 #6 0xc0ad71f8 at kern_writev+0x68 #7 0xc0ad730f at sys_write+0x4f #8 0xc0e0060e at syscall+0x2fe #9 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 6194267, on dev ada0s1e 0xccae4000: tag ufs, type VREG usecount 1, writecount 1, refcount 2 mountedhere 0 flags (VI(0x200)) v_object 0xd01b5de0 ref 0 pages 0 lock type ufs: EXCL by thread 0xcc0e4000 (pid 1091, ntpd, tid 100104) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273c3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b38c16 at vn_write+0x176 #5 0xc0ad6b85 at dofilewrite+0x95 #6 0xc0ad71f8 at kern_writev+0x68 #7 0xc0ad730f at sys_write+0x4f #8 0xc0e0060e at syscall+0x2fe #9 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 1742950, on dev ada0s1e db:0:lockedvnods> show mount 0xcbfa6b70 /dev/ad4s1a on / (ufs) 0xcbfa7000 devfs on /dev (devfs) 0xcc37c5b8 /dev/ad4s1f on /home (ufs) 0xcbfa65b8 /dev/ad4s1d on /usr (ufs) 0xcbfa62dc /dev/ad4s1e on /var (ufs) 0xcbfa6000 procfs on /proc (procfs) 0xcca88000 /dev/label/tmp on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 3212 2439 2439 1001 RL+ bash 3211 1154 1154 25 RL sendmail 3210 1159 1159 0 RL cron 3209 1766 1766 0 RL watchdogd 3208 1208 1206 1001 S piperd 0xcfa9b7d0 awk 3207 1208 1206 1001 R CPU 1 sh 3206 1208 1206 1001 S piperd 0xcc081190 sed 3205 1208 1206 1001 RL vmstat 3157 1935 1256 0 S+ piperd 0xcdbad960 egrep 3156 1935 1256 0 R+ dd 2439 2438 2439 1001 RLs+ CPU 0 bash 2438 2436 2436 1001 S select 0xcc079664 sshd 2436 1148 2436 0 Ss select 0xcac864a4 sshd 1935 1256 1256 0 S+ wait 0xcca0fb30 sh 1816 1 1816 0 Ss+ ttyin 0xcae33870 getty 1815 1 1815 0 Ss+ ttyin 0xcae33a70 getty 1814 1 1814 0 Ss+ ttyin 0xc970d270 getty 1813 1 1813 0 Ss+ ttyin 0xc970d470 getty 1812 1 1812 0 Ss+ ttyin 0xc970d670 getty 1811 1 1811 0 Ss+ ttyin 0xc970d870 getty 1810 1 1810 0 Ss+ ttyin 0xc970da70 getty 1809 1 1809 0 Ss+ ttyin 0xc970dc70 getty 1808 1 1808 0 Ss+ ttyin 0xc970de70 getty 1791 1 1791 0 Ss select 0xcc0792e4 inetd 1766 1 1766 0 LLs *page loc 0xcacfa280 watchdogd 1256 1252 1256 0 S+ wait 0xcc03d000 sh 1252 1249 1252 0 S+ wait 0xcca06598 bash 1249 1233 1249 0 S+ pause 0xcc08fb88 csh 1233 1228 1233 1001 S+ wait 0xcc08f2cc su 1228 1227 1228 1001 Ss+ wait 0xcc375598 bash 1227 1225 1225 1001 S select 0xcac866a4 sshd 1225 1148 1225 0 Ss select 0xcbfa4ba4 sshd 1209 1206 1206 1001 S piperd 0xcc0f8320 awk 1208 1206 1206 1001 S wait 0xcc3e4598 sh 1207 1202 1207 1001 Ss+ select 0xcac868e4 top 1206 1204 1206 1001 Ss wait 0xcc374b30 sh 1205 1203 1205 1001 Ds ufs 0xcc9905c8 tail 1204 1196 1196 1001 S select 0xcc0786a4 sshd 1203 1198 1198 1001 S select 0xcbfa4a24 sshd 1202 1197 1197 1001 S select 0xcac868a4 sshd 1198 1148 1198 0 Ss select 0xcbfa43e4 sshd 1197 1148 1197 0 Ss select 0xcc078e64 sshd 1196 1148 1196 0 Ss select 0xcc7b72a4 sshd 1159 1 1159 0 DLs pfault 0xc130cc7c cron 1154 1 1154 25 RLs sendmail 1151 1 1151 0 Rs sendmail 1148 1 1148 0 Ss select 0xcc0793e4 sshd 1116 1 1116 0 Ss select 0xcc7b7424 moused 1091 1 1091 0 Rs ntpd 1036 1031 1031 0 S (threaded) nfsd 100125 S rpcsvc 0xcc03ae50 nfsd: service 100124 S rpcsvc 0xcc03ae90 nfsd: service 100123 S rpcsvc 0xcc509010 nfsd: service 100082 S rpcsvc 0xcc741150 nfsd: master 1031 1 1031 0 Ss select 0xcac79164 nfsd 1029 1025 1025 0 S select 0xcac7ae64 nfsuserd 1028 1025 1025 0 S select 0xcc038564 nfsuserd 1027 1025 1025 0 S select 0xcac7ada4 nfsuserd 1026 1025 1025 0 S select 0xcc0385a4 nfsuserd 1025 1 1025 0 Ss pause 0xcbedc5f0 nfsuserd 1009 1 1009 0 Ss select 0xcc079324 mountd 976 1 976 0 Ss select 0xcc0790a4 rpcbind 961 1 961 0 Rs CPU 3 syslogd 838 1 838 0 Ss select 0xcbfa4464 devd 19 0 0 0 RL [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc13048b4 [syncer] 16 0 0 0 DL psleep 0xc1304548 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e468 [pagezero] 8 0 0 0 DL psleep 0xc130e048 [vmdaemon] 7 0 0 0 RL CPU 2 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151054 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc1307dd8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 LL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 L *ehci0 0xc95a0300 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d4e4 [yarrow] 13 0 0 0 DL (threaded) [geom] 100016 D - 0xc118a5a8 [g_down] 100015 D - 0xc118a5a4 [g_up] 100014 D - 0xc118a59c [g_event] 12 0 0 0 RL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 RunQ [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 CanRun [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 CanRun [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130bd00 [audit] 0 0 0 0 DLs (threaded) [kernel] 100069 D - 0xc118d4e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118a760 [swapper] db:0:ps> allt Tracing command bash pid 3212 tid 100197 td 0xcca6e000 sched_switch(cca6e000,0,104,1a3,2fa079fb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cca6e000,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(ccd01940,1e,0,0,f41e8b64,...) at vm_waitpfault+0x75 vm_fault_hold(cc7982b8,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc7982b8,bfbfe000,2,0,bfbfe7dc,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,cca6e000,cca2f598,cca2f598,...) at trap_pfault+0x1b9 trap(f41e8d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x806f5ea, esp = 0xbfbfe7e0, ebp = 0xbfbfe828 --- Tracing command sendmail pid 3211 tid 100149 td 0xcc9c2b80 sched_switch(cc9c2b80,0,104,1a3,4dbc0127,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cc9c2b80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(cc5be534,28,0,0,f4141b64,...) at vm_waitpfault+0x75 vm_fault_hold(c95e5910,811c000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(c95e5910,811c000,2,0,811c750,...) at vm_fault+0x8b trap_pfault(cca43000,0,f4141c74,c0aae2dc,cca122cc,...) at trap_pfault+0x1b9 trap(f4141d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a20ec, esp = 0xbfbfcf10, ebp = 0xbfbfcf78 --- Tracing command cron pid 3210 tid 100169 td 0xcc0938a0 sched_switch(cc0938a0,0,103,19e,2fc21301,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,780,...) at mi_switch+0x1ff turnstile_wait(cc0e3580,cbee88a0,0,1c5,3,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c1bb50e8,cc0938a0,0,c100e17f,5ae,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c1bb50e8,0,c100e17f,5ae,c596e000,...) at _mtx_lock_flags+0x10f _vm_map_lock(c1bb508c,c100e17f,5ae,c0df5b68,f417d7f4,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,f417d82c,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd05c7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,f417d893,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d144,871,c0a74f36,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,0,c1bb6a08,f417d940,c0d0015f,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,0,101,969,104,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,c1010cdf,0,...) at uma_zalloc_arg+0x59f vm_radix_insert(cca0d1ec,1e,0,c3c93f40,c100f607,...) at vm_radix_insert+0x8f vm_page_insert(c3c93f40,cca0d1bc,1e,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(cca0d1bc,1e,0,0,f417db64,...) at vm_page_alloc+0x51c vm_fault_hold(cc7420e8,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc7420e8,bfbfe000,2,0,bfbfec10,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,cc0938a0,cca2e864,cca2e864,...) at trap_pfault+0x1b9 trap(f417dd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c0e7, esp = 0xbfbfec10, ebp = 0xbfbfec48 --- Tracing command watchdogd pid 3209 tid 100178 td 0xcca43000 sched_switch(cca43000,0,104,1a3,2f9c04b6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cca43000,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(ceb7bde0,1e,0,0,f4198b64,...) at vm_waitpfault+0x75 vm_fault_hold(cc0d4cb0,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc0d4cb0,bfbfe000,2,0,bfbfec38,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,cca43000,cca2cb30,cca2cb30,...) at trap_pfault+0x1b9 trap(f4198d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bcea6, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command awk pid 3208 tid 100175 td 0xcca438a0 sched_switch(cca438a0,0,104,1a3,43158adf,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cca438a0,0,c0fd7fa0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cfa9b7d0,5c,c0fda927,100,0,...) at sleepq_wait_sig+0x17 _sleep(cfa9b7d0,cfa9b948,15c,c0fda927,0,...) at _sleep+0x385 pipe_read(cdee45b0,f418fc3c,cc037d00,0,cca438a0,...) at pipe_read+0x427 dofileread(f418fc3c,ffffffff,ffffffff,0,f418fc18,...) at dofileread+0x9e kern_readv(cca438a0,0,f418fc3c,f418fc5c,1,...) at kern_readv+0x68 sys_read(cca438a0,f418fcec,c1025a7c,c0fdae1a,c1027030,...) at sys_read+0x4f syscall(f418fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281aced3, esp = 0xbfbfea8c, ebp = 0xbfbfeaa8 --- Tracing command sh pid 3207 tid 100153 td 0xcca092e0 cpustop_handler(1,f414d794,c0e00b03,c93dbc80,f414d6e4,...) at cpustop_handler+0x26 ipi_nmi_handler(c93dbc80,f414d6e4,c0a742f3,c93dbc80,cca052cc,...) at ipi_nmi_handler+0x35 trap(f414d7a0) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0d15149, esp = 0xf414d7e0, ebp = 0xf414d7fc --- vm_page_rename(c30e7900,d01b5b90,0,0,0,...) at vm_page_rename+0xf9 vm_object_backing_scan(ceb7c094,0,c100eff3,6c2,c0fd05c7,...) at vm_object_backing_scan+0x5ed vm_object_collapse(d01b5b90,0,c100eff3,253,246,...) at vm_object_collapse+0x167 vm_object_deallocate(cc956d4c,ccb3b870,0,cdbde4c8,cca092e0,...) at vm_object_deallocate+0x5b9 vm_map_entry_deallocate(c12d6288,c95e4570,0,0,c95e4570,...) at vm_map_entry_deallocate+0x21 vm_map_process_deferred(c95e45b8,c100e17f,b35,c95a400c,f414d974,...) at vm_map_process_deferred+0x76 _vm_map_unlock(c95e4570,c100e17f,b35,0,c95a4000,...) at _vm_map_unlock+0x4a vm_map_remove(c95e4570,1000,bfc00000,411,c1113920,...) at vm_map_remove+0x6b exec_new_vmspace(f414db80,c1142d80,8,4,c0fd05c7,...) at exec_new_vmspace+0x1bb exec_elf32_imgact(f414db80,c0fcc280,c0fcc20e,171,ec521000,...) at exec_elf32_imgact+0x441 kern_execve(cca092e0,f414dc38,0,2880fcc8,2880fd0c,ec521000,ec52100d,ec521055,ec5211b9,ec521000,0,3fe54,2,e,0) at kern_execve+0x5c2 sys_execve(cca092e0,f414dcec,c1025a7c,c0fbc9b1,21e,...) at sys_execve+0x4c syscall(f414dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (59, FreeBSD ELF32, sys_execve), eip = 0x2815074b, esp = 0xbfbfe75c, ebp = 0xbfbfe8a8 --- Tracing command sed pid 3206 tid 100201 td 0xcca6c5c0 sched_switch(cca6c5c0,0,104,1a3,440018d7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cca6c5c0,0,c0fd7fa0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc081190,5c,c0fda927,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc081190,cc081308,15c,c0fda927,0,...) at _sleep+0x385 pipe_read(cc07fe38,f41f7c3c,cc037d00,0,cca6c5c0,...) at pipe_read+0x427 dofileread(f41f7c3c,ffffffff,ffffffff,0,f41f7c18,...) at dofileread+0x9e kern_readv(cca6c5c0,0,f41f7c3c,f41f7c5c,1,...) at kern_readv+0x68 sys_read(cca6c5c0,f41f7cec,c1025a7c,c0fdae1a,c1027030,...) at sys_read+0x4f syscall(f41f7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x2817aed3, esp = 0xbfbfea0c, ebp = 0xbfbfea28 --- Tracing command vmstat pid 3205 tid 100162 td 0xcca2bb80 sched_switch(cca2bb80,0,103,19e,2f9b40e0,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,b10,...) at mi_switch+0x1ff turnstile_wait(cc0e3600,cc0e18a0,0,1c5,c0a74727,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c1bb7608,cca2bb80,0,c100d144,969,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c1bb7608,0,c100d144,969,c6b5d720,...) at _mtx_lock_flags+0x10f uma_zalloc_arg(c1bb69a0,0,101,0,3,...) at uma_zalloc_arg+0x4bf vm_radix_insert(cc12a43c,430,0,c5027680,0,...) at vm_radix_insert+0x1e9 vm_page_insert(c5027680,cc12a40c,430,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(cc12a40c,430,0,40,f4168b64,...) at vm_page_alloc+0x51c vm_fault_hold(cd6dd000,285e7000,1,0,0,...) at vm_fault_hold+0x60a vm_fault(cd6dd000,285e7000,1,0,285e7000,...) at vm_fault+0x8b trap_pfault(c118c410,0,c10259d2,304,cca0f000,...) at trap_pfault+0x1b9 trap(f4168d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x2810e4e3, esp = 0xbfbfde10, ebp = 0xbfbfde68 --- Tracing command egrep pid 3157 tid 100140 td 0xcc03eb80 sched_switch(cc03eb80,0,104,1a3,d4e2c1a2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cc03eb80,0,c0fd7fa0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cdbad960,5c,c0fda927,100,0,...) at sleepq_wait_sig+0x17 _sleep(cdbad960,cdbadad8,15c,c0fda927,0,...) at _sleep+0x385 pipe_read(cec60bd0,f4126c3c,c95da580,0,cc03eb80,...) at pipe_read+0x427 dofileread(f4126c3c,ffffffff,ffffffff,0,f4126c18,...) at dofileread+0x9e kern_readv(cc03eb80,0,f4126c3c,f4126c5c,1,...) at kern_readv+0x68 sys_read(cc03eb80,f4126cec,c1025a7c,c0fdae1a,c1027030,...) at sys_read+0x4f syscall(f4126d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281baed3, esp = 0xbfbfe81c, ebp = 0xbfbfe8a8 --- Tracing command dd pid 3156 tid 100204 td 0xcca44b80 sched_switch(cca44b80,0,103,19e,2f0b6a60,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,768,...) at mi_switch+0x1ff turnstile_wait(cacfa800,cc9c2b80,0,1c5,c12d61d8,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130cf00,cca44b80,0,c100f607,75c,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130cf00,0,c100f607,75c,c4d12740,...) at _mtx_lock_flags+0x10f vm_page_free_toq(c4d12740,f42007ac,c0d17263,c4d12740,0,...) at vm_page_free_toq+0x17e vm_page_free(c4d12740,0,c100f607,4fe,20,...) at vm_page_free+0x15 vm_page_alloc(ccae5534,70400,0,40222,40222,...) at vm_page_alloc+0x583 vm_page_grab(ccae5534,70400,0,412a2,c12d6470,...) at vm_page_grab+0x152 allocbuf(e32d7120,4000,c0fdf52b,b00,ffffffe0,...) at allocbuf+0x672 getblk(ccb29984,1c100,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(ccb29984,70400000,0,4000,c95da580,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f4200ba4,c102d163,ccb29984,ccb29984,f4200b5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c11375a0,f4200ba4,ccb29984,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cc053a10,f4200c3c,c95da580,0,cca44b80,...) at vn_write+0x2bc dofilewrite(f4200c3c,ffffffff,ffffffff,0,f4200c18,...) at dofilewrite+0x95 kern_writev(cca44b80,4,f4200c3c,f4200c5c,1,...) at kern_writev+0x68 sys_write(cca44b80,f4200cec,c1025a7c,c0e3210f,c96ea8a0,...) at sys_write+0x4f syscall(f4200d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28177eb3, esp = 0xbfbfe9bc, ebp = 0xbfbfe9e8 --- Tracing command bash pid 2439 tid 100145 td 0xcca098a0 cpustop_handler(0,f41358e4,c0e00b03,c1bb7608,8,...) at cpustop_handler+0x26 ipi_nmi_handler(c1bb7608,8,c100d144,359,cca05864,...) at ipi_nmi_handler+0x35 trap(f41358f0) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a73ea0, esp = 0xf4135930, ebp = 0xf4135950 --- _mtx_assert(cc0e18a0,0,c0fd7fa0,35d,0,...) at _mtx_assert sleepq_broadcast(c130cc7c,0,0,0,c33b9b40,...) at sleepq_broadcast+0xee wakeup(c130cc7c,4,c100f607,706,c33b9b40,...) at wakeup+0x37 vm_page_free_toq(c33b9b40,f41359e8,c0d17263,c33b9b40,0,...) at vm_page_free_toq+0x248 vm_page_free(c33b9b40,0,c100f607,4fe,0,...) at vm_page_free+0x15 vm_page_alloc(cc9fed4c,1,0,0,f4135b64,...) at vm_page_alloc+0x583 vm_fault_hold(cc742cb0,80e8000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc742cb0,80e8000,2,0,80e8ed0,...) at vm_fault+0x8b trap_pfault(cca05864,f4135c64,c0acc418,cca098a0,cca05864,...) at trap_pfault+0x1b9 trap(f4135d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x806f7e5, esp = 0xbfbfe7e0, ebp = 0xbfbfe828 --- Tracing command sshd pid 2438 tid 100193 td 0xcca6eb80 sched_switch(cca6eb80,0,104,1a3,fd301c48,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cca6eb80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc079650,0,c0fd05c7,cca6eb80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc079664,0,f41dca50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc079664,cc079650,c0fda6f8,6ae,cc08d770,...) at _cv_wait_sig+0x260 seltdwait(cc08d770,58,cc076300,cca6eb80,58,...) at seltdwait+0xa2 kern_select(cca6eb80,c,28c0c170,28c0c1b0,0,0,20,cca532cc,0) at kern_select+0x655 sys_select(cca6eb80,f41dccec,c1025a7c,c0fdb025,f41dccb4,...) at sys_select+0x66 syscall(f41dcd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 2436 tid 100199 td 0xcca6cb80 sched_switch(cca6cb80,0,104,1a3,2629f048,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cca6cb80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86490,0,c0fd05c7,cca6cb80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac864a4,0,f41f1ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac864a4,cac86490,c0fda6f8,6ae,f41f1b3c,...) at _cv_wait_sig+0x260 seltdwait(cca85d2c,c0fda6f8,5ab,cca6cb80,696,...) at seltdwait+0xa2 sys_poll(cca6cb80,f41f1cec,c1025a7c,c0fbc9b1,c1027030,...) at sys_poll+0x321 syscall(f41f1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sh pid 1935 tid 100151 td 0xcc9c25c0 sched_switch(cc9c25c0,0,104,1a3,ca26f284,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc9c25c0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cca0fb30,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cca0fb30,cca0fbb8,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc9c25c0,ffffffff,f4147c58,2,0,...) at kern_wait+0x597 sys_wait4(cc9c25c0,f4147cec,c1025a7c,c0fdaeb7,c1027030,...) at sys_wait4+0x3b syscall(f4147d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe78c, ebp = 0xbfbfe7a8 --- Tracing command getty pid 1816 tid 100144 td 0xcc03e000 sched_switch(cc03e000,0,104,1a3,8b39fcb1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc03e000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc03e000,cc03e000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33870,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33870,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33800,cae33870,f4132c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33800,f4132c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27100,f4132c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0a5e38,f4132c3c,c95d9b80,0,cc03e000,...) at devfs_read_f+0x7e dofileread(f4132c3c,ffffffff,ffffffff,0,f4132c18,...) at dofileread+0x9e kern_readv(cc03e000,0,f4132c3c,f4132c5c,1,...) at kern_readv+0x68 sys_read(cc03e000,f4132cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4132d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1815 tid 100180 td 0xcca428a0 sched_switch(cca428a0,0,104,1a3,891fe555,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cca428a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cca428a0,cca428a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33a70,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33a70,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33a00,cae33a70,f419ec3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33a00,f419ec3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27000,f419ec3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08d380,f419ec3c,c95d9b80,0,cca428a0,...) at devfs_read_f+0x7e dofileread(f419ec3c,ffffffff,ffffffff,0,f419ec18,...) at dofileread+0x9e kern_readv(cca428a0,0,f419ec3c,f419ec5c,1,...) at kern_readv+0x68 sys_read(cca428a0,f419ecec,c1025a7c,c0e36ed2,cc9c55c0,...) at sys_read+0x4f syscall(f419ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1814 tid 100186 td 0xcca075c0 sched_switch(cca075c0,0,104,1a3,88e7f353,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cca075c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cca075c0,cca075c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d270,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d270,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d200,c970d270,f41b7c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d200,f41b7c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0ce00,f41b7c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc9a5b98,f41b7c3c,c95d9b80,0,cca075c0,...) at devfs_read_f+0x7e dofileread(f41b7c3c,ffffffff,ffffffff,0,f41b7c18,...) at dofileread+0x9e kern_readv(cca075c0,0,f41b7c3c,f41b7c5c,1,...) at kern_readv+0x68 sys_read(cca075c0,f41b7cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f41b7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1813 tid 100117 td 0xcae265c0 sched_switch(cae265c0,0,104,1a3,89e8bc14,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae265c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cae265c0,cae265c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d470,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d470,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d400,c970d470,f4070c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d400,f4070c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cd00,f4070c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc9a5770,f4070c3c,c95d9b80,0,cae265c0,...) at devfs_read_f+0x7e dofileread(f4070c3c,ffffffff,ffffffff,0,f4070c18,...) at dofileread+0x9e kern_readv(cae265c0,0,f4070c3c,f4070c5c,1,...) at kern_readv+0x68 sys_read(cae265c0,f4070cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4070d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1812 tid 100129 td 0xcc9c55c0 sched_switch(cc9c55c0,0,104,1a3,890a6f7e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9c55c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc9c55c0,cc9c55c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d670,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d670,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d600,c970d670,f4105c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d600,f4105c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cc00,f4105c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0a5ab8,f4105c3c,c95d9b80,0,cc9c55c0,...) at devfs_read_f+0x7e dofileread(f4105c3c,ffffffff,ffffffff,0,f4105c18,...) at dofileread+0x9e kern_readv(cc9c55c0,0,f4105c3c,f4105c5c,1,...) at kern_readv+0x68 sys_read(cc9c55c0,f4105cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4105d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1811 tid 100198 td 0xcca28b80 sched_switch(cca28b80,0,104,1a3,85aaa7f7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cca28b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cca28b80,cca28b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d870,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d870,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d800,c970d870,f41eec3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d800,f41eec3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c900,f41eec3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc053150,f41eec3c,c95d9b80,0,cca28b80,...) at devfs_read_f+0x7e dofileread(f41eec3c,ffffffff,ffffffff,0,f41eec18,...) at dofileread+0x9e kern_readv(cca28b80,0,f41eec3c,f41eec5c,1,...) at kern_readv+0x68 sys_read(cca28b80,f41eecec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f41eed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1810 tid 100182 td 0xcca422e0 sched_switch(cca422e0,0,104,1a3,85bb3084,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cca422e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cca422e0,cca422e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970da70,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970da70,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970da00,c970da70,f41a4c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970da00,f41a4c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c800,f41a4c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cca690a8,f41a4c3c,c95d9b80,0,cca422e0,...) at devfs_read_f+0x7e dofileread(f41a4c3c,ffffffff,ffffffff,0,f41a4c18,...) at dofileread+0x9e kern_readv(cca422e0,0,f41a4c3c,f41a4c5c,1,...) at kern_readv+0x68 sys_read(cca422e0,f41a4cec,c1025a7c,c0e36ed2,cca28b80,...) at sys_read+0x4f syscall(f41a4d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1809 tid 100139 td 0xcc044000 sched_switch(cc044000,0,104,1a3,85bd7e7e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc044000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc044000,cc044000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970dc70,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970dc70,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970dc00,c970dc70,f4123c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970dc00,f4123c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c700,f4123c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0a5038,f4123c3c,c95d9b80,0,cc044000,...) at devfs_read_f+0x7e dofileread(f4123c3c,ffffffff,ffffffff,0,f4123c18,...) at dofileread+0x9e kern_readv(cc044000,0,f4123c3c,f4123c5c,1,...) at kern_readv+0x68 sys_read(cc044000,f4123cec,c1025a7c,c0e36ed2,f4123cb4,...) at sys_read+0x4f syscall(f4123d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1808 tid 100110 td 0xcbee6b80 sched_switch(cbee6b80,0,104,1a3,79f459a4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee6b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,c970de04,0,c0fd05c7,cbee6b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970de70,0,f405aae8,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970de70,c970de04,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970de00,c970de70,f405ac3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970de00,f405ac3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(c95c6100,f405ac3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08d888,f405ac3c,c95d9b80,0,cbee6b80,...) at devfs_read_f+0x7e dofileread(f405ac3c,ffffffff,ffffffff,0,f405ac18,...) at dofileread+0x9e kern_readv(cbee6b80,0,f405ac3c,f405ac5c,1,...) at kern_readv+0x68 sys_read(cbee6b80,f405acec,c1025a7c,c0e36ed2,cca075c0,...) at sys_read+0x4f syscall(f405ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command inetd pid 1791 tid 100098 td 0xcc0442e0 sched_switch(cc0442e0,0,104,1a3,6294f9aa,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0442e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc0792d0,0,c0fd05c7,cc0442e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0792e4,0,f4019a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0792e4,cc0792d0,c0fda6f8,6ae,cc07f850,...) at _cv_wait_sig+0x260 seltdwait(cc07f850,58,c95d9b80,cc0442e0,202,...) at seltdwait+0xa2 kern_select(cc0442e0,8,bfbfdf90,0,0,0,20,cbedc2cc,4) at kern_select+0x655 sys_select(cc0442e0,f4019cec,c1025a7c,c0fdaf24,c1027030,...) at sys_select+0x66 syscall(f4019d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2819be53, esp = 0xbfbfdf4c, ebp = 0xbfbfedd8 --- Tracing command watchdogd pid 1766 tid 100156 td 0xcc9c22e0 sched_switch(cc9c22e0,0,103,19e,4dbf1bf0,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,858,...) at mi_switch+0x1ff turnstile_wait(cacfa280,cbee88a0,0,1c5,101,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d900,cc9c22e0,0,c100f607,4fe,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d900,0,c100f607,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(ceb7b818,1e,0,0,f4156b64,...) at vm_page_alloc+0x565 vm_fault_hold(cc4223a0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc4223a0,bfbfe000,2,0,bfbfec3c,...) at vm_fault+0x8b trap_pfault(cca0f864,f4156c64,c0acc418,cc9c22e0,cca0f864,...) at trap_pfault+0x1b9 trap(f4156d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bce00, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command sh pid 1256 tid 100092 td 0xcc045000 sched_switch(cc045000,0,104,1a3,81844e43,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc045000,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc03d000,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc03d000,cc03d088,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc045000,ffffffff,f3fffc58,2,0,...) at kern_wait+0x597 sys_wait4(cc045000,f3fffcec,c1025a7c,c0fdaeb7,c1027030,...) at sys_wait4+0x3b syscall(f3fffd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe6ec, ebp = 0xbfbfe708 --- Tracing command bash pid 1252 tid 100141 td 0xcc03e8a0 sched_switch(cc03e8a0,0,104,1a3,cf5f28d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc03e8a0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cca06598,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cca06598,cca06620,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc03e8a0,ffffffff,f4129c58,6,0,...) at kern_wait+0x597 sys_wait4(cc03e8a0,f4129cec,c1025a7c,c0fdadd7,c96eb2e0,...) at sys_wait4+0x3b syscall(f4129d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe76c, ebp = 0xbfbfe788 --- Tracing command csh pid 1249 tid 100099 td 0xcc097000 sched_switch(cc097000,0,104,1a3,d2026944,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc097000,0,c0fd7fa0,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08fb88,74,c0e58cee,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc08fb88,cc08fbb8,174,c0e58cee,0,...) at _sleep+0x385 kern_sigsuspend(cc097000,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc097000,f4020cec,c1025a7c,c0fdadd7,c1027030,...) at sys_sigsuspend+0x4d syscall(f4020d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x2816fd7b, esp = 0xbfbfe05c, ebp = 0xbfbfe0c8 --- Tracing command su pid 1233 tid 100118 td 0xcc0965c0 sched_switch(cc0965c0,0,104,1a3,1128674c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0965c0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc08f2cc,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc08f2cc,cc08f354,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc0965c0,4e1,f4074c58,2,0,...) at kern_wait+0x597 sys_wait4(cc0965c0,f4074cec,c1025a7c,c0fdb6ee,c1027030,...) at sys_wait4+0x3b syscall(f4074d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2810b78b, esp = 0xbfbfe5ac, ebp = 0xbfbfe5c8 --- Tracing command bash pid 1228 tid 100109 td 0xcbee8000 sched_switch(cbee8000,0,104,1a3,fa901fc1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cbee8000,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc375598,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc375598,cc375620,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cbee8000,ffffffff,f4054c58,6,0,...) at kern_wait+0x597 sys_wait4(cbee8000,f4054cec,c1025a7c,c0fdadd7,c119b340,...) at sys_wait4+0x3b syscall(f4054d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command sshd pid 1227 tid 100137 td 0xcc9c3b80 sched_switch(cc9c3b80,0,104,1a3,2677bf44,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9c3b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86690,0,c0fd05c7,cc9c3b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac866a4,0,f411da50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac866a4,cac86690,c0fda6f8,6ae,cc0a56c8,...) at _cv_wait_sig+0x260 seltdwait(cc0a56c8,58,cbfb2e00,cc9c3b80,0,...) at seltdwait+0xa2 kern_select(cc9c3b80,c,28c0c170,28c0c1b0,0,0,20,cc375b30,0) at kern_select+0x655 sys_select(cc9c3b80,f411dcec,c1025a7c,c0fdb025,f411dcb4,...) at sys_select+0x66 syscall(f411dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1225 tid 100119 td 0xcae262e0 sched_switch(cae262e0,0,104,1a3,193c4d01,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae262e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfa4b90,0,c0fd05c7,cae262e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa4ba4,0,f4078ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa4ba4,cbfa4b90,c0fda6f8,6ae,f4078b3c,...) at _cv_wait_sig+0x260 seltdwait(cc7d7b2c,c0fda6f8,5ab,cae262e0,696,...) at seltdwait+0xa2 sys_poll(cae262e0,f4078cec,c1025a7c,c0fbc9b1,c1027030,...) at sys_poll+0x321 syscall(f4078d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command awk pid 1209 tid 100135 td 0xcc9c42e0 sched_switch(cc9c42e0,0,104,1a3,af4c8550,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cc9c42e0,0,c0fd7fa0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0f8320,5c,c0fda927,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0f8320,cc0f8498,15c,c0fda927,0,...) at _sleep+0x385 pipe_read(cc053268,f4117c3c,cc037d00,0,cc9c42e0,...) at pipe_read+0x427 dofileread(f4117c3c,ffffffff,ffffffff,0,f4117c18,...) at dofileread+0x9e kern_readv(cc9c42e0,0,f4117c3c,f4117c5c,1,...) at kern_readv+0x68 sys_read(cc9c42e0,f4117cec,c1025a7c,c0e3210f,c119b340,...) at sys_read+0x4f syscall(f4117d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281aced3, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command sh pid 1208 tid 100134 td 0xcc9c45c0 sched_switch(cc9c45c0,0,104,1a3,3f82d9da,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc9c45c0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc3e4598,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc3e4598,cc3e4620,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc9c45c0,ffffffff,f4114c58,2,0,...) at kern_wait+0x597 sys_wait4(cc9c45c0,f4114cec,c1025a7c,c0fdaeb7,cca092e0,...) at sys_wait4+0x3b syscall(f4114d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe9bc, ebp = 0xbfbfe9d8 --- Tracing command top pid 1207 tid 100102 td 0xcc0e45c0 sched_switch(cc0e45c0,0,104,1a3,1b74e059,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0e45c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4032a14,c0a7478a,cac868d0,0,cc0e45c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac868e4,0,f4032a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac868e4,cac868d0,3e9,6ae,cc07f498,...) at _cv_timedwait_sig+0x270 seltdwait(f4032c08,f4032c10,cc037000,cc0e45c0,61622025,...) at seltdwait+0x8a kern_select(cc0e45c0,2,bfbfeb9c,0,0,f4032c54,20,1,0) at kern_select+0x655 sys_select(cc0e45c0,f4032cec,c1025a7c,c0fa5642,f4032cb4,...) at sys_select+0x66 syscall(f4032d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281e1e53, esp = 0xbfbfeb3c, ebp = 0xbfbfec98 --- Tracing command sh pid 1206 tid 100112 td 0xcbee65c0 sched_switch(cbee65c0,0,104,1a3,790dd3d7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cbee65c0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc374b30,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc374b30,cc374bb8,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cbee65c0,ffffffff,f4061c58,2,0,...) at kern_wait+0x597 sys_wait4(cbee65c0,f4061cec,c1025a7c,c0fdaeb7,c1027030,...) at sys_wait4+0x3b syscall(f4061d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfeb0c, ebp = 0xbfbfeb28 --- Tracing command tail pid 1205 tid 100126 td 0xcc0988a0 sched_switch(cc0988a0,0,104,1a3,2fb5e345,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cc0988a0,0,c0fd7fa0,268,60,...) at sleepq_switch+0x162 sleepq_wait(cc9905c8,60,c0fc2564,4,1,...) at sleepq_wait+0x63 __lockmgr_args(cc9905c8,200400,cc990634,0,0,...) at __lockmgr_args+0x593 ffs_lock(f40fcb48,0,f40fcb3c,c12d6a18,cc098950,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11375a0,f40fcb48,c959c7e0,c1149700,cc990570,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc990570,200400,c0fe39b0,232,0,...) at _vn_lock+0x78 vn_read(cc08d1f8,f40fcc3c,cc076c00,0,cc0988a0,...) at vn_read+0x456 dofileread(f40fcc3c,ffffffff,ffffffff,0,f40fcc18,...) at dofileread+0x9e kern_readv(cc0988a0,3,f40fcc3c,f40fcc5c,1,...) at kern_readv+0x68 sys_read(cc0988a0,f40fccec,c1025a7c,c0fdad8f,c1027030,...) at sys_read+0x4f syscall(f40fcd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28176ed3, esp = 0xbfbfea5c, ebp = 0xbfbfea78 --- Tracing command sshd pid 1204 tid 100127 td 0xcc0985c0 sched_switch(cc0985c0,0,104,1a3,5f4302b4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0985c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc078690,0,c0fd05c7,cc0985c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0786a4,0,f40ffa50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0786a4,cc078690,c0fda6f8,6ae,cc0a53f0,...) at _cv_wait_sig+0x260 seltdwait(cc0a53f0,58,c95dab00,cc0985c0,cc098670,...) at seltdwait+0xa2 kern_select(cc0985c0,c,28c0c170,28c0c1b0,0,0,20,cc995b30,0) at kern_select+0x655 sys_select(cc0985c0,f40ffcec,c1025a7c,c0fdb025,cc9c52e0,...) at sys_select+0x66 syscall(f40ffd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1203 tid 100133 td 0xcc9c48a0 sched_switch(cc9c48a0,0,104,1a3,2fb59a8f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9c48a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfa4a10,0,c0fd05c7,cc9c48a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa4a24,0,f4111a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa4a24,cbfa4a10,c0fda6f8,6ae,cc053e38,...) at _cv_wait_sig+0x260 seltdwait(cc053e38,58,cc037e80,cc9c48a0,cc9c4950,...) at seltdwait+0xa2 kern_select(cc9c48a0,c,28c0c170,28c0c1b0,0,0,20,cc3e4864,0) at kern_select+0x655 sys_select(cc9c48a0,f4111cec,c1025a7c,c0fdb025,c1027030,...) at sys_select+0x66 syscall(f4111d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1202 tid 100087 td 0xcc0458a0 sched_switch(cc0458a0,0,104,1a3,1b7751b4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0458a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86890,0,c0fd05c7,cc0458a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac868a4,0,f3fe9a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac868a4,cac86890,c0fda6f8,6ae,cc08d3b8,...) at _cv_wait_sig+0x260 seltdwait(cc08d3b8,58,cc076980,cc0458a0,cc045950,...) at seltdwait+0xa2 kern_select(cc0458a0,a,28c0c170,28c0c1b0,0,0,20,cc03d864,0) at kern_select+0x655 sys_select(cc0458a0,f3fe9cec,c1025a7c,c0fdb025,f3fe9cb4,...) at sys_select+0x66 syscall(f3fe9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1198 tid 100132 td 0xcc9c4b80 sched_switch(cc9c4b80,0,104,1a3,486187d4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9c4b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfa43d0,0,c0fd05c7,cc9c4b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa43e4,0,f410eab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa43e4,cbfa43d0,c0fda6f8,6ae,f410eb3c,...) at _cv_wait_sig+0x260 seltdwait(cc4ef02c,c0fda6f8,5ab,cc9c4b80,0,...) at seltdwait+0xa2 sys_poll(cc9c4b80,f410ecec,c1025a7c,c0e3210f,c119c040,...) at sys_poll+0x321 syscall(f410ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1197 tid 100131 td 0xcc9c5000 sched_switch(cc9c5000,0,104,1a3,4a6f22fa,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9c5000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc078e50,0,c0fd05c7,cc9c5000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc078e64,0,f410bab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc078e64,cc078e50,c0fda6f8,6ae,f410bb3c,...) at _cv_wait_sig+0x260 seltdwait(cc420b2c,c0fda6f8,5ab,cc9c5000,0,...) at seltdwait+0xa2 sys_poll(cc9c5000,f410bcec,c1025a7c,c0e3210f,cc0e15c0,...) at sys_poll+0x321 syscall(f410bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1196 tid 100130 td 0xcc9c52e0 sched_switch(cc9c52e0,0,104,1a3,486321eb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc9c52e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc7b7290,0,c0fd05c7,cc9c52e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc7b72a4,0,f4108ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc7b72a4,cc7b7290,c0fda6f8,6ae,f4108b3c,...) at _cv_wait_sig+0x260 seltdwait(cc4ee92c,c0fda6f8,5ab,cc9c52e0,0,...) at seltdwait+0xa2 sys_poll(cc9c52e0,f4108cec,c1025a7c,c0e3210f,cbee68a0,...) at sys_poll+0x321 syscall(f4108d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command cron pid 1159 tid 100106 td 0xcc0e18a0 sched_switch(cc0e18a0,0,104,1a3,5e8a731e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cc0e18a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(cc956784,1e,0,0,f4043b64,...) at vm_waitpfault+0x75 vm_fault_hold(cc0d4d98,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc0d4d98,bfbfe000,2,0,bfbfec50,...) at vm_fault+0x8b trap_pfault(f4043c58,c0a8e090,c119c6f0,cc0e18a0,cc0dc2cc,...) at trap_pfault+0x1b9 trap(f4043d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c20c, esp = 0xbfbfec50, ebp = 0xbfbfec68 --- Tracing command sendmail pid 1154 tid 100122 td 0xcc3fd5c0 sched_switch(cc3fd5c0,0,104,1a3,1dba0095,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cc3fd5c0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(cc9fcde0,1c,0,0,f408db64,...) at vm_waitpfault+0x75 vm_fault_hold(cc7989f8,bfbfc000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc7989f8,bfbfc000,2,0,bfbfcf10,...) at vm_fault+0x8b trap_pfault(cc0e02cc,f408dc64,c0acc418,cc3fd5c0,cc0e02cc,...) at trap_pfault+0x1b9 trap(f408dd28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a23a1, esp = 0xbfbfcf10, ebp = 0xbfbfcf78 --- Tracing command sendmail pid 1151 tid 100100 td 0xcc0e48a0 sched_switch(cc0e48a0,0,602,19e,2fb13dbf,...) at sched_switch+0x413 mi_switch(602,0,c0fd2c5f,cd,c,...) at mi_switch+0x1ff critical_exit(cc0e48a0,0,3,c2472500,b,...) at critical_exit+0xb5 ipi_bitmap_handler(8,28,28,0,b,...) at ipi_bitmap_handler+0x82 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2f --- interrupt, eip = 0xc0d1514c, esp = 0xf402aa98, ebp = 0xf402aab4 --- vm_page_rename(c2472500,cc7d9de0,b,0,0,...) at vm_page_rename+0xfc vm_object_backing_scan(cc7d94a0,0,c100eff3,6c2,c100e17f,...) at vm_object_backing_scan+0x5ed vm_object_collapse(cc7d9de0,0,c100e17f,b91,28800000,...) at vm_object_collapse+0x167 vmspace_fork(cc4f1740,f402ac24,2,0,c0fd248f,...) at vmspace_fork+0x461 fork1(cc0e48a0,14,2,f402ac5c,0,...) at fork1+0x2cb sys_fork(cc0e48a0,f402acec,c1025a7c,c0fdb6ee,f402acb4,...) at sys_fork+0x39 syscall(f402ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (2, FreeBSD ELF32, sys_fork), eip = 0x2833b7ab, esp = 0xbfbfc39c, ebp = 0xbfbfc3b8 --- Tracing command sshd pid 1148 tid 100111 td 0xcbee68a0 sched_switch(cbee68a0,0,104,1a3,19771174,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee68a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc0793d0,0,c0fd05c7,cbee68a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0793e4,0,f405ea50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0793e4,cc0793d0,c0fda6f8,6ae,cc08dbd0,...) at _cv_wait_sig+0x260 seltdwait(cc08dbd0,58,c95dac80,cbee68a0,f405eb14,...) at seltdwait+0xa2 kern_select(cbee68a0,9,28c0c140,0,0,0,20,cc375000,a) at kern_select+0x655 sys_select(cbee68a0,f405ecec,c1025a7c,c0fbc9b1,c119b340,...) at sys_select+0x66 syscall(f405ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfdedc, ebp = 0xbfbfede8 --- Tracing command moused pid 1116 tid 100081 td 0xcbee82e0 sched_switch(cbee82e0,0,104,1a3,6eda7b4c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee82e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc7b7410,0,c0fd05c7,cbee82e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc7b7424,0,f3fcda50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc7b7424,cc7b7410,c0fda6f8,6ae,cc07f700,...) at _cv_wait_sig+0x260 seltdwait(cc07f700,58,c95d9b80,cbee82e0,f6,...) at seltdwait+0xa2 kern_select(cbee82e0,400,bfbfe9f4,0,0,0,20,cac0b864,1) at kern_select+0x655 sys_select(cbee82e0,f3fcdcec,c1025a7c,c0e36ed2,c1027030,...) at sys_select+0x66 syscall(f3fcdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281a8e53, esp = 0xbfbfe98c, ebp = 0xbfbfeb18 --- Tracing command ntpd pid 1091 tid 100104 td 0xcc0e4000 sched_switch(cc0e4000,0,103,19e,2f9da916,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,b10,...) at mi_switch+0x1ff turnstile_wait(cacfa280,cbee88a0,0,1c5,7,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c1bb7608,cc0e4000,0,c100d144,359,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c1bb7608,0,c100d144,359,c1bb69a0,...) at _mtx_lock_flags+0x10f keg_alloc_slab(c1bb7608,4,c100d144,871,c0a74f36,...) at keg_alloc_slab+0x147 keg_fetch_slab(c1b9e418,c1bb69a0,101,f403a6a4,c0cfdef5,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,0,101,9d6,0,...) at zone_fetch_slab+0x4c zone_alloc_item(c1bb7608,0,c100d144,860,104,...) at zone_alloc_item+0x45 uma_zalloc_arg(c1bb69a0,0,101,c95859f0,0,...) at uma_zalloc_arg+0x7b6 vm_radix_insert(d01b5e10,0,0,c2ee38c0,c100f607,...) at vm_radix_insert+0x8f vm_page_insert(c2ee38c0,d01b5de0,0,0,20,...) at vm_page_insert+0x1b5 vm_page_alloc(d01b5de0,0,0,10222,10222,...) at vm_page_alloc+0x51c vm_page_grab(d01b5de0,0,0,112a2,ccae40f8,...) at vm_page_grab+0x152 allocbuf(e31b44c0,800,c0fdf52b,b00,fffffffc,...) at allocbuf+0x672 getblk(ccae4000,0,0,800,0,...) at getblk+0x5e3 ffs_balloc_ufs2(ccae4000,0,0,8,c95d9b80,...) at ffs_balloc_ufs2+0x1231 ffs_write(f403aba4,c102d163,ccae4000,ccae4000,f403ab5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c11375a0,f403aba4,ccae4000,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cc0a5b60,f403ac3c,c95d9b80,0,cc0e4000,...) at vn_write+0x2bc dofilewrite(f403ac3c,ffffffff,ffffffff,0,f403ac18,...) at dofilewrite+0x95 kern_writev(cc0e4000,4,f403ac3c,f403ac5c,1,...) at kern_writev+0x68 sys_write(cc0e4000,f403acec,c1025a7c,c0fdae1a,e,...) at sys_write+0x4f syscall(f403ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28358eb3, esp = 0xbfbfea8c, ebp = 0xbfbfeaa8 --- Tracing command nfsd pid 1036 tid 100125 td 0xcc098b80 sched_switch(cc098b80,0,104,1a3,b5255f8a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc098b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40e1bdc,c0a7478a,cbc36e80,0,cc098b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc03ae50,0,f40e1c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc03ae50,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40e1d14,c0a56e68,cbc36e80,f40e1d28,c0fcc5b0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40e1d28,c0fcc5b0,3d8,cc092b30,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40e1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1036 tid 100124 td 0xcc0e1000 sched_switch(cc0e1000,0,104,1a3,b52651fc,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0e1000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40debdc,c0a7478a,cbc36e80,0,cc0e1000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc03ae90,0,f40dec18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc03ae90,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40ded14,c0a56e68,cbc36e80,f40ded28,c0fcc5b0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40ded28,c0fcc5b0,3d8,cc092b30,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40ded28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1036 tid 100123 td 0xcc0e12e0 sched_switch(cc0e12e0,0,104,1a3,b5251f3a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0e12e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40dbbdc,c0a7478a,cbc36e80,0,cc0e12e0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc509010,0,f40dbc18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc509010,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40dbd14,c0a56e68,cbc36e80,f40dbd28,c0fcc5b0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40dbd28,c0fcc5b0,3d8,cc092b30,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40dbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1036 tid 100082 td 0xcc098000 sched_switch(cc098000,0,104,1a3,b524dfa2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc098000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fd248c,c0a7478a,cbc36e80,0,cc098000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc741150,0,f3fd24c8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc741150,cbc36e80,1388,3b1,f3fd2510,...) at _cv_timedwait_sig+0x270 svc_run_internal(cc0981b0,14,c10027da,c0fbdebb,f3fd2808,...) at svc_run_internal+0x356 svc_run(cbc36e80,0,c0fbde96,19f,cc0980b0,...) at svc_run+0x7b nfsrvd_nfsd(cc098000,f3fd2808,c,ab,c0fd05c7,...) at nfsrvd_nfsd+0x1c3 nfssvc_nfsd(cc098000,f3fd2cec,0,cc098000,f3fd2d1c,...) at nfssvc_nfsd+0x177 sys_nfssvc(cc098000,f3fd2cec,c1025a7c,c0fdb2b4,c1027030,...) at sys_nfssvc+0xeb syscall(f3fd2d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (155, FreeBSD ELF32, sys_nfssvc), eip = 0x280bf9bb, esp = 0xbfbfe81c, ebp = 0xbfbfea88 --- Tracing command nfsd pid 1031 tid 100093 td 0xcc0e5000 sched_switch(cc0e5000,0,104,1a3,1bbb51f9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0e5000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac79150,0,c0fd05c7,cc0e5000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac79164,0,f4004a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac79164,cac79150,c0fda6f8,6ae,cc08dce8,...) at _cv_wait_sig+0x260 seltdwait(cc08dce8,58,c95d9b80,cc0e5000,0,...) at seltdwait+0xa2 kern_select(cc0e5000,7,bfbfeca4,0,0,0,20,cc0dd864,1) at kern_select+0x655 sys_select(cc0e5000,f4004cec,c1025a7c,c0fdb2b4,c1027030,...) at sys_select+0x66 syscall(f4004d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28176e53, esp = 0xbfbfea8c, ebp = 0xbfbfedd8 --- Tracing command nfsuserd pid 1029 tid 100120 td 0xcc3fd8a0 sched_switch(cc3fd8a0,0,104,1a3,8fd7330a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3fd8a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f407ba14,c0a7478a,cac7ae50,0,cc3fd8a0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac7ae64,0,f407ba50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac7ae64,cac7ae50,7531,6ae,cc07f0a8,...) at _cv_timedwait_sig+0x270 seltdwait(f407bc08,f407bc10,c95d9b80,cc3fd8a0,f407bab4,...) at seltdwait+0x8a kern_select(cc3fd8a0,4,bfbfebb4,0,0,f407bc54,20,1e,0) at kern_select+0x655 sys_select(cc3fd8a0,f407bcec,c1025a7c,c0fdb00e,f407bcb4,...) at sys_select+0x66 syscall(f407bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1028 tid 100101 td 0xcc096b80 sched_switch(cc096b80,0,104,1a3,8fd4829f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc096b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f402ea14,c0a7478a,cc038550,0,cc096b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc038564,0,f402ea50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc038564,cc038550,7531,6ae,cc07f0a8,...) at _cv_timedwait_sig+0x270 seltdwait(f402ec08,f402ec10,c95d9b80,cc096b80,f402eab4,...) at seltdwait+0x8a kern_select(cc096b80,4,bfbfebb4,0,0,f402ec54,20,1e,0) at kern_select+0x655 sys_select(cc096b80,f402ecec,c1025a7c,c0fdb00e,f402ecb4,...) at sys_select+0x66 syscall(f402ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1027 tid 100096 td 0xcc0448a0 sched_switch(cc0448a0,0,104,1a3,8fd56bb1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0448a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4011a14,c0a7478a,cac7ad90,0,cc0448a0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac7ada4,0,f4011a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac7ada4,cac7ad90,7531,6ae,cc07f0a8,...) at _cv_timedwait_sig+0x270 seltdwait(f4011c08,f4011c10,c95d9b80,cc0448a0,f4011ab4,...) at seltdwait+0x8a kern_select(cc0448a0,4,bfbfebb4,0,0,f4011c54,20,1e,0) at kern_select+0x655 sys_select(cc0448a0,f4011cec,c1025a7c,c0fdb00e,f4011cb4,...) at sys_select+0x66 syscall(f4011d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1026 tid 100080 td 0xcbee85c0 sched_switch(cbee85c0,0,104,1a3,8fba57da,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee85c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fc9a14,c0a7478a,cc038590,0,cbee85c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc0385a4,0,f3fc9a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc0385a4,cc038590,7531,6ae,cc07f0a8,...) at _cv_timedwait_sig+0x270 seltdwait(f3fc9c08,f3fc9c10,c95d9b80,cbee85c0,f3fc9ab4,...) at seltdwait+0x8a kern_select(cbee85c0,4,bfbfebb4,0,0,f3fc9c54,20,1e,0) at kern_select+0x655 sys_select(cbee85c0,f3fc9cec,c1025a7c,c0fdb00e,c119c040,...) at sys_select+0x66 syscall(f3fc9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1025 tid 100097 td 0xcc0445c0 sched_switch(cc0445c0,0,104,1a3,10d00fcb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc0445c0,0,c0fd7fa0,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbedc5f0,74,c0e58cee,100,0,...) at sleepq_wait_sig+0x17 _sleep(cbedc5f0,cbedc620,174,c0e58cee,0,...) at _sleep+0x385 kern_sigsuspend(cc0445c0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc0445c0,f4015cec,c1025a7c,c0fdb2b4,c119c040,...) at sys_sigsuspend+0x4d syscall(f4015d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x280bdd7b, esp = 0xbfbfec4c, ebp = 0xbfbfede8 --- Tracing command mountd pid 1009 tid 100121 td 0xcc0962e0 sched_switch(cc0962e0,0,104,1a3,1af62240,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0962e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc079310,0,c0fd05c7,cc0962e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc079324,0,f407ea50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc079324,cc079310,c0fda6f8,6ae,cc08db98,...) at _cv_wait_sig+0x260 seltdwait(cc08db98,58,c95d9b80,cc0962e0,f407eac0,...) at seltdwait+0xa2 kern_select(cc0962e0,9,bfbfecf4,0,0,0,20,cc08f000,bfbfecf4) at kern_select+0x655 sys_select(cc0962e0,f407ecec,c1025a7c,c0fbbc17,c1027030,...) at sys_select+0x66 syscall(f407ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de53, esp = 0xbfbfec8c, ebp = 0xbfbfedd8 --- Tracing command rpcbind pid 976 tid 100089 td 0xcc0e55c0 sched_switch(cc0e55c0,0,104,1a3,9ac67a9e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0e55c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3ff1a78,c0a7478a,cc079090,0,cc0e55c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc0790a4,0,f3ff1ab4,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc0790a4,cc079090,7531,6ae,f3ff1b6c,...) at _cv_timedwait_sig+0x270 seltdwait(f3ff1c3c,f3ff1c44,5ab,cc0e55c0,b,...) at seltdwait+0x8a sys_poll(cc0e55c0,f3ff1cec,c1025a7c,c0fdb00e,f3ff1cb4,...) at sys_poll+0x321 syscall(f3ff1d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2812c3ab, esp = 0xbfbfcb9c, ebp = 0xbfbfed68 --- Tracing command syslogd pid 961 tid 100108 td 0xcc0e15c0 cpustop_handler(3,f404f64c,c0e00b03,c119b9c0,cc0938a0,...) at cpustop_handler+0x26 ipi_nmi_handler(c119b9c0,cc0938a0,f404f590,c0a8e090,cc0dc000,...) at ipi_nmi_handler+0x35 trap(f404f658) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0df4fb8, esp = 0xf404f698, ebp = 0xf404f6b8 --- smp_tlb_shootdown(f404f6e4,c0df7aac,e53bc000,e53c0000,c9587b40,...) at smp_tlb_shootdown+0x98 smp_invlpg_range(e53bc000,e53c0000,c9587b40,cc0e1670,e53c0000,...) at smp_invlpg_range+0x1c pmap_invalidate_range(c132cdc0,e53bc000,e53c0000) at pmap_invalidate_range+0x6c pmap_qremove(e53bc000,4,ccb29a7c,4,e31b8de0,...) at pmap_qremove+0x54 vfs_vmio_release(c1304594,0,c0fdf52b,792,0,...) at vfs_vmio_release+0x25 getnewbuf(4000,4000,0,ad1,c12d75c8,...) at getnewbuf+0x334 getblk(ccae4570,ffd1dff4,ffffffff,4000,0,...) at getblk+0x485 breadn_flags(ccae4570,ffd1dff4,ffffffff,4000,0,...) at breadn_flags+0x43 ffs_balloc_ufs2(ccae4570,881c0000,b,4000,c9561b80,...) at ffs_balloc_ufs2+0x1775 ffs_copyonwrite(cc370414,e3115dc0,0,e3115dc0,cc990570,...) at ffs_copyonwrite+0x3f8 ffs_geom_strategy(cc37050c,e3115dc0,c11088e8,c12d75c8,cc990570,...) at ffs_geom_strategy+0xeb ufs_strategy(f404fad0,c102cf2b,c0dea8d1,e3115e20,cc0e15c0,...) at ufs_strategy+0xd3 VOP_STRATEGY_APV(c11375a0,f404fad0,c0fdf52b,c11498e0,cc990570,...) at VOP_STRATEGY_APV+0x119 bufstrategy(cc990668,e3115dc0,3a0,1000,0,...) at bufstrategy+0x61 bufwrite(e3115dc0,0,c0fdf52b,6d6,0,...) at bufwrite+0x159 vfs_bio_awrite(e3115dc0,0,c100b1f1,106,0,...) at vfs_bio_awrite+0x328 ffs_syncvnode(cc990570,1,0,c11375a0,cc990668,...) at ffs_syncvnode+0x3a5 ffs_fsync(f404fc40,c102c6e4,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c11375a0,f404fc40,c0fe37a7,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cc0e15c0,f404fcec,c1025a7c,c0fdbd8f,f404fcb4,...) at sys_fsync+0x1df syscall(f404fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x2818ad47, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- Tracing command devd pid 838 tid 100105 td 0xcc0e1b80 sched_switch(cc0e1b80,0,104,1a3,21ddff4c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0e1b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfa4450,0,c0fd05c7,cc0e1b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfa4464,0,f403ea50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfa4464,cbfa4450,c0fda6f8,6ae,cc07fa48,...) at _cv_wait_sig+0x260 seltdwait(cc07fa48,58,c95d9b80,cc0e1b80,c11088e8,...) at seltdwait+0xa2 kern_select(cc0e1b80,5,bfbfe950,0,0,0,20,cc0dc598,34) at kern_select+0x655 sys_select(cc0e1b80,f403ecec,c93e1c80,0,c1027030,...) at sys_select+0x66 syscall(f403ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x80893bb, esp = 0xbfbfe91c, ebp = 0xbfbfedf8 --- Tracing command softdepflush pid 19 tid 100078 td 0xcbee88a0 sched_switch(cbee88a0,0,602,19e,4dbee0cc,...) at sched_switch+0x413 mi_switch(602,0,c0fd2c5f,cd,cbee88a0,...) at mi_switch+0x1ff critical_exit(c11aa84c,efbe9884,c0a742f3,c11aa84c,4,...) at critical_exit+0xb5 spinlock_exit(c11aa84c,4,c0fd05c7,119,0,c0fd8990) at spinlock_exit+0x10 _mtx_unlock_spin_flags(c11aa84c,0,c0fd8990,260,efbe98bc,...) at _mtx_unlock_spin_flags+0x133 turnstile_chain_unlock(c130cf00,0,c12d79e8,c130cf00,c130cf00,...) at turnstile_chain_unlock+0x39 _mtx_unlock_sleep(c130cf00,0,c100f607,76a,0,...) at _mtx_unlock_sleep+0xb0 _mtx_unlock_flags(c130cf00,0,c100f607,76a,c3c93f80,...) at _mtx_unlock_flags+0x157 vm_page_free_toq(c3c93f80,efbe9938,c0d17263,c3c93f80,0,...) at vm_page_free_toq+0x26c vm_page_free(c3c93f80,0,c100f607,4fe,20,...) at vm_page_free+0x15 vm_page_alloc(cc072094,6a6a38,0,40222,40222,...) at vm_page_alloc+0x583 vm_page_grab(cc072094,6a6a38,0,412a2,c12d7a08,...) at vm_page_grab+0x152 allocbuf(e3295760,4000,c0fdf52b,b00,efbe99fc,...) at allocbuf+0x672 getblk(cc370414,35351c0,0,4000,0,...) at getblk+0x5e3 breadn_flags(cc370414,35351c0,0,4000,0,...) at breadn_flags+0x43 indir_trunc(fffcf7f4,ffffffff,c1007b8c,1e0e,0,...) at indir_trunc+0x114 indir_trunc(fffff7f3,ffffffff,c1007b8c,1cd0,c12d79e8,...) at indir_trunc+0x4ba handle_workitem_freeblocks(0,efbe9c70,2,6ba,efbe9c74,...) at handle_workitem_freeblocks+0x24f process_worklist_item(c130c580,0,c1007b8c,5e9,cbfba700,...) at process_worklist_item+0x2c2 softdep_process_worklist(cbfa62dc,0,c1007b8c,562,3e8,...) at softdep_process_worklist+0x84 softdep_flush(0,efbe9d28,c0fcc5b0,3d8,cbedb000,...) at softdep_flush+0x2a0 fork_exit(c0cd7e10,0,efbe9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe9d60, ebp = 0 --- Tracing command vnlru pid 18 tid 100077 td 0xcbee8b80 sched_switch(cbee8b80,0,104,1a3,dc315f62,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cbee8b80,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cbedb2cc,60,c0fe30f9,0,0,...) at sleepq_timedwait+0x6b _sleep(cbedb2cc,c1304874,260,c0fe30f9,3e8,...) at _sleep+0x36a vnlru_proc(0,efbe6d28,c0fcc5b0,3d8,cbedb2cc,...) at vnlru_proc+0xe7 fork_exit(c0b2b970,0,efbe6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe6d60, ebp = 0 --- Tracing command syncer pid 17 tid 100076 td 0xcacf98a0 sched_switch(cacf98a0,0,104,1a3,4db88693,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf98a0,0,c0fd7fa0,28b,cacf98a0,...) at sleepq_switch+0x162 sleepq_timedwait(c13048b4,0,efbe3c78,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c13048b4,c13048a0,3e8,783,4e20,...) at _cv_timedwait+0x270 sched_sync(0,efbe3d28,c0fcc5b0,3d8,cbedb598,...) at sched_sync+0x595 fork_exit(c0b2aca0,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xcacf9b80 sched_switch(cacf9b80,0,104,1a3,2e0e1a98,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cacf9b80,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304548,54,c0fdfe72,0,0,...) at sleepq_timedwait+0x6b _sleep(c1304548,c1304550,54,c0fdfe72,3e8,...) at _sleep+0x36a buf_daemon(0,efbe0d28,c0fcc5b0,3d8,cbedb864,...) at buf_daemon+0x175 fork_exit(c0b0d9c0,0,efbe0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe0d60, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xcae24000 sched_switch(cae24000,0,104,1a3,71d2bef4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24000,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e468,0,c101129d,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e468,c130cf00,0,c101129d,493e0,...) at _sleep+0x36a vm_pagezero(0,efbddd28,c0fcc5b0,3d8,cbedbb30,...) at vm_pagezero+0xdc fork_exit(c0d1fa40,0,efbddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbddd60, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xcae242e0 sched_switch(cae242e0,0,104,1a3,dcbe41c6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cae242e0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130e048,74,c0fdfe72,0,0,...) at sleepq_wait+0x63 _sleep(c130e048,c130e04c,74,c0fdfe72,0,...) at _sleep+0x39c vm_daemon(0,efbdad28,c0fcc5b0,3d8,cbedc000,...) at vm_daemon+0x59 fork_exit(c0d18320,0,efbdad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbdad60, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xcae245c0 kdb_enter(c0e655d6,c0f57edc,0,14e2742,0) at kdb_enter+0x3a watchdog_fire(1,0,c0fca11b,21b,c95de2cc,...) at watchdog_fire+0xbd hardclock_cnt(1,0,efbd7ab0,c0ab0c65,16c6d1c3,...) at hardclock_cnt+0x2bc handleevents(efbd7ae0,c9582790,4,c12d7ba0,0,c95827f8,140d,16c6d1c3,4a1d95da,828dd00) at handleevents+0x17d timercb(c136d380,0,2,cbdc0a08,30a,...) at timercb+0x292 lapic_handle_timer(efbd7b24) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0ad3eb0, esp = 0xefbd7b64, ebp = 0xefbd7b94 --- witness_unlock(cbdc0a08,8,c100d144,30a,0,...) at witness_unlock+0x50 _mtx_unlock_flags(cbdc0a08,0,c100d144,30a,4,...) at _mtx_unlock_flags+0xca zone_drain_wait(efbd7c10,c0cfb695,cbe0d380,0,c100d144,...) at zone_drain_wait+0xf1 zone_drain(cbe0d380,0,c100d144,63c,0,...) at zone_drain+0x10 zone_foreach(c95b46c0,0,efbd7ce8,c0d19bac,c95b46cc,...) at zone_foreach+0x45 uma_reclaim(c95b46cc,0,c1010087,2ee,1388,...) at uma_reclaim+0x16 vm_pageout(0,efbd7d28,c0fcc5b0,3d8,c95de2cc,...) at vm_pageout+0x80c fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xcae248a0 sched_switch(cae248a0,0,104,1a3,53cfd0a1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae248a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1151054,5c,c0e2c8fe,0,0,...) at sleepq_wait+0x63 _sleep(c1151054,c1151098,5c,c0e2c8fe,0,...) at _sleep+0x39c xpt_scanner_thread(0,efa54d28,c0fcc5b0,3d8,c95de598,...) at xpt_scanner_thread+0x47 fork_exit(c04a72f0,0,efa54d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa54d60, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xcae24b80 sched_switch(cae24b80,0,104,1a3,b43c19e1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24b80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1307dd8,0,c0ff75bf,0,0,...) at sleepq_wait+0x63 _sleep(c1307dd8,c1307da0,0,c0ff75bf,0,...) at _sleep+0x39c sctp_iterator_thread(0,efa51d28,c0fcc5b0,3d8,c95de864,...) at sctp_iterator_thread+0x5c fork_exit(c0bc5a50,0,efa51d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa51d60, ebp = 0 --- Tracing command fdc0 pid 4 tid 100065 td 0xcae25b80 sched_switch(cae25b80,0,104,1a3,4db37930,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae25b80,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cac7223c,5c,c0fbe932,0,0,...) at sleepq_timedwait+0x6b _sleep(cac7223c,cac722f0,5c,c0fbe932,3e8,...) at _sleep+0x36a fdc_thread(cac72200,efa42d28,c0fcc5b0,3d8,c95deb30,...) at fdc_thread+0x27d fork_exit(c0da1d70,cac72200,efa42d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa42d60, ebp = 0 --- Tracing command fw0_probe pid 3 tid 100063 td 0xcace42e0 sched_switch(cace42e0,0,104,1a3,53cfeeed,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cace42e0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cad2c000,6c,c0fbe932,100,0,...) at sleepq_wait_sig+0x17 _sleep(cad2c000,cad30488,16c,c0fbe932,0,...) at _sleep+0x385 fw_bus_probe_thread(cad2c000,efa32d28,c0fcc5b0,3d8,cac0b000,...) at fw_bus_probe_thread+0xa08 fork_exit(c07017d0,cad2c000,efa32d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa32d60, ebp = 0 --- Tracing command usb pid 15 tid 100059 td 0xcacf7000 sched_switch(cacf7000,0,104,1a3,416a1a6f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf7000,0,c0fd7fa0,268,cacf7000,...) at sleepq_switch+0x162 sleepq_wait(cacfed34,0,efa1bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed34,cacfee4c,c0faffb1,74,cacfed3c,...) at _cv_wait+0x260 usb_process(cacfed2c,efa1bd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfed2c,efa1bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa1bd60, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xcacf72e0 sched_switch(cacf72e0,0,103,19e,5c674c0f,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,a50,...) at mi_switch+0x1ff turnstile_wait(c95a0300,c9622000,0,1c5,0,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(cacfee4c,cacf72e0,0,c0fd05c7,9a,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(cacfee4c,0,c0fd05c7,9a,efa18cc0,...) at _mtx_lock_flags+0x10f lock_mtx(cacfee4c,0,c0fca469,91,0,...) at lock_mtx+0x29 _cv_wait(cacfed04,cacfee4c,c0faffb1,74,cacfed0c,...) at _cv_wait+0x31e usb_process(cacfecfc,efa18d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfecfc,efa18d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa18d60, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xcacf75c0 sched_switch(cacf75c0,0,104,1a3,4103d6e9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf75c0,0,c0fd7fa0,268,cacf75c0,...) at sleepq_switch+0x162 sleepq_wait(cacfecd4,0,efa15cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfecd4,cacfee4c,c0faffb1,74,cacfecdc,...) at _cv_wait+0x260 usb_process(cacfeccc,efa15d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfeccc,efa15d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa15d60, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xcacf78a0 sched_switch(cacf78a0,0,104,1a3,4103c708,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf78a0,0,c0fd7fa0,268,cacf78a0,...) at sleepq_switch+0x162 sleepq_wait(cacfeca4,0,efa12cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfeca4,cacfee4c,c0faffb1,74,cacfecac,...) at _cv_wait+0x260 usb_process(cacfec9c,efa12d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfec9c,efa12d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa12d60, ebp = 0 --- Tracing command usb pid 15 tid 100054 td 0xcacf9000 sched_switch(cacf9000,0,104,1a3,4103b259,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf9000,0,c0fd7fa0,268,cacf9000,...) at sleepq_switch+0x162 sleepq_wait(cacefb5c,0,efa09cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb5c,cacefc74,c0faffb1,74,cacefb64,...) at _cv_wait+0x260 usb_process(cacefb54,efa09d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb54,efa09d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa09d60, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xcacf92e0 sched_switch(cacf92e0,0,104,1a3,4db6631e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf92e0,0,c0fd7fa0,268,cacf92e0,...) at sleepq_switch+0x162 sleepq_wait(cacefb2c,0,efa06cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb2c,cacefc74,c0faffb1,74,cacefb34,...) at _cv_wait+0x260 usb_process(cacefb24,efa06d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb24,efa06d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa06d60, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xcacf95c0 sched_switch(cacf95c0,0,104,1a3,409f3fdd,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf95c0,0,c0fd7fa0,268,cacf95c0,...) at sleepq_switch+0x162 sleepq_wait(cacefafc,0,efa03cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefafc,cacefc74,c0faffb1,74,cacefb04,...) at _cv_wait+0x260 usb_process(cacefaf4,efa03d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefaf4,efa03d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa03d60, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xcace2000 sched_switch(cace2000,0,104,1a3,409f2c92,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2000,0,c0fd7fa0,268,cace2000,...) at sleepq_switch+0x162 sleepq_wait(cacefacc,0,efa00cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefacc,cacefc74,c0faffb1,74,cacefad4,...) at _cv_wait+0x260 usb_process(cacefac4,efa00d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefac4,efa00d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa00d60, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xcace22e0 sched_switch(cace22e0,0,104,1a3,409f177f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace22e0,0,c0fd7fa0,268,cace22e0,...) at sleepq_switch+0x162 sleepq_wait(caceab5c,0,ef9fccac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab5c,caceac74,c0faffb1,74,caceab64,...) at _cv_wait+0x260 usb_process(caceab54,ef9fcd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab54,ef9fcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9fcd60, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xcace25c0 sched_switch(cace25c0,0,104,1a3,4db32dcd,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace25c0,0,c0fd7fa0,268,cace25c0,...) at sleepq_switch+0x162 sleepq_wait(caceab2c,0,ef9f9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab2c,caceac74,c0faffb1,74,caceab34,...) at _cv_wait+0x260 usb_process(caceab24,ef9f9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab24,ef9f9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f9d60, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xcace28a0 sched_switch(cace28a0,0,104,1a3,3acd4061,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace28a0,0,c0fd7fa0,268,cace28a0,...) at sleepq_switch+0x162 sleepq_wait(caceaafc,0,ef9f6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaafc,caceac74,c0faffb1,74,caceab04,...) at _cv_wait+0x260 usb_process(caceaaf4,ef9f6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaaf4,ef9f6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f6d60, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xcace2b80 sched_switch(cace2b80,0,104,1a3,3acd2e64,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2b80,0,c0fd7fa0,268,cace2b80,...) at sleepq_switch+0x162 sleepq_wait(caceaacc,0,ef9f3cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaacc,caceac74,c0faffb1,74,caceaad4,...) at _cv_wait+0x260 usb_process(caceaac4,ef9f3d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaac4,ef9f3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f3d60, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xcace3000 sched_switch(cace3000,0,104,1a3,3acd1a18,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace3000,0,c0fd7fa0,268,cace3000,...) at sleepq_switch+0x162 sleepq_wait(cace5b5c,0,ef9efcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b5c,cace5c74,c0faffb1,74,cace5b64,...) at _cv_wait+0x260 usb_process(cace5b54,ef9efd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b54,ef9efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9efd60, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xcace32e0 sched_switch(cace32e0,0,104,1a3,4c3f862b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace32e0,0,c0fd7fa0,268,cace32e0,...) at sleepq_switch+0x162 sleepq_wait(cace5b2c,0,ef9eccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b2c,cace5c74,c0faffb1,74,cace5b34,...) at _cv_wait+0x260 usb_process(cace5b24,ef9ecd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b24,ef9ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ecd60, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xcace35c0 sched_switch(cace35c0,0,104,1a3,3a68eabc,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace35c0,0,c0fd7fa0,268,cace35c0,...) at sleepq_switch+0x162 sleepq_wait(cace5afc,0,ef9e9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5afc,cace5c74,c0faffb1,74,cace5b04,...) at _cv_wait+0x260 usb_process(cace5af4,ef9e9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5af4,ef9e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e9d60, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xcace38a0 sched_switch(cace38a0,0,104,1a3,3a68d99e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace38a0,0,c0fd7fa0,268,cace38a0,...) at sleepq_switch+0x162 sleepq_wait(cace5acc,0,ef9e6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5acc,cace5c74,c0faffb1,74,cace5ad4,...) at _cv_wait+0x260 usb_process(cace5ac4,ef9e6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5ac4,ef9e6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e6d60, ebp = 0 --- Tracing command usb pid 15 tid 100041 td 0xcace4000 sched_switch(cace4000,0,104,1a3,3a68c21a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace4000,0,c0fd7fa0,268,cace4000,...) at sleepq_switch+0x162 sleepq_wait(cacddb5c,0,ef9dfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb5c,cacddc74,c0faffb1,74,cacddb64,...) at _cv_wait+0x260 usb_process(cacddb54,ef9dfd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb54,ef9dfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dfd60, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xcacbf000 sched_switch(cacbf000,0,104,1a3,2fc72aa3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf000,0,c0fd7fa0,268,cacbf000,...) at sleepq_switch+0x162 sleepq_wait(cacddb2c,0,ef9dccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb2c,cacddc74,c0faffb1,74,cacddb34,...) at _cv_wait+0x260 usb_process(cacddb24,ef9dcd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb24,ef9dcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dcd60, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xcacbf2e0 sched_switch(cacbf2e0,0,104,1a3,3a0546d0,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf2e0,0,c0fd7fa0,268,cacbf2e0,...) at sleepq_switch+0x162 sleepq_wait(cacddafc,0,ef9d9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddafc,cacddc74,c0faffb1,74,cacddb04,...) at _cv_wait+0x260 usb_process(cacddaf4,ef9d9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddaf4,ef9d9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d9d60, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xcacbf5c0 sched_switch(cacbf5c0,0,104,1a3,3a0534f2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf5c0,0,c0fd7fa0,268,cacbf5c0,...) at sleepq_switch+0x162 sleepq_wait(cacddacc,0,ef9d6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddacc,cacddc74,c0faffb1,74,cacddad4,...) at _cv_wait+0x260 usb_process(cacddac4,ef9d6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddac4,ef9d6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d6d60, ebp = 0 --- Tracing command usb pid 15 tid 100036 td 0xcacbfb80 sched_switch(cacbfb80,0,104,1a3,3a051f1f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbfb80,0,c0fd7fa0,268,cacbfb80,...) at sleepq_switch+0x162 sleepq_wait(cacd5b5c,0,ef9cfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b5c,cacd5c74,c0faffb1,74,cacd5b64,...) at _cv_wait+0x260 usb_process(cacd5b54,ef9cfd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b54,ef9cfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9cfd60, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xcacc1000 sched_switch(cacc1000,0,104,1a3,2fc61c33,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc1000,0,c0fd7fa0,268,cacc1000,...) at sleepq_switch+0x162 sleepq_wait(cacd5b2c,0,ef9cccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b2c,cacd5c74,c0faffb1,74,cacd5b34,...) at _cv_wait+0x260 usb_process(cacd5b24,ef9ccd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b24,ef9ccd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ccd60, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xcacc12e0 sched_switch(cacc12e0,0,104,1a3,39a154ec,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc12e0,0,c0fd7fa0,268,cacc12e0,...) at sleepq_switch+0x162 sleepq_wait(cacd5afc,0,ef9c9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5afc,cacd5c74,c0faffb1,74,cacd5b04,...) at _cv_wait+0x260 usb_process(cacd5af4,ef9c9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5af4,ef9c9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c9d60, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xcacc15c0 sched_switch(cacc15c0,0,104,1a3,39a140cb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc15c0,0,c0fd7fa0,268,cacc15c0,...) at sleepq_switch+0x162 sleepq_wait(cacd5acc,0,ef9c6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5acc,cacd5c74,c0faffb1,74,cacd5ad4,...) at _cv_wait+0x260 usb_process(cacd5ac4,ef9c6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5ac4,ef9c6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c6d60, ebp = 0 --- Tracing command ctl_thrd pid 2 tid 100028 td 0xc96eb8a0 sched_switch(c96eb8a0,0,104,1a3,39a127e5,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb8a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c97c4000,5c,c0e37370,0,0,...) at sleepq_wait+0x63 _sleep(c97c4000,c97c4000,5c,c0e37370,0,...) at _sleep+0x39c ctl_work_thread(c97c4000,ef733d28,c0fcc5b0,3d8,cac0b598,...) at ctl_work_thread+0x22f6 fork_exit(c04d08d0,c97c4000,ef733d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef733d60, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xc96ea5c0 sched_switch(c96ea5c0,0,104,1a3,2dbb83bb,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ea5c0,0,c0fd7fa0,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe932,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe932,64) at _sleep+0x36a pause(c0fbe932,64,c0f8c010,111,0,...) at pause+0xa0 random_kthread(0,e33e0d28,c0fcc5b0,3d8,c95dd000,...) at random_kthread+0x1ef fork_exit(c082d020,0,e33e0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e0d60, ebp = 0 --- Tracing command geom pid 13 tid 100016 td 0xc96ea8a0 sched_switch(c96ea8a0,0,104,1a3,2e4c66fc,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96ea8a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a5a8,5c,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c118a5a8,c118a508,25c,c0fbe932,0,...) at _sleep+0x39c g_io_schedule_down(c96ea8a0,0,c0fc0481,6c,e33ddd14,...) at g_io_schedule_down+0x5b g_down_procbody(0,e33ddd28,c0fcc5b0,3d8,c95dd2cc,...) at g_down_procbody+0x9d fork_exit(c09e6af0,0,e33ddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ddd60, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xc96eab80 sched_switch(c96eab80,0,104,1a3,2fa6c5f9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eab80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a5a4,5c,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c118a5a4,c118a528,25c,c0fbe932,0,...) at _sleep+0x39c g_io_schedule_up(c96eab80,0,c0fc0481,5f,e33dad14,...) at g_io_schedule_up+0x134 g_up_procbody(0,e33dad28,c0fcc5b0,3d8,c95dd2cc,...) at g_up_procbody+0x9d fork_exit(c09e6b90,0,e33dad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33dad60, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xc96eb000 sched_switch(c96eb000,0,104,1a3,6fef96f6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb000,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a59c,5c,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c118a59c,c118a4cc,25c,c0fbe932,0,...) at _sleep+0x39c g_run_events(c119b340,0,c0fc0481,79,e33d7d14,...) at g_run_events+0x54b g_event_procbody(0,e33d7d28,c0fcc5b0,3d8,c95dd2cc,...) at g_event_procbody+0x90 fork_exit(c09e6c30,0,e33d7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d7d60, ebp = 0 --- Tracing command intr pid 12 tid 100067 td 0xcae255c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100066 td 0xcae258a0 sched_switch(cae258a0,0,109,1a3,399d5679,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cae258a0,...) at mi_switch+0x1ff ithread_loop(cae1e990,efa45d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cae1e990,efa45d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa45d60, ebp = 0 --- Tracing command intr pid 12 tid 100064 td 0xcae26000 sched_switch(cae26000,0,109,1a3,79f426fe,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cae26000,...) at mi_switch+0x1ff ithread_loop(cae18a40,efa35d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cae18a40,efa35d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa35d60, ebp = 0 --- Tracing command intr pid 12 tid 100061 td 0xcace48a0 sched_switch(cace48a0,0,109,1a3,29783a3a,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cace48a0,...) at mi_switch+0x1ff ithread_loop(cad15880,efa29d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15880,efa29d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa29d60, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xcace4b80 sched_switch(cace4b80,0,109,1a3,1533dbf7,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cace4b80,...) at mi_switch+0x1ff ithread_loop(cad15010,efa21d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15010,efa21d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa21d60, ebp = 0 --- Tracing command intr pid 12 tid 100055 td 0xcacf7b80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100042 td 0xcace3b80 sched_switch(cace3b80,0,109,1a3,5eb2e43e,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cace3b80,...) at mi_switch+0x1ff ithread_loop(cacc5720,ef9e3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacc5720,ef9e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xcacbf8a0 sched_switch(cacbf8a0,0,109,1a3,591f501d,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cacbf8a0,...) at mi_switch+0x1ff ithread_loop(cacad590,ef9d3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad590,ef9d3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d3d60, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xcacc18a0 sched_switch(cacc18a0,0,109,1a3,56776cd7,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cacc18a0,...) at mi_switch+0x1ff ithread_loop(cacad830,ef9c3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad830,ef9c3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c3d60, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xcacc1b80 sched_switch(cacc1b80,0,109,1a3,2fa63356,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cacc1b80,...) at mi_switch+0x1ff ithread_loop(cac97ae0,ef86dd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97ae0,ef86dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef86dd60, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc96eb2e0 sched_switch(c96eb2e0,0,109,1a3,1b7a3236,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96eb2e0,...) at mi_switch+0x1ff ithread_loop(cac97c80,ef869d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97c80,ef869d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef869d60, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc96eb5c0 sched_switch(c96eb5c0,0,109,1a3,2b572919,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96eb5c0,...) at mi_switch+0x1ff ithread_loop(cac97500,ef763d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97500,ef763d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef763d60, ebp = 0 --- Tracing command intr pid 12 tid 100022 td 0xc97c1b80 sched_switch(c97c1b80,0,109,1a3,8ad52c8d,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c97c1b80,...) at mi_switch+0x1ff ithread_loop(c95b3450,e33efd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3450,e33efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33efd60, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc96228a0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100019 td 0xc96ea000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100018 td 0xc96ea2e0 sched_switch(c96ea2e0,0,109,1a3,d57d96e8,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96ea2e0,...) at mi_switch+0x1ff ithread_loop(c95b3490,e33e3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3490,e33e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc95e05c0 sched_switch(c95e05c0,0,109,1a3,2e0e057a,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c95e05c0,...) at mi_switch+0x1ff ithread_loop(c95b36f0,e33d1d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b36f0,e33d1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d1d60, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc95e08a0 sched_switch(c95e08a0,0,109,1a3,2d111f4,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c95e08a0,...) at mi_switch+0x1ff ithread_loop(c95b3700,e33ced28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3700,e33ced28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ced60, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc95e0b80 sched_switch(c95e0b80,0,109,1a3,2dbb7173,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c95e0b80,...) at mi_switch+0x1ff ithread_loop(c95b3710,e33cbd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3710,e33cbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33cbd60, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc9622000 sched_switch(c9622000,0,602,19e,5c672ade,...) at sched_switch+0x413 mi_switch(602,0,c0fd2c5f,cd,c9622000,...) at mi_switch+0x1ff critical_exit(c11a9bbc,c93feb38,c0a742f3,c11a9bbc,4,...) at critical_exit+0xb5 spinlock_exit(c11a9bbc,4,c0fd05c7,119,c93feb50,c0ac8b74) at spinlock_exit+0x10 _mtx_unlock_spin_flags(c11a9bbc,0,c0fd7fa0,10d,c93feb70,...) at _mtx_unlock_spin_flags+0x133 sleepq_release(cacfed04,1,0,0,cacfed5c,...) at sleepq_release+0x39 cv_signal(cacfed04,4,c0faffb1,128,cacfed5c,...) at cv_signal+0x4e usb_proc_msignal(cacfecfc,cacfed5c,cacfed70,776,cacfec78,...) at usb_proc_msignal+0x16a usb_needs_explore(cacfec78,0,c93febf0,c0917397,cacfec78,...) at usb_needs_explore+0x10b usb_bus_power_update(cacfec78,0,c0fab995,23f,0,...) at usb_bus_power_update+0x19 usb_power_wdog(cacfec78,1,c0fd3b12,1e7,c0aae18e,...) at usb_power_wdog+0x97 softclock(c118e720,c119b340,4,c0fd05c7,c95aa0b8,...) at softclock+0x297 intr_event_execute_handlers(c95dd598,c95aa080,c0fcc8e1,561,c9622000,...) at intr_event_execute_handlers+0x125 ithread_loop(c95b3720,c93fed28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0xa9 fork_exit(c0a5a9c0,c95b3720,c93fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fed60, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc96222e0 sched_switch(c96222e0,0,109,1a3,4a5f9e9b,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96222e0,...) at mi_switch+0x1ff ithread_loop(c95b3730,c93fbd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3730,c93fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fbd60, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc96225c0 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc95df000 sched_switch(c95df000,0,108,19e,2f1b0ad5,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df000,...) at mi_switch+0x1ff sched_idletd(0,c93f4d28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93f4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f4d60, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc95df2e0 sched_switch(c95df2e0,0,108,19e,1d83f3ef,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df2e0,...) at mi_switch+0x1ff sched_idletd(0,c93f1d28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93f1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f1d60, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc95df5c0 sched_switch(c95df5c0,0,108,19e,2f5a7c16,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df5c0,...) at mi_switch+0x1ff sched_idletd(0,c93eed28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93eed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93eed60, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc95df8a0 sched_switch(c95df8a0,0,108,19e,4dbec476,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df8a0,...) at mi_switch+0x1ff sched_idletd(0,c93ebd28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93ebd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93ebd60, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc95dfb80 sched_switch(c95dfb80,0,104,1a3,c220be58,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c95dfb80,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c95ddb30,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(c95ddb30,c95ddbb8,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(c95dfb80,ffffffff,c93e7c58,0,0,...) at kern_wait+0x597 sys_wait4(c95dfb80,c93e7cec,c1025a7c,c0fdad26,c1027030,...) at sys_wait4+0x3b syscall(c93e7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x806070f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8c8 --- Tracing command audit pid 10 tid 100001 td 0xc95e0000 sched_switch(c95e0000,0,104,1a3,399ee628,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e0000,0,c0fd7fa0,268,c95e0000,...) at sleepq_switch+0x162 sleepq_wait(c130bd00,0,c93e4c8c,1,0,...) at sleepq_wait+0x63 _cv_wait(c130bd00,c130bce4,c1004257,194,c93e4cc4,...) at _cv_wait+0x260 audit_worker(0,c93e4d28,c0fcc5b0,3d8,c95de000,...) at audit_worker+0x84 fork_exit(c0c95110,0,c93e4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93e4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xcae25000 sched_switch(cae25000,0,104,1a3,42c9e189,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae25000,0,c0fd7fa0,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe932,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe932,bb8) at _sleep+0x36a pause(c0fbe932,bb8,126,124,c118a7e8,...) at pause+0xa0 deadlkres(0,efa4ed28,c0fcc5b0,3d8,c118a760,...) at deadlkres+0x324 fork_exit(c0a398a0,0,efa4ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4ed60, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xcae252e0 sched_switch(cae252e0,0,104,1a3,a216bb99,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae252e0,0,c0fd7fa0,268,cae252e0,...) at sleepq_switch+0x162 sleepq_wait(c971ddc0,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971ddc0,c971ddd8,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c136d440,efa4bd28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c136d440,efa4bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4bd60, ebp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xcace45c0 sched_switch(cace45c0,0,104,1a3,586f2867,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace45c0,0,c0fd7fa0,268,cace45c0,...) at sleepq_switch+0x162 sleepq_wait(c971de00,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971de00,c971de18,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(cad3049c,efa2fd28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,cad3049c,efa2fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa2fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc96ebb80 sched_switch(c96ebb80,0,104,1a3,53d458d3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ebb80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de40,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c971de40,c971de58,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c118b0f8,e33fed28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c118b0f8,e33fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fed60, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc97c1000 sched_switch(c97c1000,0,104,1a3,53d445d9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c1000,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de80,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c971de80,c971de98,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c130c2e4,e33fbd28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c130c2e4,e33fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fbd60, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc97c12e0 sched_switch(c97c12e0,0,104,1a3,53d4348b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c12e0,0,c0fd7fa0,268,c97c12e0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f8d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f8d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f8d60, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc97c15c0 sched_switch(c97c15c0,0,104,1a3,53d425ae,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c15c0,0,c0fd7fa0,268,c97c15c0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f5d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f5d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f5d60, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc97c18a0 sched_switch(c97c18a0,0,104,1a3,53d4160c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c18a0,0,c0fd7fa0,268,c97c18a0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f2d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f2d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f2d60, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc9622b80 sched_switch(c9622b80,0,104,1a3,4860517,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c9622b80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971e0c0,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c971e0c0,c971e0d8,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c11aa0ac,e33e9d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11aa0ac,e33e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e9d60, ebp = 0 --- Tracing command kernel pid 0 tid 100013 td 0xc95e02e0 sched_switch(c95e02e0,0,104,1a3,470802a1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e02e0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c96e9000,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c96e9000,c96e9018,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c11a8ac0,e33d4d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11a8ac0,e33d4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc118aa30 sched_switch(c118aa30,0,104,1a3,d558b77d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c118aa30,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c118a760,54,c1004c38,0,0,...) at sleepq_timedwait+0x6b _sleep(c118a760,0,54,c1004c38,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x240 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 310 MB:lock order reversal: 1st 0xcbdc0a08 sctp_readq (UMA zone) @ vm/uma_core.c:768 2nd 0xcac11804 AHCI channel lock (AHCI channel lock) @ cam/cam_periph.h:191 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,0,0,0,0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d7bb0,bf,efbd6fac,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cac11804,c0e5501d,c9586f10,c0e2df7b,...) at _witness_debugger+0x25 witness_checkorder(cac11804,9,c0e2df72,bf,0,...) at witness_checkorder+0x86f _mtx_lock_flags(cac11804,0,c0e2df72,bf,0,...) at _mtx_lock_flags+0xd9 adadump(c95ba400,c1326160,0,6c95ea00,4,...) at adadump+0x67 dump_write(c118d200,c1326160,0,6c95ea00,4,...) at dump_write+0xbf minidumpsys(c118d200,c051c7e3,c15b84a9,efbd759c,efbd759c,...) at minidumpsys+0x356 dumpsys(c118d200,efbd764c,c051b3a9,0,0,...) at dumpsys+0x2b doadump(0,0,0,0,0,...) at doadump+0x6c db_fncall(c0abd94a,0,ffffffff,efbd7688,0,...) at db_fncall+0x149 db_command(c1153368,efbd7778,c051fcd0,c1153368,0,...) at db_command+0x3bf db_command_script(c1153368,0,c1152cc0,c1153367,c1152cc0,...) at db_command_script+0x34 db_script_exec(efbd79bc,efbd7848,c051b7a1,1,0,...) at db_script_exec+0x130 db_run_cmd(1,0,c132da60,efbd77c0,1,...) at db_run_cmd+0x41 db_command(c0e4669e,efbd78a8,c051d8ed,c0e655d6,c9775000,...) at db_command+0x381 db_command_loop(c0e655d6,c9775000,0,efbd7880,0,...) at db_command_loop+0x5a db_trap(3,0,1,46,2,...) at db_trap+0xdd kdb_trap(3,0,efbd79bc,efbd7974,c95de2cc,...) at kdb_trap+0xb6 trap(efbd79bc) at trap+0x79b calltrap() at calltrap+0x6 --- trap 0x3, eip = 0xc0abd94a, esp = 0xefbd79fc, ebp = 0xefbd7a04 --- kdb_enter(c0e655d6,c0f57edc,0,14e2742,0) at kdb_enter+0x3a watchdog_fire(1,0,c0fca11b,21b,c95de2cc,...) at watchdog_fire+0xbd hardclock_cnt(1,0,efbd7ab0,c0ab0c65,16c6d1c3,...) at hardclock_cnt+0x2bc handleevents(efbd7ae0,c9582790,4,c12d7ba0,0,c95827f8,140d,16c6d1c3,4a1d95da,828dd00) at handleevents+0x17d timercb(c136d380,0,2,cbdc0a08,30a,...) at timercb+0x292 lapic_handle_timer(efbd7b24) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0ad3eb0, esp = 0xefbd7b64, ebp = 0xefbd7b94 --- witness_unlock(cbdc0a08,8,c100d144,30a,0,...) at witness_unlock+0x50 _mtx_unlock_flags(cbdc0a08,0,c100d144,30a,4,...) at _mtx_unlock_flags+0xca zone_drain_wait(efbd7c10,c0cfb695,cbe0d380,0,c100d144,...) at zone_drain_wait+0xf1 zone_drain(cbe0d380,0,c100d144,63c,0,...) at zone_drain+0x10 zone_foreach(c95b46c0,0,efbd7ce8,c0d19bac,c95b46cc,...) at zone_foreach+0x45 uma_reclaim(c95b46cc,0,c1010087,2ee,1388,...) at uma_reclaim+0x16 vm_pageout(0,efbd7d28,c0fcc5b0,3d8,c95de2cc,...) at vm_pageout+0x80c fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- 295 279 263 247 231 215 199 183 167 151 135 119 103 87 71 55 39 23 7 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 2 (kgdb) proc 1766 [Switching to thread 118 (Thread 100156)]#0 sched_switch (td=0xcc9c22e0, newtd=0xcca098a0, flags=0x103) at ../../../kern/sched_ule.c:1902 1902 cpuid = PCPU_GET(cpuid); (kgdb) bt #0 sched_switch (td=0xcc9c22e0, newtd=0xcca098a0, flags=0x103) at ../../../kern/sched_ule.c:1902 #1 0xc0a9074f in mi_switch (flags=0x103, newtd=0x0) at ../../../kern/kern_synch.c:468 #2 0xc0acf084 in turnstile_wait (ts=0xcacfa280, owner=0xcbee88a0, queue=Variable "queue" is not available. ) at ../../../kern/subr_turnstile.c:744 #3 0xc0a74cc7 in _mtx_lock_sleep (m=0xc130d900, tid=0xcc9c22e0, opts=Variable "opts" is not available. ) at ../../../kern/kern_mutex.c:472 #4 0xc0a74edf in _mtx_lock_flags (m=0xc130d900, opts=0x0, file=0xc100f607 "../../../vm/vm_page.c", line=0x4fe) at ../../../kern/kern_mutex.c:212 #5 0xc0d17245 in vm_page_alloc (object=0xceb7b818, pindex=0x1e, req=0x0) at ../../../vm/vm_page.c:1278 #6 0xc0d02aaa in vm_fault_hold (map=0xcc4223a0, vaddr=0xbfbfe000, fault_type=0x2, fault_flags=Variable "fault_flags" is not available. ) at ../../../vm/vm_fault.c:444 #7 0xc0d0449b in vm_fault (map=0xcc4223a0, vaddr=0xbfbfe000, fault_type=Variable "fault_type" is not available. ) at ../../../vm/vm_fault.c:229 #8 0xc0e000f9 in trap_pfault (frame=0xf4156d28, usermode=0x1, eva=0xbfbfec3c) at ../../../i386/i386/trap.c:903 #9 0xc0e00e73 in trap (frame=0xf4156d28) at ../../../i386/i386/trap.c:413 #10 0xc0dea83c in calltrap () at ../../../i386/i386/exception.s:169 #11 0x280bce00 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) proc 3209 [Switching to thread 139 (Thread 100178)]#0 sched_switch (td=0xcca43000, newtd=0xcc9c22e0, flags=0x104) at ../../../kern/sched_ule.c:1902 1902 cpuid = PCPU_GET(cpuid); (kgdb) bt #0 sched_switch (td=0xcca43000, newtd=0xcc9c22e0, flags=0x104) at ../../../kern/sched_ule.c:1902 #1 0xc0a9074f in mi_switch (flags=0x104, newtd=0x0) at ../../../kern/kern_synch.c:468 #2 0xc0ac9122 in sleepq_switch (wchan=0xc130cc7c, pri=0x78) at ../../../kern/subr_sleepqueue.c:538 #3 0xc0ac9e53 in sleepq_wait (wchan=0xc130cc7c, pri=0x78) at ../../../kern/subr_sleepqueue.c:617 #4 0xc0a90e5c in _sleep (ident=0xc130cc7c, lock=0xc130cf00, priority=0x278, wmesg=0xc100f8cd "pfault", timo=0x0) at ../../../kern/kern_synch.c:234 #5 0xc0d14615 in vm_waitpfault () at ../../../vm/vm_page.c:1664 #6 0xc0d02abe in vm_fault_hold (map=0xcc0d4cb0, vaddr=0xbfbfe000, fault_type=0x2, fault_flags=Variable "fault_flags" is not available. ) at ../../../vm/vm_fault.c:449 #7 0xc0d0449b in vm_fault (map=0xcc0d4cb0, vaddr=0xbfbfe000, fault_type=Variable "fault_type" is not available. ) at ../../../vm/vm_fault.c:229 #8 0xc0e000f9 in trap_pfault (frame=0xf4198d28, usermode=0x1, eva=0xbfbfec38) at ../../../i386/i386/trap.c:903 #9 0xc0e00e73 in trap (frame=0xf4198d28) at ../../../i386/i386/trap.c:413 #10 0xc0dea83c in calltrap () at ../../../i386/i386/exception.s:169 #11 0x280bcea6 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) p cnt $1 = {v_swtch = 0x0, v_trap = 0x0, v_syscall = 0x0, v_intr = 0x0, v_soft = 0x0, v_vm_faults = 0x0, v_cow_faults = 0x0, v_cow_optim = 0x0, v_zfod = 0x0, v_ozfod = 0x0, v_swapin = 0x0, v_swapout = 0x0, v_swappgsin = 0x0, v_swappgsout = 0x0, v_vnodein = 0x0, v_vnodeout = 0x0, v_vnodepgsin = 0x0, v_vnodepgsout = 0x0, v_intrans = 0x0, v_reactivated = 0x5e4, v_pdwakeups = 0xb5d0ae, v_pdpages = 0x7788e4, v_tcached = 0x0, v_dfree = 0x0, v_pfree = 0x0, v_tfree = 0x0, v_page_size = 0x1000, v_page_count = 0xd34cd, v_free_reserved = 0x498, v_free_target = 0x5a90, v_free_min = 0x157e, v_free_count = 0x63d5, v_wire_count = 0xe7ea, v_active_count = 0x17481, v_inactive_target = 0x87d8, v_inactive_count = 0xa14d8, v_cache_count = 0x5f2d, v_cache_min = 0x5a90, v_cache_max = 0xb520, v_pageout_free_min = 0x22, v_interrupt_free_min = 0x2, v_free_severe = 0xd0b, v_forks = 0x0, v_vforks = 0x0, v_rforks = 0x0, v_kthreads = 0x0, v_forkpages = 0x0, v_vforkpages = 0x0, v_rforkpages = 0x0, v_kthreadpages = 0x0} (kgdb) $ svn diff -x -p /var/tmp/vmcontention/sys Index: /var/tmp/vmcontention/sys/vm/vm_radix.c =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.c (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.c (working copy) @@ -99,14 +99,12 @@ #define KSPLT64H(x) ((u_long)(((x) >> 32) & 0xFFFFFFFF)) #endif -CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); -CTASSERT((sizeof(u_int) * NBBY) >= VM_RADIX_LIMIT); - struct vm_radix_node { void *rn_child[VM_RADIX_COUNT]; /* Child nodes. */ volatile uint32_t rn_count; /* Valid children. */ }; +CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); CTASSERT(sizeof(struct vm_radix_node) < PAGE_SIZE); static uma_zone_t vm_radix_node_zone; @@ -356,6 +354,101 @@ vm_radix_reclaim_allnodes_internal(struct vm_radix } /* + * Remove the specified index from the tree. If possible the height of the + * tree is adjusted after deletion. May be used to cleanup intermediate + * nodes of the path if the key is not entirely present. + */ +static void +vm_radix_sweep(struct vm_radix *rtree, vm_pindex_t index, int color, int hard) +{ + struct vm_radix_node *stack[VM_RADIX_LIMIT]; + struct vm_radix_node *rnode, *root; + int level; + int slot; + + level = vm_radix_height(rtree, &root); + KASSERT(index <= VM_RADIX_MAX(level), + ("vm_radix_sweep: %p index out of range %jd.", rtree, + VM_RADIX_MAX(level))); + rnode = root; + level--; + /* + * Find the node and record the path in stack. + */ + while (level && rnode) { + stack[level] = rnode; + slot = vm_radix_slot(index, level); + CTR6(KTR_VM, + "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", + rtree, rnode, rnode->rn_child[slot], rnode->rn_count); + rnode = rnode->rn_child[slot]; + level--; + } + if (rnode == NULL) { + if (hard) + panic("vm_radix_sweep: index not present.\n"); + + /* + * Almost certainly it got an half-constructed tree it was + * expected. + */ + KASSERT(level < (VM_RADIX_LIMIT - 1), + ("vm_radix_sweep: level %d not consistent.\n", level)); + ++level; + rnode = stack[level]; + slot = vm_radix_slot(index, level); + } else { + slot = vm_radix_slot(index, 0); + if (hard && + vm_radix_match(rnode->rn_child[slot], color) == NULL) + panic("vm_radix_sweep: index not present as leaf.\n"); + } + + for (;;) { + CTR6(KTR_VM, +"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, + "remove: resetting tree %p, rnode %p, child %p, count %u", + rtree, rnode, + (rnode != NULL) ? rnode->rn_child[slot] : NULL, + (rnode != NULL) ? rnode->rn_count : 0); + rnode->rn_child[slot] = NULL; + /* + * Use atomics for the last level since red and black + * will both adjust it. + * Use a write memory barrier here in order to avoid + * rn_count reaching 0 before to fetch the actual pointer. + * Concurrent black removal, infact, may want to reclaim + * the radix node itself before to read it. + */ + MPASS(rnode->rn_count != 0); + if (level == 0) + atomic_add_rel_32(&rnode->rn_count, -1); + else + rnode->rn_count--; + + /* + * Only allow black removes to prune the tree. + */ + if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) + break; + vm_radix_node_put(rnode); + if (rnode == root) { + vm_radix_setroot(rtree, NULL, 0); + break; + } + rnode = stack[++level]; + slot = vm_radix_slot(index, level); + + } +} + +/* * Inserts the key-value pair in to the radix tree. Returns errno. * Panics if the key already exists. */ @@ -363,8 +456,7 @@ int vm_radix_insert(struct vm_radix *rtree, vm_pindex_t index, void *val) { struct vm_radix_node *iroot, *rnode, *root; - u_int allocmsk; - int clev, ilevel, level, slot; + int ilevel, level, slot; CTR4(KTR_VM, "insert: tree %p, " KFRMT64(index) ", val %p", rtree, @@ -417,13 +509,17 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ } /* Now that the tree is tall enough, fill in the path to the index. */ - allocmsk = 0; - clev = level; rnode = root; for (level = level - 1; level > 0; level--) { slot = vm_radix_slot(index, level); /* Add the required intermidiate nodes. */ if (rnode->rn_child[slot] == NULL) { + + /* + * In case of failed allocation, the vm_radix_sweep() + * will unwind back rn_count appropriately. + */ + rnode->rn_count++; rnode->rn_child[slot] = vm_radix_node_get(); if (rnode->rn_child[slot] == NULL) { CTR6(KTR_VM, @@ -434,35 +530,17 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ "insert: tree %p, rnode %p, child %p, count %u ENOMEM", rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - MPASS(level != clev || allocmsk == 0); - while (allocmsk != 0) { - rnode = root; - level = clev; - level--; - slot = vm_radix_slot(index, level); - CTR4(KTR_VM, - "insert: unwind root %p, level %d, slot %d, allocmsk: 0x%x", - root, level, slot, allocmsk); - MPASS(level >= (ffs(allocmsk) - 1)); - while (level > (ffs(allocmsk) - 1)) { - MPASS(level > 0); - slot = vm_radix_slot(index, - level); - rnode = rnode->rn_child[slot]; - level--; - } - MPASS((allocmsk & (1 << level)) != 0); - allocmsk &= ~(1 << level); - rnode->rn_count--; - vm_radix_node_put(rnode->rn_child[slot]); - rnode->rn_child[slot] = NULL; - } + vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); + + /* + * vm_radix_sweep() may have changed the shape + * of the tree, refetch the root and level. + */ + level = vm_radix_height(rtree, &root); vm_radix_unwind_heightup(rtree, root, iroot, ilevel); return (ENOMEM); } - rnode->rn_count++; - allocmsk |= (1 << level); } CTR6(KTR_VM, "insert: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", @@ -812,86 +890,13 @@ restart: } /* - * Remove the specified index from the tree. If possible the height of the - * tree is adjusted after deletion. The value stored at index is returned - * panics if the key is not present. + * Remove an entry from the tree. Expects the entry to be already present. */ -void * +void vm_radix_remove(struct vm_radix *rtree, vm_pindex_t index, int color) { - struct vm_radix_node *stack[VM_RADIX_LIMIT]; - struct vm_radix_node *rnode, *root; - void *val; - int level; - int slot; - level = vm_radix_height(rtree, &root); - KASSERT(index <= VM_RADIX_MAX(level), - ("vm_radix_remove: %p index out of range %jd.", rtree, - VM_RADIX_MAX(level))); - rnode = root; - val = NULL; - level--; - /* - * Find the node and record the path in stack. - */ - while (level && rnode) { - stack[level] = rnode; - slot = vm_radix_slot(index, level); - CTR6(KTR_VM, - "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", - rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - rnode = rnode->rn_child[slot]; - level--; - } - KASSERT(rnode != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - slot = vm_radix_slot(index, 0); - val = vm_radix_match(rnode->rn_child[slot], color); - KASSERT(val != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - - for (;;) { - CTR6(KTR_VM, -"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, - "remove: resetting tree %p, rnode %p, child %p, count %u", - rtree, rnode, - (rnode != NULL) ? rnode->rn_child[slot] : NULL, - (rnode != NULL) ? rnode->rn_count : 0); - rnode->rn_child[slot] = NULL; - /* - * Use atomics for the last level since red and black - * will both adjust it. - * Use a write memory barrier here in order to avoid - * rn_count reaching 0 before to fetch the actual pointer. - * Concurrent black removal, infact, may want to reclaim - * the radix node itself before to read it. - */ - if (level == 0) - atomic_add_rel_32(&rnode->rn_count, -1); - else - rnode->rn_count--; - /* - * Only allow black removes to prune the tree. - */ - if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) - break; - vm_radix_node_put(rnode); - if (rnode == root) { - vm_radix_setroot(rtree, NULL, 0); - break; - } - rnode = stack[++level]; - slot = vm_radix_slot(index, level); - - } - return (val); + vm_radix_sweep(rtree, index, color, 1); } /* Index: /var/tmp/vmcontention/sys/vm/vm_radix.h =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.h (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.h (working copy) @@ -63,7 +63,7 @@ int vm_radix_lookupn(struct vm_radix *, vm_pindex_ void **, int, vm_pindex_t *); void *vm_radix_lookup_le(struct vm_radix *, vm_pindex_t, int); void vm_radix_reclaim_allnodes(struct vm_radix *); -void *vm_radix_remove(struct vm_radix *, vm_pindex_t, int); +void vm_radix_remove(struct vm_radix *, vm_pindex_t, int); /* * Look up any entry at a position greater or equal to index. -- Test scenario: altbufferflushes.sh