GENERIC HEAD from 2012-05-12 19:54:57 UTC, r236123M, vmcore.383 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r236123M: Tue May 29 10:08:31 CEST 2012 pho@x4.osted.lan:/var/tmp/vmcontention/sys/i386/compile/PHO_KTR i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.01-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) lock order reversal: 1st 0xc1bb505c system map (system map) @ vm/vm_map.c:3723 2nd 0xc130cd00 vm object (kernel object) @ vm/vm_fault.c:299 3rd 0xc1bb50e8 system map (system map) @ vm/vm_map.c:1454 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,34313a63,a3435,ff000a39,c1820740,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda09d,c12d8d48,5ae,c18207d0,...) at kdb_backtrace+0x2a _witness_debugger(c0fda09d,c1bb50e8,c100db0a,c9582110,c100e188,...) at _witness_debugger+0x25 witness_checkorder(c1bb50e8,9,c100e17f,5ae,0,...) at witness_checkorder+0x86f _mtx_lock_flags(c1bb50e8,0,c100e17f,5ae,c596e000,...) at _mtx_lock_flags+0xd9 _vm_map_lock(c1bb508c,c100e17f,5ae,0,c182090b,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,c1820884,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd05c7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,c18208eb,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d144,871,c1820938,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,11,c596ff44,c1820998,c0d0015f,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,c1bb7600,101,827,c1bb1ba0,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,0,5,...) at uma_zalloc_arg+0x59f vm_radix_insert(c130cd30,23480,0,c21fbf80,0,...) at vm_radix_insert+0x1e9 vm_page_insert(c21fbf80,c130cd00,23480,0,0,...) at vm_page_insert+0x1b5 vm_page_alloc(c130cd00,23480,0,40,c1820bbc,...) at vm_page_alloc+0x51c vm_fault_hold(c1bb5000,e337f000,0,1,0,...) at vm_fault_hold+0x60a vm_fault(c1bb5000,e337f000,0,1,c1bb3a68,...) at vm_fault+0x8b vm_fault_wire(c1bb5000,e3000000,e33c9000,0,0,...) at vm_fault_wire+0x3f vm_map_wire(c1bb5000,e3000000,e33c9000,0,e3000000,...) at vm_map_wire+0x2bc kmem_alloc(c1bb5000,3c9000,c955c770,0,0,...) at kmem_alloc+0x10d vm_ksubmap_init(c131e64c,0,1,1000,0,...) at vm_ksubmap_init+0x64 cpu_startup(0,181ec00,181ec00,181e000,1825000,...) at cpu_startup+0x24f mi_startup() at mi_startup+0xac begin() at begin+0x2c avail memory = 3538612224 (3374 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 616460 free (1292 frags, 76896 blocks, 0.1% fragmentation) /dev/label/tmp: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/label/tmp: clean, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 122960 free (5272 frags, 14711 blocks, 0.5% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 2956327 free (219183 frags, 342143 blocks, 2.2% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 7577918 free (77246 frags, 937584 blocks, 0.3% fragmentation) Mounting local file systems:. Setting hostname: x4.osted.lan. re0: link state changed to DOWN re0: link state changed to UP Starting Network: lo0 re0 fwe0 fwip0. lo0: flags=8049 metric 0 mtu 16384 options=3 inet6 ::1 prefixlen 128 inet6 fe80::1%lo0 prefixlen 64 scopeid 0xa inet 127.0.0.1 netmask 0xff000000 nd6 options=21 re0: flags=8843 metric 0 mtu 1500 options=8209b ether 00:21:85:15:33:7d inet 192.168.1.101 netmask 0xffffff00 broadcast 192.168.1.255 inet6 fe80::221:85ff:fe15:337d%re0 prefixlen 64 scopeid 0x1 nd6 options=29 media: Ethernet autoselect (100baseTX ) status: active fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 Starting devd. Starting Network: fwe0. fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 Starting Network: fwip0. fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/kde4/lib /usr/local/lib/compat /usr/local/lib/gcc46 /usr/local/lib/mysql /usr/local/lib/nss /usr/local/lib/qt4 /usr/local/lib/virtualbox a.out ldconfig path: /usr/lib/aout /usr/lib/compat/aout Creating and/or trimming log files. Starting syslogd. lock order reversal: 1st 0xe310c560 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcc38651c snaplk (snaplk) @ ufs/ffs/ffs_snapshot.c:2297 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,632e746f,3932323a,ff000a37,0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d7478,8f9,f404d990,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cc38651c,c1007654,c958aa60,c10075de,...) at _witness_debugger+0x25 witness_checkorder(cc38651c,9,c10075d5,8f9,0,...) at witness_checkorder+0x86f __lockmgr_args(cc38651c,80900,cbc3937c,0,0,...) at __lockmgr_args+0x8c5 ffs_copyonwrite(cbc392b8,e310c500,e310c500,0,0,...) at ffs_copyonwrite+0x2c2 ffs_geom_strategy(cbc393b0,e310c500,cc49415c,e310c500,170f788,...) at ffs_geom_strategy+0xeb bufwrite(e310c500,0,c100aa70,811,0) at bufwrite+0x159 ffs_bufwrite(e310c500,cc498d00,100,4000,0,...) at ffs_bufwrite+0x290 ffs_update(cc495ae0,1,c100b1f1,151,0,...) at ffs_update+0x3a5 ffs_syncvnode(cc495ae0,1,0,c11375a0,cc495bd8,...) at ffs_syncvnode+0x591 ffs_fsync(f404dc40,c102c6e4,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c11375a0,f404dc40,c0fe37a7,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cc1105c0,f404dcec,c1025a7c,c0fdbd8f,c1027030,...) at sys_fsync+0x1df syscall(f404dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x2818ad47, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- No core dumps found. Additional ABI support: linux. Starting rpcbind. NFS access cache time=60 Clearing /tmp (X related). Starting mountd. Starting nfsuserd. Starting nfsd. Recovering vi editor sessions:lock order reversal: 1st 0xcc784df0 ufs (ufs) @ kern/vfs_lookup.c:542 2nd 0xe3129b00 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:260 3rd 0xcc78446c ufs (ufs) @ kern/vfs_subr.c:2159 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,3a632e73,a303632,0,0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda09d,c12d7058,86f,f402b33c,...) at kdb_backtrace+0x2a _witness_debugger(c0fda09d,cc78446c,c0fc2564,c958a928,c0fe20de,...) at _witness_debugger+0x25 witness_checkorder(cc78446c,9,c0fe20d5,86f,0,...) at witness_checkorder+0x86f __lockmgr_args(cc78446c,80100,cc7844d8,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f402b444,c0ad492c,c0fdf52b,5e2,c11088e8,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c11375a0,f402b444,cc1150b0,c1149700,cc784414,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc784414,80100,c0fe20d5,86f,eb,...) at _vn_lock+0x78 vget(cc784414,80100,cc115000,50,0,...) at vget+0xbb vfs_hash_get(cbff9b70,15ef55,80000,cc115000,f402b590,...) at vfs_hash_get+0xed ffs_vgetf(cbff9b70,15ef55,80000,f402b590,1,...) at ffs_vgetf+0x49 softdep_sync_buf(cc784d98,e3129aa0,1,106,0,...) at softdep_sync_buf+0xac9 ffs_syncvnode(cc784d98,1,0,cc1150b0,c11088e8,...) at ffs_syncvnode+0x24c ffs_truncate(cc784d98,200,0,880,cad1d380,...) at ffs_truncate+0x8b5 ufs_direnter(cc784d98,cc784414,f402b904,f402bbb8,0,...) at ufs_direnter+0x980 ufs_makeinode(f402bbb8,c1137b60,f402bafc,f402ba58,c0e28800,...) at ufs_makeinode+0x65d ufs_create(f402bafc,c102d22e,c1149980,cc784d98,f402bb14,...) at ufs_create+0x30 VOP_CREATE_APV(c11375a0,f402bafc,f402bbb8,f402ba94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f402bb78,f402bc40,1b0,0,cad1d380,...) at vn_open_cred+0x205 vn_open(f402bb78,f402bc40,1b0,cc090700,4fc48963,...) at vn_open+0x3b kern_openat(cc115000,ffffff9c,28c72f60,0,a03,...) at kern_openat+0x1ec kern_open(cc115000,28c72f60,0,a02,1b0,...) at kern_open+0x35 sys_open(cc115000,f402bcec,c1025a7c,c0fdad8f,c1027030,...) at sys_open+0x30 syscall(f402bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x283d28d3, esp = 0xbfbfc4ec, ebp = 0xbfbfc578 --- . Updating motd:. Starting ntpd. Starting default moused. Configuring syscons: keymap blanktime. Starting sshd. Starting cron. Local package initialization: backuplock order reversal: 1st 0xcc36e5c8 ufs (ufs) @ kern/vfs_mount.c:1254 2nd 0xcc08e9dc devfs (devfs) @ ufs/ffs/ffs_softdep.c:1834 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,38313a63,a3433,cc5ce46c,8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d6c28,72a,f40119a8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cc08e9dc,c0fcd480,c958a788,c1007b95,...) at _witness_debugger+0x25 witness_checkorder(cc08e9dc,9,c1007b8c,72a,cc08ea48,...) at witness_checkorder+0x86f __lockmgr_args(cc08e9dc,80400,cc08ea48,0,0,...) at __lockmgr_args+0x8c5 vop_stdlock(f4011aac,c130c580,4,c0fd05c7,eb,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c10f7d60,f4011aac,c130c580,c1149700,cc08e984,...) at VOP_LOCK1_APV+0xf3 _vn_lock(cc08e984,80400,c1007b8c,72a,0,...) at _vn_lock+0x78 softdep_flushworklist(cc3782dc,f4011b64,cc115b80,612,cc37b4d8,...) at softdep_flushworklist+0x47 ffs_sync(cc3782dc,1,f4011bb8,51f,c1bae710,...) at ffs_sync+0x3bf dounmount(cc3782dc,8080000,cc115b80,4a1,f4011c3c,...) at dounmount+0x3f8 sys_unmount(cc115b80,f4011cec,c1025a7c,c0fdad52,c1027030,...) at sys_unmount+0x421 syscall(f4011d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280beebb, esp = 0xbfbfe63c, ebp = 0xbfbfe708 --- ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5925 files, 37539076 used, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 5925 files, 37539076 used, 80774905 free (2569 frags, 10096542 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Starting inetd. Tue May 29 10:36:49 CEST 2012 FreeBSD/i386 (x4.osted.lan) (console) login: Expensive timeout(9) function: 0xc08cdd10(0xc132da60) 0.012136095 s lock order reversal: 1st 0xe313ada0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xcc13f600 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:284 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,2e687361,38323a63,c1000a34,cc080c30,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d6cf8,11c,f3fc57c8,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cc13f600,c100ba9a,c958a990,c100b72c,...) at _witness_debugger+0x25 witness_checkorder(cc13f600,9,c100b723,11c,0,...) at witness_checkorder+0x86f _sx_xlock(cc13f600,0,c100b723,11c,cc5e25e4,...) at _sx_xlock+0x9a ufsdirhash_acquire(e313ad40,cc5e25e4,200,e405a400,f3fc5898,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(cc5e25e4,f3fc5904,2400,f3fc5884,f3fc5888,...) at ufsdirhash_add+0x13 ufs_direnter(cc7842b8,ccac415c,f3fc5904,f3fc5bb8,0,...) at ufs_direnter+0x799 ufs_makeinode(f3fc5bb8,c1137b60,f3fc5afc,f3fc5a58,c0e28800,...) at ufs_makeinode+0x65d ufs_create(f3fc5afc,c102d22e,c1149980,cc7842b8,f3fc5b14,...) at ufs_create+0x30 VOP_CREATE_APV(c11375a0,f3fc5afc,f3fc5bb8,f3fc5a94,0,...) at VOP_CREATE_APV+0x100 vn_open_cred(f3fc5b78,f3fc5c40,180,0,cc12eb80,...) at vn_open_cred+0x205 vn_open(f3fc5b78,f3fc5c40,180,cc0d4e38,0,...) at vn_open+0x3b kern_openat(cc080b80,ffffff9c,bfbfe7ab,0,a02,...) at kern_openat+0x1ec kern_open(cc080b80,bfbfe7ab,0,a01,180,...) at kern_open+0x35 sys_open(cc080b80,f3fc5cec,c1025a7c,c0fdad8f,c1027030,...) at sys_open+0x30 syscall(f3fc5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x2829f8d3, esp = 0xbfbfe37c, ebp = 0xbfbfec18 --- May 29 10:44:41 x4 su: pho to root on /dev/pts/0 lock order reversal: 1st 0xcc38651c snaplk (snaplk) @ ufs/ufs/ufs_vnops.c:966 2nd 0xcc37adf0 ufs (ufs) @ ufs/ffs/ffs_snapshot.c:1627 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,a373236,a00,0,f4155680,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0ac239b,c0fda084,c12d63e8,65b,f4155718,...) at kdb_backtrace+0x2a _witness_debugger(c0fda084,cc37adf0,c0fc2564,c958a928,c10075de,...) at _witness_debugger+0x25 witness_checkorder(cc37adf0,9,c10075d5,65b,0,...) at witness_checkorder+0x86f __lockmgr_args(cc37adf0,80000,0,0,0,...) at __lockmgr_args+0x8c5 ffs_snapremove(cc37ad98,c1007513,f4155848,c11a99c4,c119b9c0,...) at ffs_snapremove+0x11f ffs_truncate(cc37ad98,0,0,c00,0,...) at ffs_truncate+0x647 ufs_inactive(f4155ad4,c102cbfd,eb,c11375a0,f4155aa8,...) at ufs_inactive+0x1f6 VOP_INACTIVE_APV(c11375a0,f4155ad4,c0fe20d5,9bf,f4155b08,...) at VOP_INACTIVE_APV+0x100 vinactive(cc37ad98,ccb885c0,c0fe20d5,8be,0,...) at vinactive+0x127 vputx(f4155c34,c0b2e9e3,cc37ad98,ffffffdf,cc37ad98,...) at vputx+0x370 vput(cc37ad98,ffffffdf,cc37ad98,f4155b8c,1d2f0,...) at vput+0x10 kern_unlinkat(ccb885c0,ffffff9c,bfbfec06,0,0,...) at kern_unlinkat+0x2a3 kern_unlink(ccb885c0,bfbfec06,0,f4155d1c,c0e0060e,...) at kern_unlink+0x2f sys_unlink(ccb885c0,f4155cec,c1025a7c,c0fdada3,0,...) at sys_unlink+0x22 syscall(f4155d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (10, FreeBSD ELF32, sys_unlink), eip = 0x2816352f, esp = 0xbfbfe97c, ebp = 0xbfbfe9f8 --- interrupt total irq1: atkbd0 1 irq4: uart0 503 irq6: fdc0 1 irq9: acpi0 1 irq14: ata0 42 irq16: hdac1 ohci0 81 irq17: ohci1 ohci3 3 irq18: ohci2 ohci4 3 irq20: fwohci0 1 irq22: ahci0 1146256 cpu0:timer 5372443 irq256: hdac0 5 irq257: re0 7583 cpu1:timer 5187444 cpu3:timer 5201015 cpu2:timer 5184865 Total 22100280 KDB: stack backtrace: db_trace_self_wrapper(c0fd65b7,41551d,f3ff1828,c0e09bbe,34eb20c5,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0fca24e,1513938,0,1513938,0,...) at kdb_backtrace+0x2a watchdog_fire(1,0,c0fca11b,21b,cc07a000,...) at watchdog_fire+0xa9 hardclock_cnt(1,0,c119c6c0,14,d0769fcf,...) at hardclock_cnt+0x2bc handleevents(f3ff1934,cc080000,f3ff1930,246,c119c6c0,f3ff1954,162b,d0769fcf,39fcd564,8290d00) at handleevents+0x17d timercb(c136d380,0,3,1,cce1eafc,...) at timercb+0x292 lapic_handle_timer(f3ff1978) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0a73f60, esp = 0xf3ff19b8, ebp = 0xf3ff19c8 --- _mtx_assert(cce1eafc,4,c100eff3,141,cce1eafc,...) at _mtx_assert+0xc0 vm_object_pip_add(cce1eafc,1,c100d9e7,12b,f3ff1b64,...) at vm_object_pip_add+0x33 vm_fault_hold(cc39c740,bfbfe000,2,0,0,...) at vm_fault_hold+0x1a9 vm_fault(cc39c740,bfbfe000,2,0,bfbfec10,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,cc080000,cc07a000,cc07a000,...) at trap_pfault+0x1b9 trap(f3ff1d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c0e7, esp = 0xbfbfec10, ebp = 0xbfbfec48 --- KDB: enter: watchdog timeout [ thread pid 2744 tid 100090 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 2744 tid 100090 td 0xcc080000 kdb_enter(c0e655d6,c0f57edc,0,1513938,0) at kdb_enter+0x3a watchdog_fire(1,0,c0fca11b,21b,cc07a000,...) at watchdog_fire+0xbd hardclock_cnt(1,0,c119c6c0,14,d0769fcf,...) at hardclock_cnt+0x2bc handleevents(f3ff1934,cc080000,f3ff1930,246,c119c6c0,f3ff1954,162b,d0769fcf,39fcd564,8290d00) at handleevents+0x17d timercb(c136d380,0,3,1,cce1eafc,...) at timercb+0x292 lapic_handle_timer(f3ff1978) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0a73f60, esp = 0xf3ff19b8, ebp = 0xf3ff19c8 --- _mtx_assert(cce1eafc,4,c100eff3,141,cce1eafc,...) at _mtx_assert+0xc0 vm_object_pip_add(cce1eafc,1,c100d9e7,12b,f3ff1b64,...) at vm_object_pip_add+0x33 vm_fault_hold(cc39c740,bfbfe000,2,0,0,...) at vm_fault_hold+0x1a9 vm_fault(cc39c740,bfbfe000,2,0,bfbfec10,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,cc080000,cc07a000,cc07a000,...) at trap_pfault+0x1b9 trap(f3ff1d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c0e7, esp = 0xbfbfec10, ebp = 0xbfbfec48 --- db:0:bt> show allpcpu Current CPU: 3 cpuid = 0 dynamic pcpu = 0x6d4d00 curthread = 0xcae245c0: pid 7 "pagedaemon" curpcb = 0xefbd7d80 fpcurthread = none idlethread = 0xc95df8a0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x828ad00 curthread = 0xcc1105c0: pid 961 "syslogd" curpcb = 0xf404dd80 fpcurthread = none idlethread = 0xc95df5c0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: exclusive spin mutex smp rendezvous (smp rendezvous) r = 0 (0xc11a9dd0) locked @ i386/i386/mp_machdep.c:1213 cpuid = 2 dynamic pcpu = 0x828dd00 curthread = 0xcc0b7b80: pid 1151 "sendmail" curpcb = 0xf3fe9d80 fpcurthread = none idlethread = 0xc95df2e0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x8290d00 curthread = 0xcc080000: pid 2744 "cron" curpcb = 0xf3ff1d80 fpcurthread = none idlethread = 0xc95df000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 2745 (sendmail) thread 0xcc9862e0 (100133) exclusive sleep mutex vm object (standard object) r = 0 (0xccb9e094) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc39c958) locked @ vm/vm_map.c:3723 Process 2744 (cron) thread 0xcc080000 (100090) exclusive sleep mutex vm object (standard object) r = 0 (0xcce1eafc) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc39c788) locked @ vm/vm_map.c:3723 Process 2742 (watchdogd) thread 0xccb88000 (100170) exclusive sleep mutex page lock (page lock) r = 0 (0xc130d980) locked @ vm/vm_page.c:1278 exclusive sleep mutex vm object (standard object) r = 0 (0xccb838ac) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xc95e4788) locked @ vm/vm_map.c:3723 Process 2695 (dd) thread 0xccaf52e0 (100166) exclusive lockmgr bufwait (bufwait) r = 0 (0xe338f3a0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe32d9bc0) locked @ kern/vfs_bio.c:2652 exclusive lockmgr ufs (ufs) r = 0 (0xccb845c8) locked @ kern/vfs_vnops.c:645 Process 1316 (sshd) thread 0xccaf65c0 (100154) exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xcc7ef460) locked @ kern/uipc_sockbuf.c:148 Process 1204 (watchdogd) thread 0xcc1155c0 (100102) exclusive sleep mutex vm object (standard object) r = 0 (0xcd2f9d4c) locked @ vm/vm_fault.c:299 shared sx user map (user map) r = 0 (0xcc0783e8) locked @ vm/vm_map.c:3723 Process 1151 (sendmail) thread 0xcc0b7b80 (100088) exclusive sleep mutex vm object (standard object) r = 0 (0xcc801c24) locked @ vm/vm_object.c:1730 exclusive sleep mutex vm object (standard object) r = 0 (0xcc133940) locked @ vm/vm_map.c:2961 exclusive sx user map (user map) r = 0 (0xcc39cec8) locked @ vm/vm_map.c:3105 exclusive sx user map (user map) r = 0 (0xc95e5c10) locked @ vm/vm_map.c:3101 Process 1091 (ntpd) thread 0xcbee85c0 (100095) exclusive sleep mutex vm object (standard object) r = 0 (0xcc133e74) locked @ vm/vm_page.c:2295 exclusive lockmgr bufwait (bufwait) r = 0 (0xe33259a0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr ufs (ufs) r = 0 (0xccb8446c) locked @ kern/vfs_vnops.c:645 Process 961 (syslogd) thread 0xcc1105c0 (100110) exclusive lockmgr bufwait (bufwait) r = 0 (0xe33657c0) locked @ kern/vfs_bio.c:1905 exclusive lockmgr snaplk (snaplk) r = 0 (0xcc38651c) locked @ ufs/ffs/ffs_snapshot.c:2297 exclusive lockmgr ufs (ufs) r = 0 (0xcc495b38) locked @ kern/vfs_syscalls.c:3587 Process 19 (softdepflush) thread 0xcbee88a0 (100078) exclusive lockmgr bufwait (bufwait) r = 0 (0xe3244a40) locked @ kern/vfs_bio.c:1905 exclusive lockmgr bufwait (bufwait) r = 0 (0xe3246c60) locked @ kern/vfs_bio.c:1905 Process 7 (pagedaemon) thread 0xcae245c0 (100072) exclusive sleep mutex mbuf_jumbo_16k (UMA zone) r = 0 (0xc1ba0588) locked @ vm/uma_core.c:785 exclusive sleep mutex UMA lock (UMA lock) r = 0 (0xc130c96c) locked @ vm/uma_core.c:1596 db:0:alllocks> show lockedvnods Locked vnodes 0xcc495ae0: tag ufs, type VREG usecount 2, writecount 1, refcount 4 mountedhere 0 flags (VI(0x200)) v_object 0xcc3ba1bc ref 0 pages 2 lock type ufs: EXCL by thread 0xcc1105c0 (pid 961, syslogd, tid 100110) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273c3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b35f66 at sys_fsync+0x116 #5 0xc0e0060e at syscall+0x2fe #6 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 6052908, on dev ada0s1e 0xccae0c3c: tag ufs, type VREG usecount 1, writecount 0, refcount 50 mountedhere 0 flags (VV_SYSTEM|VI(0x200)) v_object 0xccb68250 ref 0 pages 1972 lock type snaplk: EXCL by thread 0xcc1105c0 (pid 961, syslogd, tid 100110) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0cbf2a2 at ffs_copyonwrite+0x2c2 #2 0xc0cdc2bb at ffs_geom_strategy+0xeb #3 0xc0cf0ff3 at ufs_strategy+0xd3 #4 0xc0e27099 at VOP_STRATEGY_APV+0x119 #5 0xc0b09251 at bufstrategy+0x61 #6 0xc0b0cce9 at bufwrite+0x159 #7 0xc0b0d058 at vfs_bio_awrite+0x328 #8 0xc0ce2df5 at ffs_syncvnode+0x3a5 #9 0xc0ce304f at ffs_fsync+0x2f #10 0xc0e23b00 at VOP_FSYNC_APV+0x100 #11 0xc0b3602f at sys_fsync+0x1df #12 0xc0e0060e at syscall+0x2fe #13 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 7, on dev ada0s1e 0xccb84570: tag ufs, type VREG usecount 1, writecount 1, refcount 7078 mountedhere 0 flags (VI(0x200)) v_object 0xcac0d40c ref 0 pages 470124 lock type ufs: EXCL by thread 0xccaf52e0 (pid 2695, dd, tid 100166) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273c3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b38c16 at vn_write+0x176 #5 0xc0ad6b85 at dofilewrite+0x95 #6 0xc0ad71f8 at kern_writev+0x68 #7 0xc0ad730f at sys_write+0x4f #8 0xc0e0060e at syscall+0x2fe #9 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 6194267, on dev ada0s1e 0xccb84414: tag ufs, type VREG usecount 1, writecount 1, refcount 2 mountedhere 0 flags (VI(0x200)) v_object 0xcc133e74 ref 0 pages 0 lock type ufs: EXCL by thread 0xcbee85c0 (pid 1091, ntpd, tid 100095) #0 0xc0a6c61d at __lockmgr_args+0xd6d #1 0xc0ce2991 at ffs_lock+0xa1 #2 0xc0e273c3 at VOP_LOCK1_APV+0xf3 #3 0xc0b37dc8 at _vn_lock+0x78 #4 0xc0b38c16 at vn_write+0x176 #5 0xc0ad6b85 at dofilewrite+0x95 #6 0xc0ad71f8 at kern_writev+0x68 #7 0xc0ad730f at sys_write+0x4f #8 0xc0e0060e at syscall+0x2fe #9 0xc0dea8d1 at Xint0x80_syscall+0x21 ino 1742962, on dev ada0s1e db:0:lockedvnods> show mount 0xcbff8b70 /dev/ad4s1a on / (ufs) 0xcbff9000 devfs on /dev (devfs) 0xcc3785b8 /dev/ad4s1f on /home (ufs) 0xcc378000 /dev/ad4s1d on /usr (ufs) 0xcbff9b70 /dev/ad4s1e on /var (ufs) 0xcbff9894 procfs on /proc (procfs) 0xcca0d000 /dev/label/tmp on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 2745 1154 1154 25 RL sendmail 2744 1159 1159 0 RL CPU 3 cron 2743 1320 1319 1001 RL sh 2742 1204 1204 0 RL watchdogd 2696 1339 1339 0 S+ piperd 0xcca8d000 egrep 2695 1339 1339 0 R+ dd 1339 1302 1339 0 S+ wait 0xcc8082cc sh 1321 1319 1319 1001 S piperd 0xcc0ceaf0 awk 1320 1319 1319 1001 DL pfault 0xc130cc7c sh 1319 1318 1319 1001 Ss wait 0xcc10eb30 sh 1318 1310 1310 1001 S select 0xcbff79a4 sshd 1317 1316 1317 1001 Ss+ ttyout 0xcbfd9e78 top 1316 1309 1309 1001 R sshd 1315 1314 1315 1001 Rs tail 1314 1308 1308 1001 S select 0xcbff7d24 sshd 1310 1148 1310 0 Ss select 0xcc1315e4 sshd 1309 1148 1309 0 Ss select 0xcc7934a4 sshd 1308 1148 1308 0 Ss select 0xcac86be4 sshd 1302 1299 1302 0 S+ wait 0xcc0b32cc bash 1299 1298 1299 0 S+ pause 0xcbedc324 csh 1298 1293 1298 1001 S+ wait 0xcca90b30 su 1293 1292 1293 1001 Ss+ wait 0xcca90864 bash 1292 1290 1290 1001 S select 0xcc0aaa24 sshd 1290 1148 1290 0 Ss select 0xcc1312e4 sshd 1254 1 1254 0 Ss+ ttyin 0xcae33870 getty 1253 1 1253 0 Ss+ ttyin 0xcae33a70 getty 1252 1 1252 0 Ss+ ttyin 0xc970d270 getty 1251 1 1251 0 Ss+ ttyin 0xc970d470 getty 1250 1 1250 0 Ss+ ttyin 0xc970d670 getty 1249 1 1249 0 Ss+ ttyin 0xc970d870 getty 1248 1 1248 0 Ss+ ttyin 0xc970da70 getty 1247 1 1247 0 Ss+ ttyin 0xc970dc70 getty 1246 1 1246 0 Ss+ ttyin 0xc970de70 getty 1229 1 1229 0 Ss select 0xcc7936a4 inetd 1204 1 1204 0 LLs *page loc 0xcc0b6280 watchdogd 1159 1 1159 0 RLs cron 1154 1 1154 25 RLs sendmail 1151 1 1151 0 Rs CPU 2 sendmail 1148 1 1148 0 Ss select 0xcc793524 sshd 1116 1 1116 0 Ss select 0xcc7937a4 moused 1091 1 1091 0 Rs ntpd 1035 1031 1031 0 S (threaded) nfsd 100125 S rpcsvc 0xcaca9dd0 nfsd: service 100124 S rpcsvc 0xcaca9e10 nfsd: service 100123 S rpcsvc 0xcaca9e50 nfsd: service 100117 S rpcsvc 0xcc079c50 nfsd: master 1031 1 1031 0 Ss select 0xcac85964 nfsd 1029 1025 1025 0 S select 0xcbff7c64 nfsuserd 1028 1025 1025 0 S select 0xcac791a4 nfsuserd 1027 1025 1025 0 S select 0xcbff7c24 nfsuserd 1026 1025 1025 0 S select 0xcac60864 nfsuserd 1025 1 1025 0 Ss pause 0xcc10f8bc nfsuserd 1009 1 1009 0 Ss select 0xcbff7064 mountd 976 1 976 0 Ss select 0xcac7aea4 rpcbind 961 1 961 0 Rs CPU 1 syslogd 838 1 838 0 Ss select 0xcbff7664 devd 19 0 0 0 RL [softdepflush] 18 0 0 0 DL vlruwt 0xcbedb2cc [vnlru] 17 0 0 0 DL syncer 0xc13048b4 [syncer] 16 0 0 0 DL psleep 0xc1304548 [bufdaemon] 9 0 0 0 DL pgzero 0xc130e468 [pagezero] 8 0 0 0 DL psleep 0xc130e048 [vmdaemon] 7 0 0 0 RL CPU 0 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc1151054 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc1307dd8 [sctp_iterator] 4 0 0 0 DL - 0xcac7223c [fdc0] 3 0 0 0 SL - 0xcad2c000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcacfed34 [usbus5] 100058 D - 0xcacfed04 [usbus5] 100057 D - 0xcacfecd4 [usbus5] 100056 D - 0xcacfeca4 [usbus5] 100054 D - 0xcacefb5c [usbus4] 100053 D - 0xcacefb2c [usbus4] 100052 D - 0xcacefafc [usbus4] 100051 D - 0xcacefacc [usbus4] 100050 D - 0xcaceab5c [usbus3] 100049 D - 0xcaceab2c [usbus3] 100048 D - 0xcaceaafc [usbus3] 100047 D - 0xcaceaacc [usbus3] 100046 D - 0xcace5b5c [usbus2] 100045 D - 0xcace5b2c [usbus2] 100044 D - 0xcace5afc [usbus2] 100043 D - 0xcace5acc [usbus2] 100041 D - 0xcacddb5c [usbus1] 100040 D - 0xcacddb2c [usbus1] 100039 D - 0xcacddafc [usbus1] 100038 D - 0xcacddacc [usbus1] 100036 D - 0xcacd5b5c [usbus0] 100035 D - 0xcacd5b2c [usbus0] 100034 D - 0xcacd5afc [usbus0] 100033 D - 0xcacd5acc [usbus0] 2 0 0 0 DL ctl_work 0xc97c4000 [ctl_thrd] 14 0 0 0 DL - 0xc118d4e4 [yarrow] 13 0 0 0 DL (threaded) [geom] 100016 D - 0xc118a5a8 [g_down] 100015 D - 0xc118a5a4 [g_up] 100014 D - 0xc118a59c [g_event] 12 0 0 0 WL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100018 I [swi2: cambio] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 CanRun [idle: cpu2] 100004 CanRun [idle: cpu1] 100003 CanRun [idle: cpu0] 1 0 1 0 SLs wait 0xc95ddb30 [init] 10 0 0 0 DL audit_wo 0xc130bd00 [audit] 0 0 0 0 DLs (threaded) [kernel] 100069 D - 0xc118d4e4 [deadlkres] 100068 D - 0xc971ddc0 [mca taskq] 100062 D - 0xc971de00 [fw0_taskq] 100027 D - 0xc971de40 [kqueue taskq] 100026 D - 0xc971de80 [ffs_trim taskq] 100025 D - 0xc971e000 [acpi_task_2] 100024 D - 0xc971e000 [acpi_task_1] 100023 D - 0xc971e000 [acpi_task_0] 100020 D - 0xc971e0c0 [thread taskq] 100013 D - 0xc96e9000 [firmware taskq] 100000 D sched 0xc118a760 [swapper] db:0:ps> allt Tracing command sendmail pid 2745 tid 100133 td 0xcc9862e0 sched_switch(cc9862e0,0,103,19e,d4bcdc10,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,810,...) at mi_switch+0x1ff turnstile_wait(cc07ea00,cc115b80,0,1c5,101,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d680,cc9862e0,0,c100f607,4fe,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d680,0,c100f607,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(ccb9e094,28,0,0,f40e0b64,...) at vm_page_alloc+0x565 vm_fault_hold(cc39c910,811c000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc39c910,811c000,2,0,811c750,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,cc9862e0,cc807864,cc807864,...) at trap_pfault+0x1b9 trap(f40e0d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a20ec, esp = 0xbfbfcf10, ebp = 0xbfbfcf78 --- Tracing command cron pid 2744 tid 100090 td 0xcc080000 kdb_enter(c0e655d6,c0f57edc,0,1513938,0) at kdb_enter+0x3a watchdog_fire(1,0,c0fca11b,21b,cc07a000,...) at watchdog_fire+0xbd hardclock_cnt(1,0,c119c6c0,14,d0769fcf,...) at hardclock_cnt+0x2bc handleevents(f3ff1934,cc080000,f3ff1930,246,c119c6c0,f3ff1954,162b,d0769fcf,39fcd564,8290d00) at handleevents+0x17d timercb(c136d380,0,3,1,cce1eafc,...) at timercb+0x292 lapic_handle_timer(f3ff1978) at lapic_handle_timer+0x7f Xtimerint() at Xtimerint+0x20 --- interrupt, eip = 0xc0a73f60, esp = 0xf3ff19b8, ebp = 0xf3ff19c8 --- _mtx_assert(cce1eafc,4,c100eff3,141,cce1eafc,...) at _mtx_assert+0xc0 vm_object_pip_add(cce1eafc,1,c100d9e7,12b,f3ff1b64,...) at vm_object_pip_add+0x33 vm_fault_hold(cc39c740,bfbfe000,2,0,0,...) at vm_fault_hold+0x1a9 vm_fault(cc39c740,bfbfe000,2,0,bfbfec10,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,cc080000,cc07a000,cc07a000,...) at trap_pfault+0x1b9 trap(f3ff1d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c0e7, esp = 0xbfbfec10, ebp = 0xbfbfec48 --- Tracing command sh pid 2743 tid 100174 td 0xccb872e0 sched_switch(ccb872e0,0,104,1a3,d0e3120,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(ccb872e0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(ccb89378,1e,0,0,f4173b64,...) at vm_waitpfault+0x75 vm_fault_hold(cd097740,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cd097740,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,ccb872e0,ccb6db30,ccb6db30,...) at trap_pfault+0x1b9 trap(f4173d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- Tracing command watchdogd pid 2742 tid 100170 td 0xccb88000 sched_switch(ccb88000,0,103,19e,d534f1b,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,8e8,...) at mi_switch+0x1ff turnstile_wait(cc3a6180,cbee68a0,0,1c5,c1bb6a10,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130df80,ccb88000,0,c10245f2,10b9,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130df80,0,c10245f2,10b9,c2e9fd80,...) at _mtx_lock_flags+0x10f pmap_page_is_mapped(c2e9fd80,4,c100f607,72a,c2e9fd80,...) at pmap_page_is_mapped+0x3e vm_page_free_toq(c2e9fd80,f41679e8,c0d17263,c2e9fd80,0,...) at vm_page_free_toq+0x48 vm_page_free(c2e9fd80,0,c100f607,4fe,0,...) at vm_page_free+0x15 vm_page_alloc(ccb838ac,1e,0,0,f4167b64,...) at vm_page_alloc+0x583 vm_fault_hold(c95e4740,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(c95e4740,bfbfe000,2,0,bfbfec38,...) at vm_fault+0x8b trap_pfault(c0dffb1f,0,ccb88000,ccb6e864,ccb6e864,...) at trap_pfault+0x1b9 trap(f4167d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bcea6, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command egrep pid 2696 tid 100199 td 0xcd0c78a0 sched_switch(cd0c78a0,0,104,1a3,3087c693,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cd0c78a0,0,c0fd7fa0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cca8d000,5c,c0fda927,100,0,...) at sleepq_wait_sig+0x17 _sleep(cca8d000,cca8d178,15c,c0fda927,0,...) at _sleep+0x385 pipe_read(cc0d4188,f41ebc3c,cc005780,0,cd0c78a0,...) at pipe_read+0x427 dofileread(f41ebc3c,ffffffff,ffffffff,0,f41ebc18,...) at dofileread+0x9e kern_readv(cd0c78a0,0,f41ebc3c,f41ebc5c,1,...) at kern_readv+0x68 sys_read(cd0c78a0,f41ebcec,c1025a7c,c0fdae1a,c1027030,...) at sys_read+0x4f syscall(f41ebd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281baed3, esp = 0xbfbfe80c, ebp = 0xbfbfe898 --- Tracing command dd pid 2695 tid 100166 td 0xccaf52e0 sched_switch(ccaf52e0,0,103,19e,e5c2f74b,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,768,...) at mi_switch+0x1ff turnstile_wait(cacc0400,ccaf65c0,0,1c5,c12d63e8,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130cf00,ccaf52e0,0,c100f607,65e,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130cf00,0,c100f607,65e,1000,...) at _mtx_lock_flags+0x10f vm_wait(cac0d40c,0,c100f607,8f5,40222,...) at vm_wait+0x2a vm_page_grab(cac0d40c,72b80,0,412a2,c12d63c0,...) at vm_page_grab+0x180 allocbuf(e338f340,4000,c0fdf52b,b00,ffffffe0,...) at allocbuf+0x672 getblk(ccb84570,1cae0,0,4000,0,...) at getblk+0x5e3 ffs_balloc_ufs2(ccb84570,72b80000,0,4000,cc005780,...) at ffs_balloc_ufs2+0x1e6c ffs_write(f415bba4,c102d163,ccb84570,ccb84570,f415bb5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c11375a0,f415bba4,ccb84570,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cd0d3b60,f415bc3c,cc005780,0,ccaf52e0,...) at vn_write+0x2bc dofilewrite(f415bc3c,ffffffff,ffffffff,0,f415bc18,...) at dofilewrite+0x95 kern_writev(ccaf52e0,4,f415bc3c,f415bc5c,1,...) at kern_writev+0x68 sys_write(ccaf52e0,f415bcec,c1025a7c,c0e3210f,c119c040,...) at sys_write+0x4f syscall(f415bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28177eb3, esp = 0xbfbfe9ac, ebp = 0xbfbfe9d8 --- Tracing command sh pid 1339 tid 100130 td 0xcc0b95c0 sched_switch(cc0b95c0,0,104,1a3,a735a076,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0b95c0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc8082cc,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc8082cc,cc808354,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc0b95c0,ffffffff,f40d7c58,2,0,...) at kern_wait+0x597 sys_wait4(cc0b95c0,f40d7cec,c1025a7c,c0fdaeb7,c1027030,...) at sys_wait4+0x3b syscall(f40d7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfe77c, ebp = 0xbfbfe798 --- Tracing command awk pid 1321 tid 100119 td 0xcae265c0 sched_switch(cae265c0,0,104,1a3,825d75ce,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae265c0,0,c0fd7fa0,1a7,5c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0ceaf0,5c,c0fda927,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0ceaf0,cc0cec68,15c,c0fda927,0,...) at _sleep+0x385 pipe_read(cc0cc348,f4082c3c,cc005d80,0,cae265c0,...) at pipe_read+0x427 dofileread(f4082c3c,ffffffff,ffffffff,0,f4082c18,...) at dofileread+0x9e kern_readv(cae265c0,0,f4082c3c,f4082c5c,1,...) at kern_readv+0x68 sys_read(cae265c0,f4082cec,c1025a7c,c0e3210f,c1027030,...) at sys_read+0x4f syscall(f4082d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281aced3, esp = 0xbfbfe92c, ebp = 0xbfbfe948 --- Tracing command sh pid 1320 tid 100098 td 0xcc115b80 sched_switch(cc115b80,0,104,1a3,10271482,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cc115b80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(cec4a534,1e,0,0,f4011b64,...) at vm_waitpfault+0x75 vm_fault_hold(cc9d1570,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc9d1570,bfbfe000,2,0,bfbfea38,...) at vm_fault+0x8b trap_pfault(f4011c58,c0a8e090,c119b370,ccb872e0,cc10e598,...) at trap_pfault+0x1b9 trap(f4011d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8054b7f, esp = 0xbfbfea20, ebp = 0xbfbfea48 --- Tracing command sh pid 1319 tid 100092 td 0xcc1162e0 sched_switch(cc1162e0,0,104,1a3,d206adc6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc1162e0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc10eb30,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc10eb30,cc10ebb8,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc1162e0,ffffffff,f3ff9c58,2,0,...) at kern_wait+0x597 sys_wait4(cc1162e0,f3ff9cec,c1025a7c,c0fdaeb7,c1027030,...) at sys_wait4+0x3b syscall(f3ff9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2815078b, esp = 0xbfbfeb0c, ebp = 0xbfbfeb28 --- Tracing command sshd pid 1318 tid 100129 td 0xcc0b98a0 sched_switch(cc0b98a0,0,104,1a3,9f163570,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b98a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbff7990,0,c0fd05c7,cc0b98a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbff79a4,0,f40d4a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbff79a4,cbff7990,c0fda6f8,6ae,cc090d20,...) at _cv_wait_sig+0x260 seltdwait(cc090d20,58,cc0a8b00,cc0b98a0,cc0b9950,...) at seltdwait+0xa2 kern_select(cc0b98a0,c,28c0c170,28c0c1b0,0,0,20,cc808598,0) at kern_select+0x655 sys_select(cc0b98a0,f40d4cec,c1025a7c,c0fdb025,c1027030,...) at sys_select+0x66 syscall(f40d4d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command top pid 1317 tid 100106 td 0xcc1108a0 sched_switch(cc1108a0,0,104,1a3,c8570c40,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc1108a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbfd9e04,0,c0fd05c7,cc1108a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbfd9e78,0,f4033a00,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbfd9e78,cbfd9e04,c0fdc738,527,14,...) at _cv_wait_sig+0x260 tty_wait(cbfd9e00,cbfd9e78,c0fdc83a,65,0,...) at tty_wait+0x71 ttydisc_write(cbfd9e00,f4033c3c,0,a1,25f,...) at ttydisc_write+0x2f0 ttydev_write(ccaf1000,f4033c3c,0,0,0,...) at ttydev_write+0xfa devfs_write_f(cc0cc3f0,f4033c3c,cc005000,0,cc1108a0,...) at devfs_write_f+0xa9 dofilewrite(f4033c3c,ffffffff,ffffffff,0,f4033c18,...) at dofilewrite+0x95 kern_writev(cc1108a0,1,f4033c3c,f4033c5c,1,...) at kern_writev+0x68 sys_write(cc1108a0,f4033cec,c1825c80,0,c1027030,...) at sys_write+0x4f syscall(f4033d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x281e1eb3, esp = 0xbfbfea9c, ebp = 0xbfbfeab8 --- Tracing command sshd pid 1316 tid 100154 td 0xccaf65c0 sched_switch(ccaf65c0,0,104,1a3,d536a3b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(ccaf65c0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,54,c100f8db,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,254,c100f8db,0,...) at _sleep+0x39c vm_wait(c1bb5118,c100deab,174,172,cd0df000,...) at vm_wait+0xbe kmem_back(c1bb5118,cd0df000,1000,102,2,...) at kmem_back+0x1e5 kmem_malloc(c1bb5118,1000,102,f4137998,c0cfe329,...) at kmem_malloc+0x2a0 page_alloc(c1baaa80,1000,f413798b,102,c1baaa80,...) at page_alloc+0x27 keg_alloc_slab(c1ba0408,4,c100d144,871,c0a74f36,...) at keg_alloc_slab+0xf9 keg_fetch_slab(2,2,c1baab18,f4137a38,c0d0015f,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1baaa80,0,2,827,cd0cf000,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1baaa80,cc3cc400,2,4a,0,...) at uma_zalloc_arg+0x59f mb_zinit_pack(cc3cc400,100,2,999,617,...) at mb_zinit_pack+0x23 uma_zalloc_arg(c1baab60,f4137ad4,2,af0001,0,...) at uma_zalloc_arg+0x661 m_getm2(0,180,2,1,0,...) at m_getm2+0x128 m_uiotombuf(f4137c3c,2,8218,0,0,...) at m_uiotombuf+0x77 sosend_generic(cc7ef340,0,f4137c3c,0,0,...) at sosend_generic+0x432 sosend(cc7ef340,0,f4137c3c,0,0,...) at sosend+0x3f soo_write(cc0db1f8,f4137c3c,cc12e880,0,ccaf65c0,...) at soo_write+0x63 dofilewrite(f4137c3c,ffffffff,ffffffff,0,f4137c18,...) at dofilewrite+0x95 kern_writev(ccaf65c0,3,f4137c3c,f4137c5c,1,...) at kern_writev+0x68 sys_write(ccaf65c0,f4137cec,c1025a7c,c0fdadd7,f4137cb4,...) at sys_write+0x4f syscall(f4137d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x284daeb3, esp = 0xbfbfddac, ebp = 0xbfbfddd8 --- Tracing command tail pid 1315 tid 100121 td 0xcc3a88a0 sched_switch(cc3a88a0,0,104,1a3,e5cc6e54,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,68,...) at mi_switch+0x1ff sleepq_switch(cc3a88a0,0,c0fd7fa0,1a7,68,...) at sleepq_switch+0x162 sleepq_catch_signals(3e9,c0ac8bc0,cc3a88a0,0,100,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbfd0380,68,c0fcc162,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(cbfd0380,cbfd0380,168,c0fcc162,3e9,...) at _sleep+0x34f kern_kevent(cc3a88a0,4,0,1,f408ac3c,...) at kern_kevent+0x374 sys_kevent(cc3a88a0,f408acec,c1025a7c,c0fa5642,f408acb4,...) at sys_kevent+0x19e syscall(f408ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2815639b, esp = 0xbfbfeb0c, ebp = 0xbfbfebc8 --- Tracing command sshd pid 1314 tid 100141 td 0xcc3a7b80 sched_switch(cc3a7b80,0,104,1a3,e5d914e4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a7b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbff7d10,0,c0fd05c7,cc3a7b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbff7d24,0,f4110a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbff7d24,cbff7d10,c0fda6f8,6ae,cc0cc850,...) at _cv_wait_sig+0x260 seltdwait(cc0cc850,58,cc003200,cc3a7b80,cc3a7c30,...) at seltdwait+0xa2 kern_select(cc3a7b80,c,28c0c170,28c0c1b0,0,0,20,cca90598,0) at kern_select+0x655 sys_select(cc3a7b80,f4110cec,c1025a7c,c0fdb025,f4110cb4,...) at sys_select+0x66 syscall(f4110d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1310 tid 100153 td 0xccaf68a0 sched_switch(ccaf68a0,0,104,1a3,c969cea1,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(ccaf68a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc1315d0,0,c0fd05c7,ccaf68a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc1315e4,0,f4134ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc1315e4,cc1315d0,c0fda6f8,6ae,f4134b3c,...) at _cv_wait_sig+0x260 seltdwait(cc3ca72c,c0fda6f8,5ab,ccaf68a0,0,...) at seltdwait+0xa2 sys_poll(ccaf68a0,f4134cec,c1025a7c,c0e3210f,c1027030,...) at sys_poll+0x321 syscall(f4134d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1309 tid 100142 td 0xcc3a78a0 sched_switch(cc3a78a0,0,104,1a3,c7b8afaa,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a78a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc793490,0,c0fd05c7,cc3a78a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc7934a4,0,f4113ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc7934a4,cc793490,c0fda6f8,6ae,f4113b3c,...) at _cv_wait_sig+0x260 seltdwait(cc3cad2c,c0fda6f8,5ab,cc3a78a0,0,...) at seltdwait+0xa2 sys_poll(cc3a78a0,f4113cec,c1025a7c,c0e3210f,c1027030,...) at sys_poll+0x321 syscall(f4113d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command sshd pid 1308 tid 100115 td 0xcbee62e0 sched_switch(cbee62e0,0,104,1a3,c4c1de8c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee62e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac86bd0,0,c0fd05c7,cbee62e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac86be4,0,f4071ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac86be4,cac86bd0,c0fda6f8,6ae,f4071b3c,...) at _cv_wait_sig+0x260 seltdwait(c95c322c,c0fda6f8,5ab,cbee62e0,0,...) at seltdwait+0xa2 sys_poll(cbee62e0,f4071cec,c1025a7c,c0e3210f,c1027030,...) at sys_poll+0x321 syscall(f4071d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command bash pid 1302 tid 100083 td 0xcc0b85c0 sched_switch(cc0b85c0,0,104,1a3,4cca4ea7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc0b85c0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0b32cc,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc0b32cc,cc0b3354,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc0b85c0,ffffffff,f3fd5c58,6,0,...) at kern_wait+0x597 sys_wait4(cc0b85c0,f3fd5cec,c1025a7c,c0fdadd7,c1027030,...) at sys_wait4+0x3b syscall(f3fd5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe76c, ebp = 0xbfbfe788 --- Tracing command csh pid 1299 tid 100108 td 0xcc07f2e0 sched_switch(cc07f2e0,0,104,1a3,e245c276,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cc07f2e0,0,c0fd7fa0,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbedc324,74,c0e58cee,100,0,...) at sleepq_wait_sig+0x17 _sleep(cbedc324,cbedc354,174,c0e58cee,0,...) at _sleep+0x385 kern_sigsuspend(cc07f2e0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cc07f2e0,f403ecec,c1025a7c,c0fdadd7,c96eb2e0,...) at sys_sigsuspend+0x4d syscall(f403ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x2816fd7b, esp = 0xbfbfe05c, ebp = 0xbfbfe0c8 --- Tracing command su pid 1298 tid 100139 td 0xcc3a82e0 sched_switch(cc3a82e0,0,104,1a3,8d6f9d09,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc3a82e0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cca90b30,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cca90b30,cca90bb8,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc3a82e0,513,f410ac58,2,0,...) at kern_wait+0x597 sys_wait4(cc3a82e0,f410acec,c1025a7c,c0fdb6ee,c1027030,...) at sys_wait4+0x3b syscall(f410ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x2810b78b, esp = 0xbfbfe5ac, ebp = 0xbfbfe5c8 --- Tracing command bash pid 1293 tid 100140 td 0xcc3a8000 sched_switch(cc3a8000,0,104,1a3,890d6d3b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cc3a8000,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cca90864,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(cca90864,cca908ec,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(cc3a8000,ffffffff,f410dc58,6,0,...) at kern_wait+0x597 sys_wait4(cc3a8000,f410dcec,c1025a7c,c0fdadd7,c1027030,...) at sys_wait4+0x3b syscall(f410dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x282ad78b, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command sshd pid 1292 tid 100103 td 0xcc1152e0 sched_switch(cc1152e0,0,104,1a3,719cdfc8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc1152e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc0aaa10,0,c0fd05c7,cc1152e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc0aaa24,0,f4027a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc0aaa24,cc0aaa10,c0fda6f8,6ae,cc090380,...) at _cv_wait_sig+0x260 seltdwait(cc090380,58,cc12e480,cc1152e0,cc115390,...) at seltdwait+0xa2 kern_select(cc1152e0,c,28c0c170,28c0c1b0,0,0,20,cc10bb30,0) at kern_select+0x655 sys_select(cc1152e0,f4027cec,c1025a7c,c0fdb025,f4027cb4,...) at sys_select+0x66 syscall(f4027d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfde1c, ebp = 0xbfbfde68 --- Tracing command sshd pid 1290 tid 100143 td 0xcc3a75c0 sched_switch(cc3a75c0,0,104,1a3,9309f712,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a75c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc1312d0,0,c0fd05c7,cc3a75c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc1312e4,0,f4116ab4,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc1312e4,cc1312d0,c0fda6f8,6ae,f4116b3c,...) at _cv_wait_sig+0x260 seltdwait(cc08b72c,c0fda6f8,5ab,cc3a75c0,696,...) at seltdwait+0xa2 sys_poll(cc3a75c0,f4116cec,c1025a7c,c0fbc9b1,c1027030,...) at sys_poll+0x321 syscall(f4116d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x284723ab, esp = 0xbfbfde7c, ebp = 0xbfbfdeb8 --- Tracing command getty pid 1254 tid 100152 td 0xcc1168a0 sched_switch(cc1168a0,0,104,1a3,c82614d8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc1168a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc1168a0,cc1168a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33870,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33870,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33800,cae33870,f4131c3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33800,f4131c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27100,f4131c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08fab8,f4131c3c,c95d9b80,0,cc1168a0,...) at devfs_read_f+0x7e dofileread(f4131c3c,ffffffff,ffffffff,0,f4131c18,...) at dofileread+0x9e kern_readv(cc1168a0,0,f4131c3c,f4131c5c,1,...) at kern_readv+0x68 sys_read(cc1168a0,f4131cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4131d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1253 tid 100151 td 0xcc116b80 sched_switch(cc116b80,0,104,1a3,c75f9f14,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc116b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc116b80,cc116b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cae33a70,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cae33a70,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(cae33a00,cae33a70,f412ec3c,1,0,...) at tty_wait+0x71 ttydisc_read(cae33a00,f412ec3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cae27000,f412ec3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0ccc08,f412ec3c,c95d9b80,0,cc116b80,...) at devfs_read_f+0x7e dofileread(f412ec3c,ffffffff,ffffffff,0,f412ec18,...) at dofileread+0x9e kern_readv(cc116b80,0,f412ec3c,f412ec5c,1,...) at kern_readv+0x68 sys_read(cc116b80,f412ecec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f412ed28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1252 tid 100150 td 0xcc3a5000 sched_switch(cc3a5000,0,104,1a3,cac15768,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a5000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc3a5000,cc3a5000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d270,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d270,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d200,c970d270,f412bc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d200,f412bc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0ce00,f412bc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0ccce8,f412bc3c,c95d9b80,0,cc3a5000,...) at devfs_read_f+0x7e dofileread(f412bc3c,ffffffff,ffffffff,0,f412bc18,...) at dofileread+0x9e kern_readv(cc3a5000,0,f412bc3c,f412bc5c,1,...) at kern_readv+0x68 sys_read(cc3a5000,f412bcec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f412bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1251 tid 100149 td 0xcc3a52e0 sched_switch(cc3a52e0,0,104,1a3,cb9bef5d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a52e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc3a52e0,cc3a52e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d470,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d470,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d400,c970d470,f4128c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d400,f4128c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cd00,f4128c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0cc118,f4128c3c,c95d9b80,0,cc3a52e0,...) at devfs_read_f+0x7e dofileread(f4128c3c,ffffffff,ffffffff,0,f4128c18,...) at dofileread+0x9e kern_readv(cc3a52e0,0,f4128c3c,f4128c5c,1,...) at kern_readv+0x68 sys_read(cc3a52e0,f4128cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4128d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1250 tid 100148 td 0xcc3a55c0 sched_switch(cc3a55c0,0,104,1a3,ca663ad8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a55c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc3a55c0,cc3a55c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d670,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d670,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d600,c970d670,f4125c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d600,f4125c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0cc00,f4125c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08f1f8,f4125c3c,c95d9b80,0,cc3a55c0,...) at devfs_read_f+0x7e dofileread(f4125c3c,ffffffff,ffffffff,0,f4125c18,...) at dofileread+0x9e kern_readv(cc3a55c0,0,f4125c3c,f4125c5c,1,...) at kern_readv+0x68 sys_read(cc3a55c0,f4125cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4125d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1249 tid 100147 td 0xcc3a58a0 sched_switch(cc3a58a0,0,104,1a3,c70d053b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a58a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc3a58a0,cc3a58a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970d870,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970d870,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970d800,c970d870,f4122c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970d800,f4122c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c900,f4122c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08f1c0,f4122c3c,c95d9b80,0,cc3a58a0,...) at devfs_read_f+0x7e dofileread(f4122c3c,ffffffff,ffffffff,0,f4122c18,...) at dofileread+0x9e kern_readv(cc3a58a0,0,f4122c3c,f4122c5c,1,...) at kern_readv+0x68 sys_read(cc3a58a0,f4122cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4122d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1248 tid 100146 td 0xcc3a5b80 sched_switch(cc3a5b80,0,104,1a3,ceff4ec7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a5b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc3a5b80,cc3a5b80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970da70,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970da70,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970da00,c970da70,f411fc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970da00,f411fc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c800,f411fc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0cc930,f411fc3c,c95d9b80,0,cc3a5b80,...) at devfs_read_f+0x7e dofileread(f411fc3c,ffffffff,ffffffff,0,f411fc18,...) at dofileread+0x9e kern_readv(cc3a5b80,0,f411fc3c,f411fc5c,1,...) at kern_readv+0x68 sys_read(cc3a5b80,f411fcec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f411fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1247 tid 100145 td 0xcc3a7000 sched_switch(cc3a7000,0,104,1a3,cae63164,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a7000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,cc3a7000,cc3a7000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970dc70,0,c0fdcd60,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970dc70,c118c410,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970dc00,c970dc70,f411cc3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970dc00,f411cc3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(cac0c700,f411cc3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc08fe00,f411cc3c,c95d9b80,0,cc3a7000,...) at devfs_read_f+0x7e dofileread(f411cc3c,ffffffff,ffffffff,0,f411cc18,...) at dofileread+0x9e kern_readv(cc3a7000,0,f411cc3c,f411cc5c,1,...) at kern_readv+0x68 sys_read(cc3a7000,f411ccec,c1025a7c,c0e36ed2,cc3a55c0,...) at sys_read+0x4f syscall(f411cd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command getty pid 1246 tid 100144 td 0xcc3a72e0 sched_switch(cc3a72e0,0,104,1a3,8e059080,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc3a72e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,c970de04,0,c0fd05c7,cc3a72e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c970de70,0,f4119ae8,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c970de70,c970de04,c0fdc738,527,0,...) at _cv_wait_sig+0x260 tty_wait(c970de00,c970de70,f4119c3c,1,0,...) at tty_wait+0x71 ttydisc_read(c970de00,f4119c3c,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(c95c6100,f4119c3c,0,0,1,...) at ttydev_read+0xaa devfs_read_f(cc0cc4d0,f4119c3c,c95d9b80,0,cc3a72e0,...) at devfs_read_f+0x7e dofileread(f4119c3c,ffffffff,ffffffff,0,f4119c18,...) at dofileread+0x9e kern_readv(cc3a72e0,0,f4119c3c,f4119c5c,1,...) at kern_readv+0x68 sys_read(cc3a72e0,f4119cec,c1025a7c,c0e36ed2,c1027030,...) at sys_read+0x4f syscall(f4119d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28189ed3, esp = 0xbfbfed6c, ebp = 0xbfbfeda8 --- Tracing command inetd pid 1229 tid 100109 td 0xcc0b72e0 sched_switch(cc0b72e0,0,104,1a3,ac1aa40e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b72e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc793690,0,c0fd05c7,cc0b72e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc7936a4,0,f4047a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc7936a4,cc793690,c0fda6f8,6ae,cc0cce00,...) at _cv_wait_sig+0x260 seltdwait(cc0cce00,58,c95d9b80,cc0b72e0,202,...) at seltdwait+0xa2 kern_select(cc0b72e0,8,bfbfdf90,0,0,0,20,cc0b2000,4) at kern_select+0x655 sys_select(cc0b72e0,f4047cec,c1025a7c,c0fdaf24,c119b340,...) at sys_select+0x66 syscall(f4047d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2819be53, esp = 0xbfbfdf4c, ebp = 0xbfbfedd8 --- Tracing command watchdogd pid 1204 tid 100102 td 0xcc1155c0 sched_switch(cc1155c0,0,103,19e,d54c567,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,858,...) at mi_switch+0x1ff turnstile_wait(cc0b6280,ccb88000,0,1c5,101,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c130d980,cc1155c0,0,c100f607,4fe,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c130d980,0,c100f607,4fe,0,...) at _mtx_lock_flags+0x10f vm_page_alloc(cd2f9d4c,1e,0,0,f4023b64,...) at vm_page_alloc+0x565 vm_fault_hold(cc0783a0,bfbfe000,2,0,0,...) at vm_fault_hold+0x60a vm_fault(cc0783a0,bfbfe000,2,0,bfbfec3c,...) at vm_fault+0x8b trap_pfault(cc10e000,f4023c64,c0acc418,cc1155c0,cc10e000,...) at trap_pfault+0x1b9 trap(f4023d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x280bce00, esp = 0xbfbfec30, ebp = 0xbfbfecd8 --- Tracing command cron pid 1159 tid 100138 td 0xcc9852e0 sched_switch(cc9852e0,0,104,1a3,e5c56529,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cc9852e0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(cd8f0c24,1e,0,0,f40f7b64,...) at vm_waitpfault+0x75 vm_fault_hold(cc078658,bfbfe000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc078658,bfbfe000,2,0,bfbfec50,...) at vm_fault+0x8b trap_pfault(cc39b864,f40f7c64,c0acc418,cc9852e0,cc39b864,...) at trap_pfault+0x1b9 trap(f40f7d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x804c20c, esp = 0xbfbfec50, ebp = 0xbfbfec68 --- Tracing command sendmail pid 1154 tid 100113 td 0xcbee68a0 sched_switch(cbee68a0,0,104,1a3,d548ca8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,78,...) at mi_switch+0x1ff sleepq_switch(cbee68a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,78,c100f8cd,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,278,c100f8cd,0,...) at _sleep+0x39c vm_waitpfault(ccb89afc,1c,0,0,f4065b64,...) at vm_waitpfault+0x75 vm_fault_hold(cc078910,bfbfc000,2,0,0,...) at vm_fault_hold+0x61e vm_fault(cc078910,bfbfc000,2,0,bfbfcf10,...) at vm_fault+0x8b trap_pfault(f4065c58,c0a8e090,c119c070,cbee68a0,cc39b000,...) at trap_pfault+0x1b9 trap(f4065d28) at trap+0x3a3 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x80a23a1, esp = 0xbfbfcf10, ebp = 0xbfbfcf78 --- Tracing command sendmail pid 1151 tid 100088 td 0xcc0b7b80 cpustop_handler(2,f3fe9a4c,c0e00b03,c93dec80,f3fe999c,...) at cpustop_handler+0x26 ipi_nmi_handler(c93dec80,f3fe999c,c0a742f3,c93dec80,cc0b2864,...) at ipi_nmi_handler+0x35 trap(f3fe9a58) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0d15147, esp = 0xf3fe9a98, ebp = 0xf3fe9ab4 --- vm_page_rename(c246fac0,cc133940,b,0,0,...) at vm_page_rename+0xf7 vm_object_backing_scan(cc801c24,0,c100eff3,6c2,c100e17f,...) at vm_object_backing_scan+0x5ed vm_object_collapse(cc133940,0,c100e17f,b91,28800000,...) at vm_object_collapse+0x167 vmspace_fork(c95e5bc8,f3fe9c24,2,0,c0fd248f,...) at vmspace_fork+0x461 fork1(cc0b7b80,14,2,f3fe9c5c,0,...) at fork1+0x2cb sys_fork(cc0b7b80,f3fe9cec,c1025a7c,c0fdb6ee,e,...) at sys_fork+0x39 syscall(f3fe9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (2, FreeBSD ELF32, sys_fork), eip = 0x2833b7ab, esp = 0xbfbfc39c, ebp = 0xbfbfc3b8 --- Tracing command sshd pid 1148 tid 100099 td 0xcc07fb80 sched_switch(cc07fb80,0,104,1a3,c8d588da,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc07fb80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc793510,0,c0fd05c7,cc07fb80,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc793524,0,f4016a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc793524,cc793510,c0fda6f8,6ae,cc0d48c0,...) at _cv_wait_sig+0x260 seltdwait(cc0d48c0,58,cad1d380,cc07fb80,f4016b14,...) at seltdwait+0xa2 kern_select(cc07fb80,9,28c0c140,0,0,0,20,cbedcb30,a) at kern_select+0x655 sys_select(cc07fb80,f4016cec,c1025a7c,c0fbc9b1,c119c040,...) at sys_select+0x66 syscall(f4016d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284dae53, esp = 0xbfbfdedc, ebp = 0xbfbfede8 --- Tracing command moused pid 1116 tid 100126 td 0xcc1102e0 sched_switch(cc1102e0,0,104,1a3,218ad363,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc1102e0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cc793790,0,c0fd05c7,cc1102e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc7937a4,0,f40c9a50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cc7937a4,cc793790,c0fda6f8,6ae,cc0cc498,...) at _cv_wait_sig+0x260 seltdwait(cc0cc498,58,c95d9b80,cc1102e0,f6,...) at seltdwait+0xa2 kern_select(cc1102e0,400,bfbfe9f4,0,0,0,20,cc809000,1) at kern_select+0x655 sys_select(cc1102e0,f40c9cec,c1025a7c,c0e36ed2,0,...) at sys_select+0x66 syscall(f40c9d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281a8e53, esp = 0xbfbfe98c, ebp = 0xbfbfeb18 --- Tracing command ntpd pid 1091 tid 100095 td 0xcbee85c0 sched_switch(cbee85c0,0,103,19e,e4cfe982,...) at sched_switch+0x413 mi_switch(103,0,c0fd8990,2e0,780,...) at mi_switch+0x1ff turnstile_wait(cacc0480,ccb872e0,0,1c5,c12d6fc8,...) at turnstile_wait+0x4a4 _mtx_lock_sleep(c1bb50e8,cbee85c0,0,c100e17f,5ae,...) at _mtx_lock_sleep+0x2b7 _mtx_lock_flags(c1bb50e8,0,c100e17f,5ae,c596e000,...) at _mtx_lock_flags+0x10f _vm_map_lock(c1bb508c,c100e17f,5ae,400,cc0,...) at _vm_map_lock+0x31 vm_map_find(c1bb508c,0,0,0,f40055f0,...) at vm_map_find+0x31 kmem_alloc_nofault(c1bb508c,1000,c0fd05c7,eb,100,...) at kmem_alloc_nofault+0x67 vm_radix_node_zone_allocf(c1bb69a0,1000,f4005657,101,c1bb69a0,...) at vm_radix_node_zone_allocf+0x58 keg_alloc_slab(c1bb7608,4,c100d144,871,c0a74f36,...) at keg_alloc_slab+0xf9 keg_fetch_slab(101,2,c1bb6a38,f4005704,c0d0015f,...) at keg_fetch_slab+0x17f zone_fetch_slab(c1bb69a0,0,101,969,104,...) at zone_fetch_slab+0x4c uma_zalloc_arg(c1bb69a0,0,101,c9586bd0,0,...) at uma_zalloc_arg+0x59f vm_radix_insert(cc133ea4,0,0,c49ab440,c100f607,...) at vm_radix_insert+0x8f vm_page_insert(c49ab440,cc133e74,0,0,20,...) at vm_page_insert+0x1b5 vm_page_alloc(cc133e74,0,0,10222,10222,...) at vm_page_alloc+0x51c vm_page_grab(cc133e74,0,0,112a2,c12d6fb8,...) at vm_page_grab+0x152 allocbuf(e3325940,800,c0fdf52b,b00,fffffffc,...) at allocbuf+0x672 getblk(ccb84414,0,0,800,0,...) at getblk+0x5e3 ffs_balloc_ufs2(ccb84414,0,0,8,c95d9b80,...) at ffs_balloc_ufs2+0x1231 ffs_write(f4005ba4,c102d163,ccb84414,ccb84414,f4005b5c,...) at ffs_write+0x2bd VOP_WRITE_APV(c11375a0,f4005ba4,ccb84414,285,0,...) at VOP_WRITE_APV+0x174 vn_write(cd99f428,f4005c3c,c95d9b80,0,cbee85c0,...) at vn_write+0x2bc dofilewrite(f4005c3c,ffffffff,ffffffff,0,f4005c18,...) at dofilewrite+0x95 kern_writev(cbee85c0,4,f4005c3c,f4005c5c,1,...) at kern_writev+0x68 sys_write(cbee85c0,f4005cec,c1025a7c,c0fdae1a,e,...) at sys_write+0x4f syscall(f4005d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x28358eb3, esp = 0xbfbfea8c, ebp = 0xbfbfeaa8 --- Tracing command nfsd pid 1035 tid 100125 td 0xcc0b48a0 sched_switch(cc0b48a0,0,104,1a3,b8914696,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b48a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40c6bdc,c0a7478a,cbc36e80,0,cc0b48a0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca9dd0,0,f40c6c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca9dd0,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40c6d14,c0a56e68,cbc36e80,f40c6d28,c0fcc5b0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40c6d28,c0fcc5b0,3d8,cc39a2cc,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40c6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1035 tid 100124 td 0xcc0b4b80 sched_switch(cc0b4b80,0,104,1a3,b930a036,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b4b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40c3bdc,c0a7478a,cbc36e80,0,cc0b4b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca9e10,0,f40c3c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca9e10,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40c3d14,c0a56e68,cbc36e80,f40c3d28,c0fcc5b0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40c3d28,c0fcc5b0,3d8,cc39a2cc,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40c3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1035 tid 100123 td 0xcc0b7000 sched_switch(cc0b7000,0,104,1a3,b7d10049,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b7000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f40c0bdc,c0a7478a,cbc36e80,0,cc0b7000,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(caca9e50,0,f40c0c18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(caca9e50,cbc36e80,1388,3b1,0,...) at _cv_timedwait_sig+0x270 svc_run_internal(f40c0d14,c0a56e68,cbc36e80,f40c0d28,c0fcc5b0,...) at svc_run_internal+0x356 svc_thread_start(cbc36e80,f40c0d28,c0fcc5b0,3d8,cc39a2cc,...) at svc_thread_start+0x10 fork_exit(c0c84dc0,cbc36e80,f40c0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0x1 --- Tracing command nfsd pid 1035 tid 100117 td 0xcae26b80 sched_switch(cae26b80,0,104,1a3,b879c448,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae26b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f407b48c,c0a7478a,cbc36e80,0,cae26b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cc079c50,0,f407b4c8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cc079c50,cbc36e80,1388,3b1,f407b510,...) at _cv_timedwait_sig+0x270 svc_run_internal(cae26d30,14,c10027da,c0fbdebb,f407b808,...) at svc_run_internal+0x356 svc_run(cbc36e80,0,c0fbde96,19f,c,...) at svc_run+0x7b nfsrvd_nfsd(cae26b80,f407b808,c,4,80000000,...) at nfsrvd_nfsd+0x1c3 nfssvc_nfsd(cae26b80,f407bcec,0,cae26b80,f407bd1c,...) at nfssvc_nfsd+0x177 sys_nfssvc(cae26b80,f407bcec,c1025a7c,c0fdb2b4,c1027030,...) at sys_nfssvc+0xeb syscall(f407bd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (155, FreeBSD ELF32, sys_nfssvc), eip = 0x280bf9bb, esp = 0xbfbfe81c, ebp = 0xbfbfea88 --- Tracing command nfsd pid 1031 tid 100101 td 0xcc1158a0 sched_switch(cc1158a0,0,104,1a3,b53f78b3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc1158a0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cac85950,0,c0fd05c7,cc1158a0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cac85964,0,f401fa50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cac85964,cac85950,c0fda6f8,6ae,cc0ccd58,...) at _cv_wait_sig+0x260 seltdwait(cc0ccd58,58,c95d9b80,cc1158a0,0,...) at seltdwait+0xa2 kern_select(cc1158a0,7,bfbfeca4,0,0,0,20,cc10e2cc,1) at kern_select+0x655 sys_select(cc1158a0,f401fcec,c1025a7c,c0fdb2b4,c1027030,...) at sys_select+0x66 syscall(f401fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28176e53, esp = 0xbfbfea8c, ebp = 0xbfbfedd8 --- Tracing command nfsuserd pid 1029 tid 100094 td 0xcc0b75c0 sched_switch(cc0b75c0,0,104,1a3,7ba4ac00,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b75c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f4001a14,c0a7478a,cbff7c50,0,cc0b75c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbff7c64,0,f4001a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbff7c64,cbff7c50,7531,6ae,cc0d4428,...) at _cv_timedwait_sig+0x270 seltdwait(f4001c08,f4001c10,c95d9b80,cc0b75c0,f4001ab4,...) at seltdwait+0x8a kern_select(cc0b75c0,4,bfbfebb4,0,0,f4001c54,20,1e,0) at kern_select+0x655 sys_select(cc0b75c0,f4001cec,c1025a7c,c0fdb00e,f4001cb4,...) at sys_select+0x66 syscall(f4001d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1028 tid 100087 td 0xcc0805c0 sched_switch(cc0805c0,0,104,1a3,7ba73a22,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0805c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fe5a14,c0a7478a,cac79190,0,cc0805c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac791a4,0,f3fe5a50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac791a4,cac79190,7531,6ae,cc0d4428,...) at _cv_timedwait_sig+0x270 seltdwait(f3fe5c08,f3fe5c10,c95d9b80,cc0805c0,f3fe5ab4,...) at seltdwait+0x8a kern_select(cc0805c0,4,bfbfebb4,0,0,f3fe5c54,20,1e,0) at kern_select+0x655 sys_select(cc0805c0,f3fe5cec,c1025a7c,c0fdb00e,f3fe5cb4,...) at sys_select+0x66 syscall(f3fe5d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1027 tid 100081 td 0xcc0b8b80 sched_switch(cc0b8b80,0,104,1a3,7ba44764,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b8b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f3fcda14,c0a7478a,cbff7c10,0,cc0b8b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cbff7c24,0,f3fcda50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cbff7c24,cbff7c10,7531,6ae,cc0d4428,...) at _cv_timedwait_sig+0x270 seltdwait(f3fcdc08,f3fcdc10,c95d9b80,cc0b8b80,f3fcdab4,...) at seltdwait+0x8a kern_select(cc0b8b80,4,bfbfebb4,0,0,f3fcdc54,20,1e,0) at kern_select+0x655 sys_select(cc0b8b80,f3fcdcec,c1025a7c,c0fdb00e,f3fcdcb4,...) at sys_select+0x66 syscall(f3fcdd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1026 tid 100105 td 0xcc110b80 sched_switch(cc110b80,0,104,1a3,7b888cc7,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc110b80,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f402fa14,c0a7478a,cac60850,0,cc110b80,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac60864,0,f402fa50,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac60864,cac60850,7531,6ae,cc0d4428,...) at _cv_timedwait_sig+0x270 seltdwait(f402fc08,f402fc10,c95d9b80,cc110b80,f402fab4,...) at seltdwait+0x8a kern_select(cc110b80,4,bfbfebb4,0,0,f402fc54,20,1e,0) at kern_select+0x655 sys_select(cc110b80,f402fcec,cc110b80,c119c040,f402fcb4,...) at sys_select+0x66 syscall(f402fd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x28175e53, esp = 0xbfbfeb0c, ebp = 0xbfbfec48 --- Tracing command nfsuserd pid 1025 tid 100120 td 0xcae262e0 sched_switch(cae262e0,0,104,1a3,aa734355,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cae262e0,0,c0fd7fa0,1a7,74,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cc10f8bc,74,c0e58cee,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc10f8bc,cc10f8ec,174,c0e58cee,0,...) at _sleep+0x385 kern_sigsuspend(cae262e0,0,0,0,0,...) at kern_sigsuspend+0xba sys_sigsuspend(cae262e0,f4086cec,c1025a7c,c0fdb2b4,c119c040,...) at sys_sigsuspend+0x4d syscall(f4086d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (4, FreeBSD ELF32, sys_write), eip = 0x280bdd7b, esp = 0xbfbfec4c, ebp = 0xbfbfede8 --- Tracing command mountd pid 1009 tid 100107 td 0xcc07f5c0 sched_switch(cc07f5c0,0,104,1a3,64079264,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc07f5c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbff7050,0,c0fd05c7,cc07f5c0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbff7064,0,f403aa50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbff7064,cbff7050,c0fda6f8,6ae,cc0d4850,...) at _cv_wait_sig+0x260 seltdwait(cc0d4850,58,c95d9b80,cc07f5c0,f403aac0,...) at seltdwait+0xa2 kern_select(cc07f5c0,9,bfbfecf4,0,0,0,20,cbedc598,bfbfecf4) at kern_select+0x655 sys_select(cc07f5c0,f403acec,c1025a7c,c0fbbc17,1,...) at sys_select+0x66 syscall(f403ad28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2818de53, esp = 0xbfbfec8c, ebp = 0xbfbfedd8 --- Tracing command rpcbind pid 976 tid 100114 td 0xcbee65c0 sched_switch(cbee65c0,0,104,1a3,8662c2bf,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cbee65c0,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(f406da78,c0a7478a,cac7ae90,0,cbee65c0,...) at sleepq_catch_signals+0xf9 sleepq_timedwait_sig(cac7aea4,0,f406dab4,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(cac7aea4,cac7ae90,7531,6ae,f406db6c,...) at _cv_timedwait_sig+0x270 seltdwait(f406dc3c,f406dc44,5ab,cbee65c0,c0a7587b,...) at seltdwait+0x8a sys_poll(cbee65c0,f406dcec,c1025a7c,c0fdb00e,f406dcb4,...) at sys_poll+0x321 syscall(f406dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2812c3ab, esp = 0xbfbfcb9c, ebp = 0xbfbfed68 --- Tracing command syslogd pid 961 tid 100110 td 0xcc1105c0 cpustop_handler(1,f404d64c,c0e00b03,c0fd05c7,119,...) at cpustop_handler+0x26 ipi_nmi_handler(c0fd05c7,119,c12d6498,0,cc10b000,...) at ipi_nmi_handler+0x35 trap(f404d658) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0df4fb8, esp = 0xf404d698, ebp = 0xf404d6b8 --- smp_tlb_shootdown(f404d6e4,c0df7aac,e95ac000,e95b0000,c9588e58,...) at smp_tlb_shootdown+0x98 smp_invlpg_range(e95ac000,e95b0000,c9588e58,cc110670,e95b0000,...) at smp_invlpg_range+0x1c pmap_invalidate_range(c132cdc0,e95ac000,e95b0000) at pmap_invalidate_range+0x6c pmap_qremove(e95ac000,4,ccb84668,4,e3365760,...) at pmap_qremove+0x54 vfs_vmio_release(c1304594,0,c0fdf52b,792,0,...) at vfs_vmio_release+0x25 getnewbuf(4000,4000,0,ad1,c12d7468,...) at getnewbuf+0x334 getblk(ccae0c3c,ffd1dff4,ffffffff,4000,0,...) at getblk+0x485 breadn_flags(ccae0c3c,ffd1dff4,ffffffff,4000,0,...) at breadn_flags+0x43 ffs_balloc_ufs2(ccae0c3c,88224000,b,4000,c9561b80,...) at ffs_balloc_ufs2+0x1775 ffs_copyonwrite(cbc392b8,e3375b60,0,e3375b60,cc495ae0,...) at ffs_copyonwrite+0x3f8 ffs_geom_strategy(cbc393b0,e3375b60,c11088e8,c12d7468,cc495ae0,...) at ffs_geom_strategy+0xeb ufs_strategy(f404dad0,c102cf2b,c0dea8d1,e3375bc0,cc1105c0,...) at ufs_strategy+0xd3 VOP_STRATEGY_APV(c11375a0,f404dad0,c0fdf52b,c11498e0,cc495ae0,...) at VOP_STRATEGY_APV+0x119 bufstrategy(cc495bd8,e3375b60,3a0,2000,0,...) at bufstrategy+0x61 bufwrite(e3375b60,0,c0fdf52b,6d6,0,...) at bufwrite+0x159 vfs_bio_awrite(e3375b60,0,c100b1f1,106,0,...) at vfs_bio_awrite+0x328 ffs_syncvnode(cc495ae0,1,0,c11375a0,cc495bd8,...) at ffs_syncvnode+0x3a5 ffs_fsync(f404dc40,c102c6e4,eb,0,e03,...) at ffs_fsync+0x2f VOP_FSYNC_APV(c11375a0,f404dc40,c0fe37a7,e08,0,...) at VOP_FSYNC_APV+0x100 sys_fsync(cc1105c0,f404dcec,c1025a7c,c0fdbd8f,f404dcb4,...) at sys_fsync+0x1df syscall(f404dd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (95, FreeBSD ELF32, sys_fsync), eip = 0x2818ad47, esp = 0xbfbfe28c, ebp = 0xbfbfedd8 --- Tracing command devd pid 838 tid 100085 td 0xcc0b8000 sched_switch(cc0b8000,0,104,1a3,c78cb2b3,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cc0b8000,0,c0fd7fa0,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a7478a,cbff7650,0,c0fd05c7,cc0b8000,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cbff7664,0,f3fdda50,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(cbff7664,cbff7650,c0fda6f8,6ae,cc0cc7a8,...) at _cv_wait_sig+0x260 seltdwait(cc0cc7a8,58,c95d9b80,cc0b8000,c11088e8,...) at seltdwait+0xa2 kern_select(cc0b8000,5,bfbfe950,0,0,0,20,cc0b2b30,34) at kern_select+0x655 sys_select(cc0b8000,f3fddcec,c1025a7c,c0e3210f,c119b340,...) at sys_select+0x66 syscall(f3fddd28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x80893bb, esp = 0xbfbfe91c, ebp = 0xbfbfedf8 --- Tracing command softdepflush pid 19 tid 100078 td 0xcbee88a0 sched_switch(cbee88a0,0,104,1a3,e5a791c2,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cbee88a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130cc7c,54,c100f8db,0,0,...) at sleepq_wait+0x63 _sleep(c130cc7c,c130cf00,254,c100f8db,0,...) at _sleep+0x39c vm_wait(cc134c24,0,c100f607,8f5,40222,...) at vm_wait+0xbe vm_page_grab(cc134c24,623bd8,0,412a2,c12d7900,...) at vm_page_grab+0x180 allocbuf(e32449e0,4000,c0fdf52b,b00,efbe99fc,...) at allocbuf+0x672 getblk(cbc392b8,311dec0,0,4000,0,...) at getblk+0x5e3 breadn_flags(cbc392b8,311dec0,0,4000,0,...) at breadn_flags+0x43 indir_trunc(fffd87f4,ffffffff,c1007b8c,1e0e,0,...) at indir_trunc+0x114 indir_trunc(fffff7f3,ffffffff,c1007b8c,1cd0,c12d78e0,...) at indir_trunc+0x4ba handle_workitem_freeblocks(0,efbe9c70,2,6ba,efbe9c74,...) at handle_workitem_freeblocks+0x24f process_worklist_item(c130c580,0,c1007b8c,5e9,cc386600,...) at process_worklist_item+0x2c2 softdep_process_worklist(cbff9b70,0,c1007b8c,562,3e8,...) at softdep_process_worklist+0x84 softdep_flush(0,efbe9d28,c0fcc5b0,3d8,cbedb000,...) at softdep_flush+0x2a0 fork_exit(c0cd7e10,0,efbe9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe9d60, ebp = 0 --- Tracing command vnlru pid 18 tid 100077 td 0xcbee8b80 sched_switch(cbee8b80,0,104,1a3,ebefe58f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cbee8b80,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cbedb2cc,60,c0fe30f9,0,0,...) at sleepq_timedwait+0x6b _sleep(cbedb2cc,c1304874,260,c0fe30f9,3e8,...) at _sleep+0x36a vnlru_proc(0,efbe6d28,c0fcc5b0,3d8,cbedb2cc,...) at vnlru_proc+0xe7 fork_exit(c0b2b970,0,efbe6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe6d60, ebp = 0 --- Tracing command syncer pid 17 tid 100076 td 0xcacf98a0 sched_switch(cacf98a0,0,104,1a3,48534a9,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf98a0,0,c0fd7fa0,28b,cacf98a0,...) at sleepq_switch+0x162 sleepq_timedwait(c13048b4,0,efbe3c78,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c13048b4,c13048a0,3e8,783,4e20,...) at _cv_timedwait+0x270 sched_sync(0,efbe3d28,c0fcc5b0,3d8,cbedb598,...) at sched_sync+0x595 fork_exit(c0b2aca0,0,efbe3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe3d60, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xcacf9b80 sched_switch(cacf9b80,0,104,1a3,7f42404b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(cacf9b80,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1304548,54,c0fdfe72,0,0,...) at sleepq_timedwait+0x6b _sleep(c1304548,c1304550,54,c0fdfe72,3e8,...) at _sleep+0x36a buf_daemon(0,efbe0d28,c0fcc5b0,3d8,cbedb864,...) at buf_daemon+0x175 fork_exit(c0b0d9c0,0,efbe0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbe0d60, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xcae24000 sched_switch(cae24000,0,104,1a3,c1f7ac1a,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24000,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c130e468,0,c101129d,0,0,...) at sleepq_timedwait+0x6b _sleep(c130e468,c130cf00,0,c101129d,493e0,...) at _sleep+0x36a vm_pagezero(0,efbddd28,c0fcc5b0,3d8,cbedbb30,...) at vm_pagezero+0xdc fork_exit(c0d1fa40,0,efbddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbddd60, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xcae242e0 sched_switch(cae242e0,0,104,1a3,730dc008,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,74,...) at mi_switch+0x1ff sleepq_switch(cae242e0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c130e048,74,c0fdfe72,0,0,...) at sleepq_wait+0x63 _sleep(c130e048,c130e04c,74,c0fdfe72,0,...) at _sleep+0x39c vm_daemon(0,efbdad28,c0fcc5b0,3d8,cbedc000,...) at vm_daemon+0x59 fork_exit(c0d18320,0,efbdad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbdad60, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xcae245c0 cpustop_handler(0,efbd7b08,c0e00b03,c131e828,c1108904,...) at cpustop_handler+0x26 ipi_nmi_handler(c131e828,c1108904,cae245c0,efbd7a48,c95de2cc,...) at ipi_nmi_handler+0x35 trap(efbd7b14) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0a73fe1, esp = 0xefbd7b54, ebp = 0xefbd7b64 --- _mtx_assert(c11a95d4,4,c0fd05c7,119,c0ac8998,...) at _mtx_assert+0x141 _mtx_unlock_spin_flags(c11a95d4,0,c0fd7fa0,10d,efbd7bc0,...) at _mtx_unlock_spin_flags+0xe0 sleepq_release(c1baae00,0,0,0,c1baae00,...) at sleepq_release+0x39 wakeup(c1baae00,0,c100d144,311,0,...) at wakeup+0x41 zone_drain_wait(efbd7c10,c0cfb695,c1baae00,0,c100d144,...) at zone_drain_wait+0x12f zone_drain(c1baae00,0,c100d144,63c,0,...) at zone_drain+0x10 zone_foreach(c95b46c0,0,efbd7ce8,c0d19bac,c95b46cc,...) at zone_foreach+0x45 uma_reclaim(c95b46cc,0,c1010087,2ee,1388,...) at uma_reclaim+0x16 vm_pageout(0,efbd7d28,c0fcc5b0,3d8,c95de2cc,...) at vm_pageout+0x80c fork_exit(c0d193a0,0,efbd7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefbd7d60, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xcae248a0 sched_switch(cae248a0,0,104,1a3,ea3b72cd,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae248a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1151054,5c,c0e2c8fe,0,0,...) at sleepq_wait+0x63 _sleep(c1151054,c1151098,5c,c0e2c8fe,0,...) at _sleep+0x39c xpt_scanner_thread(0,efa54d28,c0fcc5b0,3d8,c95de598,...) at xpt_scanner_thread+0x47 fork_exit(c04a72f0,0,efa54d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa54d60, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xcae24b80 sched_switch(cae24b80,0,104,1a3,4e2d66a5,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae24b80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1307dd8,0,c0ff75bf,0,0,...) at sleepq_wait+0x63 _sleep(c1307dd8,c1307da0,0,c0ff75bf,0,...) at _sleep+0x39c sctp_iterator_thread(0,efa51d28,c0fcc5b0,3d8,c95de864,...) at sctp_iterator_thread+0x5c fork_exit(c0bc5a50,0,efa51d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa51d60, ebp = 0 --- Tracing command fdc0 pid 4 tid 100065 td 0xcae25b80 sched_switch(cae25b80,0,104,1a3,88cdbf0e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(cae25b80,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(cac7223c,5c,c0fbe932,0,0,...) at sleepq_timedwait+0x6b _sleep(cac7223c,cac722f0,5c,c0fbe932,3e8,...) at _sleep+0x36a fdc_thread(cac72200,efa42d28,c0fcc5b0,3d8,c95deb30,...) at fdc_thread+0x27d fork_exit(c0da1d70,cac72200,efa42d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa42d60, ebp = 0 --- Tracing command fw0_probe pid 3 tid 100063 td 0xcace42e0 sched_switch(cace42e0,0,104,1a3,ea3b9366,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(cace42e0,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(cad2c000,6c,c0fbe932,100,0,...) at sleepq_wait_sig+0x17 _sleep(cad2c000,cad30488,16c,c0fbe932,0,...) at _sleep+0x385 fw_bus_probe_thread(cad2c000,efa32d28,c0fcc5b0,3d8,cac0b000,...) at fw_bus_probe_thread+0xa08 fork_exit(c07017d0,cad2c000,efa32d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa32d60, ebp = 0 --- Tracing command usb pid 15 tid 100059 td 0xcacf7000 sched_switch(cacf7000,0,104,1a3,d7d26f06,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf7000,0,c0fd7fa0,268,cacf7000,...) at sleepq_switch+0x162 sleepq_wait(cacfed34,0,efa1bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed34,cacfee4c,c0faffb1,74,cacfed3c,...) at _cv_wait+0x260 usb_process(cacfed2c,efa1bd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfed2c,efa1bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa1bd60, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xcacf72e0 sched_switch(cacf72e0,0,104,1a3,4d949185,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf72e0,0,c0fd7fa0,268,cacf72e0,...) at sleepq_switch+0x162 sleepq_wait(cacfed04,0,efa18cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfed04,cacfee4c,c0faffb1,74,cacfed0c,...) at _cv_wait+0x260 usb_process(cacfecfc,efa18d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfecfc,efa18d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa18d60, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xcacf75c0 sched_switch(cacf75c0,0,104,1a3,d76b5c53,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf75c0,0,c0fd7fa0,268,cacf75c0,...) at sleepq_switch+0x162 sleepq_wait(cacfecd4,0,efa15cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfecd4,cacfee4c,c0faffb1,74,cacfecdc,...) at _cv_wait+0x260 usb_process(cacfeccc,efa15d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfeccc,efa15d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa15d60, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xcacf78a0 sched_switch(cacf78a0,0,104,1a3,d76b48ac,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf78a0,0,c0fd7fa0,268,cacf78a0,...) at sleepq_switch+0x162 sleepq_wait(cacfeca4,0,efa12cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacfeca4,cacfee4c,c0faffb1,74,cacfecac,...) at _cv_wait+0x260 usb_process(cacfec9c,efa12d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacfec9c,efa12d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa12d60, ebp = 0 --- Tracing command usb pid 15 tid 100054 td 0xcacf9000 sched_switch(cacf9000,0,104,1a3,d76b308e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf9000,0,c0fd7fa0,268,cacf9000,...) at sleepq_switch+0x162 sleepq_wait(cacefb5c,0,efa09cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb5c,cacefc74,c0faffb1,74,cacefb64,...) at _cv_wait+0x260 usb_process(cacefb54,efa09d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb54,efa09d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa09d60, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xcacf92e0 sched_switch(cacf92e0,0,104,1a3,1be78988,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf92e0,0,c0fd7fa0,268,cacf92e0,...) at sleepq_switch+0x162 sleepq_wait(cacefb2c,0,efa06cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefb2c,cacefc74,c0faffb1,74,cacefb34,...) at _cv_wait+0x260 usb_process(cacefb24,efa06d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefb24,efa06d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa06d60, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xcacf95c0 sched_switch(cacf95c0,0,104,1a3,d7069fd4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacf95c0,0,c0fd7fa0,268,cacf95c0,...) at sleepq_switch+0x162 sleepq_wait(cacefafc,0,efa03cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefafc,cacefc74,c0faffb1,74,cacefb04,...) at _cv_wait+0x260 usb_process(cacefaf4,efa03d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefaf4,efa03d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa03d60, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xcace2000 sched_switch(cace2000,0,104,1a3,d7068910,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2000,0,c0fd7fa0,268,cace2000,...) at sleepq_switch+0x162 sleepq_wait(cacefacc,0,efa00cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacefacc,cacefc74,c0faffb1,74,cacefad4,...) at _cv_wait+0x260 usb_process(cacefac4,efa00d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacefac4,efa00d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa00d60, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xcace22e0 sched_switch(cace22e0,0,104,1a3,d7066f78,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace22e0,0,c0fd7fa0,268,cace22e0,...) at sleepq_switch+0x162 sleepq_wait(caceab5c,0,ef9fccac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab5c,caceac74,c0faffb1,74,caceab64,...) at _cv_wait+0x260 usb_process(caceab54,ef9fcd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab54,ef9fcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9fcd60, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xcace25c0 sched_switch(cace25c0,0,104,1a3,1825bc5e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace25c0,0,c0fd7fa0,268,cace25c0,...) at sleepq_switch+0x162 sleepq_wait(caceab2c,0,ef9f9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceab2c,caceac74,c0faffb1,74,caceab34,...) at _cv_wait+0x260 usb_process(caceab24,ef9f9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceab24,ef9f9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f9d60, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xcace28a0 sched_switch(cace28a0,0,104,1a3,d1319e18,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace28a0,0,c0fd7fa0,268,cace28a0,...) at sleepq_switch+0x162 sleepq_wait(caceaafc,0,ef9f6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaafc,caceac74,c0faffb1,74,caceab04,...) at _cv_wait+0x260 usb_process(caceaaf4,ef9f6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaaf4,ef9f6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f6d60, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xcace2b80 sched_switch(cace2b80,0,104,1a3,d1318894,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace2b80,0,c0fd7fa0,268,cace2b80,...) at sleepq_switch+0x162 sleepq_wait(caceaacc,0,ef9f3cac,1,0,...) at sleepq_wait+0x63 _cv_wait(caceaacc,caceac74,c0faffb1,74,caceaad4,...) at _cv_wait+0x260 usb_process(caceaac4,ef9f3d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,caceaac4,ef9f3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9f3d60, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xcace3000 sched_switch(cace3000,0,104,1a3,d1317008,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace3000,0,c0fd7fa0,268,cace3000,...) at sleepq_switch+0x162 sleepq_wait(cace5b5c,0,ef9efcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b5c,cace5c74,c0faffb1,74,cace5b64,...) at _cv_wait+0x260 usb_process(cace5b54,ef9efd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b54,ef9efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9efd60, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xcace32e0 sched_switch(cace32e0,0,104,1a3,16f7ab3c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace32e0,0,c0fd7fa0,268,cace32e0,...) at sleepq_switch+0x162 sleepq_wait(cace5b2c,0,ef9eccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5b2c,cace5c74,c0faffb1,74,cace5b34,...) at _cv_wait+0x260 usb_process(cace5b24,ef9ecd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5b24,ef9ecd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ecd60, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xcace35c0 sched_switch(cace35c0,0,104,1a3,d0ccd92c,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace35c0,0,c0fd7fa0,268,cace35c0,...) at sleepq_switch+0x162 sleepq_wait(cace5afc,0,ef9e9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5afc,cace5c74,c0faffb1,74,cace5b04,...) at _cv_wait+0x260 usb_process(cace5af4,ef9e9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5af4,ef9e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e9d60, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xcace38a0 sched_switch(cace38a0,0,104,1a3,d0ccc53e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace38a0,0,c0fd7fa0,268,cace38a0,...) at sleepq_switch+0x162 sleepq_wait(cace5acc,0,ef9e6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cace5acc,cace5c74,c0faffb1,74,cace5ad4,...) at _cv_wait+0x260 usb_process(cace5ac4,ef9e6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cace5ac4,ef9e6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e6d60, ebp = 0 --- Tracing command usb pid 15 tid 100041 td 0xcace4000 sched_switch(cace4000,0,104,1a3,d0ccaa0e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace4000,0,c0fd7fa0,268,cace4000,...) at sleepq_switch+0x162 sleepq_wait(cacddb5c,0,ef9dfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb5c,cacddc74,c0faffb1,74,cacddb64,...) at _cv_wait+0x260 usb_process(cacddb54,ef9dfd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb54,ef9dfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dfd60, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xcacbf000 sched_switch(cacbf000,0,104,1a3,16014d52,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf000,0,c0fd7fa0,268,cacbf000,...) at sleepq_switch+0x162 sleepq_wait(cacddb2c,0,ef9dccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddb2c,cacddc74,c0faffb1,74,cacddb34,...) at _cv_wait+0x260 usb_process(cacddb24,ef9dcd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddb24,ef9dcd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9dcd60, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xcacbf2e0 sched_switch(cacbf2e0,0,104,1a3,d068980d,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf2e0,0,c0fd7fa0,268,cacbf2e0,...) at sleepq_switch+0x162 sleepq_wait(cacddafc,0,ef9d9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddafc,cacddc74,c0faffb1,74,cacddb04,...) at _cv_wait+0x260 usb_process(cacddaf4,ef9d9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddaf4,ef9d9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d9d60, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xcacbf5c0 sched_switch(cacbf5c0,0,104,1a3,d0688141,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbf5c0,0,c0fd7fa0,268,cacbf5c0,...) at sleepq_switch+0x162 sleepq_wait(cacddacc,0,ef9d6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacddacc,cacddc74,c0faffb1,74,cacddad4,...) at _cv_wait+0x260 usb_process(cacddac4,ef9d6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacddac4,ef9d6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d6d60, ebp = 0 --- Tracing command usb pid 15 tid 100036 td 0xcacbfb80 sched_switch(cacbfb80,0,104,1a3,d0686abe,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacbfb80,0,c0fd7fa0,268,cacbfb80,...) at sleepq_switch+0x162 sleepq_wait(cacd5b5c,0,ef9cfcac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b5c,cacd5c74,c0faffb1,74,cacd5b64,...) at _cv_wait+0x260 usb_process(cacd5b54,ef9cfd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b54,ef9cfd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9cfd60, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xcacc1000 sched_switch(cacc1000,0,104,1a3,1600a613,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc1000,0,c0fd7fa0,268,cacc1000,...) at sleepq_switch+0x162 sleepq_wait(cacd5b2c,0,ef9cccac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5b2c,cacd5c74,c0faffb1,74,cacd5b34,...) at _cv_wait+0x260 usb_process(cacd5b24,ef9ccd28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5b24,ef9ccd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9ccd60, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xcacc12e0 sched_switch(cacc12e0,0,104,1a3,d0042760,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc12e0,0,c0fd7fa0,268,cacc12e0,...) at sleepq_switch+0x162 sleepq_wait(cacd5afc,0,ef9c9cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5afc,cacd5c74,c0faffb1,74,cacd5b04,...) at _cv_wait+0x260 usb_process(cacd5af4,ef9c9d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5af4,ef9c9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c9d60, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xcacc15c0 sched_switch(cacc15c0,0,104,1a3,d0041056,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cacc15c0,0,c0fd7fa0,268,cacc15c0,...) at sleepq_switch+0x162 sleepq_wait(cacd5acc,0,ef9c6cac,1,0,...) at sleepq_wait+0x63 _cv_wait(cacd5acc,cacd5c74,c0faffb1,74,cacd5ad4,...) at _cv_wait+0x260 usb_process(cacd5ac4,ef9c6d28,c0fcc5b0,3d8,cac0b2cc,...) at usb_process+0x1a3 fork_exit(c092f7e0,cacd5ac4,ef9c6d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c6d60, ebp = 0 --- Tracing command ctl_thrd pid 2 tid 100028 td 0xc96eb8a0 sched_switch(c96eb8a0,0,104,1a3,d003f3a5,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb8a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c97c4000,5c,c0e37370,0,0,...) at sleepq_wait+0x63 _sleep(c97c4000,c97c4000,5c,c0e37370,0,...) at _sleep+0x39c ctl_work_thread(c97c4000,ef733d28,c0fcc5b0,3d8,cac0b598,...) at ctl_work_thread+0x22f6 fork_exit(c04d08d0,c97c4000,ef733d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef733d60, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xc96ea5c0 sched_switch(c96ea5c0,0,104,1a3,af1a18e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ea5c0,0,c0fd7fa0,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe932,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe932,64) at _sleep+0x36a pause(c0fbe932,64,c0f8c010,111,0,...) at pause+0xa0 random_kthread(0,e33e0d28,c0fcc5b0,3d8,c95dd000,...) at random_kthread+0x1ef fork_exit(c082d020,0,e33e0d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e0d60, ebp = 0 --- Tracing command geom pid 13 tid 100016 td 0xc96ea8a0 sched_switch(c96ea8a0,0,104,1a3,e4c30c52,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96ea8a0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a5a8,5c,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c118a5a8,c118a508,25c,c0fbe932,0,...) at _sleep+0x39c g_io_schedule_down(c96ea8a0,0,c0fc0481,6c,e33ddd14,...) at g_io_schedule_down+0x5b g_down_procbody(0,e33ddd28,c0fcc5b0,3d8,c95dd2cc,...) at g_down_procbody+0x9d fork_exit(c09e6af0,0,e33ddd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ddd60, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xc96eab80 sched_switch(c96eab80,0,104,1a3,e5c4d5c6,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eab80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a5a4,5c,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c118a5a4,c118a528,25c,c0fbe932,0,...) at _sleep+0x39c g_io_schedule_up(c96eab80,0,c0fc0481,5f,e33dad14,...) at g_io_schedule_up+0x134 g_up_procbody(0,e33dad28,c0fcc5b0,3d8,c95dd2cc,...) at g_up_procbody+0x9d fork_exit(c09e6b90,0,e33dad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33dad60, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xc96eb000 sched_switch(c96eb000,0,104,1a3,b501da0b,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c96eb000,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c118a59c,5c,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c118a59c,c118a4cc,25c,c0fbe932,0,...) at _sleep+0x39c g_run_events(c119b340,0,c0fc0481,79,e33d7d14,...) at g_run_events+0x54b g_event_procbody(0,e33d7d28,c0fcc5b0,3d8,c95dd2cc,...) at g_event_procbody+0x90 fork_exit(c09e6c30,0,e33d7d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d7d60, ebp = 0 --- Tracing command intr pid 12 tid 100067 td 0xcae255c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100066 td 0xcae258a0 sched_switch(cae258a0,0,109,1a3,d00017e6,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cae258a0,...) at mi_switch+0x1ff ithread_loop(cae1e990,efa45d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cae1e990,efa45d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa45d60, ebp = 0 --- Tracing command intr pid 12 tid 100064 td 0xcae26000 sched_switch(cae26000,0,109,1a3,8e055f9b,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cae26000,...) at mi_switch+0x1ff ithread_loop(cae18a40,efa35d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cae18a40,efa35d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa35d60, ebp = 0 --- Tracing command intr pid 12 tid 100061 td 0xcace48a0 sched_switch(cace48a0,0,109,1a3,bfda9793,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cace48a0,...) at mi_switch+0x1ff ithread_loop(cad15880,efa29d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15880,efa29d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa29d60, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xcace4b80 sched_switch(cace4b80,0,109,1a3,ab57d726,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cace4b80,...) at mi_switch+0x1ff ithread_loop(cad15010,efa21d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cad15010,efa21d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa21d60, ebp = 0 --- Tracing command intr pid 12 tid 100055 td 0xcacf7b80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100042 td 0xcace3b80 sched_switch(cace3b80,0,109,1a3,f514ec5f,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cace3b80,...) at mi_switch+0x1ff ithread_loop(cacc5720,ef9e3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacc5720,ef9e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xcacbf8a0 sched_switch(cacbf8a0,0,109,1a3,ef9c75ac,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cacbf8a0,...) at mi_switch+0x1ff ithread_loop(cacad590,ef9d3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad590,ef9d3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9d3d60, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xcacc18a0 sched_switch(cacc18a0,0,109,1a3,a0afaa14,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cacc18a0,...) at mi_switch+0x1ff ithread_loop(cacad830,ef9c3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cacad830,ef9c3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef9c3d60, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xcacc1b80 sched_switch(cacc1b80,0,109,1a3,e5c35d86,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,cacc1b80,...) at mi_switch+0x1ff ithread_loop(cac97ae0,ef86dd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97ae0,ef86dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef86dd60, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc96eb2e0 sched_switch(c96eb2e0,0,109,1a3,f09086ab,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96eb2e0,...) at mi_switch+0x1ff ithread_loop(cac97c80,ef869d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97c80,ef869d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef869d60, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc96eb5c0 sched_switch(c96eb5c0,0,109,1a3,c1bd4e0b,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96eb5c0,...) at mi_switch+0x1ff ithread_loop(cac97500,ef763d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,cac97500,ef763d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xef763d60, ebp = 0 --- Tracing command intr pid 12 tid 100022 td 0xc97c1b80 sched_switch(c97c1b80,0,109,1a3,24c17ea3,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c97c1b80,...) at mi_switch+0x1ff ithread_loop(c95b3450,e33efd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3450,e33efd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33efd60, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc96228a0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100019 td 0xc96ea000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100018 td 0xc96ea2e0 sched_switch(c96ea2e0,0,109,1a3,6bc3888c,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96ea2e0,...) at mi_switch+0x1ff ithread_loop(c95b3490,e33e3d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3490,e33e3d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e3d60, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc95e05c0 sched_switch(c95e05c0,0,109,1a3,ca91cb20,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c95e05c0,...) at mi_switch+0x1ff ithread_loop(c95b36f0,e33d1d28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b36f0,e33d1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d1d60, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc95e08a0 sched_switch(c95e08a0,0,109,1a3,36f17c,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c95e08a0,...) at mi_switch+0x1ff ithread_loop(c95b3700,e33ced28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3700,e33ced28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33ced60, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc95e0b80 sched_switch(c95e0b80,0,109,1a3,e0070cec,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c95e0b80,...) at mi_switch+0x1ff ithread_loop(c95b3710,e33cbd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3710,e33cbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33cbd60, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc9622000 sched_switch(c9622000,0,109,1a3,faa1364,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c9622000,...) at mi_switch+0x1ff ithread_loop(c95b3720,c93fed28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3720,c93fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fed60, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc96222e0 sched_switch(c96222e0,0,109,1a3,e1ea8d90,...) at sched_switch+0x413 mi_switch(109,0,c0fcc8e1,557,c96222e0,...) at mi_switch+0x1ff ithread_loop(c95b3730,c93fbd28,c0fcc5b0,3d8,c95dd598,...) at ithread_loop+0x1fb fork_exit(c0a5a9c0,c95b3730,c93fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93fbd60, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc96225c0 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc95df000 sched_switch(c95df000,0,108,19e,d3e3a7c7,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df000,...) at mi_switch+0x1ff sched_idletd(0,c93f4d28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93f4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f4d60, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc95df2e0 sched_switch(c95df2e0,0,108,19e,d0e4387,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df2e0,...) at mi_switch+0x1ff sched_idletd(0,c93f1d28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93f1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93f1d60, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc95df5c0 sched_switch(c95df5c0,0,108,19e,e5b40213,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df5c0,...) at mi_switch+0x1ff sched_idletd(0,c93eed28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93eed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93eed60, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc95df8a0 sched_switch(c95df8a0,0,108,19e,d587cc2,...) at sched_switch+0x413 mi_switch(108,0,c0fd462c,3ce,c95df8a0,...) at mi_switch+0x1ff sched_idletd(0,c93ebd28,c0fcc5b0,3d8,c95dd864,...) at sched_idletd+0x17c fork_exit(c0aaf180,0,c93ebd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93ebd60, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc95dfb80 sched_switch(c95dfb80,0,104,1a3,6829aec4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c95dfb80,0,c0fd7fa0,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0fd7fa0,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c95ddb30,6c,c0fdaed8,100,0,...) at sleepq_wait_sig+0x17 _sleep(c95ddb30,c95ddbb8,16c,c0fdaed8,0,...) at _sleep+0x385 kern_wait(c95dfb80,ffffffff,c93e7c58,0,0,...) at kern_wait+0x597 sys_wait4(c95dfb80,c93e7cec,c1025a7c,c0fdad26,c1027030,...) at sys_wait4+0x3b syscall(c93e7d28) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x806070f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8c8 --- Tracing command audit pid 10 tid 100001 td 0xc95e0000 sched_switch(c95e0000,0,104,1a3,d001a2ac,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e0000,0,c0fd7fa0,268,c95e0000,...) at sleepq_switch+0x162 sleepq_wait(c130bd00,0,c93e4c8c,1,0,...) at sleepq_wait+0x63 _cv_wait(c130bd00,c130bce4,c1004257,194,c93e4cc4,...) at _cv_wait+0x260 audit_worker(0,c93e4d28,c0fcc5b0,3d8,c95de000,...) at audit_worker+0x84 fork_exit(c0c95110,0,c93e4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc93e4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xcae25000 sched_switch(cae25000,0,104,1a3,69b1b594,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae25000,0,c0fd7fa0,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c118d4e4,0,c0fbe932,2,0,...) at sleepq_timedwait+0x6b _sleep(c118d4e4,0,0,c0fbe932,bb8) at _sleep+0x36a pause(c0fbe932,bb8,126,124,c118a7e8,...) at pause+0xa0 deadlkres(0,efa4ed28,c0fcc5b0,3d8,c118a760,...) at deadlkres+0x324 fork_exit(c0a398a0,0,efa4ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4ed60, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xcae252e0 sched_switch(cae252e0,0,104,1a3,37dd85d4,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cae252e0,0,c0fd7fa0,268,cae252e0,...) at sleepq_switch+0x162 sleepq_wait(c971ddc0,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971ddc0,c971ddd8,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c136d440,efa4bd28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c136d440,efa4bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa4bd60, ebp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xcace45c0 sched_switch(cace45c0,0,104,1a3,c56e6a5,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cace45c0,0,c0fd7fa0,268,cace45c0,...) at sleepq_switch+0x162 sleepq_wait(c971de00,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971de00,c971de18,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(cad3049c,efa2fd28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,cad3049c,efa2fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xefa2fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc96ebb80 sched_switch(c96ebb80,0,104,1a3,ea401cae,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c96ebb80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de40,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c971de40,c971de58,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c118b0f8,e33fed28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c118b0f8,e33fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fed60, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc97c1000 sched_switch(c97c1000,0,104,1a3,ea4007f8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c1000,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971de80,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c971de80,c971de98,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c130c2e4,e33fbd28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c130c2e4,e33fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33fbd60, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc97c12e0 sched_switch(c97c12e0,0,104,1a3,ea3ff12e,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c12e0,0,c0fd7fa0,268,c97c12e0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f8d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f8d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f8d60, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc97c15c0 sched_switch(c97c15c0,0,104,1a3,ea3fdeda,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c15c0,0,c0fd7fa0,268,c97c15c0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f5d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f5d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f5d60, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc97c18a0 sched_switch(c97c18a0,0,104,1a3,ea3fcb9f,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c97c18a0,0,c0fd7fa0,268,c97c18a0,...) at sleepq_switch+0x162 sleepq_wait(c971e000,0,c0fd3181,c0fbe932,0,...) at sleepq_wait+0x63 msleep_spin(c971e000,c971e018,c0fbe932,0,a55,...) at msleep_spin+0x25f taskqueue_thread_loop(c1154298,e33f2d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0x8f fork_exit(c0acc240,c1154298,e33f2d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33f2d60, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc9622b80 sched_switch(c9622b80,0,104,1a3,9aa474c8,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c9622b80,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c971e0c0,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c971e0c0,c971e0d8,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c11aa0ac,e33e9d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11aa0ac,e33e9d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33e9d60, ebp = 0 --- Tracing command kernel pid 0 tid 100013 td 0xc95e02e0 sched_switch(c95e02e0,0,104,1a3,e35aad09,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c95e02e0,0,c0fd7fa0,268,0,...) at sleepq_switch+0x162 sleepq_wait(c96e9000,0,c0fbe932,0,0,...) at sleepq_wait+0x63 _sleep(c96e9000,c96e9018,0,c0fbe932,0,...) at _sleep+0x39c taskqueue_thread_loop(c11a8ac0,e33d4d28,c0fcc5b0,3d8,c118a760,...) at taskqueue_thread_loop+0xb5 fork_exit(c0acc240,c11a8ac0,e33d4d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe33d4d60, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc118aa30 sched_switch(c118aa30,0,104,1a3,bbb23244,...) at sched_switch+0x413 mi_switch(104,0,c0fd7fa0,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c118aa30,0,c0fd7fa0,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c118a760,54,c1004c38,0,0,...) at sleepq_timedwait+0x6b _sleep(c118a760,0,54,c1004c38,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x240 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 309 MB: 294 278 262 246 230 214 198 182 166 150 134 118 102 86 70 54 38 22 6 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 3 $ svn diff -x -p /var/tmp/vmcontention/sys Index: /var/tmp/vmcontention/sys/vm/vm_radix.c =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.c (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.c (working copy) @@ -99,14 +99,12 @@ #define KSPLT64H(x) ((u_long)(((x) >> 32) & 0xFFFFFFFF)) #endif -CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); -CTASSERT((sizeof(u_int) * NBBY) >= VM_RADIX_LIMIT); - struct vm_radix_node { void *rn_child[VM_RADIX_COUNT]; /* Child nodes. */ volatile uint32_t rn_count; /* Valid children. */ }; +CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIMIT); CTASSERT(sizeof(struct vm_radix_node) < PAGE_SIZE); static uma_zone_t vm_radix_node_zone; @@ -356,6 +354,101 @@ vm_radix_reclaim_allnodes_internal(struct vm_radix } /* + * Remove the specified index from the tree. If possible the height of the + * tree is adjusted after deletion. May be used to cleanup intermediate + * nodes of the path if the key is not entirely present. + */ +static void +vm_radix_sweep(struct vm_radix *rtree, vm_pindex_t index, int color, int hard) +{ + struct vm_radix_node *stack[VM_RADIX_LIMIT]; + struct vm_radix_node *rnode, *root; + int level; + int slot; + + level = vm_radix_height(rtree, &root); + KASSERT(index <= VM_RADIX_MAX(level), + ("vm_radix_sweep: %p index out of range %jd.", rtree, + VM_RADIX_MAX(level))); + rnode = root; + level--; + /* + * Find the node and record the path in stack. + */ + while (level && rnode) { + stack[level] = rnode; + slot = vm_radix_slot(index, level); + CTR6(KTR_VM, + "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", + rtree, rnode, rnode->rn_child[slot], rnode->rn_count); + rnode = rnode->rn_child[slot]; + level--; + } + if (rnode == NULL) { + if (hard) + panic("vm_radix_sweep: index not present.\n"); + + /* + * Almost certainly it got an half-constructed tree it was + * expected. + */ + KASSERT(level < (VM_RADIX_LIMIT - 1), + ("vm_radix_sweep: level %d not consistent.\n", level)); + ++level; + rnode = stack[level]; + slot = vm_radix_slot(index, level); + } else { + slot = vm_radix_slot(index, 0); + if (hard && + vm_radix_match(rnode->rn_child[slot], color) == NULL) + panic("vm_radix_sweep: index not present as leaf.\n"); + } + + for (;;) { + CTR6(KTR_VM, +"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", + rtree, KSPLT64L(index), KSPLT64H(index), level, slot, + rnode); + CTR4(KTR_VM, + "remove: resetting tree %p, rnode %p, child %p, count %u", + rtree, rnode, + (rnode != NULL) ? rnode->rn_child[slot] : NULL, + (rnode != NULL) ? rnode->rn_count : 0); + rnode->rn_child[slot] = NULL; + /* + * Use atomics for the last level since red and black + * will both adjust it. + * Use a write memory barrier here in order to avoid + * rn_count reaching 0 before to fetch the actual pointer. + * Concurrent black removal, infact, may want to reclaim + * the radix node itself before to read it. + */ + MPASS(rnode->rn_count != 0); + if (level == 0) + atomic_add_rel_32(&rnode->rn_count, -1); + else + rnode->rn_count--; + + /* + * Only allow black removes to prune the tree. + */ + if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) + break; + vm_radix_node_put(rnode); + if (rnode == root) { + vm_radix_setroot(rtree, NULL, 0); + break; + } + rnode = stack[++level]; + slot = vm_radix_slot(index, level); + + } +} + +/* * Inserts the key-value pair in to the radix tree. Returns errno. * Panics if the key already exists. */ @@ -363,8 +456,7 @@ int vm_radix_insert(struct vm_radix *rtree, vm_pindex_t index, void *val) { struct vm_radix_node *iroot, *rnode, *root; - u_int allocmsk; - int clev, ilevel, level, slot; + int ilevel, level, slot; CTR4(KTR_VM, "insert: tree %p, " KFRMT64(index) ", val %p", rtree, @@ -417,13 +509,17 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ } /* Now that the tree is tall enough, fill in the path to the index. */ - allocmsk = 0; - clev = level; rnode = root; for (level = level - 1; level > 0; level--) { slot = vm_radix_slot(index, level); /* Add the required intermidiate nodes. */ if (rnode->rn_child[slot] == NULL) { + + /* + * In case of failed allocation, the vm_radix_sweep() + * will unwind back rn_count appropriately. + */ + rnode->rn_count++; rnode->rn_child[slot] = vm_radix_node_get(); if (rnode->rn_child[slot] == NULL) { CTR6(KTR_VM, @@ -434,35 +530,17 @@ vm_radix_insert(struct vm_radix *rtree, vm_pindex_ "insert: tree %p, rnode %p, child %p, count %u ENOMEM", rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - MPASS(level != clev || allocmsk == 0); - while (allocmsk != 0) { - rnode = root; - level = clev; - level--; - slot = vm_radix_slot(index, level); - CTR4(KTR_VM, - "insert: unwind root %p, level %d, slot %d, allocmsk: 0x%x", - root, level, slot, allocmsk); - MPASS(level >= (ffs(allocmsk) - 1)); - while (level > (ffs(allocmsk) - 1)) { - MPASS(level > 0); - slot = vm_radix_slot(index, - level); - rnode = rnode->rn_child[slot]; - level--; - } - MPASS((allocmsk & (1 << level)) != 0); - allocmsk &= ~(1 << level); - rnode->rn_count--; - vm_radix_node_put(rnode->rn_child[slot]); - rnode->rn_child[slot] = NULL; - } + vm_radix_sweep(rtree, index, VM_RADIX_BLACK, 0); + + /* + * vm_radix_sweep() may have changed the shape + * of the tree, refetch the root and level. + */ + level = vm_radix_height(rtree, &root); vm_radix_unwind_heightup(rtree, root, iroot, ilevel); return (ENOMEM); } - rnode->rn_count++; - allocmsk |= (1 << level); } CTR6(KTR_VM, "insert: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", @@ -812,86 +890,13 @@ restart: } /* - * Remove the specified index from the tree. If possible the height of the - * tree is adjusted after deletion. The value stored at index is returned - * panics if the key is not present. + * Remove an entry from the tree. Expects the entry to be already present. */ -void * +void vm_radix_remove(struct vm_radix *rtree, vm_pindex_t index, int color) { - struct vm_radix_node *stack[VM_RADIX_LIMIT]; - struct vm_radix_node *rnode, *root; - void *val; - int level; - int slot; - level = vm_radix_height(rtree, &root); - KASSERT(index <= VM_RADIX_MAX(level), - ("vm_radix_remove: %p index out of range %jd.", rtree, - VM_RADIX_MAX(level))); - rnode = root; - val = NULL; - level--; - /* - * Find the node and record the path in stack. - */ - while (level && rnode) { - stack[level] = rnode; - slot = vm_radix_slot(index, level); - CTR6(KTR_VM, - "remove: tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, "remove: tree %p, rnode %p, child %p, count %u", - rtree, rnode, rnode->rn_child[slot], rnode->rn_count); - rnode = rnode->rn_child[slot]; - level--; - } - KASSERT(rnode != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - slot = vm_radix_slot(index, 0); - val = vm_radix_match(rnode->rn_child[slot], color); - KASSERT(val != NULL, - ("vm_radix_remove: index not present in the tree.\n")); - - for (;;) { - CTR6(KTR_VM, -"remove: resetting tree %p, " KFRMT64(index) ", level %d, slot %d, rnode %p", - rtree, KSPLT64L(index), KSPLT64H(index), level, slot, - rnode); - CTR4(KTR_VM, - "remove: resetting tree %p, rnode %p, child %p, count %u", - rtree, rnode, - (rnode != NULL) ? rnode->rn_child[slot] : NULL, - (rnode != NULL) ? rnode->rn_count : 0); - rnode->rn_child[slot] = NULL; - /* - * Use atomics for the last level since red and black - * will both adjust it. - * Use a write memory barrier here in order to avoid - * rn_count reaching 0 before to fetch the actual pointer. - * Concurrent black removal, infact, may want to reclaim - * the radix node itself before to read it. - */ - if (level == 0) - atomic_add_rel_32(&rnode->rn_count, -1); - else - rnode->rn_count--; - /* - * Only allow black removes to prune the tree. - */ - if ((color & VM_RADIX_BLACK) == 0 || rnode->rn_count > 0) - break; - vm_radix_node_put(rnode); - if (rnode == root) { - vm_radix_setroot(rtree, NULL, 0); - break; - } - rnode = stack[++level]; - slot = vm_radix_slot(index, level); - - } - return (val); + vm_radix_sweep(rtree, index, color, 1); } /* Index: /var/tmp/vmcontention/sys/vm/vm_radix.h =================================================================== --- /var/tmp/vmcontention/sys/vm/vm_radix.h (revision 236123) +++ /var/tmp/vmcontention/sys/vm/vm_radix.h (working copy) @@ -63,7 +63,7 @@ int vm_radix_lookupn(struct vm_radix *, vm_pindex_ void **, int, vm_pindex_t *); void *vm_radix_lookup_le(struct vm_radix *, vm_pindex_t, int); void vm_radix_reclaim_allnodes(struct vm_radix *); -void *vm_radix_remove(struct vm_radix *, vm_pindex_t, int); +void vm_radix_remove(struct vm_radix *, vm_pindex_t, int); /* * Look up any entry at a position greater or equal to index. -- Test scenario: altbufferflushes.sh