GENERIC HEAD from 2010-01-19 11:42:15 UTC, r202614M, vmcore.47 KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2010 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 9.0-CURRENT #0 r202614M: Wed Jan 20 07:34:16 CET 2010 pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0xf24 Stepping = 4 Features=0x3febfbff real memory = 1073741824 (1024 MB) avail memory = 1031356416 (983 MB) : Trying to mount root from ufs:/dev/ad0s1a Entropy harvesting: interrupts ethernet point_to_point kickstart. /dev/ad0s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad0s1a: clean, 267431 free (1439 frags, 33249 blocks, 0.3% fragmentation) /dev/ad0s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad0s1f: clean, 446095 free (511 frags, 55698 blocks, 0.1% fragmentation) /dev/ad0s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad0s1e: clean, 2022938 free (234 frags, 252838 blocks, 0.0% fragmentation) /dev/ad0s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad0s1d: clean, 2838657 free (53241 frags, 348177 blocks, 1.0% fragmentation) /dev/ad0s1g: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad0s1g: clean, 18238038 free (23470 frags, 2276821 blocks, 0.1% fragmentation) lock order reversal: 1st 0xd8104500 bufwait (bufwait) @ kern/vfs_bio.c:2581 2nd 0xc443f600 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:283 KDB: stack backtrace: db_trace_self_wrapper(c0cac517,e692787c,c08d80b5,c08c8b0b,c0caf4c8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08c8b0b,c0caf4c8,c413bf60,c413f228,e69278d8,...) at kdb_backtrace+0x29 _witness_debugger(c0caf4c8,c443f600,c0cd2f8f,c413f228,c0cd2c21,...) at _witness_debugger+0x25 witness_checkorder(c443f600,9,c0cd2c18,11b,0,...) at witness_checkorder+0x839 _sx_xlock(c443f600,0,c0cd2c18,11b,c47a2b54,...) at _sx_xlock+0x85 ufsdirhash_acquire(d81044a0,e6927a1c,164,d87b14ac,e69279a8,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(c47a2b54,e6927a1c,4ac,e6927994,e6927998,...) at ufsdirhash_add+0x13 ufs_direnter(c47a8ae0,c47b0c3c,e6927a1c,e6927c00,d8106ee0,...) at ufs_direnter+0x749 ufs_mkdir(e6927c28,c0ce8682,0,0,e6927b6c,...) at ufs_mkdir+0x993 VOP_MKDIR_APV(c0db9ce0,e6927c28,e6927c00,e6927b6c,0,...) at VOP_MKDIR_APV+0xc5 kern_mkdirat(c4731b40,ffffff9c,bfbfef5a,0,1ff,...) at kern_mkdirat+0x21b kern_mkdir(c4731b40,bfbfef5a,0,1ff,e6927d2c,...) at kern_mkdir+0x2e mkdir(c4731b40,e6927cf8,8,c0cafd79,c0d97a40,...) at mkdir+0x29 syscall(e6927d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (136, FreeBSD ELF32, mkdir), eip = 0x2816ba63, esp = 0xbfbfed6c, ebp = 0xbfbfee38 --- fxp0: link state changed to UP Starting Network: lo0 fxp0. add net default: gateway 192.168.1.1 Additional ABI support: linux. Starting mountd. Configuring syscons: keymap blanktime. Local package initialization:lock order reversal: 1st 0xc47a9880 ufs (ufs) @ kern/vfs_mount.c:1204 2nd 0xc47645c8 devfs (devfs) @ ufs/ffs/ffs_vfsops.c:1387 KDB: stack backtrace: db_trace_self_wrapper(c0cac517,e6a20a10,c08d80b5,c08c8b0b,c0caf4c8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08c8b0b,c0caf4c8,c413f1c0,c413f088,e6a20a6c,...) at kdb_backtrace+0x29 _witness_debugger(c0caf4c8,c47645c8,c0c9d9b8,c413f088,c0cd2247,...) at _witness_debugger+0x25 witness_checkorder(c47645c8,9,c0cd223e,56b,c4764634,...) at witness_checkorder+0x839 __lockmgr_args(c47645c8,80400,c4764634,0,0,...) at __lockmgr_args+0x804 vop_stdlock(e6a20b88,c0f48fe0,c4848524,80400,c4764570,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c0d93000,e6a20b88,e6a20ba8,c0dd46c0,c4764570,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c4764570,80400,c0cd223e,56b,c0cb70f5,...) at _vn_lock+0x78 ffs_sync(c47ac2d4,1,c0cb5e5c,4f9,80,...) at ffs_sync+0x3d5 dounmount(c47ac2d4,8000000,c4848480,47e,e083d37b,...) at dounmount+0x44e unmount(c4848480,e6a20cf8,8,c4848480,c0d96dc8,...) at unmount+0x2ff syscall(e6a20d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280da8ef, esp = 0xbfbfe68c, ebp = 0xbfbfe758 --- ** /dev/ad0s1e ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 435 files, 6093 used, 2022938 free (250 frags, 252836 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** usage: kill [-s signal_name] pid ... kill -l [exit_status] kill -signal_name pid ... kill -signal_number pid ... fsck -y /tmp watchdogd. Wed Jan 20 08:03:35 CET 2010 Jan 20 08:03:47 crashbox su: pho to root on /dev/pts/0 lock order reversal: 1st 0xc4cf6724 ufs (ufs) @ kern/vfs_lookup.c:497 2nd 0xd813a3e0 bufwait (bufwait) @ ufs/ffs/ffs_softdep.c:10998 3rd 0xc4e111b4 ufs (ufs) @ kern/vfs_subr.c:2091 KDB: stack backtrace: db_trace_self_wrapper(c0cac517,e6b382ac,c08d80b5,c08c8b0b,c0caf4e1,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08c8b0b,c0caf4e1,c413bf60,c413f1c0,e6b38308,...) at kdb_backtrace+0x29 _witness_debugger(c0caf4e1,c4e111b4,c0ca1afa,c413f1c0,c0cb6667,...) at _witness_debugger+0x25 witness_checkorder(c4e111b4,9,c0cb665e,82b,0,...) at witness_checkorder+0x839 __lockmgr_args(c4e111b4,80100,c4e11220,0,0,...) at __lockmgr_args+0x804 ffs_lock(e6b38428,c08d7e5b,c0cb5b45,80100,c4e1115c,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0db9ce0,e6b38428,c4d89524,c0dd46c0,c4e1115c,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c4e1115c,80100,c0cb665e,82b,4,...) at _vn_lock+0x78 vget(c4e1115c,80100,c4d89480,50,0,...) at vget+0xbb vfs_hash_get(c46d05a8,259d,80000,c4d89480,e6b38578,...) at vfs_hash_get+0xed ffs_vgetf(c46d05a8,259d,80000,e6b38578,1,...) at ffs_vgetf+0x49 softdep_sync_metadata(c4cf66cc,0,c0cd2797,147,0,...) at softdep_sync_metadata+0xc72 ffs_syncvnode(c4cf66cc,1,c4cf67c4,c413bf60,c0f46aa0,...) at ffs_syncvnode+0x3e2 ffs_truncate(c4cf66cc,200,0,880,c4504880,...) at ffs_truncate+0x8a2 ufs_direnter(c4cf66cc,c4e1115c,e6b388d0,e6b38bd0,0,...) at ufs_direnter+0x924 ufs_makeinode(e6b38bd0,c0dba1e0,e6b38abc,e6b38a18,c0bef7f5,...) at ufs_makeinode+0x5dc ufs_create(e6b38abc,c0ce8848,0,0,e6b38ba4,...) at ufs_create+0x30 VOP_CREATE_APV(c0db9ce0,e6b38abc,e6b38bd0,e6b38a54,0,...) at VOP_CREATE_APV+0xc5 vn_open_cred(e6b38ba4,e6b38c5c,1b0,0,c4504880,...) at vn_open_cred+0x205 vn_open(e6b38ba4,e6b38c5c,1b0,c4886a10,e6b38b58,...) at vn_open+0x3b kern_openat(c4d89480,ffffff9c,bfbfe664,0,602,...) at kern_openat+0x11f kern_open(c4d89480,bfbfe664,0,601,1b0,...) at kern_open+0x35 open(c4d89480,e6b38cf8,c,c0cafcc1,c0d96bec,...) at open+0x30 syscall(e6b38d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x2817eaa3, esp = 0xbfbfe58c, ebp = 0xbfbfe5a8 --- Expensive timeout(9) function: 0xc088d110(0xc47c42a8) 0.015932270 s Expensive timeout(9) function: 0xc0a13ce0(0) 0.026405784 s lock order reversal: 1st 0xc4f741b4 syncer (syncer) @ kern/vfs_subr.c:1701 2nd 0xc5525c94 ufs (ufs) @ kern/vfs_subr.c:2091 KDB: stack backtrace: db_trace_self_wrapper(c0cac517,e46faa18,c08d80b5,c08c8b0b,c0caf4c8,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08c8b0b,c0caf4c8,c413f290,c413f1c0,e46faa74,...) at kdb_backtrace+0x29 _witness_debugger(c0caf4c8,c5525c94,c0ca1afa,c413f1c0,c0cb6667,...) at _witness_debugger+0x25 witness_checkorder(c5525c94,9,c0cb665e,82b,0,...) at witness_checkorder+0x839 __lockmgr_args(c5525c94,80500,c5525d00,0,0,...) at __lockmgr_args+0x804 ffs_lock(e46fab94,c08d7e5b,c0cb6667,80500,c5525c3c,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c0db9ce0,e46fab94,c436ce24,c0dd46c0,c5525c3c,...) at VOP_LOCK1_APV+0xb5 _vn_lock(c5525c3c,80500,c0cb665e,82b,4,...) at _vn_lock+0x78 vget(c5525c3c,80500,c436cd80,c72,e46fac20,...) at vget+0xbb vfs_msync(c62ddb50,2,c0cb665e,d6b,c62ddb50,...) at vfs_msync+0xf5 sync_fsync(e46fac7c,c0ce7e2d,c4f74220,e46fac7c,c4f7415c,...) at sync_fsync+0x17b VOP_FSYNC_APV(c0da8a00,e46fac7c,c0cb665e,6a5,c436cd80,...) at VOP_FSYNC_APV+0xc5 sync_vnode(c0f75d98,c0f75d84,3e8,6d4,4e20,...) at sync_vnode+0x16b sched_sync(0,e46fad38,c0ca4427,343,c465ed48,...) at sched_sync+0x273 fork_exit(c09293f0,0,e46fad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46fad70, ebp = 0 --- Expensive timeout(9) function: 0xc0607e20(0xc4291000) 0.076066466 s Jan 20 09:26:50 crashbox inetd[1029]: discard/udp server failing (looping), service terminated Jan 20 11:05:44 crashbox inetd[1029]: discard/udp server failing (looping), service terminated Expensive timeout(9) function: 0xc0a13ce0(0) 0.084212398 s Limiting icmp unreach response from 26206 to 200 packets/sec Limiting icmp unreach response from 25694 to 200 packets/sec Expensive timeout(9) function: 0xc0607e20(0xc4291000) 0.085534892 s Expensive timeout(9) function: 0xc08c58f0(0) 0.087564599 s Limiting icmp unreach response from 26533 to 200 packets/sec Limiting icmp unreach response from 25707 to 200 packets/sec Limiting icmp unreach response from 22818 to 200 packets/sec Limiting icmp unreach response from 5510 to 200 packets/sec Limiting icmp unreach response from 18210 to 200 packets/sec Limiting icmp unreach response from 23625 to 200 packets/sec Limiting icmp unreach response from 26202 to 200 packets/sec Limiting icmp unreach response from 9784 to 200 packets/sec Limiting icmp unreach response from 14455 to 200 packets/sec Limiting icmp unreach response from 6535 to 200 packets/sec Limiting icmp unreach response from 9554 to 200 packets/sec Limiting icmp unreach response from 10827 to 200 packets/sec Limiting icmp unreach response from 5171 to 200 packets/sec Limiting icmp unreach response from 6502 to 200 packets/sec Limiting icmp unreach response from 18312 to 200 packets/sec Limiting icmp unreach response from 17189 to 200 packets/sec Limiting icmp unreach response from 23126 to 200 packets/sec Limiting icmp unreach response from 15908 to 200 packets/sec Limiting icmp unreach response from 6927 to 200 packets/sec Limiting icmp unreach response from 12438 to 200 packets/sec Limiting icmp unreach response from 8661 to 200 packets/sec Limiting icmp unreach response from 12608 to 200 packets/sec Limiting icmp unreach response from 3735 to 200 packets/sec Limiting icmp unreach response from 10666 to 200 packets/sec Limiting icmp unreach response from 11179 to 200 packets/sec Limiting icmp unreach response from 14829 to 200 packets/sec Limiting icmp unreach response from 24678 to 200 packets/sec Limiting icmp unreach response from 10683 to 200 packets/sec Limiting icmp unreach response from 5976 to 200 packets/sec Limiting icmp unreach response from 13110 to 200 packets/sec Limiting icmp unreach response from 4416 to 200 packets/sec Limiting icmp unreach response from 18559 to 200 packets/sec Limiting icmp unreach response from 13795 to 200 packets/sec Limiting icmp unreach response from 14239 to 200 packets/sec Limiting icmp unreach response from 13606 to 200 packets/sec Limiting icmp unreach response from 10316 to 200 packets/sec Limiting icmp unreach response from 9907 to 200 packets/sec Limiting icmp unreach response from 16028 to 200 packets/sec Expensive timeout(9) function: 0xc08ce700(0xc4977240) 0.119029236 s Limiting icmp unreach response from 25743 to 200 packets/sec Limiting icmp unreach response from 24882 to 200 packets/sec Limiting icmp unreach response from 22662 to 200 packets/sec Limiting icmp unreach response from 24981 to 200 packets/sec Limiting icmp unreach response from 16305 to 200 packets/sec Limiting icmp unreach response from 4410 to 200 packets/sec Limiting icmp unreach response from 3913 to 200 packets/sec Limiting icmp unreach response from 4477 to 200 packets/sec Limiting icmp unreach response from 4165 to 200 packets/sec Limiting icmp unreach response from 23047 to 200 packets/sec Limiting icmp unreach response from 24330 to 200 packets/sec Limiting icmp unreach response from 4676 to 200 packets/sec Limiting icmp unreach response from 2941 to 200 packets/sec Limiting icmp unreach response from 16156 to 200 packets/sec Limiting icmp unreach response from 19188 to 200 packets/sec Limiting icmp unreach response from 11004 to 200 packets/sec Limiting icmp unreach response from 8328 to 200 packets/sec Limiting icmp unreach response from 1682 to 200 packets/sec Limiting icmp unreach response from 6861 to 200 packets/sec Limiting icmp unreach response from 5716 to 200 packets/sec Limiting icmp unreach response from 7436 to 200 packets/sec Limiting icmp unreach response from 14787 to 200 packets/sec Limiting icmp unreach response from 17582 to 200 packets/sec Limiting icmp unreach response from 20535 to 200 packets/sec Limiting icmp unreach response from 11591 to 200 packets/sec Limiting icmp unreach response from 21848 to 200 packets/sec Limiting icmp unreach response from 19836 to 200 packets/sec Expensive timeout(9) function: 0xc08f3020(0) 0.321327281 s Limiting icmp unreach response from 18969 to 200 packets/sec Limiting icmp unreach response from 917 to 200 packets/sec Limiting icmp unreach response from 1235 to 200 packets/sec Limiting icmp unreach response from 11387 to 200 packets/sec Limiting icmp unreach response from 3220 to 200 packets/sec Limiting icmp unreach response from 5522 to 200 packets/sec Limiting icmp unreach response from 22185 to 200 packets/sec Limiting icmp unreach response from 24305 to 200 packets/sec Limiting icmp unreach response from 25356 to 200 packets/sec Limiting icmp unreach response from 25324 to 200 packets/sec Limiting icmp unreach response from 21143 to 200 packets/sec Limiting icmp unreach response from 10725 to 200 packets/sec Limiting icmp unreach response from 14107 to 200 packets/sec Limiting icmp unreach response from 10634 to 200 packets/sec Limiting icmp unreach response from 14486 to 200 packets/sec Limiting icmp unreach response from 4492 to 200 packets/sec Limiting icmp unreach response from 21984 to 200 packets/sec Limiting icmp unreach response from 13254 to 200 packets/sec Limiting icmp unreach response from 9060 to 200 packets/sec Limiting icmp unreach response from 25168 to 200 packets/sec Limiting icmp unreach response from 19200 to 200 packets/sec Limiting icmp unreach response from 15818 to 200 packets/sec Limiting icmp unreach response from 16701 to 200 packets/sec Limiting icmp unreach response from 13876 to 200 packets/sec Limiting icmp unreach response from 23366 to 200 packets/sec Limiting icmp unreach response from 9903 to 200 packets/sec Limiting icmp unreach response from 3353 to 200 packets/sec Limiting icmp unreach response from 2701 to 200 packets/sec Limiting icmp unreach response from 15070 to 200 packets/sec Limiting icmp unreach response from 6305 to 200 packets/sec Limiting icmp unreach response from 20963 to 200 packets/sec Limiting icmp unreach response from 12583 to 200 packets/sec Limiting icmp unreach response from 19984 to 200 packets/sec Limiting icmp unreach response from 14856 to 200 packets/sec Limiting icmp unreach response from 17315 to 200 packets/sec Limiting icmp unreach response from 20236 to 200 packets/sec Limiting icmp unreach response from 9619 to 200 packets/sec Limiting icmp unreach response from 4918 to 200 packets/sec Limiting icmp unreach response from 10535 to 200 packets/sec Limiting icmp unreach response from 18181 to 200 packets/sec Limiting icmp unreach response from 16275 to 200 packets/sec Limiting icmp unreach response from 24948 to 200 packets/sec Expensive timeout(9) function: 0xc0607e20(0xc4291000) 0.924817574 s Jan 20 11:54:03 crashbox inetd[1029]: discard/udp server failing (looping), service terminated Limiting icmp unreach response from 12170 to 200 packets/sec Limiting icmp unreach response from 33266 to 200 packets/sec Limiting icmp unreach response from 28160 to 200 packets/sec Limiting icmp unreach response from 31934 to 200 packets/sec Limiting icmp unreach response from 34719 to 200 packets/sec Limiting icmp unreach response from 35179 to 200 packets/sec Limiting icmp unreach response from 33054 to 200 packets/sec Limiting icmp unreach response from 33337 to 200 packets/sec Limiting icmp unreach response from 29859 to 200 packets/sec Limiting icmp unreach response from 32851 to 200 packets/sec Limiting icmp unreach response from 34130 to 200 packets/sec Limiting icmp unreach response from 30797 to 200 packets/sec Limiting icmp unreach response from 34518 to 200 packets/sec Limiting icmp unreach response from 37568 to 200 packets/sec Limiting icmp unreach response from 29113 to 200 packets/sec Limiting icmp unreach response from 30306 to 200 packets/sec Limiting icmp unreach response from 29436 to 200 packets/sec Limiting icmp unreach response from 34279 to 200 packets/sec Limiting icmp unreach response from 28115 to 200 packets/sec Limiting icmp unreach response from 32648 to 200 packets/sec Limiting icmp unreach response from 33624 to 200 packets/sec Limiting icmp unreach response from 27873 to 200 packets/sec Limiting icmp unreach response from 9621 to 200 packets/sec Limiting icmp unreach response from 64011 to 200 packets/sec Limiting icmp unreach response from 73375 to 200 packets/sec Limiting icmp unreach response from 76608 to 200 packets/sec Limiting icmp unreach response from 76600 to 200 packets/sec Limiting icmp unreach response from 27841 to 200 packets/sec Limiting icmp unreach response from 10968 to 200 packets/sec Limiting icmp unreach response from 15521 to 200 packets/sec Limiting icmp unreach response from 9372 to 200 packets/sec Limiting icmp unreach response from 8412 to 200 packets/sec Limiting icmp unreach response from 5290 to 200 packets/sec Limiting icmp unreach response from 12495 to 200 packets/sec Limiting icmp unreach response from 18910 to 200 packets/sec Limiting icmp unreach response from 14992 to 200 packets/sec Limiting icmp unreach response from 8661 to 200 packets/sec Limiting icmp unreach response from 17016 to 200 packets/sec Limiting icmp unreach response from 19237 to 200 packets/sec Limiting icmp unreach response from 8331 to 200 packets/sec Limiting icmp unreach response from 21761 to 200 packets/sec Limiting icmp unreach response from 8750 to 200 packets/sec Limiting icmp unreach response from 9398 to 200 packets/sec Limiting icmp unreach response from 17377 to 200 packets/sec Limiting icmp unreach response from 12800 to 200 packets/sec Limiting icmp unreach response from 17420 to 200 packets/sec Limiting icmp unreach response from 10393 to 200 packets/sec Limiting icmp unreach response from 21585 to 200 packets/sec Limiting icmp unreach response from 12175 to 200 packets/sec Limiting icmp unreach response from 7027 to 200 packets/sec Limiting icmp unreach response from 16681 to 200 packets/sec Limiting icmp unreach response from 14570 to 200 packets/sec Limiting icmp unreach response from 9110 to 200 packets/sec Limiting icmp unreach response from 13424 to 200 packets/sec Limiting icmp unreach response from 15101 to 200 packets/sec Limiting icmp unreach response from 21878 to 200 packets/sec Limiting icmp unreach response from 7085 to 200 packets/sec Limiting icmp unreach response from 15453 to 200 packets/sec Limiting icmp unreach response from 17621 to 200 packets/sec Limiting icmp unreach response from 18188 to 200 packets/sec Limiting icmp unreach response from 18648 to 200 packets/sec Limiting icmp unreach response from 13239 to 200 packets/sec Limiting icmp unreach response from 13236 to 200 packets/sec Limiting icmp unreach response from 11147 to 200 packets/sec Limiting icmp unreach response from 8071 to 200 packets/sec Limiting icmp unreach response from 19301 to 200 packets/sec Limiting icmp unreach response from 13128 to 200 packets/sec Limiting icmp unreach response from 8733 to 200 packets/sec Limiting icmp unreach response from 13225 to 200 packets/sec Limiting icmp unreach response from 6911 to 200 packets/sec Limiting icmp unreach response from 17254 to 200 packets/sec Limiting icmp unreach response from 15400 to 200 packets/sec Limiting icmp unreach response from 7785 to 200 packets/sec Limiting icmp unreach response from 7033 to 200 packets/sec Limiting icmp unreach response from 10243 to 200 packets/sec Limiting icmp unreach response from 17194 to 200 packets/sec Limiting icmp unreach response from 22383 to 200 packets/sec Limiting icmp unreach response from 15901 to 200 packets/sec Limiting icmp unreach response from 14127 to 200 packets/sec Limiting icmp unreach response from 3869 to 200 packets/sec Limiting icmp unreach response from 18736 to 200 packets/sec Limiting icmp unreach response from 16020 to 200 packets/sec Limiting icmp unreach response from 10015 to 200 packets/sec Limiting icmp unreach response from 10127 to 200 packets/sec Limiting icmp unreach response from 8164 to 200 packets/sec Limiting icmp unreach response from 10723 to 200 packets/sec Limiting icmp unreach response from 6321 to 200 packets/sec Limiting icmp unreach response from 6861 to 200 packets/sec Limiting icmp unreach response from 6823 to 200 packets/sec Limiting icmp unreach response from 18966 to 200 packets/sec Limiting icmp unreach response from 11883 to 200 packets/sec Limiting icmp unreach response from 1487 to 200 packets/sec Limiting icmp unreach response from 11677 to 200 packets/sec Limiting icmp unreach response from 20529 to 200 packets/sec Limiting icmp unreach response from 8276 to 200 packets/sec Limiting icmp unreach response from 14870 to 200 packets/sec Limiting icmp unreach response from 12842 to 200 packets/sec Limiting icmp unreach response from 11957 to 200 packets/sec Limiting icmp unreach response from 8019 to 200 packets/sec Limiting icmp unreach response from 11165 to 200 packets/sec Limiting icmp unreach response from 28505 to 200 packets/sec Limiting icmp unreach response from 10925 to 200 packets/sec Limiting icmp unreach response from 20445 to 200 packets/sec Limiting icmp unreach response from 17407 to 200 packets/sec Limiting icmp unreach response from 9183 to 200 packets/sec Limiting icmp unreach response from 9281 to 200 packets/sec Limiting icmp unreach response from 24795 to 200 packets/sec Limiting icmp unreach response from 12054 to 200 packets/sec Limiting icmp unreach response from 13187 to 200 packets/sec Limiting icmp unreach response from 18521 to 200 packets/sec Limiting icmp unreach response from 5022 to 200 packets/sec Limiting icmp unreach response from 14171 to 200 packets/sec Limiting icmp unreach response from 23309 to 200 packets/sec Limiting icmp unreach response from 10715 to 200 packets/sec Limiting icmp unreach response from 12221 to 200 packets/sec Limiting icmp unreach response from 17971 to 200 packets/sec Limiting icmp unreach response from 15434 to 200 packets/sec Limiting icmp unreach response from 10225 to 200 packets/sec Limiting icmp unreach response from 17551 to 200 packets/sec Limiting icmp unreach response from 22371 to 200 packets/sec Limiting icmp unreach response from 11125 to 200 packets/sec Limiting icmp unreach response from 10157 to 200 packets/sec Limiting icmp unreach response from 4962 to 200 packets/sec Limiting icmp unreach response from 6954 to 200 packets/sec Limiting icmp unreach response from 6219 to 200 packets/sec Limiting icmp unreach response from 15265 to 200 packets/sec Limiting icmp unreach response from 14747 to 200 packets/sec Limiting icmp unreach response from 26444 to 200 packets/sec Limiting icmp unreach response from 11040 to 200 packets/sec Limiting icmp unreach response from 15506 to 200 packets/sec Limiting icmp unreach response from 10471 to 200 packets/sec Jan 20 13:12:53 crashbox su: pho to root on /dev/pts/0 panic: handle_workitem_remove: worklist not empty. jaddref cpuid = 1 KDB: enter: panic [thread pid 18 tid 100049 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 18 tid 100049 td 0xc436c900 kdb_enter(c0ca9032,c0ca9032,c0cd0590,e4700c28,1,...) at kdb_enter+0x3a panic(c0cd0590,c0cd06eb,c0ccf8ab,1d2b,c093470f,...) at panic+0x136 handle_workitem_remove(0,e4700c88,2,513,1,...) at handle_workitem_remove+0x277 process_worklist_item(c0f81798,0,c0ccf8ab,480,c436c900,...) at process_worklist_item+0x1ff softdep_process_worklist(c62ddb50,0,c0ccf8ab,403,3e8,...) at softdep_process_worklist+0x8c softdep_flush(0,e4700d38,c0ca4427,343,c465e7f8,...) at softdep_flush+0x293 fork_exit(c0ad4250,0,e4700d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4700d70, ebp = 0 --- db:0:bt> show allpcpu Current CPU: 1 cpuid = 0 dynamic pcpu = 0x650280 curthread = 0xc4180480: pid 11 "idle: cpu0" curpcb = 0xc3f0ad90 fpcurthread = none idlethread = 0xc4180480: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x310c280 curthread = 0xc436c900: pid 18 "softdepflush" curpcb = 0xe4700d90 fpcurthread = none idlethread = 0xc41806c0: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x310f280 curthread = 0xc4180900: pid 11 "idle: cpu2" curpcb = 0xc3f04d90 fpcurthread = none idlethread = 0xc4180900: pid 11 "idle: cpu2" APIC ID = 6 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x3112280 curthread = 0xc4180b40: pid 11 "idle: cpu3" curpcb = 0xc3f01d90 fpcurthread = none idlethread = 0xc4180b40: pid 11 "idle: cpu3" APIC ID = 7 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 11215 (rm) thread 0xc4d47240 (100158) exclusive sleep mutex SYSMAPS (SYSMAPS) r = 0 (0xc0fd52a0) locked @ i386/i386/pmap.c:3867 exclusive sleep mutex vm object (standard object) r = 0 (0xc46dc880) locked @ vm/vm_fault.c:679 shared sx user map (user map) r = 0 (0xc4883de0) locked @ vm/vm_map.c:3532 Process 1059 (sshd) thread 0xc4766d80 (100077) exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xc496a6fc) locked @ kern/uipc_sockbuf.c:148 Process 18 (softdepflush) thread 0xc436c900 (100049) exclusive sleep mutex Softdep Lock (Softdep Lock) r = 0 (0xc0f81798) locked @ ufs/ffs/ffs_softdep.c:7467 exclusive lockmgr ufs (ufs) r = 0 (0xc7e67c94) locked @ kern/vfs_subr.c:2091 db:0:alllocks> show lockedvnods Locked vnodes 0xc7e67c3c: tag ufs, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0 flags () v_object 0xc5c5e220 ref 0 pages 0 lock type ufs: EXCL by thread 0xc436c900 (pid 18) #0 0xc087ac0e at __lockmgr_args+0xbfe #1 0xc0add701 at ffs_lock+0xa1 #2 0xc0bee825 at VOP_LOCK1_APV+0xb5 #3 0xc0934848 at _vn_lock+0x78 #4 0xc092794b at vget+0xbb #5 0xc091a80d at vfs_hash_get+0xed #6 0xc0ad7dd9 at ffs_vgetf+0x49 #7 0xc0ac915b at process_worklist_item+0x10b #8 0xc0ad3f2c at softdep_process_worklist+0x8c #9 0xc0ad44e3 at softdep_flush+0x293 #10 0xc0867da8 at fork_exit+0xb8 #11 0xc0bb7d10 at fork_trampoline+0x8 ino 141353, on dev ad0s1e db:0:lockedvnods> show mount 0xc46cfb50 /dev/ad0s1a on / (ufs) 0xc46d0000 devfs on /dev (devfs) 0xc47ac5a8 /dev/ad0s1f on /home (ufs) 0xc47ac000 /dev/ad0s1d on /usr (ufs) 0xc46d0b50 /dev/ad0s1g on /var (ufs) 0xc62ddb50 /dev/ad0s1e on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 11215 11108 11108 1001 R+ rm 11108 1062 11108 1001 S+ wait 0xc7d542a8 sh 1062 1061 1062 1001 Ss+ wait 0xc4a20000 bash 1061 1059 1059 1001 S select 0xc497d3e4 sshd 1059 948 1059 0 SWs sbwait 0xc496a728 sshd 1058 1 1058 0 Ss+ ttyin 0xc443f070 getty 1057 1 1057 0 Ss+ ttyin 0xc443f270 getty 1056 1 1056 0 Ss+ ttyin 0xc4154e70 getty 1055 1 1055 0 Ss+ ttyin 0xc4426270 getty 1054 1 1054 0 Ss+ ttyin 0xc4426870 getty 1053 1 1053 0 Ss+ ttyin 0xc4426a70 getty 1052 1 1052 0 Ss+ ttyin 0xc4154c70 getty 1051 1 1051 0 Ss+ ttyin 0xc4291e70 getty 1029 1 1029 0 Ss select 0xc464a864 inetd 1005 1 1005 0 Ss select 0xc468a764 moused 987 1 987 0 Ss nanslp 0xc0e09864 watchdogd 965 1 965 0 Ss nanslp 0xc0e09864 cron 959 1 959 25 SWs pause 0xc496e058 sendmail 955 1 955 0 Ss select 0xc468bea4 sendmail 948 1 948 0 Ss select 0xc497d2e4 sshd 911 1 911 0 Ss select 0xc468bc64 ntpd 810 809 809 0 S (threaded) nfsd 100103 S rpcsvc 0xc46c8050 nfsd: service 100102 S rpcsvc 0xc46c8090 nfsd: service 100101 S rpcsvc 0xc46c80d0 nfsd: service 100054 S rpcsvc 0xc46d3010 nfsd: master 809 1 809 0 Ss select 0xc468a8e4 nfsd 807 1 807 0 Ss select 0xc46c9924 mountd 712 1 712 0 Ss select 0xc46c8124 rpcbind 692 1 692 0 Ss select 0xc464b2e4 syslogd 562 1 562 0 Ss select 0xc464b9a4 devd 19 0 0 0 DL flowclea 0xc0f75f88 [flowcleaner] 18 0 0 0 RL CPU 1 [softdepflush] 17 0 0 0 DL vlruwt 0xc465eaa0 [vnlru] 16 0 0 0 DL syncer 0xc0f75d98 [syncer] 15 0 0 0 DL psleep 0xc0f75ac8 [bufdaemon] 9 0 0 0 DL pgzero 0xc0f82614 [pagezero] 8 0 0 0 DL psleep 0xc0f82244 [vmdaemon] 7 0 0 0 DL psleep 0xc0f8220c [pagedaemon] 6 0 0 0 DL - 0xc429163c [fdc0] 14 0 0 0 DL (threaded) [usb] 100034 D - 0xc43a4dac [usbus0] 100033 D - 0xc43a4d7c [usbus0] 100032 D - 0xc43a4d4c [usbus0] 100031 D - 0xc43a4d1c [usbus0] 5 0 0 0 DL ccb_scan 0xc0dd5a54 [xpt_thrd] 13 0 0 0 DL - 0xc0e096c4 [yarrow] 4 0 0 0 DL - 0xc0e07464 [g_down] 3 0 0 0 DL - 0xc0e07460 [g_up] 2 0 0 0 DL - 0xc0e07458 [g_event] 12 0 0 0 WL (threaded) [intr] 100042 I [irq7: ppc0] 100040 I [swi0: uart uart] 100039 I [irq12: psm0] 100038 I [irq1: atkbd0] 100037 I [irq15: ata1] 100036 I [irq14: ata0] 100035 I [irq17: fxp0] 100030 I [irq16: uhci0] 100028 I [irq9: acpi0] 100024 I [swi2: cambio] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 Run CPU 0 [idle: cpu0] 100005 CanRun [idle: cpu1] 100004 Run CPU 2 [idle: cpu2] 100003 Run CPU 3 [idle: cpu3] 1 0 1 0 SLs wait 0xc417ed48 [init] 10 0 0 0 DL audit_wo 0xc0f81000 [audit] 0 0 0 0 DLs (threaded) [kernel] 100029 D - 0xc4388340 [em0 taskq] 100027 D - 0xc4345100 [acpi_task_2] 100026 D - 0xc4345100 [acpi_task_1] 100025 D - 0xc4345100 [acpi_task_0] 100020 D - 0xc4345380 [thread taskq] 100018 D - 0xc4345600 [kqueue taskq] 100016 D - 0xc4165e00 [firmware taskq] 100000 D sched 0xc0e07540 [swapper] db:0:ps> allt Tracing command rm pid 11215 tid 100158 td 0xc4d47240 sched_switch(c4d47240,0,207,18c,23ada5a,...) at sched_switch+0x406 mi_switch(207,0,c0cade80,d6,8,...) at mi_switch+0x200 ast(e6ac4d38) at ast+0x2b3 doreti_ast() at doreti_ast+0x17 Tracing command sh pid 11108 tid 100333 td 0xc4e28d80 sched_switch(c4e28d80,0,104,191,1c0e514,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c4e28d80,0,c0cad641,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c7d542a8,5c,c0cafe4e,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7d542a8,c7d54330,15c,c0cafe4e,0,...) at _sleep+0x354 kern_wait(c4e28d80,ffffffff,e6d59c74,2,0,...) at kern_wait+0xb76 wait4(c4e28d80,e6d59cf8,10,c0cafe2d,c0d96c24,...) at wait4+0x3b syscall(e6d59d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815f76b, esp = 0xbfbfe67c, ebp = 0xbfbfe698 --- Tracing command bash pid 1062 tid 100126 td 0xc49776c0 sched_switch(c49776c0,0,104,191,b39f570a,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c49776c0,0,c0cad641,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c4a20000,5c,c0cafe4e,100,0,...) at sleepq_wait_sig+0x17 _sleep(c4a20000,c4a20088,15c,c0cafe4e,0,...) at _sleep+0x354 kern_wait(c49776c0,ffffffff,e6a64c74,6,0,...) at kern_wait+0xb76 wait4(c49776c0,e6a64cf8,10,c0cafd3d,c0d96c24,...) at wait4+0x3b syscall(e6a64d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x282bf76b, esp = 0xbfbfe9cc, ebp = 0xbfbfe9e8 --- Tracing command sshd pid 1061 tid 100127 td 0xc4977480 sched_switch(c4977480,0,104,191,b3794100,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4977480,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c497d3d0,0,c0ca77ba,c4977480,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c497d3e4,0,e6a67a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c497d3e4,c497d3d0,c0caf8b4,603,c47161f8,...) at _cv_wait_sig+0x240 seltdwait(c47161f8,58,c4687080,c4977480,200246,...) at seltdwait+0xa2 kern_select(c4977480,a,286030b8,286030dc,0,0,20,0,280fecd0) at kern_select+0x4f4 select(c4977480,e6a67cf8,14,c0c90d1c,c0d9758c,...) at select+0x66 syscall(e6a67d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283cd5f3, esp = 0xbfbfde5c, ebp = 0xbfbfdea8 --- Tracing command sshd pid 1059 tid 100077 td 0xc4766d80 Tracing command getty pid 1058 tid 100137 td 0xc4bcb6c0 sched_switch(c4bcb6c0,0,104,191,95e8cfa,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4bcb6c0,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c4bcb6c0,c4bcb6c0,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c443f070,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c443f070,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c443f000,c443f070,e6a85c58,1,0,...) at tty_wait+0x71 ttydisc_read(c443f000,e6a85c58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c443d800,e6a85c58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c47495e8,e6a85c58,c4184380,0,c4bcb6c0,...) at devfs_read_f+0x7e dofileread(e6a85c58,ffffffff,ffffffff,0,c47495e8,...) at dofileread+0x96 kern_readv(c4bcb6c0,0,e6a85c58,e6a85c78,1,...) at kern_readv+0x58 read(c4bcb6c0,e6a85cf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e6a85d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command getty pid 1057 tid 100136 td 0xc4bcb900 sched_switch(c4bcb900,0,104,191,8ee41ca,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4bcb900,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c4bcb900,c4bcb900,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c443f270,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c443f270,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c443f200,c443f270,e6a82c58,1,0,...) at tty_wait+0x71 ttydisc_read(c443f200,e6a82c58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c443d900,e6a82c58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c4886000,e6a82c58,c4184380,0,c4bcb900,...) at devfs_read_f+0x7e dofileread(e6a82c58,ffffffff,ffffffff,0,c4886000,...) at dofileread+0x96 kern_readv(c4bcb900,0,e6a82c58,e6a82c78,1,...) at kern_readv+0x58 read(c4bcb900,e6a82cf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e6a82d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command getty pid 1056 tid 100135 td 0xc4731000 sched_switch(c4731000,0,104,191,95df88e,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4731000,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c4731000,c4731000,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c4154e70,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c4154e70,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c4154e00,c4154e70,e6a7fc58,1,0,...) at tty_wait+0x71 ttydisc_read(c4154e00,e6a7fc58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c443da00,e6a7fc58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c48868f8,e6a7fc58,c4184380,0,c4731000,...) at devfs_read_f+0x7e dofileread(e6a7fc58,ffffffff,ffffffff,0,c48868f8,...) at dofileread+0x96 kern_readv(c4731000,0,e6a7fc58,e6a7fc78,1,...) at kern_readv+0x58 read(c4731000,e6a7fcf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e6a7fd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command getty pid 1055 tid 100134 td 0xc4731240 sched_switch(c4731240,0,104,191,8fe140a,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4731240,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c4731240,c4731240,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c4426270,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c4426270,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c4426200,c4426270,e6a7cc58,1,0,...) at tty_wait+0x71 ttydisc_read(c4426200,e6a7cc58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c443db00,e6a7cc58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c4717310,e6a7cc58,c4184380,0,c4731240,...) at devfs_read_f+0x7e dofileread(e6a7cc58,ffffffff,ffffffff,0,c4717310,...) at dofileread+0x96 kern_readv(c4731240,0,e6a7cc58,e6a7cc78,1,...) at kern_readv+0x58 read(c4731240,e6a7ccf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e6a7cd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command getty pid 1054 tid 100133 td 0xc4731480 sched_switch(c4731480,0,104,191,9578396,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4731480,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c4731480,c4731480,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c4426870,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c4426870,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c4426800,c4426870,e6a79c58,1,0,...) at tty_wait+0x71 ttydisc_read(c4426800,e6a79c58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c443dc00,e6a79c58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c4886c78,e6a79c58,c4184380,0,c4731480,...) at devfs_read_f+0x7e dofileread(e6a79c58,ffffffff,ffffffff,0,c4886c78,...) at dofileread+0x96 kern_readv(c4731480,0,e6a79c58,e6a79c78,1,...) at kern_readv+0x58 read(c4731480,e6a79cf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e6a79d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command getty pid 1053 tid 100121 td 0xc497b240 sched_switch(c497b240,0,104,191,8f5ae72,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c497b240,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c497b240,c497b240,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c4426a70,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c4426a70,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c4426a00,c4426a70,e6a55c58,1,0,...) at tty_wait+0x71 ttydisc_read(c4426a00,e6a55c58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c438bb00,e6a55c58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c4886268,e6a55c58,c4184380,0,c497b240,...) at devfs_read_f+0x7e dofileread(e6a55c58,ffffffff,ffffffff,0,c4886268,...) at dofileread+0x96 kern_readv(c497b240,0,e6a55c58,e6a55c78,1,...) at kern_readv+0x58 read(c497b240,e6a55cf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e6a55d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command getty pid 1052 tid 100051 td 0xc470fd80 sched_switch(c470fd80,0,104,191,910ac32,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c470fd80,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c470fd80,c470fd80,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c4154c70,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c4154c70,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c4154c00,c4154c70,e68f3c58,1,0,...) at tty_wait+0x71 ttydisc_read(c4154c00,e68f3c58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c4369500,e68f3c58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c4749738,e68f3c58,c4184380,0,c470fd80,...) at devfs_read_f+0x7e dofileread(e68f3c58,ffffffff,ffffffff,0,c4749738,...) at dofileread+0x96 kern_readv(c470fd80,0,e68f3c58,e68f3c78,1,...) at kern_readv+0x58 read(c470fd80,e68f3cf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e68f3d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command getty pid 1051 tid 100073 td 0xc470f000 sched_switch(c470f000,0,104,191,95a0572,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c470f000,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,c470f000,c470f000,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c4291e70,0,c0cb1c19,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c4291e70,c0e09010,c0cb15e8,511,0,...) at _cv_wait_sig+0x240 tty_wait(c4291e00,c4291e70,e6960c58,1,0,...) at tty_wait+0x71 ttydisc_read(c4291e00,e6960c58,0,9f,0,...) at ttydisc_read+0x22c ttydev_read(c4427400,e6960c58,0,0,1,...) at ttydev_read+0xaa devfs_read_f(c473a508,e6960c58,c4184380,0,c470f000,...) at devfs_read_f+0x7e dofileread(e6960c58,ffffffff,ffffffff,0,c473a508,...) at dofileread+0x96 kern_readv(c470f000,0,e6960c58,e6960c78,1,...) at kern_readv+0x58 read(c470f000,e6960cf8,c,c0cc14b5,c0d96bb4,...) at read+0x4f syscall(e6960d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818e673, esp = 0xbfbfedbc, ebp = 0xbfbfede8 --- Tracing command inetd pid 1029 tid 100132 td 0xc4bb1240 sched_switch(c4bb1240,0,104,191,4289e1f2,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4bb1240,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c464a850,0,c0ca77ba,c4bb1240,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c464a864,0,e6a76a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c464a864,c464a850,c0caf8b4,603,c6951348,...) at _cv_wait_sig+0x240 seltdwait(c6951348,58,c4184380,c4bb1240,c4bb1240,...) at seltdwait+0xa2 kern_select(c4bb1240,8,bfbfdfd0,0,0,0,20,0,2822bc00) at kern_select+0x4f4 select(c4bb1240,e6a76cf8,14,c0ca6e68,c0d9758c,...) at select+0x66 syscall(e6a76d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281a25f3, esp = 0xbfbfdf8c, ebp = 0xbfbfee18 --- Tracing command moused pid 1005 tid 100130 td 0xc4868d80 sched_switch(c4868d80,0,104,191,e4d977b4,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4868d80,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c468a750,0,c0ca77ba,c4868d80,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c468a764,0,e6a70a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c468a764,c468a750,c0caf8b4,603,c473a850,...) at _cv_wait_sig+0x240 seltdwait(c473a850,58,c4184380,c4868d80,0,...) at seltdwait+0xa2 kern_select(c4868d80,6,bfbfea34,0,0,0,20,0,0) at kern_select+0x4f4 select(c4868d80,e6a70cf8,14,c0cc14b5,c0d9758c,...) at select+0x66 syscall(e6a70d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281ab5f3, esp = 0xbfbfe9bc, ebp = 0xbfbfeb58 --- Tracing command watchdogd pid 987 tid 100123 td 0xc4977d80 sched_switch(c4977d80,0,104,191,12eec140,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c4977d80,0,c0cad641,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c08ce700,c4977d80,2,100,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c0e09864,5c,c0caa2df,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0e09864,0,15c,c0caa2df,ea61,...) at _sleep+0x31e kern_nanosleep(c4977d80,e6a5bc64,e6a5bc6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c4977d80,e6a5bcf8,8,c0cc14b5,c0d985a0,...) at nanosleep+0x6f syscall(e6a5bd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28187f87, esp = 0xbfbfecec, ebp = 0xbfbfed18 --- Tracing command cron pid 965 tid 100111 td 0xc476d900 sched_switch(c476d900,0,104,191,468ae4ca,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c476d900,0,c0cad641,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(ea61,c08ce700,c476d900,0,100,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c0e09864,5c,c0caa2df,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0e09864,0,15c,c0caa2df,ea61,...) at _sleep+0x31e kern_nanosleep(c476d900,e6a2fc64,e6a2fc6c,3c,0,...) at kern_nanosleep+0xc1 nanosleep(c476d900,e6a2fcf8,8,c0cb0302,c0d985a0,...) at nanosleep+0x6f syscall(e6a2fd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2817af87, esp = 0xbfbfec8c, ebp = 0xbfbfecb8 --- Tracing command sendmail pid 959 tid 100115 td 0xc497c000 Tracing command sendmail pid 955 tid 100068 td 0xc470fb40 sched_switch(c470fb40,0,104,191,a78ef160,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c470fb40,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e6943a4c,c088238a,c468be90,0,c470fb40,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c468bea4,0,e6943a7c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c468bea4,c468be90,1389,603,c68a62a0,...) at _cv_timedwait_sig+0x250 seltdwait(e6943c28,e6943c30,c46c6880,c470fb40,c14b1014,...) at seltdwait+0x8a kern_select(c470fb40,5,bfbfc510,0,0,e6943c70,20,5,0) at kern_select+0x4f4 select(c470fb40,e6943cf8,14,c0cb0261,c0d9758c,...) at select+0x66 syscall(e6943d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283d75f3, esp = 0xbfbfc47c, ebp = 0xbfbfcfa8 --- Tracing command sshd pid 948 tid 100069 td 0xc470f900 sched_switch(c470f900,0,104,191,3ed62442,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c470f900,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c497d2d0,0,c0ca77ba,c470f900,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c497d2e4,0,e6947a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c497d2e4,c497d2d0,c0caf8b4,603,c4716af0,...) at _cv_wait_sig+0x240 seltdwait(c4716af0,58,c47de880,c470f900,c0afa9ee,...) at seltdwait+0xa2 kern_select(c470f900,5,286090b0,0,0,0,20,2861a070,1) at kern_select+0x4f4 select(c470f900,e6947cf8,14,c470f900,c0d9758c,...) at select+0x66 syscall(e6947d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283cd5f3, esp = 0xbfbfdf1c, ebp = 0xbfbfee38 --- Tracing command ntpd pid 911 tid 100112 td 0xc476d6c0 sched_switch(c476d6c0,0,104,191,bbcc2808,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c476d6c0,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c468bc50,0,c0ca77ba,c476d6c0,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c468bc64,0,e6a32a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c468bc64,c468bc50,c0caf8b4,603,c4717348,...) at _cv_wait_sig+0x240 seltdwait(c4717348,58,c4184380,c476d6c0,0,...) at seltdwait+0xa2 kern_select(c476d6c0,1c,bfbfed28,0,0,0,20,e6a32c98,246) at kern_select+0x4f4 select(c476d6c0,e6a32cf8,14,c476d6c0,c0d9758c,...) at select+0x66 syscall(e6a32d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283555f3, esp = 0xbfbfecfc, ebp = 0xbfbfedc8 --- Tracing command nfsd pid 810 tid 100103 td 0xc4848b40 sched_switch(c4848b40,0,104,191,8a26bf98,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4848b40,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e6a17bf8,c088238a,c4503c80,0,c4848b40,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c46c8050,0,e6a17c28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c46c8050,c4503c80,1388,3af,0,...) at _cv_timedwait_sig+0x250 svc_run_internal(e6a17d24,c0867da8,c4503c80,e6a17d38,c0ca4427,...) at svc_run_internal+0x356 svc_thread_start(c4503c80,e6a17d38,c0ca4427,343,c472e000,...) at svc_thread_start+0x10 fork_exit(c0a8a9b0,c4503c80,e6a17d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 810 tid 100102 td 0xc4848d80 sched_switch(c4848d80,0,104,191,4f376382,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4848d80,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e6a14bf8,c088238a,c4503c80,0,c4848d80,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c46c8090,0,e6a14c28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c46c8090,c4503c80,1388,3af,5a5a5a5a,...) at _cv_timedwait_sig+0x250 svc_run_internal(e6a14d24,c0867da8,c4503c80,e6a14d38,c0ca4427,...) at svc_run_internal+0x356 svc_thread_start(c4503c80,e6a14d38,c0ca4427,343,c472e000,...) at svc_thread_start+0x10 fork_exit(c0a8a9b0,c4503c80,e6a14d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 810 tid 100101 td 0xc4849000 sched_switch(c4849000,0,104,191,9295d31a,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4849000,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e6a11bf8,c088238a,c4503c80,0,c4849000,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c46c80d0,0,e6a11c28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c46c80d0,c4503c80,1388,3af,5a5a5a5a,...) at _cv_timedwait_sig+0x250 svc_run_internal(e6a11d24,c0867da8,c4503c80,e6a11d38,c0ca4427,...) at svc_run_internal+0x356 svc_thread_start(c4503c80,e6a11d38,c0ca4427,343,c472e000,...) at svc_thread_start+0x10 fork_exit(c0a8a9b0,c4503c80,e6a11d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 810 tid 100054 td 0xc4734900 sched_switch(c4734900,0,104,191,d9032370,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4734900,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e68ffae8,c088238a,c4503c80,0,c4734900,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c46d3010,0,e68ffb18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c46d3010,c4503c80,1388,3af,c4848b40,...) at _cv_timedwait_sig+0x250 svc_run_internal(c4734a68,14,c0ccca89,c0ccb53d,e68ffc3c,...) at svc_run_internal+0x356 svc_run(c4503c80,0,c0ccb841,1fb,0,...) at svc_run+0x7f nfssvc_nfsd(bfbfe8b0,e68ffc3c,c,c4184380,e68ffc50,...) at nfssvc_nfsd+0xad nfssvc_nfsserver(c4734900,e68ffcf8,bfbfe8b0,c4734900,c472e000,...) at nfssvc_nfsserver+0x24f nfssvc(c4734900,e68ffcf8,8,c0cb0261,c0d97c54,...) at nfssvc+0x83 syscall(e68ffd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280db28b, esp = 0xbfbfe86c, ebp = 0xbfbfead8 --- Tracing command nfsd pid 809 tid 100059 td 0xc476d000 sched_switch(c476d000,0,104,191,22e33b22,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c476d000,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c468a8d0,0,c0ca77ba,c476d000,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c468a8e4,0,e6913a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c468a8e4,c468a8d0,c0caf8b4,603,c4716070,...) at _cv_wait_sig+0x240 seltdwait(c4716070,58,c4184380,c476d000,c0afa9ee,...) at seltdwait+0xa2 kern_select(c476d000,5,bfbfece0,0,0,0,20,bfbffff4,1) at kern_select+0x4f4 select(c476d000,e6913cf8,14,c476d000,c0d9758c,...) at select+0x66 syscall(e6913d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2817f5f3, esp = 0xbfbfeadc, ebp = 0xbfbfee18 --- Tracing command mountd pid 807 tid 100078 td 0xc4766b40 sched_switch(c4766b40,0,104,191,7448358e,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4766b40,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c46c9910,0,c0ca77ba,c4766b40,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c46c9924,0,e6970a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c46c9924,c46c9910,c0caf8b4,603,c477b3b8,...) at _cv_wait_sig+0x240 seltdwait(c477b3b8,58,c4184380,c4766b40,c0f494e8,...) at seltdwait+0xa2 kern_select(c4766b40,9,bfbfed8c,0,0,0,20,0,281a5ae8) at kern_select+0x4f4 select(c4766b40,e6970cf8,14,c0ccb302,c0d9758c,...) at select+0x66 syscall(e6970d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281925f3, esp = 0xbfbfed5c, ebp = 0xbfbfee28 --- Tracing command rpcbind pid 712 tid 100082 td 0xc4766240 sched_switch(c4766240,0,104,191,ec679c94,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4766240,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(e697caa8,c088238a,c46c8110,0,c4766240,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c46c8124,0,e697cad8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c46c8124,c46c8110,7530,603,e697cb8c,...) at _cv_timedwait_sig+0x250 seltdwait(e697cc5c,e697cc64,511,c4766240,e697cb5c,...) at seltdwait+0x8a poll(c4766240,e697ccf8,c,c0caff8b,c0d9823c,...) at poll+0x300 syscall(e697cd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (209, FreeBSD ELF32, poll), eip = 0x28140b6f, esp = 0xbfbfcc0c, ebp = 0xbfbfedd8 --- Tracing command syslogd pid 692 tid 100087 td 0xc4849480 sched_switch(c4849480,0,104,191,b58dc824,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4849480,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c464b2d0,0,c0ca77ba,c4849480,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c464b2e4,0,e698ea7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c464b2e4,c464b2d0,c0caf8b4,603,c4749b98,...) at _cv_wait_sig+0x240 seltdwait(c4749b98,58,c4184380,c4849480,0,...) at seltdwait+0xa2 kern_select(c4849480,9,282290ac,0,0,0,20,0,281a4ae8) at kern_select+0x4f4 select(c4849480,e698ecf8,14,c0cafe3d,c0d9758c,...) at select+0x66 syscall(e698ed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281915f3, esp = 0xbfbfe2ac, ebp = 0xbfbfee18 --- Tracing command devd pid 562 tid 100063 td 0xc476b900 sched_switch(c476b900,0,104,191,cbcdaf8a,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c476b900,0,c0cad641,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c088238a,c464b990,0,c0ca77ba,c476b900,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c464b9a4,0,e6923a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c464b9a4,c464b990,c0caf8b4,603,c473a620,...) at _cv_wait_sig+0x240 seltdwait(c473a620,58,c4184380,c476b900,c0ca0eda,...) at seltdwait+0xa2 kern_select(c476b900,6,bfbfe9a0,0,0,0,20,0,10) at kern_select+0x4f4 select(c476b900,e6923cf8,14,c0c47d30,c0d9758c,...) at select+0x66 syscall(e6923d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x8088c03, esp = 0xbfbfe96c, ebp = 0xbfbfee48 --- Tracing command flowcleaner pid 19 tid 100050 td 0xc436c6c0 sched_switch(c436c6c0,0,104,191,b7ab2d62,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c436c6c0,0,c0cad641,283,c436c6c0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f75f88,0,e4703cc4,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0f75f88,c0f75f90,2710,3f0,0,...) at _cv_timedwait+0x250 flowtable_cleaner(0,e4703d38,c0ca4427,343,c465e550,...) at flowtable_cleaner+0x1bf fork_exit(c093e090,0,e4703d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4703d70, ebp = 0 --- Tracing command softdepflush pid 18 tid 100049 td 0xc436c900 kdb_enter(c0ca9032,c0ca9032,c0cd0590,e4700c28,1,...) at kdb_enter+0x3a panic(c0cd0590,c0cd06eb,c0ccf8ab,1d2b,c093470f,...) at panic+0x136 handle_workitem_remove(0,e4700c88,2,513,1,...) at handle_workitem_remove+0x277 process_worklist_item(c0f81798,0,c0ccf8ab,480,c436c900,...) at process_worklist_item+0x1ff softdep_process_worklist(c62ddb50,0,c0ccf8ab,403,3e8,...) at softdep_process_worklist+0x8c softdep_flush(0,e4700d38,c0ca4427,343,c465e7f8,...) at softdep_flush+0x293 fork_exit(c0ad4250,0,e4700d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe4700d70, ebp = 0 --- Tracing command vnlru pid 17 tid 100048 td 0xc436cb40 sched_switch(c436cb40,0,104,191,f058f7bc,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,50,...) at mi_switch+0x200 sleepq_switch(c436cb40,0,c0cad641,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c465eaa0,50,c0cb763f,0,0,...) at sleepq_timedwait+0x6b _sleep(c465eaa0,c0f75d58,250,c0cb763f,3e8,...) at _sleep+0x339 vnlru_proc(0,e46fdd38,c0ca4427,343,c465eaa0,...) at vnlru_proc+0xe7 fork_exit(c0929fc0,0,e46fdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46fdd70, ebp = 0 --- Tracing command syncer pid 16 tid 100047 td 0xc436cd80 sched_switch(c436cd80,0,104,191,cf883b7e,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c436cd80,0,c0cad641,283,c436cd80,...) at sleepq_switch+0x15f sleepq_timedwait(c0f75d98,0,e46fac88,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0f75d98,c0f75d84,3e8,6d4,4e20,...) at _cv_timedwait+0x250 sched_sync(0,e46fad38,c0ca4427,343,c465ed48,...) at sched_sync+0x502 fork_exit(c09293f0,0,e46fad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46fad70, ebp = 0 --- Tracing command bufdaemon pid 15 tid 100046 td 0xc4420000 sched_switch(c4420000,0,104,191,e29d17a2,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,44,...) at mi_switch+0x200 sleepq_switch(c4420000,0,c0cad641,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f75ac8,44,c0cb4b9c,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f75ac8,c0f75ad0,44,c0cb4b9c,3e8,...) at _sleep+0x339 buf_daemon(0,e46f7d38,c0ca4427,343,c417f2a8,...) at buf_daemon+0x16e fork_exit(c0911050,0,e46f7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46f7d70, ebp = 0 --- Tracing command pagezero pid 9 tid 100045 td 0xc4420240 sched_switch(c4420240,0,104,191,43af8c34,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4420240,0,c0cad641,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f82614,0,c0cd7bad,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f82614,c0f82100,0,c0cd7bad,493e0,...) at _sleep+0x339 vm_pagezero(0,e46f4d38,c0ca4427,343,c417f550,...) at vm_pagezero+0xdc fork_exit(c0b13c70,0,e46f4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46f4d70, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100044 td 0xc4420480 sched_switch(c4420480,0,104,191,4dadf62c,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,68,...) at mi_switch+0x200 sleepq_switch(c4420480,0,c0cad641,260,0,...) at sleepq_switch+0x15f sleepq_wait(c0f82244,68,c0cb4b9c,0,0,...) at sleepq_wait+0x63 _sleep(c0f82244,c0f82248,68,c0cb4b9c,0,...) at _sleep+0x36b vm_daemon(0,e46f1d38,c0ca4427,343,c417f7f8,...) at vm_daemon+0x59 fork_exit(c0b0e100,0,e46f1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46f1d70, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100043 td 0xc44206c0 sched_switch(c44206c0,0,104,191,f60ed3b4,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,44,...) at mi_switch+0x200 sleepq_switch(c44206c0,0,c0cad641,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f8220c,44,c0cb4b9c,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f8220c,c0f82100,44,c0cb4b9c,1388,...) at _sleep+0x339 vm_pageout(0,e46eed38,c0ca4427,343,c417faa0,...) at vm_pageout+0x2bb fork_exit(c0b0efa0,0,e46eed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46eed70, ebp = 0 --- Tracing command fdc0 pid 6 tid 100041 td 0xc4420b40 sched_switch(c4420b40,0,104,191,aecc5b54,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c4420b40,0,c0cad641,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c429163c,4c,c0c9e869,0,0,...) at sleepq_timedwait+0x6b _sleep(c429163c,c42916f0,4c,c0c9e869,3e8,...) at _sleep+0x339 fdc_thread(c4291600,e46e8d38,c0ca4427,343,c417fd48,...) at fdc_thread+0x27d fork_exit(c0b912b0,c4291600,e46e8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46e8d70, ebp = 0 --- Tracing command usb pid 14 tid 100034 td 0xc436a6c0 sched_switch(c436a6c0,0,104,191,57a2c53c,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c436a6c0,0,c0cad641,260,c436a6c0,...) at sleepq_switch+0x15f sleepq_wait(c43a4dac,0,c3f88cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c43a4dac,c43a4e4c,c0c97559,6c,c43a4db4,...) at _cv_wait+0x240 usb_process(c43a4da4,c3f88d38,c0ca4427,343,c4347000,...) at usb_process+0x193 fork_exit(c07c28a0,c43a4da4,c3f88d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f88d70, ebp = 0 --- Tracing command usb pid 14 tid 100033 td 0xc436a900 sched_switch(c436a900,0,104,191,cc8527b2,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c436a900,0,c0cad641,260,c436a900,...) at sleepq_switch+0x15f sleepq_wait(c43a4d7c,0,c3f85cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c43a4d7c,c43a4e4c,c0c97559,6c,c43a4d84,...) at _cv_wait+0x240 usb_process(c43a4d74,c3f85d38,c0ca4427,343,c4347000,...) at usb_process+0x193 fork_exit(c07c28a0,c43a4d74,c3f85d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f85d70, ebp = 0 --- Tracing command usb pid 14 tid 100032 td 0xc436ab40 sched_switch(c436ab40,0,104,191,572d797c,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c436ab40,0,c0cad641,260,c436ab40,...) at sleepq_switch+0x15f sleepq_wait(c43a4d4c,0,c3f82cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c43a4d4c,c43a4e4c,c0c97559,6c,c43a4d54,...) at _cv_wait+0x240 usb_process(c43a4d44,c3f82d38,c0ca4427,343,c4347000,...) at usb_process+0x193 fork_exit(c07c28a0,c43a4d44,c3f82d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f82d70, ebp = 0 --- Tracing command usb pid 14 tid 100031 td 0xc436ad80 sched_switch(c436ad80,0,104,191,572d49f8,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c436ad80,0,c0cad641,260,c436ad80,...) at sleepq_switch+0x15f sleepq_wait(c43a4d1c,0,c3f7fcbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c43a4d1c,c43a4e4c,c0c97559,6c,c43a4d24,...) at _cv_wait+0x240 usb_process(c43a4d14,c3f7fd38,c0ca4427,343,c4347000,...) at usb_process+0x193 fork_exit(c07c28a0,c43a4d14,c3f7fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f7fd70, ebp = 0 --- Tracing command xpt_thrd pid 5 tid 100023 td 0xc435a000 sched_switch(c435a000,0,104,191,572d0a68,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c435a000,0,c0cad641,260,0,...) at sleepq_switch+0x15f sleepq_wait(c0dd5a54,4c,c0c42eda,0,0,...) at sleepq_wait+0x63 _sleep(c0dd5a54,c0dd5a6c,4c,c0c42eda,0,...) at _sleep+0x36b xpt_scanner_thread(0,c3f40d38,c0ca4427,343,c43472a8,...) at xpt_scanner_thread+0x4a fork_exit(c0484d80,0,c3f40d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f40d70, ebp = 0 --- Tracing command yarrow pid 13 tid 100017 td 0xc4182240 sched_switch(c4182240,0,104,191,ff0fc148,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4182240,0,c0cad641,283,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0e096c4,0,c0c9e869,2,0,...) at sleepq_timedwait+0x6b _sleep(c0e096c4,0,0,c0c9e869,64,...) at _sleep+0x339 pause(c0c9e869,64,c0c8b254,111,0,...) at pause+0x47 random_kthread(0,c3f2ed38,c0ca4427,343,c4347550,...) at random_kthread+0x1ef fork_exit(c07393b0,0,c3f2ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f2ed70, ebp = 0 --- Tracing command g_down pid 4 tid 100015 td 0xc41826c0 sched_switch(c41826c0,0,104,191,1dcc7b0,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c41826c0,0,c0cad641,260,0,...) at sleepq_switch+0x15f sleepq_wait(c0e07464,4c,c0c9e869,0,0,...) at sleepq_wait+0x63 _sleep(c0e07464,c0e073c8,24c,c0c9e869,0,...) at _sleep+0x36b g_io_schedule_down(c41826c0,0,c0c9ff75,74,0,...) at g_io_schedule_down+0x56 g_down_procbody(0,c3f28d38,c0ca4427,343,c417e000,...) at g_down_procbody+0x8d fork_exit(c082edf0,0,c3f28d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f28d70, ebp = 0 --- Tracing command g_up pid 3 tid 100014 td 0xc4182900 sched_switch(c4182900,0,104,191,1f4a488,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c4182900,0,c0cad641,260,0,...) at sleepq_switch+0x15f sleepq_wait(c0e07460,4c,c0c9e869,0,0,...) at sleepq_wait+0x63 _sleep(c0e07460,c0e073e8,24c,c0c9e869,0,...) at _sleep+0x36b g_io_schedule_up(c4182900,0,c0c9ff75,5d,0,...) at g_io_schedule_up+0x11e g_up_procbody(0,c3f25d38,c0ca4427,343,c417e2a8,...) at g_up_procbody+0x8d fork_exit(c082ee80,0,c3f25d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f25d70, ebp = 0 --- Tracing command g_event pid 2 tid 100013 td 0xc4182b40 sched_switch(c4182b40,0,104,191,fbb5d426,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c4182b40,0,c0cad641,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e07458,4c,c0c9e869,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e07458,0,4c,c0c9e869,64,...) at _sleep+0x339 g_event_procbody(0,c3f22d38,c0ca4427,343,c417e550,...) at g_event_procbody+0xcb fork_exit(c082ef10,0,c3f22d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f22d70, ebp = 0 --- Tracing command intr pid 12 tid 100042 td 0xc4420900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100040 td 0xc4420d80 sched_switch(c4420d80,0,109,191,1d7ab51a,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c4417470,...) at mi_switch+0x200 ithread_loop(c441bb40,e46dbd38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c441bb40,e46dbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46dbd70, ebp = 0 --- Tracing command intr pid 12 tid 100039 td 0xc4421000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100038 td 0xc435ad80 sched_switch(c435ad80,0,109,191,56b54230,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41842f0,...) at mi_switch+0x200 ithread_loop(c4411ca0,e46d5d38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c4411ca0,e46d5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46d5d70, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xc436a000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100036 td 0xc436a240 sched_switch(c436a240,0,109,191,1e53140,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c4183b70,...) at mi_switch+0x200 ithread_loop(c4411820,e46ccd38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c4411820,e46ccd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46ccd70, ebp = 0 --- Tracing command intr pid 12 tid 100035 td 0xc436a480 sched_switch(c436a480,0,109,191,be43c104,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41839f0,...) at mi_switch+0x200 ithread_loop(c43b5b10,e46c6d38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c43b5b10,e46c6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe46c6d70, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc436c000 sched_switch(c436c000,0,109,191,5685cb38,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c4183a70,...) at mi_switch+0x200 ithread_loop(c438a830,c3f7cd38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c438a830,c3f7cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f7cd70, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc436c480 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100024 td 0xc41c7d80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100022 td 0xc435a240 sched_switch(c435a240,0,109,191,ba187814,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41c6c70,...) at mi_switch+0x200 ithread_loop(c4135950,c3f3dd38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c4135950,c3f3dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f3dd70, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc435a480 sched_switch(c435a480,0,109,191,cbcb7caa,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41c6cf0,...) at mi_switch+0x200 ithread_loop(c4135960,c3f3ad38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c4135960,c3f3ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f3ad70, ebp = 0 --- Tracing command intr pid 12 tid 100019 td 0xc435a900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100012 td 0xc4182d80 sched_switch(c4182d80,0,109,191,d902d248,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41c8d70,...) at mi_switch+0x200 ithread_loop(c417d090,c3f1fd38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c417d090,c3f1fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f1fd70, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc41c7000 sched_switch(c41c7000,0,109,191,fbb51194,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41c8df0,...) at mi_switch+0x200 ithread_loop(c417d0a0,c3f1cd38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c417d0a0,c3f1cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f1cd70, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc41c7240 sched_switch(c41c7240,0,109,191,ff0f7bc6,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41c8e70,...) at mi_switch+0x200 ithread_loop(c417d0b0,c3f19d38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c417d0b0,c3f19d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f19d70, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc41c7480 sched_switch(c41c7480,0,109,191,68cc3e6,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c41846f0,...) at mi_switch+0x200 ithread_loop(c417d0c0,c3f16d38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c417d0c0,c3f16d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f16d70, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc4180000 sched_switch(c4180000,0,109,191,1bf9f26e,...) at sched_switch+0x406 mi_switch(109,0,c0ca46a6,52d,c4184770,...) at mi_switch+0x200 ithread_loop(c417d0d0,c3f13d38,c0ca4427,343,c417e7f8,...) at ithread_loop+0x1f6 fork_exit(c086ad90,c417d0d0,c3f13d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f13d70, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc4180240 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc4180480 cpustop_handler(1,c3f0ac10,c0bd5c56,da7a0000,6400,...) at cpustop_handler+0x32 ipi_nmi_handler(da7a0000,6400,0,e0007ede,c417eaa0,...) at ipi_nmi_handler+0x2f trap(c3f0ac1c) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc08b77a1, esp = 0xc3f0ac5c, ebp = 0xc3f0ac84 --- cpu_search_highest(c0e1bad0,c3f0ac98,0,fffffffe,0,...) at cpu_search_highest+0x121 cpu_search_highest(c0e1bac0,c3f0acd4,c0caac2e,a09,c4180480,...) at cpu_search_highest+0x71 sched_idletd(0,c3f0ad38,c0ca4427,343,c417eaa0,...) at sched_idletd+0xe7 fork_exit(c08b77b0,0,c3f0ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f0ad70, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc41806c0 sched_switch(c41806c0,0,60c,18c,1f482fa,...) at sched_switch+0x406 mi_switch(60c,0,c0caac2e,815,1,...) at mi_switch+0x200 sched_preempt(c41806c0,1,1f4,c3f07c74,c0bb838e,...) at sched_preempt+0x9f ipi_bitmap_handler(8,c4180028,11b0028,c4291818,c4291800,...) at ipi_bitmap_handler+0x34 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0bac485, esp = 0xc3f07c74, ebp = 0xc3f07c74 --- acpi_cpu_c1(1,1,1,c08b76f1,c0e1bae0,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(c3f07cb4,c0bc3c3b,1,c3f07cf8,c08b79ee,...) at acpi_cpu_idle+0x11c cpu_idle_acpi(1,c3f07cf8,c08b79ee,1,c3f07cd4,...) at cpu_idle_acpi+0x1b cpu_idle(1,c3f07cd4,c0caac2e,a09,c41806c0,...) at cpu_idle+0x1b sched_idletd(0,c3f07d38,c0ca4427,343,c417eaa0,...) at sched_idletd+0x23e fork_exit(c08b77b0,0,c3f07d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f07d70, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc4180900 cpustop_handler(4,c3f04c28,c0bd5c56,c0e09918,c3f04bbc,...) at cpustop_handler+0x32 ipi_nmi_handler(c0e09918,c3f04bbc,c0881fd4,c0e09918,c417eaa0,...) at ipi_nmi_handler+0x2f trap(c3f04c34) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0bac485, esp = 0xc3f04c74, ebp = 0xc3f04c74 --- acpi_cpu_c1(1,2,2,c08b76f1,c0e1bae0,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(c3f04cb4,c0bc3c3b,1,c3f04cf8,c08b79ee,...) at acpi_cpu_idle+0x11c cpu_idle_acpi(1,c3f04cf8,c08b79ee,1,c3f04cd4,...) at cpu_idle_acpi+0x1b cpu_idle(1,c3f04cd4,c0caac2e,a09,c4180900,...) at cpu_idle+0x1b sched_idletd(0,c3f04d38,c0ca4427,343,c417eaa0,...) at sched_idletd+0x23e fork_exit(c08b77b0,0,c3f04d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f04d70, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc4180b40 cpustop_handler(8,c3f01c28,c0bd5c56,c0e09954,c3f01bbc,...) at cpustop_handler+0x32 ipi_nmi_handler(c0e09954,c3f01bbc,c0881fd4,c0e09954,c417eaa0,...) at ipi_nmi_handler+0x2f trap(c3f01c34) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0bac485, esp = 0xc3f01c74, ebp = 0xc3f01c74 --- acpi_cpu_c1(1,2,3,c08b76f1,c0e1bae0,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(c3f01cb4,c0bc3c3b,0,c3f01cf8,c08b79ee,...) at acpi_cpu_idle+0x11c cpu_idle_acpi(0,c3f01cf8,c08b79ee,0,c3f01cd4,...) at cpu_idle_acpi+0x1b cpu_idle(0,c3f01cd4,c0caac2e,3b0,c4180b40,...) at cpu_idle+0x1b sched_idletd(0,c3f01d38,c0ca4427,343,c417eaa0,...) at sched_idletd+0x23e fork_exit(c08b77b0,0,c3f01d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f01d70, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc4180d80 sched_switch(c4180d80,0,104,191,e11c0700,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c4180d80,0,c0cad641,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0cad641,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c417ed48,5c,c0cafe4e,100,0,...) at sleepq_wait_sig+0x17 _sleep(c417ed48,c417edd0,15c,c0cafe4e,0,...) at _sleep+0x354 kern_wait(c4180d80,ffffffff,c3efdc74,0,0,...) at kern_wait+0xb76 wait4(c4180d80,c3efdcf8,10,c0cafc7e,c0d96c24,...) at wait4+0x3b syscall(c3efdd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x805569f, esp = 0xbfbfe90c, ebp = 0xbfbfe928 --- Tracing command audit pid 10 tid 100001 td 0xc4182000 sched_switch(c4182000,0,104,191,572773f0,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4182000,0,c0cad641,260,c4182000,...) at sleepq_switch+0x15f sleepq_wait(c0f81000,0,c3efac9c,1,0,...) at sleepq_wait+0x63 _cv_wait(c0f81000,c0f80fe4,c0ccd7f0,194,0,...) at _cv_wait+0x240 audit_worker(0,c3efad38,c0ca4427,343,c417f000,...) at audit_worker+0x84 fork_exit(c0a9aae0,0,c3efad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3efad70, ebp = 0 --- Tracing command kernel pid 0 tid 100029 td 0xc436c240 sched_switch(c436c240,0,104,191,5685326c,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c436c240,0,c0cad641,260,c436c240,...) at sleepq_switch+0x15f sleepq_wait(c4388340,0,c0ca9c27,c0c9e869,0,...) at sleepq_wait+0x63 msleep_spin(c4388340,c4388358,c0c9e869,0,c0ca77ba,...) at msleep_spin+0x21d taskqueue_thread_loop(c43905a0,c3f78d38,c0ca4427,343,c0e07540,...) at taskqueue_thread_loop+0x94 fork_exit(c08d11a0,c43905a0,c3f78d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f78d70, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc41c76c0 sched_switch(c41c76c0,0,104,191,bbc63fac,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c41c76c0,0,c0cad641,260,c41c76c0,...) at sleepq_switch+0x15f sleepq_wait(c4345100,0,c0ca9c27,c0c9e869,0,...) at sleepq_wait+0x63 msleep_spin(c4345100,c4345118,c0c9e869,0,c0ca77ba,...) at msleep_spin+0x21d taskqueue_thread_loop(c0dd88a0,c3f4cd38,c0ca4427,343,c0e07540,...) at taskqueue_thread_loop+0x94 fork_exit(c08d11a0,c0dd88a0,c3f4cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f4cd70, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc41c7900 sched_switch(c41c7900,0,104,191,bbc620a0,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c41c7900,0,c0cad641,260,c41c7900,...) at sleepq_switch+0x15f sleepq_wait(c4345100,0,c0ca9c27,c0c9e869,0,...) at sleepq_wait+0x63 msleep_spin(c4345100,c4345118,c0c9e869,0,c0ca77ba,...) at msleep_spin+0x21d taskqueue_thread_loop(c0dd88a0,c3f49d38,c0ca4427,343,c0e07540,...) at taskqueue_thread_loop+0x94 fork_exit(c08d11a0,c0dd88a0,c3f49d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f49d70, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc41c7b40 sched_switch(c41c7b40,0,104,191,bbc5fe18,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c41c7b40,0,c0cad641,260,c41c7b40,...) at sleepq_switch+0x15f sleepq_wait(c4345100,0,c0ca9c27,c0c9e869,0,...) at sleepq_wait+0x63 msleep_spin(c4345100,c4345118,c0c9e869,0,c0ca77ba,...) at msleep_spin+0x21d taskqueue_thread_loop(c0dd88a0,c3f46d38,c0ca4427,343,c0e07540,...) at taskqueue_thread_loop+0x94 fork_exit(c08d11a0,c0dd88a0,c3f46d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f46d70, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc435a6c0 sched_switch(c435a6c0,0,104,191,bbc20bec,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c435a6c0,0,c0cad641,260,0,...) at sleepq_switch+0x15f sleepq_wait(c4345380,0,c0c9e869,0,0,...) at sleepq_wait+0x63 _sleep(c4345380,c4345398,0,c0c9e869,0,...) at _sleep+0x36b taskqueue_thread_loop(c0e1bcc8,c3f37d38,c0ca4427,343,c0e07540,...) at taskqueue_thread_loop+0xba fork_exit(c08d11a0,c0e1bcc8,c3f37d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f37d70, ebp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xc435ab40 sched_switch(c435ab40,0,104,191,bbaf48c0,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c435ab40,0,c0cad641,260,0,...) at sleepq_switch+0x15f sleepq_wait(c4345600,0,c0c9e869,0,0,...) at sleepq_wait+0x63 _sleep(c4345600,c4345618,0,c0c9e869,0,...) at _sleep+0x36b taskqueue_thread_loop(c0e07dd8,c3f31d38,c0ca4427,343,c0e07540,...) at taskqueue_thread_loop+0xba fork_exit(c08d11a0,c0e07dd8,c3f31d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f31d70, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc4182480 sched_switch(c4182480,0,104,191,be72145e,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,0,...) at mi_switch+0x200 sleepq_switch(c4182480,0,c0cad641,260,0,...) at sleepq_switch+0x15f sleepq_wait(c4165e00,0,c0c9e869,0,0,...) at sleepq_wait+0x63 _sleep(c4165e00,c4165e18,0,c0c9e869,0,...) at _sleep+0x36b taskqueue_thread_loop(c0e1a760,c3f2bd38,c0ca4427,343,c0e07540,...) at taskqueue_thread_loop+0xba fork_exit(c08d11a0,c0e1a760,c3f2bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc3f2bd70, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc0e077f0 sched_switch(c0e077f0,0,104,191,3d8e0774,...) at sched_switch+0x406 mi_switch(104,0,c0cad641,1eb,44,...) at mi_switch+0x200 sleepq_switch(c0e077f0,0,c0cad641,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0e07540,44,c0cab4a3,0,0,...) at sleepq_timedwait+0x6b _sleep(c0e07540,0,44,c0cab4a3,2710,...) at _sleep+0x339 scheduler(0,141ec00,141ec00,141e000,1425000,...) at scheduler+0x23e mi_startup() at mi_startup+0x96 begin() at begin+0x2c db:0:allt> call doadump Physical memory: 1007 MB Dumping 178 MB: 163 147 131 115 99 83 67 51 35 19 3 Dump complete = 0xf db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 1 (kgdb) bt #0 doadump () at pcpu.h:246 #1 0xc04d1149 in db_fncall (dummy1=0xc08c488a, dummy2=0x0, dummy3=0xffffffff, dummy4=0xe47008c8 "Ü\bpä") at ../../../ddb/db_command.c:548 #2 0xc04d157f in db_command (last_cmdp=0xc0dd6a7c, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #3 0xc04d1634 in db_command_script (command=0xc0dd7988 "call doadump") at ../../../ddb/db_command.c:516 #4 0xc04d5800 in db_script_exec (scriptname=0xc0dd72e0 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc04d5891 in db_run_cmd (addr=0x1, have_addr=0x0, count=0xc0fd5f60, modif=0xe4700a00 "") at ../../../ddb/db_script.c:375 #6 0xc04d1541 in db_command (last_cmdp=0xc0dd6a7c, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:445 #7 0xc04d169a in db_command_loop () at ../../../ddb/db_command.c:498 #8 0xc04d353d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:229 #9 0xc08c4706 in kdb_trap (type=0x3, code=0x0, tf=0xe4700ba8) at ../../../kern/subr_kdb.c:535 #10 0xc0bd624b in trap (frame=0xe4700ba8) at ../../../i386/i386/trap.c:690 #11 0xc0bb7c9b in calltrap () at ../../../i386/i386/exception.s:165 #12 0xc08c488a in kdb_enter (why=0xc0ca9032 "panic", msg=0xc0ca9032 "panic") at cpufunc.h:71 #13 0xc0891d86 in panic (fmt=0xc0cd0590 "handle_workitem_remove: worklist not empty. %s") at ../../../kern/kern_shutdown.c:562 #14 0xc0aca027 in handle_workitem_remove (dirrem=0xcc9c2a00, xp=0xc7e67c3c) at ../../../ufs/ffs/ffs_softdep.c:7506 #15 0xc0ac924f in process_worklist_item (mp=0xc62ddb50, flags=Variable "flags" is not available. ) at ../../../ufs/ffs/ffs_softdep.c:1307 #16 0xc0ad3f2c in softdep_process_worklist (mp=0xc62ddb50, full=0x0) at ../../../ufs/ffs/ffs_softdep.c:1157 #17 0xc0ad44e3 in softdep_flush () at ../../../ufs/ffs/ffs_softdep.c:1035 #18 0xc0867da8 in fork_exit (callout=0xc0ad4250 , arg=0x0, frame=0xe4700d38) at ../../../kern/kern_fork.c:843 #19 0xc0bb7d10 in fork_trampoline () at ../../../i386/i386/exception.s:270 (kgdb) f 14 #14 0xc0aca027 in handle_workitem_remove (dirrem=0xcc9c2a00, xp=0xc7e67c3c) at ../../../ufs/ffs/ffs_softdep.c:7506 7506 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), (kgdb) l 7501 panic("handle_workitem_remove: bad file delta"); 7502 if (ip->i_nlink == 0) 7503 unlinked_inodedep(mp, inodedep); 7504 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 7505 num_dirrem -= 1; 7506 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 7507 ("handle_workitem_remove: worklist not empty. %s", 7508 TYPENAME(LIST_FIRST(&dirrem->dm_jwork)->wk_type))); 7509 WORKITEM_FREE(dirrem, D_DIRREM); 7510 FREE_LOCK(&lk); (kgdb) p *dirrem $1 = {dm_list = {wk_list = {le_next = 0xcc3e1300, le_prev = 0xc5d0f460}, wk_mp = 0xc62ddb50, wk_type = 0xc, wk_state = 0x20004}, dm_next = {le_next = 0x0, le_prev = 0x0}, dm_inonext = {le_next = 0x0, le_prev = 0xc7539a50}, dm_jremrefhd = { lh_first = 0x0}, dm_oldinum = 0x22829, dm_un = {dmu_pagedep = 0x2282a, dmu_dirinum = 0x2282a}, dm_jwork = {lh_first = 0xcce8db80}} (kgdb) p *dirrem->dm_jwork.lh_first $2 = {wk_list = {le_next = 0x0, le_prev = 0xcc9c2a2c}, wk_mp = 0xc62ddb50, wk_type = 0x10, wk_state = 0x8909} (kgdb) -- Test scenario: root=/tmp for i in `jot 10000`; do rm -rf ${root}/a mkdir -p ${root}/a/b/c/d/e/f/g mkdir -p ${root}/a/b/c/d/e/f/z cd ${root}/a/b/c/d/e/f ( mv ${root}/a/b/c ${root}/a/c ) & if ! mv z g/z; then echo "FAILURE at loop $i" break fi wait done rm -rf ${root}/a