GENERIC HEAD from Mon Apr 16 07:08:58 CEST 2012, r234335M, vmcore.337 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r234335M: Mon Apr 16 07:08:58 CEST 2012 pho@x4.osted.lan:/usr/src/sys/i386/compile/PHO i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.01-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) avail memory = 3531657216 (3368 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 673249 free (1641 frags, 83951 blocks, 0.2% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 120539 free (5075 frags, 14433 blocks, 0.5% fragmentation) /dev/label/tmp: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/label/tmp: clean, 111759897 free (2089 frags, 13969726 blocks, 0.0% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 3247351 free (241375 frags, 375747 blocks, 2.4% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 14305216 free (55040 frags, 1781272 blocks, 0.2% fragmentation) Mounting local file systems:. Setting hostname: x4.osted.lan. re0: link state changed to DOWN Starting Network: lo0 re0 fwe0 fwip0. lo0: flags=8049 metric 0 mtu 16384 options=3 inet6 ::1 prefixlen 128 inet6 fe80::1%lo0 prefixlen 64 scopeid 0xa inet 127.0.0.1 netmask 0xff000000 nd6 options=21 re0: finlags=8843 metric 0 mtu 1500 options=8209b ether 00:21:85:15:33:7d inet 192.168.1.101 netmask 0xffffff00 broadcast 192.168.1.255 inet6 fe80::221:85ff:fe15:337d%re0 prefixlen 64 scopeid 0x1 nd6 options=29 media: Ethernet autoselect (100baseTX ) status: active fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 Starting devd. Starting Network: fwe0. fwe0: flags=8802 metric 0 mtu 1500 options=8 ether 02:dc:10:62:ad:eb nd6 options=29 ch 1 dma -1 Starting Network: fwip0. fwip0: flags=8802 metric 0 mtu 1500 lladdr 0.dc.10.0.1.62.ad.eb.a.2.ff.fe.0.0.0.0 nd6 options=29 add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/kde4/lib /usr/local/lib/compat /usr/local/lib/gcc46 /usr/local/lib/qt4 /usr/local/lib/virtualbox a.out ldconfig path: /usr/lib/aout /usr/lib/compat/aout Creating and/or trimming log files. Starting syslogd. No core dumps found. Additional ABI support: linux. Starting rpcbind. NFS access cache time=60 lock order reversal: 1st 0xe191e1c0 bufwait (bufwait) @ kern/vfs_bio.c:2652 2nd 0xca7e0c00 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:284 KDB: stack backtrace: db_trace_self_wrapper(c0f78117,2e687361,38323a63,c0000a34,c12569a0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0a7c8eb,c0f7bbe0,c1256990,11c,f27c9874,...) at kdb_backtrace+0x2a _witness_debugger(c0f7bbe0,ca7e0c00,c0fac31a,c7d72610,c0fabfac,...) at _witness_debugger+0x25 witness_checkorder(ca7e0c00,9,c0fabfa3,11c,0,...) at witness_checkorder+0x86f _sx_xlock(ca7e0c00,0,c0fabfa3,11c,ca838cb0,...) at _sx_xlock+0xa0 ufsdirhash_acquire(e191e160,ca838cb0,f27c99f0,e23ef7cc,f27c9944,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(ca838cb0,f27c99f0,37cc,f27c9930,f27c9934,...) at ufsdirhash_add+0x13 ufs_direnter(caaab6cc,cad14000,f27c99f0,f27c9b80,e191e7e0,...) at ufs_direnter+0x7a9 ufs_mkdir(f27c9c10,c0fcd7b2,0,0,f27c9ba8,...) at ufs_mkdir+0xa6f VOP_MKDIR_APV(c10be200,f27c9c10,f27c9b80,f27c9ba8,0,...) at VOP_MKDIR_APV+0xc5 kern_mkdirat(ca6cb000,ffffff9c,28404020,0,1c0,...) at kern_mkdirat+0x291 kern_mkdir(ca6cb000,28404020,0,1c0,f27c9d1c,...) at kern_mkdir+0x2e sys_mkdir(ca6cb000,f27c9cec,c0fc61fc,c0f7c8f2,202,...) at sys_mkdir+0x29 syscall(f27c9d28) at syscall+0x2c7 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (136, FreeBSD ELF32, sys_mkdir), eip = 0x2817fb33, esp = 0xbfbfe8cc, ebp = 0xbfbfed78 --- Clearing /tmp (X related). Starting mountd. Starting nfsuserd. Starting nfsd. Recovering vi editor sessions:lock order reversal: 1st 0xcae4a1b4 ufs (ufs) @ kern/vfs_lookup.c:542 2nd 0xe1924340 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:260 3rd 0xcae49724 ufs (ufs) @ kern/vfs_subr.c:2166 KDB: stack backtrace: db_trace_self_wrapper(c0f78117,3a632e73,a303632,0,ffffffc0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0a7c8eb,c0f7bbf9,c12564f8,876,f2824354,...) at kdb_backtrace+0x2a _witness_debugger(c0f7bbf9,cae49724,c0f6a2e4,c7d725a8,c0f83c2f,...) at _witness_debugger+0x25 witness_checkorder(cae49724,9,c0f83c26,876,0,...) at witness_checkorder+0x86f __lockmgr_args(cae49724,80100,cae49790,0,0,...) at __lockmgr_args+0x8c5 ffs_lock(f2824450,c0a8ee6c,ca7cc000,80100,cae496cc,...) at ffs_lock+0xa1 VOP_LOCK1_APV(c10be200,f2824450,c7d6b1e0,c10cf3e0,cae496cc,...) at VOP_LOCK1_APV+0xb5 _vn_lock(cae496cc,80100,c0f83c26,876,4,...) at _vn_lock+0x78 vget(cae496cc,80100,ca7cc000,50,0,...) at vget+0xbb vfs_hash_get(ca7642d0,15eebd,80000,ca7cc000,f282459c,...) at vfs_hash_get+0xed ffs_vgetf(ca7642d0,15eebd,80000,f282459c,1,...) at ffs_vgetf+0x49 softdep_sync_buf(cae4a15c,e19242e0,1,106,0,...) at softdep_sync_buf+0xac9 ffs_syncvnode(cae4a15c,1,0,f282462c,c0a8cd47,...) at ffs_syncvnode+0x24c ffs_truncate(cae4a15c,200,0,880,ca78c780,...) at ffs_truncate+0x8b5 ufs_direnter(cae4a15c,cae496cc,f2824914,f2824bbc,0,...) at ufs_direnter+0x9a1 ufs_makeinode(f2824bbc,c10be7c0,f2824b00,f2824a5c,c0dd1765,...) at ufs_makeinode+0x65d ufs_create(f2824b00,c0fcd978,0,0,f2824b7c,...) at ufs_create+0x30 VOP_CREATE_APV(c10be200,f2824b00,f2824bbc,f2824a98,0,...) at VOP_CREATE_APV+0xc5 vn_open_cred(f2824b7c,f2824c44,1b0,0,ca78c780,...) at vn_open_cred+0x205 vn_open(f2824b7c,f2824c44,1b0,ca7a6620,4f8bdb91,...) at vn_open+0x3b kern_openat(ca7cc000,ffffff9c,28839f60,0,a03,...) at kern_openat+0x1ec kern_open(ca7cc000,28839f60,0,a02,1b0,...) at kern_open+0x35 sys_open(ca7cc000,f2824cec,c0fc61fc,c0f7c8f2,206,...) at sys_open+0x30 syscall(f2824d28) at syscall+0x2c7 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (5, FreeBSD ELF32, sys_open), eip = 0x283ec193, esp = 0xbfbfc50c, ebp = 0xbfbfc598 --- . Updating motd:. Starting ntpd. Starting default moused. Configuring syscons: keymap blanktime. Starting sshd. Starting cron. Local package initialization: backuplock order reversal: 1st 0xca860724 ufs (ufs) @ kern/vfs_mount.c:1250 2nd 0xca861058 devfs (devfs) @ ufs/ffs/ffs_softdep.c:1835 KDB: stack backtrace: db_trace_self_wrapper(c0f78117,38313a63,a3533,c7d6b7f8,ca7d1c30,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0a7c8eb,c0f7bbe0,c12560c8,72b,f27249b8,...) at kdb_backtrace+0x2a _witness_debugger(c0f7bbe0,ca861058,c0f6f254,c7d72408,c0fa8415,...) at _witness_debugger+0x25 witness_checkorder(ca861058,9,c0fa840c,72b,ca8610c4,...) at witness_checkorder+0x86f __lockmgr_args(ca861058,80400,ca8610c4,0,0,...) at __lockmgr_args+0x8c5 vop_stdlock(f2724ab0,4,c0f7235b,80400,ca861000,...) at vop_stdlock+0x65 VOP_LOCK1_APV(c1093a40,f2724ab0,c1289ec0,c10cf3e0,ca861000,...) at VOP_LOCK1_APV+0xb5 _vn_lock(ca861000,80400,c0fa840c,72b,0,...) at _vn_lock+0x78 softdep_flushworklist(ca764870,f2724b68,ca7d1b80,606,f2724b38,...) at softdep_flushworklist+0x47 ffs_sync(ca764870,1,f2724bbc,51b,c7c34000,...) at ffs_sync+0x35f dounmount(ca764870,8080000,ca7d1b80,49d,f2724c04,...) at dounmount+0x3f8 sys_unmount(ca7d1b80,f2724cec,c0fc61fc,c,206,...) at sys_unmount+0x421 syscall(f2724d28) at syscall+0x2c7 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280e7f9b, esp = 0xbfbfe64c, ebp = 0xbfbfe718 --- ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 74994 files, 6554084 used, 111759897 free (2089 frags, 13969726 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** ** /dev/label/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 74994 files, 6554084 used, 111759897 free (2089 frags, 13969726 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** fsck -y /tmp watchdogd. Starting inetd. Mon Apr 16 10:47:37 CEST 2012 FreeBSD/i386 (x4.osted.lan) (console) login: Apr 16 10:54:14 x4 su: pho to root on /dev/pts/0 Apr 16 10:54:19 x4 mountd[1009]: can't delete exports for /mnt17: Invalid argument Apr 16 10:54:20 x4 mountd[1009]: can't delete exports for /mnt10: Invalid argument Apr 16 10:54:20 x4 mountd[1009]: can't delete exports for /mnt9: Invalid argument Apr 16 10:54:20 x4 mountd[1009]: can't delete exports for /mnt5: Invalid argument Apr 16 10:54:20 x4 mountd[1009]: can't delete exports for /mnt19: Invalid argument : Apr 16 11:33:34 x4 mountd[1009]: can't delete exports for /mnt7: Invalid argument Apr 16 11:33:34 x4 mountd[1009]: can't delete exports for /mnt6: Invalid argument Apr 16 11:33:35 x4 mountd[1009]: can't delete exports for /mnt8: Invalid argument Apr 16 11:33:49 x4 mountd[1009]: can't delete exports for /mnt14: Invalid argument Apr 16 11:33:49 x4 mountd[1009]: can't delete exports for /mnt10: Invalid argument Apr 16 11:34:09 x4 mountd[1009]: can't delete exports for /mnt10: Invalid argument Apr 16 11:34:09 x4 mountd[1009]: can't delete exports for /mnt14: Invalid argument Apr 16 11:34:09 x4 mountd[1009]: can't delete exports for /mnt15: Invalid argument Apr 16 11:34:09 x4 mountd[1009]: can't delete exports for /mnt12: Invalid argument Apr 16 11:34:09 x4 mountd[1009]: can't delete exports for /mnt16: Invalid argument Apr 16 11:34:22 x4 mountd[1009]: can't delete exports for /mnt16: Device busy Apr 16 11:34:26 x4 mountd[1009]: can't delete exports for /mnt14: Invalid argument fsync: giving up on dirty 0xcb8b56cc: tag devfs, type VCHR usecount 1, writecount 0, refcount 34 mountedhere 0xcaa9b700 flags () v_object 0xcb794250 ref 0 pages 294 lock type devfs: EXCL by thread 0xcbdcf5c0 (pid 80000, umount, tid 100521) #0 0xc0a2921e at __lockmgr_args+0xd6e #1 0xc0ace995 at vop_stdlock+0x65 #2 0xc0dd0795 at VOP_LOCK1_APV+0xb5 #3 0xc0af0f88 at _vn_lock+0x78 #4 0xc0c84387 at softdep_flushworklist+0x47 #5 0xc0c8933f at ffs_sync+0x35f #6 0xc0ad97f8 at dounmount+0x3f8 #7 0xc0ad9ee1 at sys_unmount+0x421 #8 0xc0dab957 at syscall+0x2c7 #9 0xc0d95d41 at Xint0x80_syscall+0x21 dev md5a fsync: giving up on dirty 0xcb8b56cc: tag devfs, type VCHR usecount 1, writecount 0, refcount 34 mountedhere 0xcaa9b700 flags () v_object 0xcb794250 ref 0 pages 294 lock type devfs: EXCL by thread 0xcbdcf5c0 (pid 80000, umount, tid 100521) #0 0xc0a2921e at __lockmgr_args+0xd6e #1 0xc0ace995 at vop_stdlock+0x65 #2 0xc0dd0795 at VOP_LOCK1_APV+0xb5 #3 0xc0af0f88 at _vn_lock+0x78 #4 0xc0c893e7 at ffs_sync+0x407 #5 0xc0ad97f8 at dounmount+0x3f8 #6 0xc0ad9ee1 at sys_unmount+0x421 #7 0xc0dab957 at syscall+0x2c7 #8 0xc0d95d41 at Xint0x80_syscall+0x21 dev md5a lock order reversal: 1st 0xcad12df0 ufs (ufs) @ kern/vfs_mount.c:860 2nd 0xcb1179dc newnfs (newnfs) @ kern/vfs_subr.c:2166 KDB: stack backtrace: db_trace_self_wrapper(c0f78117,a3636,0,c0fad9c4,246,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0a7c8eb,c0f7bbe0,c12548c8,876,f2bf0848,...) at kdb_backtrace+0x2a _witness_debugger(c0f7bbe0,cb1179dc,c0f6492b,c7d72e30,c0f83c2f,...) at _witness_debugger+0x25 witness_checkorder(cb1179dc,9,c0f83c26,876,0,...) at witness_checkorder+0x86f __lockmgr_args(cb1179dc,80100,cb117a48,0,0,...) at __lockmgr_args+0x8c5 nfs_lock1(f2bf0940,c1bb7c88,4,80100,cb117984,...) at nfs_lock1+0xcd VOP_LOCK1_APV(c1094b60,f2bf0940,c7d6b1e0,c10cf3e0,cb117984,...) at VOP_LOCK1_APV+0xb5 _vn_lock(cb117984,80100,c0f83c26,876,4,...) at _vn_lock+0x78 vget(cb117984,80100,cb3985c0,50,0,...) at vget+0xbb vfs_hash_get(cb8c2b40,7cee4531,80000,cb3985c0,f2bf0a00,...) at vfs_hash_get+0xed ncl_nget(cb8c2b40,cb77328c,1c,f2bf0a2c,80000,...) at ncl_nget+0xcf nfs_root(cb8c2b40,80000,f2bf0c14,365,c0fae9ff,...) at nfs_root+0x42 vfs_donmount(cb3985c0,0,0,cc575100,cc575100,...) at vfs_donmount+0xe99 sys_nmount(cb3985c0,f2bf0cec,c0fc61fc,c0f7ce51,206,...) at sys_nmount+0x7f syscall(f2bf0d28) at syscall+0x2c7 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (378, FreeBSD ELF32, sys_nmount), eip = 0x280e9bbb, esp = 0xbfbfe00c, ebp = 0xbfbfe808 --- Apr 16 11:36:37 x4 mountd[1009]: bad exports list line /mnt lock order reversal: 1st 0xe18e8fe0 bufwait (bufwait) @ vm/vm_pager.c:309 2nd 0xcadd5600 so_snd_sx (so_snd_sx) @ kern/uipc_sockbuf.c:148 KDB: stack backtrace: db_trace_self_wrapper(c0f78117,75626b63,3a632e66,a383431,f2d0d400,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0a7c8eb,c0f7bbe0,c1255c08,94,f2d0d538,...) at kdb_backtrace+0x2a _witness_debugger(c0f7bbe0,cadd5600,c0f6c0c2,c7d72748,c0f800de,...) at _witness_debugger+0x25 witness_checkorder(cadd5600,9,c0f800d5,94,0,...) at witness_checkorder+0x86f _sx_xlock(cadd5600,40,c0f800d5,94,f2d0d5d0,...) at _sx_xlock+0xa0 sblock(cadd55c8,1,202,2,c1ba68d4,...) at sblock+0x85 sosend_generic(cadd54e0,0,0,ca5d6a00,0,...) at sosend_generic+0xc2 sosend(cadd54e0,0,0,ca5d6a00,0,...) at sosend+0x3f clnt_vc_call(ca6f3900,f2d0d754,6,cabb9300,f2d0d868,...) at clnt_vc_call+0x4af clnt_reconnect_call(ca6fcb40,f2d0d754,6,cabb9300,f2d0d868,...) at clnt_reconnect_call+0x670 newnfs_request(f2d0d868,cb773200,0,cb773310,cb2a62b8,cbd9e2e0,cbc97780,186a3,3,0,1,0) at newnfs_request+0xc65 nfscl_request(f2d0d868,cb2a62b8,cbd9e2e0,cbc97780,0,...) at nfscl_request+0x82 nfsrpc_read(cb2a62b8,f2d0d9fc,cbc97780,cbd9e2e0,f2d0d924,...) at nfsrpc_read+0x4e9 ncl_readrpc(cb2a62b8,f2d0d9fc,cbc97780,94,4,...) at ncl_readrpc+0x4b ncl_getpages(f2d0da68,c0fccccf,cb2a62b8,cb2a62b8,cc41a378,...) at ncl_getpages+0x389 VOP_GETPAGES_APV(c1094b60,f2d0da68,c0fb1a5a,2a3,c10cf540,...) at VOP_GETPAGES_APV+0xd6 vnode_pager_getpages(cc41a378,f2d0db60,9,0,f2d0dbe4,...) at vnode_pager_getpages+0xb4 vm_fault_hold(cba1fbc8,281d0000,2,0,0,...) at vm_fault_hold+0xfff vm_fault(cba1fbc8,281d0000,2,0,281d0000,...) at vm_fault+0x8b trap_pfault(c110b790,0,c0fc6152,304,cbd59b30,...) at trap_pfault+0x1b9 trap(f2d0dd28) at trap+0x299 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x8048793, esp = 0xbfbfe830, ebp = 0xbfbfe868 --- Apr 16 11:36:38 x4 mountd[1009]: bad exports list line /mnt Stopping inetd. Waiting for PIDS: 1223. Shutting down local packages:. Stopping cron. Waiting for PIDS: 1153. Stopping sshd. Waiting for PIDS: 1142. Stopping moused. Waiting for PIDS: 1111. Stopping ntpd. Waiting for PIDS: 1091. Stopping nfsd. Waiting for PIDS: 1031 1036. Stopping mountd. Waiting for PIDS: 1009. Stopping nfsuserd. Waiting for PIDS: 1025 1026 1027 1028 1029. rpc.umntall: 127.0.0.1: MOUNTPROG: RPC: Program not registered rpc.umntall: 127.0.0.1: MOUNTPROG: RPC: Program not registered Stopping rpcbind. Waiting for PIDS: 976. Stopping watchdogd. Waiting for PIDS: 1198. Stopping devd. Waiting for PIDS: 838. Writing entropy file:. Terminated . Apr 16 12:09:08 x4 syslogd: exiting on signal 15 Apr 16 12:09:28 init: some processes would not die; ps axl advised Enter full pathname of shell or RETURN for /bin/sh: # ps -l UID PID PPID CPU PRI NI VSZ RSS MWCHAN STAT TT TIME COMMAND 0 25224 1 0 20 0 9984 1592 wait Ss u0 0:00.01 -sh (sh) 0 25225 25224 0 20 0 9752 1104 - R+ u0 0:00.00 ps -l 0 86452 1 0 20 0 9596 896 mntref D 0- 0:00.01 umount /mnt # kill -9 86452 # ps -l UID PID PPID CPU PRI NI VSZ RSS MWCHAN STAT TT TIME COMMAND 0 25224 1 0 20 0 9984 1592 wait Ss u0 0:00.01 -sh (sh) 0 25226 25224 0 20 0 9752 1104 - R+ u0 0:00.00 ps -l 0 86452 1 0 20 0 9596 896 mntref D 0- 0:00.01 umount /mnt # ~KDB: enter: Break to debugger [ thread pid 11 tid 100005 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 11 tid 100005 td 0xc7dcd2e0 kdb_enter(c0def829,c0f780be,c7b2cb9c,c08e968d,c816be00,...) at kdb_enter+0x3a kdb_break(c816be00,0,c7b2cbc4,1356576,c0d91635,...) at kdb_break+0x28 uart_intr(c816be00,c12b89b8,c7b2cbdc,c0dc64ee,0,...) at uart_intr+0x12d intr_event_handle(c7dafc00,c7b2cc08,c12eac40,0,c7ff5a00,...) at intr_event_handle+0x76 intr_execute_handlers(c7e130d0,c7b2cc08,2,c7b2cc48,c0d960a5,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,c7b2cc08) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0d91635, esp = 0xc7b2cc48, ebp = 0xc7b2cc48 --- acpi_cpu_c1(c7b2cc64,c10d0fb0,2,317,1412,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b2cc98,c0d9e849,0,2,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,2,0,2,c7b2cce8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b2ccc0,c0f76182,a1f,c7dcd2e0,...) at cpu_idle+0x89 sched_idletd(0,c7b2cd28,c0f6e390,3d8,c7dcb864,...) at sched_idletd+0x235 fork_exit(c0a696f0,0,c7b2cd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b2cd60, ebp = 0 --- db:0:bt> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x755380 curthread = 0xc7dcd8a0: pid 11 "idle: cpu0" curpcb = 0xc7b26d80 fpcurthread = none idlethread = 0xc7dcd8a0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x6a46380 curthread = 0xc7dcd5c0: pid 11 "idle: cpu1" curpcb = 0xc7b29d80 fpcurthread = none idlethread = 0xc7dcd5c0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x6a49380 curthread = 0xc7dcd2e0: pid 11 "idle: cpu2" curpcb = 0xc7b2cd80 fpcurthread = none idlethread = 0xc7dcd2e0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x6a4c380 curthread = 0xc7dcd000: pid 11 "idle: cpu3" curpcb = 0xc7b2fd80 fpcurthread = none idlethread = 0xc7dcd000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks db:0:alllocks> show lockedvnods Locked vnodes db:0:lockedvnods> show mount 0xca764b40 /dev/ad4s1a on / (ufs) 0xca765000 devfs on /dev (devfs) 0xcaab15a0 /dev/ad4s1f on /home (ufs) 0xca7645a0 /dev/ad4s1d on /usr (ufs) 0xca7642d0 /dev/ad4s1e on /var (ufs) 0xca764000 procfs on /proc (procfs) 0xcbf385a0 /dev/label/tmp on /tmp (ufs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 25224 1 25224 0 Ss+ ttyin 0xc7ef5c70 sh 86452 1 68130 0 D+ mntref 0xcb8c2b40 umount 19 0 0 0 DL sdflush 0xc1289f00 [softdepflush] 18 0 0 0 DL vlruwt 0xca6c12cc [vnlru] 17 0 0 0 DL syncer 0xc1283bf4 [syncer] 16 0 0 0 DL psleep 0xc1283888 [bufdaemon] 9 0 0 0 DL pgzero 0xc128bddc [pagezero] 8 0 0 0 DL psleep 0xc128b9c8 [vmdaemon] 7 0 0 0 DL psleep 0xc128b990 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc10d09d4 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc1285718 [sctp_iterator] 4 0 0 0 DL ctl_work 0xc818f000 [ctl_thrd] 3 0 0 0 DL - 0xc7ff523c [fdc0] 2 0 0 0 SL - 0xc8086000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100058 D - 0xc805cd34 [usbus5] 100057 D - 0xc805cd04 [usbus5] 100056 D - 0xc805ccd4 [usbus5] 100055 D - 0xc805cca4 [usbus5] 100053 D - 0xc804db5c [usbus4] 100052 D - 0xc804db2c [usbus4] 100051 D - 0xc804dafc [usbus4] 100050 D - 0xc804dacc [usbus4] 100049 D - 0xc8048b5c [usbus3] 100048 D - 0xc8048b2c [usbus3] 100047 D - 0xc8048afc [usbus3] 100046 D - 0xc8048acc [usbus3] 100045 D - 0xc8041b5c [usbus2] 100044 D - 0xc8041b2c [usbus2] 100043 D - 0xc8041afc [usbus2] 100042 D - 0xc8041acc [usbus2] 100040 D - 0xc803ab5c [usbus1] 100039 D - 0xc803ab2c [usbus1] 100038 D - 0xc803aafc [usbus1] 100037 D - 0xc803aacc [usbus1] 100035 D - 0xc802fb5c [usbus0] 100034 D - 0xc802fb2c [usbus0] 100033 D - 0xc802fafc [usbus0] 100032 D - 0xc802facc [usbus0] 14 0 0 0 DL - 0xc110c824 [yarrow] 13 0 0 0 DL (threaded) [geom] 100015 D - 0xc1109a68 [g_down] 100014 D - 0xc1109a64 [g_up] 100013 D - 0xc1109a5c [g_event] 12 0 0 0 WL (threaded) [intr] 100066 I [irq12: psm0] 100065 I [irq1: atkbd0] 100063 I [swi0: uart] 100060 I [irq20: fwohci0] 100059 I [irq14: ata0] 100054 I [irq19: ehci0] 100041 I [irq18: ohci2 ohci4] 100036 I [irq17: ohci1 ohci3] 100031 I [irq16: hdac1 ohci0] 100030 I [irq22: ahci0] 100029 I [irq257: re0] 100028 I [irq256: hdac0] 100027 I [swi6: task queue] 100026 I [swi6: Giant taskq] 100024 I [swi5: +] 100021 I [swi2: cambio] 100012 I [swi3: vm] 100011 I [swi1: netisr 0] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi4: clock] 100007 I [swi4: clock] 11 0 0 0 RL (threaded) [idle] 100006 Run CPU 3 [idle: cpu3] 100005 Run CPU 2 [idle: cpu2] 100004 Run CPU 1 [idle: cpu1] 100003 Run CPU 0 [idle: cpu0] 1 0 1 0 SLs wait 0xc7dcbb30 [init] 10 0 0 0 DL audit_wo 0xc1289640 [audit] 0 0 0 0 DLs (threaded) [kernel] 100069 D - 0xc110c824 [deadlkres] 100068 D - 0xc8187dc0 [mca taskq] 100061 D - 0xc8084540 [fw0_taskq] 100025 D - 0xc7f5e740 [thread taskq] 100023 D - 0xc7f5e7c0 [ffs_trim taskq] 100022 D - 0xc7f5e800 [kqueue taskq] 100020 D - 0xc7f5e840 [acpi_task_2] 100019 D - 0xc7f5e840 [acpi_task_1] 100018 D - 0xc7f5e840 [acpi_task_0] 100016 D - 0xc7d9fc00 [firmware taskq] 100000 D sched 0xc1109b80 [swapper] db:0:ps> allt Tracing command sh pid 25224 tid 100514 td 0xcb3152e0 sched_switch(cb3152e0,0,104,1a3,6e00dacd,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(cb3152e0,0,c0f79b00,1a7,0,...) at sleepq_switch+0x162 sleepq_catch_signals(c0a3121a,c7ef5c04,0,c0f7235b,cb3152e0,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c7ef5c70,0,f2d5eaec,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7ef5c70,c7ef5c04,c0f7e33c,527,0,...) at _cv_wait_sig+0x260 tty_wait(c7ef5c00,c7ef5c70,f2d5ec40,1,0,...) at tty_wait+0x71 ttydisc_read(c7ef5c00,f2d5ec40,0,a1,0,...) at ttydisc_read+0x22c ttydev_read(c7ed7c00,f2d5ec40,0,0,1,...) at ttydev_read+0xaa devfs_read_f(ca7e9150,f2d5ec40,c7dafe00,0,cb3152e0,...) at devfs_read_f+0x7e dofileread(f2d5ec40,ffffffff,ffffffff,0,f2d5ec1c,...) at dofileread+0x9e kern_readv(cb3152e0,0,f2d5ec40,f2d5ec60,1,...) at kern_readv+0x68 sys_read(cb3152e0,f2d5ecec,c0fc61fc,c0ddfd72,212,...) at sys_read+0x4f syscall(f2d5ed28) at syscall+0x2c7 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x282096e3, esp = 0xbfbfecac, ebp = 0xbfbfecd8 --- Tracing command umount pid 86452 tid 100296 td 0xcb2502e0 sched_switch(cb2502e0,0,104,1a3,4bda0f5,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,60,...) at mi_switch+0x1ff sleepq_switch(cb2502e0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(cb8c2b40,60,c0f83364,0,0,...) at sleepq_wait+0x63 _sleep(cb8c2b40,cb8c2b40,60,c0f83364,0,...) at _sleep+0x39c vfs_mount_destroy(cb8c2b40,10,0,559,c7c34000,...) at vfs_mount_destroy+0x78 dounmount(cb8c2b40,8000000,cb2502e0,49d,f2a86c04,...) at dounmount+0x6ae sys_unmount(cb2502e0,f2a86cec,c0fc61fc,c,206,...) at sys_unmount+0x421 syscall(f2a86d28) at syscall+0x2c7 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (22, FreeBSD ELF32, sys_unmount), eip = 0x280e7f9b, esp = 0xbfbfe0bc, ebp = 0xbfbfe188 --- Tracing command softdepflush pid 19 tid 100078 td 0xca6cd8a0 sched_switch(ca6cd8a0,0,104,1a3,8e80da94,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,54,...) at mi_switch+0x1ff sleepq_switch(ca6cd8a0,0,c0f79b00,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1289f00,54,c0faa951,0,0,...) at sleepq_timedwait+0x6b _sleep(c1289f00,c1289ec0,54,c0faa951,3e8,...) at _sleep+0x36a softdep_flush(0,ee324d28,c0f6e390,3d8,ca6c1000,...) at softdep_flush+0x266 fork_exit(c0c84550,0,ee324d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee324d60, ebp = 0 --- Tracing command vnlru pid 18 tid 100077 td 0xca6cdb80 sched_switch(ca6cdb80,0,104,1a3,5bc10d88,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,60,...) at mi_switch+0x1ff sleepq_switch(ca6cdb80,0,c0f79b00,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(ca6c12cc,60,c0f84b8e,0,0,...) at sleepq_timedwait+0x6b _sleep(ca6c12cc,c1283bb4,260,c0f84b8e,3e8,...) at _sleep+0x36a vnlru_proc(0,ee321d28,c0f6e390,3d8,ca6c12cc,...) at vnlru_proc+0xe7 fork_exit(c0ae4be0,0,ee321d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee321d60, ebp = 0 --- Tracing command syncer pid 17 tid 100076 td 0xc80568a0 sched_switch(c80568a0,0,104,1a3,8df85766,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80568a0,0,c0f79b00,28b,c80568a0,...) at sleepq_switch+0x162 sleepq_timedwait(c1283bf4,0,ee31ec78,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c1283bf4,c1283be0,3e8,78a,4e20,...) at _cv_timedwait+0x270 sched_sync(0,ee31ed28,c0f6e390,3d8,ca6c1598,...) at sched_sync+0x595 fork_exit(c0ae3f50,0,ee31ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee31ed60, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xc8056b80 sched_switch(c8056b80,0,104,1a3,60e77fe7,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c8056b80,0,c0f79b00,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1283888,54,c0f81a72,0,0,...) at sleepq_timedwait+0x6b _sleep(c1283888,c1283890,54,c0f81a72,3e8,...) at _sleep+0x36a buf_daemon(0,ee31bd28,c0f6e390,3d8,ca6c1864,...) at buf_daemon+0x175 fork_exit(c0ac7f50,0,ee31bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee31bd60, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xc8180000 sched_switch(c8180000,0,104,1a3,62015120,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8180000,0,c0f79b00,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c128bddc,0,c0fb1a06,0,0,...) at sleepq_timedwait+0x6b _sleep(c128bddc,c128a880,0,c0fb1a06,493e0,...) at _sleep+0x36a vm_pagezero(0,ee318d28,c0f6e390,3d8,ca6c1b30,...) at vm_pagezero+0xdc fork_exit(c0ccaf30,0,ee318d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee318d60, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xc81802e0 sched_switch(c81802e0,0,104,1a3,19beb991,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,74,...) at mi_switch+0x1ff sleepq_switch(c81802e0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c128b9c8,74,c0f81a72,0,0,...) at sleepq_wait+0x63 _sleep(c128b9c8,c128b9cc,74,c0f81a72,0,...) at _sleep+0x39c vm_daemon(0,ee315d28,c0f6e390,3d8,ca6c2000,...) at vm_daemon+0x59 fork_exit(c0cc4850,0,ee315d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee315d60, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xc81805c0 sched_switch(c81805c0,0,104,1a3,660f204a,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c81805c0,0,c0f79b00,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c128b990,54,c0f81a72,0,0,...) at sleepq_timedwait+0x6b _sleep(c128b990,c128a880,54,c0f81a72,1388,...) at _sleep+0x36a vm_pageout(0,ee312d28,c0f6e390,3d8,c7dcc2cc,...) at vm_pageout+0x2c7 fork_exit(c0cc58d0,0,ee312d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee312d60, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xc81808a0 sched_switch(c81808a0,0,104,1a3,def21a28,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c81808a0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c10d09d4,5c,c0dd57c7,0,0,...) at sleepq_wait+0x63 _sleep(c10d09d4,c10d0a18,5c,c0dd57c7,0,...) at _sleep+0x39c xpt_scanner_thread(0,ee18fd28,c0f6e390,3d8,c7dcc598,...) at xpt_scanner_thread+0x47 fork_exit(c049e3f0,0,ee18fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee18fd60, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xc8180b80 sched_switch(c8180b80,0,104,1a3,35f09f74,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8180b80,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1285718,0,c0f9877f,0,0,...) at sleepq_wait+0x63 _sleep(c1285718,c12856e0,0,c0f9877f,0,...) at _sleep+0x39c sctp_iterator_thread(0,ee18cd28,c0f6e390,3d8,c7dcc864,...) at sctp_iterator_thread+0x5c fork_exit(c0b76e10,0,ee18cd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee18cd60, ebp = 0 --- Tracing command ctl_thrd pid 4 tid 100067 td 0xc81815c0 sched_switch(c81815c0,0,104,1a3,cca6cdde,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c81815c0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c818f000,5c,c0de0210,0,0,...) at sleepq_wait+0x63 _sleep(c818f000,c818f000,5c,c0de0210,0,...) at _sleep+0x39c ctl_work_thread(c818f000,ee183d28,c0f6e390,3d8,c7dccb30,...) at ctl_work_thread+0x22f6 fork_exit(c04c7990,c818f000,ee183d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee183d60, ebp = 0 --- Tracing command fdc0 pid 3 tid 100064 td 0xc8182000 sched_switch(c8182000,0,104,1a3,7d718c6d,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c8182000,0,c0f79b00,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c7ff523c,5c,c0f666b2,0,0,...) at sleepq_timedwait+0x6b _sleep(c7ff523c,c7ff52f0,5c,c0f666b2,3e8,...) at _sleep+0x36a fdc_thread(c7ff5200,ee17ad28,c0f6e390,3d8,c8038000,...) at fdc_thread+0x27d fork_exit(c0d4d1f0,c7ff5200,ee17ad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee17ad60, ebp = 0 --- Tracing command fw0_probe pid 2 tid 100062 td 0xc80475c0 sched_switch(c80475c0,0,104,1a3,def25619,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c80475c0,0,c0f79b00,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0f79b00,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c8086000,6c,c0f666b2,100,0,...) at sleepq_wait_sig+0x17 _sleep(c8086000,c808a488,16c,c0f666b2,0,...) at _sleep+0x385 fw_bus_probe_thread(c8086000,ee16ad28,c0f6e390,3d8,c80382cc,...) at fw_bus_probe_thread+0xa08 fork_exit(c06f7b80,c8086000,ee16ad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee16ad60, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xc80542e0 sched_switch(c80542e0,0,104,1a3,cca172e3,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80542e0,0,c0f79b00,268,c80542e0,...) at sleepq_switch+0x162 sleepq_wait(c805cd34,0,ee153cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c805cd34,c805ce4c,c0f57d31,74,c805cd3c,...) at _cv_wait+0x260 usb_process(c805cd2c,ee153d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c805cd2c,ee153d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee153d60, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xc80545c0 sched_switch(c80545c0,0,104,1a3,19e3d9e7,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80545c0,0,c0f79b00,268,c80545c0,...) at sleepq_switch+0x162 sleepq_wait(c805cd04,0,ee150cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c805cd04,c805ce4c,c0f57d31,74,c805cd0c,...) at _cv_wait+0x260 usb_process(c805ccfc,ee150d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c805ccfc,ee150d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee150d60, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xc80548a0 sched_switch(c80548a0,0,104,1a3,cc3ac97f,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80548a0,0,c0f79b00,268,c80548a0,...) at sleepq_switch+0x162 sleepq_wait(c805ccd4,0,ee14dcac,1,0,...) at sleepq_wait+0x63 _cv_wait(c805ccd4,c805ce4c,c0f57d31,74,c805ccdc,...) at _cv_wait+0x260 usb_process(c805cccc,ee14dd28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c805cccc,ee14dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee14dd60, ebp = 0 --- Tracing command usb pid 15 tid 100055 td 0xc8054b80 sched_switch(c8054b80,0,104,1a3,cc3ab8cf,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8054b80,0,c0f79b00,268,c8054b80,...) at sleepq_switch+0x162 sleepq_wait(c805cca4,0,ee14acac,1,0,...) at sleepq_wait+0x63 _cv_wait(c805cca4,c805ce4c,c0f57d31,74,c805ccac,...) at _cv_wait+0x260 usb_process(c805cc9c,ee14ad28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c805cc9c,ee14ad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee14ad60, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xc80562e0 sched_switch(c80562e0,0,104,1a3,cc3aa454,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80562e0,0,c0f79b00,268,c80562e0,...) at sleepq_switch+0x162 sleepq_wait(c804db5c,0,ee141cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c804db5c,c804dc74,c0f57d31,74,c804db64,...) at _cv_wait+0x260 usb_process(c804db54,ee141d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c804db54,ee141d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee141d60, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xc80565c0 sched_switch(c80565c0,0,104,1a3,19e2c466,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80565c0,0,c0f79b00,268,c80565c0,...) at sleepq_switch+0x162 sleepq_wait(c804db2c,0,ee13ecac,1,0,...) at sleepq_wait+0x63 _cv_wait(c804db2c,c804dc74,c0f57d31,74,c804db34,...) at _cv_wait+0x260 usb_process(c804db24,ee13ed28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c804db24,ee13ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee13ed60, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xc8045000 sched_switch(c8045000,0,104,1a3,cbd72276,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8045000,0,c0f79b00,268,c8045000,...) at sleepq_switch+0x162 sleepq_wait(c804dafc,0,ee13bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(c804dafc,c804dc74,c0f57d31,74,c804db04,...) at _cv_wait+0x260 usb_process(c804daf4,ee13bd28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c804daf4,ee13bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee13bd60, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xc80452e0 sched_switch(c80452e0,0,104,1a3,cbd71102,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80452e0,0,c0f79b00,268,c80452e0,...) at sleepq_switch+0x162 sleepq_wait(c804dacc,0,ee138cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c804dacc,c804dc74,c0f57d31,74,c804dad4,...) at _cv_wait+0x260 usb_process(c804dac4,ee138d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c804dac4,ee138d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee138d60, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xc80455c0 sched_switch(c80455c0,0,104,1a3,cbd6f92e,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80455c0,0,c0f79b00,268,c80455c0,...) at sleepq_switch+0x162 sleepq_wait(c8048b5c,0,ee134cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8048b5c,c8048c74,c0f57d31,74,c8048b64,...) at _cv_wait+0x260 usb_process(c8048b54,ee134d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8048b54,ee134d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee134d60, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xc80458a0 sched_switch(c80458a0,0,104,1a3,18ecccc1,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80458a0,0,c0f79b00,268,c80458a0,...) at sleepq_switch+0x162 sleepq_wait(c8048b2c,0,ee131cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8048b2c,c8048c74,c0f57d31,74,c8048b34,...) at _cv_wait+0x260 usb_process(c8048b24,ee131d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8048b24,ee131d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee131d60, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xc8045b80 sched_switch(c8045b80,0,104,1a3,c6158417,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8045b80,0,c0f79b00,268,c8045b80,...) at sleepq_switch+0x162 sleepq_wait(c8048afc,0,ee12ecac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8048afc,c8048c74,c0f57d31,74,c8048b04,...) at _cv_wait+0x260 usb_process(c8048af4,ee12ed28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8048af4,ee12ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee12ed60, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xc8046000 sched_switch(c8046000,0,104,1a3,c6157228,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8046000,0,c0f79b00,268,c8046000,...) at sleepq_switch+0x162 sleepq_wait(c8048acc,0,ee12bcac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8048acc,c8048c74,c0f57d31,74,c8048ad4,...) at _cv_wait+0x260 usb_process(c8048ac4,ee12bd28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8048ac4,ee12bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee12bd60, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xc80462e0 sched_switch(c80462e0,0,104,1a3,c6155d6b,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80462e0,0,c0f79b00,268,c80462e0,...) at sleepq_switch+0x162 sleepq_wait(c8041b5c,0,ee127cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8041b5c,c8041c74,c0f57d31,74,c8041b64,...) at _cv_wait+0x260 usb_process(c8041b54,ee127d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8041b54,ee127d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee127d60, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xc80465c0 sched_switch(c80465c0,0,104,1a3,18ec27e4,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80465c0,0,c0f79b00,268,c80465c0,...) at sleepq_switch+0x162 sleepq_wait(c8041b2c,0,ee124cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8041b2c,c8041c74,c0f57d31,74,c8041b34,...) at _cv_wait+0x260 usb_process(c8041b24,ee124d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8041b24,ee124d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee124d60, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xc80468a0 sched_switch(c80468a0,0,104,1a3,c5b29062,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80468a0,0,c0f79b00,268,c80468a0,...) at sleepq_switch+0x162 sleepq_wait(c8041afc,0,ee121cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8041afc,c8041c74,c0f57d31,74,c8041b04,...) at _cv_wait+0x260 usb_process(c8041af4,ee121d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8041af4,ee121d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee121d60, ebp = 0 --- Tracing command usb pid 15 tid 100042 td 0xc8046b80 sched_switch(c8046b80,0,104,1a3,c5b27b91,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8046b80,0,c0f79b00,268,c8046b80,...) at sleepq_switch+0x162 sleepq_wait(c8041acc,0,ee11ecac,1,0,...) at sleepq_wait+0x63 _cv_wait(c8041acc,c8041c74,c0f57d31,74,c8041ad4,...) at _cv_wait+0x260 usb_process(c8041ac4,ee11ed28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c8041ac4,ee11ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee11ed60, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xc8035000 sched_switch(c8035000,0,104,1a3,c5b262f2,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8035000,0,c0f79b00,268,c8035000,...) at sleepq_switch+0x162 sleepq_wait(c803ab5c,0,ee117cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c803ab5c,c803ac74,c0f57d31,74,c803ab64,...) at _cv_wait+0x260 usb_process(c803ab54,ee117d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c803ab54,ee117d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee117d60, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xc80352e0 sched_switch(c80352e0,0,104,1a3,18eb9a71,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80352e0,0,c0f79b00,268,c80352e0,...) at sleepq_switch+0x162 sleepq_wait(c803ab2c,0,ee114cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c803ab2c,c803ac74,c0f57d31,74,c803ab34,...) at _cv_wait+0x260 usb_process(c803ab24,ee114d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c803ab24,ee114d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee114d60, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xc80355c0 sched_switch(c80355c0,0,104,1a3,c54e1b57,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80355c0,0,c0f79b00,268,c80355c0,...) at sleepq_switch+0x162 sleepq_wait(c803aafc,0,ee111cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c803aafc,c803ac74,c0f57d31,74,c803ab04,...) at _cv_wait+0x260 usb_process(c803aaf4,ee111d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c803aaf4,ee111d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee111d60, ebp = 0 --- Tracing command usb pid 15 tid 100037 td 0xc80358a0 sched_switch(c80358a0,0,104,1a3,c54e093d,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80358a0,0,c0f79b00,268,c80358a0,...) at sleepq_switch+0x162 sleepq_wait(c803aacc,0,ee10ecac,1,0,...) at sleepq_wait+0x63 _cv_wait(c803aacc,c803ac74,c0f57d31,74,c803aad4,...) at _cv_wait+0x260 usb_process(c803aac4,ee10ed28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c803aac4,ee10ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee10ed60, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xc8037000 sched_switch(c8037000,0,104,1a3,c54df504,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8037000,0,c0f79b00,268,c8037000,...) at sleepq_switch+0x162 sleepq_wait(c802fb5c,0,ee107cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c802fb5c,c802fc74,c0f57d31,74,c802fb64,...) at _cv_wait+0x260 usb_process(c802fb54,ee107d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c802fb54,ee107d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee107d60, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xc80372e0 sched_switch(c80372e0,0,104,1a3,18eb0ecf,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80372e0,0,c0f79b00,268,c80372e0,...) at sleepq_switch+0x162 sleepq_wait(c802fb2c,0,ee104cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c802fb2c,c802fc74,c0f57d31,74,c802fb34,...) at _cv_wait+0x260 usb_process(c802fb24,ee104d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c802fb24,ee104d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee104d60, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xc80375c0 sched_switch(c80375c0,0,104,1a3,c4e9794d,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80375c0,0,c0f79b00,268,c80375c0,...) at sleepq_switch+0x162 sleepq_wait(c802fafc,0,ee101cac,1,0,...) at sleepq_wait+0x63 _cv_wait(c802fafc,c802fc74,c0f57d31,74,c802fb04,...) at _cv_wait+0x260 usb_process(c802faf4,ee101d28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c802faf4,ee101d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee101d60, ebp = 0 --- Tracing command usb pid 15 tid 100032 td 0xc80378a0 sched_switch(c80378a0,0,104,1a3,c4e9662c,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80378a0,0,c0f79b00,268,c80378a0,...) at sleepq_switch+0x162 sleepq_wait(c802facc,0,ee0fecac,1,0,...) at sleepq_wait+0x63 _cv_wait(c802facc,c802fc74,c0f57d31,74,c802fad4,...) at _cv_wait+0x260 usb_process(c802fac4,ee0fed28,c0f6e390,3d8,c8038598,...) at usb_process+0x1a3 fork_exit(c091d880,c802fac4,ee0fed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee0fed60, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xc7edd5c0 sched_switch(c7edd5c0,0,104,1a3,a22362a3,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7edd5c0,0,c0f79b00,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c110c824,0,c0f666b2,2,0,...) at sleepq_timedwait+0x6b _sleep(c110c824,0,0,c0f666b2,64) at _sleep+0x36a pause(c0f666b2,64,c0f33db0,111,0,...) at pause+0xa0 random_kthread(0,c7b52d28,c0f6e390,3d8,c7dcb000,...) at random_kthread+0x1ef fork_exit(c081b540,0,c7b52d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b52d60, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xc7eddb80 sched_switch(c7eddb80,0,104,1a3,a355792,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c7eddb80,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1109a68,5c,c0f666b2,0,0,...) at sleepq_wait+0x63 _sleep(c1109a68,c11099c8,25c,c0f666b2,0,...) at _sleep+0x39c g_io_schedule_down(c7eddb80,0,c0f68201,6c,c7b4bd14,...) at g_io_schedule_down+0x5b g_down_procbody(0,c7b4bd28,c0f6e390,3d8,c7dcb2cc,...) at g_down_procbody+0x9d fork_exit(c09d4410,0,c7b4bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b4bd60, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xc7edf000 sched_switch(c7edf000,0,104,1a3,a4848d6,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c7edf000,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1109a64,5c,c0f666b2,0,0,...) at sleepq_wait+0x63 _sleep(c1109a64,c11099e8,25c,c0f666b2,0,...) at _sleep+0x39c g_io_schedule_up(c7edf000,0,c0f68201,5f,c7b48d14,...) at g_io_schedule_up+0x134 g_up_procbody(0,c7b48d28,c0f6e390,3d8,c7dcb2cc,...) at g_up_procbody+0x9d fork_exit(c09d44b0,0,c7b48d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b48d60, ebp = 0 --- Tracing command geom pid 13 tid 100013 td 0xc7dce2e0 sched_switch(c7dce2e0,0,104,1a3,fb5ab71b,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,5c,...) at mi_switch+0x1ff sleepq_switch(c7dce2e0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c1109a5c,5c,c0f666b2,0,0,...) at sleepq_wait+0x63 _sleep(c1109a5c,c110998c,25c,c0f666b2,0,...) at _sleep+0x39c g_run_events(c111a680,0,c0f68201,79,c7b45d14,...) at g_run_events+0x54b g_event_procbody(0,c7b45d28,c0f6e390,3d8,c7dcb2cc,...) at g_event_procbody+0x90 fork_exit(c09d4550,0,c7b45d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b45d60, ebp = 0 --- Tracing command intr pid 12 tid 100066 td 0xc81818a0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100065 td 0xc8181b80 sched_switch(c8181b80,0,109,1a3,c4e4de6f,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c8181b80,...) at mi_switch+0x1ff ithread_loop(c7e14380,ee17dd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14380,ee17dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee17dd60, ebp = 0 --- Tracing command intr pid 12 tid 100063 td 0xc80472e0 sched_switch(c80472e0,0,109,1a3,6e009ad9,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c80472e0,...) at mi_switch+0x1ff ithread_loop(c7e14390,ee16dd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14390,ee16dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee16dd60, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xc8047b80 sched_switch(c8047b80,0,109,1a3,491fea4e,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c8047b80,...) at mi_switch+0x1ff ithread_loop(c7e143f0,ee161d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e143f0,ee161d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee161d60, ebp = 0 --- Tracing command intr pid 12 tid 100059 td 0xc8054000 sched_switch(c8054000,0,109,1a3,71b11aeb,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c8054000,...) at mi_switch+0x1ff ithread_loop(c7e14430,ee159d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14430,ee159d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee159d60, ebp = 0 --- Tracing command intr pid 12 tid 100054 td 0xc8056000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100041 td 0xc8047000 sched_switch(c8047000,0,109,1a3,e9e97176,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c8047000,...) at mi_switch+0x1ff ithread_loop(c7e144d0,ee11bd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e144d0,ee11bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee11bd60, ebp = 0 --- Tracing command intr pid 12 tid 100036 td 0xc8035b80 sched_switch(c8035b80,0,109,1a3,e470bbf9,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c8035b80,...) at mi_switch+0x1ff ithread_loop(c7e14500,ee10bd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14500,ee10bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee10bd60, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xc8037b80 sched_switch(c8037b80,0,109,1a3,303575e3,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c8037b80,...) at mi_switch+0x1ff ithread_loop(c7e14530,ee0fbd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14530,ee0fbd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee0fbd60, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc7edf2e0 sched_switch(c7edf2e0,0,109,1a3,a46db59,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7edf2e0,...) at mi_switch+0x1ff ithread_loop(c7e14600,edfa5d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14600,edfa5d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xedfa5d60, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc7edf5c0 sched_switch(c7edf5c0,0,109,1a3,d3085ce6,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7edf5c0,...) at mi_switch+0x1ff ithread_loop(c7e14610,edfa1d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14610,edfa1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xedfa1d60, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc7edf8a0 sched_switch(c7edf8a0,0,109,1a3,b719fd01,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7edf8a0,...) at mi_switch+0x1ff ithread_loop(c7e14660,c7ba1d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14660,c7ba1d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7ba1d60, ebp = 0 --- Tracing command intr pid 12 tid 100027 td 0xc7edfb80 sched_switch(c7edfb80,0,109,1a3,2906908e,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7edfb80,...) at mi_switch+0x1ff ithread_loop(c7e14680,c7b70d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14680,c7b70d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b70d60, ebp = 0 --- Tracing command intr pid 12 tid 100026 td 0xc7fc0000 sched_switch(c7fc0000,0,109,1a3,6af6e55c,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7fc0000,...) at mi_switch+0x1ff ithread_loop(c7e14690,c7b6dd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e14690,c7b6dd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b6dd60, ebp = 0 --- Tracing command intr pid 12 tid 100024 td 0xc7fc05c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100021 td 0xc7e158a0 sched_switch(c7e158a0,0,109,1a3,a477192,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7e158a0,...) at mi_switch+0x1ff ithread_loop(c7e146b0,c7b5ed28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e146b0,c7b5ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b5ed60, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc7dce5c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100011 td 0xc7dce8a0 sched_switch(c7dce8a0,0,109,1a3,411265c8,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7dce8a0,...) at mi_switch+0x1ff ithread_loop(c7e147b0,c7b3fd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e147b0,c7b3fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b3fd60, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc7dceb80 sched_switch(c7dceb80,0,109,1a3,a2232901,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7dceb80,...) at mi_switch+0x1ff ithread_loop(c7e147c0,c7b3cd28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e147c0,c7b3cd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b3cd60, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc7e15000 sched_switch(c7e15000,0,109,1a3,97688a1a,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7e15000,...) at mi_switch+0x1ff ithread_loop(c7e147d0,c7b39d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e147d0,c7b39d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b39d60, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc7e152e0 sched_switch(c7e152e0,0,109,1a3,8ddb8e25,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7e152e0,...) at mi_switch+0x1ff ithread_loop(c7e147e0,c7b36d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e147e0,c7b36d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b36d60, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc7e155c0 sched_switch(c7e155c0,0,109,1a3,a66dad38,...) at sched_switch+0x3fc mi_switch(109,0,c0f6e6b3,557,c7e155c0,...) at mi_switch+0x1ff ithread_loop(c7e147f0,c7b33d28,c0f6e390,3d8,c7dcb598,...) at ithread_loop+0x1fb fork_exit(c0a17630,c7e147f0,c7b33d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b33d60, ebp = 0 --- Tracing command idle pid 11 tid 100006 td 0xc7dcd000 cpustop_handler(3,c7b2fbfc,c0dabe16,c7b2fba0,c0dc64ee,...) at cpustop_handler+0x26 ipi_nmi_handler(c7b2fba0,c0dc64ee,19022a0,c0d91635,c7dcb864,...) at ipi_nmi_handler+0x35 trap(c7b2fc08) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0d91635, esp = 0xc7b2fc48, ebp = 0xc7b2fc48 --- acpi_cpu_c1(c7b2fc64,c10d0fb0,3,317,1412,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b2fc98,c0d9e849,0,3,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,3,0,2,c7b2fce8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b2fcc0,c0f76182,a1f,c7dcd000,...) at cpu_idle+0x89 sched_idletd(0,c7b2fd28,c0f6e390,3d8,c7dcb864,...) at sched_idletd+0x235 fork_exit(c0a696f0,0,c7b2fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b2fd60, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc7dcd2e0 kdb_enter(c0def829,c0f780be,c7b2cb9c,c08e968d,c816be00,...) at kdb_enter+0x3a kdb_break(c816be00,0,c7b2cbc4,1356576,c0d91635,...) at kdb_break+0x28 uart_intr(c816be00,c12b89b8,c7b2cbdc,c0dc64ee,0,...) at uart_intr+0x12d intr_event_handle(c7dafc00,c7b2cc08,c12eac40,0,c7ff5a00,...) at intr_event_handle+0x76 intr_execute_handlers(c7e130d0,c7b2cc08,2,c7b2cc48,c0d960a5,...) at intr_execute_handlers+0x49 lapic_handle_intr(30,c7b2cc08) at lapic_handle_intr+0x36 Xapic_isr1() at Xapic_isr1+0x35 --- interrupt, eip = 0xc0d91635, esp = 0xc7b2cc48, ebp = 0xc7b2cc48 --- acpi_cpu_c1(c7b2cc64,c10d0fb0,2,317,1412,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b2cc98,c0d9e849,0,2,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,2,0,2,c7b2cce8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b2ccc0,c0f76182,a1f,c7dcd2e0,...) at cpu_idle+0x89 sched_idletd(0,c7b2cd28,c0f6e390,3d8,c7dcb864,...) at sched_idletd+0x235 fork_exit(c0a696f0,0,c7b2cd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b2cd60, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc7dcd5c0 cpustop_handler(1,c7b29bfc,c0dabe16,c7b29ba0,c0dc64ee,...) at cpustop_handler+0x26 ipi_nmi_handler(c7b29ba0,c0dc64ee,c110dab0,c7dcd5c0,c7dcb864,...) at ipi_nmi_handler+0x35 trap(c7b29c08) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0d91635, esp = 0xc7b29c48, ebp = 0xc7b29c48 --- acpi_cpu_c1(c7b29c64,c10d0fb0,1,317,1412,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b29c98,c0d9e849,0,1,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,1,0,2,c7b29ce8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b29cc0,c0f76182,a1f,c7dcd5c0,...) at cpu_idle+0x89 sched_idletd(0,c7b29d28,c0f6e390,3d8,c7dcb864,...) at sched_idletd+0x235 fork_exit(c0a696f0,0,c7b29d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b29d60, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc7dcd8a0 cpustop_handler(0,c7b26bfc,c0dabe16,0,c7b26bc4,...) at cpustop_handler+0x26 ipi_nmi_handler(0,c7b26bc4,1d204ba,c0d91635,c7dcb864,...) at ipi_nmi_handler+0x35 trap(c7b26c08) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0d91635, esp = 0xc7b26c48, ebp = 0xc7b26c48 --- acpi_cpu_c1(c7b26c64,c10d0fb0,0,317,1412,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b26c98,c0d9e849,0,0,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,0,0,2,c7b26ce8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b26cc0,c0f76182,a1f,c7dcd8a0,...) at cpu_idle+0x89 sched_idletd(0,c7b26d28,c0f6e390,3d8,c7dcb864,...) at sched_idletd+0x235 fork_exit(c0a696f0,0,c7b26d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b26d60, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc7dcdb80 sched_switch(c7dcdb80,0,104,1a3,b95889e2,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,6c,...) at mi_switch+0x1ff sleepq_switch(c7dcdb80,0,c0f79b00,1a7,6c,...) at sleepq_switch+0x162 sleepq_catch_signals(c0f79b00,160,0,100,100,...) at sleepq_catch_signals+0xf9 sleepq_wait_sig(c7dcbb30,6c,c0f7ca45,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7dcbb30,c7dcbbb8,16c,c0f7ca45,0,...) at _sleep+0x385 kern_wait(c7dcdb80,ffffffff,c7b22c5c,2,0,...) at kern_wait+0x597 sys_wait4(c7dcdb80,c7b22cec,c0fc61fc,c,282,...) at sys_wait4+0x3b syscall(c7b22d28) at syscall+0x2c7 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x80601ff, esp = 0xbfbfe81c, ebp = 0xbfbfe838 --- Tracing command audit pid 10 tid 100001 td 0xc7dce000 sched_switch(c7dce000,0,104,1a3,c4e78946,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7dce000,0,c0f79b00,268,c7dce000,...) at sleepq_switch+0x162 sleepq_wait(c1289640,0,c7b1fc8c,1,0,...) at sleepq_wait+0x63 _cv_wait(c1289640,c1289624,c0fa53b7,194,0,...) at _cv_wait+0x260 audit_worker(0,c7b1fd28,c0f6e390,3d8,c7dcc000,...) at audit_worker+0x84 fork_exit(c0c45050,0,c7b1fd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b1fd60, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xc8181000 sched_switch(c8181000,0,104,1a3,852bbd03,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c8181000,0,c0f79b00,28b,2,...) at sleepq_switch+0x162 sleepq_timedwait(c110c824,0,c0f666b2,2,0,...) at sleepq_timedwait+0x6b _sleep(c110c824,0,0,c0f666b2,bb8) at _sleep+0x36a pause(c0f666b2,bb8,126,124,c1109c08,...) at pause+0xa0 deadlkres(0,ee189d28,c0f6e390,3d8,c1109b80,...) at deadlkres+0x324 fork_exit(c09f68a0,0,ee189d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee189d60, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xc81812e0 sched_switch(c81812e0,0,104,1a3,3010ec93,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c81812e0,0,c0f79b00,268,c81812e0,...) at sleepq_switch+0x162 sleepq_wait(c8187dc0,0,c0f74e21,c0f666b2,0,...) at sleepq_wait+0x63 msleep_spin(c8187dc0,c8187dd8,c0f666b2,0,c0f7235b,...) at msleep_spin+0x259 taskqueue_thread_loop(c12ead00,ee186d28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0x8f fork_exit(c0a86790,c12ead00,ee186d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee186d60, ebp = 0 --- Tracing command kernel pid 0 tid 100061 td 0xc80478a0 sched_switch(c80478a0,0,104,1a3,a2240ada,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c80478a0,0,c0f79b00,268,c80478a0,...) at sleepq_switch+0x162 sleepq_wait(c8084540,0,c0f74e21,c0f666b2,0,...) at sleepq_wait+0x63 msleep_spin(c8084540,c8084558,c0f666b2,0,c0f7235b,...) at msleep_spin+0x259 taskqueue_thread_loop(c808a49c,ee167d28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0x8f fork_exit(c0a86790,c808a49c,ee167d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee167d60, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc7fc02e0 sched_switch(c7fc02e0,0,104,1a3,696dae42,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7fc02e0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c7f5e740,0,c0f666b2,0,0,...) at sleepq_wait+0x63 _sleep(c7f5e740,c7f5e758,0,c0f666b2,0,...) at _sleep+0x39c taskqueue_thread_loop(c11293ec,c7b6ad28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0xb5 fork_exit(c0a86790,c11293ec,c7b6ad28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b6ad60, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc7fc08a0 sched_switch(c7fc08a0,0,104,1a3,def67e24,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7fc08a0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c7f5e7c0,0,c0f666b2,0,0,...) at sleepq_wait+0x63 _sleep(c7f5e7c0,c7f5e7d8,0,c0f666b2,0,...) at _sleep+0x39c taskqueue_thread_loop(c1289c24,c7b64d28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0xb5 fork_exit(c0a86790,c1289c24,c7b64d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b64d60, ebp = 0 --- Tracing command kernel pid 0 tid 100022 td 0xc7fc0b80 sched_switch(c7fc0b80,0,104,1a3,def66c27,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7fc0b80,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c7f5e800,0,c0f666b2,0,0,...) at sleepq_wait+0x63 _sleep(c7f5e800,c7f5e818,0,c0f666b2,0,...) at _sleep+0x39c taskqueue_thread_loop(c110a518,c7b61d28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0xb5 fork_exit(c0a86790,c110a518,c7b61d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b61d60, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc7e15b80 sched_switch(c7e15b80,0,104,1a3,def659b1,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7e15b80,0,c0f79b00,268,c7e15b80,...) at sleepq_switch+0x162 sleepq_wait(c7f5e840,0,c0f74e21,c0f666b2,0,...) at sleepq_wait+0x63 msleep_spin(c7f5e840,c7f5e858,c0f666b2,0,c0f7235b,...) at msleep_spin+0x259 taskqueue_thread_loop(c10d3c38,c7b5bd28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0x8f fork_exit(c0a86790,c10d3c38,c7b5bd28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b5bd60, ebp = 0 --- Tracing command kernel pid 0 tid 100019 td 0xc7edd000 sched_switch(c7edd000,0,104,1a3,def64aee,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7edd000,0,c0f79b00,268,c7edd000,...) at sleepq_switch+0x162 sleepq_wait(c7f5e840,0,c0f74e21,c0f666b2,0,...) at sleepq_wait+0x63 msleep_spin(c7f5e840,c7f5e858,c0f666b2,0,c0f7235b,...) at msleep_spin+0x259 taskqueue_thread_loop(c10d3c38,c7b58d28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0x8f fork_exit(c0a86790,c10d3c38,c7b58d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b58d60, ebp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xc7edd2e0 sched_switch(c7edd2e0,0,104,1a3,def63acd,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7edd2e0,0,c0f79b00,268,c7edd2e0,...) at sleepq_switch+0x162 sleepq_wait(c7f5e840,0,c0f74e21,c0f666b2,0,...) at sleepq_wait+0x63 msleep_spin(c7f5e840,c7f5e858,c0f666b2,0,c0f7235b,...) at msleep_spin+0x259 taskqueue_thread_loop(c10d3c38,c7b55d28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0x8f fork_exit(c0a86790,c10d3c38,c7b55d28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b55d60, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc7edd8a0 sched_switch(c7edd8a0,0,104,1a3,a2e05688,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,0,...) at mi_switch+0x1ff sleepq_switch(c7edd8a0,0,c0f79b00,268,0,...) at sleepq_switch+0x162 sleepq_wait(c7d9fc00,0,c0f666b2,0,0,...) at sleepq_wait+0x63 _sleep(c7d9fc00,c7d9fc18,0,c0f666b2,0,...) at _sleep+0x39c taskqueue_thread_loop(c1127e00,c7b4ed28,c0f6e390,3d8,c1109b80,...) at taskqueue_thread_loop+0xb5 fork_exit(c0a86790,c1127e00,c7b4ed28) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b4ed60, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc1109e50 sched_switch(c1109e50,0,104,1a3,fa0588b2,...) at sched_switch+0x3fc mi_switch(104,0,c0f79b00,1f3,54,...) at mi_switch+0x1ff sleepq_switch(c1109e50,0,c0f79b00,28b,0,...) at sleepq_switch+0x162 sleepq_timedwait(c1109b80,54,c0f76a55,0,0,...) at sleepq_timedwait+0x6b _sleep(c1109b80,0,54,c0f76a55,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x240 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 204 MB: 189 173 157 141 125 109 93 77 61 45 29 13 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 2 (kgdb) proc 86452 [Switching to thread 80 (Thread 100296)]#0 sched_switch (td=0xcb2502e0, newtd=0xc7dcd2e0, flags=0x104) at ../../../kern/sched_ule.c:1902 1902 cpuid = PCPU_GET(cpuid); (kgdb) bt #0 sched_switch (td=0xcb2502e0, newtd=0xc7dcd2e0, flags=0x104) at ../../../kern/sched_ule.c:1902 #1 0xc0a4b9bf in mi_switch (flags=0x104, newtd=0x0) at ../../../kern/kern_synch.c:468 #2 0xc0a83672 in sleepq_switch (wchan=0xcb8c2b40, pri=0x60) at ../../../kern/subr_sleepqueue.c:538 #3 0xc0a843a3 in sleepq_wait (wchan=0xcb8c2b40, pri=0x60) at ../../../kern/subr_sleepqueue.c:617 #4 0xc0a4c0cc in _sleep (ident=0xcb8c2b40, lock=0xcb8c2b40, priority=0x60, wmesg=0xc0f83364 "mntref", timo=0x0) at ../../../kern/kern_synch.c:234 #5 0xc0ad7238 in vfs_mount_destroy (mp=0xcb8c2b40) at ../../../kern/vfs_mount.c:499 #6 0xc0ad9aae in dounmount (mp=0xcb8c2b40, flags=0x8000000, td=0xcb2502e0) at ../../../kern/vfs_mount.c:1375 #7 0xc0ad9ee1 in sys_unmount (td=0xcb2502e0, uap=0xf2a86cec) at ../../../kern/vfs_mount.c:1225 #8 0xc0dab957 in syscall (frame=0xf2a86d28) at subr_syscall.c:135 #9 0xc0d95d41 in Xint0x80_syscall () at ../../../i386/i386/exception.s:267 #10 0x00000033 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) f 5 #5 0xc0ad7238 in vfs_mount_destroy (mp=0xcb8c2b40) at ../../../kern/vfs_mount.c:499 499 msleep(mp, MNT_MTX(mp), PVFS, "mntref", 0); (kgdb) p *mp $1 = {mnt_mtx = {lock_object = {lo_name = 0xc0f65e71 "struct mount mtx", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc7d6b450}, mtx_lock = 0x4}, mnt_gen = 0x144, mnt_list = { tqe_next = 0x0, tqe_prev = 0xcbf385b8}, mnt_op = 0xc10953c0, mnt_vfc = 0xc1095380, mnt_vnodecovered = 0xcad12d98, mnt_syncer = 0x0, mnt_ref = 0x1, mnt_nvnodelist = {tqh_first = 0xcc40b414, tqh_last = 0xcc40b428}, mnt_nvnodelistsize = 0x1, mnt_writeopcount = 0x0, mnt_kern_flag = 0x61000028, mnt_flag = 0x1, mnt_opt = 0xcb6b43e0, mnt_optnew = 0x0, mnt_maxsymlinklen = 0x0, mnt_stat = {f_version = 0x20030518, f_type = 0x3a, f_flags = 0x1, f_bsize = 0x200, f_iosize = 0x10000, f_blocks = 0x1c354ff4, f_bfree = 0x1aa51630, f_bavail = 0x186361b8, f_files = 0x1d2d3fe, f_ffree = 0x1d1af09, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x3a00ff02, 0x3a}}, f_charspare = '\0' , f_fstypename = "nfs", '\0' , f_mntfromname = "127.0.0.1:/tmp\000\220y\n \n \n \n", f_mntonname = "/mnt", '\0' }, mnt_cred = 0xcb6ca800, mnt_data = 0xcb773200, mnt_time = 0x0, mnt_iosize_max = 0x10000, mnt_export = 0x0, mnt_label = 0x0, mnt_hashseed = 0xd76e66b0, mnt_lockref = 0x0, mnt_secondary_writes = 0x0, mnt_secondary_accwrites = 0x0, mnt_susp_owner = 0x0, mnt_gjprovider = 0x0, mnt_explock = {lock_object = {lo_name = 0xc0f65e82 "explock", lo_flags = 0x5730000, lo_data = 0x0, lo_witness = 0xc7d71088}, lk_lock = 0x1, lk_exslpfail = 0x0, lk_timo = 0x0, lk_pri = 0x60, lk_stack = {depth = 0x6, pcs = {0xc0a2921e, 0xc0ad0287, 0xc0ad8cc3, 0xc0ad93df, 0xc0dab957, 0xc0d95d41, 0x0 }}}} (kgdb) $ svn diff -x -p /usr/src/sys Index: /usr/src/sys/nfsclient/nfs_vfsops.c =================================================================== --- /usr/src/sys/nfsclient/nfs_vfsops.c (revision 234335) +++ /usr/src/sys/nfsclient/nfs_vfsops.c (working copy) @@ -1457,19 +1457,15 @@ nfs_sync(struct mount *mp, int waitfor) * Force stale buffer cache information to be flushed. */ loop: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { /* XXX Racy bv_cnt check. */ if (VOP_ISLOCKED(vp) || vp->v_bufobj.bo_dirty.bv_cnt == 0 || waitfor == MNT_LAZY) { VI_UNLOCK(vp); - MNT_ILOCK(mp); continue; } if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td)) { - MNT_ILOCK(mp); - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } error = VOP_FSYNC(vp, waitfor, td); @@ -1477,10 +1473,7 @@ loop: allerror = error; VOP_UNLOCK(vp, 0); vrele(vp); - - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); return (allerror); } Index: /usr/src/sys/nfsclient/nfs_subs.c =================================================================== --- /usr/src/sys/nfsclient/nfs_subs.c (revision 234335) +++ /usr/src/sys/nfsclient/nfs_subs.c (working copy) @@ -866,16 +866,10 @@ nfs_clearcommit(struct mount *mp) struct bufobj *bo; MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, nvp) { + MNT_VNODE_FOREACH_ALL(vp, mp, nvp) { bo = &vp->v_bufobj; - VI_LOCK(vp); - if (vp->v_iflag & VI_DOOMED) { - VI_UNLOCK(vp); - continue; - } vholdl(vp); VI_UNLOCK(vp); - MNT_IUNLOCK(mp); BO_LOCK(bo); TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { if (!BUF_ISLOCKED(bp) && @@ -885,9 +879,7 @@ nfs_clearcommit(struct mount *mp) } BO_UNLOCK(bo); vdrop(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); } /* Index: /usr/src/sys/fs/nfsclient/nfs_clvfsops.c =================================================================== --- /usr/src/sys/fs/nfsclient/nfs_clvfsops.c (revision 234335) +++ /usr/src/sys/fs/nfsclient/nfs_clvfsops.c (working copy) @@ -1508,24 +1508,21 @@ nfs_sync(struct mount *mp, int waitfor) MNT_IUNLOCK(mp); return (EBADF); } + MNT_IUNLOCK(mp); /* * Force stale buffer cache information to be flushed. */ loop: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { /* XXX Racy bv_cnt check. */ if (NFSVOPISLOCKED(vp) || vp->v_bufobj.bo_dirty.bv_cnt == 0 || waitfor == MNT_LAZY) { VI_UNLOCK(vp); - MNT_ILOCK(mp); continue; } if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td)) { - MNT_ILOCK(mp); - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } error = VOP_FSYNC(vp, waitfor, td); @@ -1533,10 +1530,7 @@ loop: allerror = error; NFSVOPUNLOCK(vp, 0); vrele(vp); - - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); return (allerror); } Index: /usr/src/sys/fs/nfsclient/nfs_clsubs.c =================================================================== --- /usr/src/sys/fs/nfsclient/nfs_clsubs.c (revision 234335) +++ /usr/src/sys/fs/nfsclient/nfs_clsubs.c (working copy) @@ -367,17 +367,10 @@ ncl_clearcommit(struct mount *mp) struct buf *bp, *nbp; struct bufobj *bo; - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, nvp) { + MNT_VNODE_FOREACH_ALL(vp, mp, nvp) { bo = &vp->v_bufobj; - VI_LOCK(vp); - if (vp->v_iflag & VI_DOOMED) { - VI_UNLOCK(vp); - continue; - } vholdl(vp); VI_UNLOCK(vp); - MNT_IUNLOCK(mp); BO_LOCK(bo); TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { if (!BUF_ISLOCKED(bp) && @@ -387,9 +380,7 @@ ncl_clearcommit(struct mount *mp) } BO_UNLOCK(bo); vdrop(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); } /* Index: /usr/src/sys/fs/coda/coda_subr.c =================================================================== --- /usr/src/sys/fs/coda/coda_subr.c (revision 234335) +++ /usr/src/sys/fs/coda/coda_subr.c (working copy) @@ -365,13 +365,7 @@ coda_checkunmounting(struct mount *mp) struct cnode *cp; int count = 0, bad = 0; - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, nvp) { - VI_LOCK(vp); - if (vp->v_iflag & VI_DOOMED) { - VI_UNLOCK(vp); - continue; - } + MNT_VNODE_FOREACH_ALL(vp, mp, nvp) { cp = VTOC(vp); count++; if (!(cp->c_flags & C_UNMOUNTING)) { @@ -381,7 +375,6 @@ coda_checkunmounting(struct mount *mp) } VI_UNLOCK(vp); } - MNT_IUNLOCK(mp); } void Index: /usr/src/sys/fs/ext2fs/ext2_vfsops.c =================================================================== --- /usr/src/sys/fs/ext2fs/ext2_vfsops.c (revision 234335) +++ /usr/src/sys/fs/ext2fs/ext2_vfsops.c (working copy) @@ -480,19 +480,12 @@ ext2_reload(struct mount *mp, struct thread *td) } loop: - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - if (vp->v_iflag & VI_DOOMED) { - VI_UNLOCK(vp); - continue; - } - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { /* * Step 4: invalidate all cached file data. */ if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td)) { - MNT_VNODE_FOREACH_ABORT(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } if (vinvalbuf(vp, 0, 0, 0)) @@ -507,7 +500,7 @@ loop: if (error) { VOP_UNLOCK(vp, 0); vrele(vp); - MNT_VNODE_FOREACH_ABORT(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); return (error); } ext2_ei2i((struct ext2fs_dinode *) ((char *)bp->b_data + @@ -515,9 +508,7 @@ loop: brelse(bp); VOP_UNLOCK(vp, 0); vrele(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); return (0); } @@ -841,27 +832,20 @@ ext2_sync(struct mount *mp, int waitfor) */ MNT_ILOCK(mp); loop: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - if (vp->v_type == VNON || (vp->v_iflag & VI_DOOMED)) { - VI_UNLOCK(vp); - continue; - } - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { ip = VTOI(vp); if ((ip->i_flag & (IN_ACCESS | IN_CHANGE | IN_MODIFIED | IN_UPDATE)) == 0 && (vp->v_bufobj.bo_dirty.bv_cnt == 0 || waitfor == MNT_LAZY)) { VI_UNLOCK(vp); - MNT_ILOCK(mp); continue; } error = vget(vp, LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK, td); if (error) { MNT_ILOCK(mp); if (error == ENOENT) { - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } continue; @@ -870,9 +854,7 @@ loop: allerror = error; VOP_UNLOCK(vp, 0); vrele(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); /* * Force stale file system control information to be flushed. Index: /usr/src/sys/fs/msdosfs/msdosfs_vfsops.c =================================================================== --- /usr/src/sys/fs/msdosfs/msdosfs_vfsops.c (revision 234335) +++ /usr/src/sys/fs/msdosfs/msdosfs_vfsops.c (working copy) @@ -923,27 +923,18 @@ msdosfs_sync(struct mount *mp, int waitfor) /* * Write back each (modified) denode. */ - MNT_ILOCK(mp); loop: - MNT_VNODE_FOREACH(vp, mp, nvp) { - VI_LOCK(vp); - if (vp->v_type == VNON || (vp->v_iflag & VI_DOOMED)) { - VI_UNLOCK(vp); - continue; - } - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, nvp) { dep = VTODE(vp); if ((dep->de_flag & (DE_ACCESS | DE_CREATE | DE_UPDATE | DE_MODIFIED)) == 0 && (vp->v_bufobj.bo_dirty.bv_cnt == 0 || waitfor == MNT_LAZY)) { VI_UNLOCK(vp); - MNT_ILOCK(mp); continue; } error = vget(vp, LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK, td); if (error) { - MNT_ILOCK(mp); if (error == ENOENT) goto loop; continue; @@ -953,9 +944,7 @@ loop: allerror = error; VOP_UNLOCK(vp, 0); vrele(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); /* * Flush filesystem control info. Index: /usr/src/sys/sys/mount.h =================================================================== --- /usr/src/sys/sys/mount.h (revision 234335) +++ /usr/src/sys/sys/mount.h (working copy) @@ -187,6 +187,30 @@ struct mount { struct lock mnt_explock; /* vfs_export walkers lock */ }; +/* + * Definitions for MNT_VNODE_FOREACH_ALL. + */ +struct vnode *__mnt_vnode_next_all(struct vnode **mvp, struct mount *mp); +struct vnode *__mnt_vnode_first_all(struct vnode **mvp, struct mount *mp); +void __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp); + +#define MNT_VNODE_FOREACH_ALL(vp, mp, mvp) \ + for (vp = __mnt_vnode_first_all(&(mvp), (mp)); \ + (vp) != NULL; vp = __mnt_vnode_next_all(&(mvp), (mp))) + +#define MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp) \ + do { \ + MNT_ILOCK(mp); \ + __mnt_vnode_markerfree_all(&(mvp), (mp)); \ + /* MNT_IUNLOCK(mp); -- done in above function */ \ + mtx_assert(MNT_MTX(mp), MA_NOTOWNED); \ + } while (0) + +/* + * Definitions for MNT_VNODE_FOREACH. + * + * This interface has been deprecated in favor of MNT_VNODE_FOREACH_ALL. + */ struct vnode *__mnt_vnode_next(struct vnode **mvp, struct mount *mp); struct vnode *__mnt_vnode_first(struct vnode **mvp, struct mount *mp); void __mnt_vnode_markerfree(struct vnode **mvp, struct mount *mp); Index: /usr/src/sys/ufs/ufs/ufs_quota.c =================================================================== --- /usr/src/sys/ufs/ufs/ufs_quota.c (revision 234335) +++ /usr/src/sys/ufs/ufs/ufs_quota.c (working copy) @@ -598,32 +598,25 @@ quotaon(struct thread *td, struct mount *mp, int t * adding references to quota file being opened. * NB: only need to add dquot's for inodes being modified. */ - MNT_ILOCK(mp); again: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td)) { - MNT_ILOCK(mp); - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto again; } if (vp->v_type == VNON || vp->v_writecount == 0) { VOP_UNLOCK(vp, 0); vrele(vp); - MNT_ILOCK(mp); continue; } error = getinoquota(VTOI(vp)); VOP_UNLOCK(vp, 0); vrele(vp); - MNT_ILOCK(mp); if (error) { - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); break; } } - MNT_IUNLOCK(mp); if (error) quotaoff_inchange(td, mp, type); @@ -669,19 +662,10 @@ quotaoff1(struct thread *td, struct mount *mp, int * Search vnodes associated with this mount point, * deleting any references to quota file being closed. */ - MNT_ILOCK(mp); again: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - MNT_IUNLOCK(mp); - if (vp->v_type == VNON) { - VI_UNLOCK(vp); - MNT_ILOCK(mp); - continue; - } + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td)) { - MNT_ILOCK(mp); - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto again; } ip = VTOI(vp); @@ -690,9 +674,7 @@ again: dqrele(vp, dq); VOP_UNLOCK(vp, 0); vrele(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); dqflush(qvp); /* Clear um_quotas before closing the quota vnode to prevent @@ -1057,20 +1039,12 @@ qsync(struct mount *mp) * Search vnodes associated with this mount point, * synchronizing any modified dquot structures. */ - MNT_ILOCK(mp); again: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - if (vp->v_type == VNON) { - VI_UNLOCK(vp); - continue; - } - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { error = vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td); if (error) { - MNT_ILOCK(mp); if (error == ENOENT) { - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto again; } continue; @@ -1081,9 +1055,7 @@ again: dqsync(vp, dq); } vput(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); return (0); } Index: /usr/src/sys/ufs/ffs/ffs_softdep.c =================================================================== --- /usr/src/sys/ufs/ffs/ffs_softdep.c (revision 234335) +++ /usr/src/sys/ufs/ffs/ffs_softdep.c (working copy) @@ -12642,29 +12642,21 @@ retry: fs->fs_cstotal.cs_nbfree <= needed) || (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 && fs->fs_cstotal.cs_nifree <= needed)) { - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(lvp, mp, mvp) { - VI_LOCK(lvp); + MNT_VNODE_FOREACH_ALL(lvp, mp, mvp) { if (TAILQ_FIRST(&lvp->v_bufobj.bo_dirty.bv_hd) == 0) { VI_UNLOCK(lvp); continue; } - MNT_IUNLOCK(mp); if (vget(lvp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT, - curthread)) { - MNT_ILOCK(mp); + curthread)) continue; - } if (lvp->v_vflag & VV_NOSYNC) { /* unlinked */ vput(lvp); - MNT_ILOCK(mp); continue; } (void) ffs_syncvnode(lvp, MNT_NOWAIT, 0); vput(lvp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); lvp = ump->um_devvp; if (vn_lock(lvp, LK_EXCLUSIVE | LK_NOWAIT) == 0) { VOP_FSYNC(lvp, MNT_NOWAIT, curthread); Index: /usr/src/sys/ufs/ffs/ffs_vfsops.c =================================================================== --- /usr/src/sys/ufs/ffs/ffs_vfsops.c (revision 234335) +++ /usr/src/sys/ufs/ffs/ffs_vfsops.c (working copy) @@ -705,19 +705,12 @@ ffs_reload(struct mount *mp, struct thread *td) } loop: - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - if (vp->v_iflag & VI_DOOMED) { - VI_UNLOCK(vp); - continue; - } - MNT_IUNLOCK(mp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { /* * Step 4: invalidate all cached file data. */ if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td)) { - MNT_VNODE_FOREACH_ABORT(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } if (vinvalbuf(vp, 0, 0, 0)) @@ -732,7 +725,7 @@ loop: if (error) { VOP_UNLOCK(vp, 0); vrele(vp); - MNT_VNODE_FOREACH_ABORT(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); return (error); } ffs_load_inode(bp, ip, fs, ip->i_number); @@ -740,9 +733,7 @@ loop: brelse(bp); VOP_UNLOCK(vp, 0); vrele(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); return (0); } @@ -1441,13 +1432,7 @@ ffs_sync_lazy(mp) td = curthread; if ((mp->mnt_flag & MNT_NOATIME) != 0) goto qupdate; - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); - if (vp->v_iflag & VI_DOOMED || vp->v_type == VNON) { - VI_UNLOCK(vp); - continue; - } + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { ip = VTOI(vp); /* @@ -1462,19 +1447,14 @@ ffs_sync_lazy(mp) VI_UNLOCK(vp); continue; } - MNT_IUNLOCK(mp); if ((error = vget(vp, LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK, - td)) != 0) { - MNT_ILOCK(mp); + td)) != 0) continue; - } error = ffs_update(vp, 0); if (error != 0) allerror = error; vput(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); qupdate: #ifdef QUOTA @@ -1538,41 +1518,33 @@ ffs_sync(mp, waitfor) lockreq = LK_EXCLUSIVE; } lockreq |= LK_INTERLOCK | LK_SLEEPFAIL; - MNT_ILOCK(mp); loop: /* Grab snapshot of secondary write counts */ + MNT_ILOCK(mp); secondary_writes = mp->mnt_secondary_writes; secondary_accwrites = mp->mnt_secondary_accwrites; + MNT_IUNLOCK(mp); /* Grab snapshot of softdep dependency counts */ - MNT_IUNLOCK(mp); softdep_get_depcounts(mp, &softdep_deps, &softdep_accdeps); - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, mvp) { + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { /* * Depend on the vnode interlock to keep things stable enough * for a quick test. Since there might be hundreds of * thousands of vnodes, we cannot afford even a subroutine * call unless there's a good chance that we have work to do. */ - VI_LOCK(vp); - if (vp->v_iflag & VI_DOOMED) { - VI_UNLOCK(vp); - continue; - } ip = VTOI(vp); - if (vp->v_type == VNON || ((ip->i_flag & + if ((ip->i_flag & (IN_ACCESS | IN_CHANGE | IN_MODIFIED | IN_UPDATE)) == 0 && - vp->v_bufobj.bo_dirty.bv_cnt == 0)) { + vp->v_bufobj.bo_dirty.bv_cnt == 0) { VI_UNLOCK(vp); continue; } - MNT_IUNLOCK(mp); if ((error = vget(vp, lockreq, td)) != 0) { - MNT_ILOCK(mp); if (error == ENOENT || error == ENOLCK) { - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } continue; @@ -1580,9 +1552,7 @@ loop: if ((error = ffs_syncvnode(vp, waitfor, 0)) != 0) allerror = error; vput(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); /* * Force stale filesystem control information to be flushed. */ @@ -1590,10 +1560,8 @@ loop: if ((error = softdep_flushworklist(ump->um_mountp, &count, td))) allerror = error; /* Flushed work items may create new vnodes to clean */ - if (allerror == 0 && count) { - MNT_ILOCK(mp); + if (allerror == 0 && count) goto loop; - } } #ifdef QUOTA qsync(mp); @@ -1608,18 +1576,18 @@ loop: if ((error = VOP_FSYNC(devvp, waitfor, td)) != 0) allerror = error; VOP_UNLOCK(devvp, 0); - if (allerror == 0 && waitfor == MNT_WAIT) { - MNT_ILOCK(mp); + if (allerror == 0 && waitfor == MNT_WAIT) goto loop; - } } else if (suspend != 0) { if (softdep_check_suspend(mp, devvp, softdep_deps, softdep_accdeps, secondary_writes, - secondary_accwrites) != 0) + secondary_accwrites) != 0) { + MNT_IUNLOCK(mp); goto loop; /* More work needed */ + } mtx_assert(MNT_MTX(mp), MA_OWNED); mp->mnt_kern_flag |= MNTK_SUSPEND2 | MNTK_SUSPENDED; MNT_IUNLOCK(mp); Index: /usr/src/sys/ufs/ffs/ffs_snapshot.c =================================================================== --- /usr/src/sys/ufs/ffs/ffs_snapshot.c (revision 234335) +++ /usr/src/sys/ufs/ffs/ffs_snapshot.c (working copy) @@ -522,17 +522,13 @@ restart: FSMAXSNAP + 1 /* superblock */ + 1 /* last block */ + 1 /* size */; MNT_ILOCK(mp); mp->mnt_kern_flag &= ~MNTK_SUSPENDED; + MNT_IUNLOCK(mp); loop: - MNT_VNODE_FOREACH(xvp, mp, mvp) { - VI_LOCK(xvp); - MNT_IUNLOCK(mp); - if ((xvp->v_iflag & VI_DOOMED) || - (xvp->v_usecount == 0 && + MNT_VNODE_FOREACH_ALL(xvp, mp, mvp) { + if ((xvp->v_usecount == 0 && (xvp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == 0) || - xvp->v_type == VNON || IS_SNAPSHOT(VTOI(xvp))) { VI_UNLOCK(xvp); - MNT_ILOCK(mp); continue; } /* @@ -541,13 +537,11 @@ loop: */ if (xvp == nd.ni_dvp) { VI_UNLOCK(xvp); - MNT_ILOCK(mp); continue; } vholdl(xvp); if (vn_lock(xvp, LK_EXCLUSIVE | LK_INTERLOCK) != 0) { - MNT_ILOCK(mp); - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); vdrop(xvp); goto loop; } @@ -557,7 +551,6 @@ loop: VI_UNLOCK(xvp); VOP_UNLOCK(xvp, 0); vdrop(xvp); - MNT_ILOCK(mp); continue; } VI_UNLOCK(xvp); @@ -567,14 +560,12 @@ loop: vat.va_nlink > 0) { VOP_UNLOCK(xvp, 0); vdrop(xvp); - MNT_ILOCK(mp); continue; } xp = VTOI(xvp); if (ffs_checkfreefile(copy_fs, vp, xp->i_number)) { VOP_UNLOCK(xvp, 0); vdrop(xvp); - MNT_ILOCK(mp); continue; } /* @@ -610,12 +601,10 @@ loop: free(copy_fs->fs_csp, M_UFSMNT); free(copy_fs, M_UFSMNT); copy_fs = NULL; - MNT_VNODE_FOREACH_ABORT(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto out1; } - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); /* * Erase the journal file from the snapshot. */ @@ -2532,31 +2521,25 @@ process_deferred_inactive(struct mount *mp) td = curthread; (void) vn_start_secondary_write(NULL, &mp, V_WAIT); - MNT_ILOCK(mp); loop: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { /* * IN_LAZYACCESS is checked here without holding any * vnode lock, but this flag is set only while holding * vnode interlock. */ - if (vp->v_type == VNON || (vp->v_iflag & VI_DOOMED) != 0 || - ((VTOI(vp)->i_flag & IN_LAZYACCESS) == 0 && - ((vp->v_iflag & VI_OWEINACT) == 0 || - vp->v_usecount > 0))) { + if ((VTOI(vp)->i_flag & IN_LAZYACCESS) == 0 && + ((vp->v_iflag & VI_OWEINACT) == 0 || vp->v_usecount > 0)) { VI_UNLOCK(vp); continue; } - MNT_IUNLOCK(mp); vholdl(vp); error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); if (error != 0) { vdrop(vp); - MNT_ILOCK(mp); if (error == ENOENT) continue; /* vnode recycled */ - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } ip = VTOI(vp); @@ -2569,7 +2552,6 @@ process_deferred_inactive(struct mount *mp) VI_UNLOCK(vp); VOP_UNLOCK(vp, 0); vdrop(vp); - MNT_ILOCK(mp); continue; } vinactive(vp, td); @@ -2578,9 +2560,7 @@ process_deferred_inactive(struct mount *mp) VI_UNLOCK(vp); VOP_UNLOCK(vp, 0); vdrop(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); vn_finished_secondary_write(mp); } Index: /usr/src/sys/kern/vfs_default.c =================================================================== --- /usr/src/sys/kern/vfs_default.c (revision 234335) +++ /usr/src/sys/kern/vfs_default.c (working copy) @@ -1114,18 +1114,15 @@ vfs_stdsync(mp, waitfor) /* * Force stale buffer cache information to be flushed. */ - MNT_ILOCK(mp); loop: - MNT_VNODE_FOREACH(vp, mp, mvp) { - /* bv_cnt is an acceptable race here. */ - if (vp->v_bufobj.bo_dirty.bv_cnt == 0) + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { + if (vp->v_bufobj.bo_dirty.bv_cnt == 0) { + VI_UNLOCK(vp); continue; - VI_LOCK(vp); - MNT_IUNLOCK(mp); + } if ((error = vget(vp, lockreq, td)) != 0) { - MNT_ILOCK(mp); if (error == ENOENT) { - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } continue; @@ -1134,9 +1131,7 @@ loop: if (error) allerror = error; vput(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); return (allerror); } Index: /usr/src/sys/kern/vfs_mount.c =================================================================== --- /usr/src/sys/kern/vfs_mount.c (revision 234335) +++ /usr/src/sys/kern/vfs_mount.c (working copy) @@ -81,7 +81,6 @@ SYSCTL_INT(_vfs, OID_AUTO, usermount, CTLFLAG_RW, "Unprivileged users may mount and unmount file systems"); MALLOC_DEFINE(M_MOUNT, "mount", "vfs mount structure"); -static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); static uma_zone_t mount_zone; /* List of mounted filesystems. */ @@ -1720,10 +1719,14 @@ vfs_copyopt(opts, name, dest, len) } /* - * This is a helper function for filesystems to traverse their - * vnodes. See MNT_VNODE_FOREACH() in sys/mount.h + * These are helper functions for filesystems to traverse all + * their vnodes. See MNT_VNODE_FOREACH() in sys/mount.h. + * + * This interface has been deprecated in favor of MNT_VNODE_FOREACH_ALL. */ +MALLOC_DECLARE(M_VNODE_MARKER); + struct vnode * __mnt_vnode_next(struct vnode **mvp, struct mount *mp) { @@ -1812,7 +1815,6 @@ __mnt_vnode_markerfree(struct vnode **mvp, struct MNT_REL(mp); } - int __vfs_statfs(struct mount *mp, struct statfs *sbp) { Index: /usr/src/sys/kern/vfs_subr.c =================================================================== --- /usr/src/sys/kern/vfs_subr.c (revision 234335) +++ /usr/src/sys/kern/vfs_subr.c (working copy) @@ -2467,17 +2467,13 @@ vflush(struct mount *mp, int rootrefs, int flags, } vput(rootvp); } - MNT_ILOCK(mp); loop: - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { vholdl(vp); - MNT_IUNLOCK(mp); error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE); if (error) { vdrop(vp); - MNT_ILOCK(mp); - MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } /* @@ -2486,7 +2482,6 @@ loop: if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) { VOP_UNLOCK(vp, 0); vdrop(vp); - MNT_ILOCK(mp); continue; } /* @@ -2504,7 +2499,7 @@ loop: if (error != 0) { VOP_UNLOCK(vp, 0); vdrop(vp); - MNT_VNODE_FOREACH_ABORT(mp, mvp); + MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); return (error); } error = VOP_GETATTR(vp, &vattr, td->td_ucred); @@ -2515,7 +2510,6 @@ loop: (vp->v_writecount == 0 || vp->v_type != VREG)) { VOP_UNLOCK(vp, 0); vdropl(vp); - MNT_ILOCK(mp); continue; } } else @@ -2540,9 +2534,7 @@ loop: } VOP_UNLOCK(vp, 0); vdropl(vp); - MNT_ILOCK(mp); } - MNT_IUNLOCK(mp); if (rootrefs > 0 && (flags & FORCECLOSE) == 0) { /* * If just the root vnode is busy, and if its refcount @@ -3279,19 +3271,15 @@ vfs_msync(struct mount *mp, int flags) struct vm_object *obj; CTR2(KTR_VFS, "%s: mp %p", __func__, mp); - MNT_ILOCK(mp); - MNT_VNODE_FOREACH(vp, mp, mvp) { - VI_LOCK(vp); + MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { obj = vp->v_object; if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 && (flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) { - MNT_IUNLOCK(mp); if (!vget(vp, LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK, curthread)) { if (vp->v_vflag & VV_NOSYNC) { /* unlinked */ vput(vp); - MNT_ILOCK(mp); continue; } @@ -3305,11 +3293,9 @@ vfs_msync(struct mount *mp, int flags) } vput(vp); } - MNT_ILOCK(mp); } else VI_UNLOCK(vp); } - MNT_IUNLOCK(mp); } /* @@ -4504,3 +4490,90 @@ vfs_unixify_accmode(accmode_t *accmode) return (0); } + +/* + * These are helper functions for filesystems to traverse all + * their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h. + * + * This interface replaces MNT_VNODE_FOREACH. + */ + +MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); + +struct vnode * +__mnt_vnode_next_all(struct vnode **mvp, struct mount *mp) +{ + struct vnode *vp; + + if (should_yield()) + kern_yield(PRI_UNCHANGED); + MNT_ILOCK(mp); + KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); + vp = TAILQ_NEXT(*mvp, v_nmntvnodes); + while (vp != NULL && (vp->v_type == VMARKER || vp->v_type == VNON || + (vp->v_iflag & VI_DOOMED) != 0)) + vp = TAILQ_NEXT(vp, v_nmntvnodes); + + /* Check if we are done */ + if (vp == NULL) { + __mnt_vnode_markerfree_all(mvp, mp); + /* MNT_IUNLOCK(mp); -- done in above function */ + mtx_assert(MNT_MTX(mp), MA_NOTOWNED); + return (NULL); + } + TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); + TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); + VI_LOCK(vp); + MNT_IUNLOCK(mp); + return (vp); +} + +struct vnode * +__mnt_vnode_first_all(struct vnode **mvp, struct mount *mp) +{ + struct vnode *vp; + + *mvp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO); + MNT_ILOCK(mp); + MNT_REF(mp); + (*mvp)->v_type = VMARKER; + + vp = TAILQ_FIRST(&mp->mnt_nvnodelist); + while (vp != NULL && (vp->v_type == VMARKER || vp->v_type == VNON || + (vp->v_iflag & VI_DOOMED) != 0)) + vp = TAILQ_NEXT(vp, v_nmntvnodes); + + /* Check if we are done */ + if (vp == NULL) { + *mvp = NULL; + MNT_REL(mp); + MNT_IUNLOCK(mp); + free(*mvp, M_VNODE_MARKER); + return (NULL); + } + (*mvp)->v_mount = mp; + TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes); + VI_LOCK(vp); + MNT_IUNLOCK(mp); + return (vp); +} + + +void +__mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp) +{ + + if (*mvp == NULL) { + MNT_IUNLOCK(mp); + return; + } + + mtx_assert(MNT_MTX(mp), MA_OWNED); + + KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch")); + TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes); + MNT_REL(mp); + MNT_IUNLOCK(mp); + free(*mvp, M_VNODE_MARKER); + *mvp = NULL; +} -- Test scenario: mountu.sh 20120416 11:36:30 all: mountu.sh NFS FAILED PID START END PRT RES PRES REF SHD FL TP PATH 82485 0x8048000 0x8049000 r-x 1 0 1 0 CN- vn /tmp/mountu 82485 0x8049000 0x804a000 rw- 1 0 1 0 --- df 82485 0x28049000 0x28080000 r-x 22 0 101 44 CN- vn /libexec/ld-elf.so.1 82485 0x28080000 0x28082000 rw- 2 0 1 0 C-- vn /libexec/ld-elf.so.1 82485 0x28082000 0x28095000 rw- 12 0 1 0 --- df 82485 0x28095000 0x281a2000 r-x 104 0 101 44 CN- vn /lib/libc.so.7 82485 0x281a2000 0x281a8000 rw- 6 0 1 0 C-- vn /lib/libc.so.7 82485 0x281a8000 0x281d0000 rw- 12 0 1 0 --- df 82485 0x281d0000 0x28810000 rw- 16 0 1 0 --- vn /mnt/file 82485 0xbfbe0000 0xbfc00000 rwx 3 0 1 0 --- df umount: unmount of /mnt failed: Device busy umount: unmount of /mnt failed: Device busy umount: unmount of /mnt failed: Device busy