GENERIC HEAD from r202153M, vmcore.414 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2010 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 9.0-CURRENT #0 r202153M: Tue Jan 12 18:27:53 CET 2010 pho@x4.osted.lan:/usr/src/sys/i386/compile/PHO i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1799.99-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) avail memory = 3536150528 (3372 MB) ACPI APIC Table: <060508 APIC1048> FreeBSD/SMP: Multiprocessor System Detected: 4 CPUs FreeBSD/SMP: 1 package(s) x 4 core(s) cpu0 (BSP): APIC ID: 0 cpu1 (AP): APIC ID: 1 cpu2 (AP): APIC ID: 2 cpu3 (AP): APIC ID: 3 ioapic0 irqs 0-23 on motherboard : Trying to mount root from ufs:/dev/ad4s1a Entropy harvesting: interrupts ethernet point_to_point kickstart. /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 201980 free (3212 frags, 24846 blocks, 0.3% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 49800178 free (1778 frags, 6224800 blocks, 0.0% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 4285219 free (267971 frags, 502156 blocks, 1.2% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 13693682 free (99962 frags, 1699215 blocks, 0.2% fragmentation) Additional TCP/IP options: rfc1323 extensions=NO no-ipv4-mapped-ipv6. re0: link state changed to DOWN Starting Network: lo0 re0 fwe0 fwip0. Starting Network: fwe0. Starting Network: fwip0. add net default: gateway 192.168.1.1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 add net fe80::: gateway ::1 add net ff02::: gateway ::1 Additional ABI support: linux. lock order reversal: 1st 0xe6d1d320 bufwait (bufwait) @ kern/vfs_bio.c:2559 2nd 0xc742be00 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:285 KDB: stack backtrace: db_trace_self_wrapper(c0ca604d,f6143878,c08d77d5,c08c822b,c0ca8ffe,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c08c822b,c0ca8ffe,c6d2efc8,c6d328a8,f61438d4,...) at kdb_backtrace+0x29 _witness_debugger(c0ca8ffe,c742be00,c0ccb72e,c6d328a8,c0ccb3c0,...) at _witness_debugger+0x25 witness_checkorder(c742be00,9,c0ccb3b7,11d,0,...) at witness_checkorder+0x839 _sx_xlock(c742be00,0,c0ccb3b7,11d,c73fa910,...) at _sx_xlock+0x85 ufsdirhash_acquire(e6d1d2c0,f6143a1c,48,e77939d4,f61439a4,...) at ufsdirhash_acquire+0x48 ufsdirhash_add(c73fa910,f6143a1c,29d4,f6143990,f6143994,...) at ufsdirhash_add+0x13 ufs_direnter(c74fe15c,c794f15c,f6143a1c,f6143c00,e6d1dae0,...) at ufs_direnter+0x779 ufs_mkdir(f6143c28,c0ce0cdf,0,0,f6143b6c,...) at ufs_mkdir+0x9f1 VOP_MKDIR_APV(c0db0fc0,f6143c28,f6143c00,f6143b6c,0,...) at VOP_MKDIR_APV+0xc5 kern_mkdirat(c7747480,ffffff9c,bfbfef5a,0,1ff,...) at kern_mkdirat+0x21b kern_mkdir(c7747480,bfbfef5a,0,1ff,f6143d2c,...) at kern_mkdir+0x2e mkdir(c7747480,f6143cf8,8,c0ca98af,c0d8ff20,...) at mkdir+0x29 syscall(f6143d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (136, FreeBSD ELF32, mkdir), eip = 0x2816a3d3, esp = 0xbfbfed6c, ebp = 0xbfbfee38 --- Configuring syscons: keymap blanktime. Local package initialization:panic: Bad link elm 0xc74fe254 prev->next != elm cpuid = 1 KDB: enter: panic [thread pid 1276 tid 100083 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 1276 tid 100083 td 0xc70ea6c0 kdb_enter(c0ca2b60,c0ca2b60,c0c3c66a,f6076adc,1,...) at kdb_enter+0x3a panic(c0c3c66a,c74fe254,c0cb0201,7a3,c76125f0,...) at panic+0x136 reassignbuf(e6d1d2c0,c70ea764,c0f3f5e0,c0d9719c,e6d1d2c0,...) at reassignbuf+0x189 bundirty(e6d1d2c0,f6076b4c,c0881a3c,0,e6d1d2c0,...) at bundirty+0x5c bufwrite(e6d1d2c0,e6d1d320,f6076ba8,c0ad788f,e6d1d2c0,...) at bufwrite+0x7a bawrite(e6d1d2c0,0,c0ccaf36,101,0,...) at bawrite+0x5c ffs_syncvnode(c74fe15c,1,c70ea6c0,545,246,...) at ffs_syncvnode+0x29f ffs_sync(c76125f0,1,c0caf9c7,507,0,...) at ffs_sync+0x26f dounmount(c76125f0,8000000,c70ea6c0,47f,ff1ca3a5,...) at dounmount+0x520 unmount(c70ea6c0,f6076cf8,8,c70ea6c0,c0d8f2a8,...) at unmount+0x2ff syscall(f6076d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d9f7f, esp = 0xbfbfe68c, ebp = 0xbfbfe758 --- db:0:bt> show allpcpu Current CPU: 1 cpuid = 0 dynamic pcpu = 0x659000 curthread = 0xc6dc0480: pid 12 "swi4: clock" curpcb = 0xc6a3dd90 fpcurthread = none idlethread = 0xc6d7a480: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x5c3e000 curthread = 0xc70ea6c0: pid 1276 "umount" curpcb = 0xf6076d90 fpcurthread = none idlethread = 0xc6d7a6c0: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x5c41000 curthread = 0xc6d7c900: pid 3 "g_up" curpcb = 0xc6a4cd90 fpcurthread = none idlethread = 0xc6d7a900: pid 11 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x5c44000 curthread = 0xc6d7ab40: pid 11 "idle: cpu3" curpcb = 0xc6a2ad90 fpcurthread = none idlethread = 0xc6d7ab40: pid 11 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 1276 (umount) thread 0xc70ea6c0 (100083) exclusive sleep mutex bufobj interlock (bufobj interlock) r = 0 (0xc74fe254) locked @ kern/vfs_subr.c:1955 exclusive sleep mutex struct mount mtx (struct mount mtx) r = 0 (0xc76125f0) locked @ kern/vfs_subr.c:1954 exclusive lockmgr bufwait (bufwait) r = 0 (0xe6d1d320) locked @ ufs/ffs/ffs_vnops.c:255 exclusive lockmgr ufs (ufs) r = 0 (0xc74fe1b4) locked @ kern/vfs_subr.c:2114 exclusive sleep mutex Giant (Giant) r = 0 (0xc0e00290) locked @ kern/vfs_mount.c:1135 exclusive lockmgr ufs (ufs) r = 0 (0xc74fe5c8) locked @ kern/vfs_mount.c:1205 db:0:alllocks> show lockedvnods Locked vnodes 0xc74fe570: tag ufs, type VDIR usecount 1, writecount 0, refcount 1 mountedhere 0xc76125f0 flags () lock type ufs: EXCL by thread 0xc70ea6c0 (pid 1276) #0 0xc087a34e at __lockmgr_args+0xbfe #1 0xc0ad7c31 at ffs_lock+0xa1 #2 0xc0be8595 at VOP_LOCK1_APV+0xb5 #3 0xc0933fd8 at _vn_lock+0x78 #4 0xc091e77c at dounmount+0x8c #5 0xc091f1df at unmount+0x2ff #6 0xc0bcf744 at syscall+0x2b4 #7 0xc0bb1ac0 at Xint0x80_syscall+0x20 ino 211968, on dev ad4s1a 0xc74fe15c: tag ufs, type VDIR usecount 1, writecount 0, refcount 5 mountedhere 0 flags (VV_ROOT) v_object 0xc787fa18 ref 0 pages 9 lock type ufs: EXCL by thread 0xc70ea6c0 (pid 1276) #0 0xc087a34e at __lockmgr_args+0xbfe #1 0xc0ad7c31 at ffs_lock+0xa1 #2 0xc0be8595 at VOP_LOCK1_APV+0xb5 #3 0xc0933fd8 at _vn_lock+0x78 #4 0xc09271bb at vget+0xbb #5 0xc0ad31d6 at ffs_sync+0x216 #6 0xc091ec10 at dounmount+0x520 #7 0xc091f1df at unmount+0x2ff #8 0xc0bcf744 at syscall+0x2b4 #9 0xc0bb1ac0 at Xint0x80_syscall+0x20 ino 2, on dev ad4s1e db:0:lockedvnods> show mount 0xc73adbe0 /dev/ad4s1a on / (ufs) 0xc73ae000 devfs on /dev (devfs) 0xc76125f0 /dev/ad4s1e on /tmp (ufs) 0xc7618000 /dev/ad4s1f on /usr (ufs) 0xc7617be0 /dev/ad4s1d on /var (ufs) 0xc76178e8 procfs on /proc (procfs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 1276 1275 20 0 R+ CPU 1 umount 1275 1274 20 0 S+ wait 0xc7631d48 sh 1274 1270 20 0 S+ wait 0xc797b550 sh 1270 20 20 0 S+ wait 0xc7633000 sh 1266 1 1266 0 Ss nanslp 0xc0e00ae4 cron 1259 1 1259 25 Ss pause 0xc7436da0 sendmail 1255 1 1255 0 Ss select 0xc797e164 sendmail 1247 1 1247 0 Ss select 0xc7430e64 sshd 1202 1 1202 0 Ss select 0xc74a2724 ntpd 1061 1 1061 0 Ss select 0xc797f2a4 moused 1028 1 1028 0 Ss rpcsvc 0xc74a26d0 NLM: master 1021 1 1021 0 Ss select 0xc73b4e24 rpc.statd 1014 1013 1013 0 S (threaded) nfsd 100145 S rpcsvc 0xc7430c90 nfsd: service 100144 S rpcsvc 0xc7430490 nfsd: service 100143 S rpcsvc 0xc7431150 nfsd: service 100133 S rpcsvc 0xc7371090 nfsd: master 1013 1 1013 0 Ss select 0xc74a2764 nfsd 1004 1 1004 0 Ss select 0xc7430124 mountd 911 1 911 0 Ss select 0xc73b4d64 rpcbind 877 1 877 0 Ss select 0xc70d68a4 syslogd 695 1 695 0 Ss select 0xc7371b64 devd 20 1 20 0 Ss+ wait 0xc74077f8 sh 19 0 0 0 DL flowclea 0xc0f6d1c8 [flowcleaner] 18 0 0 0 DL sdflush 0xc0f78920 [softdepflush] 17 0 0 0 DL vlruwt 0xc7397aa0 [vnlru] 16 0 0 0 DL psleep 0xc0f6cd48 [bufdaemon] 15 0 0 0 DL pgzero 0xc0f79714 [pagezero] 9 0 0 0 DL psleep 0xc0f79344 [vmdaemon] 8 0 0 0 DL psleep 0xc0f7930c [pagedaemon] 7 0 0 0 DL - 0xc6fc363c [fdc0] 6 0 0 0 SL - 0xc6ff7000 [fw0_probe] 14 0 0 0 DL (threaded) [usb] 100058 D - 0xc6fd8d0c [usbus5] 100057 D - 0xc6fd8cdc [usbus5] 100056 D - 0xc6fd8cac [usbus5] 100055 D - 0xc6fd8c7c [usbus5] 100053 D - 0xc6fc9b5c [usbus4] 100052 D - 0xc6fc9b2c [usbus4] 100051 D - 0xc6fc9afc [usbus4] 100050 D - 0xc6fc9acc [usbus4] 100049 D - 0xc6fc5b5c [usbus3] 100048 D - 0xc6fc5b2c [usbus3] 100047 D - 0xc6fc5afc [usbus3] 100046 D - 0xc6fc5acc [usbus3] 100045 D - 0xc6fbeb5c [usbus2] 100044 D - 0xc6fbeb2c [usbus2] 100043 D - 0xc6fbeafc [usbus2] 100042 D - 0xc6fbeacc [usbus2] 100040 D - 0xc6fb8b5c [usbus1] 100039 D - 0xc6fb8b2c [usbus1] 100038 D - 0xc6fb8afc [usbus1] 100037 D - 0xc6fb8acc [usbus1] 100035 D - 0xc6faab5c [usbus0] 100034 D - 0xc6faab2c [usbus0] 100033 D - 0xc6faaafc [usbus0] 100032 D - 0xc6faaacc [usbus0] 5 0 0 0 DL ccb_scan 0xc0dcccd4 [xpt_thrd] 13 0 0 0 DL - 0xc0e00944 [yarrow] 4 0 0 0 DL - 0xc0dfe6e4 [g_down] 3 0 0 0 RL CPU 2 [g_up] 2 0 0 0 DL - 0xc0dfe6d8 [g_event] 12 0 0 0 RL (threaded) [intr] 100066 I [irq12: psm0] 100065 I [irq1: atkbd0] 100063 I [swi0: uart] 100060 I [irq20: fwohci0] 100059 I [irq14: ata0] 100054 I [irq19: ehci0] 100041 I [irq18: ohci2 ohci4] 100036 I [irq17: ohci1 ohci3] 100031 I [irq16: hdac1 ohci0] 100030 I [irq22: atapci0] 100029 I [irq256: hdac0] 100028 I [irq9: acpi0] 100024 I [swi2: cambio] 100022 I [swi6: task queue] 100021 I [swi6: Giant taskq] 100019 I [swi5: +] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 Run CPU 0 [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu0] 100005 CanRun [idle: cpu1] 100004 CanRun [idle: cpu2] 100003 Run CPU 3 [idle: cpu3] 1 0 1 0 SLs wait 0xc6d78d48 [init] 10 0 0 0 DL audit_wo 0xc0f78240 [audit] 0 0 0 0 DLs (threaded) [kernel] 100096 D syncer 0xc7617bbc [syncer procfs] 100095 D syncer 0xc7617eb4 [syncer ufs] 100094 D syncer 0xc76182d4 [syncer ufs] 100075 D syncer 0xc73adeb4 [syncer ufs] 100074 D syncer 0xc73ae2d4 [syncer devfs] 100061 D - 0xc6fa4940 [fw0_taskq] 100027 D - 0xc6f46080 [acpi_task_2] 100026 D - 0xc6f46080 [acpi_task_1] 100025 D - 0xc6f46080 [acpi_task_0] 100020 D - 0xc6f46300 [thread taskq] 100018 D - 0xc6f46580 [kqueue taskq] 100016 D - 0xc6d5fe40 [firmware taskq] 100000 D sched 0xc0dfe7c0 [swapper] db:0:ps> allt Tracing command umount pid 1276 tid 100083 td 0xc70ea6c0 kdb_enter(c0ca2b60,c0ca2b60,c0c3c66a,f6076adc,1,...) at kdb_enter+0x3a panic(c0c3c66a,c74fe254,c0cb0201,7a3,c76125f0,...) at panic+0x136 reassignbuf(e6d1d2c0,c70ea764,c0f3f5e0,c0d9719c,e6d1d2c0,...) at reassignbuf+0x189 bundirty(e6d1d2c0,f6076b4c,c0881a3c,0,e6d1d2c0,...) at bundirty+0x5c bufwrite(e6d1d2c0,e6d1d320,f6076ba8,c0ad788f,e6d1d2c0,...) at bufwrite+0x7a bawrite(e6d1d2c0,0,c0ccaf36,101,0,...) at bawrite+0x5c ffs_syncvnode(c74fe15c,1,c70ea6c0,545,246,...) at ffs_syncvnode+0x29f ffs_sync(c76125f0,1,c0caf9c7,507,0,...) at ffs_sync+0x26f dounmount(c76125f0,8000000,c70ea6c0,47f,ff1ca3a5,...) at dounmount+0x520 unmount(c70ea6c0,f6076cf8,8,c70ea6c0,c0d8f2a8,...) at unmount+0x2ff syscall(f6076d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (22, FreeBSD ELF32, unmount), eip = 0x280d9f7f, esp = 0xbfbfe68c, ebp = 0xbfbfe758 --- Tracing command sh pid 1275 tid 100100 td 0xc74a5900 sched_switch(c74a5900,0,104,191,f06e447a,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c74a5900,0,c0ca7177,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0ca7177,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c7631d48,5c,c0ca9984,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7631d48,c7631dd0,15c,c0ca9984,0,...) at _sleep+0x354 kern_wait(c74a5900,ffffffff,f60d6c74,2,0,...) at kern_wait+0xb76 wait4(c74a5900,f60d6cf8,10,c74a5900,c0d8f104,...) at wait4+0x3b syscall(f60d6d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815de9b, esp = 0xbfbfe9ac, ebp = 0xbfbfe9c8 --- Tracing command sh pid 1274 tid 100148 td 0xc766c6c0 sched_switch(c766c6c0,0,104,191,f0142678,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c766c6c0,0,c0ca7177,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0ca7177,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c797b550,5c,c0ca9984,100,0,...) at sleepq_wait_sig+0x17 _sleep(c797b550,c797b5d8,15c,c0ca9984,0,...) at _sleep+0x354 kern_wait(c766c6c0,ffffffff,f61afc74,2,0,...) at kern_wait+0xb76 wait4(c766c6c0,f61afcf8,10,c0ca9963,c0d8f104,...) at wait4+0x3b syscall(f61afd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815de9b, esp = 0xbfbfd68c, ebp = 0xbfbfd6a8 --- Tracing command sh pid 1270 tid 100099 td 0xc74a5b40 sched_switch(c74a5b40,0,104,191,f00736d6,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c74a5b40,0,c0ca7177,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0ca7177,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c7633000,5c,c0ca9984,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7633000,c7633088,15c,c0ca9984,0,...) at _sleep+0x354 kern_wait(c74a5b40,ffffffff,f60d2c74,2,0,...) at kern_wait+0xb76 wait4(c74a5b40,f60d2cf8,10,c0ca9963,c0d8f104,...) at wait4+0x3b syscall(f60d2d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815de9b, esp = 0xbfbfd83c, ebp = 0xbfbfd858 --- Tracing command cron pid 1266 tid 100097 td 0xc74a6000 sched_switch(c74a6000,0,104,191,ee1cec35,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c74a6000,0,c0ca7177,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(55f1,c08cde20,c74a6000,2,100,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c0e00ae4,5c,c0ca3e0d,100,0,...) at sleepq_timedwait_sig+0x1a _sleep(c0e00ae4,0,15c,c0ca3e0d,55f1,...) at _sleep+0x31e kern_nanosleep(c74a6000,f60c2c64,f60c2c6c,16,0,...) at kern_nanosleep+0xc1 nanosleep(c74a6000,f60c2cf8,8,c74a6000,c0d90a80,...) at nanosleep+0x6f syscall(f60c2d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28178887, esp = 0xbfbfec8c, ebp = 0xbfbfecb8 --- Tracing command sendmail pid 1259 tid 100080 td 0xc7438d80 sched_switch(c7438d80,0,104,191,e6924118,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,68,...) at mi_switch+0x200 sleepq_switch(c7438d80,0,c0ca7177,1a0,68,...) at sleepq_switch+0x15f sleepq_catch_signals(c0ca7177,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c7436da0,68,c0c5627d,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7436da0,c7436dd0,168,c0c5627d,0,...) at _sleep+0x354 kern_sigsuspend(c7438d80,0,0,0,0,...) at kern_sigsuspend+0xae sigsuspend(c7438d80,f606acf8,4,c0ca9873,c0d9158c,...) at sigsuspend+0x4d syscall(f606ad38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (4, FreeBSD ELF32, write), eip = 0x28332d3b, esp = 0xbfbfcf8c, ebp = 0xbfbfcfb8 --- Tracing command sendmail pid 1255 tid 100162 td 0xc797c000 sched_switch(c797c000,0,104,191,df836e49,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c797c000,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f61e0a4c,c0881aca,c797e150,0,c797c000,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c797e164,0,f61e0a7c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c797e164,c797e150,1389,603,c7894ce8,...) at _cv_timedwait_sig+0x250 seltdwait(f61e0c28,f61e0c30,c7965380,c797c000,ebe,...) at seltdwait+0x8a kern_select(c797c000,5,bfbfc510,0,0,f61e0c70,20,5,0) at kern_select+0x4f4 select(c797c000,f61e0cf8,14,c797c000,c0d8fa6c,...) at select+0x66 syscall(f61e0d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283d6e43, esp = 0xbfbfc47c, ebp = 0xbfbfcfa8 --- Tracing command sshd pid 1247 tid 100161 td 0xc797c240 sched_switch(c797c240,0,104,191,b74176e4,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c797c240,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0881aca,c7430e50,0,c0ca12e8,c797c240,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c7430e64,0,f61dda7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7430e64,c7430e50,c0ca93ea,603,c740b310,...) at _cv_wait_sig+0x240 seltdwait(c740b310,58,c7965a80,c797c240,ebe,...) at seltdwait+0xa2 kern_select(c797c240,5,286090b0,0,0,0,20,28609010,1) at kern_select+0x4f4 select(c797c240,f61ddcf8,14,c797c240,c0d8fa6c,...) at select+0x66 syscall(f61ddd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x283cde43, esp = 0xbfbfdf1c, ebp = 0xbfbfee38 --- Tracing command ntpd pid 1202 tid 100081 td 0xc7438b40 sched_switch(c7438b40,0,104,191,f3365404,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c7438b40,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0881aca,c74a2710,0,c0ca12e8,c7438b40,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c74a2724,0,f606ea7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c74a2724,c74a2710,c0ca93ea,603,c74227e0,...) at _cv_wait_sig+0x240 seltdwait(c74227e0,58,c6d7e380,c7438b40,c0cacfed,...) at seltdwait+0xa2 kern_select(c7438b40,1c,bfbfed28,0,0,0,20,0,bfbfeb38) at kern_select+0x4f4 select(c7438b40,f606ecf8,14,c0ca9e38,c0d8fa6c,...) at select+0x66 syscall(f606ed38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28353e43, esp = 0xbfbfecfc, ebp = 0xbfbfedc8 --- Tracing command moused pid 1061 tid 100086 td 0xc70ea480 sched_switch(c70ea480,0,104,191,3cba0a3e,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c70ea480,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0881aca,c797f290,0,c0ca12e8,c70ea480,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c797f2a4,0,f6082a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c797f2a4,c797f290,c0ca93ea,603,c740b888,...) at _cv_wait_sig+0x240 seltdwait(c740b888,58,c6d7e380,c70ea480,0,...) at seltdwait+0xa2 kern_select(c70ea480,6,bfbfea34,0,0,0,20,0,0) at kern_select+0x4f4 select(c70ea480,f6082cf8,14,c0cbb0e2,c0d8fa6c,...) at select+0x66 syscall(f6082d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x281aae43, esp = 0xbfbfe9bc, ebp = 0xbfbfeb58 --- Tracing command rpc.lockd pid 1028 tid 100118 td 0xc7747240 sched_switch(c7747240,0,104,191,2ce798b9,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c7747240,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f61469fc,c0881aca,c7961d00,0,c7747240,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c74a26d0,0,f6146a2c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c74a26d0,c7961d00,1388,3af,c728ad00,...) at _cv_timedwait_sig+0x250 svc_run_internal(c77473a8,14,c0cc6790,c0cc5e85,f6146b8c,...) at svc_run_internal+0x356 svc_run(c7961d00,0,4,c0a7e960,f6146b8c,...) at svc_run+0x7f nlm_syscall(c7747240,f6146cf8,10,c0caa1d7,c0d90118,...) at nlm_syscall+0x77d syscall(f6146d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (154, FreeBSD ELF32, nlm_syscall), eip = 0x280fa93b, esp = 0xbfbfed4c, ebp = 0xbfbfee18 --- Tracing command rpc.statd pid 1021 tid 100147 td 0xc73b56c0 sched_switch(c73b56c0,0,104,191,2ce7004b,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c73b56c0,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f61aaa4c,c0881aca,c73b4e10,0,c73b56c0,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c73b4e24,0,f61aaa7c,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c73b4e24,c73b4e10,7531,603,c74a4888,...) at _cv_timedwait_sig+0x250 seltdwait(f61aac28,f61aac30,c6d7e380,c73b56c0,c785c500,...) at seltdwait+0x8a kern_select(c73b56c0,8,bfbfed34,0,0,f61aac70,20,1e,0) at kern_select+0x4f4 select(c73b56c0,f61aacf8,14,c0ca9aae,c0d8fa6c,...) at select+0x66 syscall(f61aad38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28186e43, esp = 0xbfbfec8c, ebp = 0xbfbfedc8 --- Tracing command nfsd pid 1014 tid 100145 td 0xc73b9480 sched_switch(c73b9480,0,104,191,1dfc34fc,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c73b9480,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f61a3bf8,c0881aca,c7258b80,0,c73b9480,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c7430c90,0,f61a3c28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7430c90,c7258b80,1388,3ea,5a5a5a5a,...) at _cv_timedwait_sig+0x250 svc_run_internal(f61a3d24,c08676d8,c7258b80,f61a3d38,c0c9e039,...) at svc_run_internal+0x356 svc_thread_start(c7258b80,f61a3d38,c0c9e039,343,c76652a8,...) at svc_thread_start+0x10 fork_exit(c0a8c9b0,c7258b80,f61a3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 1014 tid 100144 td 0xc73b96c0 sched_switch(c73b96c0,0,104,191,1dfbace1,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c73b96c0,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f61a0bf8,c0881aca,c7258b80,0,c73b96c0,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c7430490,0,f61a0c28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7430490,c7258b80,1388,3ea,79786f72,...) at _cv_timedwait_sig+0x250 svc_run_internal(f61a0d24,c08676d8,c7258b80,f61a0d38,c0c9e039,...) at svc_run_internal+0x356 svc_thread_start(c7258b80,f61a0d38,c0c9e039,343,c76652a8,...) at svc_thread_start+0x10 fork_exit(c0a8c9b0,c7258b80,f61a0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 1014 tid 100143 td 0xc73b9900 sched_switch(c73b9900,0,104,191,1dfb692e,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c73b9900,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f619dbf8,c0881aca,c7258b80,0,c73b9900,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c7431150,0,f619dc28,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7431150,c7258b80,1388,3af,5a5a5a5a,...) at _cv_timedwait_sig+0x250 svc_run_internal(f619dd24,c08676d8,c7258b80,f619dd38,c0c9e039,...) at svc_run_internal+0x356 svc_thread_start(c7258b80,f619dd38,c0c9e039,343,c76652a8,...) at svc_thread_start+0x10 fork_exit(c0a8c9b0,c7258b80,f619dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0x2e, eip = 0xc, esp = 0x33, ebp = 0 --- Tracing command nfsd pid 1014 tid 100133 td 0xc766b000 sched_switch(c766b000,0,104,191,1dfc26cf,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c766b000,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6176ae8,c0881aca,c7258b80,0,c766b000,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c7371090,0,f6176b18,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c7371090,c7258b80,1388,3ea,f6176b60,...) at _cv_timedwait_sig+0x250 svc_run_internal(c766b168,14,c0cc6790,c0cc5244,f6176c3c,...) at svc_run_internal+0x356 svc_run(c7258b80,0,c0cc5548,1fb,0,...) at svc_run+0x7f nfssvc_nfsd(bfbfe8b0,f6176c3c,c,c6d7e380,f6176c50,...) at nfssvc_nfsd+0xad nfssvc_nfsserver(c766b000,f6176cf8,bfbfe8b0,c766b000,c76652a8,...) at nfssvc_nfsserver+0x24f nfssvc(c766b000,f6176cf8,8,c0ca9d97,c0d90134,...) at nfssvc+0x83 syscall(f6176d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (155, FreeBSD ELF32, nfssvc), eip = 0x280da91b, esp = 0xbfbfe86c, ebp = 0xbfbfead8 --- Tracing command nfsd pid 1013 tid 100129 td 0xc766cb40 sched_switch(c766cb40,0,104,191,22161f34,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c766cb40,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0881aca,c74a2750,0,c0ca12e8,c766cb40,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c74a2764,0,f6168a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c74a2764,c74a2750,c0ca93ea,603,c740b8f8,...) at _cv_wait_sig+0x240 seltdwait(c740b8f8,58,c6d7e380,c766cb40,ebe,...) at seltdwait+0xa2 kern_select(c766cb40,5,bfbfece0,0,0,0,20,bfbffff4,1) at kern_select+0x4f4 select(c766cb40,f6168cf8,14,c766cb40,c0d8fa6c,...) at select+0x66 syscall(f6168d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x2817de43, esp = 0xbfbfeadc, ebp = 0xbfbfee18 --- Tracing command mountd pid 1004 tid 100124 td 0xc77456c0 sched_switch(c77456c0,0,104,191,1bb328f6,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c77456c0,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0881aca,c7430110,0,c0ca12e8,c77456c0,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c7430124,0,f6159a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7430124,c7430110,c0ca93ea,603,c74ab3b8,...) at _cv_wait_sig+0x240 seltdwait(c74ab3b8,58,c6d7e380,c77456c0,c0f3ff28,...) at seltdwait+0xa2 kern_select(c77456c0,9,bfbfed7c,0,0,0,20,0,281a6238) at kern_select+0x4f4 select(c77456c0,f6159cf8,14,c0cc5009,c0d8fa6c,...) at select+0x66 syscall(f6159d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28191e43, esp = 0xbfbfed4c, ebp = 0xbfbfee18 --- Tracing command rpcbind pid 911 tid 100127 td 0xc7745000 sched_switch(c7745000,0,104,191,2bc606c6,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c7745000,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(f6162aa8,c0881aca,c73b4d50,0,c7745000,...) at sleepq_catch_signals+0xb7 sleepq_timedwait_sig(c73b4d64,0,f6162ad8,101,0,...) at sleepq_timedwait_sig+0x1a _cv_timedwait_sig(c73b4d64,c73b4d50,7531,603,f6162b8c,...) at _cv_timedwait_sig+0x250 seltdwait(f6162c5c,f6162c64,511,c7745000,f6162b5c,...) at seltdwait+0x8a poll(c7745000,f6162cf8,c,c0cc5009,c0d9071c,...) at poll+0x300 syscall(f6162d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (209, FreeBSD ELF32, poll), eip = 0x2813ed8f, esp = 0xbfbfcc0c, ebp = 0xbfbfedd8 --- Tracing command syslogd pid 877 tid 100123 td 0xc7745900 sched_switch(c7745900,0,104,191,f1b14627,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c7745900,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0881aca,c70d6890,0,c0ca12e8,c7745900,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c70d68a4,0,f6156a7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c70d68a4,c70d6890,c0ca93ea,603,c740ad90,...) at _cv_wait_sig+0x240 seltdwait(c740ad90,58,c6d7e380,c7745900,c0f3fc68,...) at seltdwait+0xa2 kern_select(c7745900,9,282290ac,0,0,0,20,0,8) at kern_select+0x4f4 select(c7745900,f6156cf8,14,c0c41990,c0d8fa6c,...) at select+0x66 syscall(f6156d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x28194e43, esp = 0xbfbfe2ac, ebp = 0xbfbfee18 --- Tracing command devd pid 695 tid 100120 td 0xc7747000 sched_switch(c7747000,0,104,191,bb1f3ce2,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c7747000,0,c0ca7177,1a0,0,...) at sleepq_switch+0x15f sleepq_catch_signals(c0881aca,c7371b50,0,c0ca12e8,c7747000,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c7371b64,0,f614da7c,101,0,...) at sleepq_wait_sig+0x17 _cv_wait_sig(c7371b64,c7371b50,c0ca93ea,603,c740a6c8,...) at _cv_wait_sig+0x240 seltdwait(c740a6c8,58,c6d7e380,c7747000,ebe,...) at seltdwait+0xa2 kern_select(c7747000,6,bfbfe9a0,0,0,0,20,80a24d8,1) at kern_select+0x4f4 select(c7747000,f614dcf8,14,c7747000,c0d8fa6c,...) at select+0x66 syscall(f614dd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (93, FreeBSD ELF32, select), eip = 0x8088ad3, esp = 0xbfbfe96c, ebp = 0xbfbfee48 --- Tracing command sh pid 20 tid 100076 td 0xc73b8b40 sched_switch(c73b8b40,0,104,191,ee4bf791,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c73b8b40,0,c0ca7177,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0ca7177,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c74077f8,5c,c0ca9984,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74077f8,c7407880,15c,c0ca9984,0,...) at _sleep+0x354 kern_wait(c73b8b40,ffffffff,f605ac74,2,0,...) at kern_wait+0xb76 wait4(c73b8b40,f605acf8,10,c0ca9963,c0d8f104,...) at wait4+0x3b syscall(f605ad38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815de9b, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command flowcleaner pid 19 tid 100073 td 0xc70ea900 sched_switch(c70ea900,0,104,191,79d36dac,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c70ea900,0,c0ca7177,283,c70ea900,...) at sleepq_switch+0x15f sleepq_timedwait(c0f6d1c8,0,f3e64cc4,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0f6d1c8,c0f6d1d0,2710,3f0,0,...) at _cv_timedwait+0x250 flowtable_cleaner(0,f3e64d38,c0c9e039,343,c7397550,...) at flowtable_cleaner+0x1bf fork_exit(c093d820,0,f3e64d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3e64d70, ebp = 0 --- Tracing command softdepflush pid 18 tid 100072 td 0xc70eab40 sched_switch(c70eab40,0,104,191,e3333165,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,44,...) at mi_switch+0x200 sleepq_switch(c70eab40,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f78920,44,c0cca2d5,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f78920,c0f788c4,44,c0cca2d5,3e8,...) at _sleep+0x339 softdep_flush(0,f3e61d38,c0c9e039,343,c73977f8,...) at softdep_flush+0x244 fork_exit(c0acda90,0,f3e61d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3e61d70, ebp = 0 --- Tracing command vnlru pid 17 tid 100071 td 0xc70ead80 sched_switch(c70ead80,0,104,191,e332a3eb,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,50,...) at mi_switch+0x200 sleepq_switch(c70ead80,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c7397aa0,50,c0cb1263,0,0,...) at sleepq_timedwait+0x6b _sleep(c7397aa0,c0f6cfd8,250,c0cb1263,3e8,...) at _sleep+0x339 vnlru_proc(0,f3e5ed38,c0c9e039,343,c7397aa0,...) at vnlru_proc+0xe7 fork_exit(c0929000,0,f3e5ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3e5ed70, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100070 td 0xc70ec000 sched_switch(c70ec000,0,104,191,e332668f,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,44,...) at mi_switch+0x200 sleepq_switch(c70ec000,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f6cd48,44,c0cae707,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f6cd48,c0f6cd4c,44,c0cae707,3e8,...) at _sleep+0x339 buf_daemon(0,f3e5bd38,c0c9e039,343,c7397d48,...) at buf_daemon+0x138 fork_exit(c0910720,0,f3e5bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3e5bd70, ebp = 0 --- Tracing command pagezero pid 15 tid 100069 td 0xc70ec240 sched_switch(c70ec240,0,104,191,46e72773,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c70ec240,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f79714,0,c0cd0222,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f79714,c0f79200,0,c0cd0222,493e0,...) at _sleep+0x339 vm_pagezero(0,f3e58d38,c0c9e039,343,c6d792a8,...) at vm_pagezero+0xdc fork_exit(c0b0da40,0,f3e58d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3e58d70, ebp = 0 --- Tracing command vmdaemon pid 9 tid 100068 td 0xc70ec480 sched_switch(c70ec480,0,104,191,46e1d3a4,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,68,...) at mi_switch+0x200 sleepq_switch(c70ec480,0,c0ca7177,260,0,...) at sleepq_switch+0x15f sleepq_wait(c0f79344,68,c0cae707,0,0,...) at sleepq_wait+0x63 _sleep(c0f79344,c0f79348,68,c0cae707,0,...) at _sleep+0x36b vm_daemon(0,f3e55d38,c0c9e039,343,c6d79550,...) at vm_daemon+0x59 fork_exit(c0b07ee0,0,f3e55d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3e55d70, ebp = 0 --- Tracing command pagedaemon pid 8 tid 100067 td 0xc70ec6c0 sched_switch(c70ec6c0,0,104,191,77e64448,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,44,...) at mi_switch+0x200 sleepq_switch(c70ec6c0,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f7930c,44,c0cae707,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f7930c,c0f79200,44,c0cae707,1388,...) at _sleep+0x339 vm_pageout(0,f3e52d38,c0c9e039,343,c6d797f8,...) at vm_pageout+0x2bb fork_exit(c0b08d80,0,f3e52d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3e52d70, ebp = 0 --- Tracing command fdc0 pid 7 tid 100064 td 0xc70ecd80 sched_switch(c70ecd80,0,104,191,b93e54e1,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c70ecd80,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6fc363c,4c,c0c984bf,0,0,...) at sleepq_timedwait+0x6b _sleep(c6fc363c,c6fc36f0,4c,c0c984bf,3e8,...) at _sleep+0x339 fdc_thread(c6fc3600,f3de9d38,c0c9e039,343,c6d79aa0,...) at fdc_thread+0x27d fork_exit(c0b8b070,c6fc3600,f3de9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3de9d70, ebp = 0 --- Tracing command fw0_probe pid 6 tid 100062 td 0xc6fbd480 sched_switch(c6fbd480,0,104,191,96c6d810,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c6fbd480,0,c0ca7177,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0ca7177,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c6ff7000,5c,c0c984bf,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6ff7000,c6ffb488,15c,c0c984bf,0,...) at _sleep+0x354 fw_bus_probe_thread(c6ff7000,f3dd9d38,c0c9e039,343,c6d79d48,...) at fw_bus_probe_thread+0xa08 fork_exit(c065bbd0,c6ff7000,f3dd9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dd9d70, ebp = 0 --- Tracing command usb pid 14 tid 100058 td 0xc6fbdd80 sched_switch(c6fbdd80,0,104,191,2c1b2e79,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbdd80,0,c0ca7177,260,c6fbdd80,...) at sleepq_switch+0x15f sleepq_wait(c6fd8d0c,0,f3dc2cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fd8d0c,c6fd8dac,c0c911af,6c,c6fd8d14,...) at _cv_wait+0x240 usb_process(c6fd8d04,f3dc2d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fd8d04,f3dc2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dc2d70, ebp = 0 --- Tracing command usb pid 14 tid 100057 td 0xc6fd0000 sched_switch(c6fd0000,0,104,191,670d7db8,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fd0000,0,c0ca7177,260,c6fd0000,...) at sleepq_switch+0x15f sleepq_wait(c6fd8cdc,0,f3dbfcbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fd8cdc,c6fd8dac,c0c911af,6c,c6fd8ce4,...) at _cv_wait+0x240 usb_process(c6fd8cd4,f3dbfd38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fd8cd4,f3dbfd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dbfd70, ebp = 0 --- Tracing command usb pid 14 tid 100056 td 0xc6fd0240 sched_switch(c6fd0240,0,104,191,2bb50225,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fd0240,0,c0ca7177,260,c6fd0240,...) at sleepq_switch+0x15f sleepq_wait(c6fd8cac,0,f3dbccbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fd8cac,c6fd8dac,c0c911af,6c,c6fd8cb4,...) at _cv_wait+0x240 usb_process(c6fd8ca4,f3dbcd38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fd8ca4,f3dbcd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dbcd70, ebp = 0 --- Tracing command usb pid 14 tid 100055 td 0xc6fd0480 sched_switch(c6fd0480,0,104,191,2bb4f10a,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fd0480,0,c0ca7177,260,c6fd0480,...) at sleepq_switch+0x15f sleepq_wait(c6fd8c7c,0,f3db9cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fd8c7c,c6fd8dac,c0c911af,6c,c6fd8c84,...) at _cv_wait+0x240 usb_process(c6fd8c74,f3db9d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fd8c74,f3db9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3db9d70, ebp = 0 --- Tracing command usb pid 14 tid 100053 td 0xc6fd0900 sched_switch(c6fd0900,0,104,191,2bb4dd4f,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fd0900,0,c0ca7177,260,c6fd0900,...) at sleepq_switch+0x15f sleepq_wait(c6fc9b5c,0,f3db1cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc9b5c,c6fc9bfc,c0c911af,6c,c6fc9b64,...) at _cv_wait+0x240 usb_process(c6fc9b54,f3db1d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc9b54,f3db1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3db1d70, ebp = 0 --- Tracing command usb pid 14 tid 100052 td 0xc6fd0b40 sched_switch(c6fd0b40,0,104,191,5f007e00,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fd0b40,0,c0ca7177,260,c6fd0b40,...) at sleepq_switch+0x15f sleepq_wait(c6fc9b2c,0,f3daecbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc9b2c,c6fc9bfc,c0c911af,6c,c6fc9b34,...) at _cv_wait+0x240 usb_process(c6fc9b24,f3daed38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc9b24,f3daed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3daed70, ebp = 0 --- Tracing command usb pid 14 tid 100051 td 0xc6fd0d80 sched_switch(c6fd0d80,0,104,191,2b517a5f,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fd0d80,0,c0ca7177,260,c6fd0d80,...) at sleepq_switch+0x15f sleepq_wait(c6fc9afc,0,f3dabcbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc9afc,c6fc9bfc,c0c911af,6c,c6fc9b04,...) at _cv_wait+0x240 usb_process(c6fc9af4,f3dabd38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc9af4,f3dabd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dabd70, ebp = 0 --- Tracing command usb pid 14 tid 100050 td 0xc6f6f6c0 sched_switch(c6f6f6c0,0,104,191,2b516800,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6f6c0,0,c0ca7177,260,c6f6f6c0,...) at sleepq_switch+0x15f sleepq_wait(c6fc9acc,0,f3da8cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc9acc,c6fc9bfc,c0c911af,6c,c6fc9ad4,...) at _cv_wait+0x240 usb_process(c6fc9ac4,f3da8d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc9ac4,f3da8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3da8d70, ebp = 0 --- Tracing command usb pid 14 tid 100049 td 0xc6f6f900 sched_switch(c6f6f900,0,104,191,2b51547b,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6f900,0,c0ca7177,260,c6f6f900,...) at sleepq_switch+0x15f sleepq_wait(c6fc5b5c,0,f3da4cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc5b5c,c6fc5bfc,c0c911af,6c,c6fc5b64,...) at _cv_wait+0x240 usb_process(c6fc5b54,f3da4d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc5b54,f3da4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3da4d70, ebp = 0 --- Tracing command usb pid 14 tid 100048 td 0xc6f6fb40 sched_switch(c6f6fb40,0,104,191,5eadd4b0,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6fb40,0,c0ca7177,260,c6f6fb40,...) at sleepq_switch+0x15f sleepq_wait(c6fc5b2c,0,f3da1cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc5b2c,c6fc5bfc,c0c911af,6c,c6fc5b34,...) at _cv_wait+0x240 usb_process(c6fc5b24,f3da1d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc5b24,f3da1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3da1d70, ebp = 0 --- Tracing command usb pid 14 tid 100047 td 0xc6f6fd80 sched_switch(c6f6fd80,0,104,191,2aed2af4,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6fd80,0,c0ca7177,260,c6f6fd80,...) at sleepq_switch+0x15f sleepq_wait(c6fc5afc,0,f3d9ecbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc5afc,c6fc5bfc,c0c911af,6c,c6fc5b04,...) at _cv_wait+0x240 usb_process(c6fc5af4,f3d9ed38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc5af4,f3d9ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d9ed70, ebp = 0 --- Tracing command usb pid 14 tid 100046 td 0xc6fbc000 sched_switch(c6fbc000,0,104,191,2aed16ce,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbc000,0,c0ca7177,260,c6fbc000,...) at sleepq_switch+0x15f sleepq_wait(c6fc5acc,0,f3d9bcbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fc5acc,c6fc5bfc,c0c911af,6c,c6fc5ad4,...) at _cv_wait+0x240 usb_process(c6fc5ac4,f3d9bd38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fc5ac4,f3d9bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d9bd70, ebp = 0 --- Tracing command usb pid 14 tid 100045 td 0xc6fbc240 sched_switch(c6fbc240,0,104,191,2aed02f2,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbc240,0,c0ca7177,260,c6fbc240,...) at sleepq_switch+0x15f sleepq_wait(c6fbeb5c,0,f3d97cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fbeb5c,c6fbebfc,c0c911af,6c,c6fbeb64,...) at _cv_wait+0x240 usb_process(c6fbeb54,f3d97d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fbeb54,f3d97d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d97d70, ebp = 0 --- Tracing command usb pid 14 tid 100044 td 0xc6fbc480 sched_switch(c6fbc480,0,104,191,5e5b80fc,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbc480,0,c0ca7177,260,c6fbc480,...) at sleepq_switch+0x15f sleepq_wait(c6fbeb2c,0,f3d94cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fbeb2c,c6fbebfc,c0c911af,6c,c6fbeb34,...) at _cv_wait+0x240 usb_process(c6fbeb24,f3d94d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fbeb24,f3d94d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d94d70, ebp = 0 --- Tracing command usb pid 14 tid 100043 td 0xc6fbc6c0 sched_switch(c6fbc6c0,0,104,191,2a890781,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbc6c0,0,c0ca7177,260,c6fbc6c0,...) at sleepq_switch+0x15f sleepq_wait(c6fbeafc,0,f3d91cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fbeafc,c6fbebfc,c0c911af,6c,c6fbeb04,...) at _cv_wait+0x240 usb_process(c6fbeaf4,f3d91d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fbeaf4,f3d91d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d91d70, ebp = 0 --- Tracing command usb pid 14 tid 100042 td 0xc6fbc900 sched_switch(c6fbc900,0,104,191,2a88f5f7,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbc900,0,c0ca7177,260,c6fbc900,...) at sleepq_switch+0x15f sleepq_wait(c6fbeacc,0,f3d8ecbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fbeacc,c6fbebfc,c0c911af,6c,c6fbead4,...) at _cv_wait+0x240 usb_process(c6fbeac4,f3d8ed38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fbeac4,f3d8ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d8ed70, ebp = 0 --- Tracing command usb pid 14 tid 100040 td 0xc6fbcd80 sched_switch(c6fbcd80,0,104,191,2a88e240,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbcd80,0,c0ca7177,260,c6fbcd80,...) at sleepq_switch+0x15f sleepq_wait(c6fb8b5c,0,f3d87cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fb8b5c,c6fb8bfc,c0c911af,6c,c6fb8b64,...) at _cv_wait+0x240 usb_process(c6fb8b54,f3d87d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fb8b54,f3d87d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d87d70, ebp = 0 --- Tracing command usb pid 14 tid 100039 td 0xc6fbd000 sched_switch(c6fbd000,0,104,191,5e095824,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbd000,0,c0ca7177,260,c6fbd000,...) at sleepq_switch+0x15f sleepq_wait(c6fb8b2c,0,f3d84cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fb8b2c,c6fb8bfc,c0c911af,6c,c6fb8b34,...) at _cv_wait+0x240 usb_process(c6fb8b24,f3d84d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fb8b24,f3d84d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d84d70, ebp = 0 --- Tracing command usb pid 14 tid 100038 td 0xc6f52d80 sched_switch(c6f52d80,0,104,191,2a244556,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f52d80,0,c0ca7177,260,c6f52d80,...) at sleepq_switch+0x15f sleepq_wait(c6fb8afc,0,f3d81cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fb8afc,c6fb8bfc,c0c911af,6c,c6fb8b04,...) at _cv_wait+0x240 usb_process(c6fb8af4,f3d81d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fb8af4,f3d81d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d81d70, ebp = 0 --- Tracing command usb pid 14 tid 100037 td 0xc6f6d000 sched_switch(c6f6d000,0,104,191,2a243373,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6d000,0,c0ca7177,260,c6f6d000,...) at sleepq_switch+0x15f sleepq_wait(c6fb8acc,0,f3d7ecbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6fb8acc,c6fb8bfc,c0c911af,6c,c6fb8ad4,...) at _cv_wait+0x240 usb_process(c6fb8ac4,f3d7ed38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6fb8ac4,f3d7ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d7ed70, ebp = 0 --- Tracing command usb pid 14 tid 100035 td 0xc6f6d480 sched_switch(c6f6d480,0,104,191,2a2420cb,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6d480,0,c0ca7177,260,c6f6d480,...) at sleepq_switch+0x15f sleepq_wait(c6faab5c,0,f3d77cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6faab5c,c6faabfc,c0c911af,6c,c6faab64,...) at _cv_wait+0x240 usb_process(c6faab54,f3d77d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6faab54,f3d77d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d77d70, ebp = 0 --- Tracing command usb pid 14 tid 100034 td 0xc6f6d6c0 sched_switch(c6f6d6c0,0,104,191,5db6d01b,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6d6c0,0,c0ca7177,260,c6f6d6c0,...) at sleepq_switch+0x15f sleepq_wait(c6faab2c,0,f3d74cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6faab2c,c6faabfc,c0c911af,6c,c6faab34,...) at _cv_wait+0x240 usb_process(c6faab24,f3d74d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6faab24,f3d74d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d74d70, ebp = 0 --- Tracing command usb pid 14 tid 100033 td 0xc6f6d900 sched_switch(c6f6d900,0,104,191,29b99e88,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6d900,0,c0ca7177,260,c6f6d900,...) at sleepq_switch+0x15f sleepq_wait(c6faaafc,0,f3d71cbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6faaafc,c6faabfc,c0c911af,6c,c6faab04,...) at _cv_wait+0x240 usb_process(c6faaaf4,f3d71d38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6faaaf4,f3d71d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d71d70, ebp = 0 --- Tracing command usb pid 14 tid 100032 td 0xc6f6db40 sched_switch(c6f6db40,0,104,191,29b98bf8,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f6db40,0,c0ca7177,260,c6f6db40,...) at sleepq_switch+0x15f sleepq_wait(c6faaacc,0,f3d6ecbc,1,0,...) at sleepq_wait+0x63 _cv_wait(c6faaacc,c6faabfc,c0c911af,6c,c6faaad4,...) at _cv_wait+0x240 usb_process(c6faaac4,f3d6ed38,c0c9e039,343,c6f48000,...) at usb_process+0x193 fork_exit(c07c2200,c6faaac4,f3d6ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d6ed70, ebp = 0 --- Tracing command xpt_thrd pid 5 tid 100023 td 0xc6f52000 sched_switch(c6f52000,0,104,191,29b97302,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c6f52000,0,c0ca7177,260,0,...) at sleepq_switch+0x15f sleepq_wait(c0dcccd4,4c,c0c3cb3a,0,0,...) at sleepq_wait+0x63 _sleep(c0dcccd4,c0dcccec,4c,c0c3cb3a,0,...) at _sleep+0x36b xpt_scanner_thread(0,c6a67d38,c0c9e039,343,c6f482a8,...) at xpt_scanner_thread+0x4a fork_exit(c04848d0,0,c6a67d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a67d70, ebp = 0 --- Tracing command yarrow pid 13 tid 100017 td 0xc6d7c240 sched_switch(c6d7c240,0,104,191,ef6dad2c,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6d7c240,0,c0ca7177,283,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0e00944,0,c0c984bf,2,0,...) at sleepq_timedwait+0x6b _sleep(c0e00944,0,0,c0c984bf,64,...) at _sleep+0x339 pause(c0c984bf,64,c0c84eaa,111,0,...) at pause+0x47 random_kthread(0,c6a55d38,c0c9e039,343,c6f48550,...) at random_kthread+0x1ef fork_exit(c0738d10,0,c6a55d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a55d70, ebp = 0 --- Tracing command g_down pid 4 tid 100015 td 0xc6d7c6c0 sched_switch(c6d7c6c0,0,104,191,f6ce37c1,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c6d7c6c0,0,c0ca7177,260,0,...) at sleepq_switch+0x15f sleepq_wait(c0dfe6e4,4c,c0c984bf,0,0,...) at sleepq_wait+0x63 _sleep(c0dfe6e4,c0dfe648,24c,c0c984bf,0,...) at _sleep+0x36b g_io_schedule_down(c6d7c6c0,0,c0c99b87,74,0,...) at g_io_schedule_down+0x56 g_down_procbody(0,c6a4fd38,c0c9e039,343,c6d78000,...) at g_down_procbody+0x8d fork_exit(c082e720,0,c6a4fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a4fd70, ebp = 0 --- Tracing command g_up pid 3 tid 100014 td 0xc6d7c900 cpustop_handler(4,c6a4ca3c,c0bcf9c6,2,c0ca3f26,...) at cpustop_handler+0x32 ipi_nmi_handler(2,c0ca3f26,111,2,c6d782a8,...) at ipi_nmi_handler+0x2f trap(c6a4ca48) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0bc1b23, esp = 0xc6a4ca88, ebp = 0xc6a4caa8 --- smp_tlb_shootdown(c6a4cad0,c0bc650f,e7a97000,e7a98000,1,...) at smp_tlb_shootdown+0x93 smp_invlpg_range(e7a97000,e7a98000,1,c6a4cae4,e7a98000,...) at smp_invlpg_range+0x1c pmap_invalidate_range(c0fcc2e0,e7a97000,e7a98000) at pmap_invalidate_range+0x4f pmap_qremove(e7a97000,1,c14ba888,c14ba888,9e3,...) at pmap_qremove+0x54 allocbuf(e6d30c60,0,c6a4cb94,c087e2cf,c0db02f8,...) at allocbuf+0x20d brelse(e6d30c60,0,c0cc9596,1114,c6d7c900,...) at brelse+0x172 bufdone_finish(e6d30c60,e6d30c60,e6d30c60,e6d09fa0,c6a4cc68,...) at bufdone_finish+0x3f2 bufdone(e6d30c60,0,c0cca9dd,6fd,c74febd8,...) at bufdone+0x6a ffs_backgroundwritedone(e6d30c60,c149e000,e6d30c60,c7b2fa18,c6a4cc98,...) at ffs_backgroundwritedone+0xcc bufdone(e6d30c60,c6d45484) at bufdone+0x53 g_vfs_done(c7b2fa18,0,c0caddc3,c16,c7b2fa18,...) at g_vfs_done+0x85 biodone(c7b2fa18,c0dfe668,24c,c0c984bf,0,...) at biodone+0xa5 g_io_schedule_up(c6d7c900,0,c0c99b87,5d,0,...) at g_io_schedule_up+0xc7 g_up_procbody(0,c6a4cd38,c0c9e039,343,c6d782a8,...) at g_up_procbody+0x8d fork_exit(c082e7b0,0,c6a4cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a4cd70, ebp = 0 --- Tracing command g_event pid 2 tid 100013 td 0xc6d7cb40 sched_switch(c6d7cb40,0,104,191,f45e6adf,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,4c,...) at mi_switch+0x200 sleepq_switch(c6d7cb40,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0dfe6d8,4c,c0c984bf,0,0,...) at sleepq_timedwait+0x6b _sleep(c0dfe6d8,0,4c,c0c984bf,64,...) at _sleep+0x339 g_event_procbody(0,c6a49d38,c0c9e039,343,c6d78550,...) at g_event_procbody+0xcb fork_exit(c082e840,0,c6a49d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a49d70, ebp = 0 --- Tracing command intr pid 12 tid 100066 td 0xc70ec900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100065 td 0xc70ecb40 sched_switch(c70ecb40,0,109,191,29b62543,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7e2f0,...) at mi_switch+0x200 ithread_loop(c70e39a0,f3decd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c70e39a0,f3decd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3decd70, ebp = 0 --- Tracing command intr pid 12 tid 100063 td 0xc6fbd240 sched_switch(c6fbd240,0,109,191,f03f0b7d,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c70dd870,...) at mi_switch+0x200 ithread_loop(c70e38a0,f3ddcd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c70e38a0,f3ddcd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ddcd70, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xc6fbd900 sched_switch(c6fbd900,0,109,191,299d4f29,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d870,...) at mi_switch+0x200 ithread_loop(c6feebf0,f3dd0d38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6feebf0,f3dd0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dd0d70, ebp = 0 --- Tracing command intr pid 12 tid 100059 td 0xc6fbdb40 sched_switch(c6fbdb40,0,109,191,996ecf92,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7db70,...) at mi_switch+0x200 ithread_loop(c6fee450,f3dc8d38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6fee450,f3dc8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dc8d70, ebp = 0 --- Tracing command intr pid 12 tid 100054 td 0xc6fd06c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100041 td 0xc6fbcb40 sched_switch(c6fbcb40,0,109,191,4f3fbf48,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d970,...) at mi_switch+0x200 ithread_loop(c6fa2710,f3d8bd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6fa2710,f3d8bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d8bd70, ebp = 0 --- Tracing command intr pid 12 tid 100036 td 0xc6f6d240 sched_switch(c6f6d240,0,109,191,4e8e6720,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d9f0,...) at mi_switch+0x200 ithread_loop(c6fa2a30,f3d7bd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6fa2a30,f3d7bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d7bd70, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xc6f6dd80 sched_switch(c6f6dd80,0,109,191,a5e95c37,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7da70,...) at mi_switch+0x200 ithread_loop(c6faf110,f3d6bd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6faf110,f3d6bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3d6bd70, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc6f6f000 sched_switch(c6f6f000,0,109,191,fb4955a2,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d770,...) at mi_switch+0x200 ithread_loop(c6fa2500,c6badd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6fa2500,c6badd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6badd70, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc6f6f240 sched_switch(c6f6f240,0,109,191,9e97cfcb,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6f58570,...) at mi_switch+0x200 ithread_loop(c6f8e8e0,c6aa7d38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6f8e8e0,c6aa7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa7d70, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc6f6f480 sched_switch(c6f6f480,0,109,191,29b6ca3d,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7ddf0,...) at mi_switch+0x200 ithread_loop(c6d77600,c6a9dd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d77600,c6a9dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a9dd70, ebp = 0 --- Tracing command intr pid 12 tid 100024 td 0xc6dc0d80 sched_switch(c6dc0d80,0,109,191,4566af97,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7e870,...) at mi_switch+0x200 ithread_loop(c6d773c0,c6a6ad38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d773c0,c6a6ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a6ad70, ebp = 0 --- Tracing command intr pid 12 tid 100022 td 0xc6f52240 sched_switch(c6f52240,0,109,191,b7a3a357,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7e9f0,...) at mi_switch+0x200 ithread_loop(c6d0d9f0,c6a64d38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d0d9f0,c6a64d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a64d70, ebp = 0 --- Tracing command intr pid 12 tid 100021 td 0xc6f52480 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100019 td 0xc6f52900 sched_switch(c6f52900,0,109,191,f3335cd5,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7eaf0,...) at mi_switch+0x200 ithread_loop(c6d0da20,c6a5bd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d0da20,c6a5bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a5bd70, ebp = 0 --- Tracing command intr pid 12 tid 100012 td 0xc6d7cd80 sched_switch(c6d7cd80,0,109,191,e5035240,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d070,...) at mi_switch+0x200 ithread_loop(c6d771b0,c6a46d38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d771b0,c6a46d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a46d70, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xc6dc0000 sched_switch(c6dc0000,0,109,191,ef6d5398,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d0f0,...) at mi_switch+0x200 ithread_loop(c6d771c0,c6a43d38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d771c0,c6a43d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a43d70, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc6dc0240 sched_switch(c6dc0240,0,109,191,5e5088f8,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d170,...) at mi_switch+0x200 ithread_loop(c6d771d0,c6a40d38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d771d0,c6a40d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a40d70, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc6dc0480 cpustop_handler(1,c6a3db6c,c0bcf9c6,c089e4b8,df164400,...) at cpustop_handler+0x32 ipi_nmi_handler(c089e4b8,df164400,7ffee05f,369e99,c6d787f8,...) at ipi_nmi_handler+0x2f trap(c6a3db78) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0881cfc, esp = 0xc6a3dbb8, ebp = 0xc6a3dbd4 --- _mtx_lock_sleep(c0e00290,c6dc0480,0,c0ca12e8,93,...) at _mtx_lock_sleep+0x9c _mtx_lock_flags(c0e00290,0,c0ca12e8,93,c6a3dca0,...) at _mtx_lock_flags+0xf7 lock_mtx(c0e00290,1,c0ca3f26,189,4b4,...) at lock_mtx+0x29 softclock(c0e00b20,c6a3dcc8,c0881714,c0e044c0,c6d7d538,...) at softclock+0x1e1 intr_event_execute_handlers(c6d787f8,c6d7d500,c0c9e2b8,533,c6d7d570,...) at intr_event_execute_handlers+0x125 ithread_loop(c6d771e0,c6a3dd38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x9f fork_exit(c086a6c0,c6d771e0,c6a3dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a3dd70, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc6d7a000 sched_switch(c6d7a000,0,109,191,2ce6f4cf,...) at sched_switch+0x406 mi_switch(109,0,c0c9e2b8,52d,c6d7d5f0,...) at mi_switch+0x200 ithread_loop(c6d771f0,c6a3ad38,c0c9e039,343,c6d787f8,...) at ithread_loop+0x1f6 fork_exit(c086a6c0,c6d771f0,c6a3ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a3ad70, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc6d7a240 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc6d7a480 sched_switch(c6d7a480,0,108,18c,f984c55e,...) at sched_switch+0x406 mi_switch(108,0,c0ca475c,a07,c6d7a480,...) at mi_switch+0x200 sched_idletd(0,c6a33d38,c0c9e039,343,c6d78aa0,...) at sched_idletd+0x285 fork_exit(c08b6ed0,0,c6a33d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a33d70, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc6d7a6c0 sched_switch(c6d7a6c0,0,60c,18c,f6d84f4b,...) at sched_switch+0x406 mi_switch(60c,0,c0ca475c,815,1,...) at mi_switch+0x200 sched_preempt(c6d7a6c0,1,c73b9b40,c6a30cb4,c0bb214e,...) at sched_preempt+0x9f ipi_bitmap_handler(8,28,28,7,c0e04b40,...) at ipi_bitmap_handler+0x34 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0882597, esp = 0xc6a30c8c, ebp = 0xc6a30cb4 --- _thread_lock_flags(c6d7a6c0,0,c0ca475c,a07,c6d7a6c0,...) at _thread_lock_flags+0x177 sched_idletd(0,c6a30d38,c0c9e039,343,c6d78aa0,...) at sched_idletd+0x271 fork_exit(c08b6ed0,0,c6a30d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a30d70, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc6d7a900 sched_switch(c6d7a900,0,60c,18c,fb38dd64,...) at sched_switch+0x406 mi_switch(60c,0,c0ca475c,815,2,...) at mi_switch+0x200 sched_preempt(c6d7a900,2,1f4,c6a2dc6c,c0bb214e,...) at sched_preempt+0x9f ipi_bitmap_handler(8,28,28,c6fc3c18,c6fc3c00,...) at ipi_bitmap_handler+0x34 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0ba6245, esp = 0xc6a2dc6c, ebp = 0xc6a2dc6c --- acpi_cpu_c1(c0bb203f,c6a2dcd4,2,ffffffff,c6a2dcb4,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(2710,c0e051c0,c6a2dcb4,c0bbd9ab,1,...) at acpi_cpu_idle+0x11c cpu_idle_amdc1e(1,c6a2dcf8,c08b710e,1,c6a2dcd4,...) at cpu_idle_amdc1e+0x56 cpu_idle(1,c6a2dcd4,c0ca475c,a09,c6d7a900,...) at cpu_idle+0x1b sched_idletd(0,c6a2dd38,c0c9e039,343,c6d78aa0,...) at sched_idletd+0x23e fork_exit(c08b6ed0,0,c6a2dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a2dd70, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc6d7ab40 cpustop_handler(8,c6a2ac70,c0bcf9c6,c0c9c73b,199,...) at cpustop_handler+0x32 ipi_nmi_handler(c0c9c73b,199,c084e170,0,c6d78aa0,...) at ipi_nmi_handler+0x2f trap(c6a2ac7c) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc08b70e8, esp = 0xc6a2acbc, ebp = 0xc6a2acf8 --- sched_idletd(0,c6a2ad38,c0c9e039,343,c6d78aa0,...) at sched_idletd+0x218 fork_exit(c08b6ed0,0,c6a2ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a2ad70, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc6d7ad80 sched_switch(c6d7ad80,0,104,191,eab67d80,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,5c,...) at mi_switch+0x200 sleepq_switch(c6d7ad80,0,c0ca7177,1a0,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0ca7177,160,0,100,100,...) at sleepq_catch_signals+0xb7 sleepq_wait_sig(c6d78d48,5c,c0ca9984,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6d78d48,c6d78dd0,15c,c0ca9984,0,...) at _sleep+0x354 kern_wait(c6d7ad80,ffffffff,c6a26c74,2,0,...) at kern_wait+0xb76 wait4(c6d7ad80,c6a26cf8,10,c6d7ad80,c0d8f104,...) at wait4+0x3b syscall(c6a26d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x8054eaf, esp = 0xbfbfe74c, ebp = 0xbfbfe768 --- Tracing command audit pid 10 tid 100001 td 0xc6d7c000 sched_switch(c6d7c000,0,104,191,29b7358a,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6d7c000,0,c0ca7177,260,c6d7c000,...) at sleepq_switch+0x15f sleepq_wait(c0f78240,0,c6a23c9c,1,0,...) at sleepq_wait+0x63 _cv_wait(c0f78240,c0f78224,c0cc74f7,194,0,...) at _cv_wait+0x240 audit_worker(0,c6a23d38,c0c9e039,343,c6d79000,...) at audit_worker+0x84 fork_exit(c0a9cae0,0,c6a23d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a23d70, ebp = 0 --- Tracing command kernel pid 0 tid 100096 td 0xc74a6240 sched_switch(c74a6240,0,104,191,edd1ae1b,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c74a6240,0,c0ca7177,283,c74a6240,...) at sleepq_switch+0x15f sleepq_timedwait(c7617bbc,0,f60bcc84,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c7617bbc,c76178e8,3e8,6b1,4e20,...) at _cv_timedwait+0x250 sched_sync(c76178e8,f60bcd38,c0c9e039,343,c0dfe7c0,...) at sched_sync+0x521 fork_exit(c0928390,c76178e8,f60bcd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf60bcd70, ebp = 0 --- Tracing command kernel pid 0 tid 100095 td 0xc74a6480 sched_switch(c74a6480,0,104,191,e9f4bef3,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c74a6480,0,c0ca7177,283,c74a6480,...) at sleepq_switch+0x15f sleepq_timedwait(c7617eb4,0,f60b9c84,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c7617eb4,c7617be0,3e8,68e,4e20,...) at _cv_timedwait+0x250 sched_sync(c7617be0,f60b9d38,c0c9e039,343,c0dfe7c0,...) at sched_sync+0x521 fork_exit(c0928390,c7617be0,f60b9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf60b9d70, ebp = 0 --- Tracing command kernel pid 0 tid 100094 td 0xc74a66c0 sched_switch(c74a66c0,0,104,191,e8067022,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c74a66c0,0,c0ca7177,283,c74a66c0,...) at sleepq_switch+0x15f sleepq_timedwait(c76182d4,0,f60b6c84,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c76182d4,c7618000,3e8,6b1,4e20,...) at _cv_timedwait+0x250 sched_sync(c7618000,f60b6d38,c0c9e039,343,c0dfe7c0,...) at sched_sync+0x521 fork_exit(c0928390,c7618000,f60b6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf60b6d70, ebp = 0 --- Tracing command kernel pid 0 tid 100075 td 0xc73b8d80 sched_switch(c73b8d80,0,104,191,e5035d1d,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c73b8d80,0,c0ca7177,283,c73b8d80,...) at sleepq_switch+0x15f sleepq_timedwait(c73adeb4,0,f6056c84,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c73adeb4,c73adbe0,3e8,6b1,4e20,...) at _cv_timedwait+0x250 sched_sync(c73adbe0,f6056d38,c0c9e039,343,c0dfe7c0,...) at sched_sync+0x521 fork_exit(c0928390,c73adbe0,f6056d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf6056d70, ebp = 0 --- Tracing command kernel pid 0 tid 100074 td 0xc73b9000 sched_switch(c73b9000,0,104,191,e4975e95,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c73b9000,0,c0ca7177,283,c73b9000,...) at sleepq_switch+0x15f sleepq_timedwait(c73ae2d4,0,f6053c84,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c73ae2d4,c73ae000,3e8,6b1,4e20,...) at _cv_timedwait+0x250 sched_sync(c73ae000,f6053d38,c0c9e039,343,c0dfe7c0,...) at sched_sync+0x521 fork_exit(c0928390,c73ae000,f6053d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf6053d70, ebp = 0 --- Tracing command kernel pid 0 tid 100061 td 0xc6fbd6c0 sched_switch(c6fbd6c0,0,104,191,299d8b7d,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6fbd6c0,0,c0ca7177,260,c6fbd6c0,...) at sleepq_switch+0x15f sleepq_wait(c6fa4940,0,c0ca3755,c0c984bf,0,...) at sleepq_wait+0x63 msleep_spin(c6fa4940,c6fa4958,c0c984bf,0,c0ca12e8,...) at msleep_spin+0x21d taskqueue_thread_loop(c6ffb49c,f3dd6d38,c0c9e039,343,c0dfe7c0,...) at taskqueue_thread_loop+0x94 fork_exit(c08d08c0,c6ffb49c,f3dd6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3dd6d70, ebp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xc6dc06c0 sched_switch(c6dc06c0,0,104,191,96d18613,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6dc06c0,0,c0ca7177,260,c6dc06c0,...) at sleepq_switch+0x15f sleepq_wait(c6f46080,0,c0ca3755,c0c984bf,0,...) at sleepq_wait+0x63 msleep_spin(c6f46080,c6f46098,c0c984bf,0,c0ca12e8,...) at msleep_spin+0x21d taskqueue_thread_loop(c0dcfb20,c6a73d38,c0c9e039,343,c0dfe7c0,...) at taskqueue_thread_loop+0x94 fork_exit(c08d08c0,c0dcfb20,c6a73d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a73d70, ebp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xc6dc0900 sched_switch(c6dc0900,0,104,191,96d177de,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6dc0900,0,c0ca7177,260,c6dc0900,...) at sleepq_switch+0x15f sleepq_wait(c6f46080,0,c0ca3755,c0c984bf,0,...) at sleepq_wait+0x63 msleep_spin(c6f46080,c6f46098,c0c984bf,0,c0ca12e8,...) at msleep_spin+0x21d taskqueue_thread_loop(c0dcfb20,c6a70d38,c0c9e039,343,c0dfe7c0,...) at taskqueue_thread_loop+0x94 fork_exit(c08d08c0,c0dcfb20,c6a70d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a70d70, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc6dc0b40 sched_switch(c6dc0b40,0,104,191,96d16816,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6dc0b40,0,c0ca7177,260,c6dc0b40,...) at sleepq_switch+0x15f sleepq_wait(c6f46080,0,c0ca3755,c0c984bf,0,...) at sleepq_wait+0x63 msleep_spin(c6f46080,c6f46098,c0c984bf,0,c0ca12e8,...) at msleep_spin+0x21d taskqueue_thread_loop(c0dcfb20,c6a6dd38,c0c9e039,343,c0dfe7c0,...) at taskqueue_thread_loop+0x94 fork_exit(c08d08c0,c0dcfb20,c6a6dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a6dd70, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xc6f526c0 sched_switch(c6f526c0,0,104,191,45687d9e,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f526c0,0,c0ca7177,260,0,...) at sleepq_switch+0x15f sleepq_wait(c6f46300,0,c0c984bf,0,0,...) at sleepq_wait+0x63 _sleep(c6f46300,c6f46318,0,c0c984bf,0,...) at _sleep+0x36b taskqueue_thread_loop(c0e12f48,c6a5ed38,c0c9e039,343,c0dfe7c0,...) at taskqueue_thread_loop+0xba fork_exit(c08d08c0,c0e12f48,c6a5ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a5ed70, ebp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xc6f52b40 sched_switch(c6f52b40,0,104,191,96c71a1f,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6f52b40,0,c0ca7177,260,0,...) at sleepq_switch+0x15f sleepq_wait(c6f46580,0,c0c984bf,0,0,...) at sleepq_wait+0x63 _sleep(c6f46580,c6f46598,0,c0c984bf,0,...) at _sleep+0x36b taskqueue_thread_loop(c0dff058,c6a58d38,c0c9e039,343,c0dfe7c0,...) at taskqueue_thread_loop+0xba fork_exit(c08d08c0,c0dff058,c6a58d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a58d70, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc6d7c480 sched_switch(c6d7c480,0,104,191,4b9ac0c8,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,0,...) at mi_switch+0x200 sleepq_switch(c6d7c480,0,c0ca7177,260,0,...) at sleepq_switch+0x15f sleepq_wait(c6d5fe40,0,c0c984bf,0,0,...) at sleepq_wait+0x63 _sleep(c6d5fe40,c6d5fe58,0,c0c984bf,0,...) at _sleep+0x36b taskqueue_thread_loop(c0e119e0,c6a52d38,c0c9e039,343,c0dfe7c0,...) at taskqueue_thread_loop+0xba fork_exit(c08d08c0,c0e119e0,c6a52d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a52d70, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc0dfea70 sched_switch(c0dfea70,0,104,191,794d31ba,...) at sched_switch+0x406 mi_switch(104,0,c0ca7177,1eb,44,...) at mi_switch+0x200 sleepq_switch(c0dfea70,0,c0ca7177,283,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0dfe7c0,44,c0ca4fd1,0,0,...) at sleepq_timedwait+0x6b _sleep(c0dfe7c0,0,44,c0ca4fd1,2710,...) at _sleep+0x339 scheduler(0,141ec00,141ec00,141e000,1425000,...) at scheduler+0x23e mi_startup() at mi_startup+0x96 begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3439 MB Dumping 112 MB: 97 81 65 49 33 17 1 Dump complete = 0xf db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 1 (kgdb) bt #0 doadump () at pcpu.h:246 #1 0xc04d0c99 in db_fncall (dummy1=0xc08c3faa, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf607677c "\220g\aö") at ../../../ddb/db_command.c:548 #2 0xc04d10cf in db_command (last_cmdp=0xc0dcdcfc, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #3 0xc04d1184 in db_command_script (command=0xc0dcec08 "call doadump") at ../../../ddb/db_command.c:516 #4 0xc04d5350 in db_script_exec (scriptname=0xc0dce560 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc04d53e1 in db_run_cmd (addr=0x1, have_addr=0x0, count=0xc0fcd060, modif=0xf60768b4 "") at ../../../ddb/db_script.c:375 #6 0xc04d1091 in db_command (last_cmdp=0xc0dcdcfc, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:445 #7 0xc04d11ea in db_command_loop () at ../../../ddb/db_command.c:498 #8 0xc04d308d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:229 #9 0xc08c3e26 in kdb_trap (type=0x3, code=0x0, tf=0xf6076a5c) at ../../../kern/subr_kdb.c:535 #10 0xc0bcffbb in trap (frame=0xf6076a5c) at ../../../i386/i386/trap.c:690 #11 0xc0bb1a5b in calltrap () at ../../../i386/i386/exception.s:165 #12 0xc08c3faa in kdb_enter (why=0xc0ca2b60 "panic", msg=0xc0ca2b60 "panic") at cpufunc.h:71 #13 0xc08914a6 in panic (fmt=0xc0c3c66a "Bad link elm %p prev->next != elm") at ../../../kern/kern_shutdown.c:562 #14 0xc0922a19 in reassignbuf (bp=0xe6d1d2c0) at ../../../kern/vfs_subr.c:1987 #15 0xc090c9ac in bundirty (bp=0xe6d1d2c0) at ../../../kern/vfs_bio.c:1101 #16 0xc090f6fa in bufwrite (bp=0xe6d1d2c0) at ../../../kern/vfs_bio.c:866 #17 0xc090fbac in bawrite (bp=0xe6d1d2c0) at buf.h:385 #18 0xc0ad788f in ffs_syncvnode (vp=0xc74fe15c, waitfor=0x1) at ../../../ufs/ffs/ffs_vnops.c:285 #19 0xc0ad322f in ffs_sync (mp=0xc76125f0, waitfor=0x1) at ../../../ufs/ffs/ffs_vfsops.c:1358 #20 0xc091ec10 in dounmount (mp=0xc76125f0, flags=0x8000000, td=0xc70ea6c0) at ../../../kern/vfs_mount.c:1312 #21 0xc091f1df in unmount (td=0xc70ea6c0, uap=0xf6076cf8) at ../../../kern/vfs_mount.c:1180 #22 0xc0bcf744 in syscall (frame=0xf6076d38) at ../../../i386/i386/trap.c:1076 #23 0xc0bb1ac0 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261 #24 0x00000033 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) f 14 #14 0xc0922a19 in reassignbuf (bp=0xe6d1d2c0) at ../../../kern/vfs_subr.c:1987 1987 LIST_REMOVE(bo, bo_synclist); (kgdb) l 1982 buf_vlist_add(bp, bo, BX_VNDIRTY); 1983 } else { 1984 buf_vlist_add(bp, bo, BX_VNCLEAN); 1985 1986 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { 1987 LIST_REMOVE(bo, bo_synclist); 1988 mp->mnt_sync_worklist_len--; 1989 bo->bo_flag &= ~BO_ONWORKLST; 1990 } 1991 } (kgdb) info loc mp = (struct mount *) 0xc76125f0 vp = (struct vnode *) 0xc74fe15c bo = (struct bufobj *) 0xc74fe254 delay = Variable "delay" is not available. (kgdb) p *mp $1 = {mnt_mtx = {lock_object = {lo_name = 0xc0ca921c "struct mount mtx", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d2c2b0}, mtx_lock = 0xc70ea6c0}, mnt_gen = 0x1, mnt_list = {tqe_next = 0xc7618000, tqe_prev = 0xc73ae018}, mnt_op = 0xc0db0da0, mnt_vfc = 0xc0db0c00, mnt_vnodecovered = 0xc74fe570, mnt_syncer = 0x0, mnt_ref = 0xb, mnt_nvnodelist = {tqh_first = 0xc74fe15c, tqh_last = 0xc794e584}, mnt_nvnodelistsize = 0x9, mnt_writeopcount = 0x1, mnt_kern_flag = 0x6100004c, mnt_flag = 0x201100, mnt_xflag = 0x0, mnt_noasync = 0x1, mnt_opt = 0xc73bc9c0, mnt_optnew = 0x0, mnt_maxsymlinklen = 0x78, mnt_stat = {f_version = 0x20030518, f_type = 0x3, f_flags = 0x201100, f_bsize = 0x800, f_iosize = 0x4000, f_blocks = 0x306cbf3, f_bfree = 0x2f7e3f2, f_bavail = 0x2b9e81c, f_files = 0xc887fe, f_ffree = 0xc7f129, f_syncwrites = 0x0, f_asyncwrites = 0x0, f_syncreads = 0x0, f_asyncreads = 0x0, f_spare = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, f_namemax = 0xff, f_owner = 0x0, f_fsid = {val = {0x48f38e3c, 0xff1ca3a5}}, f_charspare = '\0' , f_fstypename = "ufs", '\0' , f_mntfromname = "/dev/ad4s1e", '\0' , f_mntonname = "/tmp", '\0' }, mnt_cred = 0xc73c2500, mnt_data = 0xc7378a00, mnt_time = 0x0, mnt_iosize_max = 0x20000, mnt_export = 0xc7645000, mnt_label = 0x0, mnt_hashseed = 0x29ba2e7a, mnt_lockref = 0x0, mnt_secondary_writes = 0x0, mnt_secondary_accwrites = 0x1e, mnt_susp_owner = 0x0, mnt_gjprovider = 0x0, mnt_explock = {lock_object = {lo_name = 0xc0cafa83 "explock", lo_flags = 0x5730000, lo_data = 0x0, lo_witness = 0xc6d32638}, lk_lock = 0x1, lk_exslpfail = 0x0, lk_timo = 0x0, lk_pri = 0x50, lk_stack = {depth = 0x6, pcs = { 0xc087a34e, 0xc0917597, 0xc0920281, 0xc0921975, 0xc0bcf744, 0xc0bb1ac0, 0x0 }}}, mnt_sync_wakeup = {cv_description = 0xc0cb0d85 "syncer", cv_waiters = 0x0}, mnt_sync_thr = 0x0, mnt_sync_workitem_pending = 0xc73c2480, mnt_sync_delayno = 0xa, mnt_sync_vnode_count = 0x0, mnt_sync_worklist_len = 0x1, mnt_sync_rushjob = 0x0, mnt_sync_state = SYNCER_DIED} (kgdb) p *vp $2 = {v_type = VDIR, v_tag = 0xc0c9b70c "ufs", v_op = 0xc0db0fc0, v_data = 0xc73fa910, v_mount = 0xc76125f0, v_nmntvnodes = {tqe_next = 0xc742ba00, tqe_prev = 0xc7612624}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0, vu_yield = 0x0}, v_hashlist = {le_next = 0x0, le_prev = 0xc71129f0}, v_hash = 0x2, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0x0, tqh_last = 0xc74fe18c}, v_cache_dd = 0x0, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0, v_clen = 0x0, v_lock = {lock_object = {lo_name = 0xc0c9b70c "ufs", lo_flags = 0x57b0000, lo_data = 0x0, lo_witness = 0xc6d32840}, lk_lock = 0xc70ea6c0, lk_exslpfail = 0x0, lk_timo = 0x33, lk_pri = 0x50, lk_stack = {depth = 0xa, pcs = {0xc087a34e, 0xc0ad7c31, 0xc0be8595, 0xc0933fd8, 0xc09271bb, 0xc0ad31d6, 0xc091ec10, 0xc091f1df, 0xc0bcf744, 0xc0bb1ac0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}}, v_interlock = { lock_object = {lo_name = 0xc0ca920c "vnode interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d2c178}, mtx_lock = 0x4}, v_vnlock = 0xc74fe1b4, v_holdcnt = 0x5, v_usecount = 0x1, v_iflag = 0x0, v_vflag = 0x1, v_writecount = 0x0, v_freelist = {tqe_next = 0x0, tqe_prev = 0xc7497d2c}, v_bufobj = {bo_mtx = {lock_object = {lo_name = 0xc0cb1238 "bufobj interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d31048}, mtx_lock = 0xc70ea6c0}, bo_clean = {bv_hd = {tqh_first = 0xe6d1d2c0, tqh_last = 0xe6d1d638}, bv_root = 0xe6d1d2c0, bv_cnt = 0x3}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc74fe278}, bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x1, bo_ops = 0xc0d9f880, bo_bsize = 0x4000, bo_object = 0xc787fa18, bo_synclist = {le_next = 0x0, le_prev = 0xc73c249c}, bo_private = 0xc74fe15c, __bo_vnode = 0xc74fe15c}, v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0} (kgdb) p *bo $3 = {bo_mtx = {lock_object = {lo_name = 0xc0cb1238 "bufobj interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xc6d31048}, mtx_lock = 0xc70ea6c0}, bo_clean = {bv_hd = {tqh_first = 0xe6d1d2c0, tqh_last = 0xe6d1d638}, bv_root = 0xe6d1d2c0, bv_cnt = 0x3}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xc74fe278}, bv_root = 0x0, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x1, bo_ops = 0xc0d9f880, bo_bsize = 0x4000, bo_object = 0xc787fa18, bo_synclist = {le_next = 0x0, le_prev = 0xc73c249c}, bo_private = 0xc74fe15c, __bo_vnode = 0xc74fe15c} (kgdb) $ svn diff -x -p /usr/src/sys Index: /usr/src/sys/kern/vfs_mount.c =================================================================== --- /usr/src/sys/kern/vfs_mount.c (revision 202153) +++ /usr/src/sys/kern/vfs_mount.c (working copy) @@ -505,6 +505,7 @@ vfs_mount_alloc(struct vnode *vp, struct vfsconf * mac_mount_create(cred, mp); #endif arc4rand(&mp->mnt_hashseed, sizeof mp->mnt_hashseed, 0); + vfs_syncer_attach(mp, vfsp->vfc_name); return (mp); } @@ -1253,6 +1254,19 @@ dounmount(mp, flags, td) error = msleep(&mp->mnt_lockref, MNT_MTX(mp), PVFS, "mount drain", 0); } + + /* Until the syncer thread is not entirely closed, stop the unmount. */ + MPASS(mp->mnt_sync_state != SYNCER_TO_DIE); + while (mp->mnt_sync_state != SYNCER_DIED) { + mp->mnt_sync_state = SYNCER_TO_DIE; + + /* + * Wakeup the possible sleeping syncer in order to speedup the + * operation. + */ + cv_broadcast(&mp->mnt_sync_wakeup); + msleep(&mp->mnt_sync_thr, MNT_MTX(mp), PVFS, "syncer drain", 0); + } MNT_IUNLOCK(mp); KASSERT(mp->mnt_lockref == 0, ("%s: invalid lock refcount in the drain path @ %s:%d", @@ -1272,8 +1286,14 @@ dounmount(mp, flags, td) mp->mnt_kern_flag &= ~MNTK_ASYNC; MNT_IUNLOCK(mp); cache_purgevfs(mp); /* remove cache entries for this file sys */ + + /* Do the syncer handover now. */ if (mp->mnt_syncer != NULL) vrele(mp->mnt_syncer); + mp->mnt_sync_thr = 0; + free(mp->mnt_sync_workitem_pending, M_TEMP); + cv_destroy(&mp->mnt_sync_wakeup); + /* * For forced unmounts, move process cdir/rdir refs on the fs root * vnode to the covered vnode. For non-forced unmounts we want @@ -1310,6 +1330,7 @@ dounmount(mp, flags, td) } vput(fsrootvp); } + vfs_syncer_attach(mp, "accirt"); MNT_ILOCK(mp); mp->mnt_kern_flag &= ~MNTK_NOINSMNTQ; if ((mp->mnt_flag & MNT_RDONLY) == 0 && mp->mnt_syncer == NULL) { Index: /usr/src/sys/kern/vfs_subr.c =================================================================== --- /usr/src/sys/kern/vfs_subr.c (revision 202153) +++ /usr/src/sys/kern/vfs_subr.c (working copy) @@ -89,14 +89,12 @@ __FBSDID("$FreeBSD$"); #include #endif -#define WI_MPSAFEQ 0 -#define WI_GIANTQ 1 - static MALLOC_DEFINE(M_NETADDR, "subr_export_host", "Export host address structure"); static void delmntque(struct vnode *vp); static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, int slptimeo); +static void sched_sync(void *arg); static void syncer_shutdown(void *arg, int howto); static int vtryrecycle(struct vnode *vp); static void vbusy(struct vnode *vp); @@ -216,25 +214,10 @@ int prtactive; * syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask] * */ -static int syncer_delayno; -static long syncer_mask; -LIST_HEAD(synclist, bufobj); -static struct synclist *syncer_workitem_pending[2]; -/* - * The sync_mtx protects: - * bo->bo_synclist - * sync_vnode_count - * syncer_delayno - * syncer_state - * syncer_workitem_pending - * syncer_worklist_len - * rushjob - */ -static struct mtx sync_mtx; -static struct cv sync_wakeup; #define SYNCER_MAXDELAY 32 -static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */ +#define SYNCER_MASK (SYNCER_MAXDELAY - 1) + static int syncdelay = 30; /* max time to delay syncing data */ static int filedelay = 30; /* time to delay syncing files */ SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0, ""); @@ -242,18 +225,15 @@ static int dirdelay = 29; /* time to delay syncin SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0, ""); static int metadelay = 28; /* time to delay syncing metadata */ SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0, ""); -static int rushjob; /* number of slots to run ASAP */ +#if 0 static int stat_rush_requests; /* number of times I/O speeded up */ SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0, ""); +#endif /* * When shutting down the syncer, run it at four times normal speed. */ #define SYNCER_SHUTDOWN_SPEEDUP 4 -static int sync_vnode_count; -static int syncer_worklist_len; -static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY } - syncer_state; /* * Number of vnodes we want to exist at any one time. This is mostly used @@ -279,7 +259,6 @@ SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLA #define VSHOULDFREE(vp) (!((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt) #define VSHOULDBUSY(vp) (((vp)->v_iflag & VI_FREE) && (vp)->v_holdcnt) - /* * Initialize the vnode management data structures. */ @@ -313,16 +292,6 @@ vntblinit(void *dummy __unused) NULL, NULL, UMA_ALIGN_PTR, 0); vnodepoll_zone = uma_zcreate("VNODEPOLL", sizeof (struct vpollinfo), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); - /* - * Initialize the filesystem syncer. - */ - syncer_workitem_pending[WI_MPSAFEQ] = hashinit(syncer_maxdelay, M_VNODE, - &syncer_mask); - syncer_workitem_pending[WI_GIANTQ] = hashinit(syncer_maxdelay, M_VNODE, - &syncer_mask); - syncer_maxdelay = syncer_mask + 1; - mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF); - cv_init(&sync_wakeup, "syncer"); } SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL); @@ -521,6 +490,28 @@ vfs_getnewfsid(struct mount *mp) mtx_unlock(&mntid_mtx); } +void +vfs_syncer_attach(struct mount *mp, const char *name) +{ + int error; + + MPASS(mp != NULL); + + mp->mnt_sync_delayno = 0; + mp->mnt_sync_vnode_count = 0; + mp->mnt_sync_worklist_len = 0; + mp->mnt_sync_rushjob = 0; + mp->mnt_sync_state = SYNCER_RUNNING; + cv_init(&mp->mnt_sync_wakeup, "syncer"); + mp->mnt_sync_workitem_pending = malloc(SYNCER_MAXDELAY * + sizeof (struct synclist), M_TEMP, M_WAITOK | M_ZERO); + error = kthread_add(sched_sync, mp, NULL, &mp->mnt_sync_thr, 0, + 0, "syncer %s", name); + if (error != 0) + panic("%s: kthread_add() failed with %d errno", __func__, + error); +} + /* * Knob to control the precision of file timestamps: * @@ -1590,6 +1581,7 @@ void brelvp(struct buf *bp) { struct bufobj *bo; + struct mount *mp; struct vnode *vp; CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags); @@ -1598,20 +1590,24 @@ brelvp(struct buf *bp) /* * Delete from old vnode list, if on one. */ - vp = bp->b_vp; /* XXX */ + vp = bp->b_vp; + mp = vp->v_mount; bo = bp->b_bufobj; + MNT_ILOCK(mp); BO_LOCK(bo); if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) buf_vlist_remove(bp); - else + else { + BO_UNLOCK(bo); + MNT_IUNLOCK(mp); panic("brelvp: Buffer %p not on queue.", bp); + } if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { bo->bo_flag &= ~BO_ONWORKLST; - mtx_lock(&sync_mtx); LIST_REMOVE(bo, bo_synclist); - syncer_worklist_len--; - mtx_unlock(&sync_mtx); + mp->mnt_sync_worklist_len--; } + MNT_IUNLOCK(mp); bp->b_flags &= ~B_NEEDSGIANT; bp->b_vp = NULL; bp->b_bufobj = NULL; @@ -1623,138 +1619,103 @@ brelvp(struct buf *bp) * Add an item to the syncer work queue. */ static void -vn_syncer_add_to_worklist(struct bufobj *bo, int delay) +vn_syncer_add_to_worklist(struct mount *mp, struct bufobj *bo, int delay) { - int queue, slot; + int slot; + ASSERT_MP_ILOCKED(mp); ASSERT_BO_LOCKED(bo); - mtx_lock(&sync_mtx); if (bo->bo_flag & BO_ONWORKLST) LIST_REMOVE(bo, bo_synclist); else { bo->bo_flag |= BO_ONWORKLST; - syncer_worklist_len++; + mp->mnt_sync_worklist_len++; } - - if (delay > syncer_maxdelay - 2) - delay = syncer_maxdelay - 2; - slot = (syncer_delayno + delay) & syncer_mask; - - queue = VFS_NEEDSGIANT(bo->__bo_vnode->v_mount) ? WI_GIANTQ : - WI_MPSAFEQ; - LIST_INSERT_HEAD(&syncer_workitem_pending[queue][slot], bo, - bo_synclist); - mtx_unlock(&sync_mtx); + if (delay > SYNCER_MAXDELAY - 2) + delay = SYNCER_MAXDELAY - 2; + slot = (mp->mnt_sync_delayno + delay) & SYNCER_MASK; + LIST_INSERT_HEAD(&mp->mnt_sync_workitem_pending[slot], bo, bo_synclist); } -static int -sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS) +/* + * Syncs on the filesystem a vnode (feeded through its bufobj). + * It does expects a mountpoint locked and correctly referenced, and a + * valid bufobj. + */ +static void +sync_vnode(struct mount *mp, struct bufobj *bo) { - int error, len; + struct vnode *vp; - mtx_lock(&sync_mtx); - len = syncer_worklist_len - sync_vnode_count; - mtx_unlock(&sync_mtx); - error = SYSCTL_OUT(req, &len, sizeof(len)); - return (error); -} + MPASS(mp != NULL && bo != NULL); + ASSERT_MP_ILOCKED(mp); + MPASS(mp->mnt_writeopcount > 0); -SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, CTLTYPE_INT | CTLFLAG_RD, NULL, 0, - sysctl_vfs_worklist_len, "I", "Syncer thread worklist length"); + vp = bo->__bo_vnode; -static struct proc *updateproc; -static void sched_sync(void); -static struct kproc_desc up_kp = { - "syncer", - sched_sync, - &updateproc -}; -SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp); - -static int -sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td) -{ - struct vnode *vp; - struct mount *mp; - - *bo = LIST_FIRST(slp); - if (*bo == NULL) - return (0); - vp = (*bo)->__bo_vnode; /* XXX */ - if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0) - return (1); /* - * We use vhold in case the vnode does not - * successfully sync. vhold prevents the vnode from - * going away when we unlock the sync_mtx so that - * we can acquire the vnode interlock. + * vhold() the vnode linked to the bufobj in order to avoid + * possible recycling when releasing the mountpoint interlock. */ - vholdl(vp); - mtx_unlock(&sync_mtx); - VI_UNLOCK(vp); - if (vn_start_write(vp, &mp, V_NOWAIT) != 0) { - vdrop(vp); - mtx_lock(&sync_mtx); - return (*bo == LIST_FIRST(slp)); - } + vhold(vp); + MNT_IUNLOCK(mp); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); - (void) VOP_FSYNC(vp, MNT_LAZY, td); + VOP_FSYNC(vp, MNT_LAZY, curthread); VOP_UNLOCK(vp, 0); - vn_finished_write(mp); - BO_LOCK(*bo); - if (((*bo)->bo_flag & BO_ONWORKLST) != 0) { + MNT_ILOCK(mp); + BO_LOCK(bo); + if ((bo->bo_flag & BO_ONWORKLST) != 0) { + /* * Put us back on the worklist. The worklist * routine will remove us from our current * position and then add us back in at a later * position. */ - vn_syncer_add_to_worklist(*bo, syncdelay); + vn_syncer_add_to_worklist(mp, bo, syncdelay); } - BO_UNLOCK(*bo); + BO_UNLOCK(bo); vdrop(vp); - mtx_lock(&sync_mtx); - return (0); } /* * System filesystem synchronizer daemon. */ static void -sched_sync(void) +sched_sync(void *arg) { - struct synclist *gnext, *next; - struct synclist *gslp, *slp; + eventhandler_tag shutdown_tag; + long starttime; + struct mount *mp; + struct synclist *next; + struct synclist *slp; struct bufobj *bo; - long starttime; - struct thread *td = curthread; + struct thread *td; int last_work_seen; int net_worklist_len; int syncer_final_iter; int first_printf; - int error; + int vfslocked; + mp = arg; last_work_seen = 0; syncer_final_iter = 0; first_printf = 1; - syncer_state = SYNCER_RUNNING; starttime = time_uptime; + td = curthread; td->td_pflags |= TDP_NORUNNINGBUF; - EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc, - SHUTDOWN_PRI_LAST); + shutdown_tag = EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, + mp, SHUTDOWN_PRI_LAST); - mtx_lock(&sync_mtx); - for (;;) { - if (syncer_state == SYNCER_FINAL_DELAY && - syncer_final_iter == 0) { - mtx_unlock(&sync_mtx); - kproc_suspend_check(td->td_proc); - mtx_lock(&sync_mtx); - } - net_worklist_len = syncer_worklist_len - sync_vnode_count; - if (syncer_state != SYNCER_RUNNING && + vfslocked = VFS_LOCK_GIANT(mp); + MNT_ILOCK(mp); + while (mp->mnt_sync_state != SYNCER_TO_DIE) { + MPASS(mp->mnt_sync_state != SYNCER_DIED); + net_worklist_len = mp->mnt_sync_worklist_len - + mp->mnt_sync_vnode_count; + if (mp->mnt_sync_state != SYNCER_RUNNING && starttime != time_uptime) { if (first_printf) { printf("\nSyncing disks, vnodes remaining..."); @@ -1771,27 +1732,28 @@ static void * Skip over empty worklist slots when shutting down. */ do { - slp = &syncer_workitem_pending[WI_MPSAFEQ][syncer_delayno]; - gslp = &syncer_workitem_pending[WI_GIANTQ][syncer_delayno]; - syncer_delayno += 1; - if (syncer_delayno == syncer_maxdelay) - syncer_delayno = 0; - next = &syncer_workitem_pending[WI_MPSAFEQ][syncer_delayno]; - gnext = &syncer_workitem_pending[WI_GIANTQ][syncer_delayno]; + slp = + &mp->mnt_sync_workitem_pending[mp->mnt_sync_delayno]; + mp->mnt_sync_delayno += 1; + if (mp->mnt_sync_delayno == SYNCER_MAXDELAY) + mp->mnt_sync_delayno = 0; + next = + &mp->mnt_sync_workitem_pending[mp->mnt_sync_delayno]; + /* * If the worklist has wrapped since the * it was emptied of all but syncer vnodes, * switch to the FINAL_DELAY state and run * for one more second. */ - if (syncer_state == SYNCER_SHUTTING_DOWN && + if (mp->mnt_sync_state == SYNCER_SHUTTING_DOWN && net_worklist_len == 0 && - last_work_seen == syncer_delayno) { - syncer_state = SYNCER_FINAL_DELAY; + last_work_seen == mp->mnt_sync_delayno) { + mp->mnt_sync_state = SYNCER_FINAL_DELAY; syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP; } - } while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) && - LIST_EMPTY(gslp) && syncer_worklist_len > 0); + } while (mp->mnt_sync_state != SYNCER_RUNNING && + LIST_EMPTY(slp) && mp->mnt_sync_worklist_len > 0); /* * Keep track of the last time there was anything @@ -1799,35 +1761,69 @@ static void * Return to the SHUTTING_DOWN state if any * new work appears. */ - if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING) - last_work_seen = syncer_delayno; - if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY) - syncer_state = SYNCER_SHUTTING_DOWN; - while (!LIST_EMPTY(slp)) { - error = sync_vnode(slp, &bo, td); - if (error == 1) { + if (net_worklist_len > 0 || + mp->mnt_sync_state == SYNCER_RUNNING) + last_work_seen = mp->mnt_sync_delayno; + if (net_worklist_len > 0 && + mp->mnt_sync_state == SYNCER_FINAL_DELAY) + mp->mnt_sync_state = SYNCER_SHUTTING_DOWN; + + /* + * As long as the mountpoint interlock is already held and + * a simple, non-sleeping, call to vn_start_write() is due, + * it is convenient to just emulate interesting bits + * directly in the code. + * Furthermore, it is already known that the mountpoint is + * not a filesystem bypass, thus VOP_GETWRITEMOUNT() + * inquirying is not necessary and that the syncer thread + * didn't set TDP_IGNSUSP flag, thus this check is skipped. + * Ultimately, the syncer also doesn't need to acquire a + * refcount on the mountpoint because the races against + * unmount are handled via the syncer state movements. + */ + if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) { + + /* + * The filesystem is already suspending. + * Move the bufobjs into the next syncer queue + * and skip any further magic. + */ + while (!LIST_EMPTY(slp)) { + bo = LIST_FIRST(slp); LIST_REMOVE(bo, bo_synclist); LIST_INSERT_HEAD(next, bo, bo_synclist); - continue; } + } else { + mp->mnt_writeopcount++; + + /* + * The mountpoint is ready to be written now. + * sync_buflist() may release the mountpoint + * interlock so use a strict loop for dealing with + * lost insertion races. + */ + while (!LIST_EMPTY(slp)) + sync_vnode(mp, LIST_FIRST(slp)); + + /* Emulate a vn_finished_write(). */ + mp->mnt_writeopcount--; + MPASS(mp->mnt_writeopcount >= 0); + if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 && + mp->mnt_writeopcount == 0) + wakeup(&mp->mnt_writeopcount); + + /* + * If an unmount operation started while the mountpoint + * interlock has been released during sync_buflist() + * invocation, quit the syncer thread now. + */ + if (mp->mnt_sync_state == SYNCER_TO_DIE) + break; } - if (!LIST_EMPTY(gslp)) { - mtx_unlock(&sync_mtx); - mtx_lock(&Giant); - mtx_lock(&sync_mtx); - while (!LIST_EMPTY(gslp)) { - error = sync_vnode(gslp, &bo, td); - if (error == 1) { - LIST_REMOVE(bo, bo_synclist); - LIST_INSERT_HEAD(gnext, bo, - bo_synclist); - continue; - } - } - mtx_unlock(&Giant); - } - if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0) + if (mp->mnt_sync_state == SYNCER_FINAL_DELAY && + syncer_final_iter > 0) syncer_final_iter--; + /* * The variable rushjob allows the kernel to speed up the * processing of the filesystem syncer process. A rushjob @@ -1838,10 +1834,11 @@ static void * ahead of the disk that the kernel memory pool is being * threatened with exhaustion. */ - if (rushjob > 0) { - rushjob -= 1; + if (mp->mnt_sync_rushjob > 0) { + mp->mnt_sync_rushjob -= 1; continue; } + /* * Just sleep for a short period of time between * iterations when shutting down to allow some I/O @@ -1854,12 +1851,23 @@ static void * matter as we are just trying to generally pace the * filesystem activity. */ - if (syncer_state != SYNCER_RUNNING) - cv_timedwait(&sync_wakeup, &sync_mtx, + if (mp->mnt_sync_state != SYNCER_RUNNING) + cv_timedwait(&mp->mnt_sync_wakeup, MNT_MTX(mp), hz / SYNCER_SHUTDOWN_SPEEDUP); else if (time_uptime == starttime) - cv_timedwait(&sync_wakeup, &sync_mtx, hz); + cv_timedwait(&mp->mnt_sync_wakeup, MNT_MTX(mp), hz); } + + /* + * If this point is reached, an unmount request is in progress. + * Wakeup the unmount thread and close the syncer. + */ + mp->mnt_sync_state = SYNCER_DIED; + wakeup(&mp->mnt_sync_thr); + MNT_IUNLOCK(mp); + VFS_UNLOCK_GIANT(vfslocked); + EVENTHANDLER_DEREGISTER(shutdown_pre_sync, shutdown_tag); + kthread_exit(); } /* @@ -1870,6 +1878,7 @@ static void int speedup_syncer(void) { +#if 0 int ret = 0; mtx_lock(&sync_mtx); @@ -1881,6 +1890,8 @@ speedup_syncer(void) mtx_unlock(&sync_mtx); cv_broadcast(&sync_wakeup); return (ret); +#endif + return (0); } /* @@ -1890,15 +1901,17 @@ speedup_syncer(void) static void syncer_shutdown(void *arg, int howto) { + struct mount *mp; + mp = arg; if (howto & RB_NOSYNC) return; - mtx_lock(&sync_mtx); - syncer_state = SYNCER_SHUTTING_DOWN; - rushjob = 0; - mtx_unlock(&sync_mtx); - cv_broadcast(&sync_wakeup); - kproc_shutdown(arg, howto); + MNT_ILOCK(mp); + mp->mnt_sync_state = SYNCER_SHUTTING_DOWN; + mp->mnt_sync_rushjob = 0; + MNT_IUNLOCK(mp); + cv_broadcast(&mp->mnt_sync_wakeup); + kthread_shutdown(arg, howto); } /* @@ -1909,6 +1922,7 @@ syncer_shutdown(void *arg, int howto) void reassignbuf(struct buf *bp) { + struct mount *mp; struct vnode *vp; struct bufobj *bo; int delay; @@ -1917,6 +1931,7 @@ reassignbuf(struct buf *bp) #endif vp = bp->b_vp; + mp = vp->v_mount; bo = bp->b_bufobj; ++reassignbufcalls; @@ -1931,12 +1946,21 @@ reassignbuf(struct buf *bp) /* * Delete from old vnode list, if on one. + * + * Lock the mountpoint now in order to avoid a LOR with the bufobj lock + * as we may be needing to insert a dirty buffer within the appropriate + * syncer pending worklist. */ + MNT_ILOCK(mp); BO_LOCK(bo); if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) buf_vlist_remove(bp); - else + else { + BO_UNLOCK(bo); + MNT_IUNLOCK(mp); panic("reassignbuf: Buffer %p not on queue.", bp); + } + /* * If dirty, put on list of dirty buffers; otherwise insert onto list * of clean buffers. @@ -1953,17 +1977,15 @@ reassignbuf(struct buf *bp) default: delay = filedelay; } - vn_syncer_add_to_worklist(bo, delay); + vn_syncer_add_to_worklist(mp, bo, delay); } buf_vlist_add(bp, bo, BX_VNDIRTY); } else { buf_vlist_add(bp, bo, BX_VNCLEAN); if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) { - mtx_lock(&sync_mtx); LIST_REMOVE(bo, bo_synclist); - syncer_worklist_len--; - mtx_unlock(&sync_mtx); + mp->mnt_sync_worklist_len--; bo->bo_flag &= ~BO_ONWORKLST; } } @@ -1984,6 +2006,7 @@ reassignbuf(struct buf *bp) ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); #endif BO_UNLOCK(bo); + MNT_IUNLOCK(mp); } /* @@ -3386,23 +3409,23 @@ vfs_allocate_syncvnode(struct mount *mp) * are mounted at once. */ next += incr; - if (next == 0 || next > syncer_maxdelay) { + if (next == 0 || next > SYNCER_MAXDELAY) { start /= 2; incr /= 2; if (start == 0) { - start = syncer_maxdelay / 2; - incr = syncer_maxdelay; + start = SYNCER_MAXDELAY / 2; + incr = SYNCER_MAXDELAY; } next = start; } bo = &vp->v_bufobj; + MNT_ILOCK(mp); BO_LOCK(bo); - vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0); - /* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */ - mtx_lock(&sync_mtx); - sync_vnode_count++; - mtx_unlock(&sync_mtx); + vn_syncer_add_to_worklist(mp, bo, + syncdelay > 0 ? next % syncdelay : 0); + mp->mnt_sync_vnode_count++; BO_UNLOCK(bo); + MNT_IUNLOCK(mp); mp->mnt_syncer = vp; return (0); } @@ -3428,9 +3451,11 @@ sync_fsync(struct vop_fsync_args *ap) * Move ourselves to the back of the sync list. */ bo = &syncvp->v_bufobj; + MNT_ILOCK(mp); BO_LOCK(bo); - vn_syncer_add_to_worklist(bo, syncdelay); + vn_syncer_add_to_worklist(mp, bo, syncdelay); BO_UNLOCK(bo); + MNT_IUNLOCK(mp); /* * Walk the list of vnodes pushing all that are dirty and @@ -3480,21 +3505,24 @@ sync_inactive(struct vop_inactive_args *ap) static int sync_reclaim(struct vop_reclaim_args *ap) { - struct vnode *vp = ap->a_vp; + struct mount *mp; + struct vnode *vp; struct bufobj *bo; + vp = ap->a_vp; + mp = vp->v_mount; bo = &vp->v_bufobj; + MNT_ILOCK(mp); BO_LOCK(bo); vp->v_mount->mnt_syncer = NULL; if (bo->bo_flag & BO_ONWORKLST) { - mtx_lock(&sync_mtx); LIST_REMOVE(bo, bo_synclist); - syncer_worklist_len--; - sync_vnode_count--; - mtx_unlock(&sync_mtx); + mp->mnt_sync_worklist_len--; + mp->mnt_sync_vnode_count--; bo->bo_flag &= ~BO_ONWORKLST; } BO_UNLOCK(bo); + MNT_IUNLOCK(mp); return (0); } Index: /usr/src/sys/sys/mount.h =================================================================== --- /usr/src/sys/sys/mount.h (revision 202153) +++ /usr/src/sys/sys/mount.h (working copy) @@ -36,6 +36,7 @@ #include #include #ifdef _KERNEL +#include #include #include #include @@ -128,6 +129,7 @@ struct ostatfs { }; TAILQ_HEAD(vnodelst, vnode); +LIST_HEAD(synclist, bufobj); /* Mount options list */ TAILQ_HEAD(vfsoptlist, vfsopt); @@ -187,6 +189,20 @@ struct mount { #define mnt_endzero mnt_gjprovider char *mnt_gjprovider; /* gjournal provider name */ struct lock mnt_explock; /* vfs_export walkers lock */ + struct cv mnt_sync_wakeup; + struct thread *mnt_sync_thr; + struct synclist *mnt_sync_workitem_pending; + int mnt_sync_delayno; + int mnt_sync_vnode_count; + int mnt_sync_worklist_len; + int mnt_sync_rushjob; + enum { + SYNCER_RUNNING, + SYNCER_SHUTTING_DOWN, + SYNCER_FINAL_DELAY, + SYNCER_TO_DIE, + SYNCER_DIED + } mnt_sync_state; }; struct vnode *__mnt_vnode_next(struct vnode **mvp, struct mount *mp); @@ -207,6 +223,7 @@ void __mnt_vnode_markerfree(struct vnode MNT_IUNLOCK(mp); \ } while (0) +#define ASSERT_MP_ILOCKED(mp) mtx_assert(&(mp)->mnt_mtx, MA_OWNED) #define MNT_ILOCK(mp) mtx_lock(&(mp)->mnt_mtx) #define MNT_ITRYLOCK(mp) mtx_trylock(&(mp)->mnt_mtx) #define MNT_IUNLOCK(mp) mtx_unlock(&(mp)->mnt_mtx) @@ -724,6 +741,7 @@ int vfs_setopts(struct vfsoptlist *opts, const cha const char *value); int vfs_setpublicfs /* set publicly exported fs */ (struct mount *, struct netexport *, struct export_args *); +void vfs_syncer_attach(struct mount *mp, const char *name); void vfs_msync(struct mount *, int); int vfs_busy(struct mount *, int); int vfs_export /* process mount export info */