GENERIC HEAD from 2009-06-07 05:52:22 UTC, r193624M, vmcore.186 KDB: debugger backends: ddb KDB: current backend: ddb SMAP type=01 base=0000000000000000 len=000000000009fc00 SMAP type=02 base=000000000009fc00 len=0000000000000400 SMAP type=02 base=00000000000e4000 len=000000000001c000 SMAP type=01 base=0000000000100000 len=00000000d7ed0000 SMAP type=03 base=00000000d7fd0000 len=000000000000e000 SMAP type=04 base=00000000d7fde000 len=0000000000022000 SMAP type=02 base=00000000fff00000 len=0000000000100000 SMAP type=01 base=0000000100000000 len=0000000020000000 524288K of memory above 4GB ignored Copyright (c) 1992-2009 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 8.0-CURRENT #0 r193624M: Sun Jun 7 09:49:46 CEST 2009 pho@x4.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. Preloaded elf kernel "/boot/kernel/kernel" at 0xc1158000. Preloaded elf module "/boot/kernel/snd_hda.ko" at 0xc1158174. Preloaded elf module "/boot/kernel/sound.ko" at 0xc1158220. Preloaded /boot/zfs/zpool.cache "/boot/zfs/zpool.cache" at 0xc11582cc. Timecounter "i8254" frequency 1193182 Hz quality 0 Calibrating TSC clock ... TSC clock: 1799991674 Hz CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1799.99-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant Data TLB: 48 entries, fully associative Instruction TLB: 32 entries, fully associative L1 data cache: 64 kbytes, 64 bytes/line, 1 lines/tag, 2-way associative L1 instruction cache: 64 kbytes, 64 bytes/line, 1 lines/tag, 2-way associative L2 internal cache: 512 kbytes, 64 bytes/line, 1 lines/tag, 8-way associative real memory = 4294967296 (4096 MB) Physical memory chunk(s): 0x0000000000001000 - 0x000000000009efff, 647168 bytes (158 pages) 0x0000000000100000 - 0x00000000003fffff, 3145728 bytes (768 pages) 0x0000000001425000 - 0x00000000d460cfff, 3541991424 bytes (864744 pages) avail memory = 3539734528 (3375 MB) : Trying to mount root from ufs:/dev/ad4s1a ct_to_ts([2009-06-07 08:30:46]) = 1244363446.000000000 start_init: trying /sbin/init Entropy harvesting: interrupts ethernet point_to_pointKernel page fault with the following non-sleepable locks held: exclusive sleep mutex bufobj interlock (bufobj interlock) r = 0 (0xc71cf3a4) locked @ kern/vfs_bio.c:2544 KDB: stack backtrace: db_trace_self_wrapper(c0c5d225,f62e3840,c08abaa5,c0c64def,9f0,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0c64def,9f0,ffffffff,c0eea344,f62e3878,...) at kdb_backtrace+0x29 _witness_debugger(c0c5f6e6,f62e388c,4,1,0,...) at _witness_debugger+0x25 witness_warn(5,0,c0c92c69,4,c74aed34,...) at witness_warn+0x1fd trap(f62e3918) at trap+0x173 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0xc08542af, esp = 0xf62e3958, ebp = 0xf62e39e8 --- __lockmgr_args(e6d00ffc,81900,c71cf3a4,c0c65811,50,...) at __lockmgr_args+0xadf getblk(c71cf2b0,1,0,4000,0,...) at getblk+0x163 cluster_read(c71cf2b0,a000,0,1,0,...) at cluster_read+0xfa ffs_read(f62e3bc0,c0c95068,c74b6b40,0,c74233f0,...) at ffs_read+0x349 VOP_READ_APV(c0d5eb00,f62e3bc0,c71cf2b0,217,c0c68618,...) at VOP_READ_APV+0xc5 vn_read(c74233f0,f62e3c58,c6d44300,1,c74b6b40,...) at vn_read+0x208 dofileread(f62e3c58,6000,0,1,c74233f0,...) at dofileread+0x96 kern_preadv(c74b6b40,5,f62e3c58,6000,0,...) at kern_preadv+0x78 pread(c74b6b40,f62e3cf8,14,c0bf483f,c0d411f4,...) at pread+0x5d syscall(f62e3d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (475, FreeBSD ELF32, pread), eip = 0x2819a917, esp = 0xbfbfdf9c, ebp = 0xbfbfdfc8 --- Fatal trap 12: page fault while in kernel mode cpuid = 0; apic id = 00 fault virtual address = 0x18c fault code = supervisor read, page not present instruction pointer = 0x20:0xc08542af stack pointer = 0x28:0xf62e3958 frame pointer = 0x28:0xf62e39e8 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 70 (ps) [thread pid 70 tid 100091 ] Stopped at __lockmgr_args+0xadf: movl 0x19c(%ebx),%eax db:0:kdb.enter.unknown> run pho db:1:pho> bt Tracing pid 70 tid 100091 td 0xc74b6b40 __lockmgr_args(e6d00ffc,81900,c71cf3a4,c0c65811,50,...) at __lockmgr_args+0xadf getblk(c71cf2b0,1,0,4000,0,...) at getblk+0x163 cluster_read(c71cf2b0,a000,0,1,0,...) at cluster_read+0xfa ffs_read(f62e3bc0,c0c95068,c74b6b40,0,c74233f0,...) at ffs_read+0x349 VOP_READ_APV(c0d5eb00,f62e3bc0,c71cf2b0,217,c0c68618,...) at VOP_READ_APV+0xc5 vn_read(c74233f0,f62e3c58,c6d44300,1,c74b6b40,...) at vn_read+0x208 dofileread(f62e3c58,6000,0,1,c74233f0,...) at dofileread+0x96 kern_preadv(c74b6b40,5,f62e3c58,6000,0,...) at kern_preadv+0x78 pread(c74b6b40,f62e3cf8,14,c0bf483f,c0d411f4,...) at pread+0x5d syscall(f62e3d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (475, FreeBSD ELF32, pread), eip = 0x2819a917, esp = 0xbfbfdf9c, ebp = 0xbfbfdfc8 --- db:1:bt> show allpcpu Current CPU: 0 cpuid = 0 curthread = 0xc74b6b40: pid 70 "ps" curpcb = 0xf62e3d90 fpcurthread = none idlethread = 0xc6d78480: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 curthread = 0xc6d786c0: pid 11 "idle: cpu1" curpcb = 0xc6a99d90 fpcurthread = none idlethread = 0xc6d786c0: pid 11 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 curthread = 0xc6d78900: pid 11 "idle: cpu2" curpcb = 0xc6a96d90 fpcurthread = none idlethread = 0xc6d78900: pid 11 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 curthread = 0xc6d78b40: pid 11 "idle: cpu3" curpcb = 0xc6a93d90 fpcurthread = none idlethread = 0xc6d78b40: pid 11 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:1:allpcpu> show alllocks Process 70 (ps) thread 0xc74b6b40 (100091) exclusive sleep mutex bufobj interlock (bufobj interlock) r = 0 (0xc71cf3a4) locked @ kern/vfs_bio.c:2544 shared lockmgr ufs (ufs) r = 0 (0xc71cf308) locked @ kern/vfs_vnops.c:535 db:1:alllocks> show lockedvnods Locked vnodes 0xc71cf2b0: tag ufs, type VREG usecount 1, writecount 0, refcount 4 mountedhere 0 flags () v_object 0xc743a64c ref 0 pages 8 lock type ufs: SHARED (count 1) #0 0xc0853f77 at __lockmgr_args+0x7a7 #1 0xc0a94a91 at ffs_lock+0xa1 #2 0xc0b9a755 at VOP_LOCK1_APV+0xb5 #3 0xc0904db8 at _vn_lock+0x78 #4 0xc09063ef at vn_read+0x19f #5 0xc08ae2f6 at dofileread+0x96 #6 0xc08ae3d8 at kern_preadv+0x78 #7 0xc08ae50d at pread+0x5d #8 0xc0b8c4a4 at syscall+0x2b4 #9 0xc0b6f700 at Xint0x80_syscall+0x20 ino 166450, on dev ad4s1a db:1:lockedvnods> show mount 0xc73c7b40 /dev/ad4s1a on / (ufs) 0xc73c8000 devfs on /dev (devfs) More info: show mount db:1:mount> ps pid ppid pgrp uid state wmesg wchan cmd 70 69 45 0 R+ CPU 0 ps 69 67 45 0 S+ wait 0xc74467ec sh 68 62 45 0 S+ piperd 0xc7470600 dd 67 62 45 0 S+ wait 0xc741ad34 sh 62 45 45 0 S+ wait 0xc74462a4 sh 45 1 45 0 Ss+ wait 0xc741a548 sh 44 0 0 0 SL flowclea 0xc0daaca4 [flowcleaner] 43 0 0 0 SL sdflush 0xc0f2ac60 [softdepflush] 42 0 0 0 SL vlruwt 0xc73402a4 [vnlru] 41 0 0 0 SL syncer 0xc0f17514 [syncer] 40 0 0 0 SL psleep 0xc0f17248 [bufdaemon] 39 0 0 0 SL pgzero 0xc0f2b894 [pagezero] 38 0 0 0 SL psleep 0xc0f2b4bc [vmdaemon] 9 0 0 0 SL psleep 0xc0f2b484 [pagedaemon] 37 0 0 0 SL wmsg 0xc6fdfd0c [usbus5] 36 0 0 0 SL wmsg 0xc6fdfcdc [usbus5] 35 0 0 0 SL wmsg 0xc6fdfcac [usbus5] 34 0 0 0 SL wmsg 0xc6fdfc7c [usbus5] 33 0 0 0 SL wmsg 0xc6fd9b5c [usbus4] 32 0 0 0 SL wmsg 0xc6fd9b2c [usbus4] 31 0 0 0 SL wmsg 0xc6fd9afc [usbus4] 30 0 0 0 SL wmsg 0xc6fd9acc [usbus4] 29 0 0 0 SL wmsg 0xc6fd0b5c [usbus3] 28 0 0 0 SL wmsg 0xc6fd0b2c [usbus3] 27 0 0 0 SL wmsg 0xc6fd0afc [usbus3] 26 0 0 0 SL wmsg 0xc6fd0acc [usbus3] 25 0 0 0 SL wmsg 0xc6fcdb5c [usbus2] 24 0 0 0 SL wmsg 0xc6fcdb2c [usbus2] 23 0 0 0 SL wmsg 0xc6fcdafc [usbus2] 22 0 0 0 SL wmsg 0xc6fcdacc [usbus2] 21 0 0 0 SL wmsg 0xc6fc8b5c [usbus1] 20 0 0 0 SL wmsg 0xc6fc8b2c [usbus1] 19 0 0 0 SL wmsg 0xc6fc8afc [usbus1] 18 0 0 0 SL wmsg 0xc6fc8acc [usbus1] 17 0 0 0 SL wmsg 0xc6fc2b5c [usbus0] 16 0 0 0 SL wmsg 0xc6fc2b2c [usbus0] 15 0 0 0 SL wmsg 0xc6fc2afc [usbus0] 14 0 0 0 SL wmsg 0xc6fc2acc [usbus0] 8 0 0 0 SL waiting_ 0xc0f2135c [sctp_iterator] 7 0 0 0 SL - 0xc6eb2a3c [fdc0] 6 0 0 0 SL - 0xc6ffe000 [fw0_probe] 5 0 0 0 SL ccb_scan 0xc0d77c54 [xpt_thrd] 13 0 0 0 SL - 0xc0daaca4 [yarrow] 4 0 0 0 SL - 0xc0da8a44 [g_down] 3 0 0 0 SL - 0xc0da8a40 [g_up] 2 0 0 0 SL - 0xc0da8a38 [g_event] 12 0 0 0 WL (threaded) intr 100042 I [irq12: psm0] 100041 I [irq1: atkbd0] 100039 I [swi0: uart] 100036 I [irq20: fwohci0] 100035 I [irq14: ata0] 100034 I [irq19: ehci0] 100033 I [irq18: ohci2 ohci4] 100032 I [irq17: ohci1 ohci3] 100031 I [irq16: hdac1 ohci0] 100030 I [irq22: atapci0] 100029 I [irq256: hdac0] 100028 I [irq9: acpi0] 100027 I [swi5: +] 100022 I [swi2: cambio] 100020 I [swi6: task queue] 100019 I [swi6: Giant taskq] 100012 I [swi1: netisr 0] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi4: clock] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) idle 100006 CanRun [idle: cpu0] 100005 Run CPU 1 [idle: cpu1] 100004 Run CPU 2 [idle: cpu2] 100003 Run CPU 3 [idle: cpu3] 1 0 1 0 SLs wait 0xc6d76d34 [init] 10 0 0 0 SL audit_wo 0xc0f2a580 [audit] 0 0 0 0 SLs (threaded) kernel 100037 D - 0xc700f880 [fw0_taskq] 100026 D - 0xc6f29000 [kqueue taskq] 100025 D - 0xc6f29040 [acpi_task_2] 100024 D - 0xc6f29040 [acpi_task_1] 100023 D - 0xc6f29040 [acpi_task_0] 100018 D - 0xc6f29440 [thread taskq] 100016 D - 0xc6d5eb40 [firmware taskq] 100000 D sched 0xc0da8b00 [swapper] db:1:ps> allt Tracing command ps pid 70 tid 100091 td 0xc74b6b40 __lockmgr_args(e6d00ffc,81900,c71cf3a4,c0c65811,50,...) at __lockmgr_args+0xadf getblk(c71cf2b0,1,0,4000,0,...) at getblk+0x163 cluster_read(c71cf2b0,a000,0,1,0,...) at cluster_read+0xfa ffs_read(f62e3bc0,c0c95068,c74b6b40,0,c74233f0,...) at ffs_read+0x349 VOP_READ_APV(c0d5eb00,f62e3bc0,c71cf2b0,217,c0c68618,...) at VOP_READ_APV+0xc5 vn_read(c74233f0,f62e3c58,c6d44300,1,c74b6b40,...) at vn_read+0x208 dofileread(f62e3c58,6000,0,1,c74233f0,...) at dofileread+0x96 kern_preadv(c74b6b40,5,f62e3c58,6000,0,...) at kern_preadv+0x78 pread(c74b6b40,f62e3cf8,14,c0bf483f,c0d411f4,...) at pread+0x5d syscall(f62e3d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (475, FreeBSD ELF32, pread), eip = 0x2819a917, esp = 0xbfbfdf9c, ebp = 0xbfbfdfc8 --- Tracing command sh pid 69 tid 100077 td 0xc744c480 sched_switch(c744c480,0,104,18c,977a41c6,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c744c480,0,c0c5e290,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5e290,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c74467ec,5c,c0c60afd,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74467ec,c7446874,15c,c0c60afd,0,...) at _sleep+0x344 kern_wait(c744c480,ffffffff,f62a1c74,2,0,...) at kern_wait+0xac6 wait4(c744c480,f62a1cf8,10,c0c60adc,c0d3dec4,...) at wait4+0x3b syscall(f62a1d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfd50c, ebp = 0xbfbfd528 --- Tracing command dd pid 68 tid 100087 td 0xc7447000 sched_switch(c7447000,0,104,18c,98e92677,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c7447000,0,c0c5e290,189,4c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5e290,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c7470600,4c,c0c60714,100,0,...) at sleepq_wait_sig+0x17 _sleep(c7470600,c7470768,14c,c0c60714,0,...) at _sleep+0x344 pipe_read(c7423c78,f62cdc58,c6d44300,0,c7447000,...) at pipe_read+0x417 dofileread(f62cdc58,ffffffff,ffffffff,0,c7423c78,...) at dofileread+0x96 kern_readv(c7447000,0,f62cdc58,f62cdc78,1,...) at kern_readv+0x58 read(c7447000,f62cdcf8,c,c0c61338,c0d3de54,...) at read+0x4f syscall(f62cdd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (3, FreeBSD ELF32, read), eip = 0x28179d83, esp = 0xbfbfee1c, ebp = 0xbfbfee58 --- Tracing command sh pid 67 tid 100088 td 0xc741fd80 sched_switch(c741fd80,0,104,18c,975b6157,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c741fd80,0,c0c5e290,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5e290,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c741ad34,5c,c0c60afd,100,0,...) at sleepq_wait_sig+0x17 _sleep(c741ad34,c741adbc,15c,c0c60afd,0,...) at _sleep+0x344 kern_wait(c741fd80,ffffffff,f62d1c74,2,0,...) at kern_wait+0xac6 wait4(c741fd80,f62d1cf8,10,c0c60adc,c0d3dec4,...) at wait4+0x3b syscall(f62d1d38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfd82c, ebp = 0xbfbfd848 --- Tracing command sh pid 62 tid 100080 td 0xc744c000 sched_switch(c744c000,0,104,18c,975aef9d,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c744c000,0,c0c5e290,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5e290,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c74462a4,5c,c0c60afd,100,0,...) at sleepq_wait_sig+0x17 _sleep(c74462a4,c744632c,15c,c0c60afd,0,...) at _sleep+0x344 kern_wait(c744c000,ffffffff,f62adc74,2,0,...) at kern_wait+0xac6 wait4(c744c000,f62adcf8,10,c0c60adc,c0d3dec4,...) at wait4+0x3b syscall(f62add38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfd8bc, ebp = 0xbfbfd8d8 --- Tracing command sh pid 45 tid 100076 td 0xc741f000 sched_switch(c741f000,0,104,18c,946f068c,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c741f000,0,c0c5e290,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5e290,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c741a548,5c,c0c60afd,100,0,...) at sleepq_wait_sig+0x17 _sleep(c741a548,c741a5d0,15c,c0c60afd,0,...) at _sleep+0x344 kern_wait(c741f000,ffffffff,f629dc74,2,0,...) at kern_wait+0xac6 wait4(c741f000,f629dcf8,10,c0c60adc,c0d3dec4,...) at wait4+0x3b syscall(f629dd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2815ae7b, esp = 0xbfbfe96c, ebp = 0xbfbfe988 --- Tracing command flowcleaner pid 44 tid 100075 td 0xc732a480 sched_switch(c732a480,0,104,18c,16d19675,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732a480,0,c0c5e290,26a,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0daaca4,0,c0c68f43,2,0,...) at sleepq_timedwait+0x6b _sleep(c0daaca4,0,0,c0c68f43,4e20,...) at _sleep+0x329 pause(c0c68f43,4e20,c0c68e78,41e,0,...) at pause+0x47 flowtable_cleaner(0,f40aed38,c0c55614,334,c7327d34,...) at flowtable_cleaner+0x160 fork_exit(c090ece0,0,f40aed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40aed70, ebp = 0 --- Tracing command softdepflush pid 43 tid 100074 td 0xc732a6c0 sched_switch(c732a6c0,0,104,18c,809ed546,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,44,...) at mi_switch+0x200 sleepq_switch(c732a6c0,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f2ac60,44,c0c7f8c7,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f2ac60,c0f2ac04,44,c0c7f8c7,3e8,...) at _sleep+0x329 softdep_flush(0,f40abd38,c0c55614,334,c7340000,...) at softdep_flush+0x244 fork_exit(c0a8adc0,0,f40abd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40abd70, ebp = 0 --- Tracing command vnlru pid 42 tid 100073 td 0xc732a900 sched_switch(c732a900,0,104,18c,80a78e36,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,50,...) at mi_switch+0x200 sleepq_switch(c732a900,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c73402a4,50,c0c6814f,0,0,...) at sleepq_timedwait+0x6b _sleep(c73402a4,c0f174d4,250,c0c6814f,3e8,...) at _sleep+0x329 vnlru_proc(0,f40a8d38,c0c55614,334,c73402a4,...) at vnlru_proc+0xe7 fork_exit(c08fa170,0,f40a8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40a8d70, ebp = 0 --- Tracing command syncer pid 41 tid 100072 td 0xc732ab40 sched_switch(c732ab40,0,104,18c,809f366d,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732ab40,0,c0c5e290,26a,c732ab40,...) at sleepq_switch+0x15f sleepq_timedwait(c0f17514,0,f40a5c88,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c0f17514,c0f17500,3e8,6cc,4e20,...) at _cv_timedwait+0x250 sched_sync(0,f40a5d38,c0c55614,334,c7340548,...) at sched_sync+0x502 fork_exit(c08f95a0,0,f40a5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40a5d70, ebp = 0 --- Tracing command bufdaemon pid 40 tid 100071 td 0xc732ad80 sched_switch(c732ad80,0,104,18c,809e88a4,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,44,...) at mi_switch+0x200 sleepq_switch(c732ad80,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f17248,44,c0c6572a,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f17248,c0f1724c,44,c0c6572a,3e8,...) at _sleep+0x329 buf_daemon(0,f40a2d38,c0c55614,334,c73407ec,...) at buf_daemon+0x138 fork_exit(c08e22f0,0,f40a2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf40a2d70, ebp = 0 --- Tracing command pagezero pid 39 tid 100070 td 0xc732c000 sched_switch(c732c000,0,104,18c,e5c3f93,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732c000,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f2b894,0,c0c84f3d,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f2b894,c0f2b450,0,c0c84f3d,493e0,...) at _sleep+0x329 vm_pagezero(0,f409fd38,c0c55614,334,c7340a90,...) at vm_pagezero+0xdc fork_exit(c0ac7f90,0,f409fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf409fd70, ebp = 0 --- Tracing command vmdaemon pid 38 tid 100069 td 0xc732c240 sched_switch(c732c240,0,104,18c,e564f53,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,68,...) at mi_switch+0x200 sleepq_switch(c732c240,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0f2b4bc,68,c0c6572a,0,0,...) at sleepq_wait+0x63 _sleep(c0f2b4bc,c0f2b4c0,68,c0c6572a,0,...) at _sleep+0x35b vm_daemon(0,f409cd38,c0c55614,334,c7340d34,...) at vm_daemon+0x59 fork_exit(c0ac2440,0,f409cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf409cd70, ebp = 0 --- Tracing command pagedaemon pid 9 tid 100068 td 0xc732c480 sched_switch(c732c480,0,104,18c,e5f3b76,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,44,...) at mi_switch+0x200 sleepq_switch(c732c480,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0f2b484,44,c0c6572a,0,0,...) at sleepq_timedwait+0x6b _sleep(c0f2b484,c0f2b450,44,c0c6572a,1388,...) at _sleep+0x329 vm_pageout(0,f4099d38,c0c55614,334,c7341000,...) at vm_pageout+0x2bb fork_exit(c0ac32e0,0,f4099d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4099d70, ebp = 0 --- Tracing command usbus5 pid 37 tid 100067 td 0xc732c6c0 sched_switch(c732c6c0,0,104,18c,dde3ea1c,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732c6c0,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdfd0c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fdfd0c,c6fdfdac,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4036cf8,c07a241b,c6fdfd0c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdfd0c,c6fdfdac,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fdfd04,f4036d38,c0c55614,334,c73412a4,...) at usb_process+0x18b fork_exit(c07a2290,c6fdfd04,f4036d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4036d70, ebp = 0 --- Tracing command usbus5 pid 36 tid 100066 td 0xc732c900 sched_switch(c732c900,0,104,18c,480541f6,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732c900,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdfcdc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fdfcdc,c6fdfdac,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4033cf8,c07a241b,c6fdfcdc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdfcdc,c6fdfdac,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fdfcd4,f4033d38,c0c55614,334,c7341548,...) at usb_process+0x18b fork_exit(c07a2290,c6fdfcd4,f4033d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4033d70, ebp = 0 --- Tracing command usbus5 pid 35 tid 100065 td 0xc732cb40 sched_switch(c732cb40,0,104,18c,dd748b5a,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732cb40,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdfcac,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fdfcac,c6fdfdac,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4030cf8,c07a241b,c6fdfcac,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdfcac,c6fdfdac,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fdfca4,f4030d38,c0c55614,334,c7326000,...) at usb_process+0x18b fork_exit(c07a2290,c6fdfca4,f4030d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4030d70, ebp = 0 --- Tracing command usbus5 pid 34 tid 100064 td 0xc732cd80 sched_switch(c732cd80,0,104,18c,dd747b85,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c732cd80,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fdfc7c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fdfc7c,c6fdfdac,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f402dcf8,c07a241b,c6fdfc7c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fdfc7c,c6fdfdac,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fdfc74,f402dd38,c0c55614,334,c73262a4,...) at usb_process+0x18b fork_exit(c07a2290,c6fdfc74,f402dd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf402dd70, ebp = 0 --- Tracing command usbus4 pid 33 tid 100063 td 0xc70ec240 sched_switch(c70ec240,0,104,18c,dd746ac2,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec240,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd9b5c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd9b5c,c6fd9bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f402acf8,c07a241b,c6fd9b5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd9b5c,c6fd9bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd9b54,f402ad38,c0c55614,334,c7326548,...) at usb_process+0x18b fork_exit(c07a2290,c6fd9b54,f402ad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf402ad70, ebp = 0 --- Tracing command usbus4 pid 32 tid 100062 td 0xc70ec480 sched_switch(c70ec480,0,104,18c,3f309f2f,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec480,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd9b2c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd9b2c,c6fd9bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4027cf8,c07a241b,c6fd9b2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd9b2c,c6fd9bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd9b24,f4027d38,c0c55614,334,c73267ec,...) at usb_process+0x18b fork_exit(c07a2290,c6fd9b24,f4027d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4027d70, ebp = 0 --- Tracing command usbus4 pid 31 tid 100061 td 0xc70ec6c0 sched_switch(c70ec6c0,0,104,18c,dd111803,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec6c0,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd9afc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd9afc,c6fd9bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4024cf8,c07a241b,c6fd9afc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd9afc,c6fd9bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd9af4,f4024d38,c0c55614,334,c7326a90,...) at usb_process+0x18b fork_exit(c07a2290,c6fd9af4,f4024d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4024d70, ebp = 0 --- Tracing command usbus4 pid 30 tid 100060 td 0xc70ec900 sched_switch(c70ec900,0,104,18c,dd11071b,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ec900,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd9acc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd9acc,c6fd9bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4021cf8,c07a241b,c6fd9acc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd9acc,c6fd9bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd9ac4,f4021d38,c0c55614,334,c7326d34,...) at usb_process+0x18b fork_exit(c07a2290,c6fd9ac4,f4021d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4021d70, ebp = 0 --- Tracing command usbus3 pid 29 tid 100059 td 0xc70ecb40 sched_switch(c70ecb40,0,104,18c,dd10f538,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ecb40,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd0b5c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd0b5c,c6fd0bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f401ecf8,c07a241b,c6fd0b5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd0b5c,c6fd0bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd0b54,f401ed38,c0c55614,334,c7327000,...) at usb_process+0x18b fork_exit(c07a2290,c6fd0b54,f401ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf401ed70, ebp = 0 --- Tracing command usbus3 pid 28 tid 100058 td 0xc70ecd80 sched_switch(c70ecd80,0,104,18c,3f152a78,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70ecd80,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd0b2c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd0b2c,c6fd0bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f401bcf8,c07a241b,c6fd0b2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd0b2c,c6fd0bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd0b24,f401bd38,c0c55614,334,c73272a4,...) at usb_process+0x18b fork_exit(c07a2290,c6fd0b24,f401bd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf401bd70, ebp = 0 --- Tracing command usbus3 pid 27 tid 100057 td 0xc7324000 sched_switch(c7324000,0,104,18c,dcad3e5e,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7324000,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd0afc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd0afc,c6fd0bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4018cf8,c07a241b,c6fd0afc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd0afc,c6fd0bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd0af4,f4018d38,c0c55614,334,c7327548,...) at usb_process+0x18b fork_exit(c07a2290,c6fd0af4,f4018d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4018d70, ebp = 0 --- Tracing command usbus3 pid 26 tid 100056 td 0xc7324240 sched_switch(c7324240,0,104,18c,dcad2e7b,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7324240,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fd0acc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fd0acc,c6fd0bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4015cf8,c07a241b,c6fd0acc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fd0acc,c6fd0bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fd0ac4,f4015d38,c0c55614,334,c73277ec,...) at usb_process+0x18b fork_exit(c07a2290,c6fd0ac4,f4015d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4015d70, ebp = 0 --- Tracing command usbus2 pid 25 tid 100055 td 0xc7324480 sched_switch(c7324480,0,104,18c,dcad1e38,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7324480,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdb5c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdb5c,c6fcdbfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4012cf8,c07a241b,c6fcdb5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdb5c,c6fcdbfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fcdb54,f4012d38,c0c55614,334,c6e997ec,...) at usb_process+0x18b fork_exit(c07a2290,c6fcdb54,f4012d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4012d70, ebp = 0 --- Tracing command usbus2 pid 24 tid 100054 td 0xc73246c0 sched_switch(c73246c0,0,104,18c,3ef9b267,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c73246c0,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdb2c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdb2c,c6fcdbfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f400fcf8,c07a241b,c6fcdb2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdb2c,c6fcdbfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fcdb24,f400fd38,c0c55614,334,c6e99a90,...) at usb_process+0x18b fork_exit(c07a2290,c6fcdb24,f400fd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf400fd70, ebp = 0 --- Tracing command usbus2 pid 23 tid 100053 td 0xc7324900 sched_switch(c7324900,0,104,18c,dc492e3b,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7324900,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdafc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdafc,c6fcdbfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f400ccf8,c07a241b,c6fcdafc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdafc,c6fcdbfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fcdaf4,f400cd38,c0c55614,334,c6e99d34,...) at usb_process+0x18b fork_exit(c07a2290,c6fcdaf4,f400cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf400cd70, ebp = 0 --- Tracing command usbus2 pid 22 tid 100052 td 0xc7324b40 sched_switch(c7324b40,0,104,18c,dc491d3f,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7324b40,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fcdacc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fcdacc,c6fcdbfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4009cf8,c07a241b,c6fcdacc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fcdacc,c6fcdbfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fcdac4,f4009d38,c0c55614,334,c7312000,...) at usb_process+0x18b fork_exit(c07a2290,c6fcdac4,f4009d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4009d70, ebp = 0 --- Tracing command usbus1 pid 21 tid 100051 td 0xc7324d80 sched_switch(c7324d80,0,104,18c,dc490c74,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c7324d80,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc8b5c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc8b5c,c6fc8bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4006cf8,c07a241b,c6fc8b5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc8b5c,c6fc8bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc8b54,f4006d38,c0c55614,334,c73122a4,...) at usb_process+0x18b fork_exit(c07a2290,c6fc8b54,f4006d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4006d70, ebp = 0 --- Tracing command usbus1 pid 20 tid 100050 td 0xc6f666c0 sched_switch(c6f666c0,0,104,18c,3ede2c6b,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f666c0,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc8b2c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc8b2c,c6fc8bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4003cf8,c07a241b,c6fc8b2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc8b2c,c6fc8bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc8b24,f4003d38,c0c55614,334,c7312548,...) at usb_process+0x18b fork_exit(c07a2290,c6fc8b24,f4003d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4003d70, ebp = 0 --- Tracing command usbus1 pid 19 tid 100049 td 0xc6f66900 sched_switch(c6f66900,0,104,18c,dbe5858d,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f66900,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc8afc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc8afc,c6fc8bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f4000cf8,c07a241b,c6fc8afc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc8afc,c6fc8bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc8af4,f4000d38,c0c55614,334,c73127ec,...) at usb_process+0x18b fork_exit(c07a2290,c6fc8af4,f4000d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf4000d70, ebp = 0 --- Tracing command usbus1 pid 18 tid 100048 td 0xc6f66b40 sched_switch(c6f66b40,0,104,18c,dbe575c7,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f66b40,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc8acc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc8acc,c6fc8bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f3ffdcf8,c07a241b,c6fc8acc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc8acc,c6fc8bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc8ac4,f3ffdd38,c0c55614,334,c7312a90,...) at usb_process+0x18b fork_exit(c07a2290,c6fc8ac4,f3ffdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ffdd70, ebp = 0 --- Tracing command usbus0 pid 17 tid 100047 td 0xc6f66d80 sched_switch(c6f66d80,0,104,18c,dbe56390,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f66d80,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2b5c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2b5c,c6fc2bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f3ffacf8,c07a241b,c6fc2b5c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2b5c,c6fc2bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc2b54,f3ffad38,c0c55614,334,c7312d34,...) at usb_process+0x18b fork_exit(c07a2290,c6fc2b54,f3ffad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ffad70, ebp = 0 --- Tracing command usbus0 pid 16 tid 100046 td 0xc70eb000 sched_switch(c70eb000,0,104,18c,3ec2e1cb,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70eb000,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2b2c,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2b2c,c6fc2bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f3ff7cf8,c07a241b,c6fc2b2c,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2b2c,c6fc2bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc2b24,f3ff7d38,c0c55614,334,c6d772a4,...) at usb_process+0x18b fork_exit(c07a2290,c6fc2b24,f3ff7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff7d70, ebp = 0 --- Tracing command usbus0 pid 15 tid 100045 td 0xc70eb240 sched_switch(c70eb240,0,104,18c,db7b8042,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70eb240,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2afc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2afc,c6fc2bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f3ff4cf8,c07a241b,c6fc2afc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2afc,c6fc2bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc2af4,f3ff4d38,c0c55614,334,c6d77548,...) at usb_process+0x18b fork_exit(c07a2290,c6fc2af4,f3ff4d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff4d70, ebp = 0 --- Tracing command usbus0 pid 14 tid 100044 td 0xc70eb480 sched_switch(c70eb480,0,104,18c,db7b7001,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70eb480,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6fc2acc,0,c0c490f4,0,0,...) at sleepq_wait+0x63 _sleep(c6fc2acc,c6fc2bfc,0,c0c490f4,0,...) at _sleep+0x35b usb2_msleep(c0c490f4,0,f3ff1cf8,c07a241b,c6fc2acc,...) at usb2_msleep+0x58 usb2_cv_wait(c6fc2acc,c6fc2bfc,c0c49053,51,c0daebc0,...) at usb2_cv_wait+0x23 usb_process(c6fc2ac4,f3ff1d38,c0c55614,334,c6d777ec,...) at usb_process+0x18b fork_exit(c07a2290,c6fc2ac4,f3ff1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3ff1d70, ebp = 0 --- Tracing command sctp_iterator pid 8 tid 100043 td 0xc70eb6c0 sched_switch(c70eb6c0,0,104,18c,db7b5971,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c70eb6c0,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0f2135c,0,c0c7259e,0,0,...) at sleepq_wait+0x63 _sleep(c0f2135c,c0f21270,0,c0c7259e,0,...) at _sleep+0x35b sctp_iterator_thread(0,f3feed38,c0c55614,334,c6d77a90,...) at sctp_iterator_thread+0x60 fork_exit(c0977bb0,0,f3feed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3feed70, ebp = 0 --- Tracing command fdc0 pid 7 tid 100040 td 0xc70ebd80 sched_switch(c70ebd80,0,104,18c,396597af,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c70ebd80,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c6eb2a3c,4c,c0c4fe92,0,0,...) at sleepq_timedwait+0x6b _sleep(c6eb2a3c,c6eb2af0,4c,c0c4fe92,3e8,...) at _sleep+0x329 fdc_thread(c6eb2a00,f3fe2d38,c0c55614,334,c6d77d34,...) at fdc_thread+0x2be fork_exit(c0b45500,c6eb2a00,f3fe2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fe2d70, ebp = 0 --- Tracing command fw0_probe pid 6 tid 100038 td 0xc6f4dd80 sched_switch(c6f4dd80,0,104,18c,7f903754,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c6f4dd80,0,c0c5e290,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5e290,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6ffe000,5c,c0c4fe92,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6ffe000,c7002488,15c,c0c4fe92,0,...) at _sleep+0x344 fw_bus_probe_thread(c6ffe000,f3fd2d38,c0c55614,334,c6e99000,...) at fw_bus_probe_thread+0xa08 fork_exit(c06428d0,c6ffe000,f3fd2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fd2d70, ebp = 0 --- Tracing command xpt_thrd pid 5 tid 100021 td 0xc6f4d480 sched_switch(c6f4d480,0,104,18c,dde94bd0,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6f4d480,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c0d77c54,4c,c0bf0518,0,0,...) at sleepq_wait+0x63 _sleep(c0d77c54,c0d77c6c,4c,c0bf0518,0,...) at _sleep+0x35b xpt_scanner_thread(0,c6acad38,c0c55614,334,c6e992a4,...) at xpt_scanner_thread+0x41 fork_exit(c047f660,0,c6acad38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6acad70, ebp = 0 --- Tracing command yarrow pid 13 tid 100017 td 0xc6d7a240 sched_switch(c6d7a240,0,104,18c,99321197,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d7a240,0,c0c5e290,26a,2,...) at sleepq_switch+0x15f sleepq_timedwait(c0daaca4,0,c0c4fe92,2,0,...) at sleepq_timedwait+0x6b _sleep(c0daaca4,0,0,c0c4fe92,64,...) at _sleep+0x329 pause(c0c4fe92,64,c0c36864,111,0,...) at pause+0x47 random_kthread(0,c6abed38,c0c55614,334,c6e99548,...) at random_kthread+0x1ef fork_exit(c071b210,0,c6abed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abed70, ebp = 0 --- Tracing command g_down pid 4 tid 100015 td 0xc6d7a6c0 sched_switch(c6d7a6c0,0,104,18c,99f1b5c8,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d7a6c0,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da8a44,4c,c0c4fe92,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da8a44,c0da89a8,24c,c0c4fe92,64,...) at _sleep+0x329 g_io_schedule_down(c6d7a6c0,0,c0c5155a,74,0,...) at g_io_schedule_down+0x6b g_down_procbody(0,c6ab8d38,c0c55614,334,c6d76000,...) at g_down_procbody+0x8d fork_exit(c0809100,0,c6ab8d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab8d70, ebp = 0 --- Tracing command g_up pid 3 tid 100014 td 0xc6d7a900 sched_switch(c6d7a900,0,104,18c,9a166d93,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d7a900,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da8a40,4c,c0c4fe92,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da8a40,c0da89c8,24c,c0c4fe92,64,...) at _sleep+0x329 g_io_schedule_up(c6d7a900,0,c0c5155a,5d,0,...) at g_io_schedule_up+0x133 g_up_procbody(0,c6ab5d38,c0c55614,334,c6d762a4,...) at g_up_procbody+0x8d fork_exit(c0809190,0,c6ab5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab5d70, ebp = 0 --- Tracing command g_event pid 2 tid 100013 td 0xc6d7ab40 sched_switch(c6d7ab40,0,104,18c,9931ba34,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,4c,...) at mi_switch+0x200 sleepq_switch(c6d7ab40,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da8a38,4c,c0c4fe92,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da8a38,0,4c,c0c4fe92,64,...) at _sleep+0x329 g_event_procbody(0,c6ab2d38,c0c55614,334,c6d76548,...) at g_event_procbody+0xcb fork_exit(c0809220,0,c6ab2d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ab2d70, ebp = 0 --- Tracing command intr pid 12 tid 100042 td 0xc70eb900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100041 td 0xc70ebb40 sched_switch(c70ebb40,0,109,18c,dde41187,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6d74170,...) at mi_switch+0x200 ithread_loop(c70ef320,f3fe5d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c70ef320,f3fe5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fe5d70, ebp = 0 --- Tracing command intr pid 12 tid 100039 td 0xc70ec000 sched_switch(c70ec000,0,109,18c,97625e9f,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6ff99f0,...) at mi_switch+0x200 ithread_loop(c70e45f0,f3fd5d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c70e45f0,f3fd5d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fd5d70, ebp = 0 --- Tracing command intr pid 12 tid 100036 td 0xc6f64240 sched_switch(c6f64240,0,109,18c,d9e0c827,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbd1f0,...) at mi_switch+0x200 ithread_loop(c6ffa900,f3fc9d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6ffa900,f3fc9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc9d70, ebp = 0 --- Tracing command intr pid 12 tid 100035 td 0xc6f64480 sched_switch(c6f64480,0,109,18c,eb3f6a02,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbd4f0,...) at mi_switch+0x200 ithread_loop(c6ffa170,f3fc1d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6ffa170,f3fc1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fc1d70, ebp = 0 --- Tracing command intr pid 12 tid 100034 td 0xc6f646c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100033 td 0xc6f64900 sched_switch(c6f64900,0,109,18c,87e6c2da,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbd2f0,...) at mi_switch+0x200 ithread_loop(c6fa7430,f3fafd38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6fa7430,f3fafd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fafd70, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xc6f64b40 sched_switch(c6f64b40,0,109,18c,87638d23,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbd370,...) at mi_switch+0x200 ithread_loop(c6fa76b0,f3fabd38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6fa76b0,f3fabd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fabd70, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xc6f64d80 sched_switch(c6f64d80,0,109,18c,86df2a81,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbd3f0,...) at mi_switch+0x200 ithread_loop(c6f90210,f3fa7d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6f90210,f3fa7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fa7d70, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xc6f66000 sched_switch(c6f66000,0,109,18c,9a0c144a,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbd0f0,...) at mi_switch+0x200 ithread_loop(c6f90450,f3c84d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6f90450,f3c84d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3c84d70, ebp = 0 --- Tracing command intr pid 12 tid 100029 td 0xc6f66240 sched_switch(c6f66240,0,109,18c,ee6e7b0,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6f7b1f0,...) at mi_switch+0x200 ithread_loop(c6f91620,c6b12d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6f91620,c6b12d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6b12d70, ebp = 0 --- Tracing command intr pid 12 tid 100028 td 0xc6f66480 sched_switch(c6f66480,0,109,18c,dde520f6,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6d73c70,...) at mi_switch+0x200 ithread_loop(c6f4c3f0,c6b06d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6f4c3f0,c6b06d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6b06d70, ebp = 0 --- Tracing command intr pid 12 tid 100027 td 0xc6dbe6c0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100022 td 0xc6f4d240 sched_switch(c6f4d240,0,109,18c,57c16ec,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6f4e870,...) at mi_switch+0x200 ithread_loop(c6f4b8b0,c6acdd38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6f4b8b0,c6acdd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6acdd70, ebp = 0 --- Tracing command intr pid 12 tid 100020 td 0xc6f4d6c0 sched_switch(c6f4d6c0,0,109,18c,f489b0f2,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6f4e9f0,...) at mi_switch+0x200 ithread_loop(c6f4b920,c6ac7d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6f4b920,c6ac7d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac7d70, ebp = 0 --- Tracing command intr pid 12 tid 100019 td 0xc6f4d900 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100012 td 0xc6d7ad80 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100011 td 0xc6dbe000 sched_switch(c6dbe000,0,109,18c,809f37f6,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbccf0,...) at mi_switch+0x200 ithread_loop(c6d751d0,c6aacd38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6d751d0,c6aacd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aacd70, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xc6dbe240 sched_switch(c6dbe240,0,109,18c,9931ec6c,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbcd70,...) at mi_switch+0x200 ithread_loop(c6d751e0,c6aa9d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6d751e0,c6aa9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa9d70, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xc6dbe480 sched_switch(c6dbe480,0,109,18c,838ce30b,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbcdf0,...) at mi_switch+0x200 ithread_loop(c6d751f0,c6aa6d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6d751f0,c6aa6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa6d70, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xc6d78000 sched_switch(c6d78000,0,109,18c,99cf7a59,...) at sched_switch+0x437 mi_switch(109,0,c0c55893,4e3,c6dbce70,...) at mi_switch+0x200 ithread_loop(c6d75200,c6aa3d38,c0c55614,334,c6d767ec,...) at ithread_loop+0x1f6 fork_exit(c08446b0,c6d75200,c6aa3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6aa3d70, ebp = 0 --- Tracing command intr pid 12 tid 100007 td 0xc6d78240 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xc6d78480 sched_switch(c6d78480,0,60c,187,9a01c10c,...) at sched_switch+0x437 mi_switch(60c,0,c0c5bba0,813,0,...) at mi_switch+0x200 sched_preempt(c6d78480,0,1f4,c6a9cc74,c0b6fd8e,...) at sched_preempt+0x9f ipi_bitmap_handler(8,28,28,c6f78618,c6f78600,...) at ipi_bitmap_handler+0x34 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x2e --- interrupt, eip = 0xc0b64055, esp = 0xc6a9cc74, ebp = 0xc6a9cc74 --- acpi_cpu_c1(ffffffff,c6a9ccd8,0,c088d731,c0dbd3b0,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(c6a9ccb4,c0b7afcb,1,c6a9ccf8,c088d9fe,...) at acpi_cpu_idle+0x11c cpu_idle_acpi(1,c6a9ccf8,c088d9fe,1,c6a9ccd8,...) at cpu_idle_acpi+0x1b cpu_idle(1,c6a9ccd8,c0c5bba0,a06,c6d78480,...) at cpu_idle+0x1b sched_idletd(0,c6a9cd38,c0c55614,334,c6d76a90,...) at sched_idletd+0x21e fork_exit(c088d7e0,0,c6a9cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a9cd70, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xc6d786c0 cpustop_handler(2,c6a99c28,c0b8c7ad,c6ff2a00,8c,...) at cpustop_handler+0x32 ipi_nmi_handler(c6ff2a00,8c,c6d73e80,c6a99bd4,c6d76a90,...) at ipi_nmi_handler+0x2f trap(c6a99c34) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b64055, esp = 0xc6a99c74, ebp = 0xc6a99c74 --- acpi_cpu_c1(ffffffff,c6a99cd8,1,c088d731,c0dbd3b0,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(c6a99cb4,c0b7afcb,0,c6a99cf8,c088d9fe,...) at acpi_cpu_idle+0x11c cpu_idle_acpi(0,c6a99cf8,c088d9fe,0,c6a99cd8,...) at cpu_idle_acpi+0x1b cpu_idle(0,c6a99cd8,c0c5bba0,a06,c6d786c0,...) at cpu_idle+0x1b sched_idletd(0,c6a99d38,c0c55614,334,c6d76a90,...) at sched_idletd+0x21e fork_exit(c088d7e0,0,c6a99d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a99d70, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xc6d78900 cpustop_handler(4,c6a96c28,c0b8c7ad,c6a96bb8,c085b8b4,...) at cpustop_handler+0x32 ipi_nmi_handler(c6a96bb8,c085b8b4,c0daaef0,4,c6d76a90,...) at ipi_nmi_handler+0x2f trap(c6a96c34) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b64055, esp = 0xc6a96c74, ebp = 0xc6a96c74 --- acpi_cpu_c1(ffffffff,c6a96cd8,2,c088d731,c0dbd3b0,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(c6a96cb4,c0b7afcb,0,c6a96cf8,c088d9fe,...) at acpi_cpu_idle+0x11c cpu_idle_acpi(0,c6a96cf8,c088d9fe,0,c6a96cd8,...) at cpu_idle_acpi+0x1b cpu_idle(0,c6a96cd8,c0c5bba0,a06,c6d78900,...) at cpu_idle+0x1b sched_idletd(0,c6a96d38,c0c55614,334,c6d76a90,...) at sched_idletd+0x21e fork_exit(c088d7e0,0,c6a96d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a96d70, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xc6d78b40 cpustop_handler(8,c6a93c28,c0b8c7ad,c6a93bb8,c085b8b4,...) at cpustop_handler+0x32 ipi_nmi_handler(c6a93bb8,c085b8b4,c0daaf28,4,c6d76a90,...) at ipi_nmi_handler+0x2f trap(c6a93c34) at trap+0x2d calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0b64055, esp = 0xc6a93c74, ebp = 0xc6a93c74 --- acpi_cpu_c1(ffffffff,c6a93cd8,3,c088d731,c0dbd3b0,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(c6a93cb4,c0b7afcb,0,c6a93cf8,c088d9fe,...) at acpi_cpu_idle+0x11c cpu_idle_acpi(0,c6a93cf8,c088d9fe,0,c6a93cd8,...) at cpu_idle_acpi+0x1b cpu_idle(0,c6a93cd8,c0c5bba0,a06,c6d78b40,...) at cpu_idle+0x1b sched_idletd(0,c6a93d38,c0c55614,334,c6d76a90,...) at sched_idletd+0x21e fork_exit(c088d7e0,0,c6a93d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a93d70, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xc6d78d80 sched_switch(c6d78d80,0,104,18c,54269853,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,5c,...) at mi_switch+0x200 sleepq_switch(c6d78d80,0,c0c5e290,189,5c,...) at sleepq_switch+0x15f sleepq_catch_signals(c0c5e290,159,0,100,100,...) at sleepq_catch_signals+0xb6 sleepq_wait_sig(c6d76d34,5c,c0c60afd,100,0,...) at sleepq_wait_sig+0x17 _sleep(c6d76d34,c6d76dbc,15c,c0c60afd,0,...) at _sleep+0x344 kern_wait(c6d78d80,ffffffff,c6a8fc74,2,0,...) at kern_wait+0xac6 wait4(c6d78d80,c6a8fcf8,10,c6a8fd38,c0d3dec4,...) at wait4+0x3b syscall(c6a8fd38) at syscall+0x2b4 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x8054c5f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8c8 --- Tracing command audit pid 10 tid 100001 td 0xc6d7a000 sched_switch(c6d7a000,0,104,18c,dde6bcfd,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d7a000,0,c0c5e290,247,c6d7a000,...) at sleepq_switch+0x15f sleepq_wait(c0f2a580,0,c6a8cc9c,1,0,...) at sleepq_wait+0x63 _cv_wait(c0f2a580,c0f2a564,c0c7cf8c,194,0,...) at _cv_wait+0x240 audit_worker(0,c6a8cd38,c0c55614,334,c6d77000,...) at audit_worker+0x84 fork_exit(c0a5e3f0,0,c6a8cd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6a8cd70, ebp = 0 --- Tracing command kernel pid 0 tid 100037 td 0xc6f64000 sched_switch(c6f64000,0,104,18c,d9e10594,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f64000,0,c0c5e290,247,c6f64000,...) at sleepq_switch+0x15f sleepq_wait(c700f880,0,c0c5ab1c,c0c4fe92,0,...) at sleepq_wait+0x63 msleep_spin(c700f880,c700f89c,c0c4fe92,0,c0c587b4,...) at msleep_spin+0x21d taskqueue_thread_loop(c700249c,f3fcfd38,c0c55614,334,c0da8b00,...) at taskqueue_thread_loop+0x8e fork_exit(c08a49a0,c700249c,f3fcfd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xf3fcfd70, ebp = 0 --- Tracing900,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f29000,0,c0c4fe92,0,0,...) at sleepq_wait+0x63 _sleep(c6f29000,c6f2901c,0,c0c4fe92,0,...) at _sleep+0x35b taskqueue_thread_loop(c0da93c4,c6ad9d38,c0c55614,334,c0da8b00,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a49a0,c0da93c4,c6ad9d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad9d70, ebp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xc6dbeb40 sched_switch(c6dbeb40,0,104,18c,ddf9ea8e,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6dbeb40,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f29040,0,c0c4fe92,0,0,...) at sleepq_wait+0x63 _sleep(c6f29040,c6f2905c,0,c0c4fe92,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d7a960,c6ad6d38,c0c55614,334,c0da8b00,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a49a0,c0d7a960,c6ad6d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad6d70, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xc6dbed80 sched_switch(c6dbed80,0,104,18c,ddf9d9cd,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6dbed80,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f29040,0,c0c4fe92,0,0,...) at sleepq_wait+0x63 _sleep(c6f29040,c6f2905c,0,c0c4fe92,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d7a960,c6ad3d38,c0c55614,334,c0da8b00,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a49a0,c0d7a960,c6ad3d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad3d70, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xc6f4d000 sched_switch(c6f4d000,0,104,18c,ddf9c77d,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4d000,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f29040,0,c0c4fe92,0,0,...) at sleepq_wait+0x63 _sleep(c6f29040,c6f2905c,0,c0c4fe92,0,...) at _sleep+0x35b taskqueue_thread_loop(c0d7a960,c6ad0d38,c0c55614,334,c0da8b00,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a49a0,c0d7a960,c6ad0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ad0d70, ebp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xc6f4db40 sched_switch(c6f4db40,0,104,18c,57f5fe8,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6f4db40,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6f29440,0,c0c4fe92,0,0,...) at sleepq_wait+0x63 _sleep(c6f29440,c6f2945c,0,c0c4fe92,0,...) at _sleep+0x35b taskqueue_thread_loop(c0dbd5a8,c6ac1d38,c0c55614,334,c0da8b00,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a49a0,c0dbd5a8,c6ac1d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6ac1d70, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xc6d7a480 sched_switch(c6d7a480,0,104,18c,4afab617,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,0,...) at mi_switch+0x200 sleepq_switch(c6d7a480,0,c0c5e290,247,0,...) at sleepq_switch+0x15f sleepq_wait(c6d5eb40,0,c0c4fe92,0,0,...) at sleepq_wait+0x63 _sleep(c6d5eb40,c6d5eb5c,0,c0c4fe92,0,...) at _sleep+0x35b taskqueue_thread_loop(c0dbc0e0,c6abbd38,c0c55614,334,c0da8b00,...) at taskqueue_thread_loop+0xb4 fork_exit(c08a49a0,c0dbc0e0,c6abbd38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc6abbd70, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc0da8db0 sched_switch(c0da8db0,0,104,18c,166d8954,...) at sched_switch+0x437 mi_switch(104,0,c0c5e290,1d2,44,...) at mi_switch+0x200 sleepq_switch(c0da8db0,0,c0c5e290,26a,0,...) at sleepq_switch+0x15f sleepq_timedwait(c0da8b00,44,c0c5c415,0,0,...) at sleepq_timedwait+0x6b _sleep(c0da8b00,0,44,c0c5c415,2710,...) at _sleep+0x329 scheduler(0,141ec00,141ec00,141e000,1425000,...) at scheduler+0x23e mi_startup() at mi_startup+0x96 begin() at begin+0x2c db:1:allt> call doadump Physical memory: 3439 MB Dumping 98 MB: 83 67 51 35 19 3 Dump complete = 0xf db:1:doadump> reset (kgdb) bt #0 doadump () at pcpu.h:246 #1 0xc04c3d09 in db_fncall (dummy1=0xc08542af, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf62e3554 "h5.ö\001") at ../../../ddb/db_command.c:548 #2 0xc04c413f in db_command (last_cmdp=0xc0d78c3c, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #3 0xc04c41f4 in db_command_script (command=0xc0d79bc8 "call doadump") at ../../../ddb/db_command.c:516 #4 0xc04c82f0 in db_script_exec (scriptname=0xc0d794a0 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc04c8381 in db_run_cmd (addr=0x0, have_addr=0x0, count=0x0, modif=0xf62e368c " 6.ö") at ../../../ddb/db_script.c:375 #6 0xc04c413f in db_command (last_cmdp=0xc0d78c3c, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:445 #7 0xc04c41f4 in db_command_script (command=0xc0d79b00 "run pho") at ../../../ddb/db_command.c:516 #8 0xc04c82f0 in db_script_exec (scriptname=0xf62e3798 "kdb.enter.unknown", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #9 0xc04c83d7 in db_script_kdbenter (eventname=0xc0c8eff0 "unknown") at ../../../ddb/db_script.c:324 #10 0xc04c60b8 in db_trap (type=0xc, code=0x0) at ../../../ddb/db_main.c:228 #11 0xc0898d26 in kdb_trap (type=0xc, code=0x0, tf=0xf62e3918) at ../../../kern/subr_kdb.c:534 #12 0xc0b8bf8f in trap_fatal (frame=0xf62e3918, eva=0x18c) at ../../../i386/i386/trap.c:924 #13 0xc0b8c901 in trap (frame=0xf62e3918) at ../../../i386/i386/trap.c:325 #14 0xc0b6f69b in calltrap () at ../../../i386/i386/exception.s:165 #15 0xc08542af in __lockmgr_args (lk=0xe6d00ffc, flags=0x81900, ilk=0xc71cf3a4, wmesg=0xc0c65811 "getblk", pri=0x50, timo=0x0, file=0xc0c64de6 "../../../kern/vfs_bio.c", line=0x9fe) at ../../../kern/kern_lock.c:671 #16 0xc08e2fd3 in getblk (vp=0xc71cf2b0, blkno=0x1, size=0x4000, slpflag=0x0, slptimeo=0x0, flags=0x0) at lockmgr.h:94 #17 0xc08e717a in cluster_read (vp=0xc71cf2b0, filesize=0xa000, lblkno=0x2, size=0x4000, cred=0x0, totread=0x3000, seqcount=0x0, bpp=0xf62e3b64) at ../../../kern/vfs_cluster.c:120 #18 0xc0a94289 in ffs_read (ap=0xf62e3bc0) at ../../../ufs/ffs/ffs_vnops.c:514 #19 0xc0b984a5 in VOP_READ_APV (vop=0xc0d5eb00, a=0xf62e3bc0) at vnode_if.c:887 #20 0xc0906458 in vn_read (fp=0xc74233f0, uio=0xf62e3c58, active_cred=0xc6d44300, flags=0x1, td=0xc74b6b40) at vnode_if.h:384 #21 0xc08ae2f6 in dofileread (td=0xc74b6b40, fd=0x5, fp=0xc74233f0, auio=0xf62e3c58, offset=0x6000, flags=0x1) at file.h:227 #22 0xc08ae3d8 in kern_preadv (td=0xc74b6b40, fd=0x5, auio=0xf62e3c58, offset=0x6000) at ../../../kern/sys_generic.c:284 #23 0xc08ae50d in pread (td=0xc74b6b40, uap=0xf62e3cf8) at ../../../kern/sys_generic.c:185 #24 0xc0b8c4a4 in syscall (frame=0xf62e3d38) at ../../../i386/i386/trap.c:1073 #25 0xc0b6f700 in Xint0x80_syscall () at ../../../i386/i386/exception.s:261 #26 0x00000033 in ?? () (kgdb) f 15 #15 0xc08542af in __lockmgr_args (lk=0xe6d00ffc, flags=0x81900, ilk=0xc71cf3a4, wmesg=0xc0c65811 "getblk", pri=0x50, timo=0x0, file=0xc0c64de6 "../../../kern/vfs_bio.c", line=0x9fe) at ../../../kern/kern_lock.c:671 671 owner = (struct thread *)LK_HOLDER(x); (kgdb) l 666 * the owner stops running or the state of the lock 667 * changes. 668 */ 669 x = lk->lk_lock; 670 if (LK_CAN_ADAPT(lk) && (x & LK_SHARE) == 0) { 671 owner = (struct thread *)LK_HOLDER(x); 672 if (TD_IS_RUNNING(owner)) { 673 if (LOCK_LOG_TEST(&lk->lock_object, 0)) 674 CTR3(KTR_LOCK, 675 "%s: spinning on %p held by %p", (kgdb) p x $3 = 0xfffffff0 (kgdb) p *lk $4 = {lock_object = {lo_name = 0xc0c658dc "bufwait", lo_flags = 0x5730000, lo_data = 0x0, lo_witness = 0xc6d2bd58}, lk_lock = 0x1, lk_timo = 0x0, lk_pri = 0x50, lk_stack = {depth = 0xc, pcs = {0xc0854775, 0xc08e25f3, 0xc08e3282, 0xc08e76af, 0xc0a94289, 0xc0b984a5, 0xc0906458, 0xc08ae2f6, 0xc08ae628, 0xc08ae73f, 0xc0b8c4a4, 0xc0b6f700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}} (kgdb) $ svn diff /usr/src/sys Index: /usr/src/sys/conf/options =================================================================== --- /usr/src/sys/conf/options (revision 193624) +++ /usr/src/sys/conf/options (working copy) @@ -131,6 +131,7 @@ MFI_DECODE_LOG opt_mfi.h MPROF_BUFFERS opt_mprof.h MPROF_HASH_SIZE opt_mprof.h +NO_ADAPTIVE_LOCKMGRS NO_ADAPTIVE_MUTEXES opt_adaptive_mutexes.h NO_ADAPTIVE_RWLOCKS NO_ADAPTIVE_SX Index: /usr/src/sys/kern/kern_lock.c =================================================================== --- /usr/src/sys/kern/kern_lock.c (revision 193624) +++ /usr/src/sys/kern/kern_lock.c (working copy) @@ -28,12 +28,14 @@ #include "opt_ddb.h" #include "opt_kdtrace.h" +#include "opt_no_adaptive_lockmgrs.h" #include __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -43,6 +45,7 @@ #ifdef DEBUG_LOCKS #include #endif +#include #include #include @@ -51,8 +54,15 @@ #include #endif -CTASSERT((LK_NOSHARE & LO_CLASSFLAGS) == LK_NOSHARE); +CTASSERT(((LK_NOADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) == + (LK_NOADAPTIVE | LK_NOSHARE)); +CTASSERT(LK_UNLOCKED == (LK_UNLOCKED & + ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS))); +#if defined(SMP) && !defined(NO_ADAPTIVE_LOCKMGRS) +#define ADAPTIVE_LOCKMGRS +#endif + #define SQ_EXCLUSIVE_QUEUE 0 #define SQ_SHARED_QUEUE 1 @@ -106,6 +116,7 @@ #define LK_CAN_SHARE(x) \ (((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 || \ + ((x) & LK_EXCLUSIVE_SPINNERS) == 0 || \ curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT))) #define LK_TRYOP(x) \ ((x) & LK_NOWAIT) @@ -115,6 +126,9 @@ #define LK_TRYWIT(x) \ (LK_TRYOP(x) ? LOP_TRYLOCK : 0) +#define LK_CAN_ADAPT(lk) \ + (((lk)->lock_object.lo_flags & LK_NOADAPTIVE) == 0) + #define lockmgr_disowned(lk) \ (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC) @@ -145,6 +159,14 @@ #endif }; +#ifdef ADAPTIVE_LOCKMGRS +static u_int alk_retries = 10; +static u_int alk_loops = 10000; +SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging"); +SYSCTL_UINT(_debug_lockmgr, OID_AUTO, retries, CTLFLAG_RW, &alk_retries, 0, ""); +SYSCTL_UINT(_debug_lockmgr, OID_AUTO, loops, CTLFLAG_RW, &alk_loops, 0, ""); +#endif + static __inline struct thread * lockmgr_xholder(struct lock *lk) { @@ -233,9 +255,9 @@ * lock quickly. */ if ((x & LK_ALL_WAITERS) == 0) { - MPASS(x == LK_SHARERS_LOCK(1)); - if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1), - LK_UNLOCKED)) + MPASS((x & ~LK_EXCLUSIVE_SPINNERS) == + LK_SHARERS_LOCK(1)); + if (atomic_cmpset_ptr(&lk->lk_lock, x, LK_UNLOCKED)) break; continue; } @@ -245,7 +267,7 @@ * path in order to handle wakeups correctly. */ sleepq_lock(&lk->lock_object); - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); v = LK_UNLOCKED; /* @@ -256,7 +278,8 @@ queue = SQ_EXCLUSIVE_QUEUE; v |= (x & LK_SHARED_WAITERS); } else { - MPASS(x == LK_SHARED_WAITERS); + MPASS((x & ~LK_EXCLUSIVE_SPINNERS) == + LK_SHARED_WAITERS); queue = SQ_SHARED_QUEUE; } @@ -326,7 +349,7 @@ iflags |= LO_WITNESS; if (flags & LK_QUIET) iflags |= LO_QUIET; - iflags |= flags & LK_NOSHARE; + iflags |= flags & (LK_NOADAPTIVE | LK_NOSHARE); lk->lk_lock = LK_UNLOCKED; lk->lk_recurse = 0; @@ -359,6 +382,10 @@ uint64_t waittime = 0; int contested = 0; #endif +#ifdef ADAPTIVE_LOCKMGRS + volatile struct thread *owner; + u_int i, spintries = 0; +#endif error = 0; tid = (uintptr_t)curthread; @@ -436,7 +463,46 @@ break; } +#ifdef ADAPTIVE_LOCKMGRS /* + * If the owner is running on another CPU, spin until + * the owner stops running or the state of the lock + * changes. + */ + if (LK_CAN_ADAPT(lk) && (x & LK_SHARE) == 0) { + owner = (struct thread *)LK_HOLDER(x); + if (TD_IS_RUNNING(owner)) { + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR3(KTR_LOCK, + "%s: spinning on %p held by %p", + __func__, lk, owner); + GIANT_SAVE(); + while (LK_HOLDER(lk->lk_lock) == + (uintptr_t)owner && + TD_IS_RUNNING(owner)) + cpu_spinwait(); + } + } else if (LK_CAN_ADAPT(lk) && LK_SHARERS(x) && + spintries < alk_retries) { + spintries++; + GIANT_SAVE(); + for (i = 0; i < alk_loops; i++) { + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR4(KTR_LOCK, + "%s: shared spinning on %p with %u and %u", + __func__, lk, spintries, i); + x = lk->lk_lock; + if ((x & LK_SHARE) == 0 || + LK_CAN_SHARE(x) != 0) + break; + cpu_spinwait(); + } + if (i != alk_loops) + continue; + } +#endif + + /* * Acquire the sleepqueue chain lock because we * probabilly will need to manipulate waiters flags. */ @@ -452,7 +518,24 @@ continue; } +#ifdef ADAPTIVE_LOCKMGRS /* + * The current lock owner might have started executing + * on another CPU (or the lock could have changed + * owner) while we were waiting on the turnstile + * chain lock. If so, drop the turnstile lock and try + * again. + */ + if (LK_CAN_ADAPT(lk) && (x & LK_SHARE) == 0) { + owner = (struct thread *)LK_HOLDER(x); + if (TD_IS_RUNNING(owner)) { + sleepq_release(&lk->lock_object); + continue; + } + } +#endif + + /* * Try to set the LK_SHARED_WAITERS flag. If we fail, * loop back and retry. */ @@ -497,13 +580,15 @@ break; case LK_UPGRADE: _lockmgr_assert(lk, KA_SLOCKED, file, line); - x = lk->lk_lock & LK_ALL_WAITERS; + v = lk->lk_lock; + x = v & LK_ALL_WAITERS; + v &= LK_EXCLUSIVE_SPINNERS; /* * Try to switch from one shared lock to an exclusive one. * We need to preserve waiters flags during the operation. */ - if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x, + if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, tid | x)) { LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, line); @@ -575,13 +660,55 @@ break; } +#ifdef ADAPTIVE_LOCKMGRS /* + * If the owner is running on another CPU, spin until + * the owner stops running or the state of the lock + * changes. + */ + x = lk->lk_lock; + if (LK_CAN_ADAPT(lk) && (x & LK_SHARE) == 0) { + owner = (struct thread *)LK_HOLDER(x); + if (TD_IS_RUNNING(owner)) { + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR3(KTR_LOCK, + "%s: spinning on %p held by %p", + __func__, lk, owner); + GIANT_SAVE(); + while (LK_HOLDER(lk->lk_lock) == + (uintptr_t)owner && + TD_IS_RUNNING(owner)) + cpu_spinwait(); + } + } else if (LK_CAN_ADAPT(lk) && LK_SHARERS(x) && + spintries < alk_retries) { + if ((x & LK_EXCLUSIVE_SPINNERS) == 0 && + !atomic_cmpset_ptr(&lk->lk_lock, x, + x | LK_EXCLUSIVE_SPINNERS)) + continue; + spintries++; + GIANT_SAVE(); + for (i = 0; i < alk_loops; i++) { + if (LOCK_LOG_TEST(&lk->lock_object, 0)) + CTR4(KTR_LOCK, + "%s: shared spinning on %p with %u and %u", + __func__, lk, spintries, i); + if ((lk->lk_lock & + LK_EXCLUSIVE_SPINNERS) == 0) + break; + cpu_spinwait(); + } + if (i != alk_loops) + continue; + } +#endif + + /* * Acquire the sleepqueue chain lock because we * probabilly will need to manipulate waiters flags. */ sleepq_lock(&lk->lock_object); x = lk->lk_lock; - v = x & LK_ALL_WAITERS; /* * if the lock has been released while we spun on @@ -592,7 +719,24 @@ continue; } +#ifdef ADAPTIVE_LOCKMGRS /* + * The current lock owner might have started executing + * on another CPU (or the lock could have changed + * owner) while we were waiting on the turnstile + * chain lock. If so, drop the turnstile lock and try + * again. + */ + if (LK_CAN_ADAPT(lk) && (x & LK_SHARE) == 0) { + owner = (struct thread *)LK_HOLDER(x); + if (TD_IS_RUNNING(owner)) { + sleepq_release(&lk->lock_object); + continue; + } + } +#endif + + /* * The lock can be in the state where there is a * pending queue of waiters, but still no owner. * This happens when the lock is contested and an @@ -601,7 +745,9 @@ * claim lock ownership and return, preserving waiters * flags. */ - if (x == (LK_UNLOCKED | v)) { + v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); + if ((x & ~v) == LK_UNLOCKED) { + v &= ~LK_EXCLUSIVE_SPINNERS; if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, tid | v)) { sleepq_release(&lk->lock_object); @@ -666,7 +812,9 @@ * In order to preserve waiters flags, just spin. */ for (;;) { - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock; + MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); + x &= LK_ALL_WAITERS; if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, LK_SHARERS_LOCK(1) | x)) break; @@ -712,7 +860,7 @@ break; sleepq_lock(&lk->lock_object); - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock; v = LK_UNLOCKED; /* @@ -720,11 +868,13 @@ * preference in order to avoid deadlock with * shared runners up. */ + MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); if (x & LK_EXCLUSIVE_WAITERS) { queue = SQ_EXCLUSIVE_QUEUE; v |= (x & LK_SHARED_WAITERS); } else { - MPASS(x == LK_SHARED_WAITERS); + MPASS((x & LK_ALL_WAITERS) == + LK_SHARED_WAITERS); queue = SQ_SHARED_QUEUE; } @@ -777,7 +927,6 @@ */ sleepq_lock(&lk->lock_object); x = lk->lk_lock; - v = x & LK_ALL_WAITERS; /* * if the lock has been released while we spun on @@ -788,8 +937,9 @@ continue; } - if (x == (LK_UNLOCKED | v)) { - v = x; + v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); + if ((x & ~v) == LK_UNLOCKED) { + v = (x & ~LK_EXCLUSIVE_SPINNERS); if (v & LK_EXCLUSIVE_WAITERS) { queue = SQ_EXCLUSIVE_QUEUE; v &= ~LK_EXCLUSIVE_WAITERS; @@ -902,7 +1052,9 @@ * In order to preserve waiters flags, just spin. */ for (;;) { - x = lk->lk_lock & LK_ALL_WAITERS; + x = lk->lk_lock; + MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); + x &= LK_ALL_WAITERS; if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, LK_KERNPROC | x)) return; @@ -933,6 +1085,8 @@ printf(" with exclusive waiters pending\n"); if (x & LK_SHARED_WAITERS) printf(" with shared waiters pending\n"); + if (x & LK_EXCLUSIVE_SPINNERS) + printf(" with exclusive spinners pending\n"); STACK_PRINT(lk); } @@ -1094,5 +1248,10 @@ default: db_printf("none\n"); } + db_printf(" spinners: "); + if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS) + db_printf("exclusive\n"); + else + db_printf("none\n"); } #endif Index: /usr/src/sys/sys/lockmgr.h =================================================================== --- /usr/src/sys/sys/lockmgr.h (revision 193624) +++ /usr/src/sys/sys/lockmgr.h (working copy) @@ -39,13 +39,14 @@ #define LK_SHARE 0x01 #define LK_SHARED_WAITERS 0x02 #define LK_EXCLUSIVE_WAITERS 0x04 +#define LK_EXCLUSIVE_SPINNERS 0x08 #define LK_ALL_WAITERS \ (LK_SHARED_WAITERS | LK_EXCLUSIVE_WAITERS) #define LK_FLAGMASK \ - (LK_SHARE | LK_ALL_WAITERS) + (LK_SHARE | LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS) #define LK_HOLDER(x) ((x) & ~LK_FLAGMASK) -#define LK_SHARERS_SHIFT 3 +#define LK_SHARERS_SHIFT 4 #define LK_SHARERS(x) (LK_HOLDER(x) >> LK_SHARERS_SHIFT) #define LK_SHARERS_LOCK(x) ((x) << LK_SHARERS_SHIFT | LK_SHARE) #define LK_ONE_SHARER (1 << LK_SHARERS_SHIFT) @@ -141,6 +142,7 @@ #define LK_NOSHARE 0x000008 #define LK_NOWITNESS 0x000010 #define LK_QUIET 0x000020 +#define LK_NOADAPTIVE 0x000040 /* * Additional attributes to be used in lockmgr().