GENERIC HEAD from 2012-06-07 23:33:10 UTC, r236681M, vmcore.395 KDB: debugger backends: ddb KDB: current backend: ddb 524288K of memory above 4GB ignored Copyright (c) 1992-2012 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 10.0-CURRENT #0 r236735M: Fri Jun 8 08:44:48 CEST 2012 pho@x4.osted.lan:/usr/src/sys/i386/compile/MEMGUARD i386 WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. MEMGUARD DEBUGGING ALLOCATOR INITIALIZED: MEMGUARD map base: 0xc7c00000 MEMGUARD map limit: 0xca534000 MEMGUARD map size: 42192 KBytes CPU: AMD Phenom(tm) 9150e Quad-Core Processor (1800.02-MHz 686-class CPU) Origin = "AuthenticAMD" Id = 0x100f23 Family = 10 Model = 2 Stepping = 3 Features=0x178bfbff Features2=0x802009 AMD Features=0xee500800 AMD Features2=0x7ff TSC: P-state invariant real memory = 4294967296 (4096 MB) avail memory = 3531436032 (3367 MB) : Trying to mount root from ufs:/dev/ad4s1a [rw]... Setting hostuuid: 00000000-0000-0000-0000-00218515337d. Setting hostid: 0x6b64ac17. Starting ddb. Entropy harvesting: interrupts ethernet point_to_point kickstart. Starting file system checks: /dev/ad4s1a: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1a: clean, 611909 free (1349 frags, 76320 blocks, 0.1% fragmentation) /dev/ad4s1f: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1f: clean, 122774 free (5246 frags, 14691 blocks, 0.5% fragmentation) /dev/label/tmp: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/label/tmp: clean, 80774628 free (1212 frags, 10096677 blocks, 0.0% fragmentation) /dev/ad4s1d: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1d: clean, 2012213 free (215533 frags, 224585 blocks, 2.1% fragmentation) /dev/ad4s1e: FILE SYSTEM CLEAN; SKIPPING CHECKS /dev/ad4s1e: clean, 12023988 free (77284 frags, 1493338 blocks, 0.3% fragmentation) Mounting local file systems:. panic: _mtx_lock_sleep: recursed on non-recursive mutex Softdep Lock @ ../../../ufs/ffs/ffs_softdep.c:1210 cpuid = 3 KDB: enter: panic [ thread pid 92 tid 100109 ] Stopped at kdb_enter+0x3a: movl $0,kdb_why db> run pho db:0:pho> bt Tracing pid 92 tid 100109 td 0xce2075e0 kdb_enter(c0fdee48,c0fdee48,c0fdd18d,f27886f8,3,...) at kdb_enter+0x3a panic(c0fdd18d,c1017255,c101478c,4ba,ca6b7058,...) at panic+0x18c _mtx_lock_sleep(c131af20,ce2075e0,0,c101478c,4ba,...) at _mtx_lock_sleep+0x51 _mtx_lock_flags(c131af20,0,c101478c,4ba,cdfea4fc,...) at _mtx_lock_flags+0x10f workitem_alloc(f27887b4,4,c101478c,12f8,e190b9a0,...) at workitem_alloc+0x3d bmsafemap_lookup(ce209780,0,c101478c,1271,ce87e800,...) at bmsafemap_lookup+0x161 softdep_setup_inomapdep(e190b9a0,ce5e7414,5c0000,41ed,0,...) at softdep_setup_inomapdep+0xf0 ffs_nodealloccg(ce5e7414,100,5c0000,0,41ed,...) at ffs_nodealloccg+0x5ab ffs_hashalloc(5c0000,0,41ed,0,c0cbae50,...) at ffs_hashalloc+0x85 ffs_valloc(ce878b40,41ed,ca71ee80,f27889bc,e1906d40,...) at ffs_valloc+0x4d3 ufs_mkdir(f2788bec,c103a707,c1158340,ce878b40,f2788c00,...) at ufs_mkdir+0xba VOP_MKDIR_APV(c1145ae0,f2788bec,f2788b5c,f2788b84,0,...) at VOP_MKDIR_APV+0x100 kern_mkdirat(ce2075e0,ffffff9c,bfbfef32,0,1ff,...) at kern_mkdirat+0x291 kern_mkdir(ce2075e0,bfbfef32,0,1ff,f2788cfc,...) at kern_mkdir+0x2e sys_mkdir(ce2075e0,f2788ccc,c10330bc,c0fddd52,c0aba4b7,...) at sys_mkdir+0x29 syscall(f2788d08) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (136, FreeBSD ELF32, sys_mkdir), eip = 0x28169743, esp = 0xbfbfed0c, ebp = 0xbfbfedd8 --- db:0:bt> show allpcpu Current CPU: 3 cpuid = 0 dynamic pcpu = 0x6c6380 curthread = 0xca7518d0: pid 11 "idle: cpu0" curpcb = 0xc7b25d60 fpcurthread = none idlethread = 0xca7518d0: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 dynamic pcpu = 0x69b6380 curthread = 0xca7515e0: pid 11 "idle: cpu1" curpcb = 0xc7b28d60 fpcurthread = none idlethread = 0xca7515e0: tid 100004 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 dynamic pcpu = 0x69b9380 curthread = 0xca7512f0: pid 11 "idle: cpu2" curpcb = 0xc7b2bd60 fpcurthread = none idlethread = 0xca7512f0: tid 100005 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 dynamic pcpu = 0x69bc380 curthread = 0xce2075e0: pid 92 "mkdir" curpcb = 0xf2788d60 fpcurthread = none idlethread = 0xca751000: tid 100006 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: db:0:allpcpu> show alllocks Process 92 (mkdir) thread 0xce2075e0 (100109) exclusive sleep mutex Softdep Lock (Softdep Lock) r = 0 (0xc131af20) locked @ ufs/ffs/ffs_softdep.c:2118 exclusive lockmgr bufwait (bufwait) r = 0 (0xe190ba00) locked @ kern/vfs_bio.c:1905 exclusive lockmgr ufs (ufs) r = 0 (0xce878b98) locked @ kern/vfs_subr.c:2158 db:0:alllocks> show lockedvnods Locked vnodes 0xce878b40: tag ufs, type VDIR usecount 1, writecount 0, refcount 4 mountedhere 0 flags (VV_ROOT|VI(0x200)) v_object 0xcc038250 ref 0 pages 1 lock type ufs: EXCL by thread 0xce2075e0 (pid 92, mkdir, tid 100109) #0 0xc0a74d12 at __lockmgr_args+0x662 #1 0xc0cedac1 at ffs_lock+0xa1 #2 0xc0e329c3 at VOP_LOCK1_APV+0xf3 #3 0xc0b41f08 at _vn_lock+0x78 #4 0xc0b343fb at vget+0xbb #5 0xc0b22a6d at vfs_hash_get+0xed #6 0xc0ce6dc9 at ffs_vgetf+0x49 #7 0xc0ce737e at ffs_vget+0x2e #8 0xc0cf9b18 at ufs_root+0x28 #9 0xc0b24201 at lookup+0xa21 #10 0xc0b24f2f at namei+0x6ff #11 0xc0b3745b at kern_mkdirat+0xbb #12 0xc0b3770e at kern_mkdir+0x2e #13 0xc0b37739 at sys_mkdir+0x29 #14 0xc0e0bb9e at syscall+0x2fe #15 0xc0df59f1 at Xint0x80_syscall+0x21 ino 2, on dev ada0s1e db:0:lockedvnods> show mount 0xce527b70 /dev/ad4s1a on / (ufs) 0xce528000 devfs on /dev (devfs) 0xce527894 /dev/ad4s1f on /home (ufs) 0xce5275b8 /dev/label/tmp on /tmp (ufs) 0xce5272dc /dev/ad4s1d on /usr (ufs) 0xce527000 /dev/ad4s1e on /var (ufs) 0xce88b5b8 procfs on /proc (procfs) More info: show mount db:0:mount> ps pid ppid pgrp uid state wmesg wchan cmd 92 91 20 0 R+ CPU 3 mkdir 91 20 20 0 S+ wait 0xce5932cc sh 20 1 20 0 Ss+ wait 0xce55bb30 sh 19 0 0 0 DL sdflush 0xc131af60 [softdepflush] 18 0 0 0 DL syncer 0xc1313254 [syncer] 17 0 0 0 DL vlruwt 0xce1fa598 [vnlru] 16 0 0 0 DL psleep 0xc1312ee8 [bufdaemon] 9 0 0 0 DL pgzero 0xc131cf5c [pagezero] 8 0 0 0 DL psleep 0xc131cac8 [vmdaemon] 7 0 0 0 DL psleep 0xc131ca90 [pagedaemon] 6 0 0 0 DL ccb_scan 0xc115f9d4 [xpt_thrd] 5 0 0 0 DL waiting_ 0xc1316778 [sctp_iterator] 4 0 0 0 DL - 0xcc3ab23c [fdc0] 3 0 0 0 SL - 0xcc3b0000 [fw0_probe] 15 0 0 0 DL (threaded) [usb] 100059 D - 0xcc310d34 [usbus5] 100058 D - 0xcc310d04 [usbus5] 100057 D - 0xcc310cd4 [usbus5] 100056 D - 0xcc310ca4 [usbus5] 100054 D - 0xcc2ffb5c [usbus4] 100053 D - 0xcc2ffb2c [usbus4] 100052 D - 0xcc2ffafc [usbus4] 100051 D - 0xcc2ffacc [usbus4] 100050 D - 0xcc2eab5c [usbus3] 100049 D - 0xcc2eab2c [usbus3] 100048 D - 0xcc2eaafc [usbus3] 100047 D - 0xcc2eaacc [usbus3] 100046 D - 0xcc2dcb5c [usbus2] 100045 D - 0xcc2dcb2c [usbus2] 100044 D - 0xcc2dcafc [usbus2] 100043 D - 0xcc2dcacc [usbus2] 100041 D - 0xcc2bab5c [usbus1] 100040 D - 0xcc2bab2c [usbus1] 100039 D - 0xcc2baafc [usbus1] 100038 D - 0xcc2baacc [usbus1] 100036 D - 0xcc2adb5c [usbus0] 100035 D - 0xcc2adb2c [usbus0] 100034 D - 0xcc2adafc [usbus0] 100033 D - 0xcc2adacc [usbus0] 2 0 0 0 DL ctl_work 0xcaa65000 [ctl_thrd] 14 0 0 0 DL - 0xc119be64 [yarrow] 13 0 0 0 DL (threaded) [geom] 100015 D - 0xc1198f28 [g_down] 100014 D - 0xc1198f24 [g_up] 100013 D - 0xc1198f1c [g_event] 12 0 0 0 WL (threaded) [intr] 100067 I [irq12: psm0] 100066 I [irq1: atkbd0] 100064 I [swi0: uart] 100061 I [irq20: fwohci0] 100060 I [irq14: ata0] 100055 I [irq19: ehci0] 100042 I [irq18: ohci2 ohci4] 100037 I [irq17: ohci1 ohci3] 100032 I [irq16: hdac1 ohci0] 100031 I [irq22: ahci0] 100030 I [irq257: re0] 100029 I [irq256: hdac0] 100027 I [swi2: cambio] 100026 I [swi6: task queue] 100025 I [swi6: Giant taskq] 100022 I [swi5: +] 100012 I [swi4: clock] 100011 I [swi4: clock] 100010 I [swi4: clock] 100009 I [swi4: clock] 100008 I [swi1: netisr 0] 100007 I [swi3: vm] 11 0 0 0 RL (threaded) [idle] 100006 CanRun [idle: cpu3] 100005 Run CPU 2 [idle: cpu2] 100004 Run CPU 1 [idle: cpu1] 100003 Run CPU 0 [idle: cpu0] 1 0 1 0 SLs wait 0xca74eb30 [init] 10 0 0 0 DL audit_wo 0xc131a6a0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100069 D - 0xc119be64 [deadlkres] 100068 D - 0xcc2cea80 [mca taskq] 100062 D - 0xcc26b480 [fw0_taskq] 100024 D - 0xcaa5f380 [thread taskq] 100023 D - 0xca8c5c80 [ffs_trim taskq] 100021 D - 0xca8c6a80 [kqueue taskq] 100020 D - 0xca8c6c80 [acpi_task_2] 100019 D - 0xca8c6c80 [acpi_task_1] 100018 D - 0xca8c6c80 [acpi_task_0] 100016 D - 0xca871c80 [firmware taskq] 100000 D sched 0xc11990e0 [swapper] db:0:ps> allt Tracing command mkdir pid 92 tid 100109 td 0xce2075e0 kdb_enter(c0fdee48,c0fdee48,c0fdd18d,f27886f8,3,...) at kdb_enter+0x3a panic(c0fdd18d,c1017255,c101478c,4ba,ca6b7058,...) at panic+0x18c _mtx_lock_sleep(c131af20,ce2075e0,0,c101478c,4ba,...) at _mtx_lock_sleep+0x51 _mtx_lock_flags(c131af20,0,c101478c,4ba,cdfea4fc,...) at _mtx_lock_flags+0x10f workitem_alloc(f27887b4,4,c101478c,12f8,e190b9a0,...) at workitem_alloc+0x3d bmsafemap_lookup(ce209780,0,c101478c,1271,ce87e800,...) at bmsafemap_lookup+0x161 softdep_setup_inomapdep(e190b9a0,ce5e7414,5c0000,41ed,0,...) at softdep_setup_inomapdep+0xf0 ffs_nodealloccg(ce5e7414,100,5c0000,0,41ed,...) at ffs_nodealloccg+0x5ab ffs_hashalloc(5c0000,0,41ed,0,c0cbae50,...) at ffs_hashalloc+0x85 ffs_valloc(ce878b40,41ed,ca71ee80,f27889bc,e1906d40,...) at ffs_valloc+0x4d3 ufs_mkdir(f2788bec,c103a707,c1158340,ce878b40,f2788c00,...) at ufs_mkdir+0xba VOP_MKDIR_APV(c1145ae0,f2788bec,f2788b5c,f2788b84,0,...) at VOP_MKDIR_APV+0x100 kern_mkdirat(ce2075e0,ffffff9c,bfbfef32,0,1ff,...) at kern_mkdirat+0x291 kern_mkdir(ce2075e0,bfbfef32,0,1ff,f2788cfc,...) at kern_mkdir+0x2e sys_mkdir(ce2075e0,f2788ccc,c10330bc,c0fddd52,c0aba4b7,...) at sys_mkdir+0x29 syscall(f2788d08) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (136, FreeBSD ELF32, sys_mkdir), eip = 0x28169743, esp = 0xbfbfed0c, ebp = 0xbfbfedd8 --- Tracing command sh pid 91 tid 100096 td 0xce59b5e0 sched_switch(ce59b5e0,0,104,1b3,f2744aac,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0a7e6aa,...) at mi_switch+0x243 sleepq_switch(ce59b5e0,0,c0fe4b00,1ac,6c,...) at sleepq_switch+0x1a7 sleepq_catch_signals(c0fe4b00,165,0,100,100,...) at sleepq_catch_signals+0xfc sleepq_wait_sig(ce5932cc,6c,c0fe7a38,100,0,...) at sleepq_wait_sig+0x17 _sleep(ce5932cc,ce593354,16c,c0fe7a38,0,...) at _sleep+0x385 kern_wait(ce59b5e0,ffffffff,f2744c38,2,0,...) at kern_wait+0x597 sys_wait4(ce59b5e0,f2744ccc,c10330bc,c0fe7a17,c0aba4b7,...) at sys_wait4+0x3b syscall(f2744d08) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28156d8b, esp = 0xbfbfe69c, ebp = 0xbfbfe6b8 --- Tracing command sh pid 20 tid 100079 td 0xce562bc0 sched_switch(ce562bc0,0,104,1b3,f2700aac,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0a7e6aa,...) at mi_switch+0x243 sleepq_switch(ce562bc0,0,c0fe4b00,1ac,6c,...) at sleepq_switch+0x1a7 sleepq_catch_signals(c0fe4b00,165,0,100,100,...) at sleepq_catch_signals+0xfc sleepq_wait_sig(ce55bb30,6c,c0fe7a38,100,0,...) at sleepq_wait_sig+0x17 _sleep(ce55bb30,ce55bbb8,16c,c0fe7a38,0,...) at _sleep+0x385 kern_wait(ce562bc0,ffffffff,f2700c38,2,0,...) at kern_wait+0x597 sys_wait4(ce562bc0,f2700ccc,c10330bc,c0fe7a17,c10346d0,...) at sys_wait4+0x3b syscall(f2700d08) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28156d8b, esp = 0xbfbfeaac, ebp = 0xbfbfeac8 --- Tracing command softdepflush pid 19 tid 100078 td 0xce2078d0 sched_switch(ce2078d0,0,104,1b3,ee324bf0,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(ce2078d0,0,c0fe4b00,291,0,...) at sleepq_switch+0x1a7 sleepq_timedwait(c131af60,54,c1016cd1,0,0,...) at sleepq_timedwait+0x6b _sleep(c131af60,c131af20,54,c1016cd1,3e8,...) at _sleep+0x36a softdep_flush(0,ee324d08,c0fd8f70,3d7,ce1fa000,...) at softdep_flush+0x259 fork_exit(c0ce3030,0,ee324d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee324d40, ebp = 0 --- Tracing command syncer pid 18 tid 100077 td 0xce207bc0 sched_switch(ce207bc0,0,104,1b3,ee321bcc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0fdcf47,...) at mi_switch+0x243 sleepq_switch(ce207bc0,0,c0fe4b00,291,ce207bc0,...) at sleepq_switch+0x1a7 sleepq_timedwait(c1313254,0,ee321c58,1,0,...) at sleepq_timedwait+0x6b _cv_timedwait(c1313254,c1313240,3e8,782,4e20,...) at _cv_timedwait+0x270 sched_sync(0,ee321d08,c0fd8f70,3d7,ce1fa2cc,...) at sched_sync+0x595 fork_exit(c0b34740,0,ee321d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee321d40, ebp = 0 --- Tracing command vnlru pid 17 tid 100076 td 0xcc3098d0 sched_switch(cc3098d0,0,104,1b3,ee31ebc4,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(cc3098d0,0,c0fe4b00,291,0,...) at sleepq_switch+0x1a7 sleepq_timedwait(ce1fa598,60,c0fefcf5,0,0,...) at sleepq_timedwait+0x6b _sleep(ce1fa598,c1313214,260,c0fefcf5,3e8,...) at _sleep+0x36a vnlru_proc(0,ee31ed08,c0fd8f70,3d7,ce1fa598,...) at vnlru_proc+0xe7 fork_exit(c0b35420,0,ee31ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee31ed40, ebp = 0 --- Tracing command bufdaemon pid 16 tid 100075 td 0xcc309bc0 sched_switch(cc309bc0,0,104,1b3,ee31bbfc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(cc309bc0,0,c0fe4b00,291,0,...) at sleepq_switch+0x1a7 sleepq_timedwait(c1312ee8,54,c0feca55,0,0,...) at sleepq_timedwait+0x6b _sleep(c1312ee8,c1312ef0,54,c0feca55,3e8,...) at _sleep+0x36a buf_daemon(0,ee31bd08,c0fd8f70,3d7,ce1fa864,...) at buf_daemon+0x175 fork_exit(c0b17410,0,ee31bd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee31bd40, ebp = 0 --- Tracing command pagezero pid 9 tid 100074 td 0xcc67e000 sched_switch(cc67e000,0,104,1b3,ee318c00,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(cc67e000,0,c0fe4b00,291,0,...) at sleepq_switch+0x1a7 sleepq_timedwait(c131cf5c,0,c101e0a1,0,0,...) at sleepq_timedwait+0x6b _sleep(c131cf5c,c131b900,0,c101e0a1,493e0,...) at _sleep+0x36a vm_pagezero(0,ee318d08,c0fd8f70,3d7,ce1fab30,...) at vm_pagezero+0xdc fork_exit(c0d28be0,0,ee318d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee318d40, ebp = 0 --- Tracing command vmdaemon pid 8 tid 100073 td 0xcc67e2f0 sched_switch(cc67e2f0,0,104,1b3,ee315bc4,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71bd80,...) at mi_switch+0x243 sleepq_switch(cc67e2f0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(c131cac8,74,c0feca55,0,0,...) at sleepq_wait+0x6b _sleep(c131cac8,c131cacc,74,c0feca55,0,...) at _sleep+0x39c vm_daemon(0,ee315d08,c0fd8f70,3d7,ce1fc000,...) at vm_daemon+0x59 fork_exit(c0d221a0,0,ee315d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee315d40, ebp = 0 --- Tracing command pagedaemon pid 7 tid 100072 td 0xcc67e5e0 sched_switch(cc67e5e0,0,104,1b3,ee312b4c,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(cc67e5e0,0,c0fe4b00,291,0,...) at sleepq_switch+0x1a7 sleepq_timedwait(c131ca90,54,c0feca55,0,0,...) at sleepq_timedwait+0x6b _sleep(c131ca90,c131b900,54,c0feca55,1388,...) at _sleep+0x36a vm_pageout(0,ee312d08,c0fd8f70,3d7,ca74f2cc,...) at vm_pageout+0x2c7 fork_exit(c0d23220,0,ee312d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee312d40, ebp = 0 --- Tracing command xpt_thrd pid 6 tid 100071 td 0xcc67e8d0 sched_switch(cc67e8d0,0,104,1b3,ee18fbfc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71bd00,...) at mi_switch+0x243 sleepq_switch(cc67e8d0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(c115f9d4,5c,c0e37f08,0,0,...) at sleepq_wait+0x6b _sleep(c115f9d4,c115fa18,5c,c0e37f08,0,...) at _sleep+0x39c xpt_scanner_thread(0,ee18fd08,c0fd8f70,3d7,ca74f598,...) at xpt_scanner_thread+0x47 fork_exit(c04a84a0,0,ee18fd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee18fd40, ebp = 0 --- Tracing command sctp_iterator pid 5 tid 100070 td 0xcc67ebc0 sched_switch(cc67ebc0,0,104,1b3,ee18cc04,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71bcc0,...) at mi_switch+0x243 sleepq_switch(cc67ebc0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(c1316778,0,c10041ff,0,0,...) at sleepq_wait+0x6b _sleep(c1316778,c1316740,0,c10041ff,0,...) at _sleep+0x39c sctp_iterator_thread(0,ee18cd08,c0fd8f70,3d7,ca74f864,...) at sctp_iterator_thread+0x5c fork_exit(c0bd0810,0,ee18cd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee18cd40, ebp = 0 --- Tracing command fdc0 pid 4 tid 100065 td 0xcc67fbc0 sched_switch(cc67fbc0,0,104,1b3,ee17db4c,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(cc67fbc0,0,c0fe4b00,291,0,...) at sleepq_switch+0x1a7 sleepq_timedwait(cc3ab23c,5c,c0fcb2f2,0,0,...) at sleepq_timedwait+0x6b _sleep(cc3ab23c,cc3ab2f0,5c,c0fcb2f2,3e8,...) at _sleep+0x36a fdc_thread(cc3ab200,ee17dd08,c0fd8f70,3d7,ca74fb30,...) at fdc_thread+0x27d fork_exit(c0dacdd0,cc3ab200,ee17dd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee17dd40, ebp = 0 --- Tracing command fw0_probe pid 3 tid 100063 td 0xcc2ca2f0 sched_switch(cc2ca2f0,0,104,1b3,ee16d6f0,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0a7e6aa,...) at mi_switch+0x243 sleepq_switch(cc2ca2f0,0,c0fe4b00,1ac,6c,...) at sleepq_switch+0x1a7 sleepq_catch_signals(c0fe4b00,165,0,100,100,...) at sleepq_catch_signals+0xfc sleepq_wait_sig(cc3b0000,6c,c0fcb2f2,100,0,...) at sleepq_wait_sig+0x17 _sleep(cc3b0000,cc3b4488,16c,c0fcb2f2,0,...) at _sleep+0x385 fw_bus_probe_thread(cc3b0000,ee16dd08,c0fd8f70,3d7,cc037000,...) at fw_bus_probe_thread+0xa08 fork_exit(c070a0b0,cc3b0000,ee16dd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee16dd40, ebp = 0 --- Tracing command usb pid 15 tid 100059 td 0xcc307000 sched_switch(cc307000,0,104,1b3,ee156c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc307000,0,c0fe4b00,26e,cc307000,...) at sleepq_switch+0x1a7 sleepq_wait(cc310d34,0,ee156c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc310d34,cc310e4c,c0fbc971,74,cc310d3c,...) at _cv_wait+0x260 usb_process(cc310d2c,ee156d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc310d2c,ee156d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee156d40, ebp = 0 --- Tracing command usb pid 15 tid 100058 td 0xcc3072f0 sched_switch(cc3072f0,0,104,1b3,ee153c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc3072f0,0,c0fe4b00,26e,cc3072f0,...) at sleepq_switch+0x1a7 sleepq_wait(cc310d04,0,ee153c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc310d04,cc310e4c,c0fbc971,74,cc310d0c,...) at _cv_wait+0x260 usb_process(cc310cfc,ee153d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc310cfc,ee153d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee153d40, ebp = 0 --- Tracing command usb pid 15 tid 100057 td 0xcc3075e0 sched_switch(cc3075e0,0,104,1b3,ee150c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc3075e0,0,c0fe4b00,26e,cc3075e0,...) at sleepq_switch+0x1a7 sleepq_wait(cc310cd4,0,ee150c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc310cd4,cc310e4c,c0fbc971,74,cc310cdc,...) at _cv_wait+0x260 usb_process(cc310ccc,ee150d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc310ccc,ee150d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee150d40, ebp = 0 --- Tracing command usb pid 15 tid 100056 td 0xcc3078d0 sched_switch(cc3078d0,0,104,1b3,ee14dc08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc3078d0,0,c0fe4b00,26e,cc3078d0,...) at sleepq_switch+0x1a7 sleepq_wait(cc310ca4,0,ee14dc8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc310ca4,cc310e4c,c0fbc971,74,cc310cac,...) at _cv_wait+0x260 usb_process(cc310c9c,ee14dd08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc310c9c,ee14dd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee14dd40, ebp = 0 --- Tracing command usb pid 15 tid 100054 td 0xcc309000 sched_switch(cc309000,0,104,1b3,ee144c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc309000,0,c0fe4b00,26e,cc309000,...) at sleepq_switch+0x1a7 sleepq_wait(cc2ffb5c,0,ee144c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2ffb5c,cc2ffc74,c0fbc971,74,cc2ffb64,...) at _cv_wait+0x260 usb_process(cc2ffb54,ee144d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2ffb54,ee144d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee144d40, ebp = 0 --- Tracing command usb pid 15 tid 100053 td 0xcc3092f0 sched_switch(cc3092f0,0,104,1b3,ee141c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc3092f0,0,c0fe4b00,26e,cc3092f0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2ffb2c,0,ee141c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2ffb2c,cc2ffc74,c0fbc971,74,cc2ffb34,...) at _cv_wait+0x260 usb_process(cc2ffb24,ee141d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2ffb24,ee141d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee141d40, ebp = 0 --- Tracing command usb pid 15 tid 100052 td 0xcc3095e0 sched_switch(cc3095e0,0,104,1b3,ee13ec08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc3095e0,0,c0fe4b00,26e,cc3095e0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2ffafc,0,ee13ec8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2ffafc,cc2ffc74,c0fbc971,74,cc2ffb04,...) at _cv_wait+0x260 usb_process(cc2ffaf4,ee13ed08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2ffaf4,ee13ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee13ed40, ebp = 0 --- Tracing command usb pid 15 tid 100051 td 0xcc2c8000 sched_switch(cc2c8000,0,104,1b3,ee13bc08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c8000,0,c0fe4b00,26e,cc2c8000,...) at sleepq_switch+0x1a7 sleepq_wait(cc2ffacc,0,ee13bc8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2ffacc,cc2ffc74,c0fbc971,74,cc2ffad4,...) at _cv_wait+0x260 usb_process(cc2ffac4,ee13bd08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2ffac4,ee13bd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee13bd40, ebp = 0 --- Tracing command usb pid 15 tid 100050 td 0xcc2c82f0 sched_switch(cc2c82f0,0,104,1b3,ee137c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c82f0,0,c0fe4b00,26e,cc2c82f0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2eab5c,0,ee137c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2eab5c,cc2eac74,c0fbc971,74,cc2eab64,...) at _cv_wait+0x260 usb_process(cc2eab54,ee137d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2eab54,ee137d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee137d40, ebp = 0 --- Tracing command usb pid 15 tid 100049 td 0xcc2c85e0 sched_switch(cc2c85e0,0,104,1b3,ee134c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c85e0,0,c0fe4b00,26e,cc2c85e0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2eab2c,0,ee134c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2eab2c,cc2eac74,c0fbc971,74,cc2eab34,...) at _cv_wait+0x260 usb_process(cc2eab24,ee134d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2eab24,ee134d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee134d40, ebp = 0 --- Tracing command usb pid 15 tid 100048 td 0xcc2c88d0 sched_switch(cc2c88d0,0,104,1b3,ee131c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c88d0,0,c0fe4b00,26e,cc2c88d0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2eaafc,0,ee131c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2eaafc,cc2eac74,c0fbc971,74,cc2eab04,...) at _cv_wait+0x260 usb_process(cc2eaaf4,ee131d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2eaaf4,ee131d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee131d40, ebp = 0 --- Tracing command usb pid 15 tid 100047 td 0xcc2c8bc0 sched_switch(cc2c8bc0,0,104,1b3,ee12ec08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c8bc0,0,c0fe4b00,26e,cc2c8bc0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2eaacc,0,ee12ec8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2eaacc,cc2eac74,c0fbc971,74,cc2eaad4,...) at _cv_wait+0x260 usb_process(cc2eaac4,ee12ed08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2eaac4,ee12ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee12ed40, ebp = 0 --- Tracing command usb pid 15 tid 100046 td 0xcc2c9000 sched_switch(cc2c9000,0,104,1b3,ee12ac08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c9000,0,c0fe4b00,26e,cc2c9000,...) at sleepq_switch+0x1a7 sleepq_wait(cc2dcb5c,0,ee12ac8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2dcb5c,cc2dcc74,c0fbc971,74,cc2dcb64,...) at _cv_wait+0x260 usb_process(cc2dcb54,ee12ad08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2dcb54,ee12ad08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee12ad40, ebp = 0 --- Tracing command usb pid 15 tid 100045 td 0xcc2c92f0 sched_switch(cc2c92f0,0,104,1b3,ee127c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c92f0,0,c0fe4b00,26e,cc2c92f0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2dcb2c,0,ee127c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2dcb2c,cc2dcc74,c0fbc971,74,cc2dcb34,...) at _cv_wait+0x260 usb_process(cc2dcb24,ee127d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2dcb24,ee127d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee127d40, ebp = 0 --- Tracing command usb pid 15 tid 100044 td 0xcc2c95e0 sched_switch(cc2c95e0,0,104,1b3,ee124c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c95e0,0,c0fe4b00,26e,cc2c95e0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2dcafc,0,ee124c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2dcafc,cc2dcc74,c0fbc971,74,cc2dcb04,...) at _cv_wait+0x260 usb_process(cc2dcaf4,ee124d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2dcaf4,ee124d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee124d40, ebp = 0 --- Tracing command usb pid 15 tid 100043 td 0xcc2c98d0 sched_switch(cc2c98d0,0,104,1b3,ee121c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2c98d0,0,c0fe4b00,26e,cc2c98d0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2dcacc,0,ee121c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2dcacc,cc2dcc74,c0fbc971,74,cc2dcad4,...) at _cv_wait+0x260 usb_process(cc2dcac4,ee121d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2dcac4,ee121d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee121d40, ebp = 0 --- Tracing command usb pid 15 tid 100041 td 0xcc2ca000 sched_switch(cc2ca000,0,104,1b3,ee11ac08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2ca000,0,c0fe4b00,26e,cc2ca000,...) at sleepq_switch+0x1a7 sleepq_wait(cc2bab5c,0,ee11ac8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2bab5c,cc2bac74,c0fbc971,74,cc2bab64,...) at _cv_wait+0x260 usb_process(cc2bab54,ee11ad08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2bab54,ee11ad08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee11ad40, ebp = 0 --- Tracing command usb pid 15 tid 100040 td 0xcc265000 sched_switch(cc265000,0,104,1b3,ee117c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc265000,0,c0fe4b00,26e,cc265000,...) at sleepq_switch+0x1a7 sleepq_wait(cc2bab2c,0,ee117c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2bab2c,cc2bac74,c0fbc971,74,cc2bab34,...) at _cv_wait+0x260 usb_process(cc2bab24,ee117d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2bab24,ee117d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee117d40, ebp = 0 --- Tracing command usb pid 15 tid 100039 td 0xcc2652f0 sched_switch(cc2652f0,0,104,1b3,ee114c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2652f0,0,c0fe4b00,26e,cc2652f0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2baafc,0,ee114c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2baafc,cc2bac74,c0fbc971,74,cc2bab04,...) at _cv_wait+0x260 usb_process(cc2baaf4,ee114d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2baaf4,ee114d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee114d40, ebp = 0 --- Tracing command usb pid 15 tid 100038 td 0xcc2655e0 sched_switch(cc2655e0,0,104,1b3,ee111c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc2655e0,0,c0fe4b00,26e,cc2655e0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2baacc,0,ee111c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2baacc,cc2bac74,c0fbc971,74,cc2baad4,...) at _cv_wait+0x260 usb_process(cc2baac4,ee111d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2baac4,ee111d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee111d40, ebp = 0 --- Tracing command usb pid 15 tid 100036 td 0xcc265bc0 sched_switch(cc265bc0,0,104,1b3,ee10ac08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc265bc0,0,c0fe4b00,26e,cc265bc0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2adb5c,0,ee10ac8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2adb5c,cc2adc74,c0fbc971,74,cc2adb64,...) at _cv_wait+0x260 usb_process(cc2adb54,ee10ad08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2adb54,ee10ad08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee10ad40, ebp = 0 --- Tracing command usb pid 15 tid 100035 td 0xcc26c000 sched_switch(cc26c000,0,104,1b3,ee107c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc26c000,0,c0fe4b00,26e,cc26c000,...) at sleepq_switch+0x1a7 sleepq_wait(cc2adb2c,0,ee107c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2adb2c,cc2adc74,c0fbc971,74,cc2adb34,...) at _cv_wait+0x260 usb_process(cc2adb24,ee107d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2adb24,ee107d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee107d40, ebp = 0 --- Tracing command usb pid 15 tid 100034 td 0xcc26c2f0 sched_switch(cc26c2f0,0,104,1b3,ee104c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc26c2f0,0,c0fe4b00,26e,cc26c2f0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2adafc,0,ee104c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2adafc,cc2adc74,c0fbc971,74,cc2adb04,...) at _cv_wait+0x260 usb_process(cc2adaf4,ee104d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2adaf4,ee104d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee104d40, ebp = 0 --- Tracing command usb pid 15 tid 100033 td 0xcc26c5e0 sched_switch(cc26c5e0,0,104,1b3,ee101c08,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(cc26c5e0,0,c0fe4b00,26e,cc26c5e0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2adacc,0,ee101c8c,1,0,...) at sleepq_wait+0x6b _cv_wait(cc2adacc,cc2adc74,c0fbc971,74,cc2adad4,...) at _cv_wait+0x260 usb_process(cc2adac4,ee101d08,c0fd8f70,3d7,cc0372cc,...) at usb_process+0x1a3 fork_exit(c09383e0,cc2adac4,ee101d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee101d40, ebp = 0 --- Tracing command ctl_thrd pid 2 tid 100028 td 0xca86f8d0 sched_switch(ca86f8d0,0,104,1b3,c7b72654,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71a8c0,...) at mi_switch+0x243 sleepq_switch(ca86f8d0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(caa65000,5c,c0e42fc8,0,0,...) at sleepq_wait+0x6b _sleep(caa65000,caa65000,5c,c0e42fc8,0,...) at _sleep+0x39c ctl_work_thread(caa65000,c7b72d08,c0fd8f70,3d7,cc037598,...) at ctl_work_thread+0x22f6 fork_exit(c04d5960,caa65000,c7b72d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b72d40, ebp = 0 --- Tracing command yarrow pid 14 tid 100017 td 0xca86e5e0 sched_switch(ca86e5e0,0,104,1b3,c7b50bd0,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(ca86e5e0,0,c0fe4b00,291,2,...) at sleepq_switch+0x1a7 sleepq_timedwait(c119be64,0,c0fcb2f2,2,0,...) at sleepq_timedwait+0x6b _sleep(c119be64,0,0,c0fcb2f2,64) at _sleep+0x36a pause(c0fcb2f2,64,c0f989d0,111,0,...) at pause+0xa0 random_kthread(0,c7b50d08,c0fd8f70,3d7,ca74e000,...) at random_kthread+0x1ef fork_exit(c0835be0,0,c7b50d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b50d40, ebp = 0 --- Tracing command geom pid 13 tid 100015 td 0xca86ebc0 sched_switch(ca86ebc0,0,104,1b3,c7b4abb4,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71a9c0,...) at mi_switch+0x243 sleepq_switch(ca86ebc0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(c1198f28,5c,c0fcb2f2,0,0,...) at sleepq_wait+0x6b _sleep(c1198f28,c1198e88,25c,c0fcb2f2,0,...) at _sleep+0x39c g_io_schedule_down(ca86ebc0,0,c0fcce41,6c,c7b4acf4,...) at g_io_schedule_down+0x5b g_down_procbody(0,c7b4ad08,c0fd8f70,3d7,ca74e2cc,...) at g_down_procbody+0x9d fork_exit(c09ef8b0,0,c7b4ad08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b4ad40, ebp = 0 --- Tracing command geom pid 13 tid 100014 td 0xca86f000 sched_switch(ca86f000,0,104,1b3,c7b47bdc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71a980,...) at mi_switch+0x243 sleepq_switch(ca86f000,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(c1198f24,5c,c0fcb2f2,0,0,...) at sleepq_wait+0x6b _sleep(c1198f24,c1198ea8,25c,c0fcb2f2,0,...) at _sleep+0x39c g_io_schedule_up(ca86f000,0,c0fcce41,5f,c7b47cf4,...) at g_io_schedule_up+0x134 g_up_procbody(0,c7b47d08,c0fd8f70,3d7,ca74e2cc,...) at g_up_procbody+0x9d fork_exit(c09ef950,0,c7b47d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b47d40, ebp = 0 --- Tracing command geom pid 13 tid 100013 td 0xca7522f0 sched_switch(ca7522f0,0,104,1b3,c7b44bd8,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71ad00,...) at mi_switch+0x243 sleepq_switch(ca7522f0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(c1198f1c,5c,c0fcb2f2,0,0,...) at sleepq_wait+0x6b _sleep(c1198f1c,c1198e4c,25c,c0fcb2f2,0,...) at _sleep+0x39c g_run_events(c11a9cc0,0,c0fcce41,79,c7b44cf4,...) at g_run_events+0x54b g_event_procbody(0,c7b44d08,c0fd8f70,3d7,ca74e2cc,...) at g_event_procbody+0x90 fork_exit(c09ef9f0,0,c7b44d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b44d40, ebp = 0 --- Tracing command intr pid 12 tid 100067 td 0xcc67f5e0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100066 td 0xcc67f8d0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100064 td 0xcc680000 sched_switch(cc680000,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,cc680000,...) at mi_switch+0x243 ithread_loop(cc65dd80,ee170d08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc65dd80,ee170d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee170d40, ebp = 0 --- Tracing command intr pid 12 tid 100061 td 0xcc2ca8d0 sched_switch(cc2ca8d0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,cc2ca8d0,...) at mi_switch+0x243 ithread_loop(cc0da680,ee164d08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc0da680,ee164d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee164d40, ebp = 0 --- Tracing command intr pid 12 tid 100060 td 0xcc2cabc0 sched_switch(cc2cabc0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,cc2cabc0,...) at mi_switch+0x243 ithread_loop(cc0e1d80,ee15cd08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc0e1d80,ee15cd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee15cd40, ebp = 0 --- Tracing command intr pid 12 tid 100055 td 0xcc307bc0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100042 td 0xcc2c9bc0 sched_switch(cc2c9bc0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,cc2c9bc0,...) at mi_switch+0x243 ithread_loop(cc0c2a80,ee11ed08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc0c2a80,ee11ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee11ed40, ebp = 0 --- Tracing command intr pid 12 tid 100037 td 0xcc2658d0 sched_switch(cc2658d0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,cc2658d0,...) at mi_switch+0x243 ithread_loop(cc0c9380,ee10ed08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc0c9380,ee10ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee10ed40, ebp = 0 --- Tracing command intr pid 12 tid 100032 td 0xcc26c8d0 sched_switch(cc26c8d0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,cc26c8d0,...) at mi_switch+0x243 ithread_loop(cc13fd80,ee0fed08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc13fd80,ee0fed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee0fed40, ebp = 0 --- Tracing command intr pid 12 tid 100031 td 0xcc26cbc0 sched_switch(cc26cbc0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,cc26cbc0,...) at mi_switch+0x243 ithread_loop(cc140980,edfa8d08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc140980,edfa8d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xedfa8d40, ebp = 0 --- Tracing command intr pid 12 tid 100030 td 0xca86f2f0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100029 td 0xca86f5e0 sched_switch(ca86f5e0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,ca86f5e0,...) at mi_switch+0x243 ithread_loop(cc140780,c7ba4d08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,cc140780,c7ba4d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7ba4d40, ebp = 0 --- Tracing command intr pid 12 tid 100027 td 0xca86fbc0 sched_switch(ca86fbc0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,ca86fbc0,...) at mi_switch+0x243 ithread_loop(ca70c580,c7b6fd08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,ca70c580,c7b6fd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b6fd40, ebp = 0 --- Tracing command intr pid 12 tid 100026 td 0xcaa5a000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100025 td 0xcaa5a2f0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100022 td 0xcaa5abc0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100012 td 0xca7525e0 sched_switch(ca7525e0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,ca7525e0,...) at mi_switch+0x243 ithread_loop(ca70ce80,c7b41d08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,ca70ce80,c7b41d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b41d40, ebp = 0 --- Tracing command intr pid 12 tid 100011 td 0xca7528d0 sched_switch(ca7528d0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,ca7528d0,...) at mi_switch+0x243 ithread_loop(ca79b180,c7b3ed08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,ca79b180,c7b3ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b3ed40, ebp = 0 --- Tracing command intr pid 12 tid 100010 td 0xca752bc0 sched_switch(ca752bc0,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,ca752bc0,...) at mi_switch+0x243 ithread_loop(ca70b180,c7b3bd08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,ca70b180,c7b3bd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b3bd40, ebp = 0 --- Tracing command intr pid 12 tid 100009 td 0xca79a000 sched_switch(ca79a000,0,109,1b3,0,...) at sched_switch+0x453 mi_switch(109,0,c0fd9293,557,ca79a000,...) at mi_switch+0x243 ithread_loop(ca70b380,c7b38d08,c0fd8f70,3d7,ca74e598,...) at ithread_loop+0x1fb fork_exit(c0a637e0,ca70b380,c7b38d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b38d40, ebp = 0 --- Tracing command intr pid 12 tid 100008 td 0xca79a2f0 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100007 td 0xca79a5e0 fork_trampoline() at fork_trampoline Tracing command idle pid 11 tid 100006 td 0xca751000 sched_switch(ca751000,0,608,1ae,0,...) at sched_switch+0x453 mi_switch(608,0,c0fdf6bf,cd,0,...) at mi_switch+0x243 critical_exit(0,3,0,2,c7b2ecc8,...) at critical_exit+0xb8 cpu_idle(0,c7b2eca0,c0fe10ec,a41,ca751000,...) at cpu_idle+0x97 sched_idletd(0,c7b2ed08,c0fd8f70,3d7,ca74e864,...) at sched_idletd+0x235 fork_exit(c0ab8830,0,c7b2ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b2ed40, ebp = 0 --- Tracing command idle pid 11 tid 100005 td 0xca7512f0 cpustop_handler(2,c7b2bbdc,c0e0c0e3,c7b2bb2c,c7b2bb22,...) at cpustop_handler+0x26 ipi_nmi_handler(c7b2bb2c,c7b2bb22,bb68,2570000,ca74e864,...) at ipi_nmi_handler+0x35 trap(c7b2bbe8) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0df13a5, esp = 0xc7b2bc28, ebp = 0xc7b2bc28 --- acpi_cpu_c1(c7b2bc44,c115ffb0,2,317,9,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b2bc78,c0dfe569,0,2,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,2,0,2,c7b2bcc8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b2bca0,c0fe10ec,a41,ca7512f0,...) at cpu_idle+0x89 sched_idletd(0,c7b2bd08,c0fd8f70,3d7,ca74e864,...) at sched_idletd+0x235 fork_exit(c0ab8830,0,c7b2bd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b2bd40, ebp = 0 --- Tracing command idle pid 11 tid 100004 td 0xca7515e0 cpustop_handler(1,c7b28bdc,c0e0c0e3,c7b28b2c,c7b28b22,...) at cpustop_handler+0x26 ipi_nmi_handler(c7b28b2c,c7b28b22,d122,2570000,ca74e864,...) at ipi_nmi_handler+0x35 trap(c7b28be8) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0df13a5, esp = 0xc7b28c28, ebp = 0xc7b28c28 --- acpi_cpu_c1(c7b28c44,c115ffb0,1,317,9,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b28c78,c0dfe569,0,2710,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,2710,c11aa340,2,c7b28cc8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b28ca0,c0fe10ec,a41,ca7515e0,...) at cpu_idle+0x89 sched_idletd(0,c7b28d08,c0fd8f70,3d7,ca74e864,...) at sched_idletd+0x235 fork_exit(c0ab8830,0,c7b28d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b28d40, ebp = 0 --- Tracing command idle pid 11 tid 100003 td 0xca7518d0 cpustop_handler(0,c7b25bdc,c0e0c0e3,c7b25b2c,c7b25b22,...) at cpustop_handler+0x26 ipi_nmi_handler(c7b25b2c,c7b25b22,9cc0,2570000,ca74e864,...) at ipi_nmi_handler+0x35 trap(c7b25be8) at trap+0x33 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0df13a5, esp = 0xc7b25c28, ebp = 0xc7b25c28 --- acpi_cpu_c1(c7b25c44,c115ffb0,0,317,8,...) at acpi_cpu_c1+0x5 acpi_cpu_idle(0,c7b25c78,c0dfe569,0,0,...) at acpi_cpu_idle+0x122 cpu_idle_acpi(0,0,0,2,c7b25cc8,...) at cpu_idle_acpi+0x2f cpu_idle(0,c7b25ca0,c0fe10ec,a41,ca7518d0,...) at cpu_idle+0x89 sched_idletd(0,c7b25d08,c0fd8f70,3d7,ca74e864,...) at sched_idletd+0x235 fork_exit(c0ab8830,0,c7b25d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b25d40, ebp = 0 --- Tracing command init pid 1 tid 100002 td 0xca751bc0 sched_switch(ca751bc0,0,104,1b3,c7b21aac,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0a7e6aa,...) at mi_switch+0x243 sleepq_switch(ca751bc0,0,c0fe4b00,1ac,6c,...) at sleepq_switch+0x1a7 sleepq_catch_signals(c0fe4b00,165,0,100,100,...) at sleepq_catch_signals+0xfc sleepq_wait_sig(ca74eb30,6c,c0fe7a38,100,0,...) at sleepq_wait_sig+0x17 _sleep(ca74eb30,ca74ebb8,16c,c0fe7a38,0,...) at _sleep+0x385 kern_wait(ca751bc0,ffffffff,c7b21c38,2,0,...) at kern_wait+0x597 sys_wait4(ca751bc0,c7b21ccc,c10330bc,c0fd8ffc,c10346d0,...) at sys_wait4+0x3b syscall(c7b21d08) at syscall+0x2fe Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x80606bf, esp = 0xbfbfe85c, ebp = 0xbfbfe878 --- Tracing command audit pid 10 tid 100001 td 0xca752000 sched_switch(ca752000,0,104,1b3,c7b1ebe8,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,eb,...) at mi_switch+0x243 sleepq_switch(ca752000,0,c0fe4b00,26e,ca752000,...) at sleepq_switch+0x1a7 sleepq_wait(c131a6a0,0,c7b1ec6c,1,0,...) at sleepq_wait+0x6b _cv_wait(c131a6a0,c131a684,c1010e57,194,c7b1eca4,...) at _cv_wait+0x260 audit_worker(0,c7b1ed08,c0fd8f70,3d7,ca74f000,...) at audit_worker+0x84 fork_exit(c0ca05f0,0,c7b1ed08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b1ed40, ebp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xcc67f000 sched_switch(cc67f000,0,104,1b3,ee189bc8,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(cc67f000,0,c0fe4b00,291,2,...) at sleepq_switch+0x1a7 sleepq_timedwait(c119be64,0,c0fcb2f2,2,0,...) at sleepq_timedwait+0x6b _sleep(c119be64,0,0,c0fcb2f2,bb8) at _sleep+0x36a pause(c0fcb2f2,bb8,12b,129,c1199168,...) at pause+0xa0 deadlkres(0,ee189d08,c0fd8f70,3d7,c11990e0,...) at deadlkres+0x324 fork_exit(c0a42680,0,ee189d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee189d40, ebp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xcc67f2f0 sched_switch(cc67f2f0,0,104,1b3,ee186c18,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0fe4b00,...) at mi_switch+0x243 sleepq_switch(cc67f2f0,0,c0fe4b00,26e,cc67f2f0,...) at sleepq_switch+0x1a7 sleepq_wait(cc2cea80,0,c0fdfbe1,c0fcb2f2,0,...) at sleepq_wait+0x6b msleep_spin(cc2cea80,cc2cea98,c0fcb2f2,0,a77,...) at msleep_spin+0x25f taskqueue_thread_loop(c137c3e0,ee186d08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0x8f fork_exit(c0ad5b20,c137c3e0,ee186d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee186d40, ebp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xcc2ca5e0 sched_switch(cc2ca5e0,0,104,1b3,ee16ac18,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0fe4b00,...) at mi_switch+0x243 sleepq_switch(cc2ca5e0,0,c0fe4b00,26e,cc2ca5e0,...) at sleepq_switch+0x1a7 sleepq_wait(cc26b480,0,c0fdfbe1,c0fcb2f2,0,...) at sleepq_wait+0x6b msleep_spin(cc26b480,cc26b498,c0fcb2f2,0,a77,...) at msleep_spin+0x25f taskqueue_thread_loop(cc3b449c,ee16ad08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0x8f fork_exit(c0ad5b20,cc3b449c,ee16ad08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xee16ad40, ebp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xcaa5a5e0 sched_switch(caa5a5e0,0,104,1b3,c7b66bfc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71a7c0,...) at mi_switch+0x243 sleepq_switch(caa5a5e0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(caa5f380,0,c0fcb2f2,0,0,...) at sleepq_wait+0x6b _sleep(caa5f380,caa5f398,0,c0fcb2f2,0,...) at _sleep+0x39c taskqueue_thread_loop(c11b8a4c,c7b66d08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0xb5 fork_exit(c0ad5b20,c11b8a4c,c7b66d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b66d40, ebp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xcaa5a8d0 sched_switch(caa5a8d0,0,104,1b3,c7b63bfc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71a780,...) at mi_switch+0x243 sleepq_switch(caa5a8d0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(ca8c5c80,0,c0fcb2f2,0,0,...) at sleepq_wait+0x6b _sleep(ca8c5c80,ca8c5c98,0,c0fcb2f2,0,...) at _sleep+0x39c taskqueue_thread_loop(c131ac84,c7b63d08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0xb5 fork_exit(c0ad5b20,c131ac84,c7b63d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b63d40, ebp = 0 --- Tracing command kernel pid 0 tid 100021 td 0xca79a8d0 sched_switch(ca79a8d0,0,104,1b3,c7b5dbfc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71ab40,...) at mi_switch+0x243 sleepq_switch(ca79a8d0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(ca8c6a80,0,c0fcb2f2,0,0,...) at sleepq_wait+0x6b _sleep(ca8c6a80,ca8c6a98,0,c0fcb2f2,0,...) at _sleep+0x39c taskqueue_thread_loop(c1199a78,c7b5dd08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0xb5 fork_exit(c0ad5b20,c1199a78,c7b5dd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b5dd40, ebp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xca79abc0 sched_switch(ca79abc0,0,104,1b3,c7b5ac18,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0fe4b00,...) at mi_switch+0x243 sleepq_switch(ca79abc0,0,c0fe4b00,26e,ca79abc0,...) at sleepq_switch+0x1a7 sleepq_wait(ca8c6c80,0,c0fdfbe1,c0fcb2f2,0,...) at sleepq_wait+0x6b msleep_spin(ca8c6c80,ca8c6c98,c0fcb2f2,0,a77,...) at msleep_spin+0x25f taskqueue_thread_loop(c1162c18,c7b5ad08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0x8f fork_exit(c0ad5b20,c1162c18,c7b5ad08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b5ad40, ebp = 0 --- Tracing command kernel pid 0 tid 100019 td 0xca86e000 sched_switch(ca86e000,0,104,1b3,c7b57c18,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0fe4b00,...) at mi_switch+0x243 sleepq_switch(ca86e000,0,c0fe4b00,26e,ca86e000,...) at sleepq_switch+0x1a7 sleepq_wait(ca8c6c80,0,c0fdfbe1,c0fcb2f2,0,...) at sleepq_wait+0x6b msleep_spin(ca8c6c80,ca8c6c98,c0fcb2f2,0,a77,...) at msleep_spin+0x25f taskqueue_thread_loop(c1162c18,c7b57d08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0x8f fork_exit(c0ad5b20,c1162c18,c7b57d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b57d40, ebp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xca86e2f0 sched_switch(ca86e2f0,0,104,1b3,c7b54c18,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,c0fe4b00,...) at mi_switch+0x243 sleepq_switch(ca86e2f0,0,c0fe4b00,26e,ca86e2f0,...) at sleepq_switch+0x1a7 sleepq_wait(ca8c6c80,0,c0fdfbe1,c0fcb2f2,0,...) at sleepq_wait+0x6b msleep_spin(ca8c6c80,ca8c6c98,c0fcb2f2,0,a77,...) at msleep_spin+0x25f taskqueue_thread_loop(c1162c18,c7b54d08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0x8f fork_exit(c0ad5b20,c1162c18,c7b54d08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b54d40, ebp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xca86e8d0 sched_switch(ca86e8d0,0,104,1b3,c7b4dbfc,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,ca71aa00,...) at mi_switch+0x243 sleepq_switch(ca86e8d0,0,c0fe4b00,26e,0,...) at sleepq_switch+0x1a7 sleepq_wait(ca871c80,0,c0fcb2f2,0,0,...) at sleepq_wait+0x6b _sleep(ca871c80,ca871c98,0,c0fcb2f2,0,...) at _sleep+0x39c taskqueue_thread_loop(c11b7440,c7b4dd08,c0fd8f70,3d7,c11990e0,...) at taskqueue_thread_loop+0xb5 fork_exit(c0ad5b20,c11b7440,c7b4dd08) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc7b4dd40, ebp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xc11993b0 sched_switch(c11993b0,0,104,1b3,c1820c5c,...) at sched_switch+0x453 mi_switch(104,0,c0fe4b00,1f8,352,...) at mi_switch+0x243 sleepq_switch(c11993b0,0,c0fe4b00,291,0,...) at sleepq_switch+0x1a7 sleepq_timedwait(c11990e0,54,c1011838,0,0,...) at sleepq_timedwait+0x6b _sleep(c11990e0,0,54,c1011838,2710,...) at _sleep+0x36a scheduler(0,181ec00,181ec00,181e000,1825000,...) at scheduler+0x243 mi_startup() at mi_startup+0xac begin() at begin+0x2c db:0:allt> call doadump Physical memory: 3435 MB Dumping 165 MB: 150 134 118 102 86 70 54 38 22 6 Dump complete = 0 db:0:doadump> reset cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 3 (kgdb) bt #0 doadump (textdump=0x0) at pcpu.h:244 #1 0xc0523a59 in db_fncall (dummy1=0xc0ac71ba, dummy2=0x0, dummy3=0xffffffff, dummy4=0xf2788344 "X\203xò") at ../../../ddb/db_command.c:573 #2 0xc0523e8f in db_command (last_cmdp=0xc1160ddc, cmd_table=0x0, dopager=0x0) at ../../../ddb/db_command.c:449 #3 0xc0523f44 in db_command_script (command=0xc1161ce8 "call doadump") at ../../../ddb/db_command.c:520 #4 0xc0528390 in db_script_exec (scriptname=0xc1161640 "doadump", warnifnotfound=Variable "warnifnotfound" is not available. ) at ../../../ddb/db_script.c:302 #5 0xc0528421 in db_run_cmd (addr=0x1, have_addr=0x0, count=0xc133ca00, modif=0xf278847c "") at ../../../ddb/db_script.c:375 #6 0xc0523e51 in db_command (last_cmdp=0xc1160ddc, cmd_table=0x0, dopager=0x1) at ../../../ddb/db_command.c:449 #7 0xc0523faa in db_command_loop () at ../../../ddb/db_command.c:502 #8 0xc0525f9d in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:231 #9 0xc0ac76d6 in kdb_trap (type=0x3, code=0x0, tf=0xf2788678) at ../../../kern/subr_kdb.c:654 #10 0xc0e0c84b in trap (frame=0xf2788678) at ../../../i386/i386/trap.c:710 #11 0xc0df595c in calltrap () at ../../../i386/i386/exception.s:169 #12 0xc0ac71ba in kdb_enter (why=0xc0fdee48 "panic", msg=0xc0fdee48 "panic") at cpufunc.h:71 #13 0xc0a9034c in panic (fmt=0xc0fdd18d "_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n") at ../../../kern/kern_shutdown.c:628 #14 0xc0a7d891 in _mtx_lock_sleep (m=0xc131af20, tid=0xce2075e0, opts=0x0, file=0xc101478c "../../../ufs/ffs/ffs_softdep.c", line=0x4ba) at ../../../kern/kern_mutex.c:363 #15 0xc0a7dd0f in _mtx_lock_flags (m=0xc131af20, opts=0x0, file=0xc101478c "../../../ufs/ffs/ffs_softdep.c", line=0x4ba) at ../../../kern/kern_mutex.c:212 #16 0xc0cce6bd in workitem_alloc (item=Variable "item" is not available. ) at ../../../ufs/ffs/ffs_softdep.c:1210 #17 0xc0cd0671 in bmsafemap_lookup (mp=0xce527000, bp=0xe190b9a0, cg=0x100, newbmsafemap=0xce209780) at ../../../ufs/ffs/ffs_softdep.c:4879 #18 0xc0cd7cf0 in softdep_setup_inomapdep (bp=0xe190b9a0, ip=0xce5e7414, newinum=0x5c0000, mode=0x41ed) at ../../../ufs/ffs/ffs_softdep.c:4725 #19 0xc0cbb3fb in ffs_nodealloccg (ip=0xce5e7414, cg=0x100, ipref=0x0, mode=0x41ed, unused=0x0) at ../../../ufs/ffs/ffs_alloc.c:1827 #20 0xc0cba0b5 in ffs_hashalloc (ip=0xce5e7414, cg=0x100, pref=0x5c0000, size=0x41ed, rsize=0x0, allocator=0xc0cbae50 ) at ../../../ufs/ffs/ffs_alloc.c:1308 #21 0xc0cbab13 in ffs_valloc (pvp=0xce878b40, mode=0x41ed, cred=0xca71ee80, vpp=0xf27889bc) at ../../../ufs/ffs/ffs_alloc.c:972 #22 0xc0cfdd7a in ufs_mkdir (ap=0xf2788bec) at ../../../ufs/ufs/ufs_vnops.c:1825 #23 0xc0e32f90 in VOP_MKDIR_APV (vop=0xc1146020, a=0xf2788bec) at vnode_if.c:1534 #24 0xc0b37631 in kern_mkdirat (td=0xce2075e0, fd=0xffffff9c, path=0xbfbfef32
, segflg=UIO_USERSPACE, mode=Variable "mode" is not available. ) at vnode_if.h:665 #25 0xc0b3770e in kern_mkdir (td=0xce2075e0, path=0xbfbfef32
, segflg=UIO_USERSPACE, mode=0x1ff) at ../../../kern/vfs_syscalls.c:3800 #26 0xc0b37739 in sys_mkdir (td=0xce2075e0, uap=0xf2788ccc) at ../../../kern/vfs_syscalls.c:3779 #27 0xc0e0bb9e in syscall (frame=0xf2788d08) at subr_syscall.c:135 #28 0xc0df59f1 in Xint0x80_syscall () at ../../../i386/i386/exception.s:267 #29 0x00000033 in ?? () Previous frame inner to this frame (corrupt stack?) (kgdb) f f 16 No symbol "f" in current context. (kgdb) f 16 #16 0xc0cce6bd in workitem_alloc (item=Variable "item" is not available. ) at ../../../ufs/ffs/ffs_softdep.c:1210 1210 ACQUIRE_LOCK(&lk); (kgdb) l 1205 item->wk_type = type; 1206 item->wk_mp = mp; 1207 item->wk_state = 0; 1208 1209 ump = VFSTOUFS(mp); 1210 ACQUIRE_LOCK(&lk); 1211 dep_current[type]++; 1212 dep_total[type]++; 1213 ump->softdep_deps++; 1214 ump->softdep_accdeps++; (kgdb) p lk $1 = {lock_object = {lo_name = 0xc1017255 "Softdep Lock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xca6b7058}, mtx_lock = 0xce2075e0} (kgdb) f 17 #17 0xc0cd0671 in bmsafemap_lookup (mp=0xce527000, bp=0xe190b9a0, cg=0x100, newbmsafemap=0xce209780) at ../../../ufs/ffs/ffs_softdep.c:4879 4879 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); (kgdb) l 4874 FREE_LOCK(&lk); 4875 bmsafemap = malloc(sizeof(struct bmsafemap), 4876 M_BMSAFEMAP, M_SOFTDEP_FLAGS); 4877 ACQUIRE_LOCK(&lk); 4878 } 4879 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); 4880 bmsafemap->sm_buf = bp; 4881 LIST_INIT(&bmsafemap->sm_inodedephd); 4882 LIST_INIT(&bmsafemap->sm_inodedepwr); 4883 LIST_INIT(&bmsafemap->sm_newblkhd); (kgdb) l 2118 2113 request_cleanup(mp, FLUSH_INODES); 2114 FREE_LOCK(&lk); 2115 inodedep = malloc(sizeof(struct inodedep), 2116 M_INODEDEP, M_SOFTDEP_FLAGS); 2117 workitem_alloc(&inodedep->id_list, D_INODEDEP, mp); 2118 ACQUIRE_LOCK(&lk); <-- in inodedep_lookup() 2119 if (inodedep_find(inodedephd, fs, inum, inodedeppp)) { 2120 WORKITEM_FREE(inodedep, D_INODEDEP); 2121 return (1); 2122 } (kgdb) $ svn diff -x -p /usr/src/sys Index: /usr/src/sys/ufs/ffs/ffs_softdep.c =================================================================== --- /usr/src/sys/ufs/ffs/ffs_softdep.c (revision 236735) +++ /usr/src/sys/ufs/ffs/ffs_softdep.c (working copy) @@ -920,7 +920,7 @@ static struct freefrag *allocindir_merge(struct al static int bmsafemap_find(struct bmsafemap_hashhead *, struct mount *, int, struct bmsafemap **); static struct bmsafemap *bmsafemap_lookup(struct mount *, struct buf *, - int cg); + int cg, struct bmsafemap *); static int newblk_find(struct newblk_hashhead *, struct mount *, ufs2_daddr_t, int, struct newblk **); static int newblk_lookup(struct mount *, ufs2_daddr_t, int, struct newblk **); @@ -4707,12 +4707,22 @@ softdep_setup_inomapdep(bp, ip, newinum, mode) * Panic if it already exists as something is seriously wrong. * Otherwise add it to the dependency list for the buffer holding * the cylinder group map from which it was allocated. + * + * We have to preallocate a bmsafemap entry in case it is needed + * in bmsafemap_lookup since once we allocate the inodedep, we + * have to finish initializing it before we can FREE_LOCK(). + * By preallocating, we avoid LOCK_FREE while doing a malloc + * in bmsafemap_lookup. We cannot call bmsafemap_lookup before + * creating the inodedep as it can be freed during the time + * that we FREE_LOCK() while allocating the inodedep. */ + bmsafemap = malloc(sizeof(struct bmsafemap), + M_BMSAFEMAP, M_SOFTDEP_FLAGS); ACQUIRE_LOCK(&lk); if ((inodedep_lookup(mp, newinum, DEPALLOC | NODELAY, &inodedep))) panic("softdep_setup_inomapdep: dependency %p for new" "inode already exists", inodedep); - bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum)); + bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum), bmsafemap); if (jaddref) { LIST_INSERT_HEAD(&bmsafemap->sm_jaddrefhd, jaddref, ja_bmdeps); TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, @@ -4786,7 +4796,7 @@ softdep_setup_blkmapdep(bp, mp, newblkno, frags, o if (newblk_lookup(mp, newblkno, DEPALLOC, &newblk) != 0) panic("softdep_setup_blkmapdep: found block"); newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(mp, bp, - dtog(fs, newblkno)); + dtog(fs, newblkno), NULL); if (jnewblk) { jnewblk->jn_dep = (struct worklist *)newblk; LIST_INSERT_HEAD(&bmsafemap->sm_jnewblkhd, jnewblk, jn_deps); @@ -4827,13 +4837,16 @@ bmsafemap_find(bmsafemaphd, mp, cg, bmsafemapp) * Find the bmsafemap associated with a cylinder group buffer. * If none exists, create one. The buffer must be locked when * this routine is called and this routine must be called with - * splbio interrupts blocked. + * the softdep lock held. To avoid giving up the lock while + * allocating a new bmsafemap, a preallocated bmsafemap may be + * provided. If it is provided but not needed, it is freed. */ static struct bmsafemap * -bmsafemap_lookup(mp, bp, cg) +bmsafemap_lookup(mp, bp, cg, newbmsafemap) struct mount *mp; struct buf *bp; int cg; + struct bmsafemap *newbmsafemap; { struct bmsafemap_hashhead *bmsafemaphd; struct bmsafemap *bmsafemap, *collision; @@ -4843,15 +4856,26 @@ static struct bmsafemap * mtx_assert(&lk, MA_OWNED); if (bp) LIST_FOREACH(wk, &bp->b_dep, wk_list) - if (wk->wk_type == D_BMSAFEMAP) + if (wk->wk_type == D_BMSAFEMAP) { + if (newbmsafemap) + free(newbmsafemap, M_BMSAFEMAP); return (WK_BMSAFEMAP(wk)); + } fs = VFSTOUFS(mp)->um_fs; bmsafemaphd = BMSAFEMAP_HASH(fs, cg); - if (bmsafemap_find(bmsafemaphd, mp, cg, &bmsafemap) == 1) + if (bmsafemap_find(bmsafemaphd, mp, cg, &bmsafemap) == 1) { + if (newbmsafemap) + free(newbmsafemap, M_BMSAFEMAP); return (bmsafemap); - FREE_LOCK(&lk); - bmsafemap = malloc(sizeof(struct bmsafemap), - M_BMSAFEMAP, M_SOFTDEP_FLAGS); + } + if (newbmsafemap) { + bmsafemap = newbmsafemap; + } else { + FREE_LOCK(&lk); + bmsafemap = malloc(sizeof(struct bmsafemap), + M_BMSAFEMAP, M_SOFTDEP_FLAGS); + ACQUIRE_LOCK(&lk); + } workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); bmsafemap->sm_buf = bp; LIST_INIT(&bmsafemap->sm_inodedephd); @@ -4862,7 +4886,6 @@ static struct bmsafemap * LIST_INIT(&bmsafemap->sm_jnewblkhd); LIST_INIT(&bmsafemap->sm_freehd); LIST_INIT(&bmsafemap->sm_freewr); - ACQUIRE_LOCK(&lk); if (bmsafemap_find(bmsafemaphd, mp, cg, &collision) == 1) { WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); return (collision); @@ -10221,7 +10244,7 @@ softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) ACQUIRE_LOCK(&lk); /* Lookup the bmsafemap so we track when it is dirty. */ fs = VFSTOUFS(mp)->um_fs; - bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno)); + bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL); /* * Detach any jnewblks which have been canceled. They must linger * until the bitmap is cleared again by ffs_blkfree() to prevent @@ -10267,7 +10290,7 @@ softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) * allocation dependency. */ fs = VFSTOUFS(mp)->um_fs; - bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno)); + bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL); end = blkno + frags; LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { /*