GENERIC HEAD from Oct 9 15:08 UTC, vmcore.94

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 7.0-CURRENT #0: Mon Oct  9 17:31:07 CEST 2006
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1032458240 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
GEOM_RAID3: Device raid3/data launched (3/3).
GEOM_RAID3: Device data: provider raid3/data destroyed.
GEOM_RAID3: Device data destroyed.
GEOM_RAID3: Device raid3/data launched (3/3).
GEOM_RAID3: Device data: provider raid3/data destroyed.
GEOM_RAID3: Device data destroyed.
:
GEOM_RAID3: Device raid3/data launched (3/3).
GEOM_RAID3: Device data: provider raid3/data destroyed.
GEOM_RAID3: Device data destroyed.
GEOM_RAID3: Device raid3/data launched (3/3).
Kernel page fault with the following non-sleepable locks held:
exclusive sleep mutex inp (udpinp) r = 0 (0xc46a4720) locked @ netinet/udp_usrreq.c:804
KDB: stack backtrace:
kdb_backtrace(1,c4934234,c,c48b4d80,e6bf5a50,...) at kdb_backtrace+0x29
witness_warn(5,0,c09459ed) at witness_warn+0x192
trap(8,28,28,c3d80400,c46a90e4,...) at trap+0x108
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc073dc6c, esp = 0xe6bf5a98, ebp = 0xe6bf5ae4 ---
ip_output(c4633600,0,e6bf5ab8,0,0,...) at ip_output+0x940
udp_output(c46a4690,c46a9000,0,0,c48b4d80,...) at udp_output+0x4b3
udp_send(c4a94914,0,c4896b00,0,0,...) at udp_send+0x2a
sosend_dgram(c4a94914,0,e6bf5c64,c4896b00,0,...) at sosend_dgram+0x30d
sosend(c4a94914,0,e6bf5c64,0,0,0,c48b4d80) at sosend+0x3c
soo_write(c4008c60,e6bf5c64,c4609980,0,c48b4d80) at soo_write+0x42
dofilewrite(c48b4d80,3,c4008c60,e6bf5c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c48b4d80,3,e6bf5c64,8052000,0,...) at kern_writev+0x36
write(c48b4d80,e6bf5d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---


Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address   = 0xc46a90f0
fault code              = supervisor read, page not present
instruction pointer     = 0x20:0xc073dc6c
stack pointer           = 0x28:0xe6bf5a98
frame pointer           = 0x28:0xe6bf5ae4
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 1881 (udp)
[thread pid 1881 tid 100428 ]
Stopped at      ip_output+0x940:        movzbl  0(%ecx),%eax
db> where
Tracing pid 1881 tid 100428 td 0xc48b4d80
ip_output(c4633600,0,e6bf5ab8,0,0,...) at ip_output+0x940
udp_output(c46a4690,c46a9000,0,0,c48b4d80,...) at udp_output+0x4b3
udp_send(c4a94914,0,c4896b00,0,0,...) at udp_send+0x2a
sosend_dgram(c4a94914,0,e6bf5c64,c4896b00,0,...) at sosend_dgram+0x30d
sosend(c4a94914,0,e6bf5c64,0,0,0,c48b4d80) at sosend+0x3c
soo_write(c4008c60,e6bf5c64,c4609980,0,c48b4d80) at soo_write+0x42
dofilewrite(c48b4d80,3,c4008c60,e6bf5c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c48b4d80,3,e6bf5c64,8052000,0,...) at kern_writev+0x36
write(c48b4d80,e6bf5d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---
db> show allpcpu
Current CPU: 1

cpuid        = 0
curthread    = 0xc3c37bd0: pid 4 "g_down"
curpcb       = 0xe26f1d90
fpcurthread  = none
idlethread   = 0xc3c36a20: pid 13 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc48b4d80: pid 1881 "udp"
curpcb       = 0xe6bf5d90
fpcurthread  = none
idlethread   = 0xc3c36870: pid 12 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc4777000: pid 1917 "sh"
curpcb       = 0xe6997d90
fpcurthread  = none
idlethread   = 0xc3c366c0: pid 11 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc4c2cd80: pid 1827 "md12"
curpcb       = 0xe6eb9d90
fpcurthread  = none
idlethread   = 0xc3c36510: pid 10 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db>  set $lines 1000
db> show alllocks
Process 1891 (udp) thread 0xc47036c0 (100443)
exclusive sleep mutex rtentry r = 0 (0xc443a2b8) locked @ net/route.c:197
exclusive sleep mutex radix node head r = 0 (0xc3e0937c) locked @ net/route.c:148
exclusive sleep mutex inp (udpinp) r = 0 (0xc4129d08) locked @ netinet/udp_usrreq.c:804
Process 1881 (udp) thread 0xc48b4d80 (100428)
exclusive sleep mutex inp (udpinp) r = 0 (0xc46a4720) locked @ netinet/udp_usrreq.c:804
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
 1917  1850  1819     0  RL+     CPU 2               sh
 1916   773   773     0  SL      swread   0xc1da6030 cron
 1909  2823  2823  1001  SW+     nanslp   0xc0a0cfc4 sleep
 1902  1870 93630  1001  SL+     swread   0xc2084688 swap
 1901  1870 93630  1001  SL+     swread   0xc1a6ae88 swap
 1900  1870 93630  1001  SL+     swread   0xc266a0d8 swap
 1899  1870 93630  1001  SL+     swread   0xc1f458a8 swap
 1898  1870 93630  1001  SL+     swread   0xc1c16ef8 swap
 1897  1870 93630  1001  SL+     pfault   0xc0a67038 swap
 1896  1873 93630  1001  R+                          udp
 1895  1873 93630  1001  R+                          udp
 1894  1873 93630  1001  R+                          udp
 1893  1873 93630  1001  R+                          udp
 1892  1870 93630  1001  SL+     pfault   0xc0a67038 swap
 1891  1873 93630  1001  R+                          udp
 1890  1870 93630  1001  SL+     swread   0xc240d818 swap
 1889  1873 93630  1001  R+                          udp
 1888  1870 93630  1001  SL+     swread   0xc21b1108 swap
 1887  1873 93630  1001  R+                          udp
 1886  1870 93630  1001  SL+     swread   0xc2911cc8 swap
 1885  1873 93630  1001  R+                          udp
 1884  1873 93630  1001  R+                          udp
 1883  1873 93630  1001  R+                          udp
 1882  1871 93630  1001  S+      biowr    0xd7edc454 mkdir
 1881  1873 93630  1001  R+      CPU 1               udp
 1880  1871 93630  1001  S+      biowr    0xd7cb0824 mkdir
 1879  1873 93630  1001  R+                          udp
 1878  1870 93630  1001  SL+     swread   0xc1b5ac30 swap
 1877  1871 93630  1001  S+      biowr    0xd7e7ef5c mkdir
 1876  1873 93630  1001  R+                          udp
 1875  1873 93630  1001  R+                          udp
 1874  1870 93630  1001  SL+     swread   0xc19dc8b0 swap
 1873  1868 93630  1001  SW+     wait     0xc48b5b04 udp
 1872  1870 93630  1001  SL+     swread   0xc1bd83c0 swap
 1871  1866 93630  1001  SW+     wait     0xc4b37b04 mkdir
 1870  1864 93630  1001  SW+     wait     0xc47a8b04 swap
 1868 93634 93630  1001  R+                          udp
 1867 93634 93630  1001  SL+     swread   0xc1c258f8 thr1
 1866 93634 93630  1001  R+                          mkdir
 1864 93634 93630  1001  R+                          swap
 1850  1841  1819     0  S+      wait     0xc486a468 rw
 1849  1841  1819     0  S+      biowr    0xd7ca2760 rw
 1848  1841  1819     0  S+      getblk   0xd7ca27c0 rw
 1847  1841  1819     0  S+      getblk   0xd7edc9a0 rw
 1846  1841  1819     0  S+      ufs      0xc5cbcc28 rw
 1845  1841  1819     0  S+      ufs      0xc597b2f8 rw
 1844  1841  1819     0  S+      biord    0xd7f5b858 rw
 1843  1841  1819     0  S+      getblk   0xd7d7f404 rw
 1842  1841  1819     0  S+      getblk   0xd7ca27c0 rw
 1841  1840  1819     0  SW+     wait     0xc47fbd38 rw
 1840  1819  1819     0  SW+     wait     0xc47b2b04 rw
 1837     0     0     0  SL      r3:w1    0xc4185000 [g_raid3 data]
 1829     0     0     0  SL      biowr    0xd7cc8668 [md13]
 1827     0     0     0  RL      CPU 3               [md12]
 1825     0     0     0  SL      biowr    0xd7ce6dac [md11]
 1819 91302  1819     0  SW+     wait     0xc4132b04 sh
93634 93633 93630  1001  SW+     wait     0xc41d7b04 run
93633 93632 93630  1001  SW+     wait     0xc44b569c run
93632 93630 93630  1001  R+                          run
93630 93623 93630  1001  SW+     wait     0xc477969c sh
93623 93622 93623  1001  SWs+    wait     0xc4b3269c bash
93622 93620 93620  1001  SW      select   0xc0a59abc sshd
93620   757 93620     0  SWs     sbwait   0xc412809c sshd
91302 91299 91302     0  SW+     wait     0xc4b32468 bash
91299 91298 91299     0  SW+     pause    0xc4c3e268 csh
91298   957 91298  1001  SW+     wait     0xc47b969c su
89922 89839 89922  1001  SWs     sbwait   0xc4a94718 bash
89839 89819 89819  1001  SW      select   0xc0a59abc sshd
89819   757 89819     0  SWs     sbwait   0xc4194d94 sshd
 2823  1906  2823  1001  SW+     wait     0xc41d6468 sh
 1906  1905  1906  1001  SWs+    wait     0xc4133d38 bash
 1905  1807  1807  1001  SW      select   0xc0a59abc sshd
 1807   757  1807     0  SWs     sbwait   0xc41399b0 sshd
  972   966   972  1001  R+                          top
  966   965   966  1001  SWs+    wait     0xc405d000 bash
  965   963   963  1001  S       select   0xc0a59abc sshd
  963   757   963     0  SWs     sbwait   0xc41951e8 sshd
  957   956   957  1001  SWs+    wait     0xc44b5d38 bash
  956   954   954  1001  SW      select   0xc0a59abc sshd
  954   757   954     0  SWs     sbwait   0xc411f9b0 sshd
  884     1   884     0  SWs+    ttyin    0xc3dfe810 getty
  883     1   883     0  SWs+    ttyin    0xc3e02810 getty
  882     1   882     0  SWs+    ttyin    0xc3e01810 getty
  881     1   881     0  SWs+    ttyin    0xc3e01010 getty
  880     1   880     0  SWs+    ttyin    0xc3e00c10 getty
  879     1   879     0  SWs+    ttyin    0xc3e02c10 getty
  878     1   878     0  SWs+    ttyin    0xc3e03010 getty
  877     1   877     0  SWs+    ttyin    0xc3e01c10 getty
  863     1   863     0  SWs     select   0xc0a59abc inetd
  836     1   836     0  SWs     select   0xc0a59abc moused
  821     1   821     0  Ss      nanslp   0xc0a0cfc4 watchdogd
  773     1   773     0  Ss      nanslp   0xc0a0cfc4 cron
  767     1   767    25  SWs     pause    0xc41d76d0 sendmail
  763     1   763     0  Ss      select   0xc0a59abc sendmail
  757     1   757     0  SWs     select   0xc0a59abc sshd
  738     1   738     0  Rs                          ntpd
  684   680   680     0  SW      -        0xc413be00 nfsd
  683   680   680     0  SW      -        0xc4186400 nfsd
  682   680   680     0  SW      -        0xc413c000 nfsd
  681   680   680     0  SW      -        0xc3f72200 nfsd
  680     1   680     0  SWs     select   0xc0a59abc nfsd
  678     1   678     0  SWs     select   0xc0a59abc mountd
  628     1   628     0  SWs     select   0xc0a59abc rpcbind
  609     1   609     0  ?s                          syslogd
  542     1   542     0  SWs     select   0xc0a59abc devd
   42     0     0     0  SL      -        0xe43f4cfc [schedcpu]
   41     0     0     0  SL      biord    0xd7e69f08 [softdepflush]
   40     0     0     0  RL                          [vnlru]
   39     0     0     0  SL      getblk   0xd7ca27c0 [syncer]
   38     0     0     0  SL      wdrain   0xc0a59f50 [bufdaemon]
   37     0     0     0  RL                          [pagezero]
   36     0     0     0  SL      psleep   0xc0a67260 [vmdaemon]
   35     0     0     0  SL      wswbuf0  0xc0a66b78 [pagedaemon]
   34     0     0     0  WL                          [irq7: ppc0]
   33     0     0     0  SL      -        0xc3d7e03c [fdc0]
   32     0     0     0  WL                          [swi0: sio]
   31     0     0     0  WL                          [irq12: psm0]
   30     0     0     0  WL                          [irq1: atkbd0]
   29     0     0     0  WL                          [irq15: ata1]
   28     0     0     0  WL                          [irq14: ata0]
   27     0     0     0  WL                          [irq17: fxp0]
   26     0     0     0  SL      usbtsk   0xc0a0a544 [usbtask]
   25     0     0     0  SL      usbevt   0xc3d6d210 [usb0]
   24     0     0     0  WL                          [irq16: uhci0]
   23     0     0     0  SL      -        0xc3d5a280 [em0 taskq]
   22     0     0     0  WL                          [irq9: acpi0]
   21     0     0     0  WL                          [swi5: +]
    9     0     0     0  SL      -        0xc3c85700 [thread taskq]
   20     0     0     0  WL                          [swi6: Giant taskq]
    8     0     0     0  SL      -        0xc3c85880 [acpi_task_2]
    7     0     0     0  SL      -        0xc3c85880 [acpi_task_1]
    6     0     0     0  SL      -        0xc3c85880 [acpi_task_0]
   19     0     0     0  WL                          [swi6: task queue]
    5     0     0     0  SL      -        0xc3c85a00 [kqueue taskq]
   18     0     0     0  WL                          [swi2: cambio]
   17     0     0     0  SL      -        0xc0a08180 [yarrow]
    4     0     0     0  RL      CPU 0               [g_down]
    3     0     0     0  RL                          [g_up]
    2     0     0     0  RL                          [g_event]
   16     0     0     0  WL                          [swi3: vm]
   15     0     0     0  WL                          [swi4: clock sio]
   14     0     0     0  WL                          [swi1: net]
   13     0     0     0  RL                          [idle: cpu0]
   12     0     0     0  RL                          [idle: cpu1]
   11     0     0     0  RL                          [idle: cpu2]
   10     0     0     0  RL                          [idle: cpu3]
    1     0     1     0  SLs     wait     0xc3c3a000 [init]
    0     0     0     0  SLs     vmwait   0xc0a67038 [swapper]
 1865 93634 93630  1001  Z+                          creat
 1869 93634 93630  1001  Z+                          tcp
db> allt

Tracing command sh pid 1917 tid 100261 td 0xc4777000
cpustop_handler(e69979a8,c08980b2,f424,0,f424,...) at cpustop_handler+0x31
ipi_nmi_handler(f424,0,f424,0,0,...) at ipi_nmi_handler+0x28
trap(8,28,c4770028,c4777000,0,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc089b926, esp = 0xe69979f0, ebp = 0xe6997a14 ---
DELAY(1) at DELAY+0x92
_mtx_lock_spin(c0a12494,c4777000,0,c0944dd8,2f8) at _mtx_lock_spin+0x9f
_mtx_lock_spin_flags(c0a12494,0,c0944dd8,2f8) at _mtx_lock_spin_flags+0xc2
pmap_invalidate_range(c0a94540,e0bb4000,e0bb8000,d7c9c694,e6997b0c,...) at pmap_invalidate_range+0x46
pmap_qremove(e0bb4000,4,c068ee4d,c0a583e0,0,...) at pmap_qremove+0x3d
vnode_pager_generic_getpages(c400da80,e6997c30,4000,0,c1862078,...) at vnode_pager_generic_getpages+0x7ff
ffs_getpages(e6997b6c) at ffs_getpages+0x100
VOP_GETPAGES_APV(c09db880,e6997b6c) at VOP_GETPAGES_APV+0xa9
vnode_pager_getpages(c1862078,e6997c30,4,0) at vnode_pager_getpages+0xa5
vm_fault(c46d9820,280f0000,1,0,c4777000,...) at vm_fault+0x928
trap_pfault(e6997d38,1,280f00b4,280f00b4,28069895,...) at trap_pfault+0xee
trap(3b,3b,3b,2808e600,281ccad4,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x28069895, esp = 0xbfbfe6d0, ebp = 0xbfbfe6f8 ---

Tracing command cron pid 1916 tid 100475 td 0xc47ef000
sched_switch(c47ef000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1da6030,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c1da6030) at sleepq_timedwait+0x4a
msleep(c1da6030,c52fb348,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c52fb348,e6a75c30,1,0) at swap_pager_getpages+0x348
vm_fault(c4003924,8210000,1,0,c47ef000,...) at vm_fault+0x928
trap_pfault(e6a75d38,1,8210800,8210800,2808fd9a,...) at trap_pfault+0xee
trap(805003b,3b,bfbf003b,8210800,8210800,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x2808fd9a, esp = 0xbfbfec50, ebp = 0xbfbfecb8 ---

Tracing command sleep pid 1909 tid 100604 td 0xc4c2b870

Tracing command swap pid 1902 tid 100449 td 0xc468dbd0
sched_switch(c468dbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c2084688,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c2084688) at sleepq_timedwait+0x4a
msleep(c2084688,c5dcc4b0,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c5dcc4b0,e68b6c30,8,0) at swap_pager_getpages+0x348
vm_fault(c48c0820,8080000,2,8,c468dbd0,...) at vm_fault+0x928
trap_pfault(e68b6d38,1,8080000,8080000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1901 tid 100129 td 0xc41d8a20
sched_switch(c41d8a20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1a6ae88,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c1a6ae88) at sleepq_timedwait+0x4a
msleep(c1a6ae88,c609a690,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c609a690,e6778c30,8,0) at swap_pager_getpages+0x348
vm_fault(c4a7be38,8370000,2,8,c41d8a20,...) at vm_fault+0x928
trap_pfault(e6778d38,1,8370000,8370000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1900 tid 100331 td 0xc468c000
sched_switch(c468c000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c266a0d8,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c266a0d8) at sleepq_timedwait+0x4a
msleep(c266a0d8,c4c001e0,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c4c001e0,e6886c30,8,0) at swap_pager_getpages+0x348
vm_fault(c48bf618,8070000,2,8,c468c000,...) at vm_fault+0x928
trap_pfault(e6886d38,1,8070000,8070000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1899 tid 100302 td 0xc477b870
sched_switch(c477b870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1f458a8,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c1f458a8) at sleepq_timedwait+0x4a
msleep(c1f458a8,c4905438,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c4905438,e69dcc30,8,0) at swap_pager_getpages+0x348
vm_fault(c47bf410,8060000,2,8,c477b870,...) at vm_fault+0x928
trap_pfault(e69dcd38,1,8060000,8060000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1898 tid 100223 td 0xc41db870
sched_switch(c41db870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1c16ef8,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c1c16ef8) at sleepq_timedwait+0x4a
msleep(c1c16ef8,c57514b0,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c57514b0,e6790c30,8,0) at swap_pager_getpages+0x348
vm_fault(c4a7aa28,82b0000,2,8,c41db870,...) at vm_fault+0x928
trap_pfault(e6790d38,1,82b0000,82b0000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1897 tid 100275 td 0xc477b000
sched_switch(c477b000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a67038,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a67038,0,c093a662,3e5,8,...) at sleepq_wait+0x46
msleep(c0a67038,c0a671fc,2a0,c093a8c8,0,...) at msleep+0x27d
vm_waitpfault(e69cdc00) at vm_waitpfault+0x57
vm_fault(c4999208,e3f7000,2,8,c477b000,...) at vm_fault+0xff
trap_pfault(e69cdd38,1,e3f7000,e3f7000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1896 tid 100505 td 0xc4b14870
sched_switch(c4b14870,0,2) at sched_switch+0x183
mi_switch(2,0,c4b14870,c1,c0a0c7b0,0,c0921068,101) at mi_switch+0x280
ast(e6d3cd38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command udp pid 1895 tid 100590 td 0xc4c2d1b0
sched_switch(c4c2d1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c1861088,c3c36bd0,0,c1861088,2,c091c6a0,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c1861088,c4c2d1b0,0,c09389bf,89b) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c1861088,0,c09389bf,89b,c1864bd0,...) at _mtx_lock_flags+0xae
uma_zalloc_internal(c1864b40,c46a2900,2,c1861088,0,...) at uma_zalloc_internal+0x20
uma_zalloc_arg(c1864b40,c46a2900,2) at uma_zalloc_arg+0x367
mb_zinit_pack(c46a2900,100,2) at mb_zinit_pack+0x18
uma_zalloc_internal(c1864d20,e6ebfb40,2,c1861008,0,...) at uma_zalloc_internal+0x90
uma_zalloc_arg(c1864d20,e6ebfb40,2) at uma_zalloc_arg+0x367
sosend_copyin(e6ebfc64,e6ebfba8,1,e6ebfb84,0,...) at sosend_copyin+0x58
sosend_dgram(c4f54914,0,e6ebfc64,0,0,...) at sosend_dgram+0x25e
sosend(c4f54914,0,e6ebfc64,0,0,0,c4c2d1b0) at sosend+0x3c
soo_write(c40080d8,e6ebfc64,c4609980,0,c4c2d1b0) at soo_write+0x42
dofilewrite(c4c2d1b0,3,c40080d8,e6ebfc64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c4c2d1b0,3,e6ebfc64,8050000,800,...) at kern_writev+0x36
write(c4c2d1b0,e6ebfd04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1894 tid 100393 td 0xc486ca20
sched_switch(c486ca20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a5bf4c,c3c36bd0,0,c0a5bf4c,2,c091c6a0,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a5bf4c,c486ca20,0,c092f39f,439) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a5bf4c,0,c092f39f,439,c40128b8,...) at _mtx_lock_flags+0xae
udp_disconnect(c41287c8,e6b71b8c,c06db46e,c41287c8,c40128b8,...) at udp_disconnect+0x32
sodisconnect(c41287c8) at sodisconnect+0x26
soclose(c41287c8) at soclose+0x1de
soo_close(c40128b8,c486ca20) at soo_close+0x4b
fdrop_locked(c40128b8,c486ca20,c3c13b80,0,c0919c97,...) at fdrop_locked+0x88
fdrop(c40128b8,c486ca20,6b6,c0a13374,0,...) at fdrop+0x24
closef(c40128b8,c486ca20,0,3d,0,...) at closef+0x367
kern_close(c486ca20,3,e6b71d30,c0898e32,c486ca20,...) at kern_close+0x1b6
close(c486ca20,e6b71d04) at close+0x10
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x2814530f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1893 tid 100508 td 0xc4b14360
sched_switch(c4b14360,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0c7b0,0,c091f13b,...) at mi_switch+0x280
critical_exit(c0a0c7b0,e6d33a6c,c068ee4d,c3c36bd0,e6d33a98,...) at critical_exit+0x92
spinlock_exit(c3c36bd0,e6d33a98,c06bf249,c0a0c7b0,0,...) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c0a0c7b0,0,c09210f9,371,0,...) at _mtx_unlock_spin_flags+0xb1
turnstile_unpend(c3c2e5c0,0,c414d1e0,39f,e6d33ad4,...) at turnstile_unpend+0x371
_mtx_unlock_sleep(c414d1e0,0,c092f39f,39f) at _mtx_unlock_sleep+0x6b
_mtx_unlock_flags(c414d1e0,0,c092f39f,39f,c469c100,...) at _mtx_unlock_flags+0xcc
udp_output(c414d150,c469c100,0,0,c4b14360,...) at udp_output+0x4cf
udp_send(c4c3f7c8,0,c469c100,0,0,...) at udp_send+0x2a
sosend_dgram(c4c3f7c8,0,e6d33c64,c469c100,0,...) at sosend_dgram+0x30d
sosend(c4c3f7c8,0,e6d33c64,0,0,0,c4b14360) at sosend+0x3c
soo_write(c4bb8bd0,e6d33c64,c4609980,0,c4b14360) at soo_write+0x42
dofilewrite(c4b14360,3,c4bb8bd0,e6d33c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c4b14360,3,e6d33c64,804c0a0,0,...) at kern_writev+0x36
write(c4b14360,e6d33d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1892 tid 100630 td 0xc4c3bbd0
sched_switch(c4c3bbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a67038,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a67038,0,c093a662,3e5,8,...) at sleepq_wait+0x46
msleep(c0a67038,c0a671fc,2a0,c093a8c8,0,...) at msleep+0x27d
vm_waitpfault(e6f07c00) at vm_waitpfault+0x57
vm_fault(c411a820,e34c000,2,8,c4c3bbd0,...) at vm_fault+0xff
trap_pfault(e6f07d38,1,e34c000,e34c000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1891 tid 100443 td 0xc47036c0
sched_switch(c47036c0,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0c7b0,0,c091f13b,...) at mi_switch+0x280
critical_exit(c443a258,e68e6a6c,c0883a7e,c0a50008,c4430028,...) at critical_exit+0x92
lapic_handle_timer(c0a50008,c4430028,c0a10028,e68e6abc,c3e09300,...) at lapic_handle_timer+0x149
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc07188fd, esp = 0xe68e6a10, ebp = 0xe68e6a6c ---
rtalloc1(e68e6abc,1,0,0,c46639e0,...) at rtalloc1+0x1c5
rtalloc_ign(e68e6ab8,0) at rtalloc_ign+0x98
ip_output(c4663900,0,e68e6ab8,0,0,...) at ip_output+0x320
udp_output(c4129c78,c4663900,0,0,c47036c0,...) at udp_output+0x4b3
udp_send(c4195bac,0,c4663900,0,0,...) at udp_send+0x2a
sosend_dgram(c4195bac,0,e68e6c64,c4663900,0,...) at sosend_dgram+0x30d
sosend(c4195bac,0,e68e6c64,0,0,0,c47036c0) at sosend+0x3c
soo_write(c48ee120,e68e6c64,c4609980,0,c47036c0) at soo_write+0x42
dofilewrite(c47036c0,3,c48ee120,e68e6c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c47036c0,3,e68e6c64,8050034,0,...) at kern_writev+0x36
write(c47036c0,e68e6d04) at write+0x45
syscall(c088003b,3b,3b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1890 tid 100310 td 0xc47e6360
sched_switch(c47e6360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c240d818,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c240d818) at sleepq_timedwait+0x4a
msleep(c240d818,c4e5f960,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c4e5f960,e6a60c30,1,0) at swap_pager_getpages+0x348
vm_fault(c499ad34,28154000,1,0,c47e6360,...) at vm_fault+0x928
trap_pfault(e6a60d38,1,281546e0,281546e0,280a42d0,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,0,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x280a42d0, esp = 0xbfbfe89c, ebp = 0xbfbfe8c8 ---

Tracing command udp pid 1889 tid 100424 td 0xc4931510
sched_switch(c4931510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c1861008,c3c36bd0,0,c1861008,2,c091c6a0,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c1861008,c4931510,0,c09389bf,75a) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c1861008,0,c09389bf,75a,c4c40610,...) at _mtx_lock_flags+0xae
uma_zalloc_arg(c1864960,e6c1fb30,2) at uma_zalloc_arg+0x185
sosend_copyin(e6c1fc64,e6c1fba8,1,e6c1fb84,0,...) at sosend_copyin+0xdc
sosend_dgram(c4c40530,0,e6c1fc64,0,0,...) at sosend_dgram+0x25e
sosend(c4c40530,0,e6c1fc64,0,0,0,c4931510) at sosend+0x3c
soo_write(c48deb88,e6c1fc64,c4609980,0,c4931510) at soo_write+0x42
dofilewrite(c4931510,3,c48deb88,e6c1fc64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c4931510,3,e6c1fc64,8050030,10,...) at kern_writev+0x36
write(c4931510,e6c1fd04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1888 tid 100170 td 0xc4600bd0
sched_switch(c4600bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c21b1108,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c21b1108) at sleepq_timedwait+0x4a
msleep(c21b1108,c51e75a0,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c51e75a0,e6865c30,8,0) at swap_pager_getpages+0x348
vm_fault(c47be000,80b0000,2,8,c4600bd0,...) at vm_fault+0x928
trap_pfault(e6865d38,1,80b0000,80b0000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1887 tid 100624 td 0xc4c3c6c0
sched_switch(c4c3c6c0,0,2) at sched_switch+0x183
mi_switch(2,0,c4c3c6c0,c2,c0a0c7b0,0,c0921068,101) at mi_switch+0x280
ast(e6f19d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command swap pid 1886 tid 100523 td 0xc49326c0
sched_switch(c49326c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c2911cc8,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c2911cc8) at sleepq_timedwait+0x4a
msleep(c2911cc8,c58514b0,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c58514b0,e6c3dc30,8,0) at swap_pager_getpages+0x348
vm_fault(c4b9e000,80a0000,2,8,c49326c0,...) at vm_fault+0x928
trap_pfault(e6c3dd38,1,80a0000,80a0000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1885 tid 100492 td 0xc4936d80
sched_switch(c4936d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a5bf4c,c3c36bd0,0,c0a5bf4c,2,c091c6a0,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a5bf4c,c4936d80,0,c092f39f,439) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a5bf4c,0,c092f39f,439,c48eeca8,...) at _mtx_lock_flags+0xae
udp_disconnect(c4a9314c,e6c7fb8c,c06db46e,c4a9314c,c48eeca8,...) at udp_disconnect+0x32
sodisconnect(c4a9314c) at sodisconnect+0x26
soclose(c4a9314c) at soclose+0x1de
soo_close(c48eeca8,c4936d80) at soo_close+0x4b
fdrop_locked(c48eeca8,c4936d80,c3c13160,0,c0919c97,...) at fdrop_locked+0x88
fdrop(c48eeca8,c4936d80,6b6,c0a13374,0,...) at fdrop+0x24
closef(c48eeca8,c4936d80,0,3d,0,...) at closef+0x367
kern_close(c4936d80,3,e6c7fd30,c0898e32,c4936d80,...) at kern_close+0x1b6
close(c4936d80,e6c7fd04) at close+0x10
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x2814530f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1884 tid 100157 td 0xc44b8870
sched_switch(c44b8870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c0a5bf4c,c3c36bd0,0,c0a5bf4c,2,c091c6a0,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c0a5bf4c,c44b8870,0,c092f39f,42b) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c0a5bf4c,0,c092f39f,42b,c4c40000,...) at _mtx_lock_flags+0xae
udp_detach(c4c40000) at udp_detach+0x3e
sofree(c4c40000) at sofree+0x1e9
soclose(c4c40000) at soclose+0x2d9
soo_close(c4bc6438,c44b8870) at soo_close+0x4b
fdrop_locked(c4bc6438,c44b8870,c3c130d0,0,c0919c97,...) at fdrop_locked+0x88
fdrop(c4bc6438,c44b8870,6b6,c0a13374,0,...) at fdrop+0x24
closef(c4bc6438,c44b8870,0,3d,0,...) at closef+0x367
kern_close(c44b8870,3,e67c6d30,c0898e32,c44b8870,...) at kern_close+0x1b6
close(c44b8870,e67c6d04) at close+0x10
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x2814530f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1883 tid 100497 td 0xc4b156c0
sched_switch(c4b156c0,0,2) at sched_switch+0x183
mi_switch(2,0,c4b156c0,c3,c0a0c7b0,0,c0921068,101) at mi_switch+0x280
ast(e6d54d38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command mkdir pid 1882 tid 100363 td 0xc4800d80
sched_switch(c4800d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7edc454,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7edc454,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7edc454,c0a59f88,4c,c0926578,0) at msleep+0x27d
bwait(d7edc454,4c,c0926578) at bwait+0x47
bufwait(d7edc454,d7edc454,0,e6aed8b4,c07db812,...) at bufwait+0x2d
bufwrite(d7edc454,0,0,e6aed904,c07c6445,...) at bufwrite+0x168
ffs_bufwrite(d7edc454) at ffs_bufwrite+0x282
ffs_update(c5cd1930,0,c5cd1930,0,0,...) at ffs_update+0x3b9
ffs_truncate(c5cd1930,0,0,0,c4609980,c4800d80,c0a669d8,0,c0936068,e0f) at ffs_truncate+0x5af
handle_workitem_remove(c66d33c0,0,c0a669d8,0,c0936068,cb2) at handle_workitem_remove+0x196
softdep_setup_remove(d7e0cefc,c4cb9318,c5d46e70,1,c5d46e70) at softdep_setup_remove+0x10c
ufs_dirremove(c5c413f0,c5d46e70,500800c,1,16,...) at ufs_dirremove+0x12d
ufs_rmdir(e6aedbc4) at ufs_rmdir+0xc3
VOP_RMDIR_APV(c09db880,e6aedbc4) at VOP_RMDIR_APV+0xb5
kern_rmdir(c4800d80,bfbe1c60,0,e6aedd30,c0898e32,...) at kern_rmdir+0x1b6
rmdir(c4800d80,e6aedd04) at rmdir+0x12
syscall(2805003b,bfbf003b,bfbd003b,2805188c,bfbfea24,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280c8257, esp = 0xbfbe181c, ebp = 0xbfbe1c48 ---

Tracing command udp pid 1881 tid 100428 td 0xc48b4d80
ip_output(c4633600,0,e6bf5ab8,0,0,...) at ip_output+0x940
udp_output(c46a4690,c46a9000,0,0,c48b4d80,...) at udp_output+0x4b3
udp_send(c4a94914,0,c4896b00,0,0,...) at udp_send+0x2a
sosend_dgram(c4a94914,0,e6bf5c64,c4896b00,0,...) at sosend_dgram+0x30d
sosend(c4a94914,0,e6bf5c64,0,0,0,c48b4d80) at sosend+0x3c
soo_write(c4008c60,e6bf5c64,c4609980,0,c48b4d80) at soo_write+0x42
dofilewrite(c48b4d80,3,c4008c60,e6bf5c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c48b4d80,3,e6bf5c64,8052000,0,...) at kern_writev+0x36
write(c48b4d80,e6bf5d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command mkdir pid 1880 tid 100490 td 0xc49a01b0
sched_switch(c49a01b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7cb0824,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7cb0824,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7cb0824,c0a59f88,4c,c0926578,0) at msleep+0x27d
bwait(d7cb0824,4c,c0926578) at bwait+0x47
bufwait(d7cb0824,d7cb0824,0,e6ca3b18,c07db812,...) at bufwait+0x2d
bufwrite(d7cb0824,0,0,e6ca3b68,c07c6445,...) at bufwrite+0x168
ffs_bufwrite(d7cb0824) at ffs_bufwrite+0x282
ffs_update(c4f15540,0) at ffs_update+0x3b9
ufs_inactive(e6ca3bb4) at ufs_inactive+0x286
VOP_INACTIVE_APV(c09db880,e6ca3bb4) at VOP_INACTIVE_APV+0x9b
vinactive(c4f15540,c49a01b0) at vinactive+0x8d
vrele(c4f15540) at vrele+0x155
kern_chdir(c49a01b0,804a70d,0,e6ca3d30,c0898e32,...) at kern_chdir+0x1c7
chdir(c49a01b0,e6ca3d04) at chdir+0x12
syscall(2805003b,bfbf003b,bfbc003b,2805188c,bfbfea24,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (12, FreeBSD ELF32, chdir), eip = 0x280c8737, esp = 0xbfbfaa1c, ebp = 0xbfbfae48 ---

Tracing command udp pid 1879 tid 100433 td 0xc4935d80
sched_switch(c4935d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
turnstile_wait(c4129bb8,c3c36bd0,0,c4129bb8,2,c091c6a0,231) at turnstile_wait+0x3e0
_mtx_lock_sleep(c4129bb8,c4935d80,0,c092f39f,324) at _mtx_lock_sleep+0x14d
_mtx_lock_flags(c4129bb8,0,c092f39f,324,0,...) at _mtx_lock_flags+0xae
udp_output(c4129b28,c47cd500,0,0,c4935d80,...) at udp_output+0x171
udp_send(c59d6530,0,c47cd500,0,0,...) at udp_send+0x2a
sosend_dgram(c59d6530,0,e6c64c64,c47cd500,0,...) at sosend_dgram+0x30d
sosend(c59d6530,0,e6c64c64,0,0,0,c4935d80) at sosend+0x3c
soo_write(c4a1f798,e6c64c64,c4609980,0,c4935d80) at soo_write+0x42
dofilewrite(c4935d80,3,c4a1f798,e6c64c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c4935d80,3,e6c64c64,804f080,0,...) at kern_writev+0x36
write(c4935d80,e6c64d04) at write+0x45
syscall(bfbf003b,bfbf003b,e6c6003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1878 tid 100406 td 0xc48b1bd0
sched_switch(c48b1bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1b5ac30,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c1b5ac30) at sleepq_timedwait+0x4a
msleep(c1b5ac30,c4eaa528,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c4eaa528,e6bd7c30,8,0) at swap_pager_getpages+0x348
vm_fault(c46d9d34,8060000,2,8,c48b1bd0,...) at vm_fault+0x928
trap_pfault(e6bd7d38,1,8060000,8060000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command mkdir pid 1877 tid 100509 td 0xc4b141b0
sched_switch(c4b141b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7e7ef5c,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7e7ef5c,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7e7ef5c,c0a59f88,4c,c0926578,0) at msleep+0x27d
bwait(d7e7ef5c,4c,c0926578) at bwait+0x47
bufwait(d7e7ef5c,d7e7ef5c,0,e6d308b4,c07db812,...) at bufwait+0x2d
bufwrite(d7e7ef5c,0,0,e6d30904,c07c6445,...) at bufwrite+0x168
ffs_bufwrite(d7e7ef5c) at ffs_bufwrite+0x282
ffs_update(c5269150,0,c5269150,0,0,...) at ffs_update+0x3b9
ffs_truncate(c5269150,0,0,0,c4609980,c4b141b0,c0a669d8,0,c0936068,e0f) at ffs_truncate+0x5af
handle_workitem_remove(c658ed60,0,c0a669d8,0,c0936068,cb2) at handle_workitem_remove+0x196
softdep_setup_remove(d7f0ab88,c5954294,c4a474a4,1,c4a474a4) at softdep_setup_remove+0x10c
ufs_dirremove(c580fa80,c4a474a4,500800c,1,16,...) at ufs_dirremove+0x12d
ufs_rmdir(e6d30bc4) at ufs_rmdir+0xc3
VOP_RMDIR_APV(c09db880,e6d30bc4) at VOP_RMDIR_APV+0xb5
kern_rmdir(c4b141b0,bfbd1490,0,e6d30d30,c0898e32,...) at kern_rmdir+0x1b6
rmdir(c4b141b0,e6d30d04) at rmdir+0x12
syscall(2805003b,bfbf003b,bfbd003b,2805188c,bfbfea24,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280c8257, esp = 0xbfbd104c, ebp = 0xbfbd1478 ---

Tracing command udp pid 1876 tid 100127 td 0xc41d8d80
sched_switch(c41d8d80,0,2) at sched_switch+0x183
mi_switch(2,0,c41d8d80,c2,c0a0c7b0,0,c0921068,101) at mi_switch+0x280
ast(e677ed38) at ast+0x355
doreti_ast() at doreti_ast+0x17

Tracing command udp pid 1875 tid 100517 td 0xc49351b0
sched_switch(c49351b0,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0c7b0,0,c091f13b,...) at mi_switch+0x280
critical_exit(c0a0c7b0,e6c4fa6c,c068ee4d,c3c36bd0,e6c4fa98,...) at critical_exit+0x92
spinlock_exit(c3c36bd0,e6c4fa98,c06bf249,c0a0c7b0,0,...) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c0a0c7b0,0,c09210f9,371,0,...) at _mtx_unlock_spin_flags+0xb1
turnstile_unpend(c3c2e5c0,0,c46a4c60,39f,e6c4fad4,...) at turnstile_unpend+0x371
_mtx_unlock_sleep(c46a4c60,0,c092f39f,39f) at _mtx_unlock_sleep+0x6b
_mtx_unlock_flags(c46a4c60,0,c092f39f,39f,0,...) at _mtx_unlock_flags+0xcc
udp_output(c46a4bd0,c4899900,0,0,c49351b0,...) at udp_output+0x4cf
udp_send(c411e67c,0,c462cc00,0,0,...) at udp_send+0x2a
sosend_dgram(c411e67c,0,e6c4fc64,c462cc00,0,...) at sosend_dgram+0x30d
sosend(c411e67c,0,e6c4fc64,0,0,0,c49351b0) at sosend+0x3c
soo_write(c4bb9d38,e6c4fc64,c4609980,0,c49351b0) at soo_write+0x42
dofilewrite(c49351b0,3,c4bb9d38,e6c4fc64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c49351b0,3,e6c4fc64,8050800,0,...) at kern_writev+0x36
write(c49351b0,e6c4fd04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805188c,bfbfea28,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814532f, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---

Tracing command swap pid 1874 tid 100196 td 0xc405fa20
sched_switch(c405fa20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c19dc8b0,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c19dc8b0) at sleepq_timedwait+0x4a
msleep(c19dc8b0,c52fa528,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c52fa528,e66a3c30,8,0) at swap_pager_getpages+0x348
vm_fault(c3c3c618,80d8000,2,8,c405fa20,...) at vm_fault+0x928
trap_pfault(e66a3d38,1,80d8000,80d8000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command udp pid 1873 tid 100411 td 0xc48b1360

Tracing command swap pid 1872 tid 100555 td 0xc4b34510
sched_switch(c4b34510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1bd83c0,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c1bd83c0) at sleepq_timedwait+0x4a
msleep(c1bd83c0,c57d02d0,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c57d02d0,e6d87c30,8,0) at swap_pager_getpages+0x348
vm_fault(c4073c30,83e0000,2,8,c4b34510,...) at vm_fault+0x928
trap_pfault(e6d87d38,1,83e0000,83e0000,8049180,...) at trap_pfault+0xee
trap(3b,3b,3b,2805188c,bfbfea28,...) at trap+0x273
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8049180, esp = 0xbfbfe8e0, ebp = 0xbfbfe8f8 ---

Tracing command mkdir pid 1871 tid 100544 td 0xc4b38870

Tracing command swap pid 1870 tid 100278 td 0xc477aa20

Tracing command udp pid 1868 tid 100128 td 0xc41d8bd0
sched_switch(c41d8bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0cfc4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0cfc4) at sleepq_timedwait_sig+0x1e
msleep(c0a0cfc4,0,15c,c091e70c,3e9,...) at msleep+0x244
kern_nanosleep(c41d8bd0,e677bc74,e677bc6c) at kern_nanosleep+0xab
nanosleep(c41d8bd0,e677bd04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe8ec, ebp = 0xbfbfe918 ---

Tracing command thr1 pid 1867 tid 100359 td 0xc483b510
sched_switch(c483b510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1c258f8,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c1c258f8) at sleepq_timedwait+0x4a
msleep(c1c258f8,c5e3c618,40,c0938670,4e20) at msleep+0x259
swap_pager_getpages(c5e3c618,e6b176a4,8,0) at swap_pager_getpages+0x348
vm_fault(c48bf30c,bdb88000,1,0,c483b510,...) at vm_fault+0x928
trap_pfault(e6b177ac,0,bdb88000) at trap_pfault+0xee
trap(8,c0a50028,e6b10028,da163000,bdb88000,...) at trap+0x3f5
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc0896816, esp = 0xe6b177ec, ebp = 0xe6b1781c ---
slow_copyin(da160000,4000,e6b179a0,225,1,...) at slow_copyin+0x6
ffs_write(e6b17960) at ffs_write+0x413
VOP_WRITE_APV(c09db880,e6b17960) at VOP_WRITE_APV+0x132
vn_rdwr(1,c4b8f2a0,bdb85000,10000,1fc0000,...) at vn_rdwr+0x142
vn_rdwr_inchunks(1,c4b8f2a0,bdb85000,1b000,1fbb000,...) at vn_rdwr_inchunks+0x9f
elf32_coredump(c483b510,c4b8f2a0,ffffffff,7fffffff) at elf32_coredump+0xfb
coredump(c483b510) at coredump+0x5ab
sigexit(c483b510,b,c5af9aa8,0,c091da2b,...) at sigexit+0x61
postsig(b) at postsig+0x15c
ast(e6b17d38) at ast+0x3e2
doreti_ast() at doreti_ast+0x17

Tracing command mkdir pid 1866 tid 100522 td 0xc4932870
sched_switch(c4932870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0cfc4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0cfc4) at sleepq_timedwait_sig+0x1e
msleep(c0a0cfc4,0,15c,c091e70c,3e9,...) at msleep+0x244
kern_nanosleep(c4932870,e6c40c74,e6c40c6c) at kern_nanosleep+0xab
nanosleep(c4932870,e6c40d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe8dc, ebp = 0xbfbfe908 ---

Tracing command swap pid 1864 tid 100392 td 0xc486cbd0
sched_switch(c486cbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0cfc4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0cfc4) at sleepq_timedwait_sig+0x1e
msleep(c0a0cfc4,0,15c,c091e70c,3e9,...) at msleep+0x244
kern_nanosleep(c486cbd0,e6b74c74,e6b74c6c) at kern_nanosleep+0xab
nanosleep(c486cbd0,e6b74d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe8ec, ebp = 0xbfbfe918 ---

Tracing command rw pid 1850 tid 100366 td 0xc4800870
sched_switch(c4800870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c486a468) at sleepq_switch+0xcd
sleepq_wait_sig(c486a468) at sleepq_wait_sig+0x1d
msleep(c486a468,c486a4d0,15c,c0922ac1,0) at msleep+0x26e
kern_wait(c4800870,77d,e6ae4c2c,0,0) at kern_wait+0x7cf
wait4(c4800870,e6ae4d04) at wait4+0x2a
syscall(2805003b,bfbf003b,bfbf003b,77d,77d,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280dd6cf, esp = 0xbfbfe93c, ebp = 0xbfbfea08 ---

Tracing command rw pid 1849 tid 100282 td 0xc477a360
sched_switch(c477a360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7ca2760,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7ca2760,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7ca2760,c0a59f88,4c,c0926578,0) at msleep+0x27d
bwait(d7ca2760,4c,c0926578) at bwait+0x47
bufwait(d7ca2760,d7ca2760,0,e69b8954,c07db812,...) at bufwait+0x2d
bufwrite(d7ca2760,0,0,e69b89a4,c07c63d5,...) at bufwrite+0x168
ffs_bufwrite(d7ca2760) at ffs_bufwrite+0x282
ffs_update(c4d742a0,1,c06acb5f,c0a0e920,c3c37d60,...) at ffs_update+0x349
ffs_truncate(c4d742a0,0,0,880,c3f1d400,...) at ffs_truncate+0xbd4
ufs_rmdir(e69b8bc4) at ufs_rmdir+0x1af
VOP_RMDIR_APV(c09db880,e69b8bc4) at VOP_RMDIR_APV+0xb5
kern_rmdir(c477a360,804c340,0,e69b8d30,c0898e32,...) at kern_rmdir+0x1b6
rmdir(c477a360,e69b8d04) at rmdir+0x12
syscall(2805003b,bfbf003b,bfbf003b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (137, FreeBSD ELF32, rmdir), eip = 0x280c9257, esp = 0xbfbfea0c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1848 tid 100619 td 0xc4baad80
sched_switch(c4baad80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7ca27c0,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7ca27c0,0,c5cbade8,b7,c091b310,...) at sleepq_wait+0x46
msleep(d7ca27c0,c0a0bf10,50,c0926400,0,...) at msleep+0x27d
acquire(e6e68a40,120,60000,c4baad80,0,...) at acquire+0x76
lockmgr(d7ca27c0,202122,c5cbade8,c4baad80) at lockmgr+0x46e
getblk(c5cbad20,640e0,0,4000,0,...) at getblk+0x12f
breadn(c5cbad20,640e0,0,4000,0,...) at breadn+0x2f
bread(c5cbad20,640e0,0,4000,0,e6e68b24) at bread+0x20
ffs_update(c51e9150,0) at ffs_update+0x228
ufs_inactive(e6e68b80) at ufs_inactive+0x286
VOP_INACTIVE_APV(c09db880,e6e68b80) at VOP_INACTIVE_APV+0x9b
vinactive(c51e9150,c4baad80) at vinactive+0x8d
vput(c51e9150) at vput+0x1c7
kern_unlink(c4baad80,bfbfd990,0,e6e68d30,c0898e32,...) at kern_unlink+0x1b2
unlink(c4baad80,e6e68d04) at unlink+0x12
syscall(2805003b,bfbf003b,bfbf003b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (10, FreeBSD ELF32, unlink), eip = 0x28137167, esp = 0xbfbfd97c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1847 tid 100266 td 0xc47736c0
sched_switch(c47736c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7edc9a0,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7edc9a0,0,c5cd18a8,b7,c091b310,...) at sleepq_wait+0x46
msleep(d7edc9a0,c0a0c450,50,c0926400,0,...) at msleep+0x27d
acquire(e6988864,120,60000,c47736c0,0,...) at acquire+0x76
lockmgr(d7edc9a0,202122,c5cd18a8,c47736c0) at lockmgr+0x46e
getblk(c5cd17e0,153,0,4000,0,...) at getblk+0x12f
cluster_read(c5cd17e0,54d000,0,153,0,4000,0,10000,7f,e69889a8) at cluster_read+0xde
ffs_balloc_ufs2(c5cd17e0,54d000,0,1000,c3f1d400,...) at ffs_balloc_ufs2+0x1808
ffs_write(e6988b9c) at ffs_write+0x2ac
VOP_WRITE_APV(c09db880,e6988b9c) at VOP_WRITE_APV+0x132
vn_write(c4a1f6c0,e6988c64,c3f1d400,0,c47736c0) at vn_write+0x1ea
dofilewrite(c47736c0,4,c4a1f6c0,e6988c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c47736c0,4,e6988c64,bfbfda20,1000,...) at kern_writev+0x36
write(c47736c0,e6988d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814632f, esp = 0xbfbfd97c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1846 tid 100042 td 0xc3de5000
sched_switch(c3de5000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c5cbcc28,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c5cbcc28,0,c5cbcc98,b7,c091b310,...) at sleepq_wait+0x46
msleep(c5cbcc28,c0a0c120,50,c09258cd,0,...) at msleep+0x27d
acquire(e43929d8,40,60000,c3de5000,0,...) at acquire+0x76
lockmgr(c5cbcc28,2002,c5cbcc98,c3de5000) at lockmgr+0x46e
ffs_lock(e4392a30) at ffs_lock+0x76
VOP_LOCK_APV(c09db880,e4392a30) at VOP_LOCK_APV+0x87
vn_lock(c5cbcbd0,2002,c3de5000,c5cbcbd0) at vn_lock+0xac
vget(c5cbcbd0,2002,c3de5000) at vget+0xc2
cache_lookup(c5daf000,e4392c28,e4392c3c) at cache_lookup+0x392
vfs_cache_lookup(e4392b74) at vfs_cache_lookup+0x92
VOP_LOOKUP_APV(c09db880,e4392b74) at VOP_LOOKUP_APV+0xa1
lookup(e4392c14) at lookup+0x49c
namei(e4392c14) at namei+0x382
kern_chdir(c3de5000,804aba7,0,e4392d30,c0898e32,...) at kern_chdir+0x37
chdir(c3de5000,e4392d04) at chdir+0x12
syscall(3b,3b,3b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (12, FreeBSD ELF32, chdir), eip = 0x280c9737, esp = 0xbfbfea0c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1845 tid 100132 td 0xc41d8510
sched_switch(c41d8510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c597b2f8,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c597b2f8,0,c597b368,b7,c091b310,...) at sleepq_wait+0x46
msleep(c597b2f8,c0a0c360,50,c09258cd,0,...) at msleep+0x27d
acquire(e676fa78,40,60000,c41d8510,0,...) at acquire+0x76
lockmgr(c597b2f8,3002,c597b368,c41d8510) at lockmgr+0x46e
ffs_lock(e676fad0) at ffs_lock+0x76
VOP_LOCK_APV(c09db880,e676fad0) at VOP_LOCK_APV+0x87
vn_lock(c597b2a0,1002,c41d8510,c597b2a0,e676fb0c,1) at vn_lock+0xac
vn_close(c597b2a0,2,c3f1d400,c41d8510) at vn_close+0x5e
vn_closefile(c48de828,c41d8510) at vn_closefile+0xca
fdrop_locked(c48de828,c41d8510,c3c131d8,0,c0919c97,...) at fdrop_locked+0x88
fdrop(c48de828,c41d8510,6b6,c0a13374,0,...) at fdrop+0x24
closef(c48de828,c41d8510,0,c3ca3800,0,...) at closef+0x367
kern_close(c41d8510,4,e676fd30,c0898e32,c41d8510,...) at kern_close+0x1b6
close(c41d8510,e676fd04) at close+0x10
syscall(c088003b,3b,3b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x2814630f, esp = 0xbfbfd97c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1844 tid 100150 td 0xc45e9510
sched_switch(c45e9510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7f5b858,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7f5b858,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7f5b858,c0a59f88,4c,c0926572,0) at msleep+0x27d
bwait(d7f5b858,4c,c0926572) at bwait+0x47
bufwait(d7f5b858,0,0,0,c6227000,...) at bufwait+0x1a
breadn(c5561690,0,0,800,0,...) at breadn+0x11a
bread(c5561690,0,0,800,0,e67ea7c0) at bread+0x20
ffs_blkatoff(c5561690,0,0,0,e67ea82c) at ffs_blkatoff+0x9e
ufs_lookup(e67ea8d0) at ufs_lookup+0x2cd
VOP_CACHEDLOOKUP_APV(c09db880,e67ea8d0) at VOP_CACHEDLOOKUP_APV+0x9b
vfs_cache_lookup(e67ea970) at vfs_cache_lookup+0xb5
VOP_LOOKUP_APV(c09db880,e67ea970) at VOP_LOOKUP_APV+0xa1
lookup(e67eab7c) at lookup+0x49c
namei(e67eab7c) at namei+0x382
vn_open_cred(e67eab7c,e67eac7c,0,c3f1d400,4,...) at vn_open_cred+0x28f
vn_open(e67eab7c,e67eac7c,0,4) at vn_open+0x1e
kern_open(c45e9510,bfbfd990,0,1,1000,...) at kern_open+0xb9
open(c45e9510,e67ead04) at open+0x18
syscall(2805003b,bfbf003b,bfbf003b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (5, FreeBSD ELF32, open), eip = 0x281394b3, esp = 0xbfbfd97c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1843 tid 100458 td 0xc49a61b0
sched_switch(c49a61b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7d7f404,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7d7f404,0,c4ca60c8,b7,c091b310,...) at sleepq_wait+0x46
msleep(d7d7f404,c0a0bce8,50,c0926400,0,...) at msleep+0x27d
acquire(e6cd9864,120,60000,c49a61b0,0,...) at acquire+0x76
lockmgr(d7d7f404,202122,c4ca60c8,c49a61b0) at lockmgr+0x46e
getblk(c4ca6000,1d9,0,4000,0,...) at getblk+0x12f
cluster_read(c4ca6000,765000,0,1d9,0,4000,0,10000,7f,e6cd99a8) at cluster_read+0xde
ffs_balloc_ufs2(c4ca6000,765000,0,1000,c3f1d400,...) at ffs_balloc_ufs2+0x1808
ffs_write(e6cd9b9c) at ffs_write+0x2ac
VOP_WRITE_APV(c09db880,e6cd9b9c) at VOP_WRITE_APV+0x132
vn_write(c4bc6d80,e6cd9c64,c3f1d400,0,c49a61b0) at vn_write+0x1ea
dofilewrite(c49a61b0,4,c4bc6d80,e6cd9c64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c49a61b0,4,e6cd9c64,bfbfda20,1000,...) at kern_writev+0x36
write(c49a61b0,e6cd9d04) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x2814632f, esp = 0xbfbfd97c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1842 tid 100209 td 0xc4704870
sched_switch(c4704870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7ca27c0,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7ca27c0,0,c5cbade8,b7,c091b310,...) at sleepq_wait+0x46
msleep(d7ca27c0,c0a0bf10,50,c0926400,0,...) at msleep+0x27d
acquire(e690496c,120,60000,c4704870,0,...) at acquire+0x76
lockmgr(d7ca27c0,202122,c5cbade8,c4704870) at lockmgr+0x46e
getblk(c5cbad20,640e0,0,4000,0,...) at getblk+0x12f
breadn(c5cbad20,640e0,0,4000,0,...) at breadn+0x2f
bread(c5cbad20,640e0,0,4000,0,e6904a50) at bread+0x20
ffs_update(c5364930,0) at ffs_update+0x228
ufs_inactive(e6904aac) at ufs_inactive+0x286
VOP_INACTIVE_APV(c09db880,e6904aac) at VOP_INACTIVE_APV+0x9b
vinactive(c5364930,c4704870) at vinactive+0x8d
vput(c5364930,c602e000,c09f0a00,c5364930,2,...) at vput+0x1c7
vn_close(c5364930,2,c3f1d400,c4704870) at vn_close+0x99
vn_closefile(c48eec60,c4704870) at vn_closefile+0xca
fdrop_locked(c48eec60,c4704870,c3c133b8,0,c0919c97,...) at fdrop_locked+0x88
fdrop(c48eec60,c4704870,6b6,c0a13374,0,...) at fdrop+0x24
closef(c48eec60,c4704870,0,c40444e0,0,...) at closef+0x367
kern_close(c4704870,4,e6904d30,c0898e32,c4704870,...) at kern_close+0x1b6
close(c4704870,e6904d04) at close+0x10
syscall(3b,3b,3b,2805288c,bfbfeb70,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (6, FreeBSD ELF32, close), eip = 0x2814630f, esp = 0xbfbfd97c, ebp = 0xbfbfea28 ---

Tracing command rw pid 1841 tid 100335 td 0xc48006c0

Tracing command rw pid 1840 tid 100292 td 0xc47b3a20

Tracing command g_raid3 data pid 1837 tid 100545 td 0xc4b386c0
sched_switch(c4b386c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4185000,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c4185000,0,c66504f6,7ef,c0a0c348,...) at sleepq_wait+0x46
msleep(c4185000,c418506c,24c,c6650e9a,0,...) at msleep+0x27d
g_raid3_worker(c4185000,e6da5d38) at g_raid3_worker+0xbdd
fork_exit(c6649160,c4185000,e6da5d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe6da5d6c, ebp = 0 ---

Tracing command md13 pid 1829 tid 100229 td 0xc473e6c0
sched_switch(c473e6c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7cc8668,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7cc8668,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7cc8668,c0a59f88,4c,c0926578,0) at msleep+0x27d
bwait(d7cc8668,4c,c0926578) at bwait+0x47
bufwait(d7cc8668,d7cc8668,0,e6937af8,c07db812,...) at bufwait+0x2d
bufwrite(d7cc8668,0,0,e6937b48,c07c63d5,...) at bufwrite+0x168
ffs_bufwrite(d7cc8668) at ffs_bufwrite+0x282
ffs_update(c58f6bd0,1,225,1,e6937b9c,...) at ffs_update+0x349
ffs_write(e6937c8c) at ffs_write+0x61f
VOP_WRITE_APV(c09db880,e6937c8c) at VOP_WRITE_APV+0x132
mdstart_vnode(c5423000,c45e28c4) at mdstart_vnode+0x1c4
md_kthread(c5423000,e6937d38) at md_kthread+0x11d
fork_exit(c057fae8,c5423000,e6937d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe6937d6c, ebp = 0 ---

Tracing command md12 pid 1827 tid 100592 td 0xc4c2cd80
cpustop_handler(e6eb9a4c,c08980b2,ffffffff,e6eb99fc,c06ad59b,...) at cpustop_handler+0x31
ipi_nmi_handler(ffffffff,e6eb99fc,c06ad59b,0,0,...) at ipi_nmi_handler+0x28
trap(c0008,28,d8940028,d8948000,c0a94540,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc088fc06, esp = 0xe6eb9a94, ebp = 0xe6eb9a98 ---
smp_tlb_shootdown(f5,d8948000,d894c000) at smp_tlb_shootdown+0x5a
smp_invlpg_range(d8948000,d894c000) at smp_invlpg_range+0x1c
pmap_invalidate_range(c0a94540,d8948000,d894c000,c29c2518,e6eb9afc,...) at pmap_invalidate_range+0x83
pmap_qremove(d8948000,4) at pmap_qremove+0x3d
vfs_vmio_release(d7cdf2a0) at vfs_vmio_release+0x11c
brelse(d7cdf2a0,d7cdf2a0,c537cce4,e6eb9cbc,e6eb9bbc,...) at brelse+0x489
bufwrite(d7cdf2a0) at bufwrite+0x170
ffs_write(e6eb9c8c) at ffs_write+0x489
VOP_WRITE_APV(c09db880,e6eb9c8c) at VOP_WRITE_APV+0x132
mdstart_vnode(c5b34000,c3db2528) at mdstart_vnode+0x1c4
md_kthread(c5b34000,e6eb9d38) at md_kthread+0x11d
fork_exit(c057fae8,c5b34000,e6eb9d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe6eb9d6c, ebp = 0 ---

Tracing command md11 pid 1825 tid 100403 td 0xc48b41b0
sched_switch(c48b41b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7ce6dac,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7ce6dac,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7ce6dac,c0a59f88,4c,c0926578,0) at msleep+0x27d
bwait(d7ce6dac,4c,c0926578) at bwait+0x47
bufwait(d7ce6dac,c537c8c4,e6be0cbc,e6be0bbc,c07dc6fd,...) at bufwait+0x2d
bufwrite(d7ce6dac) at bufwrite+0x168
ffs_write(e6be0c8c) at ffs_write+0x489
VOP_WRITE_APV(c09db880,e6be0c8c) at VOP_WRITE_APV+0x132
mdstart_vnode(c5b34800,c3dd2210) at mdstart_vnode+0x1c4
md_kthread(c5b34800,e6be0d38) at md_kthread+0x11d
fork_exit(c057fae8,c5b34800,e6be0d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe6be0d6c, ebp = 0 ---

Tracing command sh pid 1819 tid 100103 td 0xc3f80d80

Tracing command run pid 93634 tid 100124 td 0xc41db360

Tracing command run pid 93633 tid 100136 td 0xc4135d80

Tracing command run pid 93632 tid 100300 td 0xc477bbd0
sched_switch(c477bbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0cfc4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0cfc4) at sleepq_timedwait_sig+0x1e
msleep(c0a0cfc4,0,15c,c091e70c,3e9,...) at msleep+0x244
kern_nanosleep(c477bbd0,e69e2c74,e69e2c6c) at kern_nanosleep+0xab
nanosleep(c477bbd0,e69e2d04) at nanosleep+0x4f
syscall(2805003b,3b,bfbf003b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812bf23, esp = 0xbfbfe82c, ebp = 0xbfbfe858 ---

Tracing command sh pid 93630 tid 100269 td 0xc47731b0

Tracing command bash pid 93623 tid 100521 td 0xc4932a20

Tracing command sshd pid 93622 tid 100450 td 0xc468da20

Tracing command sshd pid 93620 tid 100357 td 0xc483b870

Tracing command bash pid 91302 tid 100520 td 0xc4932bd0

Tracing command csh pid 91299 tid 100631 td 0xc4c3ba20

Tracing command su pid 91298 tid 100297 td 0xc47b31b0

Tracing command bash pid 89922 tid 100064 td 0xc4042d80

Tracing command sshd pid 89839 tid 100369 td 0xc486bbd0

Tracing command sshd pid 89819 tid 100551 td 0xc4b34bd0

Tracing command sh pid 2823 tid 100114 td 0xc4134bd0

Tracing command bash pid 1906 tid 100111 td 0xc41351b0

Tracing command sshd pid 1905 tid 100054 td 0xc3de8a20

Tracing command sshd pid 1807 tid 100131 td 0xc41d86c0

Tracing command top pid 972 tid 100071 td 0xc40421b0
sched_switch(c40421b0,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0c7b0,0,c091f13b,...) at mi_switch+0x280
critical_exit(c0a0c7b0,e6634b98,c068ee4d,c3c36d80,e6634bc4,...) at critical_exit+0x92
spinlock_exit(c3c36d80,e6634bc4,c06bf249,c0a0c7b0,0,...) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c0a0c7b0,0,c09210f9,371,0,...) at _mtx_unlock_spin_flags+0xb1
turnstile_unpend(c45eb900,0,c0a0c7c8,52e,e6634c00,...) at turnstile_unpend+0x371
_mtx_unlock_sleep(c0a0c7c8,0,c091e104,52e) at _mtx_unlock_sleep+0x6b
_mtx_unlock_flags(c0a0c7c8,0,c091e104,52e,1d100,...) at _mtx_unlock_flags+0xcc
__sysctl(c40421b0,e6634d04) at __sysctl+0xbe
syscall(3b,82e003b,bfbf003b,3,bfbfe9bc,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (202, FreeBSD ELF32, __sysctl), eip = 0x2819c93f, esp = 0xbfbfe91c, ebp = 0xbfbfe958 ---

Tracing command bash pid 966 tid 100084 td 0xc405b1b0

Tracing command sshd pid 965 tid 100137 td 0xc4135bd0
sched_switch(c4135bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a59abc) at sleepq_switch+0xcd
sleepq_wait_sig(c0a59abc,246,c0a0c7b0,c092252d,307,...) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a59abc,c0a59aa4) at cv_wait_sig+0x18d
kern_select(c4135bd0,b,8201168,820116c,0,...) at kern_select+0x4bf
select(c4135bd0,e6715d04) at select+0x44
syscall(3b,bfbf003b,bfbf003b,0,bfbfe13c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x2838b16b, esp = 0xbfbfe0cc, ebp = 0xbfbfe118 ---

Tracing command sshd pid 963 tid 100091 td 0xc4043510

Tracing command bash pid 957 tid 100139 td 0xc44b81b0

Tracing command sshd pid 956 tid 100043 td 0xc3d46d80

Tracing command sshd pid 954 tid 100079 td 0xc405ba20

Tracing command getty pid 884 tid 100106 td 0xc4135a20

Tracing command getty pid 883 tid 100123 td 0xc41db510

Tracing command getty pid 882 tid 100120 td 0xc41341b0

Tracing command getty pid 881 tid 100096 td 0xc4060a20

Tracing command getty pid 880 tid 100117 td 0xc41346c0

Tracing command getty pid 879 tid 100056 td 0xc4001360

Tracing command getty pid 878 tid 100118 td 0xc4134510

Tracing command getty pid 877 tid 100095 td 0xc4060bd0

Tracing command inetd pid 863 tid 100040 td 0xc3de5360

Tracing command moused pid 836 tid 100102 td 0xc4000000

Tracing command watchdogd pid 821 tid 100110 td 0xc4135360
sched_switch(c4135360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0cfc4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0cfc4) at sleepq_timedwait_sig+0x1e
msleep(c0a0cfc4,0,15c,c091e70c,ea61,...) at msleep+0x244
kern_nanosleep(c4135360,e6706c74,e6706c6c) at kern_nanosleep+0xab
nanosleep(c4135360,e6706d04) at nanosleep+0x4f
syscall(1003b,1003b,bfbf003b,1,3c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28157beb, esp = 0xbfbfed3c, ebp = 0xbfbfed68 ---

Tracing command cron pid 773 tid 100119 td 0xc4134360
sched_switch(c4134360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0cfc4) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a0cfc4) at sleepq_timedwait_sig+0x1e
msleep(c0a0cfc4,0,15c,c091e70c,e679,...) at msleep+0x244
kern_nanosleep(c4134360,e66ebc74,e66ebc6c) at kern_nanosleep+0xab
nanosleep(c4134360,e66ebd04) at nanosleep+0x4f
syscall(805003b,3b,bfbf003b,805008c,3b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28147beb, esp = 0xbfbfecec, ebp = 0xbfbfed18 ---

Tracing command sendmail pid 767 tid 100122 td 0xc41db6c0

Tracing command sendmail pid 763 tid 100072 td 0xc4042000
sched_switch(c4042000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a59abc) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a59abc,c0a59abc,1389,e6631ae8,246,...) at sleepq_timedwait_sig+0x1e
cv_timedwait_sig(c0a59abc,c0a59aa4,1389) at cv_timedwait_sig+0x196
kern_select(c4042000,5,bfbfc520,0,0,...) at kern_select+0x4a9
select(c4042000,e6631d04) at select+0x44
syscall(3b,bfbf003b,bfbf003b,1,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x2834a16b, esp = 0xbfbfc42c, ebp = 0xbfbfd028 ---

Tracing command sshd pid 757 tid 100074 td 0xc4001bd0

Tracing command ntpd pid 738 tid 100094 td 0xc4060d80
sched_switch(c4060d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a59abc) at sleepq_switch+0xcd
sleepq_wait_sig(c0a59abc,246,c0a0c7b0,c092252d,307,...) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a59abc,c0a59aa4) at cv_wait_sig+0x18d
kern_select(c4060d80,9,bfbfed70,0,0,...) at kern_select+0x4bf
select(c4060d80,e66c4d04) at select+0x44
syscall(bfbf003b,3b,bfbf003b,bfbfed70,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x282cf16b, esp = 0xbfbfed2c, ebp = 0xbfbfee08 ---

Tracing command nfsd pid 684 tid 100039 td 0xc3de5510

Tracing command nfsd pid 683 tid 100060 td 0xc4000bd0

Tracing command nfsd pid 682 tid 100100 td 0xc4000360

Tracing command nfsd pid 681 tid 100069 td 0xc4042510

Tracing command nfsd pid 680 tid 100073 td 0xc4001d80

Tracing command mountd pid 678 tid 100104 td 0xc3f80bd0

Tracing command rpcbind pid 628 tid 100061 td 0xc4000a20

Tracing command syslogd pid 609 tid 100099 td 0xc4000510

Tracing command devd pid 542 tid 100083 td 0xc405b360

Tracing command schedcpu pid 42 tid 100046 td 0xc3f80870
sched_switch(c3f80870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(e43f4cfc,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(e43f4cfc) at sleepq_timedwait+0x4a
msleep(e43f4cfc,0,0,c0915ea2,3e8) at msleep+0x259
schedcpu_thread(0,e43f4d38) at schedcpu_thread+0x26
fork_exit(c06abf90,0,e43f4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43f4d6c, ebp = 0 ---

Tracing command softdepflush pid 41 tid 100047 td 0xc3f806c0
sched_switch(c3f806c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7e69f08,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7e69f08,0,c0925e5e,eb8,8,...) at sleepq_wait+0x46
msleep(d7e69f08,c0a59f88,4c,c0926572,0) at msleep+0x27d
bwait(d7e69f08,4c,c0926572) at bwait+0x47
bufwait(d7e69f08,0,0,0,0,...) at bufwait+0x1a
breadn(c4033930,33b080,0,4000,0,...) at breadn+0x11a
bread(c4033930,33b080,0,4000,0,e43f19e0) at bread+0x20
ffs_update(c4dc13f0,0,c0a13374,e43f1a10,c068ee4d,...) at ffs_update+0x228
ffs_truncate(c4dc13f0,0,0,c00,0,...) at ffs_truncate+0x43b
ufs_inactive(e43f1c0c) at ufs_inactive+0x194
VOP_INACTIVE_APV(c09db880,e43f1c0c) at VOP_INACTIVE_APV+0x9b
vinactive(c4dc13f0,c3f806c0) at vinactive+0x8d
vput(c4dc13f0,c0a669d8,0,c0936068,dfe,...) at vput+0x1c7
handle_workitem_remove(c6021b80,0) at handle_workitem_remove+0x10f
process_worklist_item(c40df2dc,0) at process_worklist_item+0x193
softdep_process_worklist(c40df2dc,0,0,c07ce128,c07ce128,...) at softdep_process_worklist+0x7c
softdep_flush(0,e43f1d38) at softdep_flush+0x13b
fork_exit(c07ce128,0,e43f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43f1d6c, ebp = 0 ---

Tracing command vnlru pid 40 tid 100048 td 0xc3f80510
sched_switch(c3f80510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3f7fd38,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3f7fd38) at sleepq_timedwait+0x4a
msleep(c3f7fd38,c0a5a1f8,250,c0927ae2,3e8,c0a5a258) at msleep+0x259
vnlru_proc(0,e43eed38) at vnlru_proc+0xdf
fork_exit(c06f4cbc,0,e43eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43eed6c, ebp = 0 ---

Tracing command syncer pid 39 tid 100049 td 0xc3f80360
sched_switch(c3f80360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7ca27c0,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(d7ca27c0,0,c5cbade8,b7,c0927a21,...) at sleepq_wait+0x46
msleep(d7ca27c0,c0a0bf10,50,c0926400,0,...) at msleep+0x27d
acquire(e43ebb44,120,60000,c3f80360,0,...) at acquire+0x76
lockmgr(d7ca27c0,202122,c5cbade8,c3f80360) at lockmgr+0x46e
getblk(c5cbad20,640e0,0,4000,0,...) at getblk+0x12f
breadn(c5cbad20,640e0,0,4000,0,...) at breadn+0x2f
bread(c5cbad20,640e0,0,4000,0,e43ebc28) at bread+0x20
ffs_update(c597b2a0,0,102,0,0,...) at ffs_update+0x228
ffs_syncvnode(c597b2a0,3) at ffs_syncvnode+0x398
ffs_fsync(e43ebcb4) at ffs_fsync+0x12
VOP_FSYNC_APV(c09db880,e43ebcb4) at VOP_FSYNC_APV+0x9b
sync_vnode(c597b3a0,c3f80360) at sync_vnode+0x106
sched_sync(0,e43ebd38) at sched_sync+0x1f5
fork_exit(c06f6964,0,e43ebd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43ebd6c, ebp = 0 ---

Tracing command bufdaemon pid 38 tid 100027 td 0xc3c3bd80
sched_switch(c3c3bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a59f50,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a59f50,0,c0925e5e,182,8,...) at sleepq_wait+0x46
msleep(c0a59f50,c0a59f54,44,c0925e6d,0) at msleep+0x27d
waitrunningbufspace(1a9,c1,0,c3fefbd0,c3c3bd80,...) at waitrunningbufspace+0x4a
flushbufqueues(2,0) at flushbufqueues+0x4d3
buf_daemon(0,e270fd38) at buf_daemon+0x95
fork_exit(c06e77ac,0,e270fd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe270fd6c, ebp = 0 ---

Tracing command pagezero pid 37 tid 100028 td 0xc3c3bbd0
sched_switch(c3c3bbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a6f330,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a6f330) at sleepq_timedwait+0x4a
msleep(c0a6f330,c0a671fc,200,c093b204,493e0,c0a671fc,0,c093b1ea,9a) at msleep+0x259
vm_pagezero(0,e270cd38) at vm_pagezero+0x6c
fork_exit(c0800bf0,0,e270cd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe270cd6c, ebp = 0 ---

Tracing command vmdaemon pid 36 tid 100029 td 0xc3c3ba20
sched_switch(c3c3ba20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a67260,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a67260,0,e2709cd0,c069d310,c093add8,...) at sleepq_wait+0x46
msleep(c0a67260,0,68,c092639c,0) at msleep+0x27d
vm_daemon(0,e2709d38) at vm_daemon+0x36
fork_exit(c07ff83c,0,e2709d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2709d6c, ebp = 0 ---

Tracing command pagedaemon pid 35 tid 100030 td 0xc3c3b870
sched_switch(c3c3b870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a66b78,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a66b78,0,c093b043,144,c093add8,...) at sleepq_wait+0x46
msleep(c0a66b78,c0a6f2f8,44,c093b077,0) at msleep+0x27d
getpbuf(c0a66b78,e2706be0,e2706aec,faa0,0,...) at getpbuf+0x44
swap_pager_putpages(c4905438,e2706be0,10,0,e2706b10) at swap_pager_putpages+0x1d3
vm_pageout_flush(e2706be0,10,0,271,c3c3b870,...) at vm_pageout_flush+0x112
vm_pageout_clean(c1e224a0) at vm_pageout_clean+0x2dd
vm_pageout_scan(0,c0a671fc,0,c093adcf,5ee) at vm_pageout_scan+0x63e
vm_pageout(0,e2706d38) at vm_pageout+0x2c7
fork_exit(c07ff4f8,0,e2706d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2706d6c, ebp = 0 ---

Tracing command irq7: ppc0 pid 34 tid 100031 td 0xc3c3b6c0
fork_trampoline() at fork_trampoline

Tracing command fdc0 pid 33 tid 100032 td 0xc3c3b510
sched_switch(c3c3b510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d7e03c,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3d7e03c) at sleepq_timedwait+0x4a
msleep(c3d7e03c,c3d7e0f0,4c,c0915ea2,3e8) at msleep+0x259
fdc_worker(c3d7e000,c3d7e0f0,0,c093debe,4a1) at fdc_worker+0x1fe
fdc_thread(c3d7e000,e2700d38) at fdc_thread+0x54
fork_exit(c085d21c,c3d7e000,e2700d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe2700d6c, ebp = 0 ---

Tracing command swi0: sio pid 32 tid 100033 td 0xc3de8000
sched_switch(c3de8000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3defc90,e43add38) at ithread_loop+0xda
fork_exit(c06830c8,c3defc90,e43add38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43add6c, ebp = 0 ---

Tracing command irq12: psm0 pid 31 tid 100034 td 0xc3de5d80
fork_trampoline() at fork_trampoline

Tracing command irq1: atkbd0 pid 30 tid 100035 td 0xc3de5bd0
sched_switch(c3de5bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3def290,e43a7d38) at ithread_loop+0xda
fork_exit(c06830c8,c3def290,e43a7d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43a7d6c, ebp = 0 ---

Tracing command irq15: ata1 pid 29 tid 100036 td 0xc3de5a20
fork_trampoline() at fork_trampoline

Tracing command irq14: ata0 pid 28 tid 100037 td 0xc3de5870
sched_switch(c3de5870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3de3460,e43a1d38) at ithread_loop+0xda
fork_exit(c06830c8,c3de3460,e43a1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43a1d6c, ebp = 0 ---

Tracing command irq17: fxp0 pid 27 tid 100017 td 0xc3c376c0
sched_switch(c3c376c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3d7b460,e26e8d38) at ithread_loop+0xda
fork_exit(c06830c8,c3d7b460,e26e8d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e8d6c, ebp = 0 ---

Tracing command usbtask pid 26 tid 100018 td 0xc3c37510
sched_switch(c3c37510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0a544,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a0a544,0,c068ebc8,c0a0c7c8,c0912eea,...) at sleepq_wait+0x46
msleep(c0a0a544,0,5c,c0912f06,0,...) at msleep+0x27d
usb_task_thread(0,e26e5d38) at usb_task_thread+0x4b
fork_exit(c0632498,0,e26e5d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e5d6c, ebp = 0 ---

Tracing command usb0 pid 25 tid 100019 td 0xc3c37360
sched_switch(c3c37360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d6d210,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c3d6d210) at sleepq_timedwait+0x4a
msleep(c3d6d210,0,5c,c0912eff,ea60,c3d798c0) at msleep+0x259
usb_event_thread(c3d798c0,e26e2d38) at usb_event_thread+0x94
fork_exit(c06323e4,c3d798c0,e26e2d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26e2d6c, ebp = 0 ---

Tracing command irq16: uhci0 pid 24 tid 100020 td 0xc3c371b0
sched_switch(c3c371b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3d690b0,e26dfd38) at ithread_loop+0xda
fork_exit(c06830c8,c3d690b0,e26dfd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26dfd6c, ebp = 0 ---

Tracing command em0 taskq pid 23 tid 100021 td 0xc3d46870
sched_switch(c3d46870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3d5a280,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c3d5a280,c3d5a29c,8,c0920fbd,47,...) at sleepq_wait+0x46
msleep_spin(c3d5a280,c3d5a29c,c0915ea2,0) at msleep_spin+0x1a1
taskqueue_thread_loop(c3d30208,e4147d38) at taskqueue_thread_loop+0x61
fork_exit(c06bd590,c3d30208,e4147d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe4147d6c, ebp = 0 ---

Tracing command irq9: acpi0 pid 22 tid 100022 td 0xc3d466c0
fork_trampoline() at fork_trampoline

Tracing command swi5: + pid 21 tid 100023 td 0xc3d46510
fork_trampoline() at fork_trampoline

Tracing command thread taskq pid 9 tid 100024 td 0xc3d46360
sched_switch(c3d46360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85700,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85700,0,c0920fc6,49,8,...) at sleepq_wait+0x46
msleep(c3c85700,c3c8571c,0,c0915ea2,0) at msleep+0x27d
taskqueue_thread_loop(c0a124d0,e413ed38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c0a124d0,e413ed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe413ed6c, ebp = 0 ---

Tracing command swi6: Giant taskq pid 20 tid 100025 td 0xc3d461b0
fork_trampoline() at fork_trampoline

Tracing command acpi_task_2 pid 8 tid 100026 td 0xc3d46000
sched_switch(c3d46000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0920fc6,49,8,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0915ea2,0) at msleep+0x27d
taskqueue_thread_loop(c109fa34,e4138d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c109fa34,e4138d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe4138d6c, ebp = 0 ---

Tracing command acpi_task_1 pid 7 tid 100008 td 0xc3c361b0
sched_switch(c3c361b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0920fc6,49,8,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0915ea2,0) at msleep+0x27d
taskqueue_thread_loop(c109fa34,e26c4d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c109fa34,e26c4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26c4d6c, ebp = 0 ---

Tracing command acpi_task_0 pid 6 tid 100009 td 0xc3c36000
sched_switch(c3c36000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85880,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85880,0,c0920fc6,49,c3d52800,...) at sleepq_wait+0x46
msleep(c3c85880,c3c8589c,0,c0915ea2,0) at msleep+0x27d
taskqueue_thread_loop(c109fa34,e26c1d38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c109fa34,e26c1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26c1d6c, ebp = 0 ---

Tracing command swi6: task queue pid 19 tid 100010 td 0xc3c3b360
sched_switch(c3c3b360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3cff590,e26fdd38) at ithread_loop+0xda
fork_exit(c06830c8,c3cff590,e26fdd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26fdd6c, ebp = 0 ---

Tracing command kqueue taskq pid 5 tid 100011 td 0xc3c3b1b0
sched_switch(c3c3b1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c85a00,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c3c85a00,0,c0920fc6,49,8,...) at sleepq_wait+0x46
msleep(c3c85a00,c3c85a1c,0,c0915ea2,0) at msleep+0x27d
taskqueue_thread_loop(c0a0b4fc,e26fad38) at taskqueue_thread_loop+0x78
fork_exit(c06bd590,c0a0b4fc,e26fad38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26fad6c, ebp = 0 ---

Tracing command swi2: cambio pid 18 tid 100012 td 0xc3c3b000
fork_trampoline() at fork_trampoline

Tracing command yarrow pid 17 tid 100013 td 0xc3c37d80
sched_switch(c3c37d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a08180,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a08180) at sleepq_timedwait+0x4a
msleep(c0a08180,0,0,c0915ea2,64) at msleep+0x259
random_kthread(0,e26f4d38) at random_kthread+0x180
fork_exit(c05d7b78,0,e26f4d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f4d6c, ebp = 0 ---

Tracing command g_down pid 4 tid 100014 td 0xc3c37bd0
cpustop_handler(e26f1b6c,c08980b2,c0a0c7b0,3,a,...) at cpustop_handler+0x31
ipi_nmi_handler(c0a0c7b0,3,a,0,0,...) at ipi_nmi_handler+0x28
trap(360008,c3c30028,e26f0028,60,c3df3c00,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc086f72c, esp = 0xe26f1bb4, ebp = 0xe26f1bc4 ---
siointr1(c3df3c00,c0a7bdec,0,c093fe42,56e,...) at siointr1+0x374
siointr(c3df3c00) at siointr+0x21
intr_execute_handlers(c3c2c0c4,e26f1c20,4,e26f1c64,c0883863,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc088d387, esp = 0xe26f1c60, ebp = 0xe26f1c64 ---
spinlock_exit(c0a57ca8,e26f1c8c,c06c26d1,c0a13374,0,...) at spinlock_exit+0x27
_mtx_unlock_spin_flags(c0a13374,0,c0921b5f,6aa,c0a0abe8,...) at _mtx_unlock_spin_flags+0xb1
witness_lock_list_get(c0a1be10,c0a0abe8,44,0,e26f1cc8,...) at witness_lock_list_get+0x8d
witness_lock(c0a0abe8,8,c09174b5,44) at witness_lock+0xc1
_mtx_lock_flags(c0a0abe8,0,c09174ac,44,e26f1cf4,...) at _mtx_lock_flags+0xc4
g_bioq_lock(c0a0abe0,0,c3c37bd0,e26f1d04,c0660682,...) at g_bioq_lock+0x18
g_io_schedule_down(c3c37bd0) at g_io_schedule_down+0x12
g_down_procbody(0,e26f1d38) at g_down_procbody+0x5a
fork_exit(c0660628,0,e26f1d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26f1d6c, ebp = 0 ---

Tracing command g_up pid 3 tid 100015 td 0xc3c37a20
sched_switch(c3c37a20,0,2) at sched_switch+0x183
mi_switch(2,0,c0a0c7b0,0,c091f13b,...) at mi_switch+0x280
critical_exit(5ab40f,e26eec28,c0883a7e,8,c3c30028,...) at critical_exit+0x92
lapic_handle_timer(8,c3c30028,c0a10028,c3c37a20,c0a12494,...) at lapic_handle_timer+0x149
Xtimerint() at Xtimerint+0x1e
--- interrupt, eip = 0xc068f0f8, esp = 0xe26eec18, ebp = 0xe26eec28 ---
_mtx_lock_spin(c0a12494,c3c37a20,0,c0944dd8,2f8) at _mtx_lock_spin+0x48
_mtx_lock_spin_flags(c0a12494,0,c0944dd8,2f8) at _mtx_lock_spin_flags+0xc2
pmap_invalidate_range(c0a94540,e0b94000,e0b95000,d7c9c4f0,e26eecac,...) at pmap_invalidate_range+0x46
pmap_qremove(e0b94000,1) at pmap_qremove+0x3d
swp_pager_async_iodone(d7c9c4f0) at swp_pager_async_iodone+0x4e
bufdone(d7c9c4f0,c07ead3c,e26eece4,c06e91e0,c468e7bc,...) at bufdone+0x8f
swapgeom_done(c468e7bc) at swapgeom_done+0x31
biodone(c468e7bc) at biodone+0x58
g_io_schedule_up(c3c37a20) at g_io_schedule_up+0xe6
g_up_procbody(0,e26eed38) at g_up_procbody+0x5a
fork_exit(c06605c8,0,e26eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26eed6c, ebp = 0 ---

Tracing command g_event pid 2 tid 100016 td 0xc3c37870
sched_switch(c3c37870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a0acf0,c0a0c7b0,0,c09209eb,236,...) at sleepq_switch+0xcd
sleepq_timedwait(c0a0acf0) at sleepq_timedwait+0x4a
msleep(c0a0acf0,0,4c,c0915ea2,64) at msleep+0x259
g_event_procbody(0,e26ebd38) at g_event_procbody+0x9e
fork_exit(c0660688,0,e26ebd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26ebd6c, ebp = 0 ---

Tracing command swi3: vm pid 16 tid 100000 td 0xc3c37000
fork_trampoline() at fork_trampoline

Tracing command swi4: clock sio pid 15 tid 100001 td 0xc3c36d80
sched_switch(c3c36d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3c118a0,e26d9d38) at ithread_loop+0xda
fork_exit(c06830c8,c3c118a0,e26d9d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d9d6c, ebp = 0 ---

Tracing command swi1: net pid 14 tid 100002 td 0xc3c36bd0
sched_switch(c3c36bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
ithread_loop(c3c118b0,e26d6d38) at ithread_loop+0xda
fork_exit(c06830c8,c3c118b0,e26d6d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d6d6c, ebp = 0 ---

Tracing command idle: cpu0 pid 13 tid 100003 td 0xc3c36a20
sched_switch(c3c36a20,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0c7b0,0,c091a79b,...) at mi_switch+0x280
idle_proc(0,e26d3d38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26d3d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d3d6c, ebp = 0 ---

Tracing command idle: cpu1 pid 12 tid 100004 td 0xc3c36870
sched_switch(c3c36870,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0c7b0,0,c091a79b,...) at mi_switch+0x280
idle_proc(0,e26d0d38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26d0d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26d0d6c, ebp = 0 ---

Tracing command idle: cpu2 pid 11 tid 100005 td 0xc3c366c0
sched_switch(c3c366c0,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0c7b0,0,c091a79b,...) at mi_switch+0x280
idle_proc(0,e26cdd38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26cdd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26cdd6c, ebp = 0 ---

Tracing command idle: cpu3 pid 10 tid 100006 td 0xc3c36510
sched_switch(c3c36510,0,1) at sched_switch+0x183
mi_switch(1,0,c0a0c7b0,0,c091a79b,...) at mi_switch+0x280
idle_proc(0,e26cad38) at idle_proc+0x8c
fork_exit(c06822dc,0,e26cad38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26cad6c, ebp = 0 ---

Tracing command init pid 1 tid 100007 td 0xc3c36360
sched_switch(c3c36360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3c3a000) at sleepq_switch+0xcd
sleepq_wait_sig(c3c3a000) at sleepq_wait_sig+0x1d
msleep(c3c3a000,c3c3a068,15c,c0922ac1,0) at msleep+0x26e
kern_wait(c3c36360,ffffffff,e26c7c2c,0,0) at kern_wait+0x7cf
wait4(c3c36360,e26c7d04) at wait4+0x2a
syscall(3b,3b,3b,0,0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x8053047, esp = 0xbfbfed9c, ebp = 0xbfbfedb8 ---

Tracing command swapper pid 0 tid 0 td 0xc0a0b038
sched_switch(c0a0b038,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a67038,c0a0c7b0,0,c09209eb,211,...) at sleepq_switch+0xcd
sleepq_wait(c0a67038,0,c093a662,3c8,8,...) at sleepq_wait+0x46
msleep(c0a67038,c0a671fc,244,c093a8c1,0) at msleep+0x27d
vm_wait(c3c19df4,c09dc3d0,1428000,c1420d88,c066d742,...) at vm_wait+0x93
scheduler(0,141ec00,141e000,0,c0451735,...) at scheduler+0x4d
mi_startup() at mi_startup+0x96
begin() at begin+0x2c
db> call doadump
Physical memory: 1007 MB
Dumping 182 MB: 167 151 135 119 103 87 71 55 39 23 7
Dump complete
= 0xf
db> reset
cpu_reset: Restarting BSP
cpu_reset_proxy: Stopped CPU 1

#6  0xc0898b15 in trap_fatal (frame=0xe6bf5a58, eva=3295318256)
    at ../../../i386/i386/trap.c:858
#7  0xc0898193 in trap (frame=
      {tf_fs = 8, tf_es = 40, tf_ds = 40, tf_edi = -1009253376, tf_esi = -999649052, tf_ebp = -423666972, tf_isp = -423667068, tf_ebx = 3, tf_edx = 1224750873, tf_ecx = -999649040, tf_eax = -1551885445, tf_trapno = 12, tf_err = -999649040, tf_eip = -1066148756, tf_cs = 32, tf_eflags = 66050, tf_esp = -1048164000, tf_ss = -1006971392})
    at ../../../i386/i386/trap.c:277
#8  0xc08834aa in calltrap () at ../../../i386/i386/exception.s:138
#9  0xc073dc6c in ip_output (m=0xc4633600, opt=0x3, ro=0xe6bf5ab8, flags=0, imo=0x0,
    inp=0xc46a4690) at fnv_hash.h:26
#10 0xc074fa0f in udp_output (inp=0xc46a4690, m=0xc46a9000, addr=0x0, control=0x0,
    td=0xc48b4d80) at ../../../netinet/udp_usrreq.c:925
#11 0xc074ffde in udp_send (so=0xa3801b7b, flags=0, m=0xc4896b00, addr=0x0,
    control=0x0, td=0xc48b4d80) at ../../../netinet/udp_usrreq.c:1107
#12 0xc06dbc71 in sosend_dgram (so=0xc4a94914, addr=0x0, uio=0xe6bf5c64,
    top=0xc4896b00, control=0x0, flags=0, td=0xc48b4d80)
    at ../../../kern/uipc_socket.c:1064
#13 0xc06dc254 in sosend (so=0x49003719, addr=0x0, uio=0xe6bf5c64, top=0x0,
    control=0x0, flags=0, td=0xc48b4d80) at ../../../kern/uipc_socket.c:1286
#14 0xc06c864e in soo_write (fp=0xa3801b7b, uio=0xe6bf5c64, active_cred=0xc4609980,
    flags=0, td=0xc48b4d80) at ../../../kern/sys_socket.c:118
#15 0xc06c3243 in dofilewrite (td=0xc48b4d80, fd=3, fp=0xc4008c60, auio=0xe6bf5c64,
    offset=Unhandled dwarf expression opcode 0x93
) at file.h:253
#16 0xc06c30e6 in kern_writev (td=0xc48b4d80, fd=3, auio=0xe6bf5c64)
    at ../../../kern/sys_generic.c:402
#17 0xc06c3011 in write (td=0xc48b4d80, uap=0xc46a90f0)
    at ../../../kern/sys_generic.c:326
#18 0xc0898e32 in syscall (frame=
      {tf_fs = 671416379, tf_es = -1078001605, tf_ds = -1078001605, tf_edi = 671422604, tf_esi = -1077941720, tf_ebp = -1077942024, tf_isp = -423666332, tf_ebx = 1, tf_edx = -1, tf_ecx = 672480640, tf_eax = 4, tf_trapno = 32, tf_err = 2, tf_eip = 672420655, tf_cs = 51, tf_eflags = 658, tf_esp = -1077942100, tf_ss = 59}) at ../../../i386/i386/trap.c:1006
#19 0xc08834ff in Xint0x80_syscall () at ../../../i386/i386/exception.s:191
(kgdb) f 12
#12 0xc06dbc71 in sosend_dgram (so=0xc4a94914, addr=0x0, uio=0xe6bf5c64,
    top=0xc4896b00, control=0x0, flags=0, td=0xc48b4d80)
    at ../../../kern/uipc_socket.c:1064
1064            error = (*so->so_proto->pr_usrreqs->pru_send)(so,
(kgdb) l
1059             * maybe we slept while doing page faults in uiomove() etc.  We could
1060             * probably recheck again inside the locking protection here, but
1061             * there are probably other places that this also happens.  We must
1062             * rethink this.
1063             */
1064            error = (*so->so_proto->pr_usrreqs->pru_send)(so,
1065                (flags & MSG_OOB) ? PRUS_OOB :
1066            /*
1067             * If the user set MSG_EOF, the protocol understands this flag and
1068             * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND.
(kgdb) p *so
$7 = {so_count = 1, so_type = 2, so_options = 0, so_linger = 0, so_state = 2,
  so_qstate = 0, so_pcb = 0xc46a4690, so_proto = 0xc09d2f34, so_head = 0x0,
  so_incomp = {tqh_first = 0x0, tqh_last = 0xc4a94930}, so_comp = {tqh_first = 0x0,
    tqh_last = 0xc4a94938}, so_list = {tqe_next = 0x0, tqe_prev = 0x0}, so_qlen = 0,
  so_incqlen = 0, so_qlimit = 0, so_timeo = 0, so_error = 0, so_sigio = 0x0,
  so_oobmark = 0, so_aiojobq = {tqh_first = 0x0, tqh_last = 0xc4a9495c}, so_rcv = {
    sb_sel = {si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0,
      si_note = {kl_list = {slh_first = 0x0}, kl_lock = 0xc067d5c8 <knlist_mtx_lock>,
        kl_unlock = 0xc067d5e4 <knlist_mtx_unlock>,
        kl_locked = 0xc067d600 <knlist_mtx_locked>, kl_lockarg = 0xc4a94988},
      si_flags = 0}, sb_mtx = {mtx_object = {lo_name = 0xc09219ae "so_rcv",
        lo_type = 0xc09219ae "so_rcv", lo_flags = 16973824, lo_witness_data = {
          lod_list = {stqe_next = 0xc0a1d148}, lod_witness = 0xc0a1d148}},
      mtx_lock = 4, mtx_recurse = 0}, sb_state = 0, sb_mb = 0x0, sb_mbtail = 0x0,
    sb_lastrecord = 0x0, sb_cc = 0, sb_hiwat = 42080, sb_mbcnt = 0, sb_mbmax = 262144,
    sb_ctl = 0, sb_lowat = 1, sb_timeo = 0, sb_flags = 0}, so_snd = {sb_sel = {
      si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {
        kl_list = {slh_first = 0x0}, kl_lock = 0xc067d5c8 <knlist_mtx_lock>,
        kl_unlock = 0xc067d5e4 <knlist_mtx_unlock>,
        kl_locked = 0xc067d600 <knlist_mtx_locked>, kl_lockarg = 0xc4a949f4},
      si_flags = 0}, sb_mtx = {mtx_object = {lo_name = 0xc09219a7 "so_snd",
        lo_type = 0xc09219a7 "so_snd", lo_flags = 16973824, lo_witness_data = {
          lod_list = {stqe_next = 0xc0a1d170}, lod_witness = 0xc0a1d170}},
      mtx_lock = 4, mtx_recurse = 0}, sb_state = 0, sb_mb = 0x0, sb_mbtail = 0x0,
    sb_lastrecord = 0x0, sb_cc = 0, sb_hiwat = 9216, sb_mbcnt = 0, sb_mbmax = 73728,
    sb_ctl = 0, sb_lowat = 2048, sb_timeo = 0, sb_flags = 0}, so_upcall = 0,
  so_upcallarg = 0x0, so_cred = 0xc4609980, so_label = 0x0, so_peerlabel = 0x0,
  so_gencnt = 108497, so_emuldata = 0x0, so_accf = 0x0}
(kgdb) p *so->so_proto
$8 = {pr_type = 2, pr_domain = 0xc09d3360, pr_protocol = 17, pr_flags = 3,
  pr_input = 0xc074e708 <udp_input>, pr_output = 0,
  pr_ctlinput = 0xc074ef58 <udp_ctlinput>, pr_ctloutput = 0xc073e188 <ip_ctloutput>,
  pr_ousrreq = 0, pr_init = 0xc074e660 <udp_init>, pr_fasttimo = 0, pr_slowtimo = 0,
  pr_drain = 0, pr_usrreqs = 0xc09d57e0}
(kgdb) p *so->so_proto->pr_usrreqs
$9 = {__Break_the_struct_layout_for_now = 0, pru_abort = 0xc074fa80 <udp_abort>,
  pru_accept = 0xc06dea84 <pru_accept_notsupp>, pru_attach = 0xc074fb24 <udp_attach>,
  pru_bind = 0xc074fbec <udp_bind>, pru_connect = 0xc074fd24 <udp_connect>,
  pru_connect2 = 0xc06deab4 <pru_connect2_notsupp>,
  pru_control = 0xc07344a8 <in_control>, pru_detach = 0xc074fe38 <udp_detach>,
  pru_disconnect = 0xc074feb8 <udp_disconnect>,
  pru_listen = 0xc06dead8 <pru_listen_notsupp>,
  pru_peeraddr = 0xc0750050 <udp_peeraddr>, pru_rcvd = 0xc06deaf0 <pru_rcvd_notsupp>,
  pru_rcvoob = 0xc06deafc <pru_rcvoob_notsupp>, pru_send = 0xc074ffb4 <udp_send>,
  pru_sense = 0xc06deb14 <pru_sense_null>, pru_shutdown = 0xc074ffe0 <udp_shutdown>,
  pru_sockaddr = 0xc0750038 <udp_sockaddr>, pru_sosend = 0xc06db964 <sosend_dgram>,
  pru_soreceive = 0xc06dc33c <soreceive_generic>,
  pru_sopoll = 0xc06ddf1c <sopoll_generic>,
  pru_sosetlabel = 0xc0738964 <in_pcbsosetlabel>, pru_close = 0xc074fc80 <udp_close>}
(kgdb) info loc
space = 1024
resid = 0
clen = 0
error = 0
dontroute = 0
atomic = 1
(kgdb) bt full
#0  doadump () at pcpu.h:166
No locals.
#1  0xc047629b in db_fncall (dummy1=-423667524, dummy2=0, dummy3=-1062678592, dummy4=0xe6bf5898 "`P") at ../../../ddb/db_command.c:484
        fn_addr = -1066831824
        args = {-1064859901, -1063953760, -423667608, -1063310368, -423667592, -1069064821, -1063310368, -1063953760, -423667564,
  -423667608}
        nargs = 0
        retval = 547934144
        t = 0
#2  0xc04760a7 in db_command (last_cmdp=0xc09f2364, cmd_table=0x0) at ../../../ddb/db_command.c:399
        cmd = (struct command *) 0xc0955aa0
        t = 0
        modif = "`P\000\000\000\000\003\224\003\000\000XF\000\000\000X\003\000\000X\005\207\003\000\000\003\000\000\r\000\000\000X\a\207X\003\000\000\f\000\017\003x\000\000\000`,\237F\000\000\000\bY\200G3\221@~GF\000\000\000`,\237uG`,\237p&\237"
        addr = -423667524
        count = -1062678592
        have_addr = 0
        result = 0
#3  0xc0476162 in db_command_loop () at ../../../ddb/db_command.c:451
No locals.
#4  0xc0477dad in db_trap (type=12, code=0) at ../../../ddb/db_main.c:228
        jb = {{_jb = {-423667384, -423667404, -423667332, 1, 70, -1069056698, 0, 13, 1, 70, -423667324, -1066677732}}}
        prev_jb = (void *) 0x0
        bkpt = 0
#5  0xc06b5ea0 in kdb_trap (type=12, code=0, tf=0xe6bf5a58) at ../../../kern/subr_kdb.c:502
        intr = 70
        did_stop_cpus = 1
        handled = -423667112
#6  0xc0898b15 in trap_fatal (frame=0xe6bf5a58, eva=3295318256) at ../../../i386/i386/trap.c:858
        code = 0
        type = 12
        ss = 40
        esp = 0
        softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27, ssd_dpl = 0, ssd_p = 1, ssd_xx = 0, ssd_xx1 = 0, ssd_def32 = 1,
  ssd_gran = 1}
        msg = 0x0
#7  0xc0898193 in trap (frame=
      {tf_fs = 8, tf_es = 40, tf_ds = 40, tf_edi = -1009253376, tf_esi = -999649052, tf_ebp = -423666972, tf_isp = -423667068, tf_ebx = 3, tf_edx = 1224750873, tf_ecx = -999649040, tf_eax = -1551885445, tf_trapno = 12, tf_err = -999649040, tf_eip = -1066148756, tf_cs = 32, tf_eflags = 66050, tf_esp = -1048164000, tf_ss = -1006971392}) at ../../../i386/i386/trap.c:277
        td = (struct thread *) 0xc48b4d80
        p = (struct proc *) 0xc4934234
        i = 0
        ucode = 0
        type = 12
        code = 0
        addr = 1180
        eva = 3295318256
        ksi = {ksi_link = {tqe_next = 0xc08f8124, tqe_prev = 0x49a}, ksi_info = {si_signo = -1009253376, si_errno = -1009253376,
    si_code = -423667180, si_pid = -1066350925, si_uid = 3285713920, si_status = 0, si_addr = 0xe6bf5a34, si_value = {
      sival_int = -1066345660, sival_ptr = 0xc070db44}, _reason = {_fault = {_trapno = -1009253376}, _timer = {_timerid = -1009253376,
        _overrun = 402785280}, _mesgq = {_mqd = -1009253376}, _poll = {_band = -1009253376}, __spare__ = {__spare1__ = -1009253376,
        __spare2__ = {402785280, 0, -999649066, -1009253376, -423667012, -423667072, -1066346052}}}}, ksi_flags = -1009253376,
  ksi_sigq = 0xc46a9000}
#8  0xc08834aa in calltrap () at ../../../i386/i386/exception.s:138
No locals.
#9  0xc073dc6c in ip_output (m=0xc4633600, opt=0x3, ro=0xe6bf5ab8, flags=0, imo=0x0, inp=0xc46a4690) at fnv_hash.h:26
        ip = (struct ip *) 0xc46a90e4
        ifp = (struct ifnet *) 0xc3d80400
        m0 = (struct mbuf *) 0xc4633d00
        hlen = 20
        mtu = 1500
        len = -997627136
        error = 0
        dst = (struct sockaddr_in *) 0xe6bf5abc
        ia = (struct in_ifaddr *) 0xc3fad600
        sia = (struct in_ifaddr *) 0xc46a90f0
        isbroadcast = -999649040
        sw_csum = 3
        iproute = {ro_rt = 0xc443a258, ro_dst = {sa_len = 16 '\020', sa_family = 2 '\002',
    sa_data = "\000\000\001\002\000\000\000\000\000\000\000"}}
        odst = {s_addr = 3}
        __func__ = "ip_output"
#10 0xc074fa0f in udp_output (inp=0xc46a4690, m=0xc46a9000, addr=0x0, control=0x0, td=0xc48b4d80) at ../../../netinet/udp_usrreq.c:925
        ui = (struct udpiphdr *) 0xc46a90e4
        len = 8192
        faddr = {s_addr = 33663168}
        laddr = {s_addr = 83994816}
        cm = (struct cmsghdr *) 0xc46a90e4
        src = {sin_len = 0 '\0', sin_family = 0 '\0', sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000c\211\204p\234"}
        error = 55
        ipflags = 0
        fport = 2304
        lport = 28361
        unlock_udbinfo = 0
        __func__ = "udp_output"
#11 0xc074ffde in udp_send (so=0xa3801b7b, flags=0, m=0xc4896b00, addr=0x0, control=0x0, td=0xc48b4d80)
    at ../../../netinet/udp_usrreq.c:1107
        inp = (struct inpcb *) 0xa3801b7b
#12 0xc06dbc71 in sosend_dgram (so=0xc4a94914, addr=0x0, uio=0xe6bf5c64, top=0xc4896b00, control=0x0, flags=0, td=0xc48b4d80)
    at ../../../kern/uipc_socket.c:1064
        space = 1024
        resid = 0
        clen = 0
        error = 0
        dontroute = 0
        atomic = 1
#13 0xc06dc254 in sosend (so=0x49003719, addr=0x0, uio=0xe6bf5c64, top=0x0, control=0x0, flags=0, td=0xc48b4d80)
    at ../../../kern/uipc_socket.c:1286
No locals.
#14 0xc06c864e in soo_write (fp=0xa3801b7b, uio=0xe6bf5c64, active_cred=0xc4609980, flags=0, td=0xc48b4d80)
    at ../../../kern/sys_socket.c:118
        so = (struct socket *) 0xc4a94914
        error = -423666588
#15 0xc06c3243 in dofilewrite (td=0xc48b4d80, fd=3, fp=0xc4008c60, auio=0xe6bf5c64, offset=Unhandled dwarf expression opcode 0x93
) at file.h:253
        cnt = 8192
        error = -1006597024
        ktruio = (struct uio *) 0x0
#16 0xc06c30e6 in kern_writev (td=0xc48b4d80, fd=3, auio=0xe6bf5c64) at ../../../kern/sys_generic.c:402
        fp = (struct file *) 0xc4008c60
        error = 0
#17 0xc06c3011 in write (td=0xc48b4d80, uap=0xc46a90f0) at ../../../kern/sys_generic.c:326
        auio = {uio_iov = 0xe6bf5c5c, uio_iovcnt = 1, uio_offset = 8191, uio_resid = 0, uio_segflg = UIO_USERSPACE, uio_rw = UIO_WRITE,
  uio_td = 0xc48b4d80}
        aiov = {iov_base = 0x8052000, iov_len = 0}
#18 0xc0898e32 in syscall (frame=
      {tf_fs = 671416379, tf_es = -1078001605, tf_ds = -1078001605, tf_edi = 671422604, tf_esi = -1077941720, tf_ebp = -1077942024, tf_isp = -423666332, tf_ebx = 1, tf_edx = -1, tf_ecx = 672480640, tf_eax = 4, tf_trapno = 32, tf_err = 2, tf_eip = 672420655, tf_cs = 51, tf_eflags = 658, tf_esp = -1077942100, tf_ss = 59}) at ../../../i386/i386/trap.c:1006
        params = 0xbfbfe8b0 <Address 0xbfbfe8b0 out of bounds>
        callp = (struct sysent *) 0xc09c1000
        td = (struct thread *) 0xc48b4d80
        p = (struct proc *) 0xc4934234
        orig_tf_eflags = 658
        error = 0
        narg = 3
        args = {3, 134545408, 8192, 625, -1062675232, -423666396, -1064786263, 1}
        code = 4
        ksi = {ksi_link = {tqe_next = 0x2, tqe_prev = 0xc091f13b}, ksi_info = {si_signo = 625, si_errno = -997503616,
    si_code = -997503616, si_pid = -1063204944, si_uid = 3231762352, si_status = -996982220, si_addr = 0x2805188c, si_value = {
      sival_int = -997503616, sival_ptr = 0xc48b4d80}, _reason = {_fault = {_trapno = -423666456}, _timer = {_timerid = -423666456,
        _overrun = 70}, _mesgq = {_mqd = -423666456}, _poll = {_band = -423666456}, __spare__ = {__spare1__ = -423666456, __spare2__ = {
          70, -1063204944, -423666444, -1066865075, -997503616, -1077941720, 671422604}}}}, ksi_flags = -423666408,
  ksi_sigq = 0xc06ad59b}
(kgdb) f 9
#9  0xc073dc6c in ip_output (m=0xc4633600, opt=0x3, ro=0xe6bf5ab8, flags=0, imo=0x0, inp=0xc46a4690) at fnv_hash.h:26
26                      hval *= FNV_32_PRIME;
(kgdb) l
21      fnv_32_buf(const void *buf, size_t len, Fnv32_t hval)
22      {
23              const u_int8_t *s = (const u_int8_t *)buf;
24
25              while (len-- != 0) {
26                      hval *= FNV_32_PRIME;
27                      hval ^= *s++;
28              }
29              return hval;
30      }
(kgdb) p len
$3 = -997627136
(kgdb) up
#10 0xc074fa0f in udp_output (inp=0xc46a4690, m=0xc46a9000, addr=0x0, control=0x0, td=0xc48b4d80) at ../../../netinet/udp_usrreq.c:925
925             error = ip_output(m, inp->inp_options, NULL, ipflags,
(kgdb) l
920             ((struct ip *)ui)->ip_tos = inp->inp_ip_tos;    /* XXX */
921             udpstat.udps_opackets++;922
923             if (unlock_udbinfo)
924                     INP_INFO_WUNLOCK(&udbinfo);
925             error = ip_output(m, inp->inp_options, NULL, ipflags,
926                 inp->inp_moptions, inp);
927             INP_UNLOCK(inp);
928             return (error);
929
(kgdb) p *inp
$4 = {inp_hash = {le_next = 0x0, le_prev = 0xc46a42a0}, inp_list = {le_next = 0xc413a150, le_prev = 0xc4129b30}, inp_flow = 0,
  inp_inc = {inc_flags = 0 '\0', inc_len = 0 '\0', inc_pad = 0, inc_ie = {ie_fport = 2304, ie_lport = 28361, ie_dependfaddr = {
        ie46_foreign = {ia46_pad32 = {0, 0, 0}, ia46_addr4 = {s_addr = 33663168}}, ie6_foreign = {__u6_addr = {
            __u6_addr8 = '\0' <repeats 12 times>, "\001\002", __u6_addr16 = {0, 0, 0, 0, 0, 0, 43200, 513}, __u6_addr32 = {0, 0, 0,
              33663168}}}}, ie_dependladdr = {ie46_local = {ia46_pad32 = {0, 0, 0}, ia46_addr4 = {s_addr = 83994816}}, ie6_local = {
          __u6_addr = {__u6_addr8 = '\0' <repeats 12 times>, "\001\005", __u6_addr16 = {0, 0, 0, 0, 0, 0, 43200, 1281}, __u6_addr32 = {
              0, 0, 0, 83994816}}}}}}, inp_ppcb = 0x0, inp_pcbinfo = 0xc0a5bf20, inp_socket = 0xc4a94914, inp_label = 0x0,
  inp_flags = 8388672, inp_sp = 0x0, inp_vflag = 1 '\001', inp_ip_ttl = 64 '@', inp_ip_p = 0 '\0', inp_ip_minttl = 0 '\0',
  inp_depend4 = {inp4_ip_tos = 0 '\0', inp4_options = 0x0, inp4_moptions = 0x0}, inp_depend6 = {inp6_options = 0x0,
    inp6_outputopts = 0x0, inp6_moptions = 0x0, inp6_icmp6filt = 0x0, inp6_cksum = 0, inp6_ifindex = 0, inp6_hops = 0}, inp_portlist = {
    le_next = 0x0, le_prev = 0xc3e05968}, inp_phd = 0xc3e05960, inp_gencnt = 160743, inp_mtx = {mtx_object = {
      lo_name = 0xc092ef94 "inp", lo_type = 0xc09219db "udpinp", lo_flags = 21692416, lo_witness_data = {lod_list = {
          stqe_next = 0xc0a1d080}, lod_witness = 0xc0a1d080}}, mtx_lock = 3297463680, mtx_recurse = 0}}
(kgdb) up
#11 0xc074ffde in udp_send (so=0xa3801b7b, flags=0, m=0xc4896b00, addr=0x0, control=0x0, td=0xc48b4d80)
    at ../../../netinet/udp_usrreq.c:1107
1107            return udp_output(inp, m, addr, control, td);
(kgdb) p *so
Cannot access memory at address 0xa3801b7b