GENERIC HEAD from Jul 11 16:19 UTC, vmcore.417
Fixed in src/sys/netinet/if_ether.c,v 1.139 2005/08/11 08:25:48 glebius
Inspired by the work of Gleb Smirnoff this
problem can be provoked in seconds while stress testing, by adding this:
while true; do
arp -d 192.168.1.2
done
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #0: Mon Jul 11 19:13:08 CEST 2005
pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf13 Stepping = 3
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory = 267583488 (255 MB)
avail memory = 252223488 (240 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
rl0: link state changed to DOWN
Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address = 0xdeadc0dc
fault code = supervisor write, page not present
instruction pointer = 0x20:0xc07ecfc5
stack pointer = 0x28:0xcc9dcbe8
frame pointer = 0x28:0xcc9dcc3c
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 37 (swi1: net)
[thread pid 37 tid 100036 ]
Stopped at memcpy+0x1d: repe movsb (%esi),%es:(%edi)
db> where
Tracing pid 37 tid 100036 td 0xc1590d80
memcpy(c1689400,c173c900,c1b01880,c17f1084,c085be75) at memcpy+0x1d
in_arpinput(c173a300,c173a300,cc9dccd4,c06a20a2,c173a300) at in_arpinput+0x5b6
arpintr(c173a300) at arpintr+0xca
netisr_processqueue(c09a53f8) at netisr_processqueue+0x6e
swi_net(0) at swi_net+0xbe
ithread_loop(c1574480,cc9dcd38,c1574480,c061fc64,0) at ithread_loop+0x11c
fork_exit(c061fc64,c1574480,cc9dcd38) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xcc9dcd6c, ebp = 0 ---
db> show alllocks
Process 974 (tcp) thread 0xc1891480 (100128)
exclusive sleep mutex inp (tcpinp) r = 0 (0xc18de630) locked @ netinet/tcp_usrreq.c:651
Process 973 (arp) thread 0xc1b3d600 (100129)
exclusive sleep mutex vm page queue mutex r = 0 (0xc09b2180) locked @ vm/vm_fault.c:1007
exclusive sleep mutex vm object (standard object) r = 0 (0xc1066420) locked @ vm/vm_fault.c:992
exclusive sx user map r = 0 (0xc1b17d28) locked @ vm/vm_map.c:2997
Process 914 (udp) thread 0xc1b3d180 (100132)
exclusive sleep mutex inp (udpinp) r = 0 (0xc17ee9b4) locked @ netinet/udp_usrreq.c:762
db> where 914
Tracing pid 914 tid 100132 td 0xc1b3d180
sched_switch(c1b3d180,c1541600,6) at sched_switch+0x177
mi_switch(6,c1541600,c1541754,c1541600,cf421950) at mi_switch+0x270
maybe_preempt(c1541600) at maybe_preempt+0x165
sched_add(c1541600,4,c1574680,c1541600,c1575a3c) at sched_add+0x15a
setrunqueue(c1541600,4) at setrunqueue+0xab
ithread_schedule(c1574680,16,c1b3d180,c0955fa0,262) at ithread_schedule+0xb3
intr_execute_handlers(c1530240,cf4219cc,16,cf421a1c,c07dcb53) at intr_execute_handlers+0xe5
lapic_handle_intr(46) at lapic_handle_intr+0x2e
Xapic_isr2() at Xapic_isr2+0x33
--- interrupt, eip = 0xc062ac9e, esp = 0xcf421a10, ebp = 0xcf421a1c ---
_mtx_assert(c0955fa0,2,c0859cda,262) at _mtx_assert+0x22
critical_exit(c10443c0,0,cf421ab0,c17dd4a0,cf421a68) at critical_exit+0x4b
uma_zfree_arg(c104a9a0,c173c900,0) at uma_zfree_arg+0xe0
m_freem(c173c900) at m_freem+0x36
arpresolve(c1689400,c17f1084,c173f700,cf421b0c,cf421ab0) at arpresolve+0x1f4
ether_output(c1689400,c173f700,cf421b0c,c17f1084,c1768000) at ether_output+0x66
ip_output(c173f700,0,cf421b08,0,0) at ip_output+0x6fc
udp_output(c17ee924,c173f700,0,0,c1b3d180) at udp_output+0x4a7
udp_send(c182ade8,0,c173f700,0,0) at udp_send+0x1a
sosend(c182ade8,0,cf421cbc,c173f700,0) at sosend+0x5e3
soo_write(c1829990,cf421cbc,c18c5580,0,c1b3d180) at soo_write+0x46
dofilewrite(c1b3d180,3,c1829990,cf421cbc,ffffffff) at dofilewrite+0x77
kern_writev(c1b3d180,3,cf421cbc,804f040,0) at kern_writev+0x3b
write(c1b3d180,cf421d04,3,53,296) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,28050288,bfbfeae0) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x280c1627, esp = 0xbfbfe95c, ebp = 0xbfbfe9a8 ---
db> where 974
Tracing pid 974 tid 100128 td 0xc1891480
sched_switch(c1891480,0,2) at sched_switch+0x177
mi_switch(2,0,c0955fa0,0,c0859cda) at mi_switch+0x270
critical_exit(c0955fa0,cf3a6b0c,c062a929,c1541900,cf3a6b38) at critical_exit+0x8b
spinlock_exit(c1541900,cf3a6b38,c06525c9,c0955fa0,0) at spinlock_exit+0x10
_mtx_unlock_spin_flags(c0955fa0,0,c085b7a6,332,0) at _mtx_unlock_spin_flags+0x8d
turnstile_unpend(c1534040,c09a616c,2b9,cf3a6b70,c062a7d0) at turnstile_unpend+0x2cd
_mtx_unlock_sleep(c09a616c,0,c0867bf2,2b9) at _mtx_unlock_sleep+0x66
_mtx_unlock_flags(c09a616c,0,c0867bf2,2b9,0) at _mtx_unlock_flags+0x98
tcp_usr_send(c182a858,0,c1a4e100,0,0) at tcp_usr_send+0x18e
sosend(c182a858,0,cf3a6cbc,c1a4e100,0) at sosend+0x5e3
soo_write(c1829a68,cf3a6cbc,c18c5580,0,c1891480) at soo_write+0x46
dofilewrite(c1891480,3,c1829a68,cf3a6cbc,ffffffff) at dofilewrite+0x77
kern_writev(c1891480,3,cf3a6cbc,8053168,0) at kern_writev+0x3b
write(c1891480,cf3a6d04,3,1d,296) at write+0x45
syscall(2805003b,bfbf003b,bfbf003b,28050288,bfbfeae0) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x280c1627, esp = 0xbfbfe94c, ebp = 0xbfbfe998 ---
db> call doadump
Dumping 254 MB (2 chunks)
chunk 0: 1MB (159 pages) ... ok
chunk 1: 255MB (65072 pages) 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 ... ok
Dump complete
= 0xf
db> reset
#26 0xc069a8cc in ether_output (ifp=0xc07ecfc5, m=0xc173c900, dst=0x20, rt0=0x0) at ../../../net/if_ethersubr.c:284
#27 0xc06bb362 in in_arpinput (m=0xc173a300) at ../../../netinet/if_ether.c:736
#28 0xc06bad9e in arpintr (m=0xc173a300) at ../../../netinet/if_ether.c:505
#29 0xc06a20a2 in netisr_processqueue (ni=0xc09a53f8) at ../../../net/netisr.c:235
#30 0xc06a2286 in swi_net (dummy=0x0) at ../../../net/netisr.c:348
#31 0xc061fd80 in ithread_loop (arg=0xc1574480) at ../../../kern/kern_intr.c:545
#32 0xc061f1b0 in fork_exit (callout=0xc061fc64 <ithread_loop>, arg=0xc1574480, frame=0xcc9dcd38) at ../../../kern/kern_fork.c:789
#33 0xc07dc78c in fork_trampoline () at ../../../i386/i386/exception.s:206
(kgdb) f 26
#26 0xc069a8cc in ether_output (ifp=0xc07ecfc5, m=0xc173c900, dst=0x20, rt0=0x0) at ../../../net/if_ethersubr.c:284
284 (void)memcpy(&eh->ether_type, &type,
(kgdb) p *eh
can not access 0xdeadc0d0, invalid address (0xdeadc0d0)
can not access 0xdeadc0d0, invalid address (0xdeadc0d0)
Cannot access memory at address 0xdeadc0d0
(kgdb) up
#27 0xc06bb362 in in_arpinput (m=0xc173a300) at ../../../netinet/if_ether.c:736
736 (*ifp->if_output)(ifp, la->la_hold, rt_key(rt), rt);
(kgdb) p *m
$1 = {m_hdr = {mh_next = 0x0, mh_nextpkt = 0x0, mh_data = 0xc173a350 "", mh_len = 0x2e, mh_flags = 0x2, mh_type = 0x1}, M_dat = {MH = {
MH_pkthdr = {rcvif = 0xc1689400, len = 0x2e, header = 0xdeadc0de, csum_flags = 0x0, csum_data = 0xdeadc0de, tags = {
slh_first = 0x0}}can not access 0xdeadc0de, invalid address (0xdeadc0de)
can not access 0xdeadc0de, invalid address (0xdeadc0de)
can not access 0xdeadc0de, invalid address (0xdeadc0de)
can not access 0xdeadc0de, invalid address (0xdeadc0de)
can not access 0xdeadc0de, invalid address (0xdeadc0de)
can not access 0xdeadc0de, invalid address (0xdeadc0de)
, MH_dat = {MH_ext = {ext_buf = 0xdeadc0de <Address 0xdeadc0de out of bounds>, ext_free = 0xdeadc0de,
ext_args = 0xdeadc0de, ext_size = 0xdeadc0de, ref_cnt = 0xe00c0de, ext_type = 0x2467d1a6},
MH_databuf = "ÞÀÞÞÀÞÞÀÞÞÀÞÞÀ\000\016¦Ñg$\000à\030¯´\025\b\006\000\001\b\000\006\004\000\002\000à\030¯´\025À¨\001\002\000\016¦Ñg$À¨\001\003", '\0' <repeats 18 times>, "ÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞ"}},
M_databuf = "\000\224hÁ.\000\000\000ÞÀÞ\000\000\000\000ÞÀÞ\000\000\000\000ÞÀÞÞÀÞÞÀÞÞÀÞÞÀ\000\016¦Ñg$\000à\030¯´\025\b\006\000\001\b\000\006\004\000\002\000à\030¯´\025À¨\001\002\000\016¦Ñg$À¨\001\003", '\0' <repeats 18 times>, "ÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞÞÀÞ"...}}
(kgdb) up
#28 0xc06bad9e in arpintr (m=0xc173a300) at ../../../netinet/if_ether.c:505
505 in_arpinput(m);
(kgdb) up
#29 0xc06a20a2 in netisr_processqueue (ni=0xc09a53f8) at ../../../net/netisr.c:235
235 ni->ni_handler(m);
(kgdb) p *ni
$2 = {ni_handler = 0xc06bacd4 <arpintr>, ni_queue = 0xc09a57c0, ni_flags = 0x1}
(kgdb) p *ni->ni_queue
$3 = {ifq_head = 0x0, ifq_tail = 0x0, ifq_len = 0x0, ifq_maxlen = 0x32, ifq_drops = 0x0, ifq_mtx = {mtx_object = {lo_class = 0xc08bd744,
lo_name = 0xc0865ad4 "arp_inq", lo_type = 0xc0865ad4 "arp_inq", lo_flags = 0x30000, lo_list = {tqe_next = 0xc09a5200,
tqe_prev = 0xc16c948c}, lo_witness = 0xc0965838}, mtx_lock = 0x4, mtx_recurse = 0x0}}