GENERIC HEAD from Aug 26 13:08 UTC, vmcore.73
Livelock (not pingable)
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #0: Sat Aug 26 15:27:45 CEST 2006
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1032491008 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
ffree inode /trmp/e27e8 2i6 nhoadde -/2t0m6p8/62247 bloc8ks25 h
ad -2068624 blocks
handle_workitem_freeblocks: block count
handle_workitem_freeblocks: block count
free inode /tmp/27825 had -2068624 blocks
free inode /tmp/27826 had -2068624 blocks
:
free inode /tmp/27825 had -2068624 blocks
free inode /tmp/27826 had -2068624 blocks
handle_workitem_freeblocks: block count
handle_workitem_freeblocks: block count
~KDB: enter: Line break on console
[thread pid 13 tid 100003 ]
Stopped at kdb_enter+0x2b: nop
db> where
Tracing pid 13 tid 100003 td 0xc3c2ea20
kdb_enter(c09125d0) at kdb_enter+0x2b
siointr1(c3decc00,c0a8b60c,0,c0942af0,56e,...) at siointr1+0xce
siointr(c3decc00) at siointr+0x21
intr_execute_handlers(c3c240c4,e26cbc90,4,e26cbcd0,c08854e3,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc109f995, esp = 0xe26cbcd0, ebp = 0xe26cbcd0 ---
acpi_cpu_c1(c0a1c0d0,1,e26cbcf0,1,c0689b10,...) at acpi_cpu_c1+0x5
acpi_cpu_idle(e26cbd04,c0689b6d,c0689b10,e26cbd24,c06898a8,...) at acpi_cpu_idle+0x15a
cpu_idle(c0689b10,e26cbd24,c06898a8,0,e26cbd38,...) at cpu_idle+0x28
idle_proc(0,e26cbd38) at idle_proc+0x5d
fork_exit(c0689b10,0,e26cbd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe26cbd6c, ebp = 0 ---
db> show allpcpu
Current CPU: 0
cpuid = 0
curthread = 0xc3c2ea20: pid 13 "idle: cpu0"
curpcb = 0xe26cbd90
fpcurthread = none
idlethread = 0xc3c2ea20: pid 13 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc3c2e870: pid 12 "idle: cpu1"
curpcb = 0xe26c8d90
fpcurthread = none
idlethread = 0xc3c2e870: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc3c2e6c0: pid 11 "idle: cpu2"
curpcb = 0xe26c5d90
fpcurthread = none
idlethread = 0xc3c2e6c0: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc3c2e510: pid 10 "idle: cpu3"
curpcb = 0xe26c2d90
fpcurthread = none
idlethread = 0xc3c2e510: pid 10 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> show alllocks
db> ps
pid ppid pgrp uid state wmesg wchan cmd
89506 0 0 0 SL mdwait 0xc9c5c000 [md4]
89499 0 0 0 SL mdwait 0xc9a9e800 [md3]
89492 0 0 0 SL mdwait 0xca5d6800 [md2]
89485 0 0 0 SL mdwait 0xca0ab800 [md1]
88113 1 1200 1001 S+ zonelimi 0xc1859000 net
88112 1 1200 1001 S+ zonelimi 0xc1859000 net
88111 1 1200 1001 S+ zonelimi 0xc1859000 net
88106 1 1200 1001 S+ zonelimi 0xc1859000 net
88101 1 1200 1001 S+ zonelimi 0xc1859000 net
88096 1 1200 1001 S+ zonelimi 0xc1859000 net
88092 1 1200 1001 S+ zonelimi 0xc1859000 net
88044 1 1200 1001 S+ zonelimi 0xc1859000 net
88037 1 1200 1001 S+ zonelimi 0xc1859000 net
88027 1 1200 1001 S+ zonelimi 0xc1859000 net
88011 1 1200 1001 S+ zonelimi 0xc1859000 net
87582 1 1200 1001 S+ zonelimi 0xc1859000 net
87553 1 1200 1001 S+ zonelimi 0xc1859000 net
87548 1 1200 1001 S+ zonelimi 0xc1859000 net
76285 1 1200 1001 S+ zonelimi 0xc1859000 net
76145 1 1200 1001 S+ zonelimi 0xc1859000 net
76111 1 1200 1001 S+ zonelimi 0xc1859000 net
1193 1192 1193 1001 Ss+ ttyin 0xc3d70010 bash
1192 1190 1190 1001 S zonelimi 0xc1859000 sshd
1190 841 1190 0 SWs sbwait 0xc411b480 sshd
978 1 978 0 SWs+ ttyin 0xc3df3810 getty
977 1 977 0 SWs+ ttyin 0xc3dfb810 getty
976 1 976 0 SWs+ ttyin 0xc3dfa810 getty
975 1 975 0 SWs+ ttyin 0xc3dfa010 getty
974 1 974 0 SWs+ ttyin 0xc3df5c10 getty
973 1 973 0 SWs+ ttyin 0xc3dfbc10 getty
972 1 972 0 SWs+ ttyin 0xc3dfc010 getty
971 1 971 0 SWs+ ttyin 0xc3dfac10 getty
954 1 954 0 SWs select 0xc0a68d9c inetd
925 1 925 0 SWs select 0xc0a68d9c moused
858 1 858 0 Ss nanslp 0xc0a1c8e4 cron
852 1 852 25 Ss pause 0xc40f7b38 sendmail
848 1 848 0 Ss select 0xc0a68d9c sendmail
841 1 841 0 Ss select 0xc0a68d9c sshd
821 1 821 0 Ss select 0xc0a68d9c ntpd
761 757 757 0 SW - 0xc3f92600 nfsd
760 757 757 0 SW - 0xc3f91200 nfsd
759 757 757 0 SW - 0xc3f61400 nfsd
758 757 757 0 SW - 0xc3f92000 nfsd
757 1 757 0 SWs select 0xc0a68d9c nfsd
755 1 755 0 Ss select 0xc0a68d9c mountd
696 1 696 0 Ss select 0xc0a68d9c rpcbind
673 1 673 0 Ss select 0xc0a68d9c syslogd
563 1 563 0 SWs select 0xc0a68d9c devd
42 0 0 0 SL - 0xe43eccfc [schedcpu]
41 0 0 0 SL sdflush 0xc0a76220 [softdepflush]
40 0 0 0 SL vlruwt 0xc3f6dd38 [vnlru]
39 0 0 0 SL syncer 0xc0a1c6b0 [syncer]
38 0 0 0 SL psleep 0xc0a69214 [bufdaemon]
37 0 0 0 SL pgzero 0xc0a7eb50 [pagezero]
36 0 0 0 SL psleep 0xc0a76a80 [vmdaemon]
35 0 0 0 SL psleep 0xc0a76a40 [pagedaemon]
34 0 0 0 WL [irq7: ppc0]
33 0 0 0 SL - 0xc3d7603c [fdc0]
32 0 0 0 WL [swi0: sio]
31 0 0 0 WL [irq12: psm0]
30 0 0 0 WL [irq1: atkbd0]
29 0 0 0 WL [irq15: ata1]
28 0 0 0 WL [irq14: ata0]
27 0 0 0 WL [irq17: fxp0]
26 0 0 0 SL usbtsk 0xc0a0ce24 [usbtask]
25 0 0 0 SL usbevt 0xc3d67210 [usb0]
24 0 0 0 WL [irq16: uhci0]
23 0 0 0 SL - 0xc3d51480 [em0 taskq]
22 0 0 0 WL [irq9: acpi0]
9 0 0 0 SL - 0xc3d40980 [thread taskq]
21 0 0 0 WL [swi6: Giant taskq]
20 0 0 0 WL [swi6: task queue]
8 0 0 0 SL - 0xc3c7b280 [acpi_task_2]
7 0 0 0 SL - 0xc3c7b280 [acpi_task_1]
6 0 0 0 SL - 0xc3c7b280 [acpi_task_0]
19 0 0 0 WL [swi2: cambio]
5 0 0 0 SL - 0xc3c7b480 [kqueue taskq]
18 0 0 0 WL [swi5: +]
17 0 0 0 SL - 0xc0a0aa60 [yarrow]
4 0 0 0 SL - 0xc0a0d5dc [g_down]
3 0 0 0 SL - 0xc0a0d5d8 [g_up]
2 0 0 0 SL - 0xc0a0d5d0 [g_event]
16 0 0 0 WL [swi3: vm]
15 0 0 0 WL [swi4: clock sio]
14 0 0 0 WL [swi1: net]
13 0 0 0 RL CPU 0 [idle: cpu0]
12 0 0 0 RL CPU 1 [idle: cpu1]
11 0 0 0 RL CPU 2 [idle: cpu2]
10 0 0 0 RL CPU 3 [idle: cpu3]
1 0 1 0 SLs wait 0xc3c32000 [init]
0 0 0 0 WLs [swapper]
db> show lockchain 88113
thread 100067 (pid 88113, net) inhibited
db> show sleepchain 88113
thread 100067 (pid 88113, net) sleeping on 0xc1859000 "zonelimit"
db> show lock 0xc1859000
class: spin mutex
name:
type:
flags: {SPIN}
state: {OWNED}
db> where 88113
Tracing pid 88113 tid 100067 td 0xc3fe6870
sched_switch(c3fe6870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c1859000,c0a1c0d0,0,c09239bc,211,...) at sleepq_switch+0xcd
sleepq_wait(c1859000,0,c093b6b2,89b,6a9,...) at sleepq_wait+0x46
msleep(c1859000,c1859008,44,c093b9f6,0) at msleep+0x27d
uma_zone_slab(c185c960,2,c1859008,0,c093b6a9,89b) at uma_zone_slab+0x124
uma_zalloc_internal(c185c960,c6d3f500,2,c1859008,0,...) at uma_zalloc_internal+0x29
uma_zalloc_arg(c185c960,c6d3f500,2) at uma_zalloc_arg+0x367
mb_zinit_pack(c6d3f500,100,2) at mb_zinit_pack+0x18
uma_zalloc_internal(c185cb40,e660eb38,2,c186ce88,0,...) at uma_zalloc_internal+0x90
uma_zalloc_arg(c185cb40,e660eb38,2) at uma_zalloc_arg+0x367
sosend_copyin(e660ec64,e660eba8,0,e660eb84,0) at sosend_copyin+0x58
sosend_generic(c5d77914,0,e660ec64,0,0,...) at sosend_generic+0x34e
sosend(c5d77914,0,e660ec64,0,0,0,c3fe6870) at sosend+0x3c
soo_write(cb2f0e10,e660ec64,c4144800,0,c3fe6870) at soo_write+0x42
dofilewrite(c3fe6870,3,cb2f0e10,e660ec64,ffffffff,...) at dofilewrite+0x7b
kern_writev(c3fe6870,3,e660ec64,bfbfe750,400,...) at kern_writev+0x36
write(c3fe6870,e660ed04) at write+0x45
syscall(3b,3b,3b,2804f88c,bfbfec08,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x28150097, esp = 0xbfbfe73c, ebp = 0xbfbfeb88 ---
db> call doadump
Physical memory: 1007 MB
Dumping 237 MB: 222 206 190 174 158 142 126 110 94 78 62 46 30 14
Dump complete
= 0xf
db> reset
GDB: no debug ports present
(kgdb) l *uma_zone_slab+0x124
0xc07eee6c is in uma_zone_slab (../../../vm/uma_core.c:2013).
2008 keg->uk_flags |= UMA_ZFLAG_FULL;
2009
2010 if (flags & M_NOWAIT)
2011 break;
2012 else
2013 msleep(keg, &keg->uk_lock, PVM,
2014 "zonelimit", 0);
2015 continue;
2016 }
2017 keg->uk_recurse++;
(kgdb) info loc
No locals.
(kgdb) p *(struct uma_zone *)0xc185c960
$2 = {uz_name = 0xc091e6c6 "mbuf_cluster", uz_lock = 0xc1859008, uz_keg = 0xc1859000, uz_link = {le_next = 0x0, le_prev = 0xc185902c},
uz_full_bucket = {lh_first = 0x0}, uz_free_bucket = {lh_first = 0x0}, uz_ctor = 0xc0695740 <mb_ctor_clust>,
uz_dtor = 0xc0695818 <mb_dtor_clust>, uz_init = 0, uz_fini = 0, uz_allocs = 0x75af64, uz_frees = 0x754b64, uz_fails = 0x59,
uz_fills = 0x4, uz_count = 0x80, uz_cpu = {{uc_freebucket = 0x0, uc_allocbucket = 0x0, uc_allocs = 0x0, uc_frees = 0x0}}}
(kgdb) p (*(struct uma_zone *)0xc185c960)->uz_keg
$3 = 0xc1859000
(kgdb) p *(*(struct uma_zone *)0xc185c960)->uz_keg
$4 = {uk_link = {le_next = 0xc186ce80, le_prev = 0xc1859080}, uk_lock = {mtx_object = {lo_name = 0xc091e6c6 "mbuf_cluster",
lo_type = 0xc091bb6a "UMA zone", lo_flags = 0x1430000, lo_witness_data = {lod_list = {stqe_next = 0xc0a2bf78},
lod_witness = 0xc0a2bf78}}, mtx_lock = 0x4, mtx_recurse = 0x0}, uk_hash = {uh_slab_hash = 0xc9c7b000, uh_hashsize = 0x4000,
uh_hashmask = 0x3fff}, uk_zones = {lh_first = 0xc185c960}, uk_part_slab = {lh_first = 0x0}, uk_free_slab = {lh_first = 0x0},
uk_full_slab = {lh_first = 0xca46fd00}, uk_recurse = 0x0, uk_align = 0x3, uk_pages = 0x3200, uk_free = 0x0, uk_size = 0x800,
uk_rsize = 0x800, uk_maxpages = 0x3200, uk_init = 0xc07f0860 <trash_init>, uk_fini = 0xc07f0878 <trash_fini>,
uk_allocf = 0xc07edbe8 <page_alloc>, uk_freef = 0xc07edd54 <page_free>, uk_obj = 0x0, uk_kva = 0x0, uk_slabzone = 0xc186b1e0,
uk_pgoff = 0x0, uk_ppera = 0x1, uk_ipers = 0x2, uk_flags = 0x40000508}
(kgdb) p *(*(struct uma_zone *)0xc185c960)->uz_lock
$5 = {mtx_object = {lo_name = 0xc091e6c6 "mbuf_cluster", lo_type = 0xc091bb6a "UMA zone", lo_flags = 0x1430000, lo_witness_data = {
lod_list = {stqe_next = 0xc0a2bf78}, lod_witness = 0xc0a2bf78}}, mtx_lock = 0x4, mtx_recurse = 0x0}
(kgdb) p cnt
$6 = {v_swtch = 0x4499dbc2, v_trap = 0x0, v_syscall = 0x0, v_intr = 0x0, v_soft = 0xd53b0a8, v_vm_faults = 0x205064dc,
v_cow_faults = 0xf461110, v_cow_optim = 0x2f3f2, v_zfod = 0xd9db34f, v_ozfod = 0x8695c86, v_swapin = 0x33d94, v_swapout = 0x6b6fc,
v_swappgsin = 0xa3b0c, v_swappgsout = 0x4324a4, v_vnodein = 0x4b74, v_vnodeout = 0x0, v_vnodepgsin = 0xe78a, v_vnodepgsout = 0x0,
v_intrans = 0x5911b8, v_reactivated = 0x2dc4fc, v_pdwakeups = 0xc2f, v_pdpages = 0x1d950849, v_dfree = 0x331, v_pfree = 0x1b35e945,
v_tfree = 0x232e5a2e, v_page_size = 0x1000, v_page_count = 0x3dd26, v_free_reserved = 0x18b, v_free_target = 0x1b77, v_free_min = 0x67b,
v_free_count = 0x316fd, v_wire_count = 0xbcd1, v_active_count = 0x5f8, v_inactive_target = 0x2932, v_inactive_count = 0x2ed,
v_cache_count = 0x1, v_cache_min = 0x1b77, v_cache_max = 0x36ee, v_pageout_free_min = 0x22, v_interrupt_free_min = 0x2,
v_free_severe = 0x403, v_forks = 0x10d8801, v_vforks = 0x3db, v_rforks = 0x0, v_kthreads = 0xd9, v_forkpages = 0x3f365c3f,
v_vforkpages = 0xe9dd3, v_rforkpages = 0x0, v_kthreadpages = 0x0}