GENERIC HEAD from Feb 12 09:40 UTC + mpsafe_vfs = 1 , vmcore.177
Livelock.
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 6.0-CURRENT #0: Sat Feb 12 14:42:33 CET 2005
pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf13 Stepping = 3
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory = 267583488 (255 MB)
avail memory = 252358656 (240 MB)
:
mount root from ufs:/dev/ad0s1a
rl0: link state changed to DOWN
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
sigreturn: eflags = 0x0
freebsd4_sigreturn: eflags = 0x0
freebsd4_sigreturn: eflags = 0x0
freebsd4_sigreturn: eflags = 0x0
kernel trap 9 with interrupts disabled
kernel trap 9 with interrupts disabled
:
kernel trap 9 with interrupts disabled
kern.maxfiles limit exceeded by uid 1001, please see tuning(7).
rl0: watchdog timeout
kernel trap 9 with interrupts disabled
kernel trap 9 with interrupts disabled
:
kernel trap 9 with interrupts disabled
kernel trap 9 with interrupts disabled
rl0: watchdog timeout
ad0: WARNING - READ_DMA interrupt was seen but timeout fired LBA=630431
ad0: WARNING - READ_DMA interrupt was seen but timeout fired LBA=694847
~KDB: enter: Line break on console
[thread pid 515 tid 100123 ]
Stopped at kdb_enter+0x2b: nop
db> x panic_pid
panic_pid: 0
db> show pcpu
cpuid = 0
curthread = 0xc1e3c450: pid 515 "swap"
curpcb = 0xcf118da0
fpcurthread = none
idlethread = 0xc15205c0: pid 11 "idle: cpu0"
APIC ID = 0
currentldt = 0x30
spin locks held:
db> show alllocks
Process 538 (sh) thread 0xc1ff35c0 (100354)
exclusive sx allproc r = 0 (0xc08f6a40) locked @ kern/kern_fork.c:287
shared sx proctree r = 0 (0xc08f6a80) locked @ kern/kern_fork.c:278
Process 514 (swap) thread 0xc27ba000 (100780)
shared sx proctree r = 0 (0xc08f6a80) locked @ kern/kern_fork.c:278
Process 92047 (top) thread 0xc271f5c0 (100614)
exclusive sx sysctl lock r = 0 (0xc08f7000) locked @ kern/kern_sysctl.c:1335
Process 451 (cron) thread 0xc162ba10 (100067)
shared sx proctree r = 0 (0xc08f6a80) locked @ kern/kern_fork.c:278
db> c
~KDB: enter: Line break on console
[thread pid 515 tid 100123 ]
Stopped at kdb_enter+0x2b: nop
db> show alllocks
Process 538 (sh) thread 0xc1ff35c0 (100354)
exclusive sx allproc r = 0 (0xc08f6a40) locked @ kern/kern_fork.c:287
shared sx proctree r = 0 (0xc08f6a80) locked @ kern/kern_fork.c:278
Process 514 (swap) thread 0xc27ba000 (100780)
shared sx proctree r = 0 (0xc08f6a80) locked @ kern/kern_fork.c:278
Process 92047 (top) thread 0xc271f5c0 (100614)
exclusive sx sysctl lock r = 0 (0xc08f7000) locked @ kern/kern_sysctl.c:1335
Process 451 (cron) thread 0xc162ba10 (100067)
shared sx proctree r = 0 (0xc08f6a80) locked @ kern/kern_fork.c:278
db> where
Tracing pid 515 tid 100123 td 0xc1e3c450
kdb_enter(c0843c22) at kdb_enter+0x2b
siointr1(c1696400,c095eb20,0,c0843a32,56f) at siointr1+0xce
siointr(c1696400) at siointr+0x21
intr_execute_handlers(c1510090,cf118d44,4,bfbfe968,c07ac303) at intr_execute_handlers+0x9d
lapic_handle_intr(34) at lapic_handle_intr+0x2e
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0x8048f20, esp = 0xbfbfe950, ebp = 0xbfbfe968 ---
db> where 538
Tracing pid 538 tid 100354 td 0xc1ff35c0
sched_switch(c1ff35c0,0,1) at sched_switch+0x17f
mi_switch(1,0) at mi_switch+0x264
turnstile_wait(c21b565c,c1e3c450) at turnstile_wait+0x32b
_mtx_lock_sleep(c21b565c,c1ff35c0,0,c08228e5,162) at _mtx_lock_sleep+0x1ce
_mtx_lock_flags(c21b565c,0,c08228e5,162) at _mtx_lock_flags+0x8a
fork1(c1ff35c0,14,0,cf486ce4,cf486d40) at fork1+0x295
fork(c1ff35c0,cf486d14,0,1,246) at fork+0x18
syscall(2f,2f,2f,806824c,806a000) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (2, FreeBSD ELF32, fork), eip = 0x281279b3, esp = 0xbfbfe8bc, ebp = 0xbfbfe8d8 ---
db> where 514
Tracing pid 514 tid 100780 td 0xc27ba000
sched_switch(c27ba000,0,1) at sched_switch+0x17f
mi_switch(1,0) at mi_switch+0x264
sleepq_switch(c08f6a70,cf9b5c58,c05ee22d,c08f6a70,c0825a54) at sleepq_switch+0xe0
sleepq_wait(c08f6a70,c0825a54,c27ba070,c08f6a40,c27ba000) at sleepq_wait+0x30
cv_wait(c08f6a70,c08f5c38,c27ba000,c27ba000,c1e3a9ec) at cv_wait+0x179
_sx_xlock(c08f6a40,c08228e5,11f,c08f6a80,c08228e5) at _sx_xlock+0x68
fork1(c27ba000,80000034,0,cf9b5ce4,c1e3aa54) at fork1+0x133
vfork(c27ba000,cf9b5d14,0,0,246) at vfork+0x1b
syscall(2f,2f,2f,2804f6dc,bfbfeaa0) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (66, FreeBSD ELF32, vfork), eip = 0x2809a41c, esp = 0xbfbfe8d0, ebp = 0xbfbfe908 ---
db> where 92047
Tracing pid 92047 tid 100614 td 0xc271f5c0
sched_switch(c271f5c0,0,1) at sched_switch+0x17f
mi_switch(1,0) at mi_switch+0x264
sleepq_switch(c08f6a64,cf7f1b78,c05ee22d,c08f6a64,c0825ea2) at sleepq_switch+0xe0
sleepq_wait(c08f6a64,c0825ea2,4f9,c08f6a40,0) at sleepq_wait+0x30
cv_wait(c08f6a64,c08f5c38,0,0,0) at cv_wait+0x179
_sx_slock(c08f6a40,c0824c2a,3bf,c0824703,0) at _sx_slock+0x68
sysctl_kern_proc(c0886b20,0,0,cf7f1c04,c0886b20) at sysctl_kern_proc+0x14c
sysctl_root(0,cf7f1c84,3,cf7f1c04,c271f5c0) at sysctl_root+0x11b
userland_sysctl(c271f5c0,cf7f1c84,3,0,bfbfe98c) at userland_sysctl+0xf4
__sysctl(c271f5c0,cf7f1d14,6,1f2c,292) at __sysctl+0x77
syscall(2f,bfbf002f,bfbf002f,3,bfbfe98c) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (202, FreeBSD ELF32, __sysctl), eip = 0x2811db93, esp = 0xbfbfe8ec, ebp = 0xbfbfe928 ---
db> where 451
Tracing pid 451 tid 100067 td 0xc162ba10
sched_switch(c162ba10,0,1) at sched_switch+0x17f
mi_switch(1,0) at mi_switch+0x264
sleepq_switch(c08f6a70,cc70ec58,c05ee22d,c08f6a70,c0825a54) at sleepq_switch+0xe0
sleepq_wait(c08f6a70,c0825a54,c162ba80,c08f6a40,c162ba10) at sleepq_wait+0x30
cv_wait(c08f6a70,c08f5c38,c162ba10,c162ba10,c1849be8) at cv_wait+0x179
_sx_xlock(c08f6a40,c08228e5,11f,c08f6a80,c08228e5) at _sx_xlock+0x68
fork1(c162ba10,14,0,cc70ece4,cc70ed40) at fork1+0x133
fork(c162ba10,cc70ed14,0,ff,246) at fork+0x18
syscall(805002f,2f,bfbf002f,0,8051040) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (2, FreeBSD ELF32, fork), eip = 0x280c89b3, esp = 0xbfbfed8c, ebp = 0xbfbfed98 ---
db> call doadump
Dumping 255 MB
16 32 48 64 80 96 112 128 144 160 176 192 208 224 240
Dump complete
0xf
db> reset
UID PID PPID CPU PRI NI VSZ RSS MWCHAN STAT TT TIME COMMAND
0 0 0 0 -16 0 0 0 allpro DLs ?? 0:01,44 [swapper]
0 61 0 0 -32 0 0 0 allpro DL ?? 0:25,74 [schedcpu]
0 435 1 0 96 0 3440 376 sysctl Ds ?? 0:09,63 [sendmail]
25 439 1 0 96 0 3320 0 sysctl Ds ?? 0:00,46 [sendmail]
0 451 1 0 96 0 1356 0 allpro Ds ?? 0:02,30 [cron]
1001 500 92071 200 121 0 1228 0 allpro D+ p0 0:00,02 [swap]
1001 502 92071 201 121 0 1228 0 allpro D+ p0 0:00,02 [mkdir]
1001 506 503 221 123 0 1228 0 sysctl D+ p0 0:00,00 [swap]
1001 509 503 222 123 0 1228 0 sysctl D+ p0 0:00,00 [swap]
1001 510 503 222 123 0 1228 0 sysctl D+ p0 0:00,00 [swap]
1001 511 503 223 123 0 1228 0 sysctl D+ p0 0:00,00 [swap]
1001 514 503 223 123 0 1232 0 allpro D+ p0 0:00,00 [swap]
1001 92047 560 2 96 0 2524 584 allpro D+ p1 1:08,60 [top]
1001 489 34285 0 96 0 0 0 proctr DE+ p2 0:00,01 [sleep]
(kgdb) btp 489
frame 0 at 0xcfbd4bbc: ebp cfbd4c0c, eip 0xc0617ad0 <mi_switch+612>: add $0xc,%esp
frame 1 at 0xcfbd4c0c: ebp cfbd4c24, eip 0xc062f238 <sleepq_switch+224>: add $0x8,%esp
frame 2 at 0xcfbd4c24: ebp cfbd4c30, eip 0xc062f418 <sleepq_wait+48>: push $0x220
frame 3 at 0xcfbd4c30: ebp cfbd4c58, eip 0xc05ee22d <cv_wait+377>: mov (%ebx),%eax
frame 4 at 0xcfbd4c58: ebp cfbd4c74, eip 0xc0616ec4 <_sx_xlock+104>: decl 0x38(%ebx)
frame 5 at 0xcfbd4c74: ebp cfbd4cdc, eip 0xc05fc6fa <exit1+1302>: mov 0x198(%edi),%eax
frame 6 at 0xcfbd4cdc: ebp cfbd4cec, eip 0xc05fc1e4 <exit1>: push %ebp
frame 7 at 0xcfbd4cec: ebp cfbd4d40, eip 0xc07be713 <syscall+531>: mov %eax,%ebx
(kgdb) l *exit1+1302
0xc05fc6fa is in exit1 (../../../kern/kern_exit.c:291).
290 sx_xlock(&proctree_lock);
(kgdb) defproc 489
There is no member named p_uarea.
(kgdb) p $proc
$1 = (struct proc *) 0xc21b6be8
(kgdb) p *$proc
$2 = {p_list = {le_next = 0xc28e99ec, le_prev = 0xc28ed5f4}, p_ksegrps = {tqh_first = 0xc1fb0b40, tqh_last = 0xc1fb0b44}, p_threads = {
tqh_first = 0xc1b59b80, tqh_last = 0xc1b59b88}, p_suspended = {tqh_first = 0x0, tqh_last = 0xc21b6c00}, p_ucred = 0xc25dca00,
p_fd = 0x0, p_fdtol = 0x0, p_stats = 0xc1ba9b00, p_limit = 0xc1cc4200, p_sigacts = 0xc1ce0000, p_flag = 0x6002, p_sflag = 0x1,
p_state = PRS_NORMAL, p_pid = 0x1e9, p_hash = {le_next = 0x0, le_prev = 0xc15113a4}, p_pglist = {le_next = 0x0, le_prev = 0xc28a9050},
p_pptr = 0xc28a9000, p_sibling = {le_next = 0x0, le_prev = 0xc28a9064}, p_children = {lh_first = 0x0}, p_mtx = {mtx_object = {
lo_class = 0xc08867e8, lo_name = 0xc0824ac4 "process lock", lo_type = 0xc0824ac4 "process lock", lo_flags = 0x430000, lo_list = {
tqe_next = 0xc21b6e4c, tqe_prev = 0xc21b6a64}, lo_witness = 0xc0907c20}, mtx_lock = 0x4, mtx_recurse = 0x0}, p_oppid = 0x0,
p_vmspace = 0xc1e545dc, p_swtime = 0x21, p_realtimer = {it_interval = {tv_sec = 0x0, tv_usec = 0x0}, it_value = {tv_sec = 0x0,
tv_usec = 0x0}}, p_rux = {rux_runtime = {sec = 0x0, frac = 0x2ec0a8434395e40}, rux_uticks = 0x0, rux_sticks = 0x1, rux_iticks = 0x0,
rux_uu = 0x0, rux_su = 0x27a1, rux_iu = 0x0}, p_crux = {rux_runtime = {sec = 0x0, frac = 0x0}, rux_uticks = 0x0, rux_sticks = 0x0,
rux_iticks = 0x0, rux_uu = 0x0, rux_su = 0x0, rux_iu = 0x0}, p_profthreads = 0x0, p_maxthrwaits = 0x0, p_traceflag = 0x0,
p_tracevp = 0x0, p_tracecred = 0x0, p_textvp = 0xc1a13138, p_siglist = {__bits = {0x0, 0x0, 0x0, 0x0}}, p_lock = 0x0, p_sigiolst = {
slh_first = 0x0}, p_sigparent = 0x14, p_sig = 0x0, p_code = 0x0, p_stops = 0x0, p_stype = 0x0, p_step = 0x0, p_pfsflags = 0x0,
p_nlminfo = 0x0, p_aioinfo = 0x0, p_singlethread = 0x0, p_suspcount = 0x0, p_xthread = 0x0, p_boundary_count = 0x0,
p_magic = 0xbeefface, p_comm = "sleep\000r", '\0' <repeats 12 times>, p_pgrp = 0xc213d9c0, p_sysent = 0xc08c7ca0, p_args = 0xc24bb840,
p_cpulimit = 0x7fffffffffffffff, p_nice = 0x0, p_xstat = 0x0, p_klist = {kl_lock = 0xc21b6c50, kl_list = {slh_first = 0x0}},
p_numthreads = 0x1, p_numksegrps = 0x1, p_md = {md_ldt = 0xc24bb360}, p_itcallout = {c_links = {sle = {sle_next = 0x0}, tqe = {
tqe_next = 0x0, tqe_prev = 0x0}}, c_time = 0x0, c_arg = 0x0, c_func = 0, c_mtx = 0x0, c_flags = 0x10}, p_acflag = 0x0,
p_ru = 0xc2be8600, p_peers = 0x0, p_leader = 0xc21b6be8, p_emuldata = 0x0, p_label = 0x0, p_sched = 0xc21b6de4}
(kgdb) p *(struct thread *)0xc1b59b80
$3 = {td_proc = 0xc21b6be8, td_ksegrp = 0xc1fb0b40, td_plist = {tqe_next = 0x0, tqe_prev = 0xc21b6bf8}, td_kglist = {tqe_next = 0x0,
tqe_prev = 0xc1fb0b4c}, td_slpq = {tqe_next = 0x0, tqe_prev = 0xc1ffcc40}, td_lockq = {tqe_next = 0xc1eb38a0, tqe_prev = 0xcf6a9b58},
td_runq = {tqe_next = 0x0, tqe_prev = 0xc1fb0b54}, td_selq = {tqh_first = 0x0, tqh_last = 0x0}, td_sleepqueue = 0x0,
td_turnstile = 0xc1b73380, td_tid = 0x18a3c, td_flags = 0x1000000, td_inhibitors = 0x2, td_pflags = 0x0, td_dupfd = 0x0,
td_wchan = 0xc08f6ab0, td_wmesg = 0xc0824b79 "proctree", td_lastcpu = 0x0, td_oncpu = 0xff, td_locks = 0x0, td_blocked = 0x0,
td_ithd = 0x0, td_lockname = 0x0, td_contested = {lh_first = 0x0}, td_sleeplocks = 0x0, td_intr_nesting_level = 0x0, td_pinned = 0x0,
td_mailbox = 0x0, td_ucred = 0xc25dca00, td_standin = 0x0, td_upcall = 0x0, td_sticks = 0x1, td_uuticks = 0x0, td_usticks = 0x0,
td_intrval = 0x0, td_oldsigmask = {__bits = {0x0, 0x0, 0x0, 0x0}}, td_sigmask = {__bits = {0x0, 0x0, 0x0, 0x0}}, td_siglist = {__bits = {
0x0, 0x0, 0x0, 0x0}}, td_waitset = 0x0, td_umtxq = 0x0, td_generation = 0x8, td_sigstk = {ss_sp = 0x0, ss_size = 0x0,
ss_flags = 0x4}, td_kflags = 0x0, td_xsig = 0x0, td_profil_addr = 0x0, td_profil_ticks = 0x0, td_base_pri = 0xb4, td_priority = 0xb4,
td_pcb = 0xcfbd4da0, td_state = TDS_INHIBITED, td_retval = {0x0, 0x28148504}, td_slpcallout = {c_links = {sle = {sle_next = 0x0}, tqe = {
tqe_next = 0x0, tqe_prev = 0xc65bcec8}}, c_time = 0x55e6c17, c_arg = 0xc1b59b80, c_func = 0xc062f870 <sleepq_timeout>,
c_mtx = 0x0, c_flags = 0x12}, td_frame = 0xcfbd4d48, td_kstack_obj = 0xc223ec60, td_kstack = 0xcfbd3000, td_kstack_pages = 0x2,
td_altkstack_obj = 0x0, td_altkstack = 0x0, td_altkstack_pages = 0x0, td_critnest = 0x1, td_md = {md_savecrit = 0x246},
td_sched = 0xc1b59cd0}
(kgdb) p proctree_lock
$1 = {sx_object = {lo_class = 0xc0887bcc, lo_name = 0xc0824b79 "proctree", lo_type = 0xc0824b79 "proctree", lo_flags = 0x3b0000,
lo_list = {tqe_next = 0xc08f6ac0, tqe_prev = 0xc08f6a50}, lo_witness = 0xc0907d38}, sx_lock = 0xc08f6634, sx_cnt = 0x3, sx_shrd_cv = {
cv_description = 0xc0824b79 "proctree", cv_waiters = 0x0}, sx_shrd_wcnt = 0x0, sx_excl_cv = {cv_description = 0xc0824b79 "proctree",
cv_waiters = 0x1}, sx_excl_wcnt = 0x1, sx_xholder = 0x0}