GENERIC HEAD from Jan 18 19:27 UTC, vmcore.146
Freeze after 0+02:08:03 of stress testing. Pingable + console freeze after login prompt.
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 6.0-CURRENT #0: Tue Jan 18 23:16:34 CET 2005
pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf13 Stepping = 3
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory = 267583488 (255 MB)
avail memory = 252448768 (240 MB)
:
mount root from ufs:/dev/ad0s1a
freebsd4_sigreturn: eflags = 0x0
~KDB: enter: Line break on console
[thread pid 14589 tid 100259 ]
Stopped at kdb_enter+0x2b: nop
db> show pcpu
cpuid = 0
curthread = 0xc2139170: pid 14589 "swap"
curpcb = 0xcf406da0
fpcurthread = none
idlethread = 0xc151f5c0: pid 11 "idle: cpu0"
APIC ID = 0
currentldt = 0x30
spin locks held:
db> where
Tracing pid 14589 tid 100259 td 0xc2139170
kdb_enter(c0845129) at kdb_enter+0x2b
siointr1(c1695400,c0953900,0,c0844f39,56f) at siointr1+0xce
siointr(c1695400) at siointr+0x21
intr_execute_handlers(c1510090,cf406d44,4,bfbfea58,c07a7b53) at intr_execute_handlers+0x9d
lapic_handle_intr(34) at lapic_handle_intr+0x2e
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0x8048ef9, esp = 0xbfbfea40, ebp = 0xbfbfea58 ---
db> show alllocks
Process 14660 (sh) thread 0xc2817450 (101123)
exclusive sleep mutex process lock r = 0 (0xc1c58458) locked @ kern/kern_fork.c:300
exclusive sx allproc r = 0 (0xc08eb8a0) locked @ kern/kern_fork.c:287
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
Process 14580 (thr1) thread 0xc24db450 (100443)
exclusive sx user map r = 0 (0xc1ce4ad0) locked @ vm/vm_map.c:2994
Process 14573 (syscall) thread 0xc24e8450 (100493)
exclusive sx user map r = 0 (0xc253b4f4) locked @ vm/vm_glue.c:171
Process 2173 (top) thread 0xc17595c0 (100074)
exclusive sx sysctl lock r = 0 (0xc08ebe60) locked @ kern/kern_sysctl.c:1335
Process 2033 (cron) thread 0xc162a5c0 (100054)
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
db> c
~KDB: enter: Line break on console
[thread pid 14589 tid 100259 ]
Stopped at kdb_enter+0x2b: nop
db> where
Tracing pid 14589 tid 100259 td 0xc2139170
kdb_enter(c0845129) at kdb_enter+0x2b
siointr1(c1695400,c0953900,0,c0844f39,56f) at siointr1+0xce
siointr(c1695400) at siointr+0x21
intr_execute_handlers(c1510090,cf406d44,4,bfbfea58,c07a7b53) at intr_execute_handlers+0x9d
lapic_handle_intr(34) at lapic_handle_intr+0x2e
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0x8048ef6, esp = 0xbfbfea40, ebp = 0xbfbfea58 ---
db> show alllocks
Process 14660 (sh) thread 0xc2817450 (101123)
exclusive sleep mutex process lock r = 0 (0xc1c58458) locked @ kern/kern_fork.c:300
exclusive sx allproc r = 0 (0xc08eb8a0) locked @ kern/kern_fork.c:287
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
Process 14580 (thr1) thread 0xc24db450 (100443)
exclusive sx user map r = 0 (0xc1ce4ad0) locked @ vm/vm_map.c:2994
Process 14573 (syscall) thread 0xc24e8450 (100493)
exclusive sx user map r = 0 (0xc253b4f4) locked @ vm/vm_glue.c:171
Process 2173 (top) thread 0xc17595c0 (100074)
exclusive sx sysctl lock r = 0 (0xc08ebe60) locked @ kern/kern_sysctl.c:1335
Process 2033 (cron) thread 0xc162a5c0 (100054)
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
db> where 14660
Tracing pid 14660 tid 101123 td 0xc2817450
sched_switch(c2817450,0,2) at sched_switch+0x17f
mi_switch(2,0,c08eb7e0,0,c08287ad) at mi_switch+0x264
critical_exit(c08cc720) at critical_exit+0x86
intr_execute_handlers(c08cc720,cfd6ab9c,c2817450,c150073c,9) at intr_execute_handlers+0xba
atpic_handle_intr(0) at atpic_handle_intr+0x92
Xatpic_intr0() at Xatpic_intr0+0x20
--- interrupt, eip = 0xc0622f96, esp = 0xcfd6abe0, ebp = 0xcfd6abe4 ---
critical_exit(c09391c4,cfd6ac30,c0632f33,c08f2b40,0) at critical_exit+0xb2
_mtx_unlock_spin_flags(c08f2b40,0,c082a8d6,325,c08287ad) at _mtx_unlock_spin_flags+0x8d
witness_checkorder(c150073c,9,c0826ec0,476) at witness_checkorder+0x29f
_mtx_lock_flags(c150073c,0,c0826eb7,476) at _mtx_lock_flags+0x5b
chgproccnt(c18eb120,1,6fd,c1c58458,0) at chgproccnt+0x23
fork1(c2817450,14,0,cfd6ace4,cfd6ad40) at fork1+0x1b3
fork(c2817450,cfd6ad14,0,3,246) at fork+0x18
syscall(2f,2f,2f,806824c,806a000) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (2, FreeBSD ELF32, fork), eip = 0x28127cf3, esp = 0xbfbfe96c, ebp = 0xbfbfe988 ---
db> call doadump
Dumping 255 MB
16 32 48 64 80 96 112 128 144 160 176 192 208 224 240
Dump complete
0
db> reset
(kgdb) bt
#0 doadump () at pcpu.h:159
#1 0xc0465b3a in db_fncall (dummy1=0x0, dummy2=0x0, dummy3=0x0, dummy4=0xcf406b4c "xk@Ï0ÑyÀdk@Ïhk@Ï\220\a")
at ../../../ddb/db_command.c:531
#2 0xc0465948 in db_command (last_cmdp=0xc08d0644, cmd_table=0x0, aux_cmd_tablep=0xc084fb7c, aux_cmd_tablep_end=0xc084fb98)
at ../../../ddb/db_command.c:349
#3 0xc0465a10 in db_command_loop () at ../../../ddb/db_command.c:455
#4 0xc0467595 in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5 0xc0629de0 in kdb_trap (type=0x3, code=0x0, tf=0xcf406c90) at ../../../kern/subr_kdb.c:421
#6 0xc07b93a0 in trap (frame=
{tf_fs = 0xcf400018, tf_es = 0xc0620010, tf_ds = 0xc0820010, tf_edi = 0xf9, tf_esi = 0xc1695400, tf_ebp = 0xcf406cd0, tf_isp = 0xcf406cbc, tf_ebx = 0xc1692600, tf_edx = 0x0, tf_ecx = 0xc1033000, tf_eax = 0x22, tf_trapno = 0x3, tf_err = 0x0, tf_eip = 0xc0629b47, tf_cs = 0x8, tf_eflags = 0x92, tf_esp = 0xcf406cec, tf_ss = 0xc0794512}) at ../../../i386/i386/trap.c:573
#7 0xc07a779a in calltrap () at ../../../i386/i386/exception.s:139
#8 0xcf400018 in ?? ()
#9 0xc0620010 in link_elf_link_preload (cls=0xc0845129, filename=---Can't read userspace from dump, or kernel process---
) at ../../../kern/link_elf.c:462
#10 0xc0794512 in siointr1 (com=0x8) at ../../../dev/sio/sio.c:1523
#11 0xc0794309 in siointr (arg=0xc1695400) at ../../../dev/sio/sio.c:1392
#12 0xc07ab4cd in intr_execute_handlers (isrc=0xc1510090, iframe=0xcf406d44) at ../../../i386/i386/intr_machdep.c:203
#13 0xc07ad9aa in lapic_handle_intr (frame=
{if_vec = 0x34, if_fs = 0x2f, if_es = 0x2f, if_ds = 0x2f, if_edi = 0x2804f6c0, if_esi = 0x0, if_ebp = 0xbfbfea58, if_ebx = 0xb, if_edx = 0x1000, if_ecx = 0xbfbfea28, if_eax = 0x829f000, if_eip = 0x8048ef6, if_cs = 0x1f, if_eflags = 0x206, if_esp = 0xbfbfea40, if_ss = 0x2f})
at ../../../i386/i386/local_apic.c:516
#14 0xc07a7b53 in Xapic_isr1 () at apic_vector.s:110
#15 0x00000034 in ?? ()
:
#43 0xc15242e0 in ?? ()
#44 0xc06222ab in sched_switch (td=0x0, newtd=0xb, flags=---Can't read userspace from dump, or kernel process---
) at ../../../kern/sched_4bsd.c:963
Previous frame inner to this frame (corrupt stack?)
(kgdb) btp 14660
frame 0 at 0xcfd6aae4: ebp cfd6ab34, eip 0xc0617dc0 <mi_switch+612>: add $0xc,%esp
frame 1 at 0xcfd6ab34: ebp cfd6ab58, eip 0xc0622f6a <critical_exit+134>: push $0x254
frame 2 at 0xcfd6ab58: ebp cfd6ab78, eip 0xc07ab4ea <intr_execute_handlers+186>: add $0x4,%esp
frame 3 at 0xcfd6ab78: ebp cfd6ab94, eip 0xc07bbeee <atpic_handle_intr+146>: lea 0xfffffff4(%ebp),%esp
frame 4 at 0xcfd6ab94: ebp cfd6abe4, eip 0xc07a7830 <Xatpic_intr0+32>: add $0x4,%esp
frame 5 at 0xcfd6abe4: ebp cfd6abf0, eip 0xc060a631 <_mtx_unlock_spin_flags+141>: mov 0xfffffffc(%ebp),%ebx
frame 6 at 0xcfd6abf0: ebp cfd6ac30, eip 0xc0632f33 <witness_checkorder+671>: jmp 0xc06331f8 <witness_checkorder+1380>
frame 7 at 0xcfd6ac30: ebp cfd6ac54, eip 0xc060a3fb <_mtx_lock_flags+91>: add $0x10,%esp
frame 8 at 0xcfd6ac54: ebp cfd6ac78, eip 0xc0611423 <chgproccnt+35>: add $0x10,%esp
frame 9 at 0xcfd6ac78: ebp cfd6accc, eip 0xc05fe7ef <fork1+435>: mov %eax,%ebx
frame 10 at 0xcfd6accc: ebp cfd6acec, eip 0xc05fe4c4 <fork+24>: mov %eax,%edx
frame 11 at 0xcfd6acec: ebp cfd6ad40, eip 0xc07b9b57 <syscall+531>: mov %eax,%ebx
(kgdb) l *chgproccnt+35
0xc0611423 is in chgproccnt (../../../kern/kern_resource.c:1144).
1139 int max;
1140 {
1141
1142 UIDINFO_LOCK(uip);
1143 /* Don't allow them to exceed max, but allow subtraction. */
1144 if (diff > 0 && uip->ui_proccnt + diff > max && max != 0) {
1145 UIDINFO_UNLOCK(uip);
1146 return (0);
1147 }
1148 uip->ui_proccnt += diff;
(kgdb) l *fork1+435
0xc05fe7ef is in fork1 (../../../kern/kern_fork.c:301).
296 /*
297 * Increment the count of procs running with this uid. Don't allow
298 * a nonprivileged user to exceed their current limit.
299 */
300 PROC_LOCK(p1);
301 ok = chgproccnt(td->td_ucred->cr_ruidinfo, 1,
302 (uid != 0) ? lim_cur(p1, RLIMIT_NPROC) : 0);
303 PROC_UNLOCK(p1);
304 if (!ok) {
305 error = EAGAIN;
$ ps -alx -o flags -N kernel.debug -M /var/crash/vmcore.146 | egrep "14660|14589"
1001 14589 14550 295 123 0 8032 0 - R+ p0 157:19,11 [swap] 2
1001 14660 14555 224 124 0 1664 0 - R+ p0 0:00,01 [sh] 4002