GENERIC HEAD from Jan 18 19:27 UTC, vmcore.146 Freeze after 0+02:08:03 of stress testing. Pingable + console freeze after login prompt. GDB: no debug ports present KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2005 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD 6.0-CURRENT #0: Tue Jan 18 23:16:34 CET 2005 pho@current.osted.lan:/usr/src/sys/i386/compile/PHO WARNING: WITNESS option enabled, expect reduced performance. ACPI APIC Table: <A M I OEMAPIC > Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0xf13 Stepping = 3 Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM> real memory = 267583488 (255 MB) avail memory = 252448768 (240 MB) : mount root from ufs:/dev/ad0s1a freebsd4_sigreturn: eflags = 0x0 ~KDB: enter: Line break on console [thread pid 14589 tid 100259 ] Stopped at kdb_enter+0x2b: nop db> show pcpu cpuid = 0 curthread = 0xc2139170: pid 14589 "swap" curpcb = 0xcf406da0 fpcurthread = none idlethread = 0xc151f5c0: pid 11 "idle: cpu0" APIC ID = 0 currentldt = 0x30 spin locks held: db> where Tracing pid 14589 tid 100259 td 0xc2139170 kdb_enter(c0845129) at kdb_enter+0x2b siointr1(c1695400,c0953900,0,c0844f39,56f) at siointr1+0xce siointr(c1695400) at siointr+0x21 intr_execute_handlers(c1510090,cf406d44,4,bfbfea58,c07a7b53) at intr_execute_handlers+0x9d lapic_handle_intr(34) at lapic_handle_intr+0x2e Xapic_isr1() at Xapic_isr1+0x33 --- interrupt, eip = 0x8048ef9, esp = 0xbfbfea40, ebp = 0xbfbfea58 --- db> show alllocks Process 14660 (sh) thread 0xc2817450 (101123) exclusive sleep mutex process lock r = 0 (0xc1c58458) locked @ kern/kern_fork.c:300 exclusive sx allproc r = 0 (0xc08eb8a0) locked @ kern/kern_fork.c:287 shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278 Process 14580 (thr1) thread 0xc24db450 (100443) exclusive sx user map r = 0 (0xc1ce4ad0) locked @ vm/vm_map.c:2994 Process 14573 (syscall) thread 0xc24e8450 (100493) exclusive sx user map r = 0 (0xc253b4f4) locked @ vm/vm_glue.c:171 Process 2173 (top) thread 0xc17595c0 (100074) exclusive sx sysctl lock r = 0 (0xc08ebe60) locked @ kern/kern_sysctl.c:1335 Process 2033 (cron) thread 0xc162a5c0 (100054) shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278 db> c ~KDB: enter: Line break on console [thread pid 14589 tid 100259 ] Stopped at kdb_enter+0x2b: nop db> where Tracing pid 14589 tid 100259 td 0xc2139170 kdb_enter(c0845129) at kdb_enter+0x2b siointr1(c1695400,c0953900,0,c0844f39,56f) at siointr1+0xce siointr(c1695400) at siointr+0x21 intr_execute_handlers(c1510090,cf406d44,4,bfbfea58,c07a7b53) at intr_execute_handlers+0x9d lapic_handle_intr(34) at lapic_handle_intr+0x2e Xapic_isr1() at Xapic_isr1+0x33 --- interrupt, eip = 0x8048ef6, esp = 0xbfbfea40, ebp = 0xbfbfea58 --- db> show alllocks Process 14660 (sh) thread 0xc2817450 (101123) exclusive sleep mutex process lock r = 0 (0xc1c58458) locked @ kern/kern_fork.c:300 exclusive sx allproc r = 0 (0xc08eb8a0) locked @ kern/kern_fork.c:287 shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278 Process 14580 (thr1) thread 0xc24db450 (100443) exclusive sx user map r = 0 (0xc1ce4ad0) locked @ vm/vm_map.c:2994 Process 14573 (syscall) thread 0xc24e8450 (100493) exclusive sx user map r = 0 (0xc253b4f4) locked @ vm/vm_glue.c:171 Process 2173 (top) thread 0xc17595c0 (100074) exclusive sx sysctl lock r = 0 (0xc08ebe60) locked @ kern/kern_sysctl.c:1335 Process 2033 (cron) thread 0xc162a5c0 (100054) shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278 db> where 14660 Tracing pid 14660 tid 101123 td 0xc2817450 sched_switch(c2817450,0,2) at sched_switch+0x17f mi_switch(2,0,c08eb7e0,0,c08287ad) at mi_switch+0x264 critical_exit(c08cc720) at critical_exit+0x86 intr_execute_handlers(c08cc720,cfd6ab9c,c2817450,c150073c,9) at intr_execute_handlers+0xba atpic_handle_intr(0) at atpic_handle_intr+0x92 Xatpic_intr0() at Xatpic_intr0+0x20 --- interrupt, eip = 0xc0622f96, esp = 0xcfd6abe0, ebp = 0xcfd6abe4 --- critical_exit(c09391c4,cfd6ac30,c0632f33,c08f2b40,0) at critical_exit+0xb2 _mtx_unlock_spin_flags(c08f2b40,0,c082a8d6,325,c08287ad) at _mtx_unlock_spin_flags+0x8d witness_checkorder(c150073c,9,c0826ec0,476) at witness_checkorder+0x29f _mtx_lock_flags(c150073c,0,c0826eb7,476) at _mtx_lock_flags+0x5b chgproccnt(c18eb120,1,6fd,c1c58458,0) at chgproccnt+0x23 fork1(c2817450,14,0,cfd6ace4,cfd6ad40) at fork1+0x1b3 fork(c2817450,cfd6ad14,0,3,246) at fork+0x18 syscall(2f,2f,2f,806824c,806a000) at syscall+0x213 Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (2, FreeBSD ELF32, fork), eip = 0x28127cf3, esp = 0xbfbfe96c, ebp = 0xbfbfe988 --- db> call doadump Dumping 255 MB 16 32 48 64 80 96 112 128 144 160 176 192 208 224 240 Dump complete 0 db> reset (kgdb) bt #0 doadump () at pcpu.h:159 #1 0xc0465b3a in db_fncall (dummy1=0x0, dummy2=0x0, dummy3=0x0, dummy4=0xcf406b4c "xk@Ï0ÑyÀdk@Ïhk@Ï\220\a") at ../../../ddb/db_command.c:531 #2 0xc0465948 in db_command (last_cmdp=0xc08d0644, cmd_table=0x0, aux_cmd_tablep=0xc084fb7c, aux_cmd_tablep_end=0xc084fb98) at ../../../ddb/db_command.c:349 #3 0xc0465a10 in db_command_loop () at ../../../ddb/db_command.c:455 #4 0xc0467595 in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228 #5 0xc0629de0 in kdb_trap (type=0x3, code=0x0, tf=0xcf406c90) at ../../../kern/subr_kdb.c:421 #6 0xc07b93a0 in trap (frame= {tf_fs = 0xcf400018, tf_es = 0xc0620010, tf_ds = 0xc0820010, tf_edi = 0xf9, tf_esi = 0xc1695400, tf_ebp = 0xcf406cd0, tf_isp = 0xcf406cbc, tf_ebx = 0xc1692600, tf_edx = 0x0, tf_ecx = 0xc1033000, tf_eax = 0x22, tf_trapno = 0x3, tf_err = 0x0, tf_eip = 0xc0629b47, tf_cs = 0x8, tf_eflags = 0x92, tf_esp = 0xcf406cec, tf_ss = 0xc0794512}) at ../../../i386/i386/trap.c:573 #7 0xc07a779a in calltrap () at ../../../i386/i386/exception.s:139 #8 0xcf400018 in ?? () #9 0xc0620010 in link_elf_link_preload (cls=0xc0845129, filename=---Can't read userspace from dump, or kernel process--- ) at ../../../kern/link_elf.c:462 #10 0xc0794512 in siointr1 (com=0x8) at ../../../dev/sio/sio.c:1523 #11 0xc0794309 in siointr (arg=0xc1695400) at ../../../dev/sio/sio.c:1392 #12 0xc07ab4cd in intr_execute_handlers (isrc=0xc1510090, iframe=0xcf406d44) at ../../../i386/i386/intr_machdep.c:203 #13 0xc07ad9aa in lapic_handle_intr (frame= {if_vec = 0x34, if_fs = 0x2f, if_es = 0x2f, if_ds = 0x2f, if_edi = 0x2804f6c0, if_esi = 0x0, if_ebp = 0xbfbfea58, if_ebx = 0xb, if_edx = 0x1000, if_ecx = 0xbfbfea28, if_eax = 0x829f000, if_eip = 0x8048ef6, if_cs = 0x1f, if_eflags = 0x206, if_esp = 0xbfbfea40, if_ss = 0x2f}) at ../../../i386/i386/local_apic.c:516 #14 0xc07a7b53 in Xapic_isr1 () at apic_vector.s:110 #15 0x00000034 in ?? () : #43 0xc15242e0 in ?? () #44 0xc06222ab in sched_switch (td=0x0, newtd=0xb, flags=---Can't read userspace from dump, or kernel process--- ) at ../../../kern/sched_4bsd.c:963 Previous frame inner to this frame (corrupt stack?) (kgdb) btp 14660 frame 0 at 0xcfd6aae4: ebp cfd6ab34, eip 0xc0617dc0 <mi_switch+612>: add $0xc,%esp frame 1 at 0xcfd6ab34: ebp cfd6ab58, eip 0xc0622f6a <critical_exit+134>: push $0x254 frame 2 at 0xcfd6ab58: ebp cfd6ab78, eip 0xc07ab4ea <intr_execute_handlers+186>: add $0x4,%esp frame 3 at 0xcfd6ab78: ebp cfd6ab94, eip 0xc07bbeee <atpic_handle_intr+146>: lea 0xfffffff4(%ebp),%esp frame 4 at 0xcfd6ab94: ebp cfd6abe4, eip 0xc07a7830 <Xatpic_intr0+32>: add $0x4,%esp frame 5 at 0xcfd6abe4: ebp cfd6abf0, eip 0xc060a631 <_mtx_unlock_spin_flags+141>: mov 0xfffffffc(%ebp),%ebx frame 6 at 0xcfd6abf0: ebp cfd6ac30, eip 0xc0632f33 <witness_checkorder+671>: jmp 0xc06331f8 <witness_checkorder+1380> frame 7 at 0xcfd6ac30: ebp cfd6ac54, eip 0xc060a3fb <_mtx_lock_flags+91>: add $0x10,%esp frame 8 at 0xcfd6ac54: ebp cfd6ac78, eip 0xc0611423 <chgproccnt+35>: add $0x10,%esp frame 9 at 0xcfd6ac78: ebp cfd6accc, eip 0xc05fe7ef <fork1+435>: mov %eax,%ebx frame 10 at 0xcfd6accc: ebp cfd6acec, eip 0xc05fe4c4 <fork+24>: mov %eax,%edx frame 11 at 0xcfd6acec: ebp cfd6ad40, eip 0xc07b9b57 <syscall+531>: mov %eax,%ebx (kgdb) l *chgproccnt+35 0xc0611423 is in chgproccnt (../../../kern/kern_resource.c:1144). 1139 int max; 1140 { 1141 1142 UIDINFO_LOCK(uip); 1143 /* Don't allow them to exceed max, but allow subtraction. */ 1144 if (diff > 0 && uip->ui_proccnt + diff > max && max != 0) { 1145 UIDINFO_UNLOCK(uip); 1146 return (0); 1147 } 1148 uip->ui_proccnt += diff; (kgdb) l *fork1+435 0xc05fe7ef is in fork1 (../../../kern/kern_fork.c:301). 296 /* 297 * Increment the count of procs running with this uid. Don't allow 298 * a nonprivileged user to exceed their current limit. 299 */ 300 PROC_LOCK(p1); 301 ok = chgproccnt(td->td_ucred->cr_ruidinfo, 1, 302 (uid != 0) ? lim_cur(p1, RLIMIT_NPROC) : 0); 303 PROC_UNLOCK(p1); 304 if (!ok) { 305 error = EAGAIN; $ ps -alx -o flags -N kernel.debug -M /var/crash/vmcore.146 | egrep "14660|14589" 1001 14589 14550 295 123 0 8032 0 - R+ p0 157:19,11 [swap] 2 1001 14660 14555 224 124 0 1664 0 - R+ p0 0:00,01 [sh] 4002