GENERIC HEAD from Jan 18 19:27 UTC, vmcore.146
Freeze after 0+02:08:03 of stress testing. Pingable + console freeze after login prompt.

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 6.0-CURRENT #0: Tue Jan 18 23:16:34 CET 2005
    pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I  OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf13  Stepping = 3
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory  = 267583488 (255 MB)
avail memory = 252448768 (240 MB)
:
mount root from ufs:/dev/ad0s1a
freebsd4_sigreturn: eflags = 0x0
~KDB: enter: Line break on console
[thread pid 14589 tid 100259 ]
Stopped at      kdb_enter+0x2b: nop
db> show pcpu
cpuid        = 0
curthread    = 0xc2139170: pid 14589 "swap"
curpcb       = 0xcf406da0
fpcurthread  = none
idlethread   = 0xc151f5c0: pid 11 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x30
spin locks held:
db> where
Tracing pid 14589 tid 100259 td 0xc2139170
kdb_enter(c0845129) at kdb_enter+0x2b
siointr1(c1695400,c0953900,0,c0844f39,56f) at siointr1+0xce
siointr(c1695400) at siointr+0x21
intr_execute_handlers(c1510090,cf406d44,4,bfbfea58,c07a7b53) at intr_execute_handlers+0x9d
lapic_handle_intr(34) at lapic_handle_intr+0x2e
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0x8048ef9, esp = 0xbfbfea40, ebp = 0xbfbfea58 ---
db> show alllocks
Process 14660 (sh) thread 0xc2817450 (101123)
exclusive sleep mutex process lock r = 0 (0xc1c58458) locked @ kern/kern_fork.c:300
exclusive sx allproc r = 0 (0xc08eb8a0) locked @ kern/kern_fork.c:287
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
Process 14580 (thr1) thread 0xc24db450 (100443)
exclusive sx user map r = 0 (0xc1ce4ad0) locked @ vm/vm_map.c:2994
Process 14573 (syscall) thread 0xc24e8450 (100493)
exclusive sx user map r = 0 (0xc253b4f4) locked @ vm/vm_glue.c:171
Process 2173 (top) thread 0xc17595c0 (100074)
exclusive sx sysctl lock r = 0 (0xc08ebe60) locked @ kern/kern_sysctl.c:1335
Process 2033 (cron) thread 0xc162a5c0 (100054)
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
db> c
~KDB: enter: Line break on console
[thread pid 14589 tid 100259 ]
Stopped at      kdb_enter+0x2b: nop
db> where
Tracing pid 14589 tid 100259 td 0xc2139170
kdb_enter(c0845129) at kdb_enter+0x2b
siointr1(c1695400,c0953900,0,c0844f39,56f) at siointr1+0xce
siointr(c1695400) at siointr+0x21
intr_execute_handlers(c1510090,cf406d44,4,bfbfea58,c07a7b53) at intr_execute_handlers+0x9d
lapic_handle_intr(34) at lapic_handle_intr+0x2e
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0x8048ef6, esp = 0xbfbfea40, ebp = 0xbfbfea58 ---
db> show alllocks
Process 14660 (sh) thread 0xc2817450 (101123)
exclusive sleep mutex process lock r = 0 (0xc1c58458) locked @ kern/kern_fork.c:300
exclusive sx allproc r = 0 (0xc08eb8a0) locked @ kern/kern_fork.c:287
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
Process 14580 (thr1) thread 0xc24db450 (100443)
exclusive sx user map r = 0 (0xc1ce4ad0) locked @ vm/vm_map.c:2994
Process 14573 (syscall) thread 0xc24e8450 (100493)
exclusive sx user map r = 0 (0xc253b4f4) locked @ vm/vm_glue.c:171
Process 2173 (top) thread 0xc17595c0 (100074)
exclusive sx sysctl lock r = 0 (0xc08ebe60) locked @ kern/kern_sysctl.c:1335
Process 2033 (cron) thread 0xc162a5c0 (100054)
shared sx proctree r = 0 (0xc08eb8e0) locked @ kern/kern_fork.c:278
db> where 14660
Tracing pid 14660 tid 101123 td 0xc2817450
sched_switch(c2817450,0,2) at sched_switch+0x17f
mi_switch(2,0,c08eb7e0,0,c08287ad) at mi_switch+0x264
critical_exit(c08cc720) at critical_exit+0x86
intr_execute_handlers(c08cc720,cfd6ab9c,c2817450,c150073c,9) at intr_execute_handlers+0xba
atpic_handle_intr(0) at atpic_handle_intr+0x92
Xatpic_intr0() at Xatpic_intr0+0x20
--- interrupt, eip = 0xc0622f96, esp = 0xcfd6abe0, ebp = 0xcfd6abe4 ---
critical_exit(c09391c4,cfd6ac30,c0632f33,c08f2b40,0) at critical_exit+0xb2
_mtx_unlock_spin_flags(c08f2b40,0,c082a8d6,325,c08287ad) at _mtx_unlock_spin_flags+0x8d
witness_checkorder(c150073c,9,c0826ec0,476) at witness_checkorder+0x29f
_mtx_lock_flags(c150073c,0,c0826eb7,476) at _mtx_lock_flags+0x5b
chgproccnt(c18eb120,1,6fd,c1c58458,0) at chgproccnt+0x23
fork1(c2817450,14,0,cfd6ace4,cfd6ad40) at fork1+0x1b3
fork(c2817450,cfd6ad14,0,3,246) at fork+0x18
syscall(2f,2f,2f,806824c,806a000) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (2, FreeBSD ELF32, fork), eip = 0x28127cf3, esp = 0xbfbfe96c, ebp = 0xbfbfe988 ---
db> call doadump
Dumping 255 MB
 16 32 48 64 80 96 112 128 144 160 176 192 208 224 240
Dump complete
0
db> reset

(kgdb) bt
#0  doadump () at pcpu.h:159
#1  0xc0465b3a in db_fncall (dummy1=0x0, dummy2=0x0, dummy3=0x0, dummy4=0xcf406b4c "xk@Ï0ÑyÀdk@Ïhk@Ï\220\a")
    at ../../../ddb/db_command.c:531
#2  0xc0465948 in db_command (last_cmdp=0xc08d0644, cmd_table=0x0, aux_cmd_tablep=0xc084fb7c, aux_cmd_tablep_end=0xc084fb98)
    at ../../../ddb/db_command.c:349
#3  0xc0465a10 in db_command_loop () at ../../../ddb/db_command.c:455
#4  0xc0467595 in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5  0xc0629de0 in kdb_trap (type=0x3, code=0x0, tf=0xcf406c90) at ../../../kern/subr_kdb.c:421
#6  0xc07b93a0 in trap (frame=
      {tf_fs = 0xcf400018, tf_es = 0xc0620010, tf_ds = 0xc0820010, tf_edi = 0xf9, tf_esi = 0xc1695400, tf_ebp = 0xcf406cd0, tf_isp = 0xcf406cbc, tf_ebx = 0xc1692600, tf_edx = 0x0, tf_ecx = 0xc1033000, tf_eax = 0x22, tf_trapno = 0x3, tf_err = 0x0, tf_eip = 0xc0629b47, tf_cs = 0x8, tf_eflags = 0x92, tf_esp = 0xcf406cec, tf_ss = 0xc0794512}) at ../../../i386/i386/trap.c:573
#7  0xc07a779a in calltrap () at ../../../i386/i386/exception.s:139
#8  0xcf400018 in ?? ()
#9  0xc0620010 in link_elf_link_preload (cls=0xc0845129, filename=---Can't read userspace from dump, or kernel process---

) at ../../../kern/link_elf.c:462
#10 0xc0794512 in siointr1 (com=0x8) at ../../../dev/sio/sio.c:1523
#11 0xc0794309 in siointr (arg=0xc1695400) at ../../../dev/sio/sio.c:1392
#12 0xc07ab4cd in intr_execute_handlers (isrc=0xc1510090, iframe=0xcf406d44) at ../../../i386/i386/intr_machdep.c:203
#13 0xc07ad9aa in lapic_handle_intr (frame=
      {if_vec = 0x34, if_fs = 0x2f, if_es = 0x2f, if_ds = 0x2f, if_edi = 0x2804f6c0, if_esi = 0x0, if_ebp = 0xbfbfea58, if_ebx = 0xb, if_edx = 0x1000, if_ecx = 0xbfbfea28, if_eax = 0x829f000, if_eip = 0x8048ef6, if_cs = 0x1f, if_eflags = 0x206, if_esp = 0xbfbfea40, if_ss = 0x2f})
    at ../../../i386/i386/local_apic.c:516
#14 0xc07a7b53 in Xapic_isr1 () at apic_vector.s:110
#15 0x00000034 in ?? ()
:
#43 0xc15242e0 in ?? ()
#44 0xc06222ab in sched_switch (td=0x0, newtd=0xb, flags=---Can't read userspace from dump, or kernel process---

) at ../../../kern/sched_4bsd.c:963
Previous frame inner to this frame (corrupt stack?)
(kgdb) btp 14660
 frame 0 at 0xcfd6aae4: ebp cfd6ab34, eip 0xc0617dc0 <mi_switch+612>:   add    $0xc,%esp
 frame 1 at 0xcfd6ab34: ebp cfd6ab58, eip 0xc0622f6a <critical_exit+134>:       push   $0x254
 frame 2 at 0xcfd6ab58: ebp cfd6ab78, eip 0xc07ab4ea <intr_execute_handlers+186>:       add    $0x4,%esp
 frame 3 at 0xcfd6ab78: ebp cfd6ab94, eip 0xc07bbeee <atpic_handle_intr+146>:   lea    0xfffffff4(%ebp),%esp
 frame 4 at 0xcfd6ab94: ebp cfd6abe4, eip 0xc07a7830 <Xatpic_intr0+32>: add    $0x4,%esp
 frame 5 at 0xcfd6abe4: ebp cfd6abf0, eip 0xc060a631 <_mtx_unlock_spin_flags+141>:      mov    0xfffffffc(%ebp),%ebx
 frame 6 at 0xcfd6abf0: ebp cfd6ac30, eip 0xc0632f33 <witness_checkorder+671>:  jmp    0xc06331f8 <witness_checkorder+1380>
 frame 7 at 0xcfd6ac30: ebp cfd6ac54, eip 0xc060a3fb <_mtx_lock_flags+91>:      add    $0x10,%esp
 frame 8 at 0xcfd6ac54: ebp cfd6ac78, eip 0xc0611423 <chgproccnt+35>:   add    $0x10,%esp
 frame 9 at 0xcfd6ac78: ebp cfd6accc, eip 0xc05fe7ef <fork1+435>:       mov    %eax,%ebx
 frame 10 at 0xcfd6accc: ebp cfd6acec, eip 0xc05fe4c4 <fork+24>:        mov    %eax,%edx
 frame 11 at 0xcfd6acec: ebp cfd6ad40, eip 0xc07b9b57 <syscall+531>:    mov    %eax,%ebx
(kgdb) l *chgproccnt+35
0xc0611423 is in chgproccnt (../../../kern/kern_resource.c:1144).
1139            int     max;
1140    {
1141
1142            UIDINFO_LOCK(uip);
1143            /* Don't allow them to exceed max, but allow subtraction. */
1144            if (diff > 0 && uip->ui_proccnt + diff > max && max != 0) {
1145                    UIDINFO_UNLOCK(uip);
1146                    return (0);
1147            }
1148            uip->ui_proccnt += diff;
(kgdb) l *fork1+435
0xc05fe7ef is in fork1 (../../../kern/kern_fork.c:301).
296             /*
297              * Increment the count of procs running with this uid. Don't allow
298              * a nonprivileged user to exceed their current limit.
299              */
300             PROC_LOCK(p1);
301             ok = chgproccnt(td->td_ucred->cr_ruidinfo, 1,
302                     (uid != 0) ? lim_cur(p1, RLIMIT_NPROC) : 0);
303             PROC_UNLOCK(p1);
304             if (!ok) {
305                     error = EAGAIN;

$ ps -alx -o flags -N kernel.debug -M /var/crash/vmcore.146 | egrep "14660|14589"
 1001 14589 14550 295 123  0  8032    0 -      R+    p0  157:19,11 [swap]   2
 1001 14660 14555 224 124  0  1664    0 -      R+    p0    0:00,01 [sh]  4002