Index: src/sys/amd64/amd64/mp_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/amd64/mp_machdep.c,v
retrieving revision 1.252
diff -u -r1.252 mp_machdep.c
--- src/sys/amd64/amd64/mp_machdep.c	6 Apr 2005 01:05:36 -0000	1.252
+++ src/sys/amd64/amd64/mp_machdep.c	30 Apr 2005 04:43:13 -0000
@@ -192,6 +192,10 @@
 }
 
 
+#ifdef KDB_STOP_NMI
+volatile cpumask_t ipi_nmi_pending;
+#endif 
+
 /*
  * Calculate usable address in base memory for AP trampoline code.
  */
@@ -972,6 +976,76 @@
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
 }
 
+#ifdef KDB_STOP_NMI
+/*
+ * send NMI IPI to selected CPUs
+ */
+
+#define	BEFORE_SPIN	1000000
+
+void
+ipi_nmi_selected(u_int32_t cpus)
+{
+
+	int cpu;
+	register_t icrlo;
+
+	icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT 
+		| APIC_TRIGMOD_EDGE; 
+	
+	CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus);
+
+
+	atomic_set_int(&ipi_nmi_pending, cpus);
+
+
+	while ((cpu = ffs(cpus)) != 0) {
+		cpu--;
+		cpus &= ~(1 << cpu);
+
+		KASSERT(cpu_apic_ids[cpu] != -1,
+		    ("IPI NMI to non-existent CPU %d", cpu));
+		
+		/* Wait for an earlier IPI to finish. */
+		if (!lapic_ipi_wait(BEFORE_SPIN))
+			panic("ipi_nmi_selected: previous IPI has not cleared");
+
+		lapic_ipi_raw(icrlo,cpu_apic_ids[cpu]);
+	}
+}
+
+
+int
+ipi_nmi_handler()
+{
+	int cpu  = PCPU_GET(cpuid);
+
+	if(!(atomic_load_acq_int(&ipi_nmi_pending) & (1 << cpu)))
+		return 1;
+
+	atomic_clear_int(&ipi_nmi_pending,1 << cpu);
+
+	savectx(&stoppcbs[cpu]);
+
+	/* Indicate that we are stopped */
+	atomic_set_int(&stopped_cpus,1 << cpu);
+
+
+	/* Wait for restart */
+	while(!(atomic_load_acq_int(&started_cpus) & (1 << cpu)))
+	    ia32_pause();
+
+	atomic_clear_int(&started_cpus,1 << cpu);
+	atomic_clear_int(&stopped_cpus,1 << cpu);
+
+	if(cpu == 0 && cpustop_restartfunc != NULL)
+		cpustop_restartfunc();
+
+	return 0;
+}
+     
+#endif /* KDB_STOP_NMI */
+
 /*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
Index: src/sys/amd64/amd64/trap.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/amd64/trap.c,v
retrieving revision 1.283
diff -u -r1.283 trap.c
--- src/sys/amd64/amd64/trap.c	20 Apr 2005 20:52:45 -0000	1.283
+++ src/sys/amd64/amd64/trap.c	30 Apr 2005 04:43:13 -0000
@@ -165,6 +165,14 @@
 	PCPU_LAZY_INC(cnt.v_trap);
 	type = frame.tf_trapno;
 
+#ifdef KDB_STOP_NMI
+	/* Handler for NMI IPIs used for debugging */
+	if (type == T_NMI) {
+	         if (ipi_nmi_handler() == 0)
+	                   goto out;
+	}
+#endif /* KDB_STOP_NMI */
+
 #ifdef KDB
 	if (kdb_active) {
 		kdb_reenter();
Index: src/sys/amd64/conf/NOTES
===================================================================
RCS file: /home/ncvs/src/sys/amd64/conf/NOTES,v
retrieving revision 1.33
diff -u -r1.33 NOTES
--- src/sys/amd64/conf/NOTES	25 Apr 2005 19:58:20 -0000	1.33
+++ src/sys/amd64/conf/NOTES	30 Apr 2005 04:43:13 -0000
@@ -33,6 +33,12 @@
 #
 options 	MP_WATCHDOG
 
+# 
+# Debugging options.
+#
+options		KDB_STOP_NMI		# Stop CPUS using NMI instead of IPI
+
+
 
 #####################################################################
 # CPU OPTIONS
Index: src/sys/amd64/include/smp.h
===================================================================
RCS file: /home/ncvs/src/sys/amd64/include/smp.h,v
retrieving revision 1.83
diff -u -r1.83 smp.h
--- src/sys/amd64/include/smp.h	28 Feb 2005 23:37:35 -0000	1.83
+++ src/sys/amd64/include/smp.h	30 Apr 2005 04:43:13 -0000
@@ -63,6 +63,11 @@
 void	smp_invltlb(void);
 void	smp_masked_invltlb(u_int mask);
 
+#ifdef KDB_STOP_NMI
+int ipi_nmi_handler(void);
+void ipi_nmi_selected(u_int32_t cpus);
+#endif
+
 #endif /* !LOCORE */
 #endif /* SMP */
 
Index: src/sys/conf/options.amd64
===================================================================
RCS file: /home/ncvs/src/sys/conf/options.amd64,v
retrieving revision 1.19
diff -u -r1.19 options.amd64
--- src/sys/conf/options.amd64	15 Apr 2005 18:48:27 -0000	1.19
+++ src/sys/conf/options.amd64	30 Apr 2005 04:43:13 -0000
@@ -55,3 +55,6 @@
 PSM_RESETAFTERSUSPEND	opt_psm.h
 PSM_DEBUG		opt_psm.h
 DEV_ATPIC		opt_atpic.h
+
+# Debugging
+KDB_STOP_NMI		opt_global.h
Index: src/sys/conf/options.i386
===================================================================
RCS file: /home/ncvs/src/sys/conf/options.i386,v
retrieving revision 1.219
diff -u -r1.219 options.i386
--- src/sys/conf/options.i386	14 Apr 2005 17:59:57 -0000	1.219
+++ src/sys/conf/options.i386	30 Apr 2005 04:43:13 -0000
@@ -160,3 +160,6 @@
 DEV_APIC		opt_apic.h
 DEV_NPX			opt_npx.h
 ASR_COMPAT		opt_asr.h
+
+# Debugging
+KDB_STOP_NMI		opt_global.h
Index: src/sys/i386/conf/NOTES
===================================================================
RCS file: /home/ncvs/src/sys/i386/conf/NOTES,v
retrieving revision 1.1197
diff -u -r1.1197 NOTES
--- src/sys/i386/conf/NOTES	25 Apr 2005 06:24:19 -0000	1.1197
+++ src/sys/i386/conf/NOTES	30 Apr 2005 04:43:13 -0000
@@ -51,6 +51,11 @@
 #
 options 	MP_WATCHDOG
 
+# Debugging options.
+#
+options		KDB_STOP_NMI		# Stop CPUS using NMI instead of IPI
+
+
 
 #####################################################################
 # CPU OPTIONS
Index: src/sys/i386/i386/mp_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/mp_machdep.c,v
retrieving revision 1.246
diff -u -r1.246 mp_machdep.c
--- src/sys/i386/i386/mp_machdep.c	13 Apr 2005 22:57:17 -0000	1.246
+++ src/sys/i386/i386/mp_machdep.c	30 Apr 2005 04:43:13 -0000
@@ -175,6 +175,10 @@
 vm_offset_t smp_tlb_addr2;
 volatile int smp_tlb_wait;
 
+#ifdef KDB_STOP_NMI
+volatile cpumask_t ipi_nmi_pending;
+#endif 
+
 /*
  * Local data and functions.
  */
@@ -1191,6 +1195,76 @@
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
 }
 
+#ifdef KDB_STOP_NMI
+/*
+ * send NMI IPI to selected CPUs
+ */
+
+#define	BEFORE_SPIN	1000000
+
+void
+ipi_nmi_selected(u_int32_t cpus)
+{
+
+	int cpu;
+	register_t icrlo;
+
+	icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT 
+		| APIC_TRIGMOD_EDGE; 
+	
+	CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus);
+
+
+	atomic_set_int(&ipi_nmi_pending, cpus);
+
+
+	while ((cpu = ffs(cpus)) != 0) {
+		cpu--;
+		cpus &= ~(1 << cpu);
+
+		KASSERT(cpu_apic_ids[cpu] != -1,
+		    ("IPI NMI to non-existent CPU %d", cpu));
+		
+		/* Wait for an earlier IPI to finish. */
+		if (!lapic_ipi_wait(BEFORE_SPIN))
+			panic("ipi_nmi_selected: previous IPI has not cleared");
+
+		lapic_ipi_raw(icrlo,cpu_apic_ids[cpu]);
+	}
+}
+
+
+int
+ipi_nmi_handler()
+{
+	int cpu  = PCPU_GET(cpuid);
+
+	if(!(atomic_load_acq_int(&ipi_nmi_pending) & (1 << cpu)))
+		return 1;
+
+	atomic_clear_int(&ipi_nmi_pending,1 << cpu);
+
+	savectx(&stoppcbs[cpu]);
+
+	/* Indicate that we are stopped */
+	atomic_set_int(&stopped_cpus,1 << cpu);
+
+
+	/* Wait for restart */
+	while(!(atomic_load_acq_int(&started_cpus) & (1 << cpu)))
+	    ia32_pause();
+
+	atomic_clear_int(&started_cpus,1 << cpu);
+	atomic_clear_int(&stopped_cpus,1 << cpu);
+
+	if(cpu == 0 && cpustop_restartfunc != NULL)
+		cpustop_restartfunc();
+
+	return 0;
+}
+     
+#endif /* KDB_STOP_NMI */
+
 /*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
Index: src/sys/i386/i386/trap.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/trap.c,v
retrieving revision 1.272
diff -u -r1.272 trap.c
--- src/sys/i386/i386/trap.c	20 Apr 2005 20:52:45 -0000	1.272
+++ src/sys/i386/i386/trap.c	30 Apr 2005 04:43:13 -0000
@@ -183,6 +183,14 @@
 	PCPU_LAZY_INC(cnt.v_trap);
 	type = frame.tf_trapno;
 
+#ifdef KDB_STOP_NMI
+	/* Handler for NMI IPIs used for debugging */
+	if (type == T_NMI) {
+	         if (ipi_nmi_handler() == 0)
+	                   goto out;
+	}
+#endif /* KDB_STOP_NMI */
+
 #ifdef KDB
 	if (kdb_active) {
 		kdb_reenter();
Index: src/sys/i386/include/smp.h
===================================================================
RCS file: /home/ncvs/src/sys/i386/include/smp.h,v
retrieving revision 1.81
diff -u -r1.81 smp.h
--- src/sys/i386/include/smp.h	8 Feb 2005 20:25:06 -0000	1.81
+++ src/sys/i386/include/smp.h	30 Apr 2005 04:43:13 -0000
@@ -79,6 +79,11 @@
 void	smp_invltlb(void);
 void	smp_masked_invltlb(u_int mask);
 
+#ifdef KDB_STOP_NMI
+int ipi_nmi_handler(void);
+void ipi_nmi_selected(u_int32_t cpus);
+#endif
+
 #endif /* !LOCORE */
 #endif /* SMP */
 
Index: src/sys/kern/subr_kdb.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/subr_kdb.c,v
retrieving revision 1.10
diff -u -r1.10 subr_kdb.c
--- src/sys/kern/subr_kdb.c	6 Jan 2005 23:35:39 -0000	1.10
+++ src/sys/kern/subr_kdb.c	30 Apr 2005 04:43:13 -0000
@@ -40,6 +40,18 @@
 #include <machine/kdb.h>
 #include <machine/pcb.h>
 
+#ifdef KDB_STOP_NMI
+#include <machine/smp.h>
+#endif
+
+/* 
+ * KDB_STOP_NMI requires SMP to pick up the right dependencies
+ * (And isn't useful on UP anyway) 
+ */
+#if defined(KDB_STOP_NMI) && !defined(SMP)
+#error "options KDB_STOP_NMI" requires "options SMP"
+#endif
+
 int kdb_active = 0;
 void *kdb_jmpbufp = NULL;
 struct kdb_dbbe *kdb_dbbe = NULL;
@@ -77,6 +89,19 @@
 SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus, CTLTYPE_INT | CTLFLAG_RW,
     &kdb_stop_cpus, 0, "");
 TUNABLE_INT("debug.kdb.stop_cpus", &kdb_stop_cpus);
+
+#ifdef KDB_STOP_NMI
+/* 
+ * Provide an alternate method of stopping other CPUs. If another CPU has
+ * disabled interrupts the conventional STOP IPI will be blocked. This 
+ * NMI-based stop should get through in that case.
+ */
+static int kdb_stop_cpus_with_nmi = 0;
+SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
+    &kdb_stop_cpus_with_nmi, 0, "");
+TUNABLE_INT("debug.kdb.stop_cpus_with_nmi", &kdb_stop_cpus_with_nmi);
+#endif /* KDB_STOP_NMI */
+
 #endif
 
 static int
@@ -308,9 +333,27 @@
 
 struct pcb *
 kdb_thr_ctx(struct thread *thr)
+#ifdef KDB_STOP_NMI
+{  
+  u_int		cpuid;
+  struct pcpu *pc;
+  
+  if (thr == curthread) 
+    return &kdb_pcb;
+
+  SLIST_FOREACH(pc, &cpuhead, pc_allcpu)  {
+    cpuid = pc->pc_cpuid;
+    if (pc->pc_curthread == thr && (atomic_load_acq_int(&stopped_cpus) & (1 << cpuid)))
+      return &stoppcbs[cpuid];
+  }
+
+  return  thr->td_pcb;
+}
+#else
 {
 	return ((thr == curthread) ? &kdb_pcb : thr->td_pcb);
 }
+#endif /* KDB_STOP_NMI */
 
 struct thread *
 kdb_thr_first(void)
@@ -407,7 +450,14 @@
 
 #ifdef SMP
 	if ((did_stop_cpus = kdb_stop_cpus) != 0)
+	  {
+#ifdef KDB_STOP_NMI
+	    if(kdb_stop_cpus_with_nmi)
+	      stop_cpus_nmi(PCPU_GET(other_cpus));
+	    else
+#endif /* KDB_STOP_NMI */
 		stop_cpus(PCPU_GET(other_cpus));
+	  }
 #endif
 
 	kdb_frame = tf;
Index: src/sys/kern/subr_smp.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/subr_smp.c,v
retrieving revision 1.194
diff -u -r1.194 subr_smp.c
--- src/sys/kern/subr_smp.c	6 Jan 2005 23:35:39 -0000	1.194
+++ src/sys/kern/subr_smp.c	30 Apr 2005 04:43:13 -0000
@@ -254,6 +254,35 @@
 	return 1;
 }
 
+#ifdef KDB_STOP_NMI
+int
+stop_cpus_nmi(cpumask_t map)
+{
+	int i;
+
+	if (!smp_started)
+		return 0;
+
+	CTR1(KTR_SMP, "stop_cpus(%x)", map);
+
+	/* send the stop IPI to all CPUs in map */
+	ipi_nmi_selected(map);
+
+	i = 0;
+	while ((atomic_load_acq_int(&stopped_cpus) & map) != map) {
+		/* spin */
+		i++;
+#ifdef DIAGNOSTIC
+		if (i == 100000) {
+			printf("timeout stopping cpus\n");
+			break;
+		}
+#endif
+	}
+
+	return 1;
+}
+#endif /* KDB_STOP_NMI */
 
 /*
  * Called by a CPU to restart stopped CPUs. 
Index: src/sys/sys/smp.h
===================================================================
RCS file: /home/ncvs/src/sys/sys/smp.h,v
retrieving revision 1.83
diff -u -r1.83 smp.h
--- src/sys/sys/smp.h	7 Jan 2005 02:29:24 -0000	1.83
+++ src/sys/sys/smp.h	30 Apr 2005 04:43:13 -0000
@@ -101,6 +101,10 @@
 int	stop_cpus(cpumask_t);
 void	smp_rendezvous_action(void);
 extern	struct mtx smp_ipi_mtx;
+
+#ifdef KDB_STOP_NMI
+int	stop_cpus_nmi(cpumask_t);
+#endif 
 #endif /* SMP */
 void	smp_rendezvous(void (*)(void *), 
 		       void (*)(void *),