diff -rubB sys.base/amd64/amd64/local_apic.c sys/amd64/amd64/local_apic.c
--- sys.base/amd64/amd64/local_apic.c	2009-08-04 09:06:58.000000000 +0200
+++ sys/amd64/amd64/local_apic.c	2009-08-04 09:29:06.000000000 +0200
@@ -1187,8 +1187,17 @@
 	KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
 	    ("%s: invalid vector %d", __func__, vector));
 
-	icrlo = vector | APIC_DELMODE_FIXED | APIC_DESTMODE_PHY |
-	    APIC_LEVEL_DEASSERT | APIC_TRIGMOD_EDGE;
+	icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE;
+
+	/*
+	 * IPI_STOP_HARD is just a "fake" vector used to send an NMI.
+	 * Use special rules regard NMI if passed, otherwise specify
+	 * the vector.
+	 */
+	if (vector == IPI_STOP_HARD)
+		icrlo |= APIC_DELMODE_NMI | APIC_LEVEL_ASSERT;
+	else
+		icrlo |= vector | APIC_DELMODE_FIXED | APIC_LEVEL_DEASSERT;
 	destfield = 0;
 	switch (dest) {
 	case APIC_IPI_DEST_SELF:
diff -rubB sys.base/amd64/amd64/mp_machdep.c sys/amd64/amd64/mp_machdep.c
--- sys.base/amd64/amd64/mp_machdep.c	2009-08-04 09:06:58.000000000 +0200
+++ sys/amd64/amd64/mp_machdep.c	2009-08-04 16:56:01.000000000 +0200
@@ -115,31 +115,12 @@
 
 extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
 
-#ifdef STOP_NMI
-volatile cpumask_t ipi_nmi_pending;
-
-static void	ipi_nmi_selected(u_int32_t cpus);
-#endif 
-
 /*
  * Local data and functions.
  */
 
-#ifdef STOP_NMI
-/* 
- * Provide an alternate method of stopping other CPUs. If another CPU has
- * disabled interrupts the conventional STOP IPI will be blocked. This 
- * NMI-based stop should get through in that case.
- */
-static int stop_cpus_with_nmi = 1;
-SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
-    &stop_cpus_with_nmi, 0, "");
-TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi);
-#else
-#define	stop_cpus_with_nmi	0
-#endif
-
 static u_int logical_cpus;
+static volatile cpumask_t ipi_nmi_pending;
 
 /* used to hold the AP's until we are ready to release them */
 static struct mtx ap_boot_mtx;
@@ -1041,12 +1022,14 @@
 		ipi = IPI_BITMAP_VECTOR;
 	}
 
-#ifdef STOP_NMI
-	if (ipi == IPI_STOP && stop_cpus_with_nmi) {
-		ipi_nmi_selected(cpus);
-		return;
-	}
-#endif
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, cpus);
+
 	CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
 	while ((cpu = ffs(cpus)) != 0) {
 		cpu--;
@@ -1077,10 +1060,19 @@
 ipi_all(u_int ipi)
 {
 
-	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+	if (IPI_IS_BITMAPED(ipi)) {
 		ipi_selected(all_cpus, ipi);
 		return;
 	}
+
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, all_cpus);
+
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL);
 }
@@ -1092,10 +1084,19 @@
 ipi_all_but_self(u_int ipi)
 {
 
-	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+	if (IPI_IS_BITMAPED(ipi)) {
 		ipi_selected(PCPU_GET(other_cpus), ipi);
 		return;
 	}
+
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus));
+
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
 }
@@ -1107,64 +1108,43 @@
 ipi_self(u_int ipi)
 {
 
-	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+	if (IPI_IS_BITMAPED(ipi)) {
 		ipi_selected(PCPU_GET(cpumask), ipi);
 		return;
 	}
-	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
-	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
-}
 
-#ifdef STOP_NMI
-/*
- * send NMI IPI to selected CPUs
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
  */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, PCPU_GET(cpumask));
 
-#define	BEFORE_SPIN	1000000
-
-void
-ipi_nmi_selected(u_int32_t cpus)
-{
-	int cpu;
-	register_t icrlo;
-
-	icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT 
-		| APIC_TRIGMOD_EDGE; 
-	
-	CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus);
-
-	atomic_set_int(&ipi_nmi_pending, cpus);
-
-	while ((cpu = ffs(cpus)) != 0) {
-		cpu--;
-		cpus &= ~(1 << cpu);
-
-		KASSERT(cpu_apic_ids[cpu] != -1,
-		    ("IPI NMI to non-existent CPU %d", cpu));
-		
-		/* Wait for an earlier IPI to finish. */
-		if (!lapic_ipi_wait(BEFORE_SPIN))
-			panic("ipi_nmi_selected: previous IPI has not cleared");
-
-		lapic_ipi_raw(icrlo, cpu_apic_ids[cpu]);
-	}
+	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
+	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
 }
 
 int
 ipi_nmi_handler(void)
 {
-	int cpumask = PCPU_GET(cpumask);
+	cpumask_t cpumask;
 
-	if (!(ipi_nmi_pending & cpumask))
-		return 1;
+	/*
+	 * As long as there is not a simple way to know about a NMI's
+	 * source, if the bitmask for the current CPU is present in
+	 * the global pending bitword an IPI_STOP_HARD has been issued
+	 * and should be handled.
+	 */
+	cpumask = PCPU_GET(cpumask);
+	if ((ipi_nmi_pending & cpumask) == 0)
+		return (1);
 
 	atomic_clear_int(&ipi_nmi_pending, cpumask);
 	cpustop_handler();
-	return 0;
+	return (0);
 }
      
-#endif /* STOP_NMI */
-
 /*
  * Handle an IPI_STOP by saving our current context and spinning until we
  * are resumed.
diff -rubB sys.base/amd64/amd64/trap.c sys/amd64/amd64/trap.c
--- sys.base/amd64/amd64/trap.c	2009-08-04 09:06:58.000000000 +0200
+++ sys/amd64/amd64/trap.c	2009-08-04 09:28:16.000000000 +0200
@@ -191,15 +191,11 @@
 	PCPU_INC(cnt.v_trap);
 	type = frame->tf_trapno;
 
-#ifdef SMP
-#ifdef STOP_NMI
 	/* Handler for NMI IPIs used for stopping CPUs. */
 	if (type == T_NMI) {
 	         if (ipi_nmi_handler() == 0)
 	                   goto out;
 	}
-#endif /* STOP_NMI */
-#endif /* SMP */
 
 #ifdef KDB
 	if (kdb_active) {
diff -rubB sys.base/amd64/conf/GENERIC sys/amd64/conf/GENERIC
--- sys.base/amd64/conf/GENERIC	2009-08-04 09:06:58.000000000 +0200
+++ sys/amd64/conf/GENERIC	2009-08-04 09:24:55.000000000 +0200
@@ -62,7 +62,6 @@
 options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
 options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
 options 	ADAPTIVE_GIANT		# Giant mutex is adaptive.
-options 	STOP_NMI		# Stop CPUS using NMI instead of IPI
 options 	AUDIT			# Security event auditing
 #options 	KDTRACE_FRAME		# Ensure frames are compiled in
 #options 	KDTRACE_HOOKS		# Kernel DTrace hooks
Only in sys/amd64/conf: LINT
diff -rubB sys.base/amd64/conf/NOTES sys/amd64/conf/NOTES
--- sys.base/amd64/conf/NOTES	2009-08-04 09:06:58.000000000 +0200
+++ sys/amd64/conf/NOTES	2009-08-04 09:24:49.000000000 +0200
@@ -30,12 +30,6 @@
 #
 options 	MP_WATCHDOG
 
-# 
-# Debugging options.
-#
-options		STOP_NMI		# Stop CPUS using NMI instead of IPI
-
-
 
 #####################################################################
 # CPU OPTIONS
diff -rubB sys.base/amd64/include/apicvar.h sys/amd64/include/apicvar.h
--- sys.base/amd64/include/apicvar.h	2009-08-04 09:06:58.000000000 +0200
+++ sys/amd64/include/apicvar.h	2009-08-04 09:24:20.000000000 +0200
@@ -102,11 +102,6 @@
  * smp_ipi_mtx and waits for the completion of the IPI (Only one IPI user 
  * at a time) The second group uses a single interrupt and a bitmap to avoid
  * redundant IPI interrupts.
- *
- * Right now IPI_STOP used by kdb shares the interrupt priority class with
- * the two IPI groups mentioned above. As such IPI_STOP may cause a deadlock.
- * Eventually IPI_STOP should use NMI IPIs - this would eliminate this and
- * other deadlocks caused by IPI_STOP.
  */ 
 
 /* Interrupts for local APIC LVT entries other than the timer. */
@@ -130,6 +125,7 @@
 #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST)
 
 #define	IPI_STOP	(APIC_IPI_INTS + 7)	/* Stop CPU until restarted. */
+#define	IPI_STOP_HARD	(APIC_IPI_INTS + 8)	/* Stop CPU with a NMI. */
 
 /*
  * The spurious interrupt can share the priority class with the IPIs since
diff -rubB sys.base/amd64/include/smp.h sys/amd64/include/smp.h
--- sys.base/amd64/include/smp.h	2009-08-04 09:06:58.000000000 +0200
+++ sys/amd64/include/smp.h	2009-08-04 09:23:49.000000000 +0200
@@ -50,6 +50,7 @@
 void	cpu_add(u_int apic_id, char boot_cpu);
 void	cpustop_handler(void);
 void	init_secondary(void);
+int	ipi_nmi_handler(void);
 void	ipi_selected(u_int cpus, u_int ipi);
 void	ipi_all(u_int ipi);
 void	ipi_all_but_self(u_int ipi);
@@ -67,10 +68,6 @@
 void	smp_invltlb(void);
 void	smp_masked_invltlb(u_int mask);
 
-#ifdef STOP_NMI
-int	ipi_nmi_handler(void);
-#endif
-
 #endif /* !LOCORE */
 #endif /* SMP */
 
diff -rubB sys.base/conf/options.amd64 sys/conf/options.amd64
--- sys.base/conf/options.amd64	2009-08-04 09:06:51.000000000 +0200
+++ sys/conf/options.amd64	2009-08-04 09:10:14.000000000 +0200
@@ -63,7 +63,6 @@
 DEV_ATPIC		opt_atpic.h
 
 # Debugging
-STOP_NMI		opt_cpu.h
 KDTRACE_FRAME		opt_kdtrace.h
 
 # BPF just-in-time compiler
diff -rubB sys.base/conf/options.i386 sys/conf/options.i386
--- sys.base/conf/options.i386	2009-08-04 09:06:51.000000000 +0200
+++ sys/conf/options.i386	2009-08-04 09:10:05.000000000 +0200
@@ -146,7 +146,6 @@
 
 # Debugging
 NPX_DEBUG		opt_npx.h
-STOP_NMI		opt_cpu.h
 
 # BPF just-in-time compiler
 BPF_JITTER		opt_bpf.h
diff -rubB sys.base/conf/options.pc98 sys/conf/options.pc98
--- sys.base/conf/options.pc98	2009-08-04 09:06:51.000000000 +0200
+++ sys/conf/options.pc98	2009-08-04 09:10:35.000000000 +0200
@@ -112,7 +112,6 @@
 
 # Debugging
 NPX_DEBUG		opt_npx.h
-STOP_NMI		opt_cpu.h
 
 # BPF just-in-time compiler
 BPF_JITTER		opt_bpf.h
Only in sys/i386/compile: GENERIC
diff -rubB sys.base/i386/conf/GENERIC sys/i386/conf/GENERIC
--- sys.base/i386/conf/GENERIC	2009-08-04 09:06:29.000000000 +0200
+++ sys/i386/conf/GENERIC	2009-08-04 09:18:11.000000000 +0200
@@ -63,7 +63,6 @@
 options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
 options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
 options 	ADAPTIVE_GIANT		# Giant mutex is adaptive.
-options 	STOP_NMI		# Stop CPUS using NMI instead of IPI
 options 	AUDIT			# Security event auditing
 #options 	KDTRACE_HOOKS		# Kernel DTrace hooks
 
Only in sys/i386/conf: LINT
diff -rubB sys.base/i386/conf/NOTES sys/i386/conf/NOTES
--- sys.base/i386/conf/NOTES	2009-08-04 09:06:29.000000000 +0200
+++ sys/i386/conf/NOTES	2009-08-04 09:18:07.000000000 +0200
@@ -49,7 +49,6 @@
 
 # Debugging options.
 #
-options		STOP_NMI		# Stop CPUS using NMI instead of IPI
 options 	COUNT_XINVLTLB_HITS	# Counters for TLB events
 options 	COUNT_IPIS		# Per-CPU IPI interrupt counters
 
diff -rubB sys.base/i386/i386/local_apic.c sys/i386/i386/local_apic.c
--- sys.base/i386/i386/local_apic.c	2009-08-04 09:06:29.000000000 +0200
+++ sys/i386/i386/local_apic.c	2009-08-04 09:23:17.000000000 +0200
@@ -1194,8 +1194,17 @@
 	KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
 	    ("%s: invalid vector %d", __func__, vector));
 
-	icrlo = vector | APIC_DELMODE_FIXED | APIC_DESTMODE_PHY |
-	    APIC_LEVEL_DEASSERT | APIC_TRIGMOD_EDGE;
+	icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE;
+
+	/*
+	 * IPI_STOP_HARD is just a "fake" vector used to send an NMI.
+	 * Use special rules regard NMI if passed, otherwise specify
+	 * the vector.
+	 */
+	if (vector == IPI_STOP_HARD)
+		icrlo |= APIC_DELMODE_NMI | APIC_LEVEL_ASSERT;
+	else
+		icrlo |= vector | APIC_DELMODE_FIXED | APIC_LEVEL_DEASSERT;
 	destfield = 0;
 	switch (dest) {
 	case APIC_IPI_DEST_SELF:
diff -rubB sys.base/i386/i386/mp_machdep.c sys/i386/i386/mp_machdep.c
--- sys.base/i386/i386/mp_machdep.c	2009-08-04 09:06:29.000000000 +0200
+++ sys/i386/i386/mp_machdep.c	2009-08-04 15:36:49.000000000 +0200
@@ -158,12 +158,6 @@
 vm_offset_t smp_tlb_addr2;
 volatile int smp_tlb_wait;
 
-#ifdef STOP_NMI
-volatile cpumask_t ipi_nmi_pending;
-
-static void	ipi_nmi_selected(u_int32_t cpus);
-#endif 
-
 #ifdef COUNT_IPIS
 /* Interrupt counts. */
 static u_long *ipi_preempt_counts[MAXCPU];
@@ -180,21 +174,8 @@
  * Local data and functions.
  */
 
-#ifdef STOP_NMI
-/* 
- * Provide an alternate method of stopping other CPUs. If another CPU has
- * disabled interrupts the conventional STOP IPI will be blocked. This 
- * NMI-based stop should get through in that case.
- */
-static int stop_cpus_with_nmi = 1;
-SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
-    &stop_cpus_with_nmi, 0, "");
-TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi);
-#else
-#define	stop_cpus_with_nmi	0
-#endif
-
 static u_int logical_cpus;
+static volatile cpumask_t ipi_nmi_pending;
 
 /* used to hold the AP's until we are ready to release them */
 static struct mtx ap_boot_mtx;
@@ -1212,12 +1193,14 @@
 		ipi = IPI_BITMAP_VECTOR;
 	}
 
-#ifdef STOP_NMI
-	if (ipi == IPI_STOP && stop_cpus_with_nmi) {
-		ipi_nmi_selected(cpus);
-		return;
-	}
-#endif
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, cpus);
+
 	CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
 	while ((cpu = ffs(cpus)) != 0) {
 		cpu--;
@@ -1248,10 +1231,19 @@
 ipi_all(u_int ipi)
 {
 
-	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+	if (IPI_IS_BITMAPED(ipi)) {
 		ipi_selected(all_cpus, ipi);
 		return;
 	}
+
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, all_cpus);
+
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL);
 }
@@ -1263,10 +1255,18 @@
 ipi_all_but_self(u_int ipi)
 {
 
-	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+	if (IPI_IS_BITMAPED(ipi)) {
 		ipi_selected(PCPU_GET(other_cpus), ipi);
 		return;
 	}
+
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus));
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
 }
@@ -1278,64 +1278,42 @@
 ipi_self(u_int ipi)
 {
 
-	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+	if (IPI_IS_BITMAPED(ipi)) {
 		ipi_selected(PCPU_GET(cpumask), ipi);
 		return;
 	}
-	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
-	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
-}
 
-#ifdef STOP_NMI
-/*
- * send NMI IPI to selected CPUs
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
  */
-
-#define	BEFORE_SPIN	1000000
-
-void
-ipi_nmi_selected(u_int32_t cpus)
-{
-	int cpu;
-	register_t icrlo;
-
-	icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT 
-		| APIC_TRIGMOD_EDGE; 
-	
-	CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus);
-
-	atomic_set_int(&ipi_nmi_pending, cpus);
-
-	while ((cpu = ffs(cpus)) != 0) {
-		cpu--;
-		cpus &= ~(1 << cpu);
-
-		KASSERT(cpu_apic_ids[cpu] != -1,
-		    ("IPI NMI to non-existent CPU %d", cpu));
-		
-		/* Wait for an earlier IPI to finish. */
-		if (!lapic_ipi_wait(BEFORE_SPIN))
-			panic("ipi_nmi_selected: previous IPI has not cleared");
-
-		lapic_ipi_raw(icrlo, cpu_apic_ids[cpu]);
-	}
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, PCPU_GET(cpumask));
+	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
+	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
 }
 
 int
-ipi_nmi_handler(void)
+ipi_nmi_handler()
 {
-	int cpumask = PCPU_GET(cpumask);
+	cpumask_t cpumask;
 
-	if (!(ipi_nmi_pending & cpumask))
-		return 1;
+	/*
+	 * As long as there is not a simple way to know about a NMI's
+	 * source, if the bitmask for the current CPU is present in
+	 * the global pending bitword an IPI_STOP_HARD has been issued
+	 * and should be handled.
+	 */
+	cpumask = PCPU_GET(cpumask);
+	if ((ipi_nmi_pending & cpumask) == 0)
+		return (1);
 
 	atomic_clear_int(&ipi_nmi_pending, cpumask);
 	cpustop_handler();
-	return 0;
+	return (0);
 }
 
-#endif /* STOP_NMI */
-
 /*
  * Handle an IPI_STOP by saving our current context and spinning until we
  * are resumed.
diff -rubB sys.base/i386/i386/trap.c sys/i386/i386/trap.c
--- sys.base/i386/i386/trap.c	2009-08-04 09:06:29.000000000 +0200
+++ sys/i386/i386/trap.c	2009-08-04 09:22:17.000000000 +0200
@@ -209,15 +209,11 @@
 	PCPU_INC(cnt.v_trap);
 	type = frame->tf_trapno;
 
-#ifdef SMP
-#ifdef STOP_NMI
 	/* Handler for NMI IPIs used for stopping CPUs. */
 	if (type == T_NMI) {
 	         if (ipi_nmi_handler() == 0)
 	                   goto out;
 	}
-#endif /* STOP_NMI */
-#endif /* SMP */
 
 #ifdef KDB
 	if (kdb_active) {
diff -rubB sys.base/i386/include/apicvar.h sys/i386/include/apicvar.h
--- sys.base/i386/include/apicvar.h	2009-08-04 09:06:30.000000000 +0200
+++ sys/i386/include/apicvar.h	2009-08-04 09:17:55.000000000 +0200
@@ -100,11 +100,6 @@
  * smp_ipi_mtx and waits for the completion of the IPI (Only one IPI user 
  * at a time) The second group uses a single interrupt and a bitmap to avoid
  * redundant IPI interrupts.
- *
- * Right now IPI_STOP used by kdb shares the interrupt priority class with
- * the two IPI groups mentioned above. As such IPI_STOP may cause a deadlock.
- * Eventually IPI_STOP should use NMI IPIs - this would eliminate this and
- * other deadlocks caused by IPI_STOP.
  */ 
 
 /* Interrupts for local APIC LVT entries other than the timer. */
@@ -129,6 +124,7 @@
 #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST)
 
 #define	IPI_STOP	(APIC_IPI_INTS + 7)	/* Stop CPU until restarted. */
+#define	IPI_STOP_HARD	(APIC_IPI_INTS + 8)	/* Stop CPU with a NMI. */
 
 /*
  * The spurious interrupt can share the priority class with the IPIs since
diff -rubB sys.base/i386/include/smp.h sys/i386/include/smp.h
--- sys.base/i386/include/smp.h	2009-08-04 09:06:30.000000000 +0200
+++ sys/i386/include/smp.h	2009-08-04 09:16:21.000000000 +0200
@@ -60,7 +60,8 @@
 void	cpu_add(u_int apic_id, char boot_cpu);
 void	cpustop_handler(void);
 void	init_secondary(void);
-void	ipi_selected(u_int cpus, u_int ipi);
+int	ipi_nmi_handler(void);
+void	ipi_selected(cpumask_t cpus, u_int ipi);
 void	ipi_all(u_int ipi);
 void	ipi_all_but_self(u_int ipi);
 void	ipi_self(u_int ipi);
@@ -77,10 +78,6 @@
 void	smp_invltlb(void);
 void	smp_masked_invltlb(u_int mask);
 
-#ifdef STOP_NMI
-int	ipi_nmi_handler(void);
-#endif
-
 #endif /* !LOCORE */
 #endif /* SMP */
 
Only in sys/ia64/conf: LINT
diff -rubB sys.base/ia64/ia64/interrupt.c sys/ia64/ia64/interrupt.c
--- sys.base/ia64/ia64/interrupt.c	2009-08-04 09:06:28.000000000 +0200
+++ sys/ia64/ia64/interrupt.c	2009-08-04 09:14:38.000000000 +0200
@@ -148,6 +148,8 @@
 	/*
 	 * Handle ExtINT interrupts by generating an INTA cycle to
 	 * read the vector.
+	 * IPI_STOP_HARD is mapped to IPI_STOP so it is not necessary
+	 * to add it to this switch-like construct.
 	 */
 	if (vector == 0) {
 		inta = ib->ib_inta;
diff -rubB sys.base/ia64/include/smp.h sys/ia64/include/smp.h
--- sys.base/ia64/include/smp.h	2009-08-04 09:06:28.000000000 +0200
+++ sys/ia64/include/smp.h	2009-08-04 09:14:52.000000000 +0200
@@ -22,6 +22,7 @@
 #define	IPI_AST			5
 #define	IPI_RENDEZVOUS		6
 #define	IPI_STOP		7
+#define IPI_STOP_HARD		7
 #define	IPI_PREEMPT		8
 
 #define	IPI_COUNT		9
diff -rubB sys.base/kern/kern_shutdown.c sys/kern/kern_shutdown.c
--- sys.base/kern/kern_shutdown.c	2009-08-04 09:06:27.000000000 +0200
+++ sys/kern/kern_shutdown.c	2009-08-04 09:11:33.000000000 +0200
@@ -411,9 +411,6 @@
 	 */
 	EVENTHANDLER_INVOKE(shutdown_post_sync, howto);
 
-	/* XXX This doesn't disable interrupts any more.  Reconsider? */
-	splhigh();
-
 	if ((howto & (RB_HALT|RB_DUMP)) == RB_DUMP && !cold && !dumping) 
 		doadump();
 
@@ -486,6 +483,12 @@
 static void
 shutdown_reset(void *junk, int howto)
 {
+	/*
+	 * Disable interrupts on CPU0 in order to avoid fast handlers
+	 * to preempt the stopping process and to deadlock against other
+	 * CPUs.
+	 */
+	spinlock_enter();
 
 	printf("Rebooting...\n");
 	DELAY(1000000);	/* wait 1 sec for printf's to complete and be read */
diff -rubB sys.base/kern/subr_kdb.c sys/kern/subr_kdb.c
--- sys.base/kern/subr_kdb.c	2009-08-04 09:06:27.000000000 +0200
+++ sys/kern/subr_kdb.c	2009-08-04 09:14:02.000000000 +0200
@@ -88,7 +88,8 @@
  * Flag indicating whether or not to IPI the other CPUs to stop them on
  * entering the debugger.  Sometimes, this will result in a deadlock as
  * stop_cpus() waits for the other cpus to stop, so we allow it to be
- * disabled.
+ * disabled.  In order to maximize the chances of success, use an hard
+ * stop for that.
  */
 #ifdef SMP
 static int kdb_stop_cpus = 1;
@@ -508,7 +509,7 @@
 
 #ifdef SMP
 	if ((did_stop_cpus = kdb_stop_cpus) != 0)
-		stop_cpus(PCPU_GET(other_cpus));
+		stop_cpus_hard(PCPU_GET(other_cpus));
 #endif
 
 	kdb_active++;
diff -rubB sys.base/kern/subr_smp.c sys/kern/subr_smp.c
--- sys.base/kern/subr_smp.c	2009-08-04 09:06:28.000000000 +0200
+++ sys/kern/subr_smp.c	2009-08-04 15:31:54.000000000 +0200
@@ -229,18 +229,21 @@
  * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
  *            from executing at same time.
  */
-int
-stop_cpus(cpumask_t map)
+static int
+generic_stop_cpus(cpumask_t map, u_int type)
 {
 	int i;
 
+	KASSERT(type == IPI_STOP || type == IPI_STOP_HARD,
+	    ("%s: invalid stop type", __func__));
+
 	if (!smp_started)
 		return 0;
 
-	CTR1(KTR_SMP, "stop_cpus(%x)", map);
+	CTR2(KTR_SMP, "stop_cpus(%x) with %u type", map, type);
 
 	/* send the stop IPI to all CPUs in map */
-	ipi_selected(map, IPI_STOP);
+	ipi_selected(map, type);
 
 	i = 0;
 	while ((stopped_cpus & map) != map) {
@@ -258,6 +261,20 @@
 	return 1;
 }
 
+int
+stop_cpus(cpumask_t map)
+{
+
+	return (generic_stop_cpus(map, IPI_STOP));
+}
+
+int
+stop_cpus_hard(cpumask_t map)
+{
+
+	return (generic_stop_cpus(map, IPI_STOP_HARD));
+}
+
 /*
  * Called by a CPU to restart stopped CPUs. 
  *
Only in sys/pc98/conf: LINT
diff -rubB sys.base/pc98/conf/NOTES sys/pc98/conf/NOTES
--- sys.base/pc98/conf/NOTES	2009-08-04 09:06:19.000000000 +0200
+++ sys/pc98/conf/NOTES	2009-08-04 09:15:43.000000000 +0200
@@ -29,11 +29,6 @@
 #
 options 	MP_WATCHDOG
 
-# Debugging options.
-#
-options		STOP_NMI		# Stop CPUS using NMI instead of IPI
-
-
 
 #####################################################################
 # CPU OPTIONS
Only in sys/powerpc/conf: LINT
Only in sys/sparc64/conf: LINT
diff -rubB sys.base/sparc64/include/smp.h sys/sparc64/include/smp.h
--- sys.base/sparc64/include/smp.h	2009-08-04 09:06:18.000000000 +0200
+++ sys/sparc64/include/smp.h	2009-08-04 09:09:23.000000000 +0200
@@ -56,6 +56,7 @@
 #define	IPI_RENDEZVOUS	PIL_RENDEZVOUS
 #define	IPI_PREEMPT	PIL_PREEMPT
 #define	IPI_STOP	PIL_STOP
+#define	IPI_STOP_HARD	PIL_STOP
 
 #define	IPI_RETRIES	5000
 
Only in sys/sun4v/conf: LINT
diff -rubB sys.base/sun4v/include/smp.h sys/sun4v/include/smp.h
--- sys.base/sun4v/include/smp.h	2009-08-04 09:06:17.000000000 +0200
+++ sys/sun4v/include/smp.h	2009-08-04 09:15:23.000000000 +0200
@@ -44,6 +44,7 @@
 #define	IPI_AST		PIL_AST
 #define	IPI_RENDEZVOUS	PIL_RENDEZVOUS
 #define	IPI_STOP	PIL_STOP
+#define	IPI_STOP_HARD	PIL_STOP
 #define IPI_PREEMPT     PIL_PREEMPT
 
 
diff -rubB sys.base/sys/smp.h sys/sys/smp.h
--- sys.base/sys/smp.h	2009-08-04 09:06:16.000000000 +0200
+++ sys/sys/smp.h	2009-08-04 09:29:27.000000000 +0200
@@ -99,6 +99,7 @@
 void	forward_roundrobin(void);
 int	restart_cpus(cpumask_t);
 int	stop_cpus(cpumask_t);
+int	stop_cpus_hard(cpumask_t);
 void	smp_rendezvous_action(void);
 extern	struct mtx smp_ipi_mtx;