diff --git a/gnu/usr.bin/gdb/kgdb/kthr.c b/gnu/usr.bin/gdb/kgdb/kthr.c index 1bc95cb18b369..4ccd351e7424c 100644 --- a/gnu/usr.bin/gdb/kgdb/kthr.c +++ b/gnu/usr.bin/gdb/kgdb/kthr.c @@ -49,6 +49,7 @@ static CORE_ADDR dumppcb; static int dumptid; static cpuset_t stopped_cpus; +static cpuset_t hard_stopped_cpus; static struct kthr *first; struct kthr *curkthr; @@ -97,7 +98,8 @@ kgdb_thr_add_procs(uintptr_t paddr) if (td.td_tid == dumptid) kt->pcb = dumppcb; else if (td.td_oncpu != NOCPU && - CPU_ISSET(td.td_oncpu, &stopped_cpus)) + (CPU_ISSET(td.td_oncpu, &stopped_cpus) || + CPU_ISSET(td.td_oncpu, &hard_stopped_cpus))) kt->pcb = kgdb_trgt_core_pcb(td.td_oncpu); else kt->pcb = (uintptr_t)td.td_pcb; @@ -148,6 +150,12 @@ kgdb_thr_init(void) if (cpusetsize != -1 && (u_long)cpusetsize <= sizeof(cpuset_t) && addr != 0) kvm_read(kvm, addr, &stopped_cpus, cpusetsize); + addr = kgdb_lookup("hard_stopped_cpus"); + CPU_ZERO(&hard_stopped_cpus); + cpusetsize = sysconf(_SC_CPUSET_SIZE); + if (cpusetsize != -1 && (u_long)cpusetsize <= sizeof(cpuset_t) && + addr != 0) + kvm_read(kvm, addr, &hard_stopped_cpus, cpusetsize); kgdb_thr_add_procs(paddr); addr = kgdb_lookup("zombproc"); diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index fc9e634940fc2..f2f5ba51f645b 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -592,31 +593,33 @@ cpu_reset() u_int cnt; if (smp_started) { - map = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &map); - CPU_NAND(&map, &stopped_cpus); - if (!CPU_EMPTY(&map)) { - printf("cpu_reset: Stopping other CPUs\n"); - stop_cpus(map); + if (panicstr == NULL && !kdb_active) { + map = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &map); + CPU_NAND(&map, &stopped_cpus); + if (!CPU_EMPTY(&map)) { + printf("cpu_reset: Stopping other CPUs\n"); + stop_cpus(map); + } } - if (PCPU_GET(cpuid) != 0) { cpu_reset_proxyid = PCPU_GET(cpuid); - cpustop_restartfunc = cpu_reset_proxy; cpu_reset_proxy_active = 0; printf("cpu_reset: Restarting BSP\n"); /* Restart CPU #0. */ - CPU_SETOF(0, &started_cpus); - wmb(); + cpustop_hook = cpu_reset_proxy; cnt = 0; - while (cpu_reset_proxy_active == 0 && cnt < 10000000) { + while (cpu_reset_proxy_active == 0 && cnt < 100000000) { ia32_pause(); cnt++; /* Wait for BSP to announce restart */ } - if (cpu_reset_proxy_active == 0) + if (cpu_reset_proxy_active == 0) { printf("cpu_reset: Failed to restart BSP\n"); + cpu_reset_real(); + } + enable_intr(); cpu_reset_proxy_active = 2; diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 65f3a0a1ae750..ab4a6a873b9a3 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -643,30 +644,32 @@ cpu_reset() u_int cnt; if (smp_started) { - map = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &map); - CPU_NAND(&map, &stopped_cpus); - if (!CPU_EMPTY(&map)) { - printf("cpu_reset: Stopping other CPUs\n"); - stop_cpus(map); + if (panicstr == NULL && !kdb_active) { + map = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &map); + CPU_NAND(&map, &stopped_cpus); + if (!CPU_EMPTY(&map)) { + printf("cpu_reset: Stopping other CPUs\n"); + stop_cpus(map); + } } if (PCPU_GET(cpuid) != 0) { cpu_reset_proxyid = PCPU_GET(cpuid); - cpustop_restartfunc = cpu_reset_proxy; cpu_reset_proxy_active = 0; printf("cpu_reset: Restarting BSP\n"); /* Restart CPU #0. */ - /* XXX: restart_cpus(1 << 0); */ - CPU_SETOF(0, &started_cpus); - wmb(); + cpustop_hook = cpu_reset_proxy; cnt = 0; - while (cpu_reset_proxy_active == 0 && cnt < 10000000) + while (cpu_reset_proxy_active == 0 && cnt < 100000000) cnt++; /* Wait for BSP to announce restart */ - if (cpu_reset_proxy_active == 0) + if (cpu_reset_proxy_active == 0) { printf("cpu_reset: Failed to restart BSP\n"); + cpu_reset_real(); + } + enable_intr(); cpu_reset_proxy_active = 2; diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c index 78227e7412dc2..9e913d8c972de 100644 --- a/sys/kern/kern_shutdown.c +++ b/sys/kern/kern_shutdown.c @@ -693,9 +693,6 @@ panic(const char *fmt, ...) void vpanic(const char *fmt, va_list ap) { -#ifdef SMP - cpuset_t other_cpus; -#endif struct thread *td = curthread; int bootopt, newpanic; static char buf[256]; @@ -704,15 +701,12 @@ vpanic(const char *fmt, va_list ap) #ifdef SMP /* - * stop_cpus_hard(other_cpus) should prevent multiple CPUs from + * stop_cpus_hard() should prevent multiple CPUs from * concurrently entering panic. Only the winner will proceed * further. */ - if (panicstr == NULL && !kdb_active) { - other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - stop_cpus_hard(other_cpus); - } + if (panicstr == NULL && !kdb_active) + stop_cpus_hard(); /* * Ensure that the scheduler is stopped while panicking, even if panic diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c index d20c56b85ef44..9a85990542f19 100644 --- a/sys/kern/subr_kdb.c +++ b/sys/kern/subr_kdb.c @@ -608,9 +608,6 @@ kdb_thr_select(struct thread *thr) int kdb_trap(int type, int code, struct trapframe *tf) { -#ifdef SMP - cpuset_t other_cpus; -#endif struct kdb_dbbe *be; register_t intr; int handled; @@ -630,9 +627,7 @@ kdb_trap(int type, int code, struct trapframe *tf) #ifdef SMP if (!SCHEDULER_STOPPED()) { - other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - stop_cpus_hard(other_cpus); + stop_cpus_hard(); did_stop_cpus = 1; } else did_stop_cpus = 0; @@ -666,7 +661,7 @@ kdb_trap(int type, int code, struct trapframe *tf) #ifdef SMP if (did_stop_cpus) - restart_cpus(stopped_cpus); + unstop_cpus_hard(); #endif intr_restore(intr); diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 4f3b51de27654..cb3d46b7ab245 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -57,10 +57,14 @@ MALLOC_DEFINE(M_TOPO, "toponodes", "SMP topology data"); volatile cpuset_t stopped_cpus; volatile cpuset_t started_cpus; volatile cpuset_t suspended_cpus; +volatile cpuset_t stopping_cpus; +volatile cpuset_t hard_stopped_cpus; +volatile cpuset_t hard_started_cpus; +volatile cpuset_t hard_stopping_cpus; cpuset_t hlt_cpus_mask; cpuset_t logical_cpus_mask; -void (*cpustop_restartfunc)(void); +void (* volatile cpustop_hook)(void); #endif static int sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS); @@ -215,15 +219,16 @@ generic_stop_cpus(cpuset_t map, u_int type) #ifdef KTR char cpusetbuf[CPUSETBUFSIZ]; #endif - static volatile u_int stopping_cpu = NOCPU; + static volatile u_int stopper_cpu = NOCPU; + int cpu; int i; volatile cpuset_t *cpus; KASSERT( #if defined(__amd64__) || defined(__i386__) - type == IPI_STOP || type == IPI_STOP_HARD || type == IPI_SUSPEND, + type == IPI_STOP || type == IPI_SUSPEND, #else - type == IPI_STOP || type == IPI_STOP_HARD, + type == IPI_STOP, #endif ("%s: invalid stop type", __func__)); @@ -245,13 +250,28 @@ generic_stop_cpus(cpuset_t map, u_int type) mtx_lock_spin(&smp_ipi_mtx); #endif - if (stopping_cpu != PCPU_GET(cpuid)) - while (atomic_cmpset_int(&stopping_cpu, NOCPU, - PCPU_GET(cpuid)) == 0) - while (stopping_cpu != NOCPU) - cpu_spinwait(); /* spin */ + /* Ensure non-preemtable context, just in case. */ + spinlock_enter(); + + cpu = PCPU_GET(cpuid); + + if (cpu != stopper_cpu) { + while (atomic_cmpset_int(&stopper_cpu, NOCPU, cpu) == 0) + while (stopper_cpu != NOCPU) { + if (CPU_ISSET(cpu, &stopping_cpus)) + cpustop_handler(); + else + cpu_spinwait(); + } + } else { + /* + * Recursion here is not expected. + */ + panic("cpu stop recursion\n"); + } /* send the stop IPI to all CPUs in map */ + stopping_cpus = map; ipi_selected(map, type); #if defined(__amd64__) || defined(__i386__) @@ -277,7 +297,8 @@ generic_stop_cpus(cpuset_t map, u_int type) mtx_unlock_spin(&smp_ipi_mtx); #endif - stopping_cpu = NOCPU; + stopper_cpu = NOCPU; + spinlock_exit(); return (1); } @@ -288,13 +309,6 @@ stop_cpus(cpuset_t map) return (generic_stop_cpus(map, IPI_STOP)); } -int -stop_cpus_hard(cpuset_t map) -{ - - return (generic_stop_cpus(map, IPI_STOP_HARD)); -} - #if defined(__amd64__) || defined(__i386__) int suspend_cpus(cpuset_t map) @@ -371,6 +385,82 @@ resume_cpus(cpuset_t map) } #endif +void +stop_cpus_hard(void) +{ + static volatile u_int hard_stopper_cpu = NOCPU; + cpuset_t mask; + u_int cpu; + int i; + + if (!smp_started) + return; + + /* Ensure non-preemtable context, just in case. */ + spinlock_enter(); + + cpu = PCPU_GET(cpuid); + + CTR1(KTR_SMP, "hard_stop_cpus() with %u type", IPI_STOP_HARD); + + if (cpu != hard_stopper_cpu) { + while (atomic_cmpset_int(&hard_stopper_cpu, NOCPU, cpu) == 0) + while (hard_stopper_cpu != NOCPU) { + if (CPU_ISSET(cpu, &hard_stopping_cpus)) + cpuhardstop_handler(); + else + cpu_spinwait(); + } + } else { + /* + * Recursion here is not expected. + */ + atomic_store_rel_int(&hard_stopper_cpu, NOCPU); + panic("hard stop recursion\n"); + } + + CPU_COPY(&all_cpus, &mask); + CPU_CLR(cpu, &mask); + CPU_COPY(&mask, &hard_stopping_cpus); + ipi_all_but_self(IPI_STOP_HARD); + + i = 0; + while (CPU_CMP(&hard_stopped_cpus, &mask) != 0) { + cpu_spinwait(); + i++; + if (i == 10000000) { + /* Should not happen; other CPU stuck in NMI handler? */ + printf("timeout stopping cpus\n"); + break; + } + } + + atomic_store_rel_int(&hard_stopper_cpu, NOCPU); + + spinlock_exit(); + return; +} + +void +unstop_cpus_hard(void) +{ + cpuset_t mask; + + if (!smp_started) + return; + + CTR0(KTR_SMP, "unstop_cpus_hard()"); + + /* signal other cpus to restart */ + CPU_COPY(&all_cpus, &mask); + CPU_CLR(PCPU_GET(cpuid), &mask); + CPU_COPY_STORE_REL(&mask, &hard_started_cpus); + + /* wait for each to clear its bit */ + while (!CPU_EMPTY(&hard_stopped_cpus)) + cpu_spinwait(); +} + /* * All-CPU rendezvous. CPUs are signalled, all execute the setup function * (if specified), rendezvous, execute the action function (if specified), diff --git a/sys/sys/smp.h b/sys/sys/smp.h index 904b9f7046f76..a1b4817569706 100644 --- a/sys/sys/smp.h +++ b/sys/sys/smp.h @@ -134,11 +134,15 @@ struct cpu_group *smp_topo_2level(int l2share, int l2count, int l1share, int l1count, int l1flags); struct cpu_group *smp_topo_find(struct cpu_group *top, int cpu); -extern void (*cpustop_restartfunc)(void); +extern void (* volatile cpustop_hook)(void); extern int smp_cpus; extern volatile cpuset_t started_cpus; extern volatile cpuset_t stopped_cpus; extern volatile cpuset_t suspended_cpus; +extern volatile cpuset_t stopping_cpus; +extern volatile cpuset_t hard_started_cpus; +extern volatile cpuset_t hard_stopped_cpus; +extern volatile cpuset_t hard_stopping_cpus; extern cpuset_t hlt_cpus_mask; extern cpuset_t logical_cpus_mask; #endif /* SMP */ @@ -227,12 +231,13 @@ void cpu_mp_start(void); void forward_signal(struct thread *); int restart_cpus(cpuset_t); int stop_cpus(cpuset_t); -int stop_cpus_hard(cpuset_t); #if defined(__amd64__) || defined(__i386__) int suspend_cpus(cpuset_t); int resume_cpus(cpuset_t); #endif - +void stop_cpus_hard(void); +void unstop_cpus_hard(void); +void cpuhardstop_handler(void); void smp_rendezvous_action(void); extern struct mtx smp_ipi_mtx; diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index d2eb2e9cf0ec1..ed4b33563ee70 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -1195,6 +1195,36 @@ ipi_all_but_self(u_int ipi) lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } +void +cpuhardstop_handler(void) +{ + u_int cpu; + + cpu = PCPU_GET(cpuid); + + /* Just return if this is a belated NMI */ + if (!CPU_ISSET(cpu, &hard_stopping_cpus)) + return; + + savectx(&stoppcbs[cpu]); + + /* Indicate that we are stopped */ + CPU_SET_ATOMIC(cpu, &hard_stopped_cpus); + CPU_CLR_ATOMIC(cpu, &hard_stopping_cpus); + + /* Wait for restart */ + while (!CPU_ISSET(cpu, &hard_started_cpus)) { + if (cpu == 0 && cpustop_hook != NULL) { + cpustop_hook(); + cpustop_hook = NULL; + } + ia32_pause(); + } + + CPU_CLR_ATOMIC(cpu, &hard_started_cpus); + CPU_CLR_ATOMIC(cpu, &hard_stopped_cpus); +} + int ipi_nmi_handler(void) { @@ -1211,7 +1241,7 @@ ipi_nmi_handler(void) return (1); CPU_CLR_ATOMIC(cpuid, &ipi_stop_nmi_pending); - cpustop_handler(); + cpuhardstop_handler(); return (0); } @@ -1226,14 +1256,23 @@ cpustop_handler(void) cpu = PCPU_GET(cpuid); + /* Just return if this is a belated NMI */ + if (!CPU_ISSET(cpu, &stopping_cpus)) + return; + savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ - while (!CPU_ISSET(cpu, &started_cpus)) - ia32_pause(); + while (!CPU_ISSET(cpu, &started_cpus)) { + if (cpu == 0 && cpustop_hook != NULL) { + cpustop_hook(); + cpustop_hook = NULL; + } + ia32_pause(); + } CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); @@ -1241,11 +1280,6 @@ cpustop_handler(void) #if defined(__amd64__) && defined(DDB) amd64_db_resume_dbreg(); #endif - - if (cpu == 0 && cpustop_restartfunc != NULL) { - cpustop_restartfunc(); - cpustop_restartfunc = NULL; - } } /*