Index: kern/subr_witness.c =================================================================== --- kern/subr_witness.c (revision 285445) +++ kern/subr_witness.c (working copy) @@ -661,6 +661,9 @@ static struct witness_order_list_entry order_lists */ { "intrcnt", &lock_class_mtx_spin }, { "icu", &lock_class_mtx_spin }, +#if defined(SMP) && defined(__sparc64__) + { "ipi", &lock_class_mtx_spin }, +#endif #ifdef __i386__ { "allpmaps", &lock_class_mtx_spin }, { "descriptor tables", &lock_class_mtx_spin }, Index: sparc64/include/smp.h =================================================================== --- sparc64/include/smp.h (revision 285445) +++ sparc64/include/smp.h (working copy) @@ -39,13 +39,15 @@ #ifndef LOCORE +#include #include +#include +#include #include #include #include #include -#include #include #define IDR_BUSY 0x0000000000000001ULL @@ -96,6 +98,7 @@ struct ipi_tlb_args { }; #define ita_va ita_start +struct pcb; struct pcpu; extern struct pcb stoppcbs[]; @@ -108,8 +111,9 @@ extern cpu_ipi_selected_t *cpu_ipi_selected; typedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long); extern cpu_ipi_single_t *cpu_ipi_single; -void mp_init(u_int cpu_impl); +void mp_init(void); +extern struct mtx ipi_mtx; extern struct ipi_cache_args ipi_cache_args; extern struct ipi_rd_args ipi_rd_args; extern struct ipi_tlb_args ipi_tlb_args; @@ -139,9 +143,15 @@ ipi_all_but_self(u_int ipi) { cpuset_t cpus; + if (__predict_false(smp_started == 0)) + return; cpus = all_cpus; + sched_pin(); CPU_CLR(PCPU_GET(cpuid), &cpus); + mtx_lock_spin(&ipi_mtx); cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); + mtx_unlock_spin(&ipi_mtx); + sched_unpin(); } static __inline void @@ -148,7 +158,11 @@ static __inline void ipi_selected(cpuset_t cpus, u_int ipi) { + if (__predict_false(smp_started == 0 || CPU_EMPTY(&cpus))) + return; + mtx_lock_spin(&ipi_mtx); cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); + mtx_unlock_spin(&ipi_mtx); } static __inline void @@ -155,7 +169,11 @@ static __inline void ipi_cpu(int cpu, u_int ipi) { + if (__predict_false(smp_started == 0)) + return; + mtx_lock_spin(&ipi_mtx); cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi); + mtx_unlock_spin(&ipi_mtx); } #if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_) @@ -165,11 +183,11 @@ ipi_dcache_page_inval(void *func, vm_paddr_t pa) { struct ipi_cache_args *ica; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); ica = &ipi_cache_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ica->ica_mask = all_cpus; CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); ica->ica_pa = pa; @@ -182,11 +200,11 @@ ipi_icache_page_inval(void *func, vm_paddr_t pa) { struct ipi_cache_args *ica; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); ica = &ipi_cache_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ica->ica_mask = all_cpus; CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); ica->ica_pa = pa; @@ -199,11 +217,11 @@ ipi_rd(u_int cpu, void *func, u_long *val) { struct ipi_rd_args *ira; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); ira = &ipi_rd_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); CPU_SETOF(cpu, &ira->ira_mask); ira->ira_val = val; cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira); @@ -216,7 +234,7 @@ ipi_tlb_context_demap(struct pmap *pm) struct ipi_tlb_args *ita; cpuset_t cpus; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); cpus = pm->pm_active; @@ -227,7 +245,7 @@ ipi_tlb_context_demap(struct pmap *pm) return (NULL); } ita = &ipi_tlb_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; ita->ita_pmap = pm; cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap, @@ -241,7 +259,7 @@ ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va struct ipi_tlb_args *ita; cpuset_t cpus; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); cpus = pm->pm_active; @@ -252,7 +270,7 @@ ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va return (NULL); } ita = &ipi_tlb_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; ita->ita_pmap = pm; ita->ita_va = va; @@ -266,7 +284,7 @@ ipi_tlb_range_demap(struct pmap *pm, vm_offset_t s struct ipi_tlb_args *ita; cpuset_t cpus; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); cpus = pm->pm_active; @@ -277,7 +295,7 @@ ipi_tlb_range_demap(struct pmap *pm, vm_offset_t s return (NULL); } ita = &ipi_tlb_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; ita->ita_pmap = pm; ita->ita_start = start; @@ -292,10 +310,10 @@ ipi_wait(void *cookie) { volatile cpuset_t *mask; - if ((mask = cookie) != NULL) { + if (__predict_false((mask = cookie) != NULL)) { while (!CPU_EMPTY(mask)) ; - mtx_unlock_spin(&smp_ipi_mtx); + mtx_unlock_spin(&ipi_mtx); sched_unpin(); } } Index: sparc64/sparc64/machdep.c =================================================================== --- sparc64/sparc64/machdep.c (revision 285445) +++ sparc64/sparc64/machdep.c (working copy) @@ -499,7 +499,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_ } #ifdef SMP - mp_init(cpu_impl); + mp_init(); #endif /* Index: sparc64/sparc64/mp_machdep.c =================================================================== --- sparc64/sparc64/mp_machdep.c (revision 285445) +++ sparc64/sparc64/mp_machdep.c (working copy) @@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -113,12 +114,13 @@ struct ipi_rd_args ipi_rd_args; struct ipi_tlb_args ipi_tlb_args; struct pcb stoppcbs[MAXCPU]; +struct mtx ipi_mtx; + cpu_ipi_selected_t *cpu_ipi_selected; cpu_ipi_single_t *cpu_ipi_single; static vm_offset_t mp_tramp; static u_int cpuid_to_mid[MAXCPU]; -static int isjbus; static volatile cpuset_t shutdown_cpus; static void ap_count(phandle_t node, u_int mid, u_int cpu_impl); @@ -138,7 +140,7 @@ static cpu_ipi_single_t spitfire_ipi_single; SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); void -mp_init(u_int cpu_impl) +mp_init(void) { struct tte *tp; int i; @@ -157,24 +159,6 @@ void } for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) flush(mp_tramp + i); - - /* - * On UP systems cpu_ipi_selected() can be called while - * cpu_mp_start() wasn't so initialize these here. - */ - if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || - cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { - isjbus = 1; - cpu_ipi_selected = jalapeno_ipi_selected; - cpu_ipi_single = jalapeno_ipi_single; - } else if (cpu_impl == CPU_IMPL_SPARC64V || - cpu_impl >= CPU_IMPL_ULTRASPARCIII) { - cpu_ipi_selected = cheetah_ipi_selected; - cpu_ipi_single = cheetah_ipi_single; - } else { - cpu_ipi_selected = spitfire_ipi_selected; - cpu_ipi_single = spitfire_ipi_single; - } } static void @@ -219,7 +203,7 @@ foreach_ap(phandle_t node, void (*func)(phandle_t * Probe for other CPUs. */ void -cpu_mp_setmaxid() +cpu_mp_setmaxid(void) { CPU_SETOF(curcpu, &all_cpus); @@ -277,7 +261,26 @@ sun4u_startcpu(phandle_t cpu, void *func, u_long a void cpu_mp_start(void) { + u_int cpu_impl, isjbus; + mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); + + isjbus = 0; + cpu_impl = PCPU_GET(impl); + if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || + cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { + isjbus = 1; + cpu_ipi_selected = jalapeno_ipi_selected; + cpu_ipi_single = jalapeno_ipi_single; + } else if (cpu_impl == CPU_IMPL_SPARC64V || + cpu_impl >= CPU_IMPL_ULTRASPARCIII) { + cpu_ipi_selected = cheetah_ipi_selected; + cpu_ipi_single = cheetah_ipi_single; + } else { + cpu_ipi_selected = spitfire_ipi_selected; + cpu_ipi_single = spitfire_ipi_single; + } + intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, -1, NULL, NULL); @@ -360,7 +363,7 @@ cpu_mp_announce(void) } static void -cpu_mp_unleash(void *v) +cpu_mp_unleash(void *v __unused) { volatile struct cpu_start_args *csa; struct pcpu *pc; @@ -407,7 +410,6 @@ static void membar(StoreLoad); csa->csa_count = 0; - smp_started = 1; } void @@ -464,6 +466,9 @@ cpu_mp_bootstrap(struct pcpu *pc) while (csa->csa_count != 0) ; + if (smp_cpus == mp_ncpus) + atomic_store_rel_int(&smp_started, 1); + /* Start per-CPU event timers. */ cpu_initclocks_ap(); @@ -530,7 +535,7 @@ cpu_ipi_stop(struct trapframe *tf __unused) } static void -cpu_ipi_preempt(struct trapframe *tf) +cpu_ipi_preempt(struct trapframe *tf __unused) { sched_preempt(curthread); @@ -601,12 +606,6 @@ spitfire_ipi_single(u_int cpu, u_long d0, u_long d intr_restore(s); if ((ids & (IDR_BUSY | IDR_NACK)) == 0) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPU is also - * trying to send an IPI. - */ - DELAY(2); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI to module 0x%u\n", @@ -644,12 +643,6 @@ cheetah_ipi_single(u_int cpu, u_long d0, u_long d1 intr_restore(s); if ((ids & (IDR_BUSY | IDR_NACK)) == 0) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPU is also - * trying to send an IPI. - */ - DELAY(2); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI to module 0x%u\n", @@ -669,13 +662,12 @@ cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_l u_int cpu; int i; + KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_CHEETAH_ALL_BUSY) == 0, ("%s: outstanding dispatch", __func__)); - if (CPU_EMPTY(&cpus)) - return; ids = 0; for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { s = intr_disable(); @@ -709,12 +701,6 @@ cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_l } if (CPU_EMPTY(&cpus)) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPUs are - * also trying to send IPIs. - */ - DELAY(2 * mp_ncpus); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", @@ -754,12 +740,6 @@ jalapeno_ipi_single(u_int cpu, u_long d0, u_long d intr_restore(s); if ((ids & busynack) == 0) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPU is also - * trying to send an IPI. - */ - DELAY(2); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI to module 0x%u\n", @@ -778,13 +758,12 @@ jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_ u_int cpu; int i; + KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_CHEETAH_ALL_BUSY) == 0, ("%s: outstanding dispatch", __func__)); - if (CPU_EMPTY(&cpus)) - return; ids = 0; for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { s = intr_disable(); @@ -811,12 +790,6 @@ jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_ if ((ids & (IDR_NACK << (2 * cpuid_to_mid[cpu]))) == 0) CPU_CLR(cpu, &cpus); - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPUs are - * also trying to send IPIs. - */ - DELAY(2 * mp_ncpus); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", Index: sparc64/sparc64/tick.c =================================================================== --- sparc64/sparc64/tick.c (revision 285445) +++ sparc64/sparc64/tick.c (working copy) @@ -31,8 +31,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include -#include #include #include #include @@ -46,7 +44,6 @@ __FBSDID("$FreeBSD$"); #include #include -#include #include #include #include @@ -326,7 +323,7 @@ tick_get_timecount_up(struct timecounter *tc) static u_int stick_get_timecount_mp(struct timecounter *tc) { - u_long stick; + static u_long stick; sched_pin(); if (curcpu == 0) @@ -340,7 +337,7 @@ stick_get_timecount_mp(struct timecounter *tc) static u_int tick_get_timecount_mp(struct timecounter *tc) { - u_long tick; + static u_long tick; sched_pin(); if (curcpu == 0)