Index: sys/kern/subr_smp.c =================================================================== --- sys/kern/subr_smp.c (revision 223034) +++ sys/kern/subr_smp.c (working copy) @@ -142,7 +142,7 @@ /* Probe for MP hardware. */ if (smp_disabled != 0 || cpu_mp_probe() == 0) { mp_ncpus = 1; - all_cpus = PCPU_GET(cpumask); + CPU_SETOF(PCPU_GET(cpuid), &all_cpus); return; } @@ -708,7 +708,7 @@ { mp_ncpus = 1; mp_maxid = PCPU_GET(cpuid); - all_cpus = PCPU_GET(cpumask); + CPU_SETOF(mp_maxid, &all_cpus); KASSERT(PCPU_GET(cpuid) == 0, ("UP must have a CPU ID of zero")); } SYSINIT(cpu_mp_setvariables, SI_SUB_TUNABLES, SI_ORDER_FIRST, Index: sys/kern/subr_kdb.c =================================================================== --- sys/kern/subr_kdb.c (revision 223034) +++ sys/kern/subr_kdb.c (working copy) @@ -211,9 +211,12 @@ void kdb_panic(const char *msg) { - #ifdef SMP - stop_cpus_hard(PCPU_GET(other_cpus)); + cpuset_t other_cpus; + + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + stop_cpus_hard(other_cpus); #endif printf("KDB: panic\n"); panic("%s", msg); @@ -414,7 +417,7 @@ #if defined(SMP) && defined(KDB_STOPPEDPCB) STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { if (pc->pc_curthread == thr && - CPU_OVERLAP(&stopped_cpus, &pc->pc_cpumask)) + CPU_ISSET(pc->pc_cpuid, &stopped_cpus)) return (KDB_STOPPEDPCB(pc)); } #endif @@ -501,6 +504,7 @@ struct kdb_dbbe *be; register_t intr; #ifdef SMP + cpuset_t other_cpus; int did_stop_cpus; #endif int handled; @@ -516,8 +520,11 @@ intr = intr_disable(); #ifdef SMP - if ((did_stop_cpus = kdb_stop_cpus) != 0) - stop_cpus_hard(PCPU_GET(other_cpus)); + if ((did_stop_cpus = kdb_stop_cpus) != 0) { + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + stop_cpus_hard(other_cpus); + } #endif kdb_active++; Index: sys/kern/sched_4bsd.c =================================================================== --- sys/kern/sched_4bsd.c (revision 223034) +++ sys/kern/sched_4bsd.c (working copy) @@ -951,8 +951,7 @@ if (td->td_flags & TDF_IDLETD) { TD_SET_CAN_RUN(td); #ifdef SMP - /* Spinlock held here, assume no migration. */ - CPU_NAND(&idle_cpus_mask, PCPU_PTR(cpumask)); + CPU_CLR(PCPU_GET(cpuid), &idle_cpus_mask); #endif } else { if (TD_IS_RUNNING(td)) { @@ -1026,7 +1025,7 @@ #ifdef SMP if (td->td_flags & TDF_IDLETD) - CPU_OR(&idle_cpus_mask, PCPU_PTR(cpumask)); + CPU_SET(PCPU_GET(cpuid), &idle_cpus_mask); #endif sched_lock.mtx_lock = (uintptr_t)td; td->td_oncpu = PCPU_GET(cpuid); @@ -1055,7 +1054,8 @@ forward_wakeup(int cpunum) { struct pcpu *pc; - cpuset_t dontuse, id, map, map2, me; + cpuset_t dontuse, map, map2; + u_int id, me; int iscpuset; mtx_assert(&sched_lock, MA_OWNED); @@ -1073,27 +1073,24 @@ /* * Check the idle mask we received against what we calculated * before in the old version. - * - * Also note that sched_lock is held now, thus no migration is - * expected. */ - me = PCPU_GET(cpumask); + me = PCPU_GET(cpuid); /* Don't bother if we should be doing it ourself. */ - if (CPU_OVERLAP(&me, &idle_cpus_mask) && - (cpunum == NOCPU || CPU_ISSET(cpunum, &me))) + if (CPU_ISSET(me, &idle_cpus_mask) && + (cpunum == NOCPU || me == cpunum)) return (0); - dontuse = me; + CPU_SETOF(me, &dontuse); CPU_OR(&dontuse, &stopped_cpus); CPU_OR(&dontuse, &hlt_cpus_mask); CPU_ZERO(&map2); if (forward_wakeup_use_loop) { STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { - id = pc->pc_cpumask; - if (!CPU_OVERLAP(&id, &dontuse) && + id = pc->pc_cpuid; + if (!CPU_ISSET(id, &dontuse) && pc->pc_curthread == pc->pc_idlethread) { - CPU_OR(&map2, &id); + CPU_SET(id, &map2); } } } @@ -1125,11 +1122,11 @@ if (!CPU_EMPTY(&map)) { forward_wakeups_delivered++; STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { - id = pc->pc_cpumask; - if (!CPU_OVERLAP(&map, &id)) + id = pc->pc_cpuid; + if (!CPU_ISSET(id, &map)) continue; if (cpu_idle_wakeup(pc->pc_cpuid)) - CPU_NAND(&map, &id); + CPU_CLR(id, &map); } if (!CPU_EMPTY(&map)) ipi_selected(map, IPI_AST); @@ -1147,7 +1144,7 @@ int cpri; pcpu = pcpu_find(cpuid); - if (CPU_OVERLAP(&idle_cpus_mask, &pcpu->pc_cpumask)) { + if (CPU_ISSET(cpuid, &idle_cpus_mask)) { forward_wakeups_delivered++; if (!cpu_idle_wakeup(cpuid)) ipi_cpu(cpuid, IPI_AST); @@ -1205,10 +1202,10 @@ sched_add(struct thread *td, int flags) #ifdef SMP { - cpuset_t idle, me, tidlemsk; + cpuset_t tidlemsk; struct td_sched *ts; + u_int cpu, cpuid; int forwarded = 0; - int cpu; int single_cpu = 0; ts = td->td_sched; @@ -1271,23 +1268,17 @@ ts->ts_runq = &runq; } - if (single_cpu && (cpu != PCPU_GET(cpuid))) { + cpuid = PCPU_GET(cpuid); + if (single_cpu && cpu != cpuid) { kick_other_cpu(td->td_priority, cpu); } else { if (!single_cpu) { + tidlemsk = idle_cpus_mask; + CPU_NAND(&tidlemsk, &hlt_cpus_mask); + CPU_CLR(cpuid, &tidlemsk); - /* - * Thread spinlock is held here, assume no - * migration is possible. - */ - me = PCPU_GET(cpumask); - idle = idle_cpus_mask; - tidlemsk = idle; - CPU_AND(&idle, &me); - CPU_OR(&me, &hlt_cpus_mask); - CPU_NAND(&tidlemsk, &me); - - if (CPU_EMPTY(&idle) && ((flags & SRQ_INTR) == 0) && + if (!CPU_ISSET(cpuid, &idle_cpus_mask) && + ((flags & SRQ_INTR) == 0) && !CPU_EMPTY(&tidlemsk)) forwarded = forward_wakeup(cpu); } Index: sys/kern/kern_rmlock.c =================================================================== --- sys/kern/kern_rmlock.c (revision 223034) +++ sys/kern/kern_rmlock.c (working copy) @@ -263,7 +263,7 @@ pc = pcpu_find(curcpu); /* Check if we just need to do a proper critical_exit. */ - if (!CPU_OVERLAP(&pc->pc_cpumask, &rm->rm_writecpus)) { + if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) { critical_exit(); return (1); } @@ -325,7 +325,7 @@ critical_enter(); pc = pcpu_find(curcpu); - CPU_NAND(&rm->rm_writecpus, &pc->pc_cpumask); + CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus); rm_tracker_add(pc, tracker); sched_pin(); critical_exit(); @@ -367,7 +367,7 @@ * conditional jump. */ if (0 == (td->td_owepreempt | - CPU_OVERLAP(&rm->rm_writecpus, &pc->pc_cpumask))) + CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus))) return (1); /* We do not have a read token and need to acquire one. */ Index: sys/dev/xen/control/control.c =================================================================== --- sys/dev/xen/control/control.c (revision 223034) +++ sys/dev/xen/control/control.c (working copy) @@ -197,6 +197,7 @@ static void xctrl_suspend() { + u_int cpuid; int i, j, k, fpp; unsigned long max_pfn, start_info_mfn; @@ -210,11 +211,11 @@ thread_lock(td); sched_bind(td, 0); thread_unlock(td); - KASSERT(PCPU_GET(cpuid) == 0, ("xen_suspend: not running on cpu 0")); + cpuid = PCPU_GET(cpuid); + KASSERT(cpuid == 0, ("xen_suspend: not running on cpu 0")); - sched_pin(); - map = PCPU_GET(other_cpus); - sched_unpin(); + map = all_cpus; + CPU_CLR(cpuid, &map); CPU_NAND(&map, &stopped_cpus); if (!CPU_EMPTY(&map)) stop_cpus(map); Index: sys/sparc64/sparc64/tlb.c =================================================================== --- sys/sparc64/sparc64/tlb.c (revision 223053) +++ sys/sparc64/sparc64/tlb.c (working copy) @@ -80,7 +80,7 @@ PMAP_STATS_INC(tlb_ncontext_demap); cookie = ipi_tlb_context_demap(pm); s = intr_disable(); - if (CPU_OVERLAP(&pm->pm_active, PCPU_PTR(cpumask))) { + if (CPU_ISSET(PCPU_GET(cpuid), &pm->pm_active)) { KASSERT(pm->pm_context[curcpu] != -1, ("tlb_context_demap: inactive pmap?")); stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_DMMU_DEMAP, 0); @@ -101,7 +101,7 @@ PMAP_STATS_INC(tlb_npage_demap); cookie = ipi_tlb_page_demap(pm, va); s = intr_disable(); - if (CPU_OVERLAP(&pm->pm_active, PCPU_PTR(cpumask))) { + if (CPU_ISSET(PCPU_GET(cpuid), &pm->pm_active)) { KASSERT(pm->pm_context[curcpu] != -1, ("tlb_page_demap: inactive pmap?")); if (pm == kernel_pmap) @@ -128,7 +128,7 @@ PMAP_STATS_INC(tlb_nrange_demap); cookie = ipi_tlb_range_demap(pm, start, end); s = intr_disable(); - if (CPU_OVERLAP(&pm->pm_active, PCPU_PTR(cpumask))) { + if (CPU_ISSET(PCPU_GET(cpuid), &pm->pm_active)) { KASSERT(pm->pm_context[curcpu] != -1, ("tlb_range_demap: inactive pmap?")); if (pm == kernel_pmap) Index: sys/sparc64/sparc64/pmap.c =================================================================== --- sys/sparc64/sparc64/pmap.c (revision 223053) +++ sys/sparc64/sparc64/pmap.c (working copy) @@ -2230,7 +2230,7 @@ PCPU_SET(tlb_ctx, context + 1); pm->pm_context[curcpu] = context; - CPU_OR(&pm->pm_active, PCPU_PTR(cpumask)); + CPU_SET(PCPU_GET(cpuid), &pm->pm_active); PCPU_SET(pmap, pm); stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); Index: sys/sparc64/sparc64/mp_machdep.c =================================================================== --- sys/sparc64/sparc64/mp_machdep.c (revision 223053) +++ sys/sparc64/sparc64/mp_machdep.c (working copy) @@ -283,7 +283,6 @@ void cpu_mp_start(void) { - cpuset_t ocpus; mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); @@ -300,9 +299,6 @@ KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, ("%s: can only IPI a maximum of %d JBus-CPUs", __func__, IDR_JALAPENO_MAX_BN_PAIRS)); - ocpus = all_cpus; - CPU_CLR(curcpu, &ocpus); - PCPU_SET(other_cpus, ocpus); smp_active = 1; } @@ -424,7 +420,6 @@ void cpu_mp_bootstrap(struct pcpu *pc) { - cpuset_t ocpus; volatile struct cpu_start_args *csa; csa = &cpu_start_args; @@ -466,9 +461,6 @@ smp_cpus++; KASSERT(curthread != NULL, ("%s: curthread", __func__)); - ocpus = all_cpus; - CPU_CLR(curcpu, &ocpus); - PCPU_SET(other_cpus, ocpus); printf("SMP: AP CPU #%d Launched!\n", curcpu); csa->csa_count--; @@ -491,13 +483,14 @@ int i; critical_enter(); - shutdown_cpus = PCPU_GET(other_cpus); + shutdown_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); cpus = shutdown_cpus; /* XXX: Stop all the CPUs which aren't already. */ if (CPU_CMP(&stopped_cpus, &cpus)) { - /* pc_other_cpus is just a flat "on" mask without curcpu. */ + /* cpus is just a flat "on" mask without curcpu. */ CPU_NAND(&cpus, &stopped_cpus); stop_cpus(cpus); } @@ -520,23 +513,23 @@ static void cpu_ipi_stop(struct trapframe *tf) { - cpuset_t tcmask; + u_int cpuid; CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); sched_pin(); savectx(&stoppcbs[curcpu]); - tcmask = PCPU_GET(cpumask); - CPU_OR_ATOMIC(&stopped_cpus, &tcmask); - while (!CPU_OVERLAP(&started_cpus, &tcmask)) { - if (CPU_OVERLAP(&shutdown_cpus, &tcmask)) { - CPU_NAND_ATOMIC(&shutdown_cpus, &tcmask); + cpuid = PCPU_GET(cpuid); + CPU_SET_ATOMIC(cpuid, &stopped_cpus); + while (!CPU_ISSET(cpuid, &started_cpus)) { + if (CPU_ISSET(cpuid, &shutdown_cpus)) { + CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); (void)intr_disable(); for (;;) ; } } - CPU_NAND_ATOMIC(&started_cpus, &tcmask); - CPU_NAND_ATOMIC(&stopped_cpus, &tcmask); + CPU_CLR_ATOMIC(cpuid, &started_cpus); + CPU_CLR_ATOMIC(cpuid, &stopped_cpus); sched_unpin(); CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); } Index: sys/sparc64/include/smp.h =================================================================== --- sys/sparc64/include/smp.h (revision 223053) +++ sys/sparc64/include/smp.h (working copy) @@ -136,8 +136,11 @@ static __inline void ipi_all_but_self(u_int ipi) { + cpuset_t other_cpus; - cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)tl_ipi_level, ipi); + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + cpu_ipi_selected(other_cpus, 0, (u_long)tl_ipi_level, ipi); } static __inline void @@ -159,6 +162,7 @@ static __inline void * ipi_dcache_page_inval(void *func, vm_paddr_t pa) { + cpuset_t other_cpus; struct ipi_cache_args *ica; if (smp_cpus == 1) @@ -168,13 +172,16 @@ mtx_lock_spin(&ipi_mtx); ica->ica_mask = all_cpus; ica->ica_pa = pa; - cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)func, (u_long)ica); + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + cpu_ipi_selected(other_cpus, 0, (u_long)func, (u_long)ica); return (&ica->ica_mask); } static __inline void * ipi_icache_page_inval(void *func, vm_paddr_t pa) { + cpuset_t other_cpus; struct ipi_cache_args *ica; if (smp_cpus == 1) @@ -184,7 +191,9 @@ mtx_lock_spin(&ipi_mtx); ica->ica_mask = all_cpus; ica->ica_pa = pa; - cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)func, (u_long)ica); + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + cpu_ipi_selected(other_cpus, 0, (u_long)func, (u_long)ica); return (&ica->ica_mask); } @@ -198,7 +207,7 @@ sched_pin(); ira = &ipi_rd_args; mtx_lock_spin(&ipi_mtx); - ira->ira_mask = PCPU_GET(cpumask); + CPU_SETOF(PCPU_GET(cpuid), &ira->ira_mask); CPU_SET(cpu, &ira->ira_mask); ira->ira_val = val; cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira); @@ -215,7 +224,8 @@ return (NULL); sched_pin(); cpus = pm->pm_active; - CPU_AND(&cpus, PCPU_PTR(other_cpus)); + CPU_AND(&cpus, &all_cpus); + CPU_CLR(PCPU_GET(cpuid), &cpus); if (CPU_EMPTY(&cpus)) { sched_unpin(); return (NULL); @@ -223,7 +233,7 @@ ita = &ipi_tlb_args; mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; - CPU_OR(&ita->ita_mask, PCPU_PTR(cpumask)); + CPU_SET(PCPU_GET(cpuid), &ita->ita_mask); ita->ita_pmap = pm; cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap, (u_long)ita); @@ -240,7 +250,8 @@ return (NULL); sched_pin(); cpus = pm->pm_active; - CPU_AND(&cpus, PCPU_PTR(other_cpus)); + CPU_AND(&cpus, &all_cpus); + CPU_CLR(PCPU_GET(cpuid), &cpus); if (CPU_EMPTY(&cpus)) { sched_unpin(); return (NULL); @@ -248,7 +259,7 @@ ita = &ipi_tlb_args; mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; - CPU_OR(&ita->ita_mask, PCPU_PTR(cpumask)); + CPU_SET(PCPU_GET(cpuid), &ita->ita_mask); ita->ita_pmap = pm; ita->ita_va = va; cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_page_demap, (u_long)ita); @@ -265,7 +276,8 @@ return (NULL); sched_pin(); cpus = pm->pm_active; - CPU_AND(&cpus, PCPU_PTR(other_cpus)); + CPU_AND(&cpus, &all_cpus); + CPU_CLR(PCPU_GET(cpuid), &cpus); if (CPU_EMPTY(&cpus)) { sched_unpin(); return (NULL); @@ -273,7 +285,7 @@ ita = &ipi_tlb_args; mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; - CPU_OR(&ita->ita_mask, PCPU_PTR(cpumask)); + CPU_SET(PCPU_GET(cpuid), &ita->ita_mask); ita->ita_pmap = pm; ita->ita_start = start; ita->ita_end = end; @@ -288,7 +300,7 @@ volatile cpuset_t *mask; if ((mask = cookie) != NULL) { - CPU_NAND_ATOMIC(mask, PCPU_PTR(cpumask)); + CPU_CLR_ATOMIC(PCPU_GET(cpuid), mask); while (!CPU_EMPTY(mask)) ; mtx_unlock_spin(&ipi_mtx);