Index: sys/kern/subr_smp.c =================================================================== --- sys/kern/subr_smp.c (revision 220900) +++ sys/kern/subr_smp.c (working copy) @@ -53,16 +53,15 @@ #include "opt_sched.h" #ifdef SMP -volatile cpumask_t stopped_cpus; -volatile cpumask_t started_cpus; -cpumask_t idle_cpus_mask; -cpumask_t hlt_cpus_mask; -cpumask_t logical_cpus_mask; +volatile cpuset_t stopped_cpus; +volatile cpuset_t started_cpus; +cpuset_t hlt_cpus_mask; +cpuset_t logical_cpus_mask; void (*cpustop_restartfunc)(void); #endif /* This is used in modules that need to work in both SMP and UP. */ -cpumask_t all_cpus; +cpuset_t all_cpus; int mp_ncpus; /* export this for libkvm consumers. */ @@ -200,8 +199,11 @@ * */ static int -generic_stop_cpus(cpumask_t map, u_int type) +generic_stop_cpus(cpuset_t map, u_int type) { +#ifdef KTR + char cpusetbuf[CPUSETBUFSIZ]; +#endif static volatile u_int stopping_cpu = NOCPU; int i; @@ -216,7 +218,8 @@ if (!smp_started) return (0); - CTR2(KTR_SMP, "stop_cpus(%x) with %u type", map, type); + CTR2(KTR_SMP, "stop_cpus(%s) with %u type", + cpuset_strprint(cpusetbuf, &map), type); if (stopping_cpu != PCPU_GET(cpuid)) while (atomic_cmpset_int(&stopping_cpu, NOCPU, @@ -228,7 +231,7 @@ ipi_selected(map, type); i = 0; - while ((stopped_cpus & map) != map) { + while (!CPU_SUBSET(&stopped_cpus, &map)) { /* spin */ cpu_spinwait(); i++; @@ -245,14 +248,14 @@ } int -stop_cpus(cpumask_t map) +stop_cpus(cpuset_t map) { return (generic_stop_cpus(map, IPI_STOP)); } int -stop_cpus_hard(cpumask_t map) +stop_cpus_hard(cpuset_t map) { return (generic_stop_cpus(map, IPI_STOP_HARD)); @@ -260,7 +263,7 @@ #if defined(__amd64__) int -suspend_cpus(cpumask_t map) +suspend_cpus(cpuset_t map) { return (generic_stop_cpus(map, IPI_SUSPEND)); @@ -281,19 +284,22 @@ * 1: ok */ int -restart_cpus(cpumask_t map) +restart_cpus(cpuset_t map) { +#ifdef KTR + char cpusetbuf[CPUSETBUFSIZ]; +#endif if (!smp_started) return 0; - CTR1(KTR_SMP, "restart_cpus(%x)", map); + CTR1(KTR_SMP, "restart_cpus(%s)", cpuset_strprint(cpusetbuf, &map)); /* signal other cpus to restart */ - atomic_store_rel_int(&started_cpus, map); + CPU_COPY_STORE_REL(&map, &started_cpus); /* wait for each to clear its bit */ - while ((stopped_cpus & map) != 0) + while (CPU_OVERLAP(&stopped_cpus, &map)) cpu_spinwait(); return 1; @@ -349,12 +355,13 @@ } void -smp_rendezvous_cpus(cpumask_t map, +smp_rendezvous_cpus(cpuset_t map, void (* setup_func)(void *), void (* action_func)(void *), void (* teardown_func)(void *), void *arg) { + cpuset_t tmap; int i, ncpus = 0; if (!smp_started) { @@ -368,11 +375,11 @@ } CPU_FOREACH(i) { - if (((1 << i) & map) != 0) + if (CPU_ISSET(i, &map)) ncpus++; } if (ncpus == 0) - panic("ncpus is 0 with map=0x%x", map); + panic("ncpus is 0 with non-zero map"); /* obtain rendezvous lock */ mtx_lock_spin(&smp_ipi_mtx); @@ -388,10 +395,12 @@ atomic_store_rel_int(&smp_rv_waiters[0], 0); /* signal other processors, which will enter the IPI with interrupts off */ - ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS); + CPU_COPY(&map, &tmap); + CPU_CLR(curcpu, &tmap); + ipi_selected(tmap, IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ - if ((map & (1 << curcpu)) != 0) + if (CPU_ISSET(curcpu, &map)) smp_rendezvous_action(); if (teardown_func == smp_no_rendevous_barrier) @@ -416,6 +425,7 @@ struct cpu_group * smp_topo(void) { + char cpusetbuf[CPUSETBUFSIZ]; struct cpu_group *top; /* @@ -462,9 +472,10 @@ if (top->cg_count != mp_ncpus) panic("Built bad topology at %p. CPU count %d != %d", top, top->cg_count, mp_ncpus); - if (top->cg_mask != all_cpus) - panic("Built bad topology at %p. CPU mask 0x%X != 0x%X", - top, top->cg_mask, all_cpus); + if (CPU_CMP(&top->cg_mask, &all_cpus)) + panic("Built bad topology at %p. CPU mask (%s) != (%s)", + top, cpuset_strprint(cpusetbuf, &top->cg_mask), + cpuset_strprint(cpusetbuf, &all_cpus)); return (top); } @@ -489,11 +500,13 @@ smp_topo_addleaf(struct cpu_group *parent, struct cpu_group *child, int share, int count, int flags, int start) { - cpumask_t mask; + char cpusetbuf[CPUSETBUFSIZ]; + cpuset_t mask; int i; - for (mask = 0, i = 0; i < count; i++, start++) - mask |= (1 << start); + CPU_ZERO(&mask); + for (i = 0; i < count; i++, start++) + CPU_SET(start, &mask); child->cg_parent = parent; child->cg_child = NULL; child->cg_children = 0; @@ -503,10 +516,12 @@ child->cg_mask = mask; parent->cg_children++; for (; parent != NULL; parent = parent->cg_parent) { - if ((parent->cg_mask & child->cg_mask) != 0) - panic("Duplicate children in %p. mask 0x%X child 0x%X", - parent, parent->cg_mask, child->cg_mask); - parent->cg_mask |= child->cg_mask; + if (CPU_OVERLAP(&parent->cg_mask, &child->cg_mask)) + panic("Duplicate children in %p. mask (%s) child (%s)", + parent, + cpuset_strprint(cpusetbuf, &parent->cg_mask), + cpuset_strprint(cpusetbuf, &child->cg_mask)); + CPU_OR(&parent->cg_mask, &child->cg_mask); parent->cg_count += child->cg_count; } @@ -566,20 +581,20 @@ smp_topo_find(struct cpu_group *top, int cpu) { struct cpu_group *cg; - cpumask_t mask; + cpuset_t mask; int children; int i; - mask = (1 << cpu); + CPU_SMASSIGN(cpu, &mask); cg = top; for (;;) { - if ((cg->cg_mask & mask) == 0) + if (!CPU_OVERLAP(&cg->cg_mask, &mask)) return (NULL); if (cg->cg_children == 0) return (cg); children = cg->cg_children; for (i = 0, cg = cg->cg_child; i < children; cg++, i++) - if ((cg->cg_mask & mask) != 0) + if (CPU_OVERLAP(&cg->cg_mask, &mask)) break; } return (NULL); @@ -587,7 +602,7 @@ #else /* !SMP */ void -smp_rendezvous_cpus(cpumask_t map, +smp_rendezvous_cpus(cpuset_t map, void (*setup_func)(void *), void (*action_func)(void *), void (*teardown_func)(void *), Index: sys/kern/sched_ule.c =================================================================== --- sys/kern/sched_ule.c (revision 220900) +++ sys/kern/sched_ule.c (working copy) @@ -564,7 +564,7 @@ #define CPUSET_FOREACH(cpu, mask) \ for ((cpu) = 0; (cpu) <= mp_maxid; (cpu)++) \ - if ((mask) & 1 << (cpu)) + if (CPU_ISSET(cpu, &mask)) static __inline int cpu_search(struct cpu_group *cg, struct cpu_search *low, struct cpu_search *high, const int match); @@ -2650,15 +2650,16 @@ sysctl_kern_sched_topology_spec_internal(struct sbuf *sb, struct cpu_group *cg, int indent) { + char cpusetbuf[CPUSETBUFSIZ]; int i, first; sbuf_printf(sb, "%*s\n", indent, "", 1 + indent / 2, cg->cg_level); - sbuf_printf(sb, "%*s ", indent, "", - cg->cg_count, cg->cg_mask); + sbuf_printf(sb, "%*s ", indent, "", + cg->cg_count, cpuset_strprint(cpusetbuf, &cg->cg_mask)); first = TRUE; for (i = 0; i < MAXCPU; i++) { - if ((cg->cg_mask & (1 << i)) != 0) { + if (CPU_ISSET(i, &cg->cg_mask)) { if (!first) sbuf_printf(sb, ", "); else Index: sys/kern/subr_kdb.c =================================================================== --- sys/kern/subr_kdb.c (revision 220900) +++ sys/kern/subr_kdb.c (working copy) @@ -413,7 +413,8 @@ #if defined(SMP) && defined(KDB_STOPPEDPCB) SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { - if (pc->pc_curthread == thr && (stopped_cpus & pc->pc_cpumask)) + if (pc->pc_curthread == thr && + CPU_OVERLAP(&stopped_cpus, &pc->pc_cpumask)) return (KDB_STOPPEDPCB(pc)); } #endif Index: sys/kern/kern_pmc.c =================================================================== --- sys/kern/kern_pmc.c (revision 220900) +++ sys/kern/kern_pmc.c (working copy) @@ -55,7 +55,7 @@ int (*pmc_intr)(int cpu, struct trapframe *tf) = NULL; /* Bitmask of CPUs requiring servicing at hardclock time */ -volatile cpumask_t pmc_cpumask; +volatile cpuset_t pmc_cpumask; /* * A global count of SS mode PMCs. When non-zero, this means that @@ -112,7 +112,7 @@ { #ifdef SMP return (pmc_cpu_is_present(cpu) && - (hlt_cpus_mask & (1 << cpu)) == 0); + !CPU_ISSET(cpu, &hlt_cpus_mask)); #else return (1); #endif @@ -139,7 +139,7 @@ pmc_cpu_is_primary(int cpu) { #ifdef SMP - return ((logical_cpus_mask & (1 << cpu)) == 0); + return (!CPU_ISSET(cpu, &logical_cpus_mask)); #else return (1); #endif Index: sys/kern/subr_pcpu.c =================================================================== --- sys/kern/subr_pcpu.c (revision 220900) +++ sys/kern/subr_pcpu.c (working copy) @@ -87,7 +87,7 @@ KASSERT(cpuid >= 0 && cpuid < MAXCPU, ("pcpu_init: invalid cpuid %d", cpuid)); pcpu->pc_cpuid = cpuid; - pcpu->pc_cpumask = 1 << cpuid; + CPU_SMASSIGN(cpuid, &pcpu->pc_cpumask); cpuid_to_pcpu[cpuid] = pcpu; SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu); cpu_pcpu_init(pcpu, cpuid, size); Index: sys/kern/sched_4bsd.c =================================================================== --- sys/kern/sched_4bsd.c (revision 220900) +++ sys/kern/sched_4bsd.c (working copy) @@ -155,6 +155,9 @@ */ static struct runq runq_pcpu[MAXCPU]; long runq_length[MAXCPU]; + +/* Mask of idle cpus. */ +static cpuset_t idle_cpus_mask; #endif struct pcpuidlestat { @@ -1062,7 +1065,7 @@ forward_wakeup(int cpunum) { struct pcpu *pc; - cpumask_t dontuse, id, map, map2, map3, me; + cpuset_t dontuse, id, map, map2, map3, me, metest; mtx_assert(&sched_lock, MA_OWNED); @@ -1083,30 +1086,36 @@ me = PCPU_GET(cpumask); /* Don't bother if we should be doing it ourself. */ - if ((me & idle_cpus_mask) && (cpunum == NOCPU || me == (1 << cpunum))) + if (cpunum != NOCPU) + CPU_SMASSIGN(cpunum, &metest); + if (CPU_OVERLAP(&me, &idle_cpus_mask) && + (cpunum == NOCPU || !CPU_CMP(&me, &metest))) return (0); - dontuse = me | stopped_cpus | hlt_cpus_mask; - map3 = 0; + dontuse = me; + CPU_OR(&dontuse, &stopped_cpus); + CPU_OR(&dontuse, &hlt_cpus_mask); + CPU_ZERO(&map3); if (forward_wakeup_use_loop) { SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { id = pc->pc_cpumask; - if ((id & dontuse) == 0 && - pc->pc_curthread == pc->pc_idlethread) { - map3 |= id; - } + if (!CPU_OVERLAP(&id, &dontuse) && + pc->pc_curthread == pc->pc_idlethread) + CPU_OR(&map3, &id); } } if (forward_wakeup_use_mask) { - map = 0; - map = idle_cpus_mask & ~dontuse; + map = idle_cpus_mask; + CPU_NAND(&map, &dontuse); /* If they are both on, compare and use loop if different. */ if (forward_wakeup_use_loop) { - if (map != map3) { + if (CPU_CMP(&map, &map3)) { +#if 0 printf("map (%02X) != map3 (%02X)\n", map, map3); +#endif map = map3; } } @@ -1117,8 +1126,9 @@ /* If we only allow a specific CPU, then mask off all the others. */ if (cpunum != NOCPU) { KASSERT((cpunum <= mp_maxcpus),("forward_wakeup: bad cpunum.")); - map &= (1 << cpunum); + CPU_KEEP(cpunum, &map); } else { +#if 0 /* Try choose an idle die. */ if (forward_wakeup_use_htt) { map2 = (map & (map >> 1)) & 0x5555; @@ -1131,17 +1141,18 @@ if (forward_wakeup_use_single) { map = map & ((~map) + 1); } +#endif } - if (map) { + if (!CPU_EMPTY(&map)) { forward_wakeups_delivered++; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { id = pc->pc_cpumask; - if ((map & id) == 0) + if (!CPU_OVERLAP(&map, &id)) continue; if (cpu_idle_wakeup(pc->pc_cpuid)) - map &= ~id; + CPU_NAND(&map, &id); } - if (map) + if (!CPU_EMPTY(&map)) ipi_selected(map, IPI_AST); return (1); } @@ -1282,12 +1293,15 @@ kick_other_cpu(td->td_priority, cpu); } else { if (!single_cpu) { - cpumask_t me = PCPU_GET(cpumask); - cpumask_t idle = idle_cpus_mask & me; + cpuset_t me = PCPU_GET(cpumask); + cpuset_t idle = idle_cpus_mask; - if (!idle && ((flags & SRQ_INTR) == 0) && + CPU_AND(&idle, &me); +#if 0 + if (CPU_EMPTY(&idle) && ((flags & SRQ_INTR) == 0) && (idle_cpus_mask & ~(hlt_cpus_mask | me))) forwarded = forward_wakeup(cpu); +#endif } if (!forwarded) { Index: sys/kern/kern_cpuset.c =================================================================== --- sys/kern/kern_cpuset.c (revision 220900) +++ sys/kern/kern_cpuset.c (working copy) @@ -617,6 +617,68 @@ } /* + * Calculate the ffs() of the cpuset. + */ +int +cpuset_ffs(cpuset_t set) +{ + size_t i; + int cbit; + + cbit = 0; + for (i = 0; i < _NCPUWORDS; i++) { + if (set.__bits[i] != 0) { + cbit = ffsl(set.__bits[i]); + cbit += i * _NCPUBITS; + break; + } + } + return (cbit); +} + +/* + * Calculate the window function of the cpuset. + */ +int +cpuset_bitcount(cpuset_t set) +{ + size_t i; + int cbit, shifter; + + cbit = 0; + for (i = 0; i < _NCPUWORDS; i++) { + while ((shifter = ffsl(set.__bits[i])) != 0) { + set.__bits[i] >>= shifter; + cbit++; + } + } + return (cbit); +} + +/* + * Return a string representing a valid layout for a cpuset_t object. + * It expects an incoming buffer at least sized as CPUSETBUFSIZ. + */ +char * +cpuset_strprint(char *buf, cpuset_t *set) +{ + char *tbuf; + size_t i, bytesp, bufsiz; + + tbuf = buf; + bytesp = 0; + bufsiz = CPUSETBUFSIZ; + + for (i = 0; i < (_NCPUWORDS - 1); i++) { + bytesp = snprintf(tbuf, bufsiz, "%lx, ", set->__bits[i]); + bufsiz -= bytesp; + tbuf += bytesp; + } + snprintf(tbuf, bufsiz, "%lx", set->__bits[_NCPUWORDS - 1]); + return (buf); +} + +/* * Apply an anonymous mask to a single thread. */ int @@ -754,11 +816,10 @@ { cpuset_t mask; - CPU_ZERO(&mask); #ifdef SMP - mask.__bits[0] = all_cpus; + mask = all_cpus; #else - mask.__bits[0] = 1; + CPU_SMASSIGN(0, &mask); #endif if (cpuset_modify(cpuset_zero, &mask)) panic("Can't set initial cpuset mask.\n"); Index: sys/kern/kern_rmlock.c =================================================================== --- sys/kern/kern_rmlock.c (revision 220900) +++ sys/kern/kern_rmlock.c (working copy) @@ -263,7 +263,7 @@ pc = pcpu_find(curcpu); /* Check if we just need to do a proper critical_exit. */ - if (!(pc->pc_cpumask & rm->rm_writecpus)) { + if (!CPU_OVERLAP(&pc->pc_cpumask, &rm->rm_writecpus)) { critical_exit(); return (1); } @@ -325,7 +325,7 @@ critical_enter(); pc = pcpu_find(curcpu); - rm->rm_writecpus &= ~pc->pc_cpumask; + CPU_NAND(&rm->rm_writecpus, &pc->pc_cpumask); rm_tracker_add(pc, tracker); sched_pin(); critical_exit(); @@ -366,7 +366,8 @@ * Fast path to combine two common conditions into a single * conditional jump. */ - if (0 == (td->td_owepreempt | (rm->rm_writecpus & pc->pc_cpumask))) + if (0 == (td->td_owepreempt | + CPU_OVERLAP(&rm->rm_writecpus, &pc->pc_cpumask))) return (1); /* We do not have a read token and need to acquire one. */ @@ -429,17 +430,17 @@ { struct rm_priotracker *prio; struct turnstile *ts; - cpumask_t readcpus; + cpuset_t readcpus; if (rm->lock_object.lo_flags & RM_SLEEPABLE) sx_xlock(&rm->rm_lock_sx); else mtx_lock(&rm->rm_lock_mtx); - if (rm->rm_writecpus != all_cpus) { + if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) { /* Get all read tokens back */ - - readcpus = all_cpus & (all_cpus & ~rm->rm_writecpus); + readcpus = all_cpus; + CPU_NAND(&readcpus, &rm->rm_writecpus); rm->rm_writecpus = all_cpus; /* Index: sys/dev/hwpmc/hwpmc_mod.c =================================================================== --- sys/dev/hwpmc/hwpmc_mod.c (revision 220900) +++ sys/dev/hwpmc/hwpmc_mod.c (working copy) @@ -1991,7 +1991,7 @@ * had already processed the interrupt). We don't * lose the interrupt sample. */ - atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid))); + CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask); pmc_process_samples(PCPU_GET(cpuid)); break; @@ -4083,7 +4083,7 @@ done: /* mark CPU as needing processing */ - atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); + CPU_SET_ATOMIC_REL(cpu, &pmc_cpumask); return (error); } @@ -4193,7 +4193,7 @@ break; if (ps->ps_nsamples == PMC_SAMPLE_INUSE) { /* Need a rescan at a later time. */ - atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); + CPU_SET_ATOMIC_REL(cpu, &pmc_cpumask); break; } @@ -4782,7 +4782,7 @@ PMCDBG(MOD,INI,0, "%s", "cleanup"); /* switch off sampling */ - atomic_store_rel_int(&pmc_cpumask, 0); + CPU_ZERO(&pmc_cpumask); pmc_intr = NULL; sx_xlock(&pmc_sx); Index: sys/dev/xen/control/control.c =================================================================== --- sys/dev/xen/control/control.c (revision 220900) +++ sys/dev/xen/control/control.c (working copy) @@ -203,7 +203,7 @@ unsigned long max_pfn, start_info_mfn; #ifdef SMP - cpumask_t map; + cpuset_t map; /* * Bind us to CPU 0 and stop any other VCPUs. */ @@ -212,15 +212,16 @@ thread_unlock(curthread); KASSERT(PCPU_GET(cpuid) == 0, ("xen_suspend: not running on cpu 0")); - map = PCPU_GET(other_cpus) & ~stopped_cpus; - if (map) + map = PCPU_GET(other_cpus); + CPU_NAND(&map, &stopped_cpus); + if (!CPU_EMPTY(&map)) stop_cpus(map); #endif if (DEVICE_SUSPEND(root_bus) != 0) { printf("xen_suspend: device_suspend failed\n"); #ifdef SMP - if (map) + if (!CPU_EMPTY(&map)) restart_cpus(map); #endif return; @@ -289,7 +290,7 @@ thread_lock(curthread); sched_unbind(curthread); thread_unlock(curthread); - if (map) + if (!CPU_EMPTY(&map)) restart_cpus(map); #endif } Index: sys/x86/x86/local_apic.c =================================================================== --- sys/x86/x86/local_apic.c (revision 220900) +++ sys/x86/x86/local_apic.c (working copy) @@ -807,7 +807,7 @@ * and unlike other schedulers it actually schedules threads to * those CPUs. */ - if ((hlt_cpus_mask & (1 << PCPU_GET(cpuid))) != 0) + if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)) return; #endif Index: sys/geom/eli/g_eli.c =================================================================== --- sys/geom/eli/g_eli.c (revision 220900) +++ sys/geom/eli/g_eli.c (working copy) @@ -689,7 +689,7 @@ g_eli_cpu_is_disabled(int cpu) { #ifdef SMP - return ((hlt_cpus_mask & (1 << cpu)) != 0); + return (CPU_ISSET(cpu, &hlt_cpus_mask)); #else return (0); #endif Index: sys/xen/evtchn/evtchn.c =================================================================== --- sys/xen/evtchn/evtchn.c (revision 220900) +++ sys/xen/evtchn/evtchn.c (working copy) @@ -607,12 +607,6 @@ mtx_unlock_spin(&irq_mapping_update_lock); } - -static void set_affinity_irq(unsigned irq, cpumask_t dest) -{ - unsigned tcpu = ffs(dest) - 1; - rebind_irq_to_cpu(irq, tcpu); -} #endif /* Index: sys/cddl/dev/dtrace/amd64/dtrace_subr.c =================================================================== --- sys/cddl/dev/dtrace/amd64/dtrace_subr.c (revision 220900) +++ sys/cddl/dev/dtrace/amd64/dtrace_subr.c (working copy) @@ -113,12 +113,12 @@ void dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) { - cpumask_t cpus; + cpuset_t cpus; if (cpu == DTRACE_CPUALL) cpus = all_cpus; else - cpus = (cpumask_t)1 << cpu; + CPU_SMASSIGN(cpu, &cpus); smp_rendezvous_cpus(cpus, smp_no_rendevous_barrier, func, smp_no_rendevous_barrier, arg); @@ -394,7 +394,7 @@ { struct pcpu *pc; uint64_t tsc_f; - cpumask_t map; + cpuset_t map; int i; /* @@ -432,7 +432,8 @@ continue; pc = pcpu_find(i); - map = PCPU_GET(cpumask) | pc->pc_cpumask; + map = PCPU_GET(cpumask); + CPU_OR(&map, &pc->pc_cpumask); smp_rendezvous_cpus(map, dtrace_gethrtime_init_sync, dtrace_gethrtime_init_cpu, Index: sys/amd64/acpica/acpi_wakeup.c =================================================================== --- sys/amd64/acpica/acpi_wakeup.c (revision 220900) +++ sys/amd64/acpica/acpi_wakeup.c (working copy) @@ -78,7 +78,7 @@ #ifdef SMP static int acpi_wakeup_ap(struct acpi_softc *, int); -static void acpi_wakeup_cpus(struct acpi_softc *, cpumask_t); +static void acpi_wakeup_cpus(struct acpi_softc *, cpuset_t *); #endif #define WAKECODE_VADDR(sc) ((sc)->acpi_wakeaddr + (3 * PAGE_SIZE)) @@ -173,7 +173,7 @@ #define BIOS_WARM (0x0a) static void -acpi_wakeup_cpus(struct acpi_softc *sc, cpumask_t wakeup_cpus) +acpi_wakeup_cpus(struct acpi_softc *sc, cpuset_t *wakeup_cpus) { uint32_t mpbioswarmvec; int cpu; @@ -192,7 +192,7 @@ /* Wake up each AP. */ for (cpu = 1; cpu < mp_ncpus; cpu++) { - if ((wakeup_cpus & (1 << cpu)) == 0) + if (!CPU_ISSET(cpu, wakeup_cpus)) continue; if (acpi_wakeup_ap(sc, cpu) == 0) { /* restore the warmstart vector */ @@ -214,7 +214,7 @@ acpi_sleep_machdep(struct acpi_softc *sc, int state) { #ifdef SMP - cpumask_t wakeup_cpus; + cpuset_t wakeup_cpus; #endif register_t cr3, rf; ACPI_STATUS status; @@ -244,10 +244,9 @@ if (savectx(susppcbs[0])) { #ifdef SMP - if (wakeup_cpus != 0 && suspend_cpus(wakeup_cpus) == 0) { - device_printf(sc->acpi_dev, - "Failed to suspend APs: CPU mask = 0x%jx\n", - (uintmax_t)(wakeup_cpus & ~stopped_cpus)); + if (!CPU_EMPTY(&wakeup_cpus) && + suspend_cpus(wakeup_cpus) == 0) { + device_printf(sc->acpi_dev, "Failed to suspend APs\n"); goto out; } #endif @@ -282,8 +281,8 @@ PCPU_SET(switchtime, 0); PCPU_SET(switchticks, ticks); #ifdef SMP - if (wakeup_cpus != 0) - acpi_wakeup_cpus(sc, wakeup_cpus); + if (!CPU_EMPTY(&wakeup_cpus)) + acpi_wakeup_cpus(sc, &wakeup_cpus); #endif acpi_resync_clock(sc); ret = 0; @@ -291,7 +290,7 @@ out: #ifdef SMP - if (wakeup_cpus != 0) + if (!CPU_EMPTY(&wakeup_cpus)) restart_cpus(wakeup_cpus); #endif Index: sys/amd64/include/smp.h =================================================================== --- sys/amd64/include/smp.h (revision 220900) +++ sys/amd64/include/smp.h (working copy) @@ -63,17 +63,17 @@ void ipi_bitmap_handler(struct trapframe frame); void ipi_cpu(int cpu, u_int ipi); int ipi_nmi_handler(void); -void ipi_selected(cpumask_t cpus, u_int ipi); +void ipi_selected(cpuset_t cpus, u_int ipi); u_int mp_bootaddress(u_int); int mp_grab_cpu_hlt(void); void smp_cache_flush(void); void smp_invlpg(vm_offset_t addr); -void smp_masked_invlpg(cpumask_t mask, vm_offset_t addr); +void smp_masked_invlpg(cpuset_t mask, vm_offset_t addr); void smp_invlpg_range(vm_offset_t startva, vm_offset_t endva); -void smp_masked_invlpg_range(cpumask_t mask, vm_offset_t startva, +void smp_masked_invlpg_range(cpuset_t mask, vm_offset_t startva, vm_offset_t endva); void smp_invltlb(void); -void smp_masked_invltlb(cpumask_t mask); +void smp_masked_invltlb(cpuset_t mask); #endif /* !LOCORE */ #endif /* SMP */ Index: sys/amd64/include/_types.h =================================================================== --- sys/amd64/include/_types.h (revision 220900) +++ sys/amd64/include/_types.h (working copy) @@ -61,7 +61,6 @@ * Standard type definitions. */ typedef __int32_t __clock_t; /* clock()... */ -typedef unsigned int __cpumask_t; typedef __int64_t __critical_t; typedef double __double_t; typedef float __float_t; Index: sys/amd64/include/pmap.h =================================================================== --- sys/amd64/include/pmap.h (revision 220900) +++ sys/amd64/include/pmap.h (working copy) @@ -152,6 +152,7 @@ #ifndef LOCORE #include +#include #include #include @@ -251,7 +252,7 @@ struct mtx pm_mtx; pml4_entry_t *pm_pml4; /* KVA of level 4 page table */ TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */ - cpumask_t pm_active; /* active on cpus */ + cpuset_t pm_active; /* active on cpus */ /* spare u_int here due to padding */ struct pmap_statistics pm_stats; /* pmap statistics */ vm_page_t pm_root; /* spare page table pages */ Index: sys/amd64/amd64/vm_machdep.c =================================================================== --- sys/amd64/amd64/vm_machdep.c (revision 220900) +++ sys/amd64/amd64/vm_machdep.c (working copy) @@ -512,11 +512,13 @@ static void cpu_reset_proxy() { + cpuset_t tcrp; cpu_reset_proxy_active = 1; while (cpu_reset_proxy_active == 1) ; /* Wait for other cpu to see that we've started */ - stop_cpus((1< #include #include +#include #ifdef GPROF #include #endif @@ -125,7 +126,7 @@ * Local data and functions. */ -static volatile cpumask_t ipi_nmi_pending; +static volatile cpuset_t ipi_nmi_pending; /* used to hold the AP's until we are ready to release them */ static struct mtx ap_boot_mtx; @@ -161,7 +162,7 @@ static int hlt_logical_cpus; static u_int hyperthreading_cpus; /* logical cpus sharing L1 cache */ -static cpumask_t hyperthreading_cpus_mask; +static cpuset_t hyperthreading_cpus_mask; static int hyperthreading_allowed = 1; static struct sysctl_ctx_list logical_cpu_clist; static u_int bootMP_size; @@ -311,7 +312,7 @@ if (cpu_topo_probed) return; - logical_cpus_mask = 0; + CPU_ZERO(&logical_cpus_mask); if (mp_ncpus <= 1) cpu_cores = cpu_logical = 1; else if (cpu_vendor_id == CPU_VENDOR_AMD) @@ -455,7 +456,7 @@ * Always record BSP in CPU map so that the mbuf init code works * correctly. */ - all_cpus = 1; + CPU_SET(0, &all_cpus); if (mp_ncpus == 0) { /* * No CPUs were found, so this must be a UP system. Setup @@ -582,6 +583,7 @@ void init_secondary(void) { + cpuset_t tcpuset, tallcpus; struct pcpu *pc; struct nmi_pcpu *np; u_int64_t msr, cr0; @@ -713,19 +715,22 @@ CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); + tcpuset = PCPU_GET(cpumask); /* Determine if we are a logical CPU. */ /* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */ if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0) - logical_cpus_mask |= PCPU_GET(cpumask); - + CPU_OR(&logical_cpus_mask, &tcpuset); + /* Determine if we are a hyperthread. */ if (hyperthreading_cpus > 1 && PCPU_GET(apic_id) % hyperthreading_cpus != 0) - hyperthreading_cpus_mask |= PCPU_GET(cpumask); + CPU_OR(&hyperthreading_cpus_mask, &tcpuset); /* Build our map of 'other' CPUs. */ - PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); + tallcpus = all_cpus; + CPU_NAND(&tallcpus, &tcpuset); + PCPU_SET(other_cpus, tallcpus); if (bootverbose) lapic_dump("AP"); @@ -868,6 +873,7 @@ static int start_all_aps(void) { + cpuset_t tallcpus, tcpuset; vm_offset_t va = boot_address + KERNBASE; u_int64_t *pt4, *pt3, *pt2; u_int32_t mpbioswarmvec; @@ -932,11 +938,14 @@ panic("AP #%d (PHY# %d) failed!", cpu, apic_id); } - all_cpus |= (1 << cpu); /* record AP in CPU map */ + CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ - PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); + tallcpus = all_cpus; + tcpuset = PCPU_GET(cpumask); + CPU_NAND(&tallcpus, &tcpuset); + PCPU_SET(other_cpus, tallcpus); /* restore the warmstart vector */ *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; @@ -1088,20 +1097,22 @@ } static void -smp_targeted_tlb_shootdown(cpumask_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) +smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { + cpuset_t tmp; int ncpu, othercpus; othercpus = mp_ncpus - 1; - if (mask == (cpumask_t)-1) { + if (CPU_ISFULLSET(&mask)) { ncpu = othercpus; if (ncpu < 1) return; } else { - mask &= ~PCPU_GET(cpumask); - if (mask == 0) + tmp = PCPU_GET(cpumask); + CPU_NAND(&mask, &tmp); + if (CPU_EMPTY(&mask)) return; - ncpu = bitcount32(mask); + ncpu = cpuset_bitcount(mask); if (ncpu > othercpus) { /* XXX this should be a panic offence */ printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", @@ -1118,7 +1129,7 @@ smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); - if (mask == (cpumask_t)-1) + if (CPU_ISFULLSET(&mask)) ipi_all_but_self(vector); else ipi_selected(mask, vector); @@ -1197,7 +1208,7 @@ } void -smp_masked_invltlb(cpumask_t mask) +smp_masked_invltlb(cpuset_t mask) { if (smp_started) { @@ -1209,7 +1220,7 @@ } void -smp_masked_invlpg(cpumask_t mask, vm_offset_t addr) +smp_masked_invlpg(cpuset_t mask, vm_offset_t addr) { if (smp_started) { @@ -1221,7 +1232,7 @@ } void -smp_masked_invlpg_range(cpumask_t mask, vm_offset_t addr1, vm_offset_t addr2) +smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2) { if (smp_started) { @@ -1274,7 +1285,7 @@ * send an IPI to a set of cpus. */ void -ipi_selected(cpumask_t cpus, u_int ipi) +ipi_selected(cpuset_t cpus, u_int ipi) { int cpu; @@ -1284,12 +1295,12 @@ * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, cpus); + CPU_OR_ATOMIC(&ipi_nmi_pending, &cpus); - CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); - while ((cpu = ffs(cpus)) != 0) { + while ((cpu = cpuset_ffs(cpus)) != 0) { cpu--; - cpus &= ~(1 << cpu); + CPU_CLR(cpu, &cpus); + CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); ipi_send_cpu(cpu, ipi); } } @@ -1307,7 +1318,7 @@ * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, 1 << cpu); + CPU_SET_ATOMIC(cpu, &ipi_nmi_pending); CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); ipi_send_cpu(cpu, ipi); @@ -1319,6 +1330,7 @@ void ipi_all_but_self(u_int ipi) { + cpuset_t tmp; if (IPI_IS_BITMAPED(ipi)) { ipi_selected(PCPU_GET(other_cpus), ipi); @@ -1330,8 +1342,10 @@ * of help in order to understand what is the source. * Set the mask of receiving CPUs for this purpose. */ - if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus)); + if (ipi == IPI_STOP_HARD) { + tmp = PCPU_GET(other_cpus); + CPU_OR_ATOMIC(&ipi_nmi_pending, &tmp); + } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); @@ -1340,7 +1354,7 @@ int ipi_nmi_handler() { - cpumask_t cpumask; + cpuset_t cpumask; /* * As long as there is not a simple way to know about a NMI's @@ -1349,10 +1363,10 @@ * and should be handled. */ cpumask = PCPU_GET(cpumask); - if ((ipi_nmi_pending & cpumask) == 0) + if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) return (1); - atomic_clear_int(&ipi_nmi_pending, cpumask); + CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); cpustop_handler(); return (0); } @@ -1364,7 +1378,7 @@ void cpustop_handler(void) { - cpumask_t cpumask; + cpuset_t cpumask; u_int cpu; cpu = PCPU_GET(cpuid); @@ -1373,14 +1387,14 @@ savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - atomic_set_int(&stopped_cpus, cpumask); + CPU_OR_ATOMIC(&stopped_cpus, &cpumask); /* Wait for restart */ - while (!(started_cpus & cpumask)) + while (!CPU_OVERLAP(&started_cpus, &cpumask)) ia32_pause(); - atomic_clear_int(&started_cpus, cpumask); - atomic_clear_int(&stopped_cpus, cpumask); + CPU_NAND_ATOMIC(&started_cpus, &cpumask); + CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); @@ -1395,7 +1409,7 @@ void cpususpend_handler(void) { - cpumask_t cpumask; + cpuset_t cpumask; register_t cr3, rf; u_int cpu; @@ -1407,7 +1421,7 @@ if (savectx(susppcbs[cpu])) { wbinvd(); - atomic_set_int(&stopped_cpus, cpumask); + CPU_OR_ATOMIC(&stopped_cpus, &cpumask); } else { pmap_init_pat(); PCPU_SET(switchtime, 0); @@ -1415,11 +1429,11 @@ } /* Wait for resume */ - while (!(started_cpus & cpumask)) + while (!CPU_OVERLAP(&started_cpus, &cpumask)) ia32_pause(); - atomic_clear_int(&started_cpus, cpumask); - atomic_clear_int(&stopped_cpus, cpumask); + CPU_NAND_ATOMIC(&started_cpus, &cpumask); + CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); /* Restore CR3 and enable interrupts */ load_cr3(cr3); @@ -1444,6 +1458,7 @@ } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); +#if 0 static int sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS) { @@ -1472,6 +1487,7 @@ SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_hlt_cpus, "IU", "Bitmap of CPUs to halt. 101 (binary) will halt CPUs 0 and 2."); +#endif static int sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS) @@ -1484,15 +1500,15 @@ return (error); if (disable) - hlt_cpus_mask |= logical_cpus_mask; + CPU_OR(&hlt_cpus_mask, &logical_cpus_mask); else - hlt_cpus_mask &= ~logical_cpus_mask; + CPU_NAND(&hlt_cpus_mask, &logical_cpus_mask); if (! hyperthreading_allowed) - hlt_cpus_mask |= hyperthreading_cpus_mask; + CPU_OR(&hlt_cpus_mask, &hyperthreading_cpus_mask); - if ((hlt_cpus_mask & all_cpus) == all_cpus) - hlt_cpus_mask &= ~(1<<0); + if (CPU_SUBSET(&hlt_cpus_mask, &all_cpus)) + CPU_CLR(0, &hlt_cpus_mask); hlt_logical_cpus = disable; return (error); @@ -1519,18 +1535,18 @@ #endif if (allowed) - hlt_cpus_mask &= ~hyperthreading_cpus_mask; + CPU_NAND(&hlt_cpus_mask, &hyperthreading_cpus_mask); else - hlt_cpus_mask |= hyperthreading_cpus_mask; + CPU_OR(&hlt_cpus_mask, &hyperthreading_cpus_mask); - if (logical_cpus_mask != 0 && - (hlt_cpus_mask & logical_cpus_mask) == logical_cpus_mask) + if (!CPU_EMPTY(&logical_cpus_mask) && + CPU_SUBSET(&hlt_cpus_mask, &logical_cpus_mask)) hlt_logical_cpus = 1; else hlt_logical_cpus = 0; - if ((hlt_cpus_mask & all_cpus) == all_cpus) - hlt_cpus_mask &= ~(1<<0); + if (CPU_SUBSET(&hlt_cpus_mask, &all_cpus)) + CPU_CLR(0, &hlt_cpus_mask); hyperthreading_allowed = allowed; return (error); @@ -1540,7 +1556,7 @@ cpu_hlt_setup(void *dummy __unused) { - if (logical_cpus_mask != 0) { + if (!CPU_EMPTY(&logical_cpus_mask)) { TUNABLE_INT_FETCH("machdep.hlt_logical_cpus", &hlt_logical_cpus); sysctl_ctx_init(&logical_cpu_clist); @@ -1554,20 +1570,21 @@ &logical_cpus_mask, 0, ""); if (hlt_logical_cpus) - hlt_cpus_mask |= logical_cpus_mask; + CPU_OR(&hlt_cpus_mask, &logical_cpus_mask); /* * If necessary for security purposes, force * hyperthreading off, regardless of the value * of hlt_logical_cpus. */ - if (hyperthreading_cpus_mask) { + if (!CPU_EMPTY(&hyperthreading_cpus_mask)) { SYSCTL_ADD_PROC(&logical_cpu_clist, SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, "hyperthreading_allowed", CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_hyperthreading_allowed, "IU", ""); if (! hyperthreading_allowed) - hlt_cpus_mask |= hyperthreading_cpus_mask; + CPU_OR(&hlt_cpus_mask, + &hyperthreading_cpus_mask); } } } @@ -1576,7 +1593,7 @@ int mp_grab_cpu_hlt(void) { - cpumask_t mask; + cpuset_t mask; #ifdef MP_WATCHDOG u_int cpuid; #endif @@ -1589,7 +1606,7 @@ #endif retval = 0; - while (mask & hlt_cpus_mask) { + while (CPU_OVERLAP(&mask, &hlt_cpus_mask)) { retval = 1; __asm __volatile("sti; hlt" : : : "memory"); } Index: sys/amd64/amd64/pmap.c =================================================================== --- sys/amd64/amd64/pmap.c (revision 220900) +++ sys/amd64/amd64/pmap.c (working copy) @@ -581,7 +581,7 @@ PMAP_LOCK_INIT(kernel_pmap); kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); kernel_pmap->pm_root = NULL; - kernel_pmap->pm_active = -1; /* don't allow deactivation */ + CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); /* @@ -923,19 +923,20 @@ void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpumask_t cpumask, other_cpus; + cpuset_t cpumask, other_cpus; sched_pin(); - if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { + if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invlpg(va); smp_invlpg(va); } else { cpumask = PCPU_GET(cpumask); other_cpus = PCPU_GET(other_cpus); - if (pmap->pm_active & cpumask) + if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) invlpg(va); - if (pmap->pm_active & other_cpus) - smp_masked_invlpg(pmap->pm_active & other_cpus, va); + CPU_AND(&other_cpus, &pmap->pm_active); + if (!CPU_EMPTY(&other_cpus)) + smp_masked_invlpg(other_cpus, va); } sched_unpin(); } @@ -943,23 +944,23 @@ void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpumask_t cpumask, other_cpus; + cpuset_t cpumask, other_cpus; vm_offset_t addr; sched_pin(); - if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { + if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); smp_invlpg_range(sva, eva); } else { cpumask = PCPU_GET(cpumask); other_cpus = PCPU_GET(other_cpus); - if (pmap->pm_active & cpumask) + if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); - if (pmap->pm_active & other_cpus) - smp_masked_invlpg_range(pmap->pm_active & other_cpus, - sva, eva); + CPU_AND(&other_cpus, &pmap->pm_active); + if (!CPU_EMPTY(&other_cpus)) + smp_masked_invlpg_range(other_cpus, sva, eva); } sched_unpin(); } @@ -967,19 +968,20 @@ void pmap_invalidate_all(pmap_t pmap) { - cpumask_t cpumask, other_cpus; + cpuset_t cpumask, other_cpus; sched_pin(); - if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { + if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invltlb(); smp_invltlb(); } else { cpumask = PCPU_GET(cpumask); other_cpus = PCPU_GET(other_cpus); - if (pmap->pm_active & cpumask) + if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) invltlb(); - if (pmap->pm_active & other_cpus) - smp_masked_invltlb(pmap->pm_active & other_cpus); + CPU_AND(&other_cpus, &pmap->pm_active); + if (!CPU_EMPTY(&other_cpus)) + smp_masked_invltlb(other_cpus); } sched_unpin(); } @@ -995,8 +997,8 @@ } struct pde_action { - cpumask_t store; /* processor that updates the PDE */ - cpumask_t invalidate; /* processors that invalidate their TLB */ + cpuset_t store; /* processor that updates the PDE */ + cpuset_t invalidate; /* processors that invalidate their TLB */ vm_offset_t va; pd_entry_t *pde; pd_entry_t newpde; @@ -1005,18 +1007,22 @@ static void pmap_update_pde_action(void *arg) { + cpuset_t cpumask; struct pde_action *act = arg; - if (act->store == PCPU_GET(cpumask)) + cpumask = PCPU_GET(cpumask); + if (!CPU_CMP(&act->store, &cpumask)) pde_store(act->pde, act->newpde); } static void pmap_update_pde_teardown(void *arg) { + cpuset_t cpumask; struct pde_action *act = arg; - if ((act->invalidate & PCPU_GET(cpumask)) != 0) + cpumask = PCPU_GET(cpumask); + if (CPU_OVERLAP(&act->invalidate, &cpumask)) pmap_update_pde_invalidate(act->va, act->newpde); } @@ -1032,26 +1038,28 @@ pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { struct pde_action act; - cpumask_t active, cpumask; + cpuset_t active, cpumask, other_cpus; sched_pin(); cpumask = PCPU_GET(cpumask); + other_cpus = PCPU_GET(other_cpus); if (pmap == kernel_pmap) active = all_cpus; else active = pmap->pm_active; - if ((active & PCPU_GET(other_cpus)) != 0) { + if (CPU_OVERLAP(&active, &other_cpus)) { act.store = cpumask; act.invalidate = active; act.va = va; act.pde = pde; act.newpde = newpde; - smp_rendezvous_cpus(cpumask | active, + CPU_OR(&cpumask, &active); + smp_rendezvous_cpus(cpumask, smp_no_rendevous_barrier, pmap_update_pde_action, pmap_update_pde_teardown, &act); } else { pde_store(pde, newpde); - if ((active & cpumask) != 0) + if (CPU_OVERLAP(&active, &cpumask)) pmap_update_pde_invalidate(va, newpde); } sched_unpin(); @@ -1065,7 +1073,7 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) invlpg(va); } @@ -1074,7 +1082,7 @@ { vm_offset_t addr; - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); } @@ -1083,7 +1091,7 @@ pmap_invalidate_all(pmap_t pmap) { - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) invltlb(); } @@ -1099,7 +1107,7 @@ { pde_store(pde, newpde); - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) pmap_update_pde_invalidate(va, newpde); } #endif /* !SMP */ @@ -1607,7 +1615,7 @@ PMAP_LOCK_INIT(pmap); pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); pmap->pm_root = NULL; - pmap->pm_active = 0; + CPU_ZERO(&pmap->pm_active); PCPU_SET(curpmap, pmap); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -1649,7 +1657,7 @@ pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M; pmap->pm_root = NULL; - pmap->pm_active = 0; + CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -5081,17 +5089,19 @@ pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; + cpuset_t tcmsk; u_int64_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); + tcmsk = PCPU_GET(cpumask); #ifdef SMP - atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask)); - atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask)); + CPU_NAND_ATOMIC(&oldpmap->pm_active, &tcmsk); + CPU_OR_ATOMIC(&pmap->pm_active, &tcmsk); #else - oldpmap->pm_active &= ~PCPU_GET(cpumask); - pmap->pm_active |= PCPU_GET(cpumask); + CPU_NAND(&oldpmap->pm_active, &tcmsk); + CPU_OR(&pmap->pm_active, &tcmsk); #endif cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4); td->td_pcb->pcb_cr3 = cr3; Index: sys/amd64/amd64/intr_machdep.c =================================================================== --- sys/amd64/amd64/intr_machdep.c (revision 220900) +++ sys/amd64/amd64/intr_machdep.c (working copy) @@ -443,8 +443,7 @@ * allocate CPUs round-robin. */ -/* The BSP is always a valid target. */ -static cpumask_t intr_cpus = (1 << 0); +static cpuset_t intr_cpus; static int current_cpu; /* @@ -466,7 +465,7 @@ current_cpu++; if (current_cpu > mp_maxid) current_cpu = 0; - } while (!(intr_cpus & (1 << current_cpu))); + } while (!CPU_ISSET(current_cpu, &intr_cpus)); mtx_unlock_spin(&icu_lock); return (apic_id); } @@ -497,7 +496,7 @@ printf("INTR: Adding local APIC %d as a target\n", cpu_apic_ids[cpu]); - intr_cpus |= (1 << cpu); + CPU_SET(cpu, &intr_cpus); } /* @@ -510,6 +509,9 @@ struct intsrc *isrc; int i; + /* The BSP is always a valid target. */ + CPU_SMASSIGN(0, &intr_cpus); + /* Don't bother on UP. */ if (mp_ncpus == 1) return; Index: sys/sys/smp.h =================================================================== --- sys/sys/smp.h (revision 220900) +++ sys/sys/smp.h (working copy) @@ -16,6 +16,8 @@ #ifndef LOCORE +#include + /* * Topology of a NUMA or HTT system. * @@ -30,9 +32,9 @@ */ struct cpu_group { + cpuset_t cg_mask; /* Mask of cpus in this group. */ struct cpu_group *cg_parent; /* Our parent group. */ struct cpu_group *cg_child; /* Optional children groups. */ - cpumask_t cg_mask; /* Mask of cpus in this group. */ int8_t cg_count; /* Count of cpus in this group. */ int8_t cg_children; /* Number of children groups. */ int8_t cg_level; /* Shared cache level. */ @@ -71,11 +73,10 @@ extern void (*cpustop_restartfunc)(void); extern int smp_active; extern int smp_cpus; -extern volatile cpumask_t started_cpus; -extern volatile cpumask_t stopped_cpus; -extern cpumask_t idle_cpus_mask; -extern cpumask_t hlt_cpus_mask; -extern cpumask_t logical_cpus_mask; +extern volatile cpuset_t started_cpus; +extern volatile cpuset_t stopped_cpus; +extern cpuset_t hlt_cpus_mask; +extern cpuset_t logical_cpus_mask; #endif /* SMP */ extern u_int mp_maxid; @@ -83,14 +84,14 @@ extern int mp_ncpus; extern volatile int smp_started; -extern cpumask_t all_cpus; +extern cpuset_t all_cpus; /* * Macro allowing us to determine whether a CPU is absent at any given * time, thus permitting us to configure sparse maps of cpuid-dependent * (per-CPU) structures. */ -#define CPU_ABSENT(x_cpu) ((all_cpus & (1 << (x_cpu))) == 0) +#define CPU_ABSENT(x_cpu) (!CPU_ISSET(x_cpu, &all_cpus)) /* * Macros to iterate over non-absent CPUs. CPU_FOREACH() takes an @@ -159,11 +160,11 @@ void cpu_mp_start(void); void forward_signal(struct thread *); -int restart_cpus(cpumask_t); -int stop_cpus(cpumask_t); -int stop_cpus_hard(cpumask_t); +int restart_cpus(cpuset_t); +int stop_cpus(cpuset_t); +int stop_cpus_hard(cpuset_t); #if defined(__amd64__) -int suspend_cpus(cpumask_t); +int suspend_cpus(cpuset_t); #endif void smp_rendezvous_action(void); extern struct mtx smp_ipi_mtx; @@ -174,7 +175,7 @@ void (*)(void *), void (*)(void *), void *arg); -void smp_rendezvous_cpus(cpumask_t, +void smp_rendezvous_cpus(cpuset_t, void (*)(void *), void (*)(void *), void (*)(void *), Index: sys/sys/pmckern.h =================================================================== --- sys/sys/pmckern.h (revision 220900) +++ sys/sys/pmckern.h (working copy) @@ -76,7 +76,7 @@ extern struct sx pmc_sx; /* Per-cpu flags indicating availability of sampling data */ -extern volatile cpumask_t pmc_cpumask; +extern volatile cpuset_t pmc_cpumask; /* Count of system-wide sampling PMCs in existence */ extern volatile int pmc_ss_count; @@ -122,7 +122,7 @@ #define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) /* Check if a CPU has recorded samples. */ -#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(pmc_cpumask & (1 << (C)))) +#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(CPU_ISSET(C, &pmc_cpumask))) /* * Helper functions. Index: sys/sys/_rmlock.h =================================================================== --- sys/sys/_rmlock.h (revision 220900) +++ sys/sys/_rmlock.h (working copy) @@ -45,7 +45,7 @@ struct rmlock { struct lock_object lock_object; - volatile cpumask_t rm_writecpus; + volatile cpuset_t rm_writecpus; LIST_HEAD(,rm_priotracker) rm_activeReaders; union { struct mtx _rm_lock_mtx; Index: sys/sys/pcpu.h =================================================================== --- sys/sys/pcpu.h (revision 220900) +++ sys/sys/pcpu.h (working copy) @@ -37,6 +37,7 @@ #error "no assembler-serviceable parts inside" #endif +#include #include #include #include @@ -162,8 +163,8 @@ uint64_t pc_switchtime; /* cpu_ticks() at last csw */ int pc_switchticks; /* `ticks' at last csw */ u_int pc_cpuid; /* This cpu number */ - cpumask_t pc_cpumask; /* This cpu mask */ - cpumask_t pc_other_cpus; /* Mask of all other cpus */ + cpuset_t pc_cpumask; /* This cpu mask */ + cpuset_t pc_other_cpus; /* Mask of all other cpus */ SLIST_ENTRY(pcpu) pc_allcpu; struct lock_list_entry *pc_spinlocks; #ifdef KTR Index: sys/sys/cpuset.h =================================================================== --- sys/sys/cpuset.h (revision 220900) +++ sys/sys/cpuset.h (working copy) @@ -32,98 +32,8 @@ #ifndef _SYS_CPUSET_H_ #define _SYS_CPUSET_H_ -#ifdef _KERNEL -#define CPU_SETSIZE MAXCPU -#endif +#include -#define CPU_MAXSIZE 128 - -#ifndef CPU_SETSIZE -#define CPU_SETSIZE CPU_MAXSIZE -#endif - -#define _NCPUBITS (sizeof(long) * NBBY) /* bits per mask */ -#define _NCPUWORDS howmany(CPU_SETSIZE, _NCPUBITS) - -typedef struct _cpuset { - long __bits[howmany(CPU_SETSIZE, _NCPUBITS)]; -} cpuset_t; - -#define __cpuset_mask(n) ((long)1 << ((n) % _NCPUBITS)) -#define CPU_CLR(n, p) ((p)->__bits[(n)/_NCPUBITS] &= ~__cpuset_mask(n)) -#define CPU_COPY(f, t) (void)(*(t) = *(f)) -#define CPU_ISSET(n, p) (((p)->__bits[(n)/_NCPUBITS] & __cpuset_mask(n)) != 0) -#define CPU_SET(n, p) ((p)->__bits[(n)/_NCPUBITS] |= __cpuset_mask(n)) -#define CPU_ZERO(p) do { \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - (p)->__bits[__i] = 0; \ -} while (0) - -#define CPU_FILL(p) do { \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - (p)->__bits[__i] = -1; \ -} while (0) - -/* Is p empty. */ -#define CPU_EMPTY(p) __extension__ ({ \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - if ((p)->__bits[__i]) \ - break; \ - __i == _NCPUWORDS; \ -}) - -/* Is c a subset of p. */ -#define CPU_SUBSET(p, c) __extension__ ({ \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - if (((c)->__bits[__i] & \ - (p)->__bits[__i]) != \ - (c)->__bits[__i]) \ - break; \ - __i == _NCPUWORDS; \ -}) - -/* Are there any common bits between b & c? */ -#define CPU_OVERLAP(p, c) __extension__ ({ \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - if (((c)->__bits[__i] & \ - (p)->__bits[__i]) != 0) \ - break; \ - __i != _NCPUWORDS; \ -}) - -/* Compare two sets, returns 0 if equal 1 otherwise. */ -#define CPU_CMP(p, c) __extension__ ({ \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - if (((c)->__bits[__i] != \ - (p)->__bits[__i])) \ - break; \ - __i != _NCPUWORDS; \ -}) - -#define CPU_OR(d, s) do { \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - (d)->__bits[__i] |= (s)->__bits[__i]; \ -} while (0) - -#define CPU_AND(d, s) do { \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - (d)->__bits[__i] &= (s)->__bits[__i]; \ -} while (0) - -#define CPU_NAND(d, s) do { \ - __size_t __i; \ - for (__i = 0; __i < _NCPUWORDS; __i++) \ - (d)->__bits[__i] &= ~(s)->__bits[__i]; \ -} while (0) - /* * Valid cpulevel_t values. */ Index: sys/sys/types.h =================================================================== --- sys/sys/types.h (revision 220900) +++ sys/sys/types.h (working copy) @@ -143,7 +143,6 @@ #define _CLOCKID_T_DECLARED #endif -typedef __cpumask_t cpumask_t; typedef __critical_t critical_t; /* Critical section value */ typedef __int64_t daddr_t; /* disk address */