Index: kern/subr_smp.c =================================================================== --- kern/subr_smp.c (revision 207451) +++ kern/subr_smp.c (working copy) @@ -180,7 +180,7 @@ id = td->td_oncpu; if (id == NOCPU) return; - ipi_selected(1 << id, IPI_AST); + ipi_cpu(id, IPI_AST); } /* @@ -317,7 +317,7 @@ CTR1(KTR_SMP, "restart_cpus(%x)", map); /* signal other cpus to restart */ - atomic_store_rel_int(&started_cpus, map); + atomic_store_rel_long(&started_cpus, map); /* wait for each to clear its bit */ while ((stopped_cpus & map) != 0) @@ -395,10 +395,10 @@ } for (i = 0; i <= mp_maxid; i++) - if (((1 << i) & map) != 0 && !CPU_ABSENT(i)) + if ((cputomask(i) & map) != 0 && !CPU_ABSENT(i)) ncpus++; if (ncpus == 0) - panic("ncpus is 0 with map=0x%x", map); + panic("ncpus is 0 with map=0x%lx", map); /* obtain rendezvous lock */ mtx_lock_spin(&smp_ipi_mtx); @@ -414,10 +414,10 @@ atomic_store_rel_int(&smp_rv_waiters[0], 0); /* signal other processors, which will enter the IPI with interrupts off */ - ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS); + ipi_selected(map & ~cputomask(curcpu), IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ - if ((map & (1 << curcpu)) != 0) + if ((map & cputomask(curcpu)) != 0) smp_rendezvous_action(); if (teardown_func == smp_no_rendevous_barrier) @@ -489,7 +489,7 @@ panic("Built bad topology at %p. CPU count %d != %d", top, top->cg_count, mp_ncpus); if (top->cg_mask != all_cpus) - panic("Built bad topology at %p. CPU mask 0x%X != 0x%X", + panic("Built bad topology at %p. CPU mask 0x%lX != 0x%lX", top, top->cg_mask, all_cpus); return (top); } @@ -502,7 +502,7 @@ top = &group[0]; top->cg_parent = NULL; top->cg_child = NULL; - top->cg_mask = (1 << mp_ncpus) - 1; + top->cg_mask = cputomask(mp_ncpus) - 1; top->cg_count = mp_ncpus; top->cg_children = 0; top->cg_level = CG_SHARE_NONE; @@ -519,7 +519,7 @@ int i; for (mask = 0, i = 0; i < count; i++, start++) - mask |= (1 << start); + mask |= cputomask(start); child->cg_parent = parent; child->cg_child = NULL; child->cg_children = 0; @@ -530,7 +530,7 @@ parent->cg_children++; for (; parent != NULL; parent = parent->cg_parent) { if ((parent->cg_mask & child->cg_mask) != 0) - panic("Duplicate children in %p. mask 0x%X child 0x%X", + panic("Duplicate children in %p. mask 0x%lX child 0x%lX", parent, parent->cg_mask, child->cg_mask); parent->cg_mask |= child->cg_mask; parent->cg_count += child->cg_count; @@ -596,7 +596,7 @@ int children; int i; - mask = (1 << cpu); + mask = cputomask(cpu); cg = top; for (;;) { if ((cg->cg_mask & mask) == 0) Index: kern/sched_ule.c =================================================================== --- kern/sched_ule.c (revision 207451) +++ kern/sched_ule.c (working copy) @@ -552,7 +552,7 @@ #define CPUSET_FOREACH(cpu, mask) \ for ((cpu) = 0; (cpu) <= mp_maxid; (cpu)++) \ - if ((mask) & 1 << (cpu)) + if ((mask) & cputomask(cpu)) static __inline int cpu_search(struct cpu_group *cg, struct cpu_search *low, struct cpu_search *high, const int match); @@ -851,7 +851,7 @@ * IPI the target cpu to force it to reschedule with the new * workload. */ - ipi_selected(1 << TDQ_ID(low), IPI_PREEMPT); + ipi_cpu(TDQ_ID(low), IPI_PREEMPT); } tdq_unlock_pair(high, low); return (moved); @@ -974,7 +974,7 @@ return; } tdq->tdq_ipipending = 1; - ipi_selected(1 << cpu, IPI_PREEMPT); + ipi_cpu(cpu, IPI_PREEMPT); } /* @@ -2414,7 +2414,7 @@ cpu = ts->ts_cpu; ts->ts_cpu = sched_pickcpu(td, 0); if (cpu != PCPU_GET(cpuid)) - ipi_selected(1 << cpu, IPI_PREEMPT); + ipi_cpu(cpu, IPI_PREEMPT); #endif } @@ -2643,11 +2643,11 @@ sbuf_printf(sb, "%*s\n", indent, "", indent, cg->cg_level); - sbuf_printf(sb, "%*s ", indent, "", + sbuf_printf(sb, "%*s ", indent, "", cg->cg_count, cg->cg_mask); first = TRUE; for (i = 0; i < MAXCPU; i++) { - if ((cg->cg_mask & (1 << i)) != 0) { + if ((cg->cg_mask & cputomask(i)) != 0) { if (!first) sbuf_printf(sb, ", "); else Index: kern/kern_ktr.c =================================================================== --- kern/kern_ktr.c (revision 207451) +++ kern/kern_ktr.c (working copy) @@ -82,9 +82,9 @@ SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RD, 0, "KTR options"); -int ktr_cpumask = KTR_CPUMASK; -TUNABLE_INT("debug.ktr.cpumask", &ktr_cpumask); -SYSCTL_INT(_debug_ktr, OID_AUTO, cpumask, CTLFLAG_RW, +long ktr_cpumask = KTR_CPUMASK; +TUNABLE_LONG("debug.ktr.cpumask", &ktr_cpumask); +SYSCTL_LONG(_debug_ktr, OID_AUTO, cpumask, CTLFLAG_RW, &ktr_cpumask, 0, "Bitmask of CPUs on which KTR logging is enabled"); int ktr_mask = KTR_MASK; @@ -211,7 +211,7 @@ if ((ktr_mask & mask) == 0) return; cpu = KTR_CPU; - if (((1 << cpu) & ktr_cpumask) == 0) + if ((cputomask(cpu) & ktr_cpumask) == 0) return; #if defined(KTR_VERBOSE) || defined(KTR_ALQ) td = curthread; Index: kern/kern_pmc.c =================================================================== --- kern/kern_pmc.c (revision 207451) +++ kern/kern_pmc.c (working copy) @@ -34,6 +34,7 @@ #include "opt_hwpmc_hooks.h" #include +#include #include #include #include @@ -110,7 +111,7 @@ { #ifdef SMP return (pmc_cpu_is_present(cpu) && - (hlt_cpus_mask & (1 << cpu)) == 0); + (hlt_cpus_mask & cputomask(cpu)) == 0); #else return (1); #endif @@ -137,7 +138,7 @@ pmc_cpu_is_primary(int cpu) { #ifdef SMP - return ((logical_cpus_mask & (1 << cpu)) == 0); + return ((logical_cpus_mask & cputomask(cpu)) == 0); #else return (1); #endif Index: kern/subr_pcpu.c =================================================================== --- kern/subr_pcpu.c (revision 207451) +++ kern/subr_pcpu.c (working copy) @@ -88,7 +88,7 @@ KASSERT(cpuid >= 0 && cpuid < MAXCPU, ("pcpu_init: invalid cpuid %d", cpuid)); pcpu->pc_cpuid = cpuid; - pcpu->pc_cpumask = 1 << cpuid; + pcpu->pc_cpumask = cputomask(cpuid); cpuid_to_pcpu[cpuid] = pcpu; SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu); cpu_pcpu_init(pcpu, cpuid, size); Index: kern/sched_4bsd.c =================================================================== --- kern/sched_4bsd.c (revision 207451) +++ kern/sched_4bsd.c (working copy) @@ -1086,7 +1086,7 @@ me = PCPU_GET(cpumask); /* Don't bother if we should be doing it ourself. */ - if ((me & idle_cpus_mask) && (cpunum == NOCPU || me == (1 << cpunum))) + if ((me & idle_cpus_mask) && (cpunum == NOCPU || me == cputomask(cpunum))) return (0); dontuse = me | stopped_cpus | hlt_cpus_mask; @@ -1108,7 +1108,7 @@ /* If they are both on, compare and use loop if different. */ if (forward_wakeup_use_loop) { if (map != map3) { - printf("map (%02X) != map3 (%02X)\n", map, + printf("map (%02X) != map3 (%02lX)\n", map, map3); map = map3; } @@ -1120,7 +1120,7 @@ /* If we only allow a specific CPU, then mask off all the others. */ if (cpunum != NOCPU) { KASSERT((cpunum <= mp_maxcpus),("forward_wakeup: bad cpunum.")); - map &= (1 << cpunum); + map &= cputomask(cpunum); } else { /* Try choose an idle die. */ if (forward_wakeup_use_htt) { @@ -1669,7 +1669,7 @@ td->td_flags |= TDF_NEEDRESCHED; if (td != curthread) - ipi_selected(1 << cpu, IPI_AST); + ipi_cpu(cpu, IPI_AST); break; default: break; Index: dev/hwpmc/hwpmc_mod.c =================================================================== --- dev/hwpmc/hwpmc_mod.c (revision 207451) +++ dev/hwpmc/hwpmc_mod.c (working copy) @@ -1991,7 +1991,7 @@ * had already processed the interrupt). We don't * lose the interrupt sample. */ - atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid))); + atomic_clear_long(&pmc_cpumask, (1 << PCPU_GET(cpuid))); pmc_process_samples(PCPU_GET(cpuid)); break; @@ -4086,7 +4086,7 @@ done: /* mark CPU as needing processing */ - atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); + atomic_set_rel_long(&pmc_cpumask, (1 << cpu)); return (error); } @@ -4196,7 +4196,7 @@ break; if (ps->ps_nsamples == PMC_SAMPLE_INUSE) { /* Need a rescan at a later time. */ - atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); + atomic_set_rel_long(&pmc_cpumask, (1 << cpu)); break; } @@ -4785,7 +4785,7 @@ PMCDBG(MOD,INI,0, "%s", "cleanup"); /* switch off sampling */ - atomic_store_rel_int(&pmc_cpumask, 0); + atomic_store_rel_long(&pmc_cpumask, 0); pmc_intr = NULL; sx_xlock(&pmc_sx); Index: geom/eli/g_eli.c =================================================================== --- geom/eli/g_eli.c (revision 207451) +++ geom/eli/g_eli.c (working copy) @@ -499,7 +499,7 @@ g_eli_cpu_is_disabled(int cpu) { #ifdef SMP - return ((hlt_cpus_mask & (1 << cpu)) != 0); + return ((hlt_cpus_mask & cputomask(cpu)) != 0); #else return (0); #endif Index: i386/include/smp.h =================================================================== --- i386/include/smp.h (revision 207451) +++ i386/include/smp.h (working copy) @@ -85,6 +85,9 @@ #define NR_IPIS 2 #endif + +#define ipi_cpu(_c, _i) ipi_selected(cputomask(_c), _i) + #endif /* !LOCORE */ #endif /* SMP */ Index: i386/include/_types.h =================================================================== --- i386/include/_types.h (revision 207451) +++ i386/include/_types.h (working copy) @@ -74,7 +74,7 @@ * Standard type definitions. */ typedef unsigned long __clock_t; /* clock()... */ -typedef unsigned int __cpumask_t; +typedef unsigned long __cpumask_t; typedef __int32_t __critical_t; typedef long double __double_t; typedef long double __float_t; Index: i386/i386/vm_machdep.c =================================================================== --- i386/i386/vm_machdep.c (revision 207451) +++ i386/i386/vm_machdep.c (working copy) @@ -609,7 +609,7 @@ /* Restart CPU #0. */ /* XXX: restart_cpus(1 << 0); */ - atomic_store_rel_int(&started_cpus, (1 << 0)); + atomic_store_rel_long(&started_cpus, (1 << 0)); cnt = 0; while (cpu_reset_proxy_active == 0 && cnt < 10000000) Index: i386/i386/mp_machdep.c =================================================================== --- i386/i386/mp_machdep.c (revision 207451) +++ i386/i386/mp_machdep.c (working copy) @@ -1306,7 +1306,7 @@ * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, cpus); + atomic_set_long(&ipi_nmi_pending, cpus); CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); while ((cpu = ffs(cpus)) != 0) { @@ -1349,7 +1349,7 @@ * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus)); + atomic_set_long(&ipi_nmi_pending, PCPU_GET(other_cpus)); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } @@ -1369,7 +1369,7 @@ if ((ipi_nmi_pending & cpumask) == 0) return (1); - atomic_clear_int(&ipi_nmi_pending, cpumask); + atomic_clear_long(&ipi_nmi_pending, cpumask); cpustop_handler(); return (0); } @@ -1387,14 +1387,14 @@ savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - atomic_set_int(&stopped_cpus, cpumask); + atomic_set_long(&stopped_cpus, cpumask); /* Wait for restart */ while (!(started_cpus & cpumask)) ia32_pause(); - atomic_clear_int(&started_cpus, cpumask); - atomic_clear_int(&stopped_cpus, cpumask); + atomic_clear_long(&started_cpus, cpumask); + atomic_clear_long(&stopped_cpus, cpumask); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); Index: cddl/dev/dtrace/amd64/dtrace_subr.c =================================================================== --- cddl/dev/dtrace/amd64/dtrace_subr.c (revision 207451) +++ cddl/dev/dtrace/amd64/dtrace_subr.c (working copy) @@ -120,14 +120,14 @@ if (cpu == DTRACE_CPUALL) cpus = all_cpus; else - cpus = (cpumask_t) (1 << cpu); + cpus = cputomask(cpu); /* If the current CPU is in the set, call the function directly: */ - if ((cpus & (1 << curcpu)) != 0) { + if ((cpus & cputomask(curcpu)) != 0) { (*func)(arg); /* Mask the current CPU from the set */ - cpus &= ~(1 << curcpu); + cpus &= ~cputomask(curcpu); } /* If there are any CPUs in the set, cross-call to those CPUs */ Index: amd64/include/smp.h =================================================================== --- amd64/include/smp.h (revision 207451) +++ amd64/include/smp.h (working copy) @@ -54,6 +54,7 @@ void init_secondary(void); int ipi_nmi_handler(void); void ipi_selected(cpumask_t cpus, u_int ipi); +void ipi_cpu(int cpu, u_int ipi); void ipi_all_but_self(u_int ipi); void ipi_bitmap_handler(struct trapframe frame); u_int mp_bootaddress(u_int); Index: amd64/include/param.h =================================================================== --- amd64/include/param.h (revision 207451) +++ amd64/include/param.h (working copy) @@ -61,7 +61,7 @@ #endif #if defined(SMP) || defined(KLD_MODULE) -#define MAXCPU 32 +#define MAXCPU 64 #else #define MAXCPU 1 #endif Index: amd64/include/_types.h =================================================================== --- amd64/include/_types.h (revision 207451) +++ amd64/include/_types.h (working copy) @@ -61,7 +61,7 @@ * Standard type definitions. */ typedef __int32_t __clock_t; /* clock()... */ -typedef unsigned int __cpumask_t; +typedef unsigned long __cpumask_t; typedef __int64_t __critical_t; typedef double __double_t; typedef float __float_t; Index: amd64/include/pmap.h =================================================================== --- amd64/include/pmap.h (revision 207451) +++ amd64/include/pmap.h (working copy) @@ -244,10 +244,9 @@ struct mtx pm_mtx; pml4_entry_t *pm_pml4; /* KVA of level 4 page table */ TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */ - u_int pm_active; /* active on cpus */ + cpumask_t pm_active; /* active on cpus */ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ u_int pm_retries; - /* spare u_int here due to padding */ struct pmap_statistics pm_stats; /* pmap statistics */ vm_page_t pm_root; /* spare page table pages */ }; Index: amd64/amd64/vm_machdep.c =================================================================== --- amd64/amd64/vm_machdep.c (revision 207451) +++ amd64/amd64/vm_machdep.c (working copy) @@ -536,7 +536,7 @@ printf("cpu_reset: Restarting BSP\n"); /* Restart CPU #0. */ - atomic_store_rel_int(&started_cpus, 1 << 0); + atomic_store_rel_long(&started_cpus, cputomask(0)); cnt = 0; while (cpu_reset_proxy_active == 0 && cnt < 10000000) Index: amd64/amd64/mptable.c =================================================================== --- amd64/amd64/mptable.c (revision 207451) +++ amd64/amd64/mptable.c (working copy) @@ -888,13 +888,13 @@ * already in the table, then kill the fixup. */ for (id = 0; id <= MAX_LAPIC_ID; id++) { - if ((id_mask & 1 << id) == 0) + if ((id_mask & (1ul << id)) == 0) continue; /* First, make sure we are on a logical_cpus boundary. */ if (id % logical_cpus != 0) return; for (i = id + 1; i < id + logical_cpus; i++) - if ((id_mask & 1 << i) != 0) + if ((id_mask & (1ul << i)) != 0) return; } @@ -911,7 +911,7 @@ i, id); lapic_create(i, 0); } - id_mask &= ~(1 << id); + id_mask &= ~(1ul << id); } } #endif /* MPTABLE_FORCE_HTT */ Index: amd64/amd64/cpu_switch.S =================================================================== --- amd64/amd64/cpu_switch.S (revision 207451) +++ amd64/amd64/cpu_switch.S (working copy) @@ -78,7 +78,7 @@ /* release bit from old pm_active */ movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */ movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */ - LK btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ + LK btrq %rax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ movq TD_PCB(%rsi),%r8 /* newtd->td_proc */ movq PCB_CR3(%r8),%rdx movq %rdx,%cr3 /* new address space */ @@ -142,13 +142,13 @@ /* Release bit from old pmap->pm_active */ movq TD_PROC(%rdi), %rcx /* oldproc */ movq P_VMSPACE(%rcx), %rcx - LK btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) /* clear old */ + LK btrq %rax, VM_PMAP+PM_ACTIVE(%rcx) /* clear old */ SETLK %rdx, TD_LOCK(%rdi) /* Release the old thread */ swact: /* Set bit in new pmap->pm_active */ movq TD_PROC(%rsi),%rdx /* newproc */ movq P_VMSPACE(%rdx), %rdx - LK btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ + LK btsq %rax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ sw1: #if defined(SCHED_ULE) && defined(SMP) Index: amd64/amd64/pmap.c =================================================================== --- amd64/amd64/pmap.c (revision 207451) +++ amd64/amd64/pmap.c (working copy) @@ -567,7 +567,7 @@ PMAP_LOCK_INIT(kernel_pmap); kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); kernel_pmap->pm_root = NULL; - kernel_pmap->pm_active = -1; /* don't allow deactivation */ + kernel_pmap->pm_active = (cpumask_t)-1; /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); /* @@ -926,8 +926,8 @@ void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - u_int cpumask; - u_int other_cpus; + cpumask_t cpumask; + cpumask_t other_cpus; sched_pin(); if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { @@ -947,8 +947,8 @@ void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - u_int cpumask; - u_int other_cpus; + cpumask_t cpumask; + cpumask_t other_cpus; vm_offset_t addr; sched_pin(); @@ -4957,8 +4957,8 @@ oldpmap = PCPU_GET(curpmap); #ifdef SMP if (oldpmap) /* XXX FIXME */ - atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask)); - atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask)); + atomic_clear_long(&oldpmap->pm_active, PCPU_GET(cpumask)); + atomic_set_long(&pmap->pm_active, PCPU_GET(cpumask)); #else if (oldpmap) /* XXX FIXME */ oldpmap->pm_active &= ~PCPU_GET(cpumask); Index: amd64/amd64/mp_machdep.c =================================================================== --- amd64/amd64/mp_machdep.c (revision 207451) +++ amd64/amd64/mp_machdep.c (working copy) @@ -112,7 +112,7 @@ * Local data and functions. */ -static u_int logical_cpus; +static cpumask_t logical_cpus; static volatile cpumask_t ipi_nmi_pending; /* used to hold the AP's until we are ready to release them */ @@ -874,7 +874,7 @@ panic("AP #%d (PHY# %d) failed!", cpu, apic_id); } - all_cpus |= (1 << cpu); /* record AP in CPU map */ + all_cpus |= cputomask(cpu); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ @@ -996,27 +996,16 @@ static void smp_targeted_tlb_shootdown(cpumask_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { - int ncpu, othercpus; + int cpu, ncpu, othercpus; othercpus = mp_ncpus - 1; if (mask == (u_int)-1) { - ncpu = othercpus; - if (ncpu < 1) + if (othercpus < 1) return; } else { mask &= ~PCPU_GET(cpumask); if (mask == 0) return; - ncpu = bitcount32(mask); - if (ncpu > othercpus) { - /* XXX this should be a panic offence */ - printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", - ncpu, othercpus); - ncpu = othercpus; - } - /* XXX should be a panic, implied by mask == 0 above */ - if (ncpu < 1) - return; } if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); @@ -1024,10 +1013,18 @@ smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); - if (mask == (u_int)-1) + if (mask == (cpumask_t)-1) { + ncpu = othercpus; ipi_all_but_self(vector); - else - ipi_selected(mask, vector); + } else { + ncpu = 0; + while ((cpu = ffsl(mask)) != 0) { + cpu--; + mask &= ~cputomask(cpu); + lapic_ipi_vectored(vector, cpu_apic_ids[cpu]); + ncpu++; + } + } while (smp_tlb_wait < ncpu) ia32_pause(); mtx_unlock_spin(&smp_ipi_mtx); @@ -1045,9 +1042,8 @@ smp_invltlb(void) { - if (smp_started) { + if (smp_started) smp_tlb_shootdown(IPI_INVLTLB, 0, 0); - } } void @@ -1062,36 +1058,32 @@ smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) { - if (smp_started) { + if (smp_started) smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); - } } void smp_masked_invltlb(cpumask_t mask) { - if (smp_started) { + if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); - } } void smp_masked_invlpg(cpumask_t mask, vm_offset_t addr) { - if (smp_started) { + if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); - } } void smp_masked_invlpg_range(cpumask_t mask, vm_offset_t addr1, vm_offset_t addr2) { - if (smp_started) { + if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); - } } void @@ -1139,12 +1131,12 @@ * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, cpus); + atomic_set_long(&ipi_nmi_pending, cpus); CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); - while ((cpu = ffs(cpus)) != 0) { + while ((cpu = ffsl(cpus)) != 0) { cpu--; - cpus &= ~(1 << cpu); + cpus &= ~cputomask(cpu); KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu)); @@ -1165,6 +1157,44 @@ } /* + * send an IPI to a specific cpu. + */ +void +ipi_cpu(int cpu, u_int ipi) +{ + u_int bitmap = 0; + u_int old_pending; + u_int new_pending; + + if (IPI_IS_BITMAPED(ipi)) { + bitmap = 1 << ipi; + ipi = IPI_BITMAP_VECTOR; + } + + /* + * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit + * of help in order to understand what is the source. + * Set the mask of receiving CPUs for this purpose. + */ + if (ipi == IPI_STOP_HARD) + atomic_set_long(&ipi_nmi_pending, cputomask(cpu)); + + CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); + + KASSERT(cpu_apic_ids[cpu] != -1, + ("IPI to non-existent CPU %d", cpu)); + + if (bitmap) { + do { + old_pending = cpu_ipi_pending[cpu]; + new_pending = old_pending | bitmap; + } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending)); + } + + lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); +} + +/* * send an IPI to all CPUs EXCEPT myself */ void @@ -1182,7 +1212,7 @@ * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus)); + atomic_set_long(&ipi_nmi_pending, PCPU_GET(other_cpus)); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); @@ -1203,7 +1233,7 @@ if ((ipi_nmi_pending & cpumask) == 0) return (1); - atomic_clear_int(&ipi_nmi_pending, cpumask); + atomic_clear_long(&ipi_nmi_pending, cpumask); cpustop_handler(); return (0); } @@ -1216,19 +1246,19 @@ cpustop_handler(void) { int cpu = PCPU_GET(cpuid); - int cpumask = PCPU_GET(cpumask); + cpumask_t cpumask = PCPU_GET(cpumask); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - atomic_set_int(&stopped_cpus, cpumask); + atomic_set_long(&stopped_cpus, cpumask); /* Wait for restart */ while (!(started_cpus & cpumask)) ia32_pause(); - atomic_clear_int(&started_cpus, cpumask); - atomic_clear_int(&stopped_cpus, cpumask); + atomic_clear_long(&started_cpus, cpumask); + atomic_clear_long(&stopped_cpus, cpumask); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); @@ -1246,7 +1276,7 @@ struct savefpu *stopfpu; register_t cr3, rf; int cpu = PCPU_GET(cpuid); - int cpumask = PCPU_GET(cpumask); + cpumask_t cpumask = PCPU_GET(cpumask); rf = intr_disable(); cr3 = rcr3(); @@ -1254,7 +1284,7 @@ if (savectx2(stopxpcbs[cpu])) { fpugetregs(curthread, stopfpu); wbinvd(); - atomic_set_int(&stopped_cpus, cpumask); + atomic_set_long(&stopped_cpus, cpumask); } else fpusetregs(curthread, stopfpu); @@ -1262,8 +1292,8 @@ while (!(started_cpus & cpumask)) ia32_pause(); - atomic_clear_int(&started_cpus, cpumask); - atomic_clear_int(&stopped_cpus, cpumask); + atomic_clear_long(&started_cpus, cpumask); + atomic_clear_long(&stopped_cpus, cpumask); /* Restore CR3 and enable interrupts */ load_cr3(cr3); @@ -1294,7 +1324,7 @@ int error; mask = hlt_cpus_mask; - error = sysctl_handle_int(oidp, &mask, 0, req); + error = sysctl_handle_long(oidp, &mask, 0, req); if (error || !req->newptr) return (error); @@ -1308,12 +1338,12 @@ mask |= hyperthreading_cpus_mask; if ((mask & all_cpus) == all_cpus) - mask &= ~(1<<0); + mask &= ~cputomask(0); hlt_cpus_mask = mask; return (error); } -SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW, - 0, 0, sysctl_hlt_cpus, "IU", +SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_LONG|CTLFLAG_RW, + 0, 0, sysctl_hlt_cpus, "LU", "Bitmap of CPUs to halt. 101 (binary) will halt CPUs 0 and 2."); static int @@ -1335,7 +1365,7 @@ hlt_cpus_mask |= hyperthreading_cpus_mask; if ((hlt_cpus_mask & all_cpus) == all_cpus) - hlt_cpus_mask &= ~(1<<0); + hlt_cpus_mask &= ~cputomask(0); hlt_logical_cpus = disable; return (error); @@ -1373,7 +1403,7 @@ hlt_logical_cpus = 0; if ((hlt_cpus_mask & all_cpus) == all_cpus) - hlt_cpus_mask &= ~(1<<0); + hlt_cpus_mask &= ~cputomask(0); hyperthreading_allowed = allowed; return (error); @@ -1419,7 +1449,7 @@ int mp_grab_cpu_hlt(void) { - u_int mask = PCPU_GET(cpumask); + cpumask_t mask = PCPU_GET(cpumask); #ifdef MP_WATCHDOG u_int cpuid = PCPU_GET(cpuid); #endif Index: amd64/amd64/local_apic.c =================================================================== --- amd64/amd64/local_apic.c (revision 207451) +++ amd64/amd64/local_apic.c (working copy) @@ -762,7 +762,7 @@ * and unlike other schedulers it actually schedules threads to * those CPUs. */ - if ((hlt_cpus_mask & (1 << PCPU_GET(cpuid))) != 0) + if ((hlt_cpus_mask & PCPU_GET(cpumask)) != 0) return; #endif Index: amd64/amd64/intr_machdep.c =================================================================== --- amd64/amd64/intr_machdep.c (revision 207451) +++ amd64/amd64/intr_machdep.c (working copy) @@ -444,7 +444,7 @@ */ /* The BSP is always a valid target. */ -static cpumask_t intr_cpus = (1 << 0); +static cpumask_t intr_cpus = cputomask(0); static int current_cpu; /* @@ -466,7 +466,7 @@ current_cpu++; if (current_cpu > mp_maxid) current_cpu = 0; - } while (!(intr_cpus & (1 << current_cpu))); + } while (!(intr_cpus & cputomask(current_cpu))); mtx_unlock_spin(&icu_lock); return (apic_id); } @@ -497,7 +497,7 @@ printf("INTR: Adding local APIC %d as a target\n", cpu_apic_ids[cpu]); - intr_cpus |= (1 << cpu); + intr_cpus |= cputomask(cpu); } /* Index: sys/smp.h =================================================================== --- sys/smp.h (revision 207451) +++ sys/smp.h (working copy) @@ -89,7 +89,7 @@ * time, thus permitting us to configure sparse maps of cpuid-dependent * (per-CPU) structures. */ -#define CPU_ABSENT(x_cpu) ((all_cpus & (1 << (x_cpu))) == 0) +#define CPU_ABSENT(x_cpu) ((all_cpus & cputomask(x_cpu)) == 0) #ifdef SMP /* Index: sys/systm.h =================================================================== --- sys/systm.h (revision 207451) +++ sys/systm.h (working copy) @@ -414,4 +414,5 @@ return (x); } +#define cputomask(_cpu) ((__cpumask_t)1 << _cpu) #endif /* !_SYS_SYSTM_H_ */