Index: sys/i386/i386/mp_machdep.c =================================================================== --- sys/i386/i386/mp_machdep.c (revision 223572) +++ sys/i386/i386/mp_machdep.c (working copy) @@ -658,12 +658,11 @@ void init_secondary(void) { - cpuset_t tcpuset; struct pcpu *pc; vm_offset_t addr; int gsel_tss; int x, myid; - u_int cr0; + u_int cpuid, cr0; /* bootAP is set in start_ap() to our ID. */ myid = bootAP; @@ -758,8 +757,9 @@ #endif /* A quick check from sanity claus */ + cpuid = PCPU_GET(cpuid); if (PCPU_GET(apic_id) != lapic_id()) { - printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); + printf("SMP: cpuid = %d\n", cpuid); printf("SMP: actual apic_id = %d\n", lapic_id()); printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); panic("cpuid mismatch! boom!!"); @@ -781,14 +781,13 @@ smp_cpus++; - CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - tcpuset = PCPU_GET(cpumask); + CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); + printf("SMP: AP CPU #%d Launched!\n", cpuid); /* Determine if we are a logical CPU. */ /* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */ if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0) - CPU_OR(&logical_cpus_mask, &tcpuset); + CPU_SET(cpuid, &logical_cpus_mask); if (bootverbose) lapic_dump("AP"); @@ -1242,9 +1241,7 @@ if (othercpus < 1) return; } else { - sched_pin(); - CPU_NAND(&mask, PCPU_PTR(cpumask)); - sched_unpin(); + CPU_CLR(PCPU_GET(cpuid), &mask); if (CPU_EMPTY(&mask)) return; } @@ -1465,7 +1462,7 @@ int ipi_nmi_handler() { - cpuset_t cpumask; + u_int cpuid; /* * As long as there is not a simple way to know about a NMI's @@ -1473,13 +1470,11 @@ * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ - sched_pin(); - cpumask = PCPU_GET(cpumask); - sched_unpin(); - if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &ipi_nmi_pending)) return (1); - CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); + CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending); cpustop_handler(); return (0); } @@ -1491,25 +1486,21 @@ void cpustop_handler(void) { - cpuset_t cpumask; u_int cpu; - sched_pin(); cpu = PCPU_GET(cpuid); - cpumask = PCPU_GET(cpumask); - sched_unpin(); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - CPU_OR_ATOMIC(&stopped_cpus, &cpumask); + CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ - while (!CPU_OVERLAP(&started_cpus, &cpumask)) + while (!CPU_ISSET(cpu, &started_cpus)) ia32_pause(); - CPU_NAND_ATOMIC(&started_cpus, &cpumask); - CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); Index: sys/i386/i386/pmap.c =================================================================== --- sys/i386/i386/pmap.c (revision 223572) +++ sys/i386/i386/pmap.c (working copy) @@ -1012,11 +1012,11 @@ } struct pde_action { - cpuset_t store; /* processor that updates the PDE */ cpuset_t invalidate; /* processors that invalidate their TLB */ vm_offset_t va; pd_entry_t *pde; pd_entry_t newpde; + u_int store; /* processor that updates the PDE */ }; static void @@ -1026,9 +1026,7 @@ pd_entry_t *pde; pmap_t pmap; - sched_pin(); - if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { - sched_unpin(); + if (act->store == PCPU_GET(cpuid)) { /* * Elsewhere, this operation requires allpmaps_lock for @@ -1039,8 +1037,7 @@ pde = pmap_pde(pmap, act->va); pde_store(pde, act->newpde); } - } else - sched_unpin(); + } } static void @@ -1048,12 +1045,8 @@ { struct pde_action *act = arg; - sched_pin(); - if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { - sched_unpin(); + if (act->store == PCPU_GET(cpuid)) pde_store(act->pde, act->newpde); - } else - sched_unpin(); } static void @@ -1077,24 +1070,25 @@ pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { struct pde_action act; - cpuset_t active, cpumask, other_cpus; + cpuset_t active, other_cpus; + u_int cpuid; sched_pin(); - cpumask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); + CPU_CLR(cpuid, &other_cpus); if (pmap == kernel_pmap) active = all_cpus; else active = pmap->pm_active; if (CPU_OVERLAP(&active, &other_cpus)) { - act.store = cpumask; + act.store = cpuid; act.invalidate = active; act.va = va; act.pde = pde; act.newpde = newpde; - CPU_OR(&cpumask, &active); - smp_rendezvous_cpus(cpumask, + CPU_SET(cpuid, &active); + smp_rendezvous_cpus(active, smp_no_rendevous_barrier, pmap == kernel_pmap ? pmap_update_pde_kernel : pmap_update_pde_user, pmap_update_pde_teardown, &act); @@ -1103,7 +1097,7 @@ pmap_kenter_pde(va, newpde); else pde_store(pde, newpde); - if (CPU_OVERLAP(&active, &cpumask)) + if (CPU_ISSET(cpuid, &active)) pmap_update_pde_invalidate(va, newpde); } sched_unpin(); @@ -1928,12 +1922,12 @@ } static void -pmap_lazyfix_self(cpuset_t mymask) +pmap_lazyfix_self(u_int cpuid) { if (rcr3() == lazyptd) load_cr3(PCPU_GET(curpcb)->pcb_cr3); - CPU_NAND_ATOMIC(lazymask, &mymask); + CPU_CLR_ATOMIC(cpuid, lazymask); } @@ -1941,7 +1935,7 @@ pmap_lazyfix(pmap_t pmap) { cpuset_t mymask, mask; - u_int spins; + u_int cpuid, spins; int lsb; mask = pmap->pm_active; @@ -1959,10 +1953,13 @@ #else lazyptd = vtophys(pmap->pm_pdir); #endif - mymask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); + + /* Use a cpuset just for having an easy check. */ + CPU_SETOF(cpuid, &mymask); if (!CPU_CMP(&mask, &mymask)) { lazymask = &pmap->pm_active; - pmap_lazyfix_self(mymask); + pmap_lazyfix_self(cpuid); } else { atomic_store_rel_int((u_int *)&lazymask, (u_int)&pmap->pm_active); @@ -5101,17 +5098,19 @@ pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; + u_int cpuid; u_int32_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); + cpuid = PCPU_GET(cpuid); #if defined(SMP) - CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); + CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else - CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR(cpuid, &oldpmap->pm_active); + CPU_SET(cpuid, &pmap->pm_active); #endif #ifdef PAE cr3 = vtophys(pmap->pm_pdpt); Index: sys/i386/xen/mp_machdep.c =================================================================== --- sys/i386/xen/mp_machdep.c (revision 223572) +++ sys/i386/xen/mp_machdep.c (working copy) @@ -523,8 +523,8 @@ void init_secondary(void) { - cpuset_t tcpuset; vm_offset_t addr; + u_int cpuid; int gsel_tss; @@ -601,18 +601,18 @@ #endif smp_cpus++; - CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - tcpuset = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); + CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); + printf("SMP: AP CPU #%d Launched!\n", cpuid); /* Determine if we are a logical CPU. */ if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0) - CPU_OR(&logical_cpus_mask, &tcpuset); + CPU_SET(cpuid, &logical_cpus_mask); /* Determine if we are a hyperthread. */ if (hyperthreading_cpus > 1 && PCPU_GET(apic_id) % hyperthreading_cpus != 0) - CPU_OR(&hyperthreading_cpus_mask, &tcpuset); + CPU_SET(cpuid, &hyperthreading_cpus_mask); #if 0 if (bootverbose) lapic_dump("AP"); @@ -1020,9 +1020,7 @@ if (othercpus < 1) return; } else { - critical_enter(); - CPU_NAND(&mask, PCPU_PTR(cpumask)); - critical_exit(); + CPU_CLR(PCPU_GET(cpuid), &mask); if (CPU_EMPTY(&mask)) return; } @@ -1185,7 +1183,7 @@ int ipi_nmi_handler() { - cpuset_t cpumask; + u_int cpuid; /* * As long as there is not a simple way to know about a NMI's @@ -1193,13 +1191,11 @@ * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ - sched_pin(); - cpumask = PCPU_GET(cpumask); - sched_unpin(); - if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &ipi_nmi_pending)) return (1); - CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); + CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending); cpustop_handler(); return (0); } @@ -1211,25 +1207,21 @@ void cpustop_handler(void) { - cpuset_t cpumask; int cpu; - sched_pin(); - cpumask = PCPU_GET(cpumask); cpu = PCPU_GET(cpuid); - sched_unpin(); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - CPU_OR_ATOMIC(&stopped_cpus, &cpumask); + CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ - while (!CPU_OVERLAP(&started_cpus, &cpumask)) + while (!CPU_ISSET(cpu, &started_cpus)) ia32_pause(); - CPU_NAND_ATOMIC(&started_cpus, &cpumask); - CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); Index: sys/i386/xen/pmap.c =================================================================== --- sys/i386/xen/pmap.c (revision 223572) +++ sys/i386/xen/pmap.c (working copy) @@ -802,7 +802,8 @@ void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x", pmap, va); @@ -812,10 +813,10 @@ invlpg(va); smp_invlpg(va); } else { - cpumask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invlpg(va); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -828,8 +829,9 @@ void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; vm_offset_t addr; + u_int cpuid; CTR3(KTR_PMAP, "pmap_invalidate_page: pmap=%p eva=0x%x sva=0x%x", pmap, sva, eva); @@ -840,10 +842,10 @@ invlpg(addr); smp_invlpg_range(sva, eva); } else { - cpumask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); CPU_AND(&other_cpus, &pmap->pm_active); @@ -857,7 +859,8 @@ void pmap_invalidate_all(pmap_t pmap) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; CTR1(KTR_PMAP, "pmap_invalidate_page: pmap=%p", pmap); @@ -866,10 +869,10 @@ invltlb(); smp_invltlb(); } else { - cpumask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -1711,12 +1714,12 @@ } static void -pmap_lazyfix_self(cpuset_t mymask) +pmap_lazyfix_self(u_int cpuid) { if (rcr3() == lazyptd) load_cr3(PCPU_GET(curpcb)->pcb_cr3); - CPU_NAND_ATOMIC(lazymask, &mymask); + CPU_CLR_ATOMIC(cpuid, lazymask); } @@ -1724,7 +1727,7 @@ pmap_lazyfix(pmap_t pmap) { cpuset_t mymask, mask; - u_int spins; + u_int cpuid, spins; int lsb; mask = pmap->pm_active; @@ -1742,10 +1745,13 @@ #else lazyptd = vtophys(pmap->pm_pdir); #endif - mymask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); + + /* Use a cpuset just for having an easy check. */ + CPU_SETOF(cpuid, &mymask); if (!CPU_CMP(&mask, &mymask)) { lazymask = &pmap->pm_active; - pmap_lazyfix_self(mymask); + pmap_lazyfix_self(cpuid); } else { atomic_store_rel_int((u_int *)&lazymask, (u_int)&pmap->pm_active); @@ -4129,17 +4135,19 @@ pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; + u_int cpuid; u_int32_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); + cpuid = PCPU_GET(cpuid); #if defined(SMP) - CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); + CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else - CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR(cpuid, &oldpmap->pm_active); + CPU_SET(cpuid, &pmap->pm_active); #endif #ifdef PAE cr3 = vtophys(pmap->pm_pdpt);