Index: amd64/include/counter.h =================================================================== --- amd64/include/counter.h (revision 302252) +++ amd64/include/counter.h (working copy) @@ -51,7 +51,7 @@ int i; r = 0; - for (i = 0; i < mp_ncpus; i++) + CPU_FOREACH(i) r += counter_u64_read_one((uint64_t *)p, i); return (r); Index: cddl/compat/opensolaris/sys/proc.h =================================================================== --- cddl/compat/opensolaris/sys/proc.h (revision 302252) +++ cddl/compat/opensolaris/sys/proc.h (working copy) @@ -45,8 +45,8 @@ #define CPU curcpu #define minclsyspri PRIBIO #define maxclsyspri PVM -#define max_ncpus mp_ncpus -#define boot_max_ncpus mp_ncpus +#define max_ncpus (mp_maxid + 1) +#define boot_max_ncpus (mp_maxid + 1) #define TS_RUN 0 Index: dev/cpuctl/cpuctl.c =================================================================== --- dev/cpuctl/cpuctl.c (revision 302252) +++ dev/cpuctl/cpuctl.c (working copy) @@ -120,7 +120,7 @@ set_cpu(int cpu, struct thread *td) { - KASSERT(cpu >= 0 && cpu < mp_ncpus && cpu_enabled(cpu), + KASSERT(cpu >= 0 && cpu <= mp_maxid && cpu_enabled(cpu), ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); thread_lock(td); sched_bind(td, cpu); @@ -133,7 +133,7 @@ restore_cpu(int oldcpu, int is_bound, struct thread *td) { - KASSERT(oldcpu >= 0 && oldcpu < mp_ncpus && cpu_enabled(oldcpu), + KASSERT(oldcpu >= 0 && oldcpu <= mp_maxid && cpu_enabled(oldcpu), ("[cpuctl,%d]: bad cpu number %d", __LINE__, oldcpu)); thread_lock(td); if (is_bound == 0) @@ -150,7 +150,7 @@ int ret; int cpu = dev2unit(dev); - if (cpu >= mp_ncpus || !cpu_enabled(cpu)) { + if (cpu > mp_maxid || !cpu_enabled(cpu)) { DPRINTF("[cpuctl,%d]: bad cpu number %d\n", __LINE__, cpu); return (ENXIO); } @@ -201,7 +201,7 @@ int is_bound = 0; int oldcpu; - KASSERT(cpu >= 0 && cpu < mp_ncpus, + KASSERT(cpu >= 0 && cpu <= mp_maxid, ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); /* Explicitly clear cpuid data to avoid returning stale info. */ @@ -245,7 +245,7 @@ int oldcpu; int ret; - KASSERT(cpu >= 0 && cpu < mp_ncpus, + KASSERT(cpu >= 0 && cpu <= mp_maxid, ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); /* @@ -296,7 +296,7 @@ char vendor[13]; int ret; - KASSERT(cpu >= 0 && cpu < mp_ncpus, + KASSERT(cpu >= 0 && cpu <= mp_maxid, ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); DPRINTF("[cpuctl,%d]: XXX %d", __LINE__, cpu); @@ -512,7 +512,7 @@ int cpu; cpu = dev2unit(dev); - if (cpu >= mp_ncpus || !cpu_enabled(cpu)) { + if (cpu > mp_maxid || !cpu_enabled(cpu)) { DPRINTF("[cpuctl,%d]: incorrect cpu number %d\n", __LINE__, cpu); return (ENXIO); @@ -531,15 +531,15 @@ case MOD_LOAD: if (bootverbose) printf("cpuctl: access to MSR registers/cpuid info.\n"); - cpuctl_devs = malloc(sizeof(*cpuctl_devs) * mp_ncpus, M_CPUCTL, + cpuctl_devs = malloc(sizeof(*cpuctl_devs) * (mp_maxid + 1), M_CPUCTL, M_WAITOK | M_ZERO); - for (cpu = 0; cpu < mp_ncpus; cpu++) + CPU_FOREACH(cpu) if (cpu_enabled(cpu)) cpuctl_devs[cpu] = make_dev(&cpuctl_cdevsw, cpu, UID_ROOT, GID_KMEM, 0640, "cpuctl%d", cpu); break; case MOD_UNLOAD: - for (cpu = 0; cpu < mp_ncpus; cpu++) { + CPU_FOREACH(cpu) { if (cpuctl_devs[cpu] != NULL) destroy_dev(cpuctl_devs[cpu]); } Index: i386/include/counter.h =================================================================== --- i386/include/counter.h (revision 302252) +++ i386/include/counter.h (working copy) @@ -98,13 +98,13 @@ * critical section as well. */ critical_enter(); - for (i = 0; i < mp_ncpus; i++) { + CPU_FOREACH(i) { res += *(uint64_t *)((char *)p + sizeof(struct pcpu) * i); } critical_exit(); } else { - for (i = 0; i < mp_ncpus; i++) + CPU_FOREACH(i) res += counter_u64_read_one_8b((uint64_t *)((char *)p + sizeof(struct pcpu) * i)); } @@ -144,7 +144,7 @@ if ((cpu_feature & CPUID_CX8) == 0) { critical_enter(); - for (i = 0; i < mp_ncpus; i++) + CPU_FOREACH(i) *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0; critical_exit(); } else { Index: kern/subr_pcpu.c =================================================================== --- kern/subr_pcpu.c (revision 302252) +++ kern/subr_pcpu.c (working copy) @@ -248,7 +248,7 @@ uintptr_t dpcpu; int i; - for (i = 0; i < mp_ncpus; ++i) { + CPU_FOREACH(i) { dpcpu = dpcpu_off[i]; if (dpcpu == 0) continue; @@ -289,7 +289,7 @@ int i; count = 0; - for (i = 0; i < mp_ncpus; ++i) { + CPU_FOREACH(i) { dpcpu = dpcpu_off[i]; if (dpcpu == 0) continue; @@ -306,7 +306,7 @@ int i; count = 0; - for (i = 0; i < mp_ncpus; ++i) { + CPU_FOREACH(i) { dpcpu = dpcpu_off[i]; if (dpcpu == 0) continue; @@ -323,7 +323,7 @@ int i; count = 0; - for (i = 0; i < mp_ncpus; ++i) { + CPU_FOREACH(i) { dpcpu = dpcpu_off[i]; if (dpcpu == 0) continue; Index: kern/subr_taskqueue.c =================================================================== --- kern/subr_taskqueue.c (revision 302252) +++ kern/subr_taskqueue.c (working copy) @@ -832,6 +832,7 @@ taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx) { struct taskqgroup_cpu *qcpu; + int i, j; qcpu = &qgroup->tqg_queue[idx]; LIST_INIT(&qcpu->tgc_tasks); @@ -839,7 +840,15 @@ taskqueue_thread_enqueue, &qcpu->tgc_taskq); taskqueue_start_threads(&qcpu->tgc_taskq, 1, PI_SOFT, "%s_%d", qgroup->tqg_name, idx); - qcpu->tgc_cpu = idx * qgroup->tqg_stride; + + for (i = CPU_FIRST(), j = 0; j < idx * qgroup->tqg_stride; + j++, i = CPU_NEXT(i)) { + /* + * Wait: evaluate the idx * qgroup->tqg_stride'th CPU, + * potentially wrapping the actual count + */ + } + qcpu->tgc_cpu = i; } static void @@ -1017,13 +1026,14 @@ LIST_HEAD(, grouptask) gtask_head = LIST_HEAD_INITIALIZER(NULL); cpuset_t mask; struct grouptask *gtask; - int i, old_cnt, qid; + int i, k, old_cnt, qid, cpu; mtx_assert(&qgroup->tqg_lock, MA_OWNED); if (cnt < 1 || cnt * stride > mp_ncpus || !smp_started) { - printf("taskqgroup_adjust failed cnt: %d stride: %d mp_ncpus: %d smp_started: %d\n", - cnt, stride, mp_ncpus, smp_started); + printf("taskqgroup_adjust failed cnt: %d stride: %d " + "mp_ncpus: %d smp_started: %d\n", cnt, stride, mp_ncpus, + smp_started); return (EINVAL); } if (qgroup->tqg_adjusting) { @@ -1081,8 +1091,11 @@ /* * Set new CPU and IRQ affinity */ + cpu = CPU_FIRST(); for (i = 0; i < cnt; i++) { - qgroup->tqg_queue[i].tgc_cpu = i * qgroup->tqg_stride; + qgroup->tqg_queue[i].tgc_cpu = cpu; + for (k = 0; k < qgroup->tqg_stride; k++) + cpu = CPU_NEXT(cpu); CPU_ZERO(&mask); CPU_SET(qgroup->tqg_queue[i].tgc_cpu, &mask); LIST_FOREACH(gtask, &qgroup->tqg_queue[i].tgc_tasks, gt_list) { Index: net/flowtable.c =================================================================== --- net/flowtable.c (revision 302252) +++ net/flowtable.c (working copy) @@ -746,7 +746,7 @@ ft->ft_table[i] = uma_zalloc(pcpu_zone_ptr, M_WAITOK | M_ZERO); ft->ft_masks = uma_zalloc(pcpu_zone_ptr, M_WAITOK); - for (int i = 0; i < mp_ncpus; i++) { + CPU_FOREACH(i) { bitstr_t **b; b = zpcpu_get_cpu(ft->ft_masks, i); Index: net/iflib.c =================================================================== --- net/iflib.c (revision 302252) +++ net/iflib.c (working copy) @@ -3848,7 +3848,7 @@ iflib_txq_t txq; iflib_rxq_t rxq; iflib_fl_t fl = NULL; - int i, j, err, txconf, rxconf, fl_ifdi_offset; + int i, j, cpu, err, txconf, rxconf, fl_ifdi_offset; iflib_dma_info_t ifdip; uint32_t *rxqsizes = sctx->isc_rxqsizes; uint32_t *txqsizes = sctx->isc_txqsizes; @@ -3897,7 +3897,7 @@ /* * XXX handle allocation failure */ - for (txconf = i = 0; i < ntxqsets; i++, txconf++, txq++) { + for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) { /* Set up some basics */ if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) { @@ -3917,8 +3917,8 @@ txq->ift_ctx = ctx; txq->ift_id = i; /* XXX fix this */ - txq->ift_timer.c_cpu = i % mp_ncpus; - txq->ift_db_check.c_cpu = i % mp_ncpus; + txq->ift_timer.c_cpu = cpu; + txq->ift_db_check.c_cpu = cpu; txq->ift_nbr = nbuf_rings; if (iflib_txsd_alloc(txq)) { Index: netinet/ip_id.c =================================================================== --- netinet/ip_id.c (revision 302252) +++ netinet/ip_id.c (working copy) @@ -275,10 +275,12 @@ static void ipid_sysinit(void) { + int i; mtx_init(&V_ip_id_mtx, "ip_id_mtx", NULL, MTX_DEF); V_ip_id = counter_u64_alloc(M_WAITOK); - for (int i = 0; i < mp_ncpus; i++) + + CPU_FOREACH(i) arc4rand(zpcpu_get_cpu(V_ip_id, i), sizeof(uint64_t), 0); } VNET_SYSINIT(ip_id, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, ipid_sysinit, NULL); Index: powerpc/include/counter.h =================================================================== --- powerpc/include/counter.h (revision 302252) +++ powerpc/include/counter.h (working copy) @@ -54,7 +54,7 @@ int i; r = 0; - for (i = 0; i < mp_ncpus; i++) + CPU_FOREACH(i) r += counter_u64_read_one((uint64_t *)p, i); return (r); Index: powerpc/powerpc/mp_machdep.c =================================================================== --- powerpc/powerpc/mp_machdep.c (revision 302252) +++ powerpc/powerpc/mp_machdep.c (working copy) @@ -113,20 +113,16 @@ int error; mp_ncpus = 0; + mp_maxid = 0; error = platform_smp_first_cpu(&cpuref); while (!error) { mp_ncpus++; + mp_maxid = max(cpuref.cr_cpuid, mp_maxid); error = platform_smp_next_cpu(&cpuref); } /* Sanity. */ if (mp_ncpus == 0) mp_ncpus = 1; - - /* - * Set the largest cpuid we're going to use. This is necessary - * for VM initialization. - */ - mp_maxid = min(mp_ncpus, MAXCPU) - 1; } int Index: vm/uma.h =================================================================== --- vm/uma.h (revision 302252) +++ vm/uma.h (working copy) @@ -276,7 +276,7 @@ * mini-dumps. */ #define UMA_ZONE_PCPU 0x8000 /* - * Allocates mp_ncpus slabs sized to + * Allocates mp_maxid + 1 slabs sized to * sizeof(struct pcpu). */ Index: vm/uma_core.c =================================================================== --- vm/uma_core.c (revision 302252) +++ vm/uma_core.c (working copy) @@ -1227,7 +1227,7 @@ u_int shsize; if (keg->uk_flags & UMA_ZONE_PCPU) { - u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU; + u_int ncpus = (mp_maxid + 1) ? (mp_maxid + 1) : MAXCPU; keg->uk_slabsize = sizeof(struct pcpu); keg->uk_ppera = howmany(ncpus * sizeof(struct pcpu), @@ -3265,9 +3265,10 @@ static void uma_zero_item(void *item, uma_zone_t zone) { + int i; if (zone->uz_flags & UMA_ZONE_PCPU) { - for (int i = 0; i < mp_ncpus; i++) + CPU_FOREACH(i) bzero(zpcpu_get_cpu(item, i), zone->uz_size); } else bzero(item, zone->uz_size);