Index: sys/sysctl.h =================================================================== --- sys/sysctl.h (revision 194811) +++ sys/sysctl.h (working copy) @@ -179,6 +179,7 @@ int sysctl_handle_string(SYSCTL_HANDLER_ARGS); int sysctl_handle_opaque(SYSCTL_HANDLER_ARGS); int sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS); +int sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS); int sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS); #ifdef VIMAGE Index: sys/sched.h =================================================================== --- sys/sched.h (revision 194811) +++ sys/sched.h (working copy) @@ -162,14 +162,27 @@ sched_unpin(void) /* Scheduler stats. */ #ifdef SCHED_STATS -extern long sched_switch_stats[SWT_COUNT]; +DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]); #define SCHED_STAT_DEFINE_VAR(name, ptr, descr) \ - SYSCTL_LONG(_kern_sched_stats, OID_AUTO, name, CTLFLAG_RD, ptr, 0, descr) +static void name ## _add_proc(void *dummy __unused) \ +{ \ + \ + SYSCTL_ADD_PROC(NULL, \ + SYSCTL_STATIC_CHILDREN(_kern_sched_stats), OID_AUTO, \ + #name, CTLTYPE_LONG|CTLFLAG_RD|CTLFLAG_MPSAFE, \ + ptr, 0, sysctl_dpcpu_long, "LU", descr); \ +} \ +SYSINIT(name, SI_SUB_RUN_SCHEDULER, SI_ORDER_ANY, name ## _add_proc, NULL); + #define SCHED_STAT_DEFINE(name, descr) \ - unsigned long name; \ - SCHED_STAT_DEFINE_VAR(name, &name, descr) -#define SCHED_STAT_INC(var) atomic_add_long(&(var), 1) + DPCPU_DEFINE(unsigned long, name); \ + SCHED_STAT_DEFINE_VAR(name, &DPCPU_NAME(name), descr) +/* + * Sched stats are always incremented in critical sections so no atomic + * is necesssary to increment them. + */ +#define SCHED_STAT_INC(var) DPCPU_GET(var)++; #else #define SCHED_STAT_DEFINE_VAR(name, descr, ptr) #define SCHED_STAT_DEFINE(name, descr) Index: kern/kern_switch.c =================================================================== --- kern/kern_switch.c (revision 194811) +++ kern/kern_switch.c (working copy) @@ -79,31 +79,45 @@ SYSCTL_INT(_kern_sched, OID_AUTO, preemption, CTLF * with SCHED_STAT_DEFINE(). */ #ifdef SCHED_STATS -long sched_switch_stats[SWT_COUNT]; /* Switch reasons from mi_switch(). */ - SYSCTL_NODE(_kern_sched, OID_AUTO, stats, CTLFLAG_RW, 0, "switch stats"); -SCHED_STAT_DEFINE_VAR(uncategorized, &sched_switch_stats[SWT_NONE], ""); -SCHED_STAT_DEFINE_VAR(preempt, &sched_switch_stats[SWT_PREEMPT], ""); -SCHED_STAT_DEFINE_VAR(owepreempt, &sched_switch_stats[SWT_OWEPREEMPT], ""); -SCHED_STAT_DEFINE_VAR(turnstile, &sched_switch_stats[SWT_TURNSTILE], ""); -SCHED_STAT_DEFINE_VAR(sleepq, &sched_switch_stats[SWT_SLEEPQ], ""); -SCHED_STAT_DEFINE_VAR(sleepqtimo, &sched_switch_stats[SWT_SLEEPQTIMO], ""); -SCHED_STAT_DEFINE_VAR(relinquish, &sched_switch_stats[SWT_RELINQUISH], ""); -SCHED_STAT_DEFINE_VAR(needresched, &sched_switch_stats[SWT_NEEDRESCHED], ""); -SCHED_STAT_DEFINE_VAR(idle, &sched_switch_stats[SWT_IDLE], ""); -SCHED_STAT_DEFINE_VAR(iwait, &sched_switch_stats[SWT_IWAIT], ""); -SCHED_STAT_DEFINE_VAR(suspend, &sched_switch_stats[SWT_SUSPEND], ""); -SCHED_STAT_DEFINE_VAR(remotepreempt, &sched_switch_stats[SWT_REMOTEPREEMPT], - ""); -SCHED_STAT_DEFINE_VAR(remotewakeidle, &sched_switch_stats[SWT_REMOTEWAKEIDLE], - ""); +/* Switch reasons from mi_switch(). */ +DPCPU_DEFINE(long, sched_switch_stats[SWT_COUNT]); +SCHED_STAT_DEFINE_VAR(uncategorized, + &DPCPU_NAME(sched_switch_stats[SWT_NONE]), ""); +SCHED_STAT_DEFINE_VAR(preempt, + &DPCPU_NAME(sched_switch_stats[SWT_PREEMPT]), ""); +SCHED_STAT_DEFINE_VAR(owepreempt, + &DPCPU_NAME(sched_switch_stats[SWT_OWEPREEMPT]), ""); +SCHED_STAT_DEFINE_VAR(turnstile, + &DPCPU_NAME(sched_switch_stats[SWT_TURNSTILE]), ""); +SCHED_STAT_DEFINE_VAR(sleepq, + &DPCPU_NAME(sched_switch_stats[SWT_SLEEPQ]), ""); +SCHED_STAT_DEFINE_VAR(sleepqtimo, + &DPCPU_NAME(sched_switch_stats[SWT_SLEEPQTIMO]), ""); +SCHED_STAT_DEFINE_VAR(relinquish, + &DPCPU_NAME(sched_switch_stats[SWT_RELINQUISH]), ""); +SCHED_STAT_DEFINE_VAR(needresched, + &DPCPU_NAME(sched_switch_stats[SWT_NEEDRESCHED]), ""); +SCHED_STAT_DEFINE_VAR(idle, + &DPCPU_NAME(sched_switch_stats[SWT_IDLE]), ""); +SCHED_STAT_DEFINE_VAR(iwait, + &DPCPU_NAME(sched_switch_stats[SWT_IWAIT]), ""); +SCHED_STAT_DEFINE_VAR(suspend, + &DPCPU_NAME(sched_switch_stats[SWT_SUSPEND]), ""); +SCHED_STAT_DEFINE_VAR(remotepreempt, + &DPCPU_NAME(sched_switch_stats[SWT_REMOTEPREEMPT]), ""); +SCHED_STAT_DEFINE_VAR(remotewakeidle, + &DPCPU_NAME(sched_switch_stats[SWT_REMOTEWAKEIDLE]), ""); + static int sysctl_stats_reset(SYSCTL_HANDLER_ARGS) { struct sysctl_oid *p; + uintptr_t counter; int error; int val; + int i; val = 0; error = sysctl_handle_int(oidp, &val, 0, req); @@ -118,7 +132,12 @@ sysctl_stats_reset(SYSCTL_HANDLER_ARGS) SLIST_FOREACH(p, oidp->oid_parent, oid_link) { if (p == oidp || p->oid_arg1 == NULL) continue; - *(long *)p->oid_arg1 = 0; + counter = (uintptr_t)p->oid_arg1; + for (i = 0; i <= mp_maxid; i++) { + if (CPU_ABSENT(i)) + continue; + *(long *)(dpcpu_off[i] + counter) = 0; + } } return (0); } Index: kern/subr_pcpu.c =================================================================== --- kern/subr_pcpu.c (revision 194811) +++ kern/subr_pcpu.c (working copy) @@ -264,9 +264,8 @@ pcpu_find(u_int cpuid) int sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) { + uintptr_t dpcpu; int64_t count; -#ifdef SMP - uintptr_t dpcpu; int i; count = 0; @@ -276,18 +275,31 @@ sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) continue; count += *(int64_t *)(dpcpu + (uintptr_t)arg1); } -#else - count = *(int64_t *)(dpcpu_off[0] + (uintptr_t)arg1); -#endif return (SYSCTL_OUT(req, &count, sizeof(count))); } int +sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS) +{ + uintptr_t dpcpu; + long count; + int i; + + count = 0; + for (i = 0; i < mp_ncpus; ++i) { + dpcpu = dpcpu_off[i]; + if (dpcpu == 0) + continue; + count += *(long *)(dpcpu + (uintptr_t)arg1); + } + return (SYSCTL_OUT(req, &count, sizeof(count))); +} + +int sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) { + uintptr_t dpcpu; int count; -#ifdef SMP - uintptr_t dpcpu; int i; count = 0; @@ -297,9 +309,6 @@ sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) continue; count += *(int *)(dpcpu + (uintptr_t)arg1); } -#else - count = *(int *)(dpcpu_off[0] + (uintptr_t)arg1); -#endif return (SYSCTL_OUT(req, &count, sizeof(count))); }