diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index abce826..1d7178f 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -658,7 +658,7 @@ cpu_halt(void) halt(); } -void (*cpu_idle_hook)(void) = NULL; /* ACPI idle hook. */ +void (*cpu_idle_hook)(sbintime_t) = NULL; /* ACPI idle hook. */ static int cpu_ident_amdc1e = 0; /* AMD C1E supported. */ static int idle_mwait = 1; /* Use MONITOR/MWAIT for short idle. */ TUNABLE_INT("machdep.idle_mwait", &idle_mwait); @@ -670,7 +670,7 @@ SYSCTL_INT(_machdep, OID_AUTO, idle_mwait, CTLFLAG_RW, &idle_mwait, #define STATE_SLEEPING 0x2 static void -cpu_idle_acpi(int busy) +cpu_idle_acpi(sbintime_t sbt) { int *state; @@ -682,14 +682,14 @@ cpu_idle_acpi(int busy) if (sched_runnable()) enable_intr(); else if (cpu_idle_hook) - cpu_idle_hook(); + cpu_idle_hook(sbt); else __asm __volatile("sti; hlt"); *state = STATE_RUNNING; } static void -cpu_idle_hlt(int busy) +cpu_idle_hlt(sbintime_t sbt) { int *state; @@ -730,7 +730,7 @@ cpu_idle_hlt(int busy) #define MWAIT_C4 0x30 static void -cpu_idle_mwait(int busy) +cpu_idle_mwait(sbintime_t sbt) { int *state; @@ -753,7 +753,7 @@ cpu_idle_mwait(int busy) } static void -cpu_idle_spin(int busy) +cpu_idle_spin(sbintime_t sbt) { int *state; int i; @@ -802,12 +802,13 @@ cpu_probe_amdc1e(void) } } -void (*cpu_idle_fn)(int) = cpu_idle_acpi; +void (*cpu_idle_fn)(sbintime_t) = cpu_idle_acpi; void cpu_idle(int busy) { uint64_t msr; + sbintime_t sbt = -1; CTR2(KTR_SPARE2, "cpu_idle(%d) at %d", busy, curcpu); @@ -825,7 +826,7 @@ cpu_idle(int busy) /* If we have time - switch timers into idle mode. */ if (!busy) { critical_enter(); - cpu_idleclock(); + sbt = cpu_idleclock(); } /* Apply AMD APIC timer C1E workaround. */ @@ -836,7 +837,7 @@ cpu_idle(int busy) } /* Call main idle method. */ - cpu_idle_fn(busy); + cpu_idle_fn(sbt); /* Switch timers mack into active mode. */ if (!busy) { diff --git a/sys/dev/acpica/acpi_cpu.c b/sys/dev/acpica/acpi_cpu.c index 7f008f7..6e842d6 100644 --- a/sys/dev/acpica/acpi_cpu.c +++ b/sys/dev/acpica/acpi_cpu.c @@ -168,7 +168,7 @@ static int acpi_cpu_cx_cst(struct acpi_cpu_softc *sc); static void acpi_cpu_startup(void *arg); static void acpi_cpu_startup_cx(struct acpi_cpu_softc *sc); static void acpi_cpu_cx_list(struct acpi_cpu_softc *sc); -static void acpi_cpu_idle(void); +static void acpi_cpu_idle(sbintime_t sbt); static void acpi_cpu_notify(ACPI_HANDLE h, UINT32 notify, void *context); static int acpi_cpu_quirks(void); static int acpi_cpu_usage_sysctl(SYSCTL_HANDLER_ARGS); @@ -954,13 +954,13 @@ acpi_cpu_startup_cx(struct acpi_cpu_softc *sc) * interrupts are re-enabled. */ static void -acpi_cpu_idle() +acpi_cpu_idle(sbintime_t sbt) { struct acpi_cpu_softc *sc; struct acpi_cx *cx_next; uint64_t cputicks; uint32_t start_time, end_time; - int bm_active, cx_next_idx, i; + int bm_active, cx_next_idx, i, us; /* * Look up our CPU id to get our softc. If it's NULL, we'll use C1 @@ -980,13 +980,16 @@ acpi_cpu_idle() } /* Find the lowest state that has small enough latency. */ + us = sc->cpu_prev_sleep; + if (sbt >= 0 && us > sbt / SBT_1US) + us = sbt / SBT_1US; cx_next_idx = 0; if (cpu_disable_deep_sleep) i = min(sc->cpu_cx_lowest, sc->cpu_non_c3); else i = sc->cpu_cx_lowest; for (; i >= 0; i--) { - if (sc->cpu_cx_states[i].trans_lat * 3 <= sc->cpu_prev_sleep) { + if (sc->cpu_cx_states[i].trans_lat * 3 <= us) { cx_next_idx = i; break; } diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 744c3cb..3f15f08 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -1220,7 +1220,7 @@ cpu_halt(void) int scheduler_running; static void -cpu_idle_hlt(int busy) +cpu_idle_hlt(sbintime_t sbt) { scheduler_running = 1; @@ -1241,7 +1241,7 @@ cpu_halt(void) #endif -void (*cpu_idle_hook)(void) = NULL; /* ACPI idle hook. */ +void (*cpu_idle_hook)(sbintime_t) = NULL; /* ACPI idle hook. */ static int cpu_ident_amdc1e = 0; /* AMD C1E supported. */ static int idle_mwait = 1; /* Use MONITOR/MWAIT for short idle. */ TUNABLE_INT("machdep.idle_mwait", &idle_mwait); @@ -1253,7 +1253,7 @@ SYSCTL_INT(_machdep, OID_AUTO, idle_mwait, CTLFLAG_RW, &idle_mwait, #define STATE_SLEEPING 0x2 static void -cpu_idle_acpi(int busy) +cpu_idle_acpi(sbintime_t sbt) { int *state; @@ -1265,7 +1265,7 @@ cpu_idle_acpi(int busy) if (sched_runnable()) enable_intr(); else if (cpu_idle_hook) - cpu_idle_hook(); + cpu_idle_hook(sbt); else __asm __volatile("sti; hlt"); *state = STATE_RUNNING; @@ -1273,7 +1273,7 @@ cpu_idle_acpi(int busy) #ifndef XEN static void -cpu_idle_hlt(int busy) +cpu_idle_hlt(sbintime_t sbt) { int *state; @@ -1315,7 +1315,7 @@ cpu_idle_hlt(int busy) #define MWAIT_C4 0x30 static void -cpu_idle_mwait(int busy) +cpu_idle_mwait(sbintime_t sbt) { int *state; @@ -1338,7 +1338,7 @@ cpu_idle_mwait(int busy) } static void -cpu_idle_spin(int busy) +cpu_idle_spin(sbintime_t sbt) { int *state; int i; @@ -1388,9 +1388,9 @@ cpu_probe_amdc1e(void) } #ifdef XEN -void (*cpu_idle_fn)(int) = cpu_idle_hlt; +void (*cpu_idle_fn)(sbintime_t) = cpu_idle_hlt; #else -void (*cpu_idle_fn)(int) = cpu_idle_acpi; +void (*cpu_idle_fn)(sbintime_t) = cpu_idle_acpi; #endif void @@ -1399,6 +1399,7 @@ cpu_idle(int busy) #ifndef XEN uint64_t msr; #endif + sbintime_t sbt = -1; CTR2(KTR_SPARE2, "cpu_idle(%d) at %d", busy, curcpu); @@ -1418,7 +1419,7 @@ cpu_idle(int busy) /* If we have time - switch timers into idle mode. */ if (!busy) { critical_enter(); - cpu_idleclock(); + sbt = cpu_idleclock(); } #ifndef XEN @@ -1431,7 +1432,7 @@ cpu_idle(int busy) #endif /* Call main idle method. */ - cpu_idle_fn(busy); + cpu_idle_fn(sbt); /* Switch timers mack into active mode. */ if (!busy) { diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index 98bcf44..ac6a829 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -171,7 +171,7 @@ extern vm_offset_t ksym_start, ksym_end; struct msgbuf *msgbufp = NULL; /* Other subsystems (e.g., ACPI) can hook this later. */ -void (*cpu_idle_hook)(void) = NULL; +void (*cpu_idle_hook)(sbintime_t) = NULL; struct kva_md_info kmi; @@ -408,10 +408,11 @@ void cpu_idle(int busy) { register_t ie; + sbintime_t sbt = -1; if (!busy) { critical_enter(); - cpu_idleclock(); + sbt = cpu_idleclock(); } ie = intr_disable(); @@ -420,7 +421,7 @@ cpu_idle(int busy) if (sched_runnable()) ia64_enable_intr(); else if (cpu_idle_hook != NULL) { - (*cpu_idle_hook)(); + (*cpu_idle_hook)(sbt); /* The hook must enable interrupts! */ } else { ia64_call_pal_static(PAL_HALT_LIGHT, 0, 0, 0); diff --git a/sys/kern/kern_clocksource.c b/sys/kern/kern_clocksource.c index 6fd40a8..0b49236 100644 --- a/sys/kern/kern_clocksource.c +++ b/sys/kern/kern_clocksource.c @@ -761,7 +761,7 @@ cpu_stopprofclock(void) /* * Switch to idle mode (all ticks handled). */ -void +sbintime_t cpu_idleclock(void) { struct bintime now, t; @@ -773,7 +773,7 @@ cpu_idleclock(void) || curcpu == CPU_FIRST() #endif ) - return; + return (-1); state = DPCPU_PTR(timerstate); if (periodic) now = state->now; @@ -789,6 +789,8 @@ cpu_idleclock(void) if (!periodic) loadtimer(&now, 0); ET_HW_UNLOCK(state); + bintime_sub(&t, &now); + return (MAX(bintime2sbintime(t), 0)); } /* diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c index 3133d1b..b0d5acd 100644 --- a/sys/pc98/pc98/machdep.c +++ b/sys/pc98/pc98/machdep.c @@ -1145,7 +1145,7 @@ SYSCTL_INT(_machdep, OID_AUTO, idle_mwait, CTLFLAG_RW, &idle_mwait, #define STATE_SLEEPING 0x2 static void -cpu_idle_hlt(int busy) +cpu_idle_hlt(sbintime_t sbt) { int *state; @@ -1186,7 +1186,7 @@ cpu_idle_hlt(int busy) #define MWAIT_C4 0x30 static void -cpu_idle_mwait(int busy) +cpu_idle_mwait(sbintime_t sbt) { int *state; @@ -1209,7 +1209,7 @@ cpu_idle_mwait(int busy) } static void -cpu_idle_spin(int busy) +cpu_idle_spin(sbintime_t sbt) { int *state; int i; @@ -1229,11 +1229,12 @@ cpu_idle_spin(int busy) } } -void (*cpu_idle_fn)(int) = cpu_idle_hlt; +void (*cpu_idle_fn)(sbintime_t) = cpu_idle_hlt; void cpu_idle(int busy) { + sbintime_t sbt = -1; CTR2(KTR_SPARE2, "cpu_idle(%d) at %d", busy, curcpu); @@ -1251,11 +1252,11 @@ cpu_idle(int busy) /* If we have time - switch timers into idle mode. */ if (!busy) { critical_enter(); - cpu_idleclock(); + sbt = cpu_idleclock(); } /* Call main idle method. */ - cpu_idle_fn(busy); + cpu_idle_fn(sbt); /* Switch timers mack into active mode. */ if (!busy) { diff --git a/sys/powerpc/powerpc/cpu.c b/sys/powerpc/powerpc/cpu.c index d177c8b..d67f359 100644 --- a/sys/powerpc/powerpc/cpu.c +++ b/sys/powerpc/powerpc/cpu.c @@ -79,9 +79,9 @@ static void cpu_970_setup(int cpuid, uint16_t vers); static void cpu_booke_setup(int cpuid, uint16_t vers); int powerpc_pow_enabled; -void (*cpu_idle_hook)(void) = NULL; -static void cpu_idle_60x(void); -static void cpu_idle_booke(void); +void (*cpu_idle_hook)(sbintime_t) = NULL; +static void cpu_idle_60x(sbintime_t); +static void cpu_idle_booke(sbintime_t); struct cputab { const char *name; @@ -516,6 +516,7 @@ cpu_feature_bit(SYSCTL_HANDLER_ARGS) void cpu_idle(int busy) { + sbintime_t sbt = -1; #ifdef INVARIANTS if ((mfmsr() & PSL_EE) != PSL_EE) { @@ -531,9 +532,9 @@ cpu_idle(int busy) if (cpu_idle_hook != NULL) { if (!busy) { critical_enter(); - cpu_idleclock(); + sbt = cpu_idleclock(); } - cpu_idle_hook(); + cpu_idle_hook(sbt); if (!busy) { cpu_activeclock(); critical_exit(); @@ -551,7 +552,7 @@ cpu_idle_wakeup(int cpu) } static void -cpu_idle_60x(void) +cpu_idle_60x(sbintime_t sbt) { register_t msr; uint16_t vers; @@ -586,7 +587,7 @@ cpu_idle_60x(void) } static void -cpu_idle_booke(void) +cpu_idle_booke(sbintime_t sbt) { register_t msr; diff --git a/sys/powerpc/ps3/platform_ps3.c b/sys/powerpc/ps3/platform_ps3.c index 61ce873..207382d 100644 --- a/sys/powerpc/ps3/platform_ps3.c +++ b/sys/powerpc/ps3/platform_ps3.c @@ -70,7 +70,7 @@ static int ps3_smp_start_cpu(platform_t, struct pcpu *cpu); static struct cpu_group *ps3_smp_topo(platform_t); #endif static void ps3_reset(platform_t); -static void ps3_cpu_idle(void); +static void ps3_cpu_idle(sbintime_t); static platform_method_t ps3_methods[] = { PLATFORMMETHOD(platform_probe, ps3_probe), @@ -245,7 +245,7 @@ ps3_real_maxaddr(platform_t plat) } static void -ps3_cpu_idle(void) +ps3_cpu_idle(sbintime_t sbt) { lv1_pause(0); } diff --git a/sys/powerpc/wii/platform_wii.c b/sys/powerpc/wii/platform_wii.c index 2bb6022..bc35105 100644 --- a/sys/powerpc/wii/platform_wii.c +++ b/sys/powerpc/wii/platform_wii.c @@ -60,7 +60,7 @@ static void wii_mem_regions(platform_t, struct mem_region **, int *, struct mem_region **, int *); static unsigned long wii_timebase_freq(platform_t, struct cpuref *cpuref); static void wii_reset(platform_t); -static void wii_cpu_idle(void); +static void wii_cpu_idle(sbintime_t sbt); static platform_method_t wii_methods[] = { PLATFORMMETHOD(platform_probe, wii_probe), @@ -155,6 +155,6 @@ wii_reset(platform_t plat) } static void -wii_cpu_idle(void) +wii_cpu_idle(sbintime_t sbt) { } diff --git a/sys/sys/proc.h b/sys/sys/proc.h index ade2864..46f5820 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -908,7 +908,7 @@ void tidhash_add(struct thread *); void tidhash_remove(struct thread *); void cpu_idle(int); int cpu_idle_wakeup(int); -extern void (*cpu_idle_hook)(void); /* Hook to machdep CPU idler. */ +extern void (*cpu_idle_hook)(sbintime_t); /* Hook to machdep CPU idler. */ void cpu_switch(struct thread *, struct thread *, struct mtx *); void cpu_throw(struct thread *, struct thread *) __dead2; void unsleep(struct thread *); diff --git a/sys/sys/systm.h b/sys/sys/systm.h index dd96d70..82881bd 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -44,6 +44,7 @@ #include #include #include /* for people using printf mainly */ +#include extern int cold; /* nonzero if we are doing a cold boot */ extern int rebooting; /* kern_reboot() has been called. */ @@ -267,7 +268,7 @@ void startprofclock(struct proc *); void stopprofclock(struct proc *); void cpu_startprofclock(void); void cpu_stopprofclock(void); -void cpu_idleclock(void); +sbintime_t cpu_idleclock(void); void cpu_activeclock(void); extern int cpu_can_deep_sleep; extern int cpu_disable_deep_sleep;