--- sys/amd64/amd64/local_apic.c.orig 2008-05-24 14:32:26.000000000 +0800 +++ sys/amd64/amd64/local_apic.c 2008-05-25 13:02:15.000000000 +0800 @@ -329,29 +329,6 @@ /* XXX: Error and thermal LVTs */ - if (cpu_vendor_id == CPU_VENDOR_AMD) { - /* - * Detect the presence of C1E capability mostly on latest - * dual-cores (or future) k8 family. This feature renders - * the local APIC timer dead, so we disable it by reading - * the Interrupt Pending Message register and clearing both - * C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27). - * - * Reference: - * "BIOS and Kernel Developer's Guide for AMD NPT - * Family 0Fh Processors" - * #32559 revision 3.00 - */ - if ((cpu_id & 0x00000f00) == 0x00000f00 && - (cpu_id & 0x0fff0000) >= 0x00040000) { - uint64_t msr; - - msr = rdmsr(0xc0010055); - if (msr & 0x18000000) - wrmsr(0xc0010055, msr & ~0x18000000ULL); - } - } - intr_restore(eflags); } --- sys/amd64/amd64/machdep.c.orig 2008-04-25 13:18:47.000000000 +0800 +++ sys/amd64/amd64/machdep.c 2008-05-25 19:48:50.000000000 +0800 @@ -588,6 +588,69 @@ __asm __volatile("sti; hlt"); } +static int cpu_ident_amdc1e = 0; + +static int +cpu_probe_amdc1e(void) +{ + int i; + + /* + * Forget it, if we're not using local APIC timer. + */ + if (resource_disabled("apic", 0) || + (resource_int_value("apic", 0, "clock", &i) == 0 && i == 0)) + return (0); + + /* + * Detect the presence of C1E capability mostly on latest + * dual-cores (or future) k8 family. + */ + if (cpu_vendor_id == CPU_VENDOR_AMD && + (cpu_id & 0x00000f00) == 0x00000f00 && + (cpu_id & 0x0fff0000) >= 0x00040000) { + cpu_ident_amdc1e = 1; + return (1); + } + + return (0); +} + +/* + * C1E renders the local APIC timer dead, so we disable it by + * reading the Interrupt Pending Message register and clearing + * both C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27). + * + * Reference: + * "BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh Processors" + * #32559 revision 3.00+ + */ +#define MSR_AMDK8_IPM 0xc0010055 +#define AMDK8_SMIONCMPHALT (1ULL << 27) +#define AMDK8_C1EONCMPHALT (1ULL << 28) +#define AMDK8_CMPHALT (AMDK8_SMIONCMPHALT | AMDK8_C1EONCMPHALT) + +static void +cpu_idle_amdc1e(int busy) +{ + + disable_intr(); + if (sched_runnable()) + enable_intr(); + else { + uint64_t msr; + + msr = rdmsr(MSR_AMDK8_IPM); + if (msr & AMDK8_CMPHALT) + wrmsr(MSR_AMDK8_IPM, msr & ~AMDK8_CMPHALT); + + if (cpu_idle_hook) + cpu_idle_hook(); + else + __asm __volatile("sti; hlt"); + } +} + static void cpu_idle_spin(int busy) { @@ -685,6 +748,7 @@ { cpu_idle_spin, "spin" }, { cpu_idle_mwait, "mwait" }, { cpu_idle_mwait_hlt, "mwait_hlt" }, + { cpu_idle_amdc1e, "amdc1e" }, { cpu_idle_hlt, "hlt" }, { cpu_idle_acpi, "acpi" }, { NULL, NULL } @@ -703,6 +767,9 @@ if (strstr(idle_tbl[i].id_name, "mwait") && (cpu_feature2 & CPUID2_MON) == 0) continue; + if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 && + cpu_ident_amdc1e == 0) + continue; p += sprintf(p, "%s, ", idle_tbl[i].id_name); } error = sysctl_handle_string(oidp, avail, 0, req); @@ -733,6 +800,9 @@ if (strstr(idle_tbl[i].id_name, "mwait") && (cpu_feature2 & CPUID2_MON) == 0) continue; + if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 && + cpu_ident_amdc1e == 0) + continue; if (strcmp(idle_tbl[i].id_name, buf)) continue; cpu_idle_fn = idle_tbl[i].id_fn; @@ -1581,6 +1651,9 @@ } #endif + if (cpu_probe_amdc1e()) + cpu_idle_fn = cpu_idle_amdc1e; + /* Location of kernel stack for locore */ return ((u_int64_t)thread0.td_pcb); } --- sys/i386/i386/local_apic.c.orig 2008-05-24 14:27:02.000000000 +0800 +++ sys/i386/i386/local_apic.c 2008-05-25 13:02:15.000000000 +0800 @@ -331,29 +331,6 @@ /* XXX: Error and thermal LVTs */ - if (cpu_vendor_id == CPU_VENDOR_AMD) { - /* - * Detect the presence of C1E capability mostly on latest - * dual-cores (or future) k8 family. This feature renders - * the local APIC timer dead, so we disable it by reading - * the Interrupt Pending Message register and clearing both - * C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27). - * - * Reference: - * "BIOS and Kernel Developer's Guide for AMD NPT - * Family 0Fh Processors" - * #32559 revision 3.00 - */ - if ((cpu_id & 0x00000f00) == 0x00000f00 && - (cpu_id & 0x0fff0000) >= 0x00040000) { - uint64_t msr; - - msr = rdmsr(0xc0010055); - if (msr & 0x18000000) - wrmsr(0xc0010055, msr & ~0x18000000ULL); - } - } - intr_restore(eflags); } --- sys/i386/i386/machdep.c.orig 2008-04-25 13:18:48.000000000 +0800 +++ sys/i386/i386/machdep.c 2008-05-25 19:47:49.000000000 +0800 @@ -1219,6 +1219,70 @@ __asm __volatile("sti; hlt"); } +static int cpu_ident_amdc1e = 0; + +static int +cpu_probe_amdc1e(void) +{ +#ifdef DEV_APIC + int i; + + /* + * Forget it, if we're not using local APIC timer. + */ + if (resource_disabled("apic", 0) || + (resource_int_value("apic", 0, "clock", &i) == 0 && i == 0)) + return (0); + + /* + * Detect the presence of C1E capability mostly on latest + * dual-cores (or future) k8 family. + */ + if (cpu_vendor_id == CPU_VENDOR_AMD && + (cpu_id & 0x00000f00) == 0x00000f00 && + (cpu_id & 0x0fff0000) >= 0x00040000) { + cpu_ident_amdc1e = 1; + return (1); + } +#endif + return (0); +} + +/* + * C1E renders the local APIC timer dead, so we disable it by + * reading the Interrupt Pending Message register and clearing + * both C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27). + * + * Reference: + * "BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh Processors" + * #32559 revision 3.00+ + */ +#define MSR_AMDK8_IPM 0xc0010055 +#define AMDK8_SMIONCMPHALT (1ULL << 27) +#define AMDK8_C1EONCMPHALT (1ULL << 28) +#define AMDK8_CMPHALT (AMDK8_SMIONCMPHALT | AMDK8_C1EONCMPHALT) + +static void +cpu_idle_amdc1e(int busy) +{ + + disable_intr(); + if (sched_runnable()) + enable_intr(); + else { + uint64_t msr; + + msr = rdmsr(MSR_AMDK8_IPM); + if (msr & AMDK8_CMPHALT) + wrmsr(MSR_AMDK8_IPM, msr & ~AMDK8_CMPHALT); + + if (cpu_idle_hook) + cpu_idle_hook(); + else + __asm __volatile("sti; hlt"); + } +} + static void cpu_idle_spin(int busy) { @@ -1320,6 +1384,7 @@ { cpu_idle_spin, "spin" }, { cpu_idle_mwait, "mwait" }, { cpu_idle_mwait_hlt, "mwait_hlt" }, + { cpu_idle_amdc1e, "amdc1e" }, { cpu_idle_hlt, "hlt" }, { cpu_idle_acpi, "acpi" }, { NULL, NULL } @@ -1338,6 +1403,9 @@ if (strstr(idle_tbl[i].id_name, "mwait") && (cpu_feature2 & CPUID2_MON) == 0) continue; + if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 && + cpu_ident_amdc1e == 0) + continue; p += sprintf(p, "%s, ", idle_tbl[i].id_name); } error = sysctl_handle_string(oidp, avail, 0, req); @@ -1368,6 +1436,9 @@ if (strstr(idle_tbl[i].id_name, "mwait") && (cpu_feature2 & CPUID2_MON) == 0) continue; + if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 && + cpu_ident_amdc1e == 0) + continue; if (strcmp(idle_tbl[i].id_name, buf)) continue; cpu_idle_fn = idle_tbl[i].id_fn; @@ -2571,6 +2642,9 @@ thread0.td_frame = &proc0_tf; thread0.td_pcb->pcb_fsd = PCPU_GET(fsgs_gdt)[0]; thread0.td_pcb->pcb_gsd = PCPU_GET(fsgs_gdt)[1]; + + if (cpu_probe_amdc1e()) + cpu_idle_fn = cpu_idle_amdc1e; } #else @@ -2835,6 +2909,9 @@ #endif thread0.td_pcb->pcb_ext = 0; thread0.td_frame = &proc0_tf; + + if (cpu_probe_amdc1e()) + cpu_idle_fn = cpu_idle_amdc1e; } #endif --- sys/i386/isa/clock.c.orig 2009-05-04 01:47:21.000000000 +0800 +++ sys/i386/isa/clock.c 2009-05-04 23:42:52.000000000 +0800 @@ -108,6 +108,9 @@ static int using_atrtc_timer; static int using_lapic_timer; +static u_int stat_ticks = 0; +static u_int prof_ticks = 0; + /* Values for timerX_state: */ #define RELEASED 0 #define RELEASE_PENDING 1 @@ -137,8 +140,6 @@ hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); else hardclock_cpu(TRAPF_USERMODE(frame)); - if (!using_atrtc_timer) - statclockintr(frame); return (FILTER_HANDLED); } @@ -146,8 +147,6 @@ statclockintr(struct trapframe *frame) { - if (profprocs != 0) - profclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); statclock(TRAPF_USERMODE(frame)); return (FILTER_HANDLED); } @@ -193,6 +192,30 @@ ipi_all_but_self(IPI_HARDCLOCK); #endif hardclockintr(frame); + + if (!using_atrtc_timer) { + prof_ticks += profhz; + if (prof_ticks >= hz) { + prof_ticks -= hz; + if (profprocs != 0) { +#ifdef SMP + if (smp_started) + ipi_all_but_self(IPI_PROFCLOCK); +#endif + profclockintr(frame); + } + } + stat_ticks += stathz; + if (stat_ticks >= hz) { + stat_ticks -= hz; +#ifdef SMP + if (smp_started) + ipi_all_but_self(IPI_STATCLOCK); +#endif + statclockintr(frame); + } + } + #ifdef DEV_MCA /* Reset clock interrupt by asserting bit 7 of port 0x61 */ if (MCA_system) @@ -549,7 +572,8 @@ INTR_TYPE_CLK, NULL); atrtc_enable_intr(); } else { - profhz = stathz = hz; + profhz = min(RTC_PROFRATE, hz); + stathz = min(RTC_NOPROFRATE, hz); } } --- sys/amd64/isa/clock.c.orig 2009-05-04 01:47:21.000000000 +0800 +++ sys/amd64/isa/clock.c 2009-05-04 23:52:24.000000000 +0800 @@ -93,6 +93,9 @@ static int using_atrtc_timer; static int using_lapic_timer; +static u_int stat_ticks = 0; +static u_int prof_ticks = 0; + /* Values for timerX_state: */ #define RELEASED 0 #define RELEASE_PENDING 1 @@ -122,8 +125,6 @@ hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); else hardclock_cpu(TRAPF_USERMODE(frame)); - if (!using_atrtc_timer) - statclockintr(frame); return (FILTER_HANDLED); } @@ -131,8 +132,6 @@ statclockintr(struct trapframe *frame) { - if (profprocs != 0) - profclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); statclock(TRAPF_USERMODE(frame)); return (FILTER_HANDLED); } @@ -166,6 +165,30 @@ ipi_all_but_self(IPI_HARDCLOCK); #endif hardclockintr(frame); + + if (!using_atrtc_timer) { + prof_ticks += profhz; + if (prof_ticks >= hz) { + prof_ticks -= hz; + if (profprocs != 0) { +#ifdef SMP + if (smp_started) + ipi_all_but_self(IPI_PROFCLOCK); +#endif + profclockintr(frame); + } + } + stat_ticks += stathz; + if (stat_ticks >= hz) { + stat_ticks -= hz; +#ifdef SMP + if (smp_started) + ipi_all_but_self(IPI_STATCLOCK); +#endif + statclockintr(frame); + } + } + return (FILTER_HANDLED); } @@ -500,7 +523,8 @@ INTR_TYPE_CLK, NULL); atrtc_enable_intr(); } else { - profhz = stathz = hz; + profhz = min(RTC_PROFRATE, hz); + stathz = min(RTC_NOPROFRATE, hz); } }