Index: sys/amd64/amd64/pmap.c =================================================================== --- sys/amd64/amd64/pmap.c (revision 215402) +++ sys/amd64/amd64/pmap.c (working copy) @@ -180,10 +180,18 @@ pt_entry_t pg_nx; SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); +static int pat_works = 1; +TUNABLE_INT("vm.pmap.pat_works", &pat_works); +SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RDTUN, &pat_works, 1, + "Is page attribute table fully functional?"); + static int pg_ps_enabled = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN, &pg_ps_enabled, 0, "Are large page mappings enabled?"); +#define PAT_INDEX_SIZE 8 +static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ + static u_int64_t KPTphys; /* phys addr of kernel level 1 */ static u_int64_t KPDphys; /* phys addr of kernel level 2 */ u_int64_t KPDPphys; /* phys addr of kernel level 3 */ @@ -592,22 +600,80 @@ pmap_bootstrap(vm_paddr_t *firstaddr) void pmap_init_pat(void) { + int pat_table[PAT_INDEX_SIZE]; uint64_t pat_msr; + u_long cr0, cr4; + int i; /* Bail if this CPU doesn't implement PAT. */ - if (!(cpu_feature & CPUID_PAT)) + if ((cpu_feature & CPUID_PAT) == 0) panic("no PAT??"); - /* - * Leave the indices 0-3 at the default of WB, WT, UC, and UC-. - * Program 4 and 5 as WP and WC. - * Leave 6 and 7 as UC and UC-. - */ - pat_msr = rdmsr(MSR_PAT); - pat_msr &= ~(PAT_MASK(4) | PAT_MASK(5)); - pat_msr |= PAT_VALUE(4, PAT_WRITE_PROTECTED) | - PAT_VALUE(5, PAT_WRITE_COMBINING); + /* Set default PAT index table. */ + for (i = 0; i < PAT_INDEX_SIZE; i++) + pat_table[i] = -1; + pat_table[PAT_WRITE_BACK] = 0; + pat_table[PAT_WRITE_THROUGH] = 1; + pat_table[PAT_UNCACHEABLE] = 3; + pat_table[PAT_WRITE_COMBINING] = 3; + pat_table[PAT_WRITE_PROTECTED] = 3; + pat_table[PAT_UNCACHED] = 3; + + /* Initialize default PAT entries. */ + pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) | + PAT_VALUE(1, PAT_WRITE_THROUGH) | + PAT_VALUE(2, PAT_UNCACHED) | + PAT_VALUE(3, PAT_UNCACHEABLE) | + PAT_VALUE(4, PAT_WRITE_BACK) | + PAT_VALUE(5, PAT_WRITE_THROUGH) | + PAT_VALUE(6, PAT_UNCACHED) | + PAT_VALUE(7, PAT_UNCACHEABLE); + + if (pat_works) { + /* + * Leave the indices 0-3 at the default of WB, WT, UC-, and UC. + * Program 5 and 6 as WP and WC. + * Leave 4 and 7 as WB and UC. + */ + pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6)); + pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) | + PAT_VALUE(6, PAT_WRITE_COMBINING); + pat_table[PAT_UNCACHED] = 2; + pat_table[PAT_WRITE_PROTECTED] = 5; + pat_table[PAT_WRITE_COMBINING] = 6; + } else { + /* + * Just replace PAT Index 2 with WC instead of UC-. + */ + pat_msr &= ~PAT_MASK(2); + pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING); + pat_table[PAT_WRITE_COMBINING] = 2; + } + + /* Disable PGE. */ + cr4 = rcr4(); + load_cr4(cr4 & ~CR4_PGE); + + /* Disable caches (CD = 1, NW = 0). */ + cr0 = rcr0(); + load_cr0((cr0 & ~CR0_NW) | CR0_CD); + + /* Flushes caches and TLBs. */ + wbinvd(); + invltlb(); + + /* Update PAT and index table. */ wrmsr(MSR_PAT, pat_msr); + for (i = 0; i < PAT_INDEX_SIZE; i++) + pat_index[i] = pat_table[i]; + + /* Flush caches and TLBs again. */ + wbinvd(); + invltlb(); + + /* Restore caches and PGE. */ + load_cr0(cr0); + load_cr4(cr4); } /* @@ -759,42 +825,24 @@ SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, C static int pmap_cache_bits(int mode, boolean_t is_pde) { - int pat_flag, pat_index, cache_bits; + int cache_bits, pat_flag, pat_idx; + if (mode < 0 || mode >= PAT_INDEX_SIZE || pat_index[mode] < 0) + panic("Unknown caching mode %d\n", mode); + /* The PAT bit is different for PTE's and PDE's. */ pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT; /* Map the caching mode to a PAT index. */ - switch (mode) { - case PAT_UNCACHEABLE: - pat_index = 3; - break; - case PAT_WRITE_THROUGH: - pat_index = 1; - break; - case PAT_WRITE_BACK: - pat_index = 0; - break; - case PAT_UNCACHED: - pat_index = 2; - break; - case PAT_WRITE_COMBINING: - pat_index = 5; - break; - case PAT_WRITE_PROTECTED: - pat_index = 4; - break; - default: - panic("Unknown caching mode %d\n", mode); - } + pat_idx = pat_index[mode]; /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */ cache_bits = 0; - if (pat_index & 0x4) + if (pat_idx & 0x4) cache_bits |= pat_flag; - if (pat_index & 0x2) + if (pat_idx & 0x2) cache_bits |= PG_NC_PCD; - if (pat_index & 0x1) + if (pat_idx & 0x1) cache_bits |= PG_NC_PWT; return (cache_bits); } Index: sys/i386/i386/pmap.c =================================================================== --- sys/i386/i386/pmap.c (revision 215402) +++ sys/i386/i386/pmap.c (working copy) @@ -213,14 +213,20 @@ pt_entry_t pg_nx; static uma_zone_t pdptzone; #endif -static int pat_works; /* Is page attribute table sane? */ - SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); +static int pat_works = 1; +TUNABLE_INT("vm.pmap.pat_works", &pat_works); +SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RDTUN, &pat_works, 1, + "Is page attribute table fully functional?"); + static int pg_ps_enabled; SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN, &pg_ps_enabled, 0, "Are large page mappings enabled?"); +#define PAT_INDEX_SIZE 8 +static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ + /* * Data for the pv entry allocation mechanism */ @@ -505,43 +511,99 @@ pmap_bootstrap(vm_paddr_t firstaddr) void pmap_init_pat(void) { + int pat_table[PAT_INDEX_SIZE]; uint64_t pat_msr; + u_long cr0, cr4; + int i; + /* Set default PAT index table. */ + for (i = 0; i < PAT_INDEX_SIZE; i++) + pat_table[i] = -1; + pat_table[PAT_WRITE_BACK] = 0; + pat_table[PAT_WRITE_THROUGH] = 1; + pat_table[PAT_UNCACHEABLE] = 3; + pat_table[PAT_WRITE_COMBINING] = 3; + pat_table[PAT_WRITE_PROTECTED] = 3; + pat_table[PAT_UNCACHED] = 3; + /* Bail if this CPU doesn't implement PAT. */ - if (!(cpu_feature & CPUID_PAT)) + if ((cpu_feature & CPUID_PAT) == 0) { + for (i = 0; i < PAT_INDEX_SIZE; i++) + pat_index[i] = pat_table[i]; + pat_works = 0; return; + } - if (cpu_vendor_id != CPU_VENDOR_INTEL || - (CPUID_TO_FAMILY(cpu_id) == 6 && CPUID_TO_MODEL(cpu_id) >= 0xe)) { + /* + * Due to some Intel errata, we can only safely use the lower 4 + * PAT entries. + * + * Intel Pentium III Processor Specification Update + * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B + * or Mode C Paging) + * + * Intel Pentium IV Processor Specification Update + * Errata N46 (PAT Index MSB May Be Calculated Incorrectly) + */ + if (cpu_vendor_id == CPU_VENDOR_INTEL && + !(CPUID_TO_FAMILY(cpu_id) == 6 && CPUID_TO_MODEL(cpu_id) >= 0xe)) + pat_works = 0; + + /* Initialize default PAT entries. */ + pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) | + PAT_VALUE(1, PAT_WRITE_THROUGH) | + PAT_VALUE(2, PAT_UNCACHED) | + PAT_VALUE(3, PAT_UNCACHEABLE) | + PAT_VALUE(4, PAT_WRITE_BACK) | + PAT_VALUE(5, PAT_WRITE_THROUGH) | + PAT_VALUE(6, PAT_UNCACHED) | + PAT_VALUE(7, PAT_UNCACHEABLE); + + if (pat_works) { /* - * Leave the indices 0-3 at the default of WB, WT, UC, and UC-. - * Program 4 and 5 as WP and WC. - * Leave 6 and 7 as UC and UC-. + * Leave the indices 0-3 at the default of WB, WT, UC-, and UC. + * Program 5 and 6 as WP and WC. + * Leave 4 and 7 as WB and UC. */ - pat_msr = rdmsr(MSR_PAT); - pat_msr &= ~(PAT_MASK(4) | PAT_MASK(5)); - pat_msr |= PAT_VALUE(4, PAT_WRITE_PROTECTED) | - PAT_VALUE(5, PAT_WRITE_COMBINING); - pat_works = 1; + pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6)); + pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) | + PAT_VALUE(6, PAT_WRITE_COMBINING); + pat_table[PAT_UNCACHED] = 2; + pat_table[PAT_WRITE_PROTECTED] = 5; + pat_table[PAT_WRITE_COMBINING] = 6; } else { /* - * Due to some Intel errata, we can only safely use the lower 4 - * PAT entries. Thus, just replace PAT Index 2 with WC instead - * of UC-. - * - * Intel Pentium III Processor Specification Update - * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B - * or Mode C Paging) - * - * Intel Pentium IV Processor Specification Update - * Errata N46 (PAT Index MSB May Be Calculated Incorrectly) + * Just replace PAT Index 2 with WC instead of UC-. */ - pat_msr = rdmsr(MSR_PAT); pat_msr &= ~PAT_MASK(2); pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING); - pat_works = 0; + pat_table[PAT_WRITE_COMBINING] = 2; } + + /* Disable PGE. */ + cr4 = rcr4(); + load_cr4(cr4 & ~CR4_PGE); + + /* Disable caches (CD = 1, NW = 0). */ + cr0 = rcr0(); + load_cr0((cr0 & ~CR0_NW) | CR0_CD); + + /* Flushes caches and TLBs. */ + wbinvd(); + invltlb(); + + /* Update PAT and index table. */ wrmsr(MSR_PAT, pat_msr); + for (i = 0; i < PAT_INDEX_SIZE; i++) + pat_index[i] = pat_table[i]; + + /* Flush caches and TLBs again. */ + wbinvd(); + invltlb(); + + /* Restore caches and PGE. */ + load_cr0(cr0); + load_cr4(cr4); } /* @@ -777,78 +839,24 @@ SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, C int pmap_cache_bits(int mode, boolean_t is_pde) { - int pat_flag, pat_index, cache_bits; + int cache_bits, pat_flag, pat_idx; + if (mode < 0 || mode >= PAT_INDEX_SIZE || pat_index[mode] < 0) + panic("Unknown caching mode %d\n", mode); + /* The PAT bit is different for PTE's and PDE's. */ pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT; - /* If we don't support PAT, map extended modes to older ones. */ - if (!(cpu_feature & CPUID_PAT)) { - switch (mode) { - case PAT_UNCACHEABLE: - case PAT_WRITE_THROUGH: - case PAT_WRITE_BACK: - break; - case PAT_UNCACHED: - case PAT_WRITE_COMBINING: - case PAT_WRITE_PROTECTED: - mode = PAT_UNCACHEABLE; - break; - } - } - /* Map the caching mode to a PAT index. */ - if (pat_works) { - switch (mode) { - case PAT_UNCACHEABLE: - pat_index = 3; - break; - case PAT_WRITE_THROUGH: - pat_index = 1; - break; - case PAT_WRITE_BACK: - pat_index = 0; - break; - case PAT_UNCACHED: - pat_index = 2; - break; - case PAT_WRITE_COMBINING: - pat_index = 5; - break; - case PAT_WRITE_PROTECTED: - pat_index = 4; - break; - default: - panic("Unknown caching mode %d\n", mode); - } - } else { - switch (mode) { - case PAT_UNCACHED: - case PAT_UNCACHEABLE: - case PAT_WRITE_PROTECTED: - pat_index = 3; - break; - case PAT_WRITE_THROUGH: - pat_index = 1; - break; - case PAT_WRITE_BACK: - pat_index = 0; - break; - case PAT_WRITE_COMBINING: - pat_index = 2; - break; - default: - panic("Unknown caching mode %d\n", mode); - } - } + pat_idx = pat_index[mode]; /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */ cache_bits = 0; - if (pat_index & 0x4) + if (pat_idx & 0x4) cache_bits |= pat_flag; - if (pat_index & 0x2) + if (pat_idx & 0x2) cache_bits |= PG_NC_PCD; - if (pat_index & 0x1) + if (pat_idx & 0x1) cache_bits |= PG_NC_PWT; return (cache_bits); }