Index: sparc64/include/cache.h =================================================================== --- sparc64/include/cache.h (revision 223042) +++ sparc64/include/cache.h (working copy) @@ -110,15 +110,16 @@ cache_flush_t spitfire_cache_flush; dcache_page_inval_t spitfire_dcache_page_inval; icache_page_inval_t spitfire_icache_page_inval; +cache_enable_t zeus_cache_enable; +cache_flush_t zeus_cache_flush; +dcache_page_inval_t zeus_dcache_page_inval; +icache_page_inval_t zeus_icache_page_inval; + extern cache_enable_t *cache_enable; extern cache_flush_t *cache_flush; extern dcache_page_inval_t *dcache_page_inval; extern icache_page_inval_t *icache_page_inval; -cache_flush_t zeus_cache_flush; -dcache_page_inval_t zeus_dcache_page_inval; -icache_page_inval_t zeus_icache_page_inval; - #endif /* KERNEL */ #endif /* !LOCORE */ Index: sparc64/include/cpu.h =================================================================== --- sparc64/include/cpu.h (revision 223042) +++ sparc64/include/cpu.h (working copy) @@ -57,6 +57,7 @@ void cpu_halt(void); void cpu_reset(void); void fork_trampoline(void); void swi_vm(void *v); +void zeus_init(u_int cpu_impl); static __inline u_int64_t get_cyclecount(void) Index: sparc64/include/tlb.h =================================================================== --- sparc64/include/tlb.h (revision 223042) +++ sparc64/include/tlb.h (working copy) @@ -44,7 +44,17 @@ (TD_V | TD_4M | (TLB_DIRECT_ADDRESS_MASK - TLB_DIRECT_PAGE_MASK)) #define TLB_DAR_SLOT_SHIFT (3) -#define TLB_DAR_SLOT(slot) ((slot) << TLB_DAR_SLOT_SHIFT) +#define TLB_DAR_TLB_SHIFT (16) +#define TLB_DAR_SLOT(tlb, slot) \ + ((tlb) << TLB_DAR_TLB_SHIFT | (slot) << TLB_DAR_SLOT_SHIFT) +#define TLB_DAR_T16 (0) /* US-III{,i,+}, US-IV{,+} */ +#define TLB_DAR_T32 (0) /* US-I, II{,e,i} */ +#define TLB_DAR_DT512_0 (2) /* US-III{,i,+}, US-IV{,+} */ +#define TLB_DAR_DT512_1 (3) /* US-III{,i,+}, US-IV{,+} */ +#define TLB_DAR_IT128 (2) /* US-III{,i,+}, US-IV */ +#define TLB_DAR_IT512 (2) /* US-IV+ */ +#define TLB_DAR_FTLB (0) /* SPARC64 V, VI, VII, VIIIfx */ +#define TLB_DAR_STLB (2) /* SPARC64 V, VI, VII, VIIIfx */ #define TAR_VPN_SHIFT (13) #define TAR_CTX_MASK ((1 << TAR_VPN_SHIFT) - 1) @@ -156,6 +166,9 @@ tlb_flush_user_t cheetah_tlb_flush_user; tlb_flush_nonlocked_t spitfire_tlb_flush_nonlocked; tlb_flush_user_t spitfire_tlb_flush_user; +tlb_flush_nonlocked_t zeus_tlb_flush_nonlocked; +tlb_flush_user_t zeus_tlb_flush_user; + extern tlb_flush_nonlocked_t *tlb_flush_nonlocked; extern tlb_flush_user_t *tlb_flush_user; Index: sparc64/sparc64/cache.c =================================================================== --- sparc64/sparc64/cache.c (revision 223042) +++ sparc64/sparc64/cache.c (working copy) @@ -169,12 +169,12 @@ cache_init(struct pcpu *pcpu) tlb_flush_nonlocked = cheetah_tlb_flush_nonlocked; tlb_flush_user = cheetah_tlb_flush_user; } else if (pcpu->pc_impl == CPU_IMPL_SPARC64V) { - cache_enable = cheetah_cache_enable; + cache_enable = zeus_cache_enable; cache_flush = zeus_cache_flush; dcache_page_inval = zeus_dcache_page_inval; icache_page_inval = zeus_icache_page_inval; - tlb_flush_nonlocked = cheetah_tlb_flush_nonlocked; - tlb_flush_user = cheetah_tlb_flush_user; + tlb_flush_nonlocked = zeus_tlb_flush_nonlocked; + tlb_flush_user = zeus_tlb_flush_user; } else if (pcpu->pc_impl >= CPU_IMPL_ULTRASPARCI && pcpu->pc_impl < CPU_IMPL_ULTRASPARCIII) { cache_enable = spitfire_cache_enable; Index: sparc64/sparc64/cheetah.c =================================================================== --- sparc64/sparc64/cheetah.c (revision 223042) +++ sparc64/sparc64/cheetah.c (working copy) @@ -1,6 +1,6 @@ /*- * Copyright (c) 2003 Jake Burkholder. - * Copyright (c) 2005, 2008, 2010 Marius Strobl + * Copyright (c) 2005 - 2011 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -45,17 +45,19 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include #include #include #define CHEETAH_ICACHE_TAG_LOWER 0x30 +#define CHEETAH_T16_ENTRIES 16 +#define CHEETAH_DT512_ENTRIES 512 +#define CHEETAH_IT128_ENTRIES 128 +#define CHEETAH_IT512_ENTRIES 512 /* - * CPU-specific initialization - this is used for both the Sun Cheetah and - * later as well as the Fujitsu Zeus and later CPUs. + * CPU-specific initialization for Sun Cheetah and later CPUs */ void cheetah_init(u_int cpu_impl) @@ -78,14 +80,6 @@ cheetah_init(u_int cpu_impl) stxa(AA_IMMU_TSB_NEXT_REG, ASI_IMMU, 0); membar(Sync); - if (cpu_impl == CPU_IMPL_SPARC64V) { - /* Ensure MCNTL_JPS1_TSBP is 0. */ - val = ldxa(AA_MCNTL, ASI_MCNTL); - val &= ~MCNTL_JPS1_TSBP; - stxa(AA_MCNTL, ASI_MCNTL, val); - return; - } - /* * Configure the first large dTLB to hold 4MB pages (e.g. for direct * mappings) for all three contexts and ensure the second one is set @@ -223,33 +217,92 @@ cheetah_icache_page_inval(vm_paddr_t pa __unused) } -#define cheetah_dmap_all() do { \ - stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); \ - stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0); \ - flush(KERNBASE); \ -} while (0) - /* - * Flush all non-locked mappings from the TLB. + * Flush all non-locked mappings from the TLBs. */ void cheetah_tlb_flush_nonlocked(void) { - cheetah_dmap_all(); + stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); + stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0); + flush(KERNBASE); } /* - * Flush all user mappings from the TLB. + * Flush all user mappings from the TLBs. */ void -cheetah_tlb_flush_user() +cheetah_tlb_flush_user(void) { + u_long data, tag; + register_t s; + u_int i, slot; /* - * Just use cheetah_dmap_all() and accept somes TLB misses - * rather than searching all 1040 D-TLB and 144 I-TLB slots - * for non-kernel mappings. + * We read ASI_{D,I}TLB_DATA_ACCESS_REG twice back-to-back in order + * to work around errata of USIII and beyond. */ - cheetah_dmap_all(); + for (i = 0; i < CHEETAH_T16_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_T16, i); + s = intr_disable(); + (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + for (i = 0; i < CHEETAH_DT512_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_DT512_0, i); + s = intr_disable(); + (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + slot = TLB_DAR_SLOT(TLB_DAR_DT512_1, i); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + } + if (PCPU_GET(impl) == CPU_IMPL_ULTRASPARCIVp) { + for (i = 0; i < CHEETAH_IT512_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_IT512, i); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + intr_restore(s); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + } else { + for (i = 0; i < CHEETAH_IT128_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_IT128, i); + s = intr_disable(); + (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + intr_restore(s); + if ((data & TD_V) != 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + } } Index: sparc64/sparc64/machdep.c =================================================================== --- sparc64/sparc64/machdep.c (revision 223042) +++ sparc64/sparc64/machdep.c (working copy) @@ -348,9 +348,10 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_ /* * Do CPU-specific initialization. */ - if (cpu_impl == CPU_IMPL_SPARC64V || - cpu_impl >= CPU_IMPL_ULTRASPARCIII) + if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) cheetah_init(cpu_impl); + else if (cpu_impl == CPU_IMPL_SPARC64V) + zeus_init(cpu_impl); /* * Clear (S)TICK timer (including NPT). Index: sparc64/sparc64/mp_machdep.c =================================================================== --- sparc64/sparc64/mp_machdep.c (revision 223347) +++ sparc64/sparc64/mp_machdep.c (working copy) @@ -428,9 +426,11 @@ cpu_mp_bootstrap(struct pcpu *pc) csa = &cpu_start_args; /* Do CPU-specific initialization. */ - if (pc->pc_impl == CPU_IMPL_SPARC64V || - pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) + if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) cheetah_init(pc->pc_impl); + else if (pc->pc_impl == CPU_IMPL_SPARC64V) + zeus_init(pc->pc_impl); + /* * Enable the caches. Note that his may include applying workarounds. */ Index: sparc64/sparc64/pmap.c =================================================================== --- sparc64/sparc64/pmap.c (revision 223378) +++ sparc64/sparc64/pmap.c (working copy) @@ -247,7 +247,7 @@ PMAP_STATS_VAR(pmap_ncopy_page_soc); PMAP_STATS_VAR(pmap_nnew_thread); PMAP_STATS_VAR(pmap_nnew_thread_oc); -static inline u_long dtlb_get_data(u_int slot); +static inline u_long dtlb_get_data(u_int tlb, u_int slot); /* * Quick sort callout for comparing memory regions @@ -288,15 +288,21 @@ om_cmp(const void *a, const void *b) } static inline u_long -dtlb_get_data(u_int slot) +dtlb_get_data(u_int tlb, u_int slot) { + u_long data; + register_t s; + slot = TLB_DAR_SLOT(tlb, slot); /* - * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work - * around errata of USIII and beyond. + * We read ASI_DTLB_DATA_ACCESS_REG twice back-to-back in order to + * work around errata of USIII and beyond. */ - (void)ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG); - return (ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG)); + s = intr_disable(); + (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + intr_restore(s); + return (data); } /* @@ -392,7 +398,9 @@ pmap_bootstrap(u_int cpu_impl) } else { dtlb_slots_avail = 0; for (i = 0; i < dtlb_slots; i++) { - data = dtlb_get_data(i); + data = dtlb_get_data(cpu_impl == + CPU_IMPL_ULTRASPARCIII ? TLB_DAR_T16 : + TLB_DAR_T32, i); if ((data & (TD_V | TD_L)) != (TD_V | TD_L)) dtlb_slots_avail++; } Index: sparc64/sparc64/spitfire.c =================================================================== --- sparc64/sparc64/spitfire.c (revision 223042) +++ sparc64/sparc64/spitfire.c (working copy) @@ -140,47 +140,45 @@ spitfire_icache_page_inval(vm_paddr_t pa) } /* - * Flush all non-locked mappings from the TLB. + * Flush all non-locked mappings from the TLBs. */ void spitfire_tlb_flush_nonlocked(void) { - int i; + u_int i; + u_int slot; for (i = 0; i < SPITFIRE_TLB_ENTRIES; i++) { - if ((ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG) & - TD_L) == 0) - stxa_sync(TLB_DAR_SLOT(i), - ASI_DTLB_DATA_ACCESS_REG, 0); - if ((ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG) & - TD_L) == 0) - stxa_sync(TLB_DAR_SLOT(i), - ASI_ITLB_DATA_ACCESS_REG, 0); + slot = TLB_DAR_SLOT(TLB_DAR_T32, i); + if ((ldxa(slot, ASI_DTLB_DATA_ACCESS_REG) & TD_L) == 0) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + if ((ldxa(slot, ASI_ITLB_DATA_ACCESS_REG) & TD_L) == 0) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); } } /* - * Flush all user mappings from the TLB. + * Flush all user mappings from the TLBs. */ void spitfire_tlb_flush_user(void) { u_long data; u_long tag; - int i; + u_int i; + u_int slot; for (i = 0; i < SPITFIRE_TLB_ENTRIES; i++) { - data = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG); - tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); + slot = TLB_DAR_SLOT(TLB_DAR_T32, i); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); if ((data & TD_V) != 0 && (data & TD_L) == 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) - stxa_sync(TLB_DAR_SLOT(i), - ASI_DTLB_DATA_ACCESS_REG, 0); - data = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG); - tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); if ((data & TD_V) != 0 && (data & TD_L) == 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) - stxa_sync(TLB_DAR_SLOT(i), - ASI_ITLB_DATA_ACCESS_REG, 0); + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); } } Index: sparc64/sparc64/zeus.c =================================================================== --- sparc64/sparc64/zeus.c (revision 223042) +++ sparc64/sparc64/zeus.c (working copy) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2010 Marius Strobl + * Copyright (c) 2010 - 2011 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,9 +32,66 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include +#include +#include +#include +#include +#include +#define ZEUS_FTLB_ENTRIES 32 +#define ZEUS_STLB_ENTRIES 2048 + /* + * CPU-specific initialization for Fujitsu Zeus CPUs + */ +void +zeus_init(u_int cpu_impl) +{ + u_long val; + + /* Ensure the TSB Extension Registers hold 0 as TSB_Base. */ + + stxa(AA_DMMU_TSB_PEXT_REG, ASI_DMMU, 0); + stxa(AA_IMMU_TSB_PEXT_REG, ASI_IMMU, 0); + membar(Sync); + + stxa(AA_DMMU_TSB_SEXT_REG, ASI_DMMU, 0); + /* + * NB: the secondary context was removed from the iMMU. + */ + membar(Sync); + + stxa(AA_DMMU_TSB_NEXT_REG, ASI_DMMU, 0); + stxa(AA_IMMU_TSB_NEXT_REG, ASI_IMMU, 0); + membar(Sync); + + val = ldxa(AA_MCNTL, ASI_MCNTL); + /* Ensure MCNTL_JPS1_TSBP is 0. */ + val &= ~MCNTL_JPS1_TSBP; + /* + * Ensure 4-Mbyte page entries are stored in 1024-entry, 2-way set + * associative TLB. + */ + val = (val & ~MCNTL_RMD_MASK) | MCNTL_RMD_1024; + stxa(AA_MCNTL, ASI_MCNTL, val); +} + +/* + * Enable level 1 caches. + */ +void +zeus_cache_enable(u_int cpu_impl) +{ + u_long lsu; + + lsu = ldxa(0, ASI_LSU_CTL_REG); + stxa(0, ASI_LSU_CTL_REG, lsu | LSU_IC | LSU_DC); + flush(KERNBASE); +} + +/* * Flush all lines from the level 1 caches. */ void @@ -63,3 +120,52 @@ zeus_icache_page_inval(vm_paddr_t pa __unused) { } + +/* + * Flush all non-locked mappings from the TLBs. + */ +void +zeus_tlb_flush_nonlocked(void) +{ + + stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); + stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0); + flush(KERNBASE); +} + +/* + * Flush all user mappings from the TLBs. + */ +void +zeus_tlb_flush_user(void) +{ + u_long data, tag; + u_int i, slot; + + for (i = 0; i < ZEUS_FTLB_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_FTLB, i); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } + for (i = 0; i < ZEUS_STLB_ENTRIES; i++) { + slot = TLB_DAR_SLOT(TLB_DAR_STLB, i); + data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); + data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); + tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); + if ((data & TD_V) != 0 && (data & TD_L) == 0 && + TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) + stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); + } +}