# HG changeset patch # Parent f1d96097ee6c21cfc5b52511b5118ba6389fd956 diff -r f1d96097ee6c sys/mips/include/asm.h --- a/sys/mips/include/asm.h +++ b/sys/mips/include/asm.h @@ -666,27 +666,5 @@ 9: .asciiz msg; \ /* Only valid with the _JB_MAGIC_SETJMP magic */ #define _JB_SIGMASK 13 -/* - * Various macros for dealing with TLB hazards - * (a) why so many? - * (b) when to use? - * (c) why not used everywhere? - */ -/* - * Assume that w alaways need nops to escape CP0 hazard - * For more info on CP0 hazards see Chapter 7 (p.99) of "MIPS32 Architecture - * For Programmers Volume III: The MIPS32 Privileged Resource Architecture" - */ -#if defined(CPU_RMI) || defined(CPU_CNMIPS) -#define HAZARD_DELAY -#define ITLBNOPFIX -#elif defined(CPU_MIPS74KC) || defined(CPU_MIPS64_R2) || defined(CPU_MIPS32_R2) || defined(CPU_NLM) -#define HAZARD_DELAY sll $0,$0,3 -#define ITLBNOPFIX sll $0,$0,3 -#else -#define ITLBNOPFIX nop;nop;nop;nop;nop;nop;nop;nop;nop;sll $0,$0,3; -#define HAZARD_DELAY nop;nop;nop;nop;sll $0,$0,3; -#endif - #endif /* !_MACHINE_ASM_H_ */ diff -r f1d96097ee6c sys/mips/include/cpufunc.h --- a/sys/mips/include/cpufunc.h +++ b/sys/mips/include/cpufunc.h @@ -60,40 +60,15 @@ #define _MACHINE_CPUFUNC_H_ #include #include +#include /* * These functions are required by user-land atomi ops */ static __inline void -mips_barrier(void) -{ -#if defined(CPU_CNMIPS) || defined(CPU_RMI) || defined(CPU_NLM) - __compiler_membar(); -#else - __asm __volatile (".set noreorder\n\t" - "nop\n\t" - "nop\n\t" - "nop\n\t" - "nop\n\t" - "nop\n\t" - "nop\n\t" - "nop\n\t" - "nop\n\t" - ".set reorder\n\t" - : : : "memory"); -#endif -} - -static __inline void -mips_cp0_sync(void) -{ - __asm __volatile (__XSTRING(COP0_SYNC)); -} - -static __inline void mips_wbflush(void) { #if defined(CPU_CNMIPS) __asm __volatile ( @@ -123,41 +98,31 @@ mips_rd_ ## n (void) \ { \ int v0; \ __asm __volatile ("dmfc0 %[v0], $"__XSTRING(r)";" \ : [v0] "=&r"(v0)); \ - mips_barrier(); \ return (v0); \ } \ static __inline void \ mips_wr_ ## n (uint64_t a0) \ { \ __asm __volatile ("dmtc0 %[a0], $"__XSTRING(r)";" \ - __XSTRING(COP0_SYNC)";" \ - "nop;" \ - "nop;" \ - : \ - : [a0] "r"(a0)); \ - mips_barrier(); \ + : : [a0] "r"(a0)); \ } struct __hack #define MIPS_RW64_COP0_SEL(n,r,s) \ static __inline uint64_t \ mips_rd_ ## n(void) \ { \ int v0; \ - __asm __volatile ("dmfc0 %[v0], $"__XSTRING(r)", "__XSTRING(s)";" \ + __asm __volatile ("dmfc0 %[v0], $"__XSTRING(r)", "__XSTRING(s)";" \ : [v0] "=&r"(v0)); \ - mips_barrier(); \ return (v0); \ } \ static __inline void \ mips_wr_ ## n(uint64_t a0) \ { \ - __asm __volatile ("dmtc0 %[a0], $"__XSTRING(r)", "__XSTRING(s)";" \ - __XSTRING(COP0_SYNC)";" \ - : \ - : [a0] "r"(a0)); \ - mips_barrier(); \ + __asm __volatile ("dmtc0 %[a0], $"__XSTRING(r)", "__XSTRING(s)";" \ + : : [a0] "r"(a0)); \ } struct __hack #if defined(__mips_n64) MIPS_RW64_COP0(excpc, MIPS_COP_0_EXC_PC); @@ -187,21 +152,15 @@ mips_rd_ ## n (void) \ { \ int v0; \ __asm __volatile ("mfc0 %[v0], $"__XSTRING(r)";" \ : [v0] "=&r"(v0)); \ - mips_barrier(); \ return (v0); \ } \ static __inline void \ mips_wr_ ## n (uint32_t a0) \ { \ __asm __volatile ("mtc0 %[a0], $"__XSTRING(r)";" \ - __XSTRING(COP0_SYNC)";" \ - "nop;" \ - "nop;" \ - : \ - : [a0] "r"(a0)); \ - mips_barrier(); \ + : : [a0] "r"(a0)); \ } struct __hack #define MIPS_RW32_COP0_SEL(n,r,s) \ static __inline uint32_t \ @@ -209,21 +168,15 @@ mips_rd_ ## n(void) \ { \ int v0; \ __asm __volatile ("mfc0 %[v0], $"__XSTRING(r)", "__XSTRING(s)";" \ : [v0] "=&r"(v0)); \ - mips_barrier(); \ return (v0); \ } \ static __inline void \ mips_wr_ ## n(uint32_t a0) \ { \ __asm __volatile ("mtc0 %[a0], $"__XSTRING(r)", "__XSTRING(s)";" \ - __XSTRING(COP0_SYNC)";" \ - "nop;" \ - "nop;" \ - : \ - : [a0] "r"(a0)); \ - mips_barrier(); \ + : : [a0] "r"(a0)); \ } struct __hack #ifdef CPU_CNMIPS static __inline void mips_sync_icache (void) @@ -297,8 +250,9 @@ intr_disable(void) register_t s; s = mips_rd_status(); mips_wr_status(s & ~MIPS_SR_INT_IE); + hazard_di(); return (s & MIPS_SR_INT_IE); } @@ -308,8 +262,9 @@ intr_enable(void) register_t s; s = mips_rd_status(); mips_wr_status(s | MIPS_SR_INT_IE); + hazard_ei(); return (s); } @@ -328,8 +283,9 @@ set_intr_mask(uint32_t mask) ostatus = mips_rd_status(); mask = (ostatus & ~MIPS_SR_INT_MASK) | (mask & MIPS_SR_INT_MASK); mips_wr_status(mask); + hazard_di(); return (ostatus); } static __inline uint32_t diff -r f1d96097ee6c sys/mips/include/cpuregs.h --- a/sys/mips/include/cpuregs.h +++ b/sys/mips/include/cpuregs.h @@ -201,26 +201,8 @@ #define MIPS_PHYS_TO_DIRECT_UNCACHED(pa) MIPS_PHYS_TO_KSEG1(pa) #define MIPS_DIRECT_TO_PHYS(va) MIPS_KSEG0_TO_PHYS(va) #endif -/* CPU dependent mtc0 hazard hook */ -#if defined(CPU_CNMIPS) || defined(CPU_RMI) -#define COP0_SYNC -#elif defined(CPU_NLM) -#define COP0_SYNC .word 0xc0 /* ehb */ -#elif defined(CPU_SB1) -#define COP0_SYNC ssnop; ssnop; ssnop; ssnop; ssnop; ssnop; ssnop; ssnop; ssnop -#elif defined(CPU_MIPS74KC) || defined(CPU_MIPS64_R2) || defined(CPU_MIPS32_R2) -#define COP0_SYNC .word 0xc0 /* ehb */ -#else -/* - * Pick a reasonable default based on the "typical" spacing described in the - * "CP0 Hazards" chapter of MIPS Architecture Book Vol III. - */ -#define COP0_SYNC ssnop; ssnop; ssnop; ssnop; .word 0xc0; -#endif -#define COP0_HAZARD_FPUENABLE nop; nop; nop; nop; - /* * The bits in the cause register. * * Bits common to r3000 and r4000: diff -r f1d96097ee6c sys/mips/include/hazard.h --- a/sys/mips/include/hazard.h +++ b/sys/mips/include/hazard.h @@ -43,9 +43,9 @@ * foo - when building .s: sequence of instructions that can be dropped into * an instruction sequence. * foo - when building .c: static inline __asm__ function with _foo inside. * __X - the X instruction - * + */ /* * With MIPS CPUs, there are different 'CPU' hazards that exists. These hazards * vary across the generations of MIPS hardware, but generally involve needing to @@ -53,9 +53,9 @@ * don't affect the pipeline, or to give changes a chance to take effect before * the data is used. * * hazard_mtc0_tlbw - Padding between mtc0 instructions affecting EntryHi, EntryLo0, - * EntryLo1, PageMask and Random or Index and a tlbwr or tlbwi instruction. + * EntryLo1, PageMask and Random or Index and a tlbwr, tlbwi, or tlbp instruction. * * hazard_tlbw_use - Padding after a tlbwr or tlbwi and a tlbp, load/store through * that address. This does not cover tlbwX to I-Fetch, which is unused in FreeBSD. * @@ -64,25 +64,26 @@ * hazard_ei - Padding after enabling interrupts (with EI or mtc0 Status) * * hazard_di - Padding after disabling interrupts (with DI or mtc0 Status) * - * hazard_c0c0 - Padding between back-to-back CP0 register accesses + * hazard_c0c0 - Padding between back-to-back CP0 register accesses for the same register. (unneeded?) * * hazard_epc_eret - Padding between mtc0 EPC and eret (unneeded in FreeBSD?) * - * hazard_status_eret - Padding between mtc0 Statusand eret (unneeded in FreeBSD?) + * hazard_status_eret - Padding between mtc0 Status and eret * * hazard_instruction - Instruction hazard padding */ #if defined(CPU_MIPS32) || defined(CPU_MIPS64) /* - * We add the MIPSr2 ehb as the last nop here so mipsr1 kernels could + * We replace the last nop with the MIPSr2 ehb so mipsr1 kernels could * work on mipsr2 and newer CPUs */ #define _hazard_mtc0_tlbw __ssnop; __ehb #define _hazard_tlbw_use __ssnop; __ssnop; __ehb -#define _hazard_tlbp_use __ssnop; __ssnop; __ehb +#define _hazard_tlbp_use __ssnop; __ehb +#define _hazard_tlbr_use __ssnop; __ssnop; __ehb #define _hazard_ei __ssnop; __ssnop; __ehb #define _hazard_di __ssnop; __ssnop; __ehb #define _hazard_c0c0 __ssnop; __ssnop; __ehb #define _hazard_epc_eret __ssnop; __ehb @@ -94,8 +95,9 @@ */ #define _hazard_mtc0_tlbw __ehb #define _hazard_tlbw_use __ehb #define _hazard_tlbp_use __ehb +#define _hazard_tlbr_use __ehb #define _hazard_ei __ehb #define _hazard_di __ehb #define _hazard_c0c0 __ehb #define _hazard_epc_eret __ehb @@ -119,8 +121,9 @@ static inline hazard_instruction() */ #define _hazard_mtc0_tlbw #define _hazard_tlbw_use #define _hazard_tlbp_use +#define _hazard_tlbr_use #define _hazard_ei #define _hazard_di #define _hazard_c0c0 #define _hazard_epc_eret @@ -129,25 +132,27 @@ static inline hazard_instruction() #elif defined(CPU_SB1) /* * SB1 almost gets all the hazards right in hardware + */ #define _hazard_mtc0_tlbw #define _hazard_tlbw_use #define _hazard_tlbp_use +#define _hazard_tlbr_use #define _hazard_ei #define _hazard_di __ssnop; __ssnop; __ssnop #define _hazard_c0c0 #define _hazard_epc_eret #define _hazard_status_eret #define hazard_instruction() do { } while (0) #else /* - * Default values that should work on most other MIPS R4000-like CPUs. We use - * the + * Default values that should work on most other MIPS R4000-like CPUs */ #define _hazard_mtc0_tlbw __ssnop; __ssnop #define _hazard_tlbw_use __ssnop; __ssnop; __ssnop -#define _hazard_tlbp_use __ssnop; __ssnop; __ssnop +#define _hazard_tlbp_use __ssnop; __ssnop +#define _hazard_tlbr_use __ssnop; __ssnop; __ssnop #define _hazard_ei __ssnop; __ssnop; __ssnop #define _hazard_di __ssnop; __ssnop; __ssnop #define _hazard_c0c0 __ssnop; __ssnop; __ssnop #define _hazard_epc_eret __ssnop; __ssnop @@ -158,24 +163,28 @@ static inline hazard_instruction() #ifdef __ASSEMBLER__ #define hazard_mtc0_tlbw _hazard_mtc0_tlbw #define hazard_tlbw_use _hazard_tlbw_use #define hazard_tlbp_use _hazard_tlbp_use +#define hazard_tlbr_use _hazard_tlbr_use #define hazard_ei _hazard_ei #define hazard_di _hazard_di #define hazard_c0c0 _hazard_c0c0 #define hazard_epc_eret _hazard_epc_eret #define hazard_status_eret _hazard_status_eret /* hazard_instruction cannot be used */ #else -#define _hazard_gen(y) do {__asm __volatile(__XSTRING(y)) } while (0) -#define hazard_mtc0_tlbw _hazard_gen(_hazard_mtc0_tlbw) -#define hazard_tlbw_use _hazard_gen(_hazard_tlbw_use) -#define hazard_tlbp_use _hazard_gen(_hazard_tlbp_use) -#define hazard_ei _hazard_gen(_hazard_ei) -#define hazard_di _hazard_gen(_hazard_di) -#define hazard_c0c0 _hazard_gen(_hazard_c0c0) -#define hazard_epc_eret _hazard_gen(_hazard_epc_eret) -#define hazard_status_eret _hazard_gen(_hazard_status_eret) +#define __YSTRING1(y...) #y +#define __YSTRING(y...) __YSTRING1(y) +#define _hazard_gen(y) do {__asm __volatile(__YSTRING(y)); } while (0) +#define hazard_mtc0_tlbw() _hazard_gen(_hazard_mtc0_tlbw) +#define hazard_tlbw_use() _hazard_gen(_hazard_tlbw_use) +#define hazard_tlbp_use() _hazard_gen(_hazard_tlbp_use) +#define hazard_tlbr_use() _hazard_gen(_hazard_tlbr_use) +#define hazard_ei() _hazard_gen(_hazard_ei) +#define hazard_di() _hazard_gen(_hazard_di) +#define hazard_c0c0() _hazard_gen(_hazard_c0c0) +#define hazard_epc_eret() _hazard_gen(_hazard_epc_eret) +#define hazard_status_eret() _hazard_gen(_hazard_status_eret) #endif /* * fpu hazards @@ -184,5 +193,30 @@ static inline hazard_instruction() * * hazard_fpu_disable - Padding after disabling CP1 to flush pipeline */ +#if defined(CPU_SB1) +#define _hazard_fpu_enable \ + .set push; \ + .set noreorder; \ + __ssnop; \ + bnezl $0, .+4; \ + __ssnop; \ + .set pop +#define _hazard_fpu_disable /* nothing */ +#elif defined(CPU_MIPS32_R2) || defined(CPU_MIPS64_R2) +#define _hazard_fpu_enable __ehb +#define _hazard_fpu_disable __ehb +#else +#define _hazard_fpu_enable __ssnop; __ssnop; __ssnop; __ehb +#define _hazard_fpu_disable __ssnop; __ssnop; __ssnop; __ehb #endif + +#ifdef __ASSEMBLER__ +#define hazard_fpu_enable _hazard_fpu_enable +#define hazard_fpu_disable _hazard_fpu_disable +#else +#define hazard_fpu_enable _hazard_gen(_hazard_fpu_enable) +#define hazard_fpu_disable _hazard_gen(_hazard_fpu_disable) +#endif + +#endif /* _MIPS_HAZARD_H */ diff -r f1d96097ee6c sys/mips/include/pcb.h --- a/sys/mips/include/pcb.h +++ b/sys/mips/include/pcb.h @@ -63,16 +63,17 @@ /* * Call ast if required * * XXX Do we really need to disable interrupts? + * XXX and do we really need to read the COP0 status twice? */ #define DO_AST \ 44: \ mfc0 t0, MIPS_COP_0_STATUS ;\ and a0, t0, MIPS_SR_INT_IE ;\ xor t0, a0, t0 ;\ mtc0 t0, MIPS_COP_0_STATUS ;\ - COP0_SYNC ;\ + hazard_di ;\ GET_CPU_PCPU(s1) ;\ PTR_L s3, PC_CURPCB(s1) ;\ PTR_L s1, PC_CURTHREAD(s1) ;\ lw s2, TD_FLAGS(s1) ;\ @@ -80,9 +81,9 @@ 44: \ and s2, s0 ;\ mfc0 t0, MIPS_COP_0_STATUS ;\ or t0, a0, t0 ;\ mtc0 t0, MIPS_COP_0_STATUS ;\ - COP0_SYNC ;\ + hazard_ei ;\ beq s2, zero, 4f ;\ nop ;\ PTR_LA s0, _C_LABEL(ast) ;\ jalr s0 ;\ diff -r f1d96097ee6c sys/mips/mips/exception.S --- a/sys/mips/mips/exception.S +++ b/sys/mips/mips/exception.S @@ -61,8 +61,9 @@ #include #include #include #include +#include #include "assym.s" .set noreorder # Noreorder is default style! @@ -155,14 +156,13 @@ MipsDoTLBMiss: PTE_L k0, 0(k1) #0e: k0=lo0 pte PTE_L k1, PTESIZE(k1) #0f: k1=lo0 pte CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 #12: lo0 is loaded - COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 #15: lo1 is loaded - COP0_SYNC + hazard_mtc0_tlbw tlbwr #1a: write to tlb - HAZARD_DELAY + hazard_tlbw_use eret #1f: retUrn from exception 1: j MipsTLBMissException #20: kernel exception nop #21: branch delay slot 2: j SlowFault #22: no page table present @@ -245,25 +245,25 @@ SlowFault: or a0, a0, a2 ; \ li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | MIPS_SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, MIPS_COP_0_STATUS ; \ - ITLBNOPFIX + hazard_di #elif defined(CPU_RMI) || defined(CPU_NLM) #define CLEAR_STATUS \ mfc0 a0, MIPS_COP_0_STATUS ;\ li a2, (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_COP_2_BIT) ; \ or a0, a0, a2 ; \ li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | MIPS_SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, MIPS_COP_0_STATUS ; \ - ITLBNOPFIX + hazard_di #else #define CLEAR_STATUS \ mfc0 a0, MIPS_COP_0_STATUS ;\ li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | MIPS_SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, MIPS_COP_0_STATUS ; \ - ITLBNOPFIX + hazard_di #endif /* * Save CPU and CP0 register state. @@ -321,10 +321,9 @@ SlowFault: SAVE_REG(ra, RA, sp) ;\ PTR_ADDU v0, sp, KERN_EXC_FRAME_SIZE ;\ SAVE_REG(v0, SP, sp) ;\ CLEAR_STATUS ;\ - PTR_ADDU a0, sp, CALLFRAME_SIZ ;\ - ITLBNOPFIX + PTR_ADDU a0, sp, CALLFRAME_SIZ #define RESTORE_REG(reg, offs, base) \ REG_L reg, CALLFRAME_SIZ + (SZREG * offs) (base) @@ -488,9 +487,9 @@ NESTED_NOPROFILE(MipsUserGenException, C or t0, t0, (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_COP_2_BIT) #endif mtc0 t0, MIPS_COP_0_STATUS PTR_ADDU a0, k1, U_PCB_REGS - ITLBNOPFIX + hazard_ei /* * Call the exception handler. */ @@ -564,9 +563,9 @@ NESTED_NOPROFILE(MipsUserGenException, C .set noat RESTORE_U_PCB_REG(AT, AST, k1) mtc0 k0, MIPS_COP_0_STATUS # still exception level - ITLBNOPFIX + hazard_status_eret sync eret .set at END(MipsUserGenException) @@ -579,9 +578,9 @@ NESTED(mips_wait, CALLFRAME_SIZ, ra) REG_S ra, CALLFRAME_RA(sp) # save RA mfc0 t0, MIPS_COP_0_STATUS xori t1, t0, MIPS_SR_INT_IE mtc0 t1, MIPS_COP_0_STATUS - COP0_SYNC + hazard_di jal sched_runnable nop REG_L ra, CALLFRAME_RA(sp) mfc0 t0, MIPS_COP_0_STATUS @@ -746,9 +745,9 @@ NESTED_NOPROFILE(MipsUserIntr, CALLFRAME #elif defined(CPU_RMI) || defined(CPU_NLM) or t0, t0, (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_COP_2_BIT) #endif mtc0 t0, MIPS_COP_0_STATUS - ITLBNOPFIX + hazard_di PTR_ADDU a0, k1, U_PCB_REGS /* * Call the interrupt handler. */ @@ -765,9 +764,9 @@ NESTED_NOPROFILE(MipsUserIntr, CALLFRAME */ mfc0 a0, MIPS_COP_0_STATUS or a0, a0, MIPS_SR_INT_IE mtc0 a0, MIPS_COP_0_STATUS - ITLBNOPFIX + hazard_ei /* * DO_AST enabled interrupts */ @@ -831,9 +830,9 @@ NESTED_NOPROFILE(MipsUserIntr, CALLFRAME .set noat RESTORE_U_PCB_REG(AT, AST, k1) mtc0 k0, MIPS_COP_0_STATUS # SR with EXL set. - ITLBNOPFIX + hazard_status_eret sync eret .set at END(MipsUserIntr) @@ -903,12 +902,11 @@ 2: /* Validate page directory pointer. PTE_L k0, 0(k1) PTE_L k1, PTESIZE(k1) CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 - COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 - COP0_SYNC + hazard_mtc0_tlbw b tlb_insert_entry nop @@ -916,27 +914,27 @@ odd_page: PTE_L k0, -PTESIZE(k1) PTE_L k1, 0(k1) CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 - COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 - COP0_SYNC + hazard_mtc0_tlbw tlb_insert_entry: tlbp - HAZARD_DELAY + hazard_tlbp_use mfc0 k0, MIPS_COP_0_TLB_INDEX + hazard_mtc0_tlbw bltz k0, tlb_insert_random nop tlbwi + hazard_tlbw_use eret - ssnop tlb_insert_random: tlbwr + hazard_tlbw_use eret - ssnop 3: /* * Branch to the comprehensive exception processing. @@ -1058,14 +1056,13 @@ LEAF_NOPROFILE(MipsTLBMissException) PTE_L k0, 0(k1) # k0=lo0 pte PTE_L k1, PTESIZE(k1) # k1=lo1 pte CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 # lo0 is loaded - COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 # lo1 is loaded - COP0_SYNC + hazard_mtc0_tlbw tlbwr # write to tlb - HAZARD_DELAY + hazard_tlbw_use eret # return from exception .set at END(MipsTLBMissException) @@ -1095,9 +1092,9 @@ NESTED(MipsFPTrap, CALLFRAME_SIZ, ra) .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) or t1, t0, MIPS_SR_COP_1_BIT mtc0 t1, MIPS_COP_0_STATUS - ITLBNOPFIX + hazard_fpu_enable cfc1 t1, MIPS_FPU_CSR # stall til FP done cfc1 t1, MIPS_FPU_CSR # now get status nop sll t2, t1, (31 - 17) # unimplemented operation? @@ -1178,9 +1175,9 @@ FPReturn: mfc0 t0, MIPS_COP_0_STATUS PTR_L ra, CALLFRAME_RA(sp) and t0, t0, ~MIPS_SR_COP_1_BIT mtc0 t0, MIPS_COP_0_STATUS - ITLBNOPFIX + hazard_fpu_disable j ra PTR_ADDU sp, sp, CALLFRAME_SIZ END(MipsFPTrap) @@ -1246,9 +1243,9 @@ NESTED_NOPROFILE(MipsCacheException, KER mfc0 k0, MIPS_COP_0_STATUS # restore status li k1, MIPS_SR_DIAG_PE # ignore further errors or k0, k1 mtc0 k0, MIPS_COP_0_STATUS # restore status - COP0_SYNC + hazard_status_eret eret MSG("cache error @ EPC 0x%x CachErr 0x%x"); diff -r f1d96097ee6c sys/mips/mips/locore.S --- a/sys/mips/mips/locore.S +++ b/sys/mips/mips/locore.S @@ -68,8 +68,9 @@ #include #include #include #include +#include #include "assym.s" .data @@ -129,14 +130,14 @@ VECTOR(_locore, unknown) mfc0 t2, MIPS_COP_0_STATUS and t2, t0 or t2, t1 mtc0 t2, MIPS_COP_0_STATUS - COP0_SYNC + hazard_di /* Make sure KSEG0 is cached */ li t0, MIPS_CCA_CACHED mtc0 t0, MIPS_COP_0_CONFIG - COP0_SYNC + /* No hazard here beacuse cached vs uncached yields same data */ /*xxximp * now that we pass a0...a3 to the platform_init routine, do we need * to stash this stuff here? diff -r f1d96097ee6c sys/mips/mips/mpboot.S --- a/sys/mips/mips/mpboot.S +++ b/sys/mips/mips/mpboot.S @@ -28,8 +28,9 @@ #include #include #include +#include #include "assym.s" .text @@ -57,12 +58,11 @@ GLOBAL(mpentry) CLEAR_STATUS /* disable interrupts */ mtc0 zero, MIPS_COP_0_CAUSE /* clear soft interrupts */ - li t0, MIPS_CCA_CACHED /* make sure kseg0 is cached */ mtc0 t0, MIPS_COP_0_CONFIG - COP0_SYNC + hazard_di jal platform_processor_id /* get the processor number */ nop move s0, v0 diff -r f1d96097ee6c sys/mips/mips/support.S --- a/sys/mips/mips/support.S +++ b/sys/mips/mips/support.S @@ -92,8 +92,9 @@ #include #include #include #include +#include #include "assym.s" .set noreorder # Noreorder is default style! @@ -1017,9 +1018,9 @@ LEAF(longjmp) REG_L s7, (SZREG * PCB_REG_S7)(a0) REG_L s8, (SZREG * PCB_REG_S8)(a0) REG_L sp, (SZREG * PCB_REG_SP)(a0) mtc0 v0, MIPS_COP_0_STATUS # Later the "real" spl value! - ITLBNOPFIX + hazard_ei jr ra li v0, 1 # longjmp return END(longjmp) @@ -1030,12 +1031,9 @@ LEAF(mips3_ld) #if defined(__mips_o32) mfc0 t0, MIPS_COP_0_STATUS # turn off interrupts and t1, t0, ~(MIPS_SR_INT_IE) mtc0 t1, MIPS_COP_0_STATUS - COP0_SYNC - nop - nop - nop + hazard_di ld v0, 0(a0) #if _BYTE_ORDER == _BIG_ENDIAN dsll v1, v0, 32 @@ -1047,10 +1045,9 @@ LEAF(mips3_ld) dsra v0, v0, 32 # low word in v0 #endif mtc0 t0, MIPS_COP_0_STATUS # restore intr status. - COP0_SYNC - nop + hazard_ei #else /* !__mips_o32 */ ld v0, 0(a0) #endif /* !__mips_o32 */ @@ -1066,12 +1063,9 @@ LEAF(mips3_sd) #if defined(__mips_o32) mfc0 t0, MIPS_COP_0_STATUS # turn off interrupts and t1, t0, ~(MIPS_SR_INT_IE) mtc0 t1, MIPS_COP_0_STATUS - COP0_SYNC - nop - nop - nop + hazard_di # NOTE: a1 is padding! #if _BYTE_ORDER == _BIG_ENDIAN @@ -1086,10 +1080,9 @@ LEAF(mips3_sd) or a1, a2, a3 sd a1, 0(a0) mtc0 t0, MIPS_COP_0_STATUS # restore intr status. - COP0_SYNC - nop + hazard_ei #else /* !__mips_o32 */ sd a1, 0(a0) #endif /* !__mips_o32 */ diff -r f1d96097ee6c sys/mips/mips/swtch.S --- a/sys/mips/mips/swtch.S +++ b/sys/mips/mips/swtch.S @@ -62,8 +62,9 @@ #include #include #include #include +#include #include "assym.s" .set noreorder # Noreorder is default style! @@ -81,9 +82,9 @@ LEAF(fork_trampoline) mfc0 v0, MIPS_COP_0_STATUS and v0, ~(MIPS_SR_INT_IE) mtc0 v0, MIPS_COP_0_STATUS # disable interrupts - COP0_SYNC + hazard_di /* * The use of k1 for storing the PCB pointer must be done only * after interrupts are disabled. Otherwise it will get overwritten * by the interrupt code. @@ -134,9 +135,9 @@ LEAF(fork_trampoline) mfc0 k1, MIPS_COP_0_STATUS and k1, k1, MIPS_SR_INT_MASK or k0, k0, k1 mtc0 k0, MIPS_COP_0_STATUS # switch to user mode (when eret...) - HAZARD_DELAY + hazard_status_eret sync eret .set at END(fork_trampoline) @@ -180,9 +181,9 @@ NESTED(cpu_throw, CALLFRAME_SIZ, ra) nop nop and a3, t0, ~(MIPS_SR_INT_IE) mtc0 a3, MIPS_COP_0_STATUS # Disable all interrupts - ITLBNOPFIX + hazard_di j mips_sw1 # We're not interested in old # thread's context, so jump # right to action nop # BDSLOT @@ -200,9 +201,9 @@ NESTED(cpu_switch, CALLFRAME_SIZ, ra) nop nop and a3, t0, ~(MIPS_SR_INT_IE) mtc0 a3, MIPS_COP_0_STATUS # Disable all interrupts - ITLBNOPFIX + hazard_di beqz a0, mips_sw1 move a3, a0 PTR_L a0, TD_PCB(a0) # load PCB addr of curproc SAVE_U_PCB_CONTEXT(sp, PCB_REG_SP, a0) # save old sp @@ -315,13 +316,12 @@ blocked_loop: * NOTE: This is hard coded to UPAGES == 2. * Also, there should be no TLB faults at this point. */ MTC0 v0, MIPS_COP_0_TLB_HI # VPN = va - HAZARD_DELAY + hazard_mtc0_tlbw tlbp # probe VPN - HAZARD_DELAY + hazard_tlbp_use mfc0 s0, MIPS_COP_0_TLB_INDEX - HAZARD_DELAY PTR_LI t1, MIPS_KSEG0_START # invalidate tlb entry bltz s0, entry0set nop @@ -329,23 +329,21 @@ blocked_loop: addu t1, s0 MTC0 t1, MIPS_COP_0_TLB_HI PTE_MTC0 zero, MIPS_COP_0_TLB_LO0 PTE_MTC0 zero, MIPS_COP_0_TLB_LO1 - HAZARD_DELAY + hazard_mtc0_tlbw tlbwi - HAZARD_DELAY + hazard_tlbw_use MTC0 v0, MIPS_COP_0_TLB_HI # set VPN again entry0set: /* SMP!! - Works only for unshared TLB case - i.e. no v-cpus */ mtc0 zero, MIPS_COP_0_TLB_INDEX # TLB entry #0 - HAZARD_DELAY PTE_MTC0 a1, MIPS_COP_0_TLB_LO0 # upte[0] - HAZARD_DELAY PTE_MTC0 a2, MIPS_COP_0_TLB_LO1 # upte[1] - HAZARD_DELAY + hazard_mtc0_tlbw tlbwi # set TLB entry #0 - HAZARD_DELAY + hazard_tlbw_use /* * Now running on new u struct. */ sw2: @@ -375,9 +373,9 @@ sw2: and t0, t0, MIPS_SR_INT_MASK and v0, v0, ~MIPS_SR_INT_MASK or v0, v0, t0 mtc0 v0, MIPS_COP_0_STATUS - ITLBNOPFIX + hazard_ei j ra nop END(cpu_switch) @@ -403,9 +401,9 @@ END(cpu_switch) LEAF(MipsSwitchFPState) mfc0 t1, MIPS_COP_0_STATUS # Save old SR li t0, MIPS_SR_COP_1_BIT # enable the coprocessor mtc0 t0, MIPS_COP_0_STATUS - ITLBNOPFIX + hazard_fpu_enable beq a0, zero, 1f # skip save if NULL pointer nop /* @@ -498,9 +496,9 @@ 1: ctc1 t0, MIPS_FPU_CSR nop mtc0 t1, MIPS_COP_0_STATUS # Restore the status register. - ITLBNOPFIX + hazard_fpu_disable j ra nop END(MipsSwitchFPState) @@ -525,9 +523,9 @@ LEAF(MipsSaveCurFPState) PTR_L a0, TD_PCB(a0) # get pointer to pcb for thread mfc0 t1, MIPS_COP_0_STATUS # Disable interrupts and li t0, MIPS_SR_COP_1_BIT # enable the coprocessor mtc0 t0, MIPS_COP_0_STATUS - ITLBNOPFIX + hazard_fpu_enable GET_CPU_PCPU(a1) PTR_S zero, PC_FPCURTHREAD(a1) # indicate state has been saved /* * First read out the status register to make sure that all FP operations @@ -576,9 +574,9 @@ LEAF(MipsSaveCurFPState) SAVE_U_PCB_FPREG($f30, F30_NUM, a0) SAVE_U_PCB_FPREG($f31, F31_NUM, a0) mtc0 t1, MIPS_COP_0_STATUS # Restore the status register. - ITLBNOPFIX + hazard_fpu_disable j ra nop END(MipsSaveCurFPState) diff -r f1d96097ee6c sys/mips/mips/tlb.c --- a/sys/mips/mips/tlb.c +++ b/sys/mips/mips/tlb.c @@ -70,30 +70,26 @@ COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) static inline void tlb_probe(void) { __asm __volatile ("tlbp" : : : "memory"); - mips_cp0_sync(); } static inline void tlb_read(void) { __asm __volatile ("tlbr" : : : "memory"); - mips_cp0_sync(); } static inline void tlb_write_indexed(void) { __asm __volatile ("tlbwi" : : : "memory"); - mips_cp0_sync(); } static inline void tlb_write_random(void) { __asm __volatile ("tlbwr" : : : "memory"); - mips_cp0_sync(); } static void tlb_invalidate_one(unsigned); @@ -112,10 +108,11 @@ tlb_insert_wired(unsigned i, vm_offset_t mips_wr_pagemask(0); mips_wr_entryhi(TLBHI_ENTRY(va, 0)); mips_wr_entrylo0(pte0); mips_wr_entrylo1(pte1); + hazard_mtc0_tlbw(); tlb_write_indexed(); - + hazard_tlbw_use(); mips_wr_entryhi(asid); intr_restore(s); } @@ -132,9 +129,11 @@ tlb_invalidate_address(struct pmap *pmap asid = mips_rd_entryhi() & TLBHI_ASID_MASK; mips_wr_pagemask(0); mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap))); + hazard_mtc0_tlbw(); tlb_probe(); + hazard_tlbp_use(); i = mips_rd_index(); if (i >= 0) tlb_invalidate_one(i); @@ -155,8 +154,9 @@ tlb_invalidate_all(void) for (i = mips_rd_wired(); i < num_tlbentries; i++) tlb_invalidate_one(i); mips_wr_entryhi(asid); + hazard_mtc0_tlbw(); /* xxx is this the right one? */ intr_restore(s); } void @@ -172,10 +172,11 @@ tlb_invalidate_all_user(struct pmap *pma for (i = mips_rd_wired(); i < num_tlbentries; i++) { register_t uasid; mips_wr_index(i); + hazard_mtc0_tlbw(); tlb_read(); - + hazard_tlbr_use(); uasid = mips_rd_entryhi() & TLBHI_ASID_MASK; if (pmap == NULL) { /* * Invalidate all non-kernel entries. @@ -192,8 +193,9 @@ tlb_invalidate_all_user(struct pmap *pma tlb_invalidate_one(i); } mips_wr_entryhi(asid); + hazard_mtc0_tlbw(); /* xxx is this the right one? */ intr_restore(s); } /* @@ -218,9 +220,8 @@ tlb_invalidate_range(pmap_t pmap, vm_off end = (end + (1 << TLBMASK_SHIFT) - 1) & ~((1 << TLBMASK_SHIFT) - 1); s = intr_disable(); save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK; - asid = pmap_asid(pmap); start_hi = TLBHI_ENTRY(start, asid); end_hi = TLBHI_ENTRY(end, asid); @@ -236,9 +237,11 @@ tlb_invalidate_range(pmap_t pmap, vm_off */ for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) { mips_wr_pagemask(0); mips_wr_entryhi(hi); + hazard_mtc0_tlbw(); tlb_probe(); + hazard_tlbp_use(); i = mips_rd_index(); if (i >= 0) tlb_invalidate_one(i); } @@ -248,9 +251,11 @@ tlb_invalidate_range(pmap_t pmap, vm_off * the TLB. Test every non-wired TLB entry. */ for (i = mips_rd_wired(); i < num_tlbentries; i++) { mips_wr_index(i); + hazard_mtc0_tlbw(); tlb_read(); + hazard_tlbr_use(); hi = mips_rd_entryhi(); if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi || end == 0)) { /* @@ -265,8 +270,9 @@ tlb_invalidate_range(pmap_t pmap, vm_off } } mips_wr_entryhi(save_asid); + hazard_mtc0_tlbw(); /* xxx is this the right one? */ intr_restore(s); } /* XXX Only if DDB? */ @@ -282,9 +288,11 @@ tlb_save(void) ntlb = num_tlbentries; tlb_state[cpu].wired = mips_rd_wired(); for (i = 0; i < ntlb; i++) { mips_wr_index(i); + hazard_mtc0_tlbw(); tlb_read(); + hazard_tlbr_use(); tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi(); tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask(); tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0(); @@ -306,35 +314,42 @@ tlb_update(struct pmap *pmap, vm_offset_ asid = mips_rd_entryhi() & TLBHI_ASID_MASK; mips_wr_pagemask(0); mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap))); + hazard_mtc0_tlbw(); tlb_probe(); + hazard_tlbp_use(); i = mips_rd_index(); if (i >= 0) { tlb_read(); + /* tlbr hazard only on entry_hi */ if ((va & PAGE_SIZE) == 0) { mips_wr_entrylo0(pte); } else { mips_wr_entrylo1(pte); } + hazard_mtc0_tlbw(); tlb_write_indexed(); + hazard_tlbw_use(); } mips_wr_entryhi(asid); + hazard_mtc0_tlbw(); /* xxx is this the right one? */ intr_restore(s); } static void tlb_invalidate_one(unsigned i) { - /* XXX an invalid ASID? */ mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0)); mips_wr_entrylo0(0); mips_wr_entrylo1(0); mips_wr_pagemask(0); mips_wr_index(i); + hazard_mtc0_tlbw(); tlb_write_indexed(); + hazard_tlbw_use(); } #ifdef DDB #include diff -r f1d96097ee6c sys/mips/sentry5/s5reg.h --- a/sys/mips/sentry5/s5reg.h +++ b/sys/mips/sentry5/s5reg.h @@ -11,34 +11,21 @@ #define SENTRY5_DORESET 0x80 /* * Custom CP0 register macros. - * XXX: This really needs the mips cpuregs.h file for the barrier. */ #define S5_RDRW32_C0P0_CUST22(n,r) \ static __inline u_int32_t \ s5_rd_ ## n (void) \ { \ int v0; \ __asm __volatile ("mfc0 %[v0], $22, "__XSTRING(r)" ;" \ : [v0] "=&r"(v0)); \ - /*mips_barrier();*/ \ return (v0); \ -} \ -static __inline void \ -s5_wr_ ## n (u_int32_t a0) \ -{ \ - __asm __volatile ("mtc0 %[a0], $22, "__XSTRING(r)" ;" \ - __XSTRING(COP0_SYNC)";" \ - "nop;" \ - "nop;" \ - : \ - : [a0] "r"(a0)); \ - /*mips_barrier();*/ \ } struct __hack /* - * All 5 of these sub-registers are used by Linux. + * These 5 sub-registers are used by Linux. We only use clkcfg1. * There is a further custom register at 25 which is not used. */ #define S5_CP0_DIAG 0 #define S5_CP0_CLKCFG1 1 @@ -46,13 +33,7 @@ s5_wr_ ## n (u_int32_t a0) \ #define S5_CP0_SYNC 3 #define S5_CP0_CLKCFG3 4 #define S5_CP0_RESET 5 -/* s5_[rd|wr]_xxx() */ -S5_RDRW32_C0P0_CUST22(diag, S5_CP0_DIAG); S5_RDRW32_C0P0_CUST22(clkcfg1, S5_CP0_CLKCFG1); -S5_RDRW32_C0P0_CUST22(clkcfg2, S5_CP0_CLKCFG2); -S5_RDRW32_C0P0_CUST22(sync, S5_CP0_SYNC); -S5_RDRW32_C0P0_CUST22(clkcfg3, S5_CP0_CLKCFG3); -S5_RDRW32_C0P0_CUST22(reset, S5_CP0_RESET); #endif /* _MIPS32_SENTRY5_SENTRY5REG_H_ */