commit 11e61618abb84cffd3f2e771f6548d2b023cb857 Author: Stacey Son Date: Sat Feb 8 15:34:16 2014 -0600 Enable use of UserLocal register if detected in hardware. This change detects if the MIPS hardware supports the UserLocal Register Implementation (ULRI) and, if found, will enable it (or disable it if not found). The UserLocal register is used for the TLS pointer and is much faster than calling the sysarch(MIPS_GET_TLS,...) system call and emulating the rdhwr $29 instruction with the kernel trap() handler. diff --git a/sys/mips/include/cpufunc.h b/sys/mips/include/cpufunc.h index 3c307ae..3ab7e1d 100644 --- a/sys/mips/include/cpufunc.h +++ b/sys/mips/include/cpufunc.h @@ -159,6 +159,7 @@ mips_wr_ ## n(uint64_t a0) \ MIPS_RW64_COP0(excpc, MIPS_COP_0_EXC_PC); MIPS_RW64_COP0(entryhi, MIPS_COP_0_TLB_HI); MIPS_RW64_COP0(pagemask, MIPS_COP_0_TLB_PG_MASK); +MIPS_RW64_COP0_SEL(userlocal, MIPS_COP_0_USERLOCAL, 2); #ifdef CPU_CNMIPS MIPS_RW64_COP0_SEL(cvmcount, MIPS_COP_0_COUNT, 6); MIPS_RW64_COP0_SEL(cvmctl, MIPS_COP_0_COUNT, 7); @@ -264,6 +265,7 @@ MIPS_RW32_COP0(status, MIPS_COP_0_STATUS); #if !defined(__mips_n64) MIPS_RW32_COP0(entryhi, MIPS_COP_0_TLB_HI); MIPS_RW32_COP0(pagemask, MIPS_COP_0_TLB_PG_MASK); +MIPS_RW32_COP0_SEL(userlocal, MIPS_COP_0_USERLOCAL, 2); #endif #ifdef CPU_NLM MIPS_RW32_COP0_SEL(pagegrain, MIPS_COP_0_TLB_PG_MASK, 1); @@ -288,6 +290,7 @@ MIPS_RW32_COP0_SEL(perfcnt0, MIPS_COP_0_PERFCNT, 0); MIPS_RW32_COP0_SEL(perfcnt1, MIPS_COP_0_PERFCNT, 1); MIPS_RW32_COP0_SEL(perfcnt2, MIPS_COP_0_PERFCNT, 2); MIPS_RW32_COP0_SEL(perfcnt3, MIPS_COP_0_PERFCNT, 3); +MIPS_RW32_COP0(hwrena, MIPS_COP_0_HWRENA); #undef MIPS_RW32_COP0 #undef MIPS_RW32_COP0_SEL diff --git a/sys/mips/include/cpuinfo.h b/sys/mips/include/cpuinfo.h index 05b5a0f..036e3f3 100644 --- a/sys/mips/include/cpuinfo.h +++ b/sys/mips/include/cpuinfo.h @@ -57,6 +57,7 @@ struct mips_cpuinfo { u_int16_t tlb_nentries; u_int8_t icache_virtual; boolean_t cache_coherent_dma; + boolean_t userlocal_reg; struct { u_int32_t ic_size; u_int8_t ic_linesize; diff --git a/sys/mips/include/cpuregs.h b/sys/mips/include/cpuregs.h index 8a06d43..a4169af 100644 --- a/sys/mips/include/cpuregs.h +++ b/sys/mips/include/cpuregs.h @@ -427,9 +427,10 @@ * 2 MIPS_COP_0_TLB_LO0 .636 r4k TLB entry low. * 3 MIPS_COP_0_TLB_LO1 .636 r4k TLB entry low, extended. * 4 MIPS_COP_0_TLB_CONTEXT 3636 TLB Context. + * 4/2 MIPS_COP_0_USERLOCAL ..36 UserLocal. * 5 MIPS_COP_0_TLB_PG_MASK .333 TLB Page Mask register. * 6 MIPS_COP_0_TLB_WIRED .333 Wired TLB number. - * 7 MIPS_COP_0_INFO ..33 Info registers + * 7 MIPS_COP_0_HWRENA ..33 rdHWR Enable. * 8 MIPS_COP_0_BAD_VADDR 3636 Bad virtual address. * 9 MIPS_COP_0_COUNT .333 Count register. * 10 MIPS_COP_0_TLB_HI 3636 TLB entry high. @@ -507,7 +508,8 @@ #define MIPS_COP_0_ERROR_PC _(30) /* MIPS32/64 */ -#define MIPS_COP_0_INFO _(7) +#define MIPS_COP_0_USERLOCAL _(4) /* sel 2 is userlevel register */ +#define MIPS_COP_0_HWRENA _(7) #define MIPS_COP_0_DEBUG _(23) #define MIPS_COP_0_DEPC _(24) #define MIPS_COP_0_PERFCNT _(25) @@ -521,11 +523,21 @@ #define MIPS_MMU_BAT 0x02 /* Standard BAT */ #define MIPS_MMU_FIXED 0x03 /* Standard fixed mapping */ -#define MIPS_CONFIG0_MT_MASK 0x00000380 /* bits 9..7 MMU Type */ -#define MIPS_CONFIG0_MT_SHIFT 7 -#define MIPS_CONFIG0_BE 0x00008000 /* data is big-endian */ -#define MIPS_CONFIG0_VI 0x00000004 /* instruction cache is virtual */ +/* + * Config Register Fields + * (See "MIPS Architecture for Programmers Volume III", MD00091, Table 9.39) + */ +#define MIPS_CONFIG0_M 0x80000000 /* Flag: Config1 is present. */ +#define MIPS_CONFIG0_MT_MASK 0x00000380 /* bits 9..7 MMU Type */ +#define MIPS_CONFIG0_MT_SHIFT 7 +#define MIPS_CONFIG0_BE 0x00008000 /* data is big-endian */ +#define MIPS_CONFIG0_VI 0x00000004 /* inst cache is virtual */ +/* + * Config1 Register Fields + * (See "MIPS Architecture for Programmers Volume III", MD00091, Table 9-1) + */ +#define MIPS_CONFIG1_M 0x80000000 /* Flag: Config2 is present. */ #define MIPS_CONFIG1_TLBSZ_MASK 0x7E000000 /* bits 30..25 # tlb entries minus one */ #define MIPS_CONFIG1_TLBSZ_SHIFT 25 @@ -550,6 +562,19 @@ #define MIPS_CONFIG1_EP 0x00000002 /* EJTAG implemented */ #define MIPS_CONFIG1_FP 0x00000001 /* FPU implemented */ +/* + * Config2 Register Fields + * (See "MIPS Architecture for Programmers Volume III", MD00091, Table 9.40) + */ +#define MIPS_CONFIG2_M 0x80000000 /* Flag: Config3 is present. */ + +/* + * Config3 Register Fields + * (See "MIPS Architecture for Programmers Volume III", MD00091, Table 9.41) + */ +#define MIPS_CONFIG3_M 0x80000000 /* Flag: Config4 is present */ +#define MIPS_CONFIG3_ULR 0x00002000 /* UserLocal reg implemented */ + #define MIPS_CONFIG4_MMUSIZEEXT 0x000000FF /* bits 7.. 0 MMU Size Extension */ #define MIPS_CONFIG4_MMUEXTDEF 0x0000C000 /* bits 15.14 MMU Extension Definition */ #define MIPS_CONFIG4_MMUEXTDEF_MMUSIZEEXT 0x00004000 /* This values denotes CONFIG4 bits */ @@ -627,4 +652,15 @@ #define MIPS_OPCODE_SHIFT 26 #define MIPS_OPCODE_C1 0x11 +/* + * Bits defined for for the HWREna (CP0 register 7, select 0). + */ +#define MIPS_HWRENA_CPUNUM (1<<0) /* CPU number program is running on */ +#define MIPS_HWRENA_SYNCI_STEP (1<<1) /* Address step sized used with SYNCI */ +#define MIPS_HWRENA_CC (1<<2) /* Hi Res cycle counter */ +#define MIPS_HWRENA_CCRES (1<<3) /* Cycle counter resolution */ +#define MIPS_HWRENA_UL (1<<29) /* UserLocal Register */ +#define MIPS_HWRENA_IMPL30 (1<<30) /* Implementation-dependent 30 */ +#define MIPS_HWRENA_IMPL31 (1<<31) /* Implementation-dependent 31 */ + #endif /* _MIPS_CPUREGS_H_ */ diff --git a/sys/mips/mips/cpu.c b/sys/mips/mips/cpu.c index 0473711..02b6bb0 100644 --- a/sys/mips/mips/cpu.c +++ b/sys/mips/mips/cpu.c @@ -30,6 +30,7 @@ __FBSDID("$FreeBSD: release/10.0.0/sys/mips/mips/cpu.c 257528 2013-11-01 21:17:4 #include #include #include +#include #include #include @@ -49,6 +50,9 @@ __FBSDID("$FreeBSD: release/10.0.0/sys/mips/mips/cpu.c 257528 2013-11-01 21:17:4 #include #include #include +#include +#include +#include #if defined(CPU_CNMIPS) #include @@ -59,6 +63,63 @@ static void cpu_identify(void); struct mips_cpuinfo cpuinfo; +#define _ENCODE_INSN(a,b,c,d,e) \ + ((uint32_t)(((a) << 26)|((b) << 21)|((c) << 16)|((d) << 11)|(e))) + +#if defined(__mips_n64) + +# define _LOAD_T0_MDTLS_A1 \ + _ENCODE_INSN(OP_LD, A1, T0, 0, offsetof(struct thread, td_md.md_tls)) + +# if defined(COMPAT_FREEBSD32) +# define _ADDIU_V0_T0_TLS_OFFSET \ + _ENCODE_INSN(OP_DADDIU, T0, V0, 0, (TLS_TP_OFFSET + TLS_TCB_SIZE32)) +# else +# define _ADDIU_V0_T0_TLS_OFFSET \ + _ENCODE_INSN(OP_DADDIU, T0, V0, 0, (TLS_TP_OFFSET + TLS_TCB_SIZE)) +# endif /* ! COMPAT_FREEBSD32 */ + +# define _MTC0_V0_USERLOCAL \ + _ENCODE_INSN(OP_COP0, OP_DMT, V0, 4, 2) + +#else /* mips 32 */ + +# define _LOAD_T0_MDTLS_A1 \ + _ENCODE_INSN(OP_LW, A1, T0, 0, offsetof(struct thread, td_md.md_tls)) +# define _ADDIU_V0_T0_TLS_OFFSET \ + _ENCODE_INSN(OP_ADDIU, T0, _V0, 0, (TLS_TP_OFFSET + TLS_TCB_SIZE32)) +# define _MTC0_V0_USERLOCAL \ + _ENCODE_INSN(OP_COP0, OP_MT, _V0, 4, 2) + +#endif /* ! __mips_n64 */ + +#define _JR_RA _ENCODE_INSN(OP_SPECIAL, RA, 0, 0, OP_JR) +#define _NOP 0 + +/* + * Patch cpu_switch() by removing the UserLocal register code at the end. + * For MIPS hardware that don't support UserLocal Register Implementation + * we remove the instructions that update this register which may cause a + * reserved instruction exception in the kernel. + */ +static void +remove_userlocal_code(uint32_t *cpu_switch_code) +{ + uint32_t *instructp; + + for (instructp = cpu_switch_code;; instructp++) { + if (instructp[0] == _JR_RA) + panic("%s: Unable to patch cpu_switch().", __func__); + if (instructp[0] == _LOAD_T0_MDTLS_A1 && + instructp[1] == _ADDIU_V0_T0_TLS_OFFSET && + instructp[2] == _MTC0_V0_USERLOCAL) { + instructp[0] = _JR_RA; + instructp[1] = _NOP; + break; + } + } +} + /* * Attempt to identify the MIPS CPU as much as possible. * @@ -71,8 +132,7 @@ static void mips_get_identity(struct mips_cpuinfo *cpuinfo) { u_int32_t prid; - u_int32_t cfg0; - u_int32_t cfg1; + u_int32_t cfg0, cfg1, cfg2 = 0, cfg3 = 0; #if defined(CPU_CNMIPS) u_int32_t cfg4; #endif @@ -93,13 +153,36 @@ mips_get_identity(struct mips_cpuinfo *cpuinfo) ((cfg0 & MIPS_CONFIG0_MT_MASK) >> MIPS_CONFIG0_MT_SHIFT); cpuinfo->icache_virtual = cfg0 & MIPS_CONFIG0_VI; - /* If config register selection 1 does not exist, exit. */ - if (!(cfg0 & MIPS_CONFIG_CM)) + /* If config register selection 1 does not exist, return. */ + if (!(cfg0 & MIPS_CONFIG0_M)) return; /* Learn TLB size and L1 cache geometry. */ cfg1 = mips_rd_config1(); + /* Get the Config2 and Config3 registers as well. */ + if (cfg1 & MIPS_CONFIG1_M) { + cfg2 = mips_rd_config2(); + if (cfg2 & MIPS_CONFIG2_M) + cfg3 = mips_rd_config3(); + } + + /* Check to see if UserLocal register is implemented. */ + if (cfg3 & MIPS_CONFIG3_ULR) { + /* UserLocal register is implemented, enable it. */ + cpuinfo->userlocal_reg = true; + tmp = mips_rd_hwrena(); + mips_wr_hwrena(tmp | MIPS_HWRENA_UL); + } else { + /* + * UserLocal register is not implemented. Patch + * cpu_switch() and remove unsupported code. + */ + cpuinfo->userlocal_reg = false; + remove_userlocal_code((uint32_t *)cpu_switch); + } + + #if defined(CPU_NLM) /* Account for Extended TLB entries in XLP */ tmp = mips_rd_config6(); @@ -307,7 +390,7 @@ cpu_identify(void) /* Print Config3 if it contains any useful info */ if (cfg3 & ~(0x80000000)) - printf(" Config3=0x%b\n", cfg3, "\20\2SmartMIPS\1TraceLogic"); + printf(" Config3=0x%b\n", cfg3, "\20\14ULRI\2SmartMIPS\1TraceLogic"); } static struct rman cpu_hardirq_rman; diff --git a/sys/mips/mips/genassym.c b/sys/mips/mips/genassym.c index 5df4564..e770314 100644 --- a/sys/mips/mips/genassym.c +++ b/sys/mips/mips/genassym.c @@ -57,6 +57,7 @@ __FBSDID("$FreeBSD: release/10.0.0/sys/mips/mips/genassym.c 232587 2012-03-06 08 #include #include #include +#include #ifdef CPU_CNMIPS #include @@ -72,6 +73,13 @@ ASSYM(TD_KSTACK, offsetof(struct thread, td_kstack)); ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TD_LOCK, offsetof(struct thread, td_lock)); ASSYM(TD_MDFLAGS, offsetof(struct thread, td_md.md_flags)); +ASSYM(TD_MDTLS, offsetof(struct thread, td_md.md_tls)); + +#if defined(__mips_n64) && defined(COMPAT_FREEBSD32) +ASSYM(TLS_TCB_OFFSET, (TLS_TP_OFFSET + TLS_TCB_SIZE32)); +#else +ASSYM(TLS_TCB_OFFSET, (TLS_TP_OFFSET + TLS_TCB_SIZE)); +#endif ASSYM(U_PCB_REGS, offsetof(struct pcb, pcb_regs.zero)); ASSYM(U_PCB_CONTEXT, offsetof(struct pcb, pcb_context)); diff --git a/sys/mips/mips/swtch.S b/sys/mips/mips/swtch.S index ae64883..8b58979 100644 --- a/sys/mips/mips/swtch.S +++ b/sys/mips/mips/swtch.S @@ -383,6 +383,7 @@ sw2: * Restore registers and return. */ move a0, s0 + move a1, s7 RESTORE_U_PCB_CONTEXT(gp, PCB_REG_GP, a0) RESTORE_U_PCB_CONTEXT(v0, PCB_REG_SR, a0) # restore kernel context RESTORE_U_PCB_CONTEXT(ra, PCB_REG_RA, a0) @@ -402,6 +403,15 @@ sw2: or v0, v0, t0 mtc0 v0, MIPS_COP_0_STATUS ITLBNOPFIX +/* + * Set the new thread's TLS pointer. + * + * Note that this code is removed if the CPU doesn't support ULRI by + * remove_userlocal_code() in cpu.c. + */ + PTR_L t0, TD_MDTLS(a1) # Get TLS pointer + PTR_ADDIU v0, t0, TLS_TCB_OFFSET # Add TLS/TCB offset + MTC0 v0, MIPS_COP_0_USERLOCAL, 2 # write it to ULR for rdhwr j ra nop diff --git a/sys/mips/mips/sys_machdep.c b/sys/mips/mips/sys_machdep.c index d04012c..d679b91 100644 --- a/sys/mips/mips/sys_machdep.c +++ b/sys/mips/mips/sys_machdep.c @@ -43,7 +43,11 @@ __FBSDID("$FreeBSD: release/10.0.0/sys/mips/mips/sys_machdep.c 232767 2012-03-10 #include #include +#include +#include #include +#include +#include #ifndef _SYS_SYSPROTO_H_ struct sysarch_args { @@ -61,6 +65,22 @@ sysarch(struct thread *td, struct sysarch_args *uap) switch (uap->op) { case MIPS_SET_TLS: td->td_md.md_tls = uap->parms; + + /* + * If there is an user local register implementation (ULRI) + * update it as well. Add the TLS and TCB offsets so the + * value in this register is adjusted like in the case of the + * rdhwr trap() instruction handler. + */ + if (cpuinfo.userlocal_reg == true) { +#if defined(__mips_n64) && defined(COMPAT_FREEBSD32) + mips_wr_userlocal((unsigned long)(uap->parms + + TLS_TP_OFFSET + TLS_TCB_SIZE32)); +#else + mips_wr_userlocal((unsigned long)(uap->parms + + TLS_TP_OFFSET + TLS_TCB_SIZE)); +#endif + } return (0); case MIPS_GET_TLS: tlsbase = td->td_md.md_tls; diff --git a/sys/mips/mips/vm_machdep.c b/sys/mips/mips/vm_machdep.c index c39bbe6..54dc601 100644 --- a/sys/mips/mips/vm_machdep.c +++ b/sys/mips/mips/vm_machdep.c @@ -60,8 +60,11 @@ __FBSDID("$FreeBSD: release/10.0.0/sys/mips/mips/vm_machdep.c 255786 2013-09-22 #include #include #include +#include +#include #include #include +#include #include #include @@ -606,6 +609,15 @@ cpu_set_user_tls(struct thread *td, void *tls_base) { td->td_md.md_tls = (char*)tls_base; + if (td == curthread && cpuinfo.userlocal_reg == true) { +#if defined(__mips_n64) && defined(COMPAT_FREEBSD32) + mips_wr_userlocal((unsigned long)tls_base + TLS_TP_OFFSET + + TLS_TCB_SIZE32); +#else + mips_wr_userlocal((unsigned long)tls_base + TLS_TP_OFFSET + + TLS_TCB_SIZE); +#endif + } return (0); }