Index: sys/mips/include/pcb.h =================================================================== --- sys/mips/include/pcb.h (revision 203271) +++ sys/mips/include/pcb.h (working copy) @@ -77,8 +77,9 @@ #ifdef _KERNEL extern struct pcb *curpcb; /* the current running pcb */ -void makectx(struct trapframe *, struct pcb *); -int savectx(struct pcb *); +void makectx(struct trapframe *, struct pcb *); +int savectx(struct pcb *); +void restorectx(struct pcb *); #endif #endif /* !_MACHINE_PCB_H_ */ Index: sys/mips/include/cpu.h =================================================================== --- sys/mips/include/cpu.h (revision 203271) +++ sys/mips/include/cpu.h (working copy) @@ -484,13 +484,51 @@ void Mips_InvalidateICache(vm_offset_t, int); void Mips_TLBFlush(int); -void Mips_TLBFlushAddr(vm_offset_t); -void Mips_TLBWriteIndexed(int, struct tlb *); void Mips_TLBUpdate(vm_offset_t, unsigned); void Mips_TLBRead(int, struct tlb *); void mips_TBIAP(int); void wbflush(void); +/* + * Remove the TLB entry mapping the (address,pid) tuple specified by 'entryhi'. + * + * We make the following optimizations/assumptions when called from the + * context switch code path (ctxsw != 0): + * - interrupts are already disabled + * - entryhi and pagemask do not need to be preserved + * + * Most callers should not call this function directly and instead use + * the wrapper function Mips_TLBFlushAddr(). + */ +void Mips_TLBFlushAddr2(vm_offset_t tlbhi, int ctxsw); + +static __inline void +Mips_TLBFlushAddr(vm_offset_t tlbhi) +{ + + Mips_TLBFlushAddr2(tlbhi, 0); +} + +/* + * Write the contents of 'tlb' to the TLB entry at 'index'. + * + * We make the following optimizations/assumptions when called from the + * context switch code path (ctxsw != 0): + * - interrupts are already disabled + * - entryhi and pagemask do not need to be preserved + * + * Most callers should not call this function directly and instead use + * the wrapper function Mips_TLBWriteIndexed(). + */ +void Mips_TLBWriteIndexed2(int index, struct tlb *tlb, int ctxsw); + +static __inline void +Mips_TLBWriteIndexed(int index, struct tlb *tlb) +{ + + Mips_TLBWriteIndexed2(index, tlb, 0); +} + extern u_int32_t cpu_counter_interval; /* Number of counter ticks/tick */ extern u_int32_t cpu_counter_last; /* Last compare value loaded */ extern int num_tlbentries; Index: sys/mips/mips/tlb.S =================================================================== --- sys/mips/mips/tlb.S (revision 203271) +++ sys/mips/mips/tlb.S (working copy) @@ -107,10 +107,15 @@ /*-------------------------------------------------------------------------- * - * Mips_TLBWriteIndexed(unsigned index, tlb *tlb); + * Mips_TLBWriteIndexed2(unsigned index, tlb *tlb, int ctxsw); * * Write the given entry into the TLB at the given index. * + * The function makes the following optimizations/assumptions when + * called from the context switch code path (ctxsw != 0): + * - interrupts are already disabled + * - entryhi and pagemask do not need to be preserved + * * Results: * None. * @@ -119,33 +124,43 @@ * *-------------------------------------------------------------------------- */ -LEAF(Mips_TLBWriteIndexed) +LEAF(Mips_TLBWriteIndexed2) + bne a2, zero, 1f # ctxsw != 0 + nop + mfc0 v1, COP_0_STATUS_REG # Save the status register. mtc0 zero, COP_0_STATUS_REG # Disable interrupts ITLBNOPFIX - lw a2, 8(a1) - lw a3, 12(a1) - _MFC0 t0, COP_0_TLB_HI # Save the current PID. + _MFC0 v0, COP_0_TLB_HI # Save the current PID. - _MTC0 a2, COP_0_TLB_LO0 # Set up entry low0. - _MTC0 a3, COP_0_TLB_LO1 # Set up entry low1. - lw a2, 0(a1) - lw a3, 4(a1) +1: + lw t0, 8(a1) + lw t1, 12(a1) + _MTC0 t0, COP_0_TLB_LO0 # Set up entry low0. + _MTC0 t1, COP_0_TLB_LO1 # Set up entry low1. + + lw t2, 0(a1) + lw t3, 4(a1) mtc0 a0, COP_0_TLB_INDEX # Set the index. - _MTC0 a2, COP_0_TLB_PG_MASK # Set up entry mask. - _MTC0 a3, COP_0_TLB_HI # Set up entry high. + _MTC0 t2, COP_0_TLB_PG_MASK # Set up entry mask. + _MTC0 t3, COP_0_TLB_HI # Set up entry high. MIPS_CPU_NOP_DELAY tlbwi # Write the TLB MIPS_CPU_NOP_DELAY + + bne a2, zero, 2f # ctxsw != 0 + nop - _MTC0 t0, COP_0_TLB_HI # Restore the PID. + _MTC0 v0, COP_0_TLB_HI # Restore the PID. nop _MTC0 zero, COP_0_TLB_PG_MASK # Default mask value. mtc0 v1, COP_0_STATUS_REG # Restore the status register ITLBNOPFIX + +2: j ra nop -END(Mips_TLBWriteIndexed) +END(Mips_TLBWriteIndexed2) /*-------------------------------------------------------------------------- * @@ -267,10 +282,15 @@ /*-------------------------------------------------------------------------- * - * Mips_TLBFlushAddr(unsigned TLBhi); + * Mips_TLBFlushAddr2(unsigned TLBhi, int ctxsw) * * Flush any TLB entries for the given address and TLB PID. * + * The function makes the following optimizations/assumptions when + * called from the context switch code path (ctxsw != 0): + * - interrupts are already disabled + * - entryhi and pagemask do not need to be preserved + * * Results: * None. * @@ -279,22 +299,25 @@ * *-------------------------------------------------------------------------- */ -LEAF(Mips_TLBFlushAddr) +LEAF(Mips_TLBFlushAddr2) + bne a1, zero, 1f # ctxsw != 0 + li v0, (PTE_HVPN | PTE_ASID) + mfc0 v1, COP_0_STATUS_REG # Save the status register. mtc0 zero, COP_0_STATUS_REG # Disable interrupts ITLBNOPFIX - li v0, (PTE_HVPN | PTE_ASID) - and a0, a0, v0 # Make shure valid hi value. _MFC0 t0, COP_0_TLB_HI # Get current PID mfc0 t3, COP_0_TLB_PG_MASK # Save current pgMask + +1: + and a0, a0, v0 # get the valid bits for tlbhi _MTC0 a0, COP_0_TLB_HI # look for addr & PID MIPS_CPU_NOP_DELAY tlbp # Probe for the entry. MIPS_CPU_NOP_DELAY mfc0 v0, COP_0_TLB_INDEX # See what we got + bltz v0, 2f # index < 0 => !found li t1, MIPS_KSEG0_START - bltz v0, 1f # index < 0 => !found - nop # Load invalid entry, each TLB entry should have it's own bogus # address calculated by following expression: # MIPS_KSEG0_START + 2 * i * PAGE_SIZE; @@ -308,14 +331,17 @@ MIPS_CPU_NOP_DELAY tlbwi MIPS_CPU_NOP_DELAY -1: +2: + bne a1, zero, 3f # ctxsw != 0 + nop _MTC0 t0, COP_0_TLB_HI # restore PID mtc0 t3, COP_0_TLB_PG_MASK # Restore pgMask mtc0 v1, COP_0_STATUS_REG # Restore the status register ITLBNOPFIX +3: j ra nop -END(Mips_TLBFlushAddr) +END(Mips_TLBFlushAddr2) /*-------------------------------------------------------------------------- * Index: sys/mips/mips/vm_machdep.c =================================================================== --- sys/mips/mips/vm_machdep.c (revision 203271) +++ sys/mips/mips/vm_machdep.c (working copy) @@ -100,6 +100,8 @@ */ static struct mtx sf_buf_lock; +static void switch_to_new_thread(struct thread *new) __dead2; + /* * Finish a fork operation, with process p2 nearly set up. * Copy and update the pcb, set up the stack so that the child @@ -582,14 +584,58 @@ return (0); } +static void +switch_to_new_thread(struct thread *new) +{ + vm_offset_t newstack; + struct pcb *newpcb; + struct tlb tlb; + +#if defined(SCHED_ULE) && defined(SMP) + while (atomic_load_acq_ptr(&new->td_lock) == &blocked_lock) + ; +#endif + newpcb = new->td_pcb; + newstack = new->td_md.md_realstack; + KASSERT((newstack & ((PAGE_SIZE * 2) - 1)) == 0, + ("newstack is not aligned on a 8192 byte boundary")); + PCPU_SET(curthread, new); + PCPU_SET(curpcb, newpcb); + if (newstack >= MIPS_KSEG2_START) { + tlb.tlb_mask = 0; + tlb.tlb_hi = newstack; + tlb.tlb_lo0 = new->td_md.md_upte[0]; + tlb.tlb_lo1 = new->td_md.md_upte[1]; + Mips_TLBFlushAddr2(newstack, 1); + Mips_TLBWriteIndexed2(KSTACK_TLB_ENTRY, &tlb, 1); + } + pmap_activate(new); + restorectx(newpcb); + panic("restorectx returned!"); +} + void -cpu_throw(struct thread *old, struct thread *new) +cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) { + struct pcb *oldpcb; - func_2args_asmmacro(&mips_cpu_throw, old, new); - panic("mips_cpu_throw() returned"); + oldpcb = old->td_pcb; + if (savectx(oldpcb) == 0) { + atomic_store_rel_ptr((volatile uintptr_t *)&old->td_lock, + (uintptr_t)mtx); + switch_to_new_thread(new); + /* NOTREACHED */ + } } +void +cpu_throw(struct thread *old __unused, struct thread *new) +{ + + switch_to_new_thread(new); + /* NOTREACHED */ +} + #ifdef DDB #include Index: sys/mips/mips/swtch.S =================================================================== --- sys/mips/mips/swtch.S (revision 203271) +++ sys/mips/mips/swtch.S (working copy) @@ -231,174 +231,43 @@ END(fork_trampoline) /* - * Update pcb, saving current processor state. - * Note: this only works if pcbp != curproc's pcb since - * cpu_switch() will copy over pcb_context. - * - * savectx(struct pcb *pcbp); + * int savectx(struct pcb *pcb); + * Returns 0 when called directly to save the context. + * Returns 1 when it returns via restorectx(). */ LEAF(savectx) + mfc0 v0, COP_0_STATUS_REG SAVE_U_PCB_CONTEXT(s0, PREG_S0, a0) SAVE_U_PCB_CONTEXT(s1, PREG_S1, a0) SAVE_U_PCB_CONTEXT(s2, PREG_S2, a0) SAVE_U_PCB_CONTEXT(s3, PREG_S3, a0) - mfc0 v0, COP_0_STATUS_REG SAVE_U_PCB_CONTEXT(s4, PREG_S4, a0) SAVE_U_PCB_CONTEXT(s5, PREG_S5, a0) SAVE_U_PCB_CONTEXT(s6, PREG_S6, a0) SAVE_U_PCB_CONTEXT(s7, PREG_S7, a0) - SAVE_U_PCB_CONTEXT(sp, PREG_SP, a0) SAVE_U_PCB_CONTEXT(s8, PREG_S8, a0) + SAVE_U_PCB_CONTEXT(sp, PREG_SP, a0) SAVE_U_PCB_CONTEXT(ra, PREG_RA, a0) - SAVE_U_PCB_CONTEXT(v0, PREG_SR, a0) SAVE_U_PCB_CONTEXT(gp, PREG_GP, a0) + SAVE_U_PCB_CONTEXT(v0, PREG_SR, a0) + move v1, ra /* save 'ra' before we trash it */ + jal getpc + nop +getpc: + SAVE_U_PCB_CONTEXT(ra, PREG_PC, a0) + move ra, v1 /* restore 'ra' before returning */ /* - * FREEBSD_DEVELOPERS_FIXME: - * In case there are CPU-specific registers that need - * to be saved with the other registers do so here. + * Save any CPU-specific registers here. */ j ra move v0, zero END(savectx) - -KSEG0TEXT_START; - -NON_LEAF(mips_cpu_throw, STAND_FRAME_SIZE, ra) - mfc0 t0, COP_0_STATUS_REG # t0 = saved status register - nop - nop - and a3, t0, ~(SR_INT_ENAB) - mtc0 a3, COP_0_STATUS_REG # Disable all interrupts - ITLBNOPFIX - j mips_sw1 # We're not interested in old - # thread's context, so jump - # right to action - nop # BDSLOT -END(mips_cpu_throw) - /* - * cpu_switch(struct thread *old, struct thread *new, struct mutex *mtx); - * a0 - old - * a1 - new - * a2 - mtx - * Find the highest priority process and resume it. + * void restorectx(struct pcb *pcb); + * Return via savectx() with a return value of 1. */ -NON_LEAF(cpu_switch, STAND_FRAME_SIZE, ra) - mfc0 t0, COP_0_STATUS_REG # t0 = saved status register - nop - nop - and a3, t0, ~(SR_INT_ENAB) - mtc0 a3, COP_0_STATUS_REG # Disable all interrupts - ITLBNOPFIX - beqz a0, mips_sw1 - move a3, a0 - lw a0, TD_PCB(a0) # load PCB addr of curproc - SAVE_U_PCB_CONTEXT(sp, PREG_SP, a0) # save old sp - subu sp, sp, STAND_FRAME_SIZE - sw ra, STAND_RA_OFFSET(sp) - .mask 0x80000000, (STAND_RA_OFFSET - STAND_FRAME_SIZE) - SAVE_U_PCB_CONTEXT(s0, PREG_S0, a0) # do a 'savectx()' - SAVE_U_PCB_CONTEXT(s1, PREG_S1, a0) - SAVE_U_PCB_CONTEXT(s2, PREG_S2, a0) - SAVE_U_PCB_CONTEXT(s3, PREG_S3, a0) - SAVE_U_PCB_CONTEXT(s4, PREG_S4, a0) - SAVE_U_PCB_CONTEXT(s5, PREG_S5, a0) - SAVE_U_PCB_CONTEXT(s6, PREG_S6, a0) - SAVE_U_PCB_CONTEXT(s7, PREG_S7, a0) - SAVE_U_PCB_CONTEXT(s8, PREG_S8, a0) - SAVE_U_PCB_CONTEXT(ra, PREG_RA, a0) # save return address - SAVE_U_PCB_CONTEXT(t0, PREG_SR, a0) # save status register - SAVE_U_PCB_CONTEXT(gp, PREG_GP, a0) - jal getpc - nop -getpc: - SAVE_U_PCB_CONTEXT(ra, PREG_PC, a0) # save return address - /* - * FREEBSD_DEVELOPERS_FIXME: - * In case there are CPU-specific registers that need - * to be saved with the other registers do so here. - */ - - sw a2, TD_LOCK(a3) # Switchout td_lock - -mips_sw1: -#if defined(SMP) && defined(SCHED_ULE) - PTR_LA t0, _C_LABEL(blocked_lock) -blocked_loop: - lw t1, TD_LOCK(a1) - beq t0, t1, blocked_loop - nop -#endif - move s7, a1 # Store newthread -/* - * Switch to new context. - */ - GET_CPU_PCPU(a3) - sw a1, PC_CURTHREAD(a3) - lw a2, TD_PCB(a1) - sw a2, PC_CURPCB(a3) - lw v0, TD_REALKSTACK(a1) - li s0, (MIPS_KSEG2_START+VM_KERNEL_ALLOC_OFFSET) # If Uarea addr is below kseg2, - bltu v0, s0, sw2 # no need to insert in TLB. - lw a1, TD_UPTE+0(s7) # t0 = first u. pte - lw a2, TD_UPTE+4(s7) # t1 = 2nd u. pte - and s0, v0, PTE_ODDPG - beq s0, zero, entry0 - nop - - PANIC_KSEG0("USPACE sat on odd page boundary", t1) - -/* - * Wiredown the USPACE of newproc in TLB entry#0. Check whether target - * USPACE is already in another place of TLB before that, and if so - * invalidate that TLB entry. - * NOTE: This is hard coded to UPAGES == 2. - * Also, there should be no TLB faults at this point. - */ -entry0: - mtc0 v0, COP_0_TLB_HI # VPN = va - HAZARD_DELAY - tlbp # probe VPN - HAZARD_DELAY - mfc0 s0, COP_0_TLB_INDEX - nop -pgm: - bltz s0, entry0set - li t1, MIPS_KSEG0_START # invalidate tlb entry - sll s0, PAGE_SHIFT + 1 - addu t1, s0 - mtc0 t1, COP_0_TLB_HI - mtc0 zero, COP_0_TLB_LO0 - mtc0 zero, COP_0_TLB_LO1 - HAZARD_DELAY - tlbwi - HAZARD_DELAY - mtc0 v0, COP_0_TLB_HI # set VPN again -entry0set: -/* SMP!! - Works only for unshared TLB case - i.e. no v-cpus */ - mtc0 zero, COP_0_TLB_INDEX # TLB entry #0 -# or a1, PG_G - mtc0 a1, COP_0_TLB_LO0 # upte[0] -# or a2, PG_G - mtc0 a2, COP_0_TLB_LO1 # upte[1] - HAZARD_DELAY - tlbwi # set TLB entry #0 - HAZARD_DELAY -/* - * Now running on new u struct. - */ -sw2: - PTR_LA t1, _C_LABEL(pmap_activate) # s7 = new proc pointer - jalr t1 # s7 = new proc pointer - move a0, s7 # BDSLOT -/* - * Restore registers and return. - */ - lw a0, TD_PCB(s7) - RESTORE_U_PCB_CONTEXT(gp, PREG_GP, a0) - RESTORE_U_PCB_CONTEXT(v0, PREG_SR, a0) # restore kernel context - RESTORE_U_PCB_CONTEXT(ra, PREG_RA, a0) +LEAF(restorectx) RESTORE_U_PCB_CONTEXT(s0, PREG_S0, a0) RESTORE_U_PCB_CONTEXT(s1, PREG_S1, a0) RESTORE_U_PCB_CONTEXT(s2, PREG_S2, a0) @@ -407,24 +276,24 @@ RESTORE_U_PCB_CONTEXT(s5, PREG_S5, a0) RESTORE_U_PCB_CONTEXT(s6, PREG_S6, a0) RESTORE_U_PCB_CONTEXT(s7, PREG_S7, a0) - RESTORE_U_PCB_CONTEXT(sp, PREG_SP, a0) RESTORE_U_PCB_CONTEXT(s8, PREG_S8, a0) - /* - * FREEBSD_DEVELOPERS_FIXME: - * In case there are CPU-specific registers that need - * to be restored with the other registers do so here. - */ + RESTORE_U_PCB_CONTEXT(sp, PREG_SP, a0) + RESTORE_U_PCB_CONTEXT(ra, PREG_RA, a0) + RESTORE_U_PCB_CONTEXT(gp, PREG_GP, a0) + + RESTORE_U_PCB_CONTEXT(v0, PREG_SR, a0) mfc0 t0, COP_0_STATUS_REG and t0, t0, SR_INT_MASK and v0, v0, ~SR_INT_MASK or v0, v0, t0 mtc0 v0, COP_0_STATUS_REG ITLBNOPFIX - + /* + * Restore any CPU-specific registers here. + */ j ra - nop -END(cpu_switch) -KSEG0TEXT_END; + li v0, 1 +END(restorectx) /*---------------------------------------------------------------------------- *