Index: sys/mips/include/db_machdep.h =================================================================== --- sys/mips/include/db_machdep.h (revision 205197) +++ sys/mips/include/db_machdep.h (working copy) @@ -92,7 +92,6 @@ #define DB_SMALL_VALUE_MIN (-0x400001) int db_inst_type(int); -void db_dump_tlb(int, int); db_addr_t branch_taken(int inst, db_addr_t pc); void stacktrace_subr(register_t pc, register_t sp, register_t ra, int (*)(const char *, ...)); int kdbpeek(int *); Index: sys/mips/include/trap.h =================================================================== --- sys/mips/include/trap.h (revision 205197) +++ sys/mips/include/trap.h (working copy) @@ -111,11 +111,10 @@ void MipsFPTrap(u_int, u_int, u_int); void MipsKernGenException(void); void MipsKernIntr(void); -void MipsKernTLBInvalidException(void); +void MipsTLBInvalidException(void); void MipsTLBMissException(void); void MipsUserGenException(void); void MipsUserIntr(void); -void MipsUserTLBInvalidException(void); u_int trap(struct trapframe *); Index: sys/mips/mips/db_trace.c =================================================================== --- sys/mips/mips/db_trace.c (revision 205197) +++ sys/mips/mips/db_trace.c (working copy) @@ -162,13 +162,10 @@ subr = (uintptr_t)MipsUserGenException; else if (pcBetween(MipsKernIntr, MipsUserIntr)) subr = (uintptr_t)MipsKernIntr; - else if (pcBetween(MipsUserIntr, MipsKernTLBInvalidException)) + else if (pcBetween(MipsUserIntr, MipsTLBInvalidException)) subr = (uintptr_t)MipsUserIntr; - else if (pcBetween(MipsKernTLBInvalidException, - MipsUserTLBInvalidException)) - subr = (uintptr_t)MipsKernTLBInvalidException; - else if (pcBetween(MipsUserTLBInvalidException, MipsTLBMissException)) - subr = (uintptr_t)MipsUserTLBInvalidException; + else if (pcBetween(MipsTLBInvalidException, MipsTLBMissException)) + subr = (uintptr_t)MipsTLBInvalidException; else if (pcBetween(fork_trampoline, savectx)) subr = (uintptr_t)fork_trampoline; else if (pcBetween(savectx, mips_cpu_throw)) Index: sys/mips/mips/exception.S =================================================================== --- sys/mips/mips/exception.S (revision 205197) +++ sys/mips/mips/exception.S (working copy) @@ -264,13 +264,13 @@ mfc0 a0, COP_0_STATUS_REG ;\ li a2, (MIPS_SR_KX | MIPS_SR_SX | MIPS_SR_UX) ; \ or a0, a0, a2 ; \ - li a2, ~(MIPS_SR_INT_IE|MIPS_SR_EXL) ; \ + li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, COP_0_STATUS_REG #else #define CLEAR_STATUS \ mfc0 a0, COP_0_STATUS_REG ;\ - li a2, ~(MIPS_SR_INT_IE|MIPS_SR_EXL) ; \ + li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, COP_0_STATUS_REG #endif @@ -827,178 +827,169 @@ .set at END(MipsUserIntr) -NLEAF(MipsKernTLBInvalidException) - .set noat - mfc0 k0, COP_0_BAD_VADDR # get the fault address +NLEAF(MipsTLBInvalidException) + .set push + .set noat + .set noreorder - + mfc0 k0, COP_0_BAD_VADDR li k1, VM_MAXUSER_ADDRESS sltu k1, k0, k1 - beqz k1, 1f + bnez k1, 1f nop - GET_CPU_PCPU(k1) - lw k1, PC_SEGBASE(k1) # works for single cpu???? - beqz k1, _C_LABEL(MipsKernGenException) # seg tab is null - nop + + /* badvaddr = kernel address */ + lui k1, %hi(_C_LABEL(kernel_segmap)) b 2f - nop + lw k1, %lo(_C_LABEL(kernel_segmap))(k1) + 1: - li k1, (VM_MAX_KERNEL_ADDRESS) - bgez k0, _C_LABEL(MipsKernGenException) # full trap processing - sltu k1, k1, k0 # check fault address against - bnez k1, _C_LABEL(MipsKernGenException) # kernel_segmap upper bound - lui k1, %hi(_C_LABEL(kernel_segmap)) # k1=hi of segbase - lw k1, %lo(_C_LABEL(kernel_segmap))(k1) # k1=segment tab base - beqz k1, _C_LABEL(MipsKernGenException) # seg tab is null + /* badvaddr = user address */ + GET_CPU_PCPU(k1) + lw k1, PC_SEGBASE(k1) + 2: - srl k0, 20 # k0=seg offset (almost) - andi k0, k0, 0xffc # k0=seg offset (mask 0x3) -#xxx mips64 unsafe? - addu k1, k0, k1 # k1=seg entry address - lw k1, 0(k1) # k1=seg entry - mfc0 k0, COP_0_BAD_VADDR # k0=bad address (again) - beq k1, zero, _C_LABEL(MipsKernGenException) # ==0 -- no page table - srl k0, k0, PGSHIFT-2 - andi k0, k0, 0xffc # compute offset from index - tlbp # Probe the invalid entry -#xxx mips64 unsafe? + beqz k1, 3f /* invalid page directory pointer */ + nop + + srl k0, SEGSHIFT - 2 + andi k0, 0xffc addu k1, k1, k0 - and k0, k0, 4 # check even/odd page - nop # required for QED 5230 - bne k0, zero, KernTLBIOdd + lw k1, 0(k1) + beqz k1, 3f /* invalid page table page pointer */ nop - mfc0 k0, COP_0_TLB_INDEX + mfc0 k0, COP_0_BAD_VADDR + srl k0, PGSHIFT - 2 + andi k0, 0xffc + addu k1, k1, k0 + + lw k0, 0(k1) + andi k0, PTE_V + beqz k0, 3f /* invalid page table entry */ nop - bltz k0, sys_stk_chk - sltiu k0, k0, VMWIRED_ENTRIES # index below wired entries? - bne k0, zero, sys_stk_chk - lw k0, 0(k1) # get PTE entry + andi k0, k1, 4 + bnez k0, odd_page + nop - _SLL k0, k0, WIRED_SHIFT # get rid of "wired" bit +even_page: + lw k0, 0(k1) + _SLL k0, k0, WIRED_SHIFT _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO0 # load PTE entry - and k0, k0, PTE_V # check for valid entry - nop # required for QED5230 - beq k0, zero, _C_LABEL(MipsKernGenException) # PTE invalid - lw k0, 4(k1) # get odd PTE entry + _MTC0 k0, COP_0_TLB_LO0 + + lw k0, 4(k1) _SLL k0, k0, WIRED_SHIFT _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO1 # load PTE entry - HAZARD_DELAY - tlbwi # write TLB - HAZARD_DELAY - eret + _MTC0 k0, COP_0_TLB_LO1 -KernTLBIOdd: - mfc0 k0, COP_0_TLB_INDEX + b tlb_insert_entry nop - bltz k0, sys_stk_chk - sltiu k0, k0, VMWIRED_ENTRIES # index below wired entries? - bne k0, zero, sys_stk_chk - lw k0, 0(k1) # get PTE entry - - _SLL k0, k0, WIRED_SHIFT # get rid of wired bit +odd_page: + lw k0, 0(k1) + _SLL k0, k0, WIRED_SHIFT _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO1 # save PTE entry - and k0, k0, PTE_V # check for valid entry - nop # required for QED5230 - beq k0, zero, _C_LABEL(MipsKernGenException) # PTE invalid - lw k0, -4(k1) # get even PTE entry + _MTC0 k0, COP_0_TLB_LO1 + + lw k0, -4(k1) _SLL k0, k0, WIRED_SHIFT _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO0 # save PTE entry + _MTC0 k0, COP_0_TLB_LO0 + +tlb_insert_entry: + tlbp HAZARD_DELAY - tlbwi # update TLB + mfc0 k0, COP_0_TLB_INDEX HAZARD_DELAY + bltz k0, tlb_insert_random + nop + tlbwi eret + ssnop - .set at -END(MipsKernTLBInvalidException) +tlb_insert_random: + tlbwr + eret + ssnop - -NLEAF(MipsUserTLBInvalidException) - .set noat - mfc0 k0, COP_0_BAD_VADDR # get the fault address - - li k1, VM_MAXUSER_ADDRESS - sltu k1, k0, k1 - beqz k1, _C_LABEL(MipsUserGenException) +3: + /* + * Branch to the comprehensive exception processing. + */ + mfc0 k1, COP_0_STATUS_REG + andi k1, k1, SR_KSU_USER + bnez k1, _C_LABEL(MipsUserGenException) nop + + /* + * Check for kernel stack overflow. + */ GET_CPU_PCPU(k1) - lw k1, PC_SEGBASE(k1) # works for single cpu???? - beqz k1, _C_LABEL(MipsUserGenException) # seg tab is null + lw k0, PC_CURTHREAD(k1) + lw k0, TD_REALKSTACK(k0) + sltu k0, k0, sp + bnez k0, _C_LABEL(MipsKernGenException) nop -2: - srl k0, 20 # k0=seg offset (almost) - andi k0, k0, 0xffc # k0=seg offset (mask 0x3) -#xxx mips64 unsafe? - addu k1, k0, k1 # k1=seg entry address - lw k1, 0(k1) # k1=seg entry - mfc0 k0, COP_0_BAD_VADDR # k0=bad address (again) - beq k1, zero, _C_LABEL(MipsUserGenException) # ==0 -- no page table - srl k0, k0, PGSHIFT-2 - andi k0, k0, 0xffc # compute offset from index - tlbp # Probe the invalid entry -#xxx mips64 unsafe? - addu k1, k1, k0 - and k0, k0, 4 # check even/odd page - nop # required for QED 5230 - bne k0, zero, UserTLBIOdd - nop - mfc0 k0, COP_0_TLB_INDEX - nop - bltz k0, _C_LABEL(MipsUserGenException) + /* + * Kernel stack overflow. + * + * Move to a valid stack before we call panic. We use the boot stack + * for this purpose. + */ + GET_CPU_PCPU(k1) + lw k1, PC_CPUID(k1) + sll k1, k1, PAGE_SHIFT + 1 - sltiu k0, k0, VMWIRED_ENTRIES # index below wired entries? - bne k0, zero, _C_LABEL(MipsUserGenException) - lw k0, 0(k1) # get PTE entry + PTR_LA k0, _C_LABEL(pcpu_space) + addiu k0, (NBPG * 2) + addu k0, k0, k1 - _SLL k0, k0, WIRED_SHIFT # get rid of "wired" bit - _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO0 # load PTE entry - and k0, k0, PTE_V # check for valid entry - nop # required for QED5230 - beq k0, zero, _C_LABEL(MipsUserGenException) # PTE invalid - lw k0, 4(k1) # get odd PTE entry - _SLL k0, k0, WIRED_SHIFT - _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO1 # load PTE entry - HAZARD_DELAY - tlbwi # write TLB - HAZARD_DELAY - eret + /* + * Stash the original value of 'sp' so we can update trapframe later. + * We assume that SAVE_CPU does not trash 'k1'. + */ + move k1, sp -UserTLBIOdd: - mfc0 k0, COP_0_TLB_INDEX - nop - bltz k0, _C_LABEL(MipsUserGenException) - sltiu k0, k0, VMWIRED_ENTRIES # index below wired entries? + move sp, k0 + subu sp, sp, KERN_EXC_FRAME_SIZE - bne k0, zero, _C_LABEL(MipsUserGenException) - lw k0, 0(k1) # get PTE entry + move k0, ra + move ra, zero + sw ra, CALLFRAME_RA(sp) /* stop the ddb backtrace right here */ + sw zero, CALLFRAME_SP(sp) + move ra, k0 - _SLL k0, k0, WIRED_SHIFT # get rid of wired bit - _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO1 # save PTE entry - and k0, k0, PTE_V # check for valid entry - nop # required for QED5230 - beq k0, zero, _C_LABEL(MipsUserGenException) # PTE invalid - lw k0, -4(k1) # get even PTE entry - _SLL k0, k0, WIRED_SHIFT - _SRL k0, k0, WIRED_SHIFT - _MTC0 k0, COP_0_TLB_LO0 # save PTE entry - HAZARD_DELAY - tlbwi # update TLB - HAZARD_DELAY - eret + SAVE_CPU - .set at -END(MipsUserTLBInvalidException) + /* + * Now restore the value of 'sp' at the time of the tlb exception in + * the trapframe. + */ + SAVE_REG(k1, SP, sp) + /* + * Squelch any more overflow checks by setting the stack base to 0. + */ + GET_CPU_PCPU(k1) + lw k0, PC_CURTHREAD(k1) + sw zero, TD_REALKSTACK(k0) + + move a1, a0 + PANIC("kernel stack overflow - trapframe at %p") + + /* + * This nop is necessary so that the 'ra' remains within the bounds + * of this handler. Otherwise the ddb backtrace code will think that + * the panic() was called from MipsTLBMissException. + */ + nop + + .set pop +END(MipsTLBInvalidException) + /*---------------------------------------------------------------------------- * * MipsTLBMissException -- @@ -1048,68 +1039,6 @@ tlbwr # write to tlb HAZARD_DELAY eret # return from exception - -sys_stk_chk: - GET_CPU_PCPU(k0) - lw k0, PC_CURTHREAD(k0) - lw k0, TD_REALKSTACK(k0) - sltu k0, sp, k0 # check for stack overflow - beqz k0, _C_LABEL(MipsKernGenException) # not stack overflow - nop - -# stack overflow - PTR_LA a0, _C_LABEL(_start) - CALLFRAME_SIZ - 8 # set sp to a valid place - sw sp, 24(a0) - move sp, a0 - PTR_LA a0, 1f - mfc0 a2, COP_0_STATUS_REG - mfc0 a3, COP_0_CAUSE_REG - _MFC0 a1, COP_0_EXC_PC - sw a2, 16(sp) - sw a3, 20(sp) - move a2, ra - PTR_LA k0, _C_LABEL(printf) - jalr k0 - mfc0 a3, COP_0_BAD_VADDR - - PTR_LA sp, _C_LABEL(_start) - CALLFRAME_SIZ # set sp to a valid place - -#if !defined(SMP) && defined(DDB) - PTR_LA a0, 2f - PTR_LA k0, _C_LABEL(trapDump) - jalr k0 - nop - - li a0, 0 - lw a1, _C_LABEL(num_tlbentries) - PTR_LA k0, _C_LABEL(db_dump_tlb) - jalr k0 - addu a1, -1 - -3: - b 3b - nop -#endif - - PANIC("kernel stack overflow") - - .data - .globl lastktlbmiss -lastktlbmiss: - .word 0 -lastktlbmisspc: - .word 0 -lastutlbmiss: - .word 0 -lastutlbmisspc: - .word 0 - -1: - .asciiz "ktlbmiss: PC %x RA %x ADR %x\nSR %x CR %x SP %x\n" -2: - .asciiz "stack ovf" - .text - .set at END(MipsTLBMissException) Index: sys/mips/mips/pmap.c =================================================================== --- sys/mips/mips/pmap.c (revision 205197) +++ sys/mips/mips/pmap.c (working copy) @@ -949,10 +949,21 @@ static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { + vm_offset_t pteva; /* * unmap the page table page */ + pteva = (vm_offset_t)pmap->pm_segtab[m->pindex]; + if (pteva >= VM_MIN_KERNEL_ADDRESS) { + pmap_kremove(pteva); + kmem_free(kernel_map, pteva, PAGE_SIZE); + } else { + KASSERT(MIPS_IS_KSEG0_ADDR(pteva), + ("_pmap_unwire_pte_hold: 0x%0lx is not in kseg0", + (long)pteva)); + } + pmap->pm_segtab[m->pindex] = 0; --pmap->pm_stats.resident_count; @@ -997,7 +1008,7 @@ mpte = pmap->pm_ptphint; } else { pteva = *pmap_pde(pmap, va); - mpte = PHYS_TO_VM_PAGE(MIPS_KSEG0_TO_PHYS(pteva)); + mpte = PHYS_TO_VM_PAGE(vtophys(pteva)); pmap->pm_ptphint = mpte; } } @@ -1029,6 +1040,8 @@ int pmap_pinit(pmap_t pmap) { + vm_offset_t ptdva; + vm_paddr_t ptdpa; vm_page_t ptdpg; int i; int req; @@ -1050,8 +1063,17 @@ ptdpg->valid = VM_PAGE_BITS_ALL; - pmap->pm_segtab = (pd_entry_t *) - MIPS_PHYS_TO_KSEG0(VM_PAGE_TO_PHYS(ptdpg)); + ptdpa = VM_PAGE_TO_PHYS(ptdpg); + if (ptdpa < MIPS_KSEG0_LARGEST_PHYS) { + ptdva = MIPS_PHYS_TO_KSEG0(ptdpa); + } else { + ptdva = kmem_alloc_nofault(kernel_map, PAGE_SIZE); + if (ptdva == 0) + panic("pmap_pinit: unable to allocate kva"); + pmap_kenter(ptdva, ptdpa); + } + + pmap->pm_segtab = (pd_entry_t *)ptdva; if ((ptdpg->flags & PG_ZERO) == 0) bzero(pmap->pm_segtab, PAGE_SIZE); @@ -1118,7 +1140,15 @@ pmap->pm_stats.resident_count++; ptepa = VM_PAGE_TO_PHYS(m); - pteva = MIPS_PHYS_TO_KSEG0(ptepa); + if (ptepa < MIPS_KSEG0_LARGEST_PHYS) { + pteva = MIPS_PHYS_TO_KSEG0(ptepa); + } else { + pteva = kmem_alloc_nofault(kernel_map, PAGE_SIZE); + if (pteva == 0) + panic("_pmap_allocpte: unable to allocate kva"); + pmap_kenter(pteva, ptepa); + } + pmap->pm_segtab[ptepindex] = (pd_entry_t)pteva; /* @@ -1172,7 +1202,7 @@ (pmap->pm_ptphint->pindex == ptepindex)) { m = pmap->pm_ptphint; } else { - m = PHYS_TO_VM_PAGE(MIPS_KSEG0_TO_PHYS(pteva)); + m = PHYS_TO_VM_PAGE(vtophys(pteva)); pmap->pm_ptphint = m; } m->wire_count++; @@ -1212,13 +1242,24 @@ void pmap_release(pmap_t pmap) { + vm_offset_t ptdva; vm_page_t ptdpg; KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); - ptdpg = PHYS_TO_VM_PAGE(MIPS_KSEG0_TO_PHYS(pmap->pm_segtab)); + ptdva = (vm_offset_t)pmap->pm_segtab; + ptdpg = PHYS_TO_VM_PAGE(vtophys(ptdva)); + + if (ptdva >= VM_MIN_KERNEL_ADDRESS) { + pmap_kremove(ptdva); + kmem_free(kernel_map, ptdva, PAGE_SIZE); + } else { + KASSERT(MIPS_IS_KSEG0_ADDR(ptdva), + ("pmap_release: 0x%0lx is not in kseg0", (long)ptdva)); + } + ptdpg->wire_count--; atomic_subtract_int(&cnt.v_wire_count, 1); vm_page_free_zero(ptdpg); @@ -2030,7 +2071,7 @@ (pmap->pm_ptphint->pindex == ptepindex)) { mpte = pmap->pm_ptphint; } else { - mpte = PHYS_TO_VM_PAGE(MIPS_KSEG0_TO_PHYS(pteva)); + mpte = PHYS_TO_VM_PAGE(vtophys(pteva)); pmap->pm_ptphint = mpte; } mpte->wire_count++; @@ -3171,18 +3212,6 @@ pmap->pm_asid[PCPU_GET(cpuid)].gen = PCPU_GET(asid_generation); PCPU_SET(next_asid, PCPU_GET(next_asid) + 1); } - -#ifdef DEBUG - if (pmapdebug & (PDB_FOLLOW | PDB_TLBPID)) { - if (curproc) - printf("pmap_asid_alloc: curproc %d '%s' ", - curproc->p_pid, curproc->p_comm); - else - printf("pmap_asid_alloc: curproc "); - printf("segtab %p asid %d\n", pmap->pm_segtab, - pmap->pm_asid[PCPU_GET(cpuid)].asid); - } -#endif } int @@ -3251,43 +3280,7 @@ PHYS_TO_VM_PAGE(pa)->md.pv_flags |= (PV_TABLE_REF | PV_TABLE_MOD); } -#include - /* - * Dump the translation buffer (TLB) in readable form. - */ - -void -db_dump_tlb(int first, int last) -{ - struct tlb tlb; - int tlbno; - - tlbno = first; - - while (tlbno <= last) { - MachTLBRead(tlbno, &tlb); - if (tlb.tlb_lo0 & PTE_V || tlb.tlb_lo1 & PTE_V) { - printf("TLB %2d vad 0x%08x ", tlbno, (tlb.tlb_hi & 0xffffff00)); - } else { - printf("TLB*%2d vad 0x%08x ", tlbno, (tlb.tlb_hi & 0xffffff00)); - } - printf("0=0x%08x ", pfn_to_vad(tlb.tlb_lo0)); - printf("%c", tlb.tlb_lo0 & PTE_M ? 'M' : ' '); - printf("%c", tlb.tlb_lo0 & PTE_G ? 'G' : ' '); - printf(" atr %x ", (tlb.tlb_lo0 >> 3) & 7); - printf("1=0x%08x ", pfn_to_vad(tlb.tlb_lo1)); - printf("%c", tlb.tlb_lo1 & PTE_M ? 'M' : ' '); - printf("%c", tlb.tlb_lo1 & PTE_G ? 'G' : ' '); - printf(" atr %x ", (tlb.tlb_lo1 >> 3) & 7); - printf(" sz=%x pid=%x\n", tlb.tlb_mask, - (tlb.tlb_hi & 0x000000ff) - ); - tlbno++; - } -} - -/* * Routine: pmap_kextract * Function: * Extract the physical page address associated Index: sys/mips/mips/trap.c =================================================================== --- sys/mips/mips/trap.c (revision 205197) +++ sys/mips/mips/trap.c (working copy) @@ -118,8 +118,8 @@ */ MipsKernIntr, /* external interrupt */ MipsKernGenException, /* TLB modification */ - MipsKernTLBInvalidException, /* TLB miss (load or instr. fetch) */ - MipsKernTLBInvalidException, /* TLB miss (store) */ + MipsTLBInvalidException,/* TLB miss (load or instr. fetch) */ + MipsTLBInvalidException,/* TLB miss (store) */ MipsKernGenException, /* address error (load or I-fetch) */ MipsKernGenException, /* address error (store) */ MipsKernGenException, /* bus error (I-fetch) */ @@ -153,8 +153,8 @@ */ MipsUserIntr, /* 0 */ MipsUserGenException, /* 1 */ - MipsUserTLBInvalidException, /* 2 */ - MipsUserTLBInvalidException, /* 3 */ + MipsTLBInvalidException,/* 2 */ + MipsTLBInvalidException,/* 3 */ MipsUserGenException, /* 4 */ MipsUserGenException, /* 5 */ MipsUserGenException, /* 6 */