From a3de8dc606eadb91776a4415f9723013433b00f3 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Mon, 28 Nov 2022 12:50:12 -0500 Subject: [PATCH 26/52] mips: Add initial support for creating largepage mappings Page size index 5 (512MB) is not yet handled since that requires some changes to the leaf PTE layout. Currently we have three bits for the page size index, which lets us encode 8 page sizes, up to 64MB. To handle 256MB pages, we need to widen the index, and a naive attempt at doing so causes panics during boot. I'm not sure exactly why, but it seems to cause something to treat non-leaf PTEs as leaf PTEs. --- sys/mips/mips/pmap_mips64.c | 183 ++++++++++++++++++++++++++++++++---- 1 file changed, 166 insertions(+), 17 deletions(-) diff --git a/sys/mips/mips/pmap_mips64.c b/sys/mips/mips/pmap_mips64.c index da5f92f47be1..f7e8229570bf 100644 --- a/sys/mips/mips/pmap_mips64.c +++ b/sys/mips/mips/pmap_mips64.c @@ -276,6 +276,49 @@ pmap_ps_enabled(pmap_t pmap) return (pg_ps_enabled && pmap != kernel_pmap); } +static pt_entry_t +pmap_psind2pspte(int8_t psind) +{ + KASSERT(psind > 0 && psind < MAXPAGESIZES, + ("%s: invalid psind %d", __func__, psind)); + + switch (psind) { + case 1: + return (PTE_PS_1M); + case 2: + return (PTE_PS_4M); + case 3: + return (PTE_PS_16M); + case 4: + return (PTE_PS_64M); +#ifdef notyet + case 5: + return (PTE_PS_256M); +#endif + } + panic("%s: unhandled psind %d", psind); +} + +static int +pmap_pspte2psind(pt_entry_t pde) +{ + switch (pde & PTE_PS_IDX_MASK) { + case PTE_PS_1M: + return (1); + case PTE_PS_4M: + return (2); + case PTE_PS_16M: + return (3); + case PTE_PS_64M: + return (4); +#ifdef notyet + case PTE_PS_256M: + return (1); +#endif + } + panic("%s: unrecognized PTE %#lx", pde); +} + /* * Page table entry lookup routines. */ @@ -295,6 +338,8 @@ pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) pd_entry_t *pde; pde = (pd_entry_t *)*pdpe; + KASSERT(!pde_is_superpage(pde), + ("%s: pde %#lx is a leaf", __func__, *pde)); return (&pde[pmap_pde_index(va)]); } @@ -318,6 +363,8 @@ pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) pt_entry_t *pte; pte = (pt_entry_t *)*pde; + KASSERT(!pde_is_superpage(pde), + ("%s: pde %#lx is a leaf", __func__, *pde)); return (&pte[pmap_pte_index(va)]); } @@ -859,7 +906,7 @@ pmap_extract(pmap_t pmap, vm_offset_t va) pa = 0; PMAP_LOCK(pmap); pde = pmap_pde(pmap, va); - if (pde_is_1m_superpage(pde)) { + if (pde_is_superpage(pde)) { pa = TLBLO_PDE_TO_PA(*pde) | (va & PDRMASK); } else { pte = pmap_pde_to_pte(pde, va); @@ -892,7 +939,7 @@ pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) retry: pdep = pmap_pde(pmap, va); if (pdep != NULL && *pdep != NULL) { - if (pde_is_1m_superpage(pdep)) { + if (pde_is_superpage(pdep)) { if (!pde_test(pdep, PTE_RO) || (prot & VM_PROT_WRITE) == 0) { pde_pa = TLBLO_PDE_TO_PA(*pdep) | @@ -957,7 +1004,7 @@ pmap_kextract(vm_offset_t va) pdep = pmap_pde(&curproc->p_vmspace->vm_pmap, va); if (pdep == NULL || *pdep == NULL) return (0); - if (pde_is_1m_superpage(pdep)) { + if (pde_is_superpage(pdep)) { ptep = (pt_entry_t *)pdep; return (TLBLO_PTE_TO_PA(*ptep) | (va & PDRMASK)); @@ -990,7 +1037,7 @@ pmap_kextract(vm_offset_t va) pdep = pmap_pde(kernel_pmap, va); if (pdep == NULL || *pdep == NULL) return (0); - if (pde_is_1m_superpage(pdep)) { + if (pde_is_superpage(pdep)) { ptep = (pt_entry_t *)pdep; return (TLBLO_PTE_TO_PA(*ptep) | (va & PDRMASK)); @@ -2531,10 +2578,11 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) */ if (ptpaddr == NULL) continue; + /* * Check for superpage. */ - if (pde_is_1m_superpage(&ptpaddr)) { + if (pde_is_superpage(&ptpaddr)) { /* * Are we removing the entire superpage? If not, * demote the mapping and fall through. @@ -2774,7 +2822,7 @@ resume: /* * Check for superpage. */ - if (pde_is_1m_superpage(pde)) { + if (pde_is_superpage(pde)) { /* * Are we protecting the entire superpage? If not, * demote the mapping and fall through. @@ -2807,6 +2855,7 @@ resume: } } } + /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the @@ -2982,6 +3031,95 @@ setpte: pmap); } +static int +pmap_enter_largepage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, + u_int flags, int8_t psind, struct rwlock **lockp) +{ + pd_entry_t *pde, *pdep; + pt_entry_t newpde; + uint64_t pspte; + vm_page_t mpde; + vm_paddr_t pa; + vm_size_t size; + + size = pagesizes[psind]; + pspte = pmap_psind2pspte(psind); + + KASSERT(psind > 0 && psind < MAXPAGESIZES, + ("%s: invalid psind %d", __func__, psind)); + KASSERT((m->oflags & VPO_UNMANAGED) != 0, + ("%s: managed page %p", __func__, m)); + KASSERT((va & (size - 1)) == 0, + ("%s: unaligned va %#lx", __func__, va)); + KASSERT((VM_PAGE_TO_PHYS(m) & (size - 1)) == 0, + ("%s: unaligned pa %#lx", __func__, VM_PAGE_TO_PHYS(m))); + rw_assert(&pvh_global_lock, RA_LOCKED); + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + + if (is_kernel_pmap(pmap)) { + CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p: kernel map", + __func__, va, pmap); + return (KERN_FAILURE); + } + + if ((mpde = pmap_allocpde(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? + NULL : lockp)) == NULL) { + CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p", + __func__, va, pmap); + return (KERN_RESOURCE_SHORTAGE); + } + pde = (pd_entry_t *)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(mpde)); + for (vm_size_t i = 0; i < size / NBPDR; i++) { + pa = VM_PAGE_TO_PHYS(m) + i * NBPDR; + newpde = TLBLO_PA_TO_PFN(pa) | pspte | + init_pte_prot(m, flags, prot); + if (is_cacheable_mem(pa)) { + if (m->md.pv_memattr == VM_MEMATTR_UNCACHEABLE) + newpde |= PTE_C_UNCACHED; + else + newpde |= PTE_C_CACHE; + } else + newpde |= PTE_C_UNCACHED; + + pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); + + if ((flags & PMAP_ENTER_WIRED) != 0) { + newpde |= PTE_W; + pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE; + } + + if (i != 0) + mpde->wire_count++; + + pdep = &pde[pmap_pde_index(va + i * NBPDR)]; + KASSERT(*pdep == 0, ("%s: PDE for %#lx is non-zero: %#lx", + __func__, va + i * NBPDR, *pdep)); + sched_pin(); + pde_store(pdep, newpde); + + /* + * Sync I & D caches for executable pages. Do this only if the + * target pmap belongs to the current process. Otherwise, an + * unresolvable TLB miss may occur. + * + * XXX-MJ or simply disallow executable largepage mappings. + */ + if (pmap == &curproc->p_vmspace->vm_pmap && + (prot & VM_PROT_EXECUTE)) { + mips_icache_sync_range(va, NBPDR); + mips_dcache_wbinv_range(va, NBPDR); + } + + sched_unpin(); + + atomic_add_long(&pmap_pde_mappings, 1); + CTR3(KTR_PMAP, "%s: success for va %#lx in pmap %p", __func__, va, + pmap); + } + + return (KERN_SUCCESS); +} + /* * Insert the given physical page (p) at * the specified virtual address (v) in the @@ -3021,6 +3159,13 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); + if ((flags & PMAP_ENTER_LARGEPAGE) != 0) { + KASSERT((m->oflags & VPO_UNMANAGED) != 0, + ("managed largepage va %#lx flags %#x", va, flags)); + rv = pmap_enter_largepage(pmap, va, m, prot, flags, psind, + &lock); + goto out; + } if (psind == 1) { /* XXX-MJ needs more review */ KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned")); @@ -3051,7 +3196,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, } } pde = pmap_pde(pmap, va); - if (pde_is_1m_superpage(pde)) { + if (pde_is_superpage(pde)) { panic("%s: attempted pmap_enter on superpage", __func__); } pte = pmap_pde_to_pte(pde, va); @@ -3321,7 +3466,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, * increment the hold count, and activate it. */ if (pde && *pde != 0) { - if (pde_is_1m_superpage(pde)) + if (pde_is_superpage(pde)) return (NULL); mpte = PHYS_TO_VM_PAGE( MIPS_DIRECT_TO_PHYS(*pde)); @@ -3751,7 +3896,7 @@ resume: pde = pmap_pdpe_to_pde(pdpe, sva); if (*pde == NULL) continue; - if (pde_is_1m_superpage(pde)) { + if (pde_is_superpage(pde)) { if (!pde_test(pde, PTE_W)) panic("pmap_unwire: pde %#jx is missing PTE_W", (uintmax_t)*pde); @@ -3777,6 +3922,7 @@ resume: panic("pmap_unwire: demotion failed"); } } + if (va_next > eva) va_next = eva; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, @@ -4921,7 +5067,7 @@ pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) PMAP_LOCK(pmap); pde = pmap_pde(pmap, addr); if (pde != NULL && *pde != 0) { - if (pde_is_1m_superpage(pde)) + if (pde_is_superpage(pde)) pte = (pt_entry_t *)pde; else pte = pmap_pde_to_pte(pde, addr); @@ -4984,7 +5130,7 @@ resume: oldpde = *pde; if (pde == NULL || *pde == 0) continue; - else if (pde_is_1m_superpage(pde)) { + else if (pde_is_superpage(pde)) { if (!pde_test(&oldpde, PTE_MANAGED)) continue; if (!pv_lists_locked) { @@ -5026,6 +5172,7 @@ resume: if (lock != NULL) rw_wunlock(lock); } + /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the @@ -5236,10 +5383,10 @@ pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) PMAP_LOCK(pmap); pdep = pmap_pde(pmap, addr); if (pdep != NULL) { - if (pde_is_1m_superpage(pdep)) { + if (pde_is_superpage(pdep)) { pte = (pt_entry_t)*pdep; pa = TLBLO_PTE_TO_PA(pte); - val = MINCORE_SUPER; + val = MINCORE_PSIND(pmap_pspte2psind(pte)); } else { ptep = pmap_pde_to_pte(pdep, addr); pte = (ptep != NULL) ? *ptep : 0; @@ -5499,10 +5646,11 @@ pmap_emulate_modified(pmap_t pmap, vm_offset_t va) PMAP_LOCK(pmap); pde = pmap_pde(pmap, va); - if (pde_is_1m_superpage(pde)) + if (pde_is_superpage(pde)) pte = (pt_entry_t *)pde; - else + else { pte = pmap_pde_to_pte(pde, va); + } if (pte == NULL) panic("pmap_emulate_modified: can't find PTE"); #ifdef SMP @@ -5551,10 +5699,11 @@ pmap_emulate_referenced(pmap_t pmap, vm_offset_t va) /* Invalid page directory. */ goto dofault; } - if (pde_is_1m_superpage(pde)) { + if (pde_is_superpage(pde)) { pte = (pt_entry_t *)pde; - } else + } else { pte = pmap_pde_to_pte(pde, va); + } if (pte == NULL) { /* Invalid page table. */ goto dofault; -- 2.41.0