From 39685cb74413f9377c5714ea68e7fff778e4dd6c Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Fri, 9 Dec 2022 15:16:16 -0500 Subject: [PATCH 30/52] mips: Support a largepage size of 512MB This requires an extra bit in the page size index. To steal a bit from the left side of the existing index, we need to be able to disambiguate between non-leaf PDEs and superpage mappings. In particular, the former are direct map addresses, which will typically have bit 60 set. Thus, define a mask of reserved PTE bits and test whether they are non-zero before following a PDE. --- sys/mips/include/pte.h | 39 +++++++++++++++++++++++++------------ sys/mips/mips/exception.S | 36 ++++++++++++++++++++++++++++++++-- sys/mips/mips/pmap_mips64.c | 6 +----- sys/mips/mips/tlb.c | 2 -- sys/mips/mips/tlbdemo.c | 2 -- 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/sys/mips/include/pte.h b/sys/mips/include/pte.h index d7085645398c..a0ba3aa9c88b 100644 --- a/sys/mips/include/pte.h +++ b/sys/mips/include/pte.h @@ -186,10 +186,10 @@ typedef pt_entry_t *pd_entry_t; * * Upper bits of a 64 bit PTE: * - * 63-62 61-60 59 58 -- 56 55 54 53 - * --------------------------------------------- - * | RG | | SV | PG SZ IDX | MN | W | RO | - * --------------------------------------------- + * 63-62 61 60 59 -- 56 55 54 53 + * ------------------------------------------- + * | RG | | SV | PG SZ IDX | MN | W | RO | + * ------------------------------------------- * * VM flags managed in software: * RG: Region. (Reserved. Currently not used.) @@ -206,7 +206,7 @@ typedef pt_entry_t *pd_entry_t; * 6 0000 1111 1111 1111 16M * (MIPS 3:) * 7 0011 1111 1111 1111 64M - * 8 1111 1111 1111 1111 256M (Not currently supported) + * 8 1111 1111 1111 1111 256M * * RO: Read only. Never set PTE_D on this page, and don't * listen to requests to write to it. @@ -215,6 +215,10 @@ typedef pt_entry_t *pd_entry_t; * * These bits should not be written into the TLB, so must first be masked out * explicitly in C, or using CLEAR_PTE_SWBITS() in assembly. + * + * To distinguish between non-leaf PDEs and superpages, check whether the + * top three bits of an entry are non-zero. If they are, then the PDE is + * a direct map address and is thus not a leaf. See pde_is_superpage(). */ #define PTE_RO ((pt_entry_t)0x01 << TLBLO_SWBITS_SHIFT) #define PTE_W ((pt_entry_t)0x02 << TLBLO_SWBITS_SHIFT) @@ -227,14 +231,17 @@ typedef pt_entry_t *pd_entry_t; #define PTE_PS_4M ((pt_entry_t)0x28 << TLBLO_SWBITS_SHIFT) #define PTE_PS_16M ((pt_entry_t)0x30 << TLBLO_SWBITS_SHIFT) #define PTE_PS_64M ((pt_entry_t)0x38 << TLBLO_SWBITS_SHIFT) -#define PTE_PS_IDX_MASK ((pt_entry_t)0x38 << TLBLO_SWBITS_SHIFT) -#define PTE_PSIDX_NBITS_TO_LEFT 5 +#define PTE_PS_256M ((pt_entry_t)0x40 << TLBLO_SWBITS_SHIFT) +#define PTE_PS_IDX_MASK ((pt_entry_t)0x78 << TLBLO_SWBITS_SHIFT) +#define PTE_PSIDX_NBITS_TO_LEFT 4 #define PTE_PSIDX_NBITS_TO_RIGHT 56 #define PTE_PFN_NBITS_TO_LEFT 11 #define PTE_PFN_NBITS_TO_RIGHT 6 #define PTE_HWFLAGS_NBITS_TO_LEFT 58 -#define SW_VALID 0x40 +#define SW_VALID 0x80 +#define SW_RSV 0x700 #define PTE_SV ((pt_entry_t)SW_VALID << TLBLO_SWBITS_SHIFT) +#define PTE_RSV ((pt_entry_t)SW_RSV << TLBLO_SWBITS_SHIFT) #else #define PTE_PS_IDX_MASK 0 #define PTE_SV 0 @@ -260,6 +267,7 @@ typedef pt_entry_t *pd_entry_t; #define TLBMASK_IDX_4M 5 #define TLBMASK_IDX_16M 6 #define TLBMASK_IDX_64M 7 +#define TLBMASK_IDX_256M 8 #define TLBMASK_4K_PAGE TLBMASK_IDX_TO_MASK(TLBMASK_IDX_4K) #define TLBMASK_16K_PAGE TLBMASK_IDX_TO_MASK(TLBMASK_IDX_16K) #define TLBMASK_64K_PAGE TLBMASK_IDX_TO_MASK(TLBMASK_IDX_64K) @@ -268,6 +276,7 @@ typedef pt_entry_t *pd_entry_t; #define TLBMASK_4M_PAGE TLBMASK_IDX_TO_MASK(TLBMASK_IDX_4M) #define TLBMASK_16M_PAGE TLBMASK_IDX_TO_MASK(TLBMASK_IDX_16M) #define TLBMASK_64M_PAGE TLBMASK_IDX_TO_MASK(TLBMASK_IDX_64M) +#define TLBMASK_256M_PAGE TLBMASK_IDX_TO_MASK(TLBMASK_IDX_256M) #else /* ! MIPS64_NEW_PMAP */ #define TLBLO_PTE_TO_IDX(pte) 0 #define TLBLO_PTE_TO_MASK(pte) 0 @@ -419,7 +428,8 @@ static __inline int pde_is_superpage(pd_entry_t *pde) { - return (((pt_entry_t)*pde & PTE_PS_IDX_MASK) != 0); + return (((pt_entry_t)*pde & PTE_PS_IDX_MASK) != 0 && + ((pt_entry_t)*pde & PTE_RSV) == 0); } /* Is the PTE a superpage of any size? */ @@ -434,7 +444,7 @@ pte_is_superpage(pt_entry_t *pte) static __inline int pde_is_1m_superpage(pd_entry_t *pde) { - return (((pt_entry_t)*pde & PTE_PS_IDX_MASK) == PTE_PS_1M); + return (((pt_entry_t)*pde & (PTE_PS_IDX_MASK | PTE_RSV)) == PTE_PS_1M); } /* Is the PTE an 1MB superpage? */ @@ -544,7 +554,7 @@ TLBLO_PTE_TO_PA(pt_entry_t pte) /* * GET_HW_TLB_FLAGS(r) * - * Get the lower hardware TLB flags but shifting left then right. + * Get the lower hardware TLB flags by shifting left then right. */ #define GET_HW_TLB_FLAGS(r) \ dsll32 r, (PTE_HWFLAGS_NBITS_TO_LEFT - 32); \ @@ -555,12 +565,17 @@ TLBLO_PTE_TO_PA(pt_entry_t pte) * * Get the odd superpage PFN (TLB lo1) from the even PTE. First, mask out the * PFN from the even PTE. Then add "size" worth of pages to it (256 = 1MB). + * To ensure that the immediate fits 16 bits, as would otherwise not be the + * case with a 256MB mapping (2^16 pages), we perform four additions. * Finally, shift it back to its position in the PTE. */ #define GET_ODD_PFN_FROM_EVEN(size, r) \ dsll r, (PTE_PFN_NBITS_TO_LEFT); \ dsrl r, (PTE_PFN_NBITS_TO_LEFT + PTE_PFN_NBITS_TO_RIGHT); \ - daddiu r, r, ((size) / PAGE_SIZE); \ + daddiu r, r, ((size) / PAGE_SIZE / 4); \ + daddiu r, r, ((size) / PAGE_SIZE / 4); \ + daddiu r, r, ((size) / PAGE_SIZE / 4); \ + daddiu r, r, ((size) / PAGE_SIZE / 4); \ dsll r, (PTE_PFN_NBITS_TO_RIGHT) /* diff --git a/sys/mips/mips/exception.S b/sys/mips/mips/exception.S index 21df77f448dd..a7dfe6b0c0c9 100644 --- a/sys/mips/mips/exception.S +++ b/sys/mips/mips/exception.S @@ -176,7 +176,7 @@ MipsDoTLBMiss: PTR_ADDU k0, k0, k1 # k1=pde entry address PTR_L k1, 0(k0) # k1=pde entry beq k1, zero, 2f # ==0 -- no page table - nop + nop # branch delay slot #ifdef MIPS64_NEW_PMAP @@ -201,6 +201,13 @@ MipsDoTLBMiss: COP0_SYNC ;\ eret + # Check for a superpage. First make sure we're dealing with a + # leaf PTE: if any of the top three bits is set then we are + # dealing with a PDE and need to keep descending. + dsrl32 k1, TLBLO_SWBITS_SHIFT - 32 + andi k1, k1, SW_RSV + bne k1, zero, not_superpage + PTR_L k1, 0(k0) # k1=pde entry GET_SUPERPAGE_IDX(k1) # k1=superpage index from PTE beq k1, zero, not_superpage # ==0 -- not superpage nop # branch delay slot @@ -212,6 +219,8 @@ MipsDoTLBMiss: beq k1, zero, miss_super_32m sub k1, k1, 1 beq k1, zero, miss_super_128m + sub k1, k1, 1 + beq k1, zero, miss_super_512m nop # branch delay slot PANIC("unknown page size index") @@ -223,6 +232,8 @@ miss_super_32m: LOAD_SUPERPAGE_TLB(16 * 1024 * 1024, TLBMASK_16M_PAGE) miss_super_128m: LOAD_SUPERPAGE_TLB(64 * 1024 * 1024, TLBMASK_64M_PAGE) +miss_super_512m: + LOAD_SUPERPAGE_TLB(256 * 1024 * 1024, TLBMASK_256M_PAGE) #undef LOAD_SUPERPAGE_TLB not_superpage: @@ -1025,6 +1036,13 @@ LEAF_NOPROFILE(MipsTLBInvalidException) b tlb_insert_entry ;\ nop + # Check for a superpage. First make sure we're dealing with a + # leaf PTE: if any of the top three bits is set then we are + # dealing with a PDE and need to keep descending. + dsrl32 k1, TLBLO_SWBITS_SHIFT - 32 + andi k1, k1, SW_RSV + bne k1, zero, not_spg + PTR_L k1, 0(k0) # k1=pde entry GET_SUPERPAGE_IDX(k1) # k1=superpage index from PTE beq k1, zero, not_spg # ==0 -- not superpage nop # branch delay slot @@ -1036,6 +1054,8 @@ LEAF_NOPROFILE(MipsTLBInvalidException) beq k1, zero, invalid_super_32m sub k1, k1, 1 beq k1, zero, invalid_super_128m + sub k1, k1, 1 + beq k1, zero, invalid_super_512m nop # branch delay slot PANIC("unknown page size index") @@ -1047,6 +1067,8 @@ invalid_super_32m: LOAD_SUPERPAGE_TLB(16 * 1024 * 1024, TLBMASK_16M_PAGE) invalid_super_128m: LOAD_SUPERPAGE_TLB(64 * 1024 * 1024, TLBMASK_64M_PAGE) +invalid_super_512m: + LOAD_SUPERPAGE_TLB(256 * 1024 * 1024, TLBMASK_256M_PAGE) #undef LOAD_SUPERPAGE_TLB not_spg: @@ -1254,7 +1276,13 @@ LEAF_NOPROFILE(MipsTLBMissException) COP0_SYNC ;\ eret - # Check for superpage + # Check for a superpage. First make sure we're dealing with a + # leaf PTE: if any of the top three bits is set then we are + # dealing with a PDE and need to keep descending. + dsrl32 k1, TLBLO_SWBITS_SHIFT - 32 + andi k1, k1, SW_RSV + bne k1, zero, not_spg + PTR_L k1, 0(k0) # k1=pde entry GET_SUPERPAGE_IDX(k1) # k1=superpage index from PTE beq k1, zero, not_kspg # ==0 -- not superpage nop # branch delay slot @@ -1266,6 +1294,8 @@ LEAF_NOPROFILE(MipsTLBMissException) beq k1, zero, miss_ksuper_32m sub k1, k1, 1 beq k1, zero, miss_ksuper_128m + sub k1, k1, 1 + beq k1, zero, miss_ksuper_512m nop # branch delay slot PANIC("unknown page size index") @@ -1277,6 +1307,8 @@ miss_ksuper_32m: LOAD_SUPERPAGE_TLB(16 * 1024 * 1024, TLBMASK_16M_PAGE) miss_ksuper_128m: LOAD_SUPERPAGE_TLB(64 * 1024 * 1024, TLBMASK_64M_PAGE) +miss_ksuper_512m: + LOAD_SUPERPAGE_TLB(256 * 1024 * 1024, TLBMASK_256M_PAGE) #undef LOAD_SUPERPAGE_TLB not_kspg: diff --git a/sys/mips/mips/pmap_mips64.c b/sys/mips/mips/pmap_mips64.c index 5cedfce7e006..e3992d1fc070 100644 --- a/sys/mips/mips/pmap_mips64.c +++ b/sys/mips/mips/pmap_mips64.c @@ -291,10 +291,8 @@ pmap_psind2pspte(int8_t psind) return (PTE_PS_16M); case 4: return (PTE_PS_64M); -#ifdef notyet case 5: return (PTE_PS_256M); -#endif } panic("%s: unhandled psind %d", psind); } @@ -311,10 +309,8 @@ pmap_pspte2psind(pt_entry_t pde) return (3); case PTE_PS_64M: return (4); -#ifdef notyet case PTE_PS_256M: - return (1); -#endif + return (5); } panic("%s: unrecognized PTE %#lx", pde); } diff --git a/sys/mips/mips/tlb.c b/sys/mips/mips/tlb.c index 2e39f06c7f31..7bad48a162c6 100644 --- a/sys/mips/mips/tlb.c +++ b/sys/mips/mips/tlb.c @@ -420,9 +420,7 @@ tlb_sysctl(SYSCTL_HANDLER_ARGS) VAL2STR(pagesize, TLBMASK_4M_PAGE, "4M"); VAL2STR(pagesize, TLBMASK_16M_PAGE, "16M"); VAL2STR(pagesize, TLBMASK_64M_PAGE, "64M"); -#if 0 VAL2STR(pagesize, TLBMASK_256M_PAGE, "256M"); -#endif default: panic("unrecognized pagemask %#jx", tlbe->pagemask); } diff --git a/sys/mips/mips/tlbdemo.c b/sys/mips/mips/tlbdemo.c index 4af4ccea0ed6..7cb712e9651d 100644 --- a/sys/mips/mips/tlbdemo.c +++ b/sys/mips/mips/tlbdemo.c @@ -92,9 +92,7 @@ static struct tlbdemo_store tlbdemo_reservations[MAXPAGESIZES] = { [2] = { .count = 1 }, /* 8MB */ [3] = { .count = 1 }, /* 32MB */ [4] = { .count = 1 }, /* 128MB */ -#ifdef notyet [5] = { .count = 1 }, /* 512MB */ -#endif }; /* -- 2.41.0