commit c473d80fcee414b3f10bb2f817ed877fb88daa2b
Author: Stacey Son <sson@FreeBSD.org>
Date:   Sat Feb 8 16:31:13 2014 -0600

    Initial code change for 16K pages on mips64.
    
    This changes the default page size to be 16K from 4K. Of course,
    alignment issues in userland need to be resolved before this can
    work.

diff --git a/sys/mips/include/param.h b/sys/mips/include/param.h
index 264a847..c2df3a8 100644
--- a/sys/mips/include/param.h
+++ b/sys/mips/include/param.h
@@ -128,7 +128,7 @@
 #define	CACHE_LINE_SHIFT	6
 #define	CACHE_LINE_SIZE		(1 << CACHE_LINE_SHIFT)
 
-#define	PAGE_SHIFT		12		/* LOG2(PAGE_SIZE) */
+#define	PAGE_SHIFT		14		/* LOG2(PAGE_SIZE) */
 #define	PAGE_SIZE		(1<<PAGE_SHIFT) /* bytes/page */
 #define	PAGE_MASK		(PAGE_SIZE-1)
 
diff --git a/sys/mips/include/pmap.h b/sys/mips/include/pmap.h
index 715390b..2e44f5e 100644
--- a/sys/mips/include/pmap.h
+++ b/sys/mips/include/pmap.h
@@ -128,17 +128,30 @@ typedef struct pv_entry {
  * need to track per-pmap assignments.
  */
 #ifdef __mips_n64
+#if PAGE_SIZE == 4096
 #define	_NPCM	3
 #define	_NPCPV	168
+#define	_NPCS	0
+#elif PAGE_SIZE == 16384
+#define	_NPCM	11
+#define	_NPCPV	677
+#define	_NPCS	1
+#else
+#error PAGE_SIZE not 4K or 16K
+#endif /* PAGE_SIZE */
 #else
 #define	_NPCM	11
 #define	_NPCPV	336
+#define	_NPCS	0
 #endif
 struct pv_chunk {
 	pmap_t			pc_pmap;
 	TAILQ_ENTRY(pv_chunk)	pc_list;
 	u_long			pc_map[_NPCM];	/* bitmap; 1 = free */
 	TAILQ_ENTRY(pv_chunk)	pc_lru;
+#if _NPCS != 0
+	u_long			pc_spare[_NPCS];
+#endif
 	struct pv_entry		pc_pventry[_NPCPV];
 };
 
diff --git a/sys/mips/include/pte.h b/sys/mips/include/pte.h
index 9e9c87b..cdff1d0 100644
--- a/sys/mips/include/pte.h
+++ b/sys/mips/include/pte.h
@@ -53,7 +53,9 @@ typedef	pt_entry_t *pd_entry_t;
  * TLB PageMask register.  Has mask bits set above the default, 4K, page mask.
  */
 #define	TLBMASK_SHIFT	(13)
-#define	TLBMASK_MASK	((PAGE_MASK >> TLBMASK_SHIFT) << TLBMASK_SHIFT)
+#define	TLBMASK_MASK	((TLB_PAGE_MASK >> (TLBMASK_SHIFT - 1)) << \
+				TLBMASK_SHIFT)
+#define	TLBMASK_MASK_KSTACK	(0x3 << TLBMASK_SHIFT)
 
 /*
  * FreeBSD/mips page-table entries take a near-identical format to MIPS TLB
@@ -82,10 +84,12 @@ typedef	pt_entry_t *pd_entry_t;
 #define	TLBLO_SWBITS_CLEAR_SHIFT	(3)
 #define	TLBLO_PFN_MASK		(0x1FFFFFC0)
 #endif
-#define	TLBLO_PFN_SHIFT		(6)
+#define	TLBLO_PFN_SHIFT		(PAGE_SHIFT - 6)
 #define	TLBLO_SWBITS_MASK	((pt_entry_t)0x7 << TLBLO_SWBITS_SHIFT)
-#define	TLBLO_PA_TO_PFN(pa)	((((pa) >> TLB_PAGE_SHIFT) << TLBLO_PFN_SHIFT) & TLBLO_PFN_MASK)
-#define	TLBLO_PFN_TO_PA(pfn)	((vm_paddr_t)((pfn) >> TLBLO_PFN_SHIFT) << TLB_PAGE_SHIFT)
+#define	TLBLO_PA_TO_PFN(pa)	((((pa) >> TLB_PAGE_SHIFT) << TLBLO_PFN_SHIFT) \
+					& TLBLO_PFN_MASK)
+#define	TLBLO_PFN_TO_PA(pfn)	((vm_paddr_t)((pfn) >> TLBLO_PFN_SHIFT) << \
+					TLB_PAGE_SHIFT)
 #define	TLBLO_PTE_TO_PFN(pte)	((pte) & TLBLO_PFN_MASK)
 #define	TLBLO_PTE_TO_PA(pte)	(TLBLO_PFN_TO_PA(TLBLO_PTE_TO_PFN((pte))))
 
@@ -108,16 +112,17 @@ typedef	pt_entry_t *pd_entry_t;
 #define	TLBHI_R_KERNEL		(0x03UL << TLBHI_R_SHIFT)
 #define	TLBHI_R_MASK		(0x03UL << TLBHI_R_SHIFT)
 #define	TLBHI_VA_R(va)		((va) & TLBHI_R_MASK)
-#define	TLBHI_FILL_SHIFT	40
 #define	TLBHI_VPN2_SHIFT	(TLB_PAGE_SHIFT + 1)
-#define	TLBHI_VPN2_MASK		(((~((1UL << TLBHI_VPN2_SHIFT) - 1)) << (63 - TLBHI_FILL_SHIFT)) >> (63 - TLBHI_FILL_SHIFT))
+#define	TLBHI_VPN2_MASK		(~((1ULL << TLBHI_VPN2_SHIFT) - 1) & \
+					~TLBHI_R_MASK)
 #define	TLBHI_VA_TO_VPN2(va)	((va) & TLBHI_VPN2_MASK)
 #define	TLBHI_ENTRY(va, asid)	((TLBHI_VA_R((va))) /* Region. */ | \
 				 (TLBHI_VA_TO_VPN2((va))) /* VPN2. */ | \
 				 ((asid) & TLBHI_ASID_MASK))
 #else /* !defined(__mips_n64) */
 #define	TLBHI_PAGE_MASK		(2 * PAGE_SIZE - 1)
-#define	TLBHI_ENTRY(va, asid)	(((va) & ~TLBHI_PAGE_MASK) | ((asid) & TLBHI_ASID_MASK))
+#define	TLBHI_ENTRY(va, asid)	(((va) & ~TLBHI_PAGE_MASK) | ((asid) & \
+					TLBHI_ASID_MASK))
 #endif /* defined(__mips_n64) */
 
 /*
@@ -163,16 +168,16 @@ typedef	pt_entry_t *pd_entry_t;
 #ifdef LOCORE
 #if defined(__mips_n64) || defined(__mips_n32) /*  PHYSADDR_64_BIT */
 #define	PTESHIFT		3
-#define	PTE2MASK		0xff0	/* for the 2-page lo0/lo1 */
-#define	PTEMASK			0xff8
+#define	PTE2MASK		(TLB_PAGE_MASK & ~15)
+#define	PTEMASK			(TLB_PAGE_MASK & ~7)
 #define	PTESIZE			8
 #define	PTE_L			ld
 #define	PTE_MTC0		dmtc0
 #define	CLEAR_PTE_SWBITS(pr)
 #else
 #define	PTESHIFT		2
-#define	PTE2MASK		0xff8	/* for the 2-page lo0/lo1 */
-#define	PTEMASK			0xffc
+#define	PTE2MASK		(TLB_PAGE_MASK & ~7)
+#define	PTEMASK			(TLB_PAGE_MASK & ~3)
 #define	PTESIZE			4
 #define	PTE_L			lw
 #define	PTE_MTC0		mtc0
@@ -181,10 +186,10 @@ typedef	pt_entry_t *pd_entry_t;
 
 #if defined(__mips_n64)
 #define	PTRSHIFT		3
-#define	PDEPTRMASK		0xff8
+#define	PDEPTRMASK		(TLB_PAGE_MASK & ~7)
 #else
 #define	PTRSHIFT		2
-#define	PDEPTRMASK		0xffc
+#define	PDEPTRMASK		(TLB_PAGE_MASK & ~3)
 #endif
 
 #endif /* LOCORE */
diff --git a/sys/mips/mips/exception.S b/sys/mips/mips/exception.S
index 9c05e3c..cf8fa8e 100644
--- a/sys/mips/mips/exception.S
+++ b/sys/mips/mips/exception.S
@@ -159,6 +159,9 @@ MipsDoTLBMiss:
 	CLEAR_PTE_SWBITS(k1)
 	PTE_MTC0	k1, MIPS_COP_0_TLB_LO1		#15: lo1 is loaded
 	COP0_SYNC
+	li		k0, TLBMASK_MASK
+	MTC0		k0, MIPS_COP_0_TLB_PG_MASK
+	COP0_SYNC
 	tlbwr						#1a: write to tlb
 	HAZARD_DELAY
 	eret						#1f: retUrn from exception
@@ -913,6 +916,9 @@ NLEAF(MipsTLBInvalidException)
 	CLEAR_PTE_SWBITS(k1)
 	PTE_MTC0	k1, MIPS_COP_0_TLB_LO1
 	COP0_SYNC
+	li		k0, TLBMASK_MASK
+	MTC0		k0, MIPS_COP_0_TLB_PG_MASK
+	COP0_SYNC
 
 	b		tlb_insert_entry
 	nop
@@ -926,6 +932,9 @@ odd_page:
 	CLEAR_PTE_SWBITS(k1)
 	PTE_MTC0	k1, MIPS_COP_0_TLB_LO1
 	COP0_SYNC
+	li		k0, TLBMASK_MASK
+	MTC0		k0, MIPS_COP_0_TLB_PG_MASK
+	COP0_SYNC
 
 tlb_insert_entry:
 	tlbp
@@ -972,7 +981,8 @@ tlb_insert_random:
 	sll	k1, k1, PAGE_SHIFT + 1
 
 	PTR_LA	k0, _C_LABEL(pcpu_space)
-	PTR_ADDU	k0, PAGE_SIZE * 2
+	PTR_ADDU	k0, PAGE_SIZE
+	PTR_ADDU	k0, PAGE_SIZE
 	PTR_ADDU	k0, k0, k1
 
 	/*
@@ -1068,6 +1078,9 @@ NLEAF(MipsTLBMissException)
 	CLEAR_PTE_SWBITS(k1)
 	PTE_MTC0	k1, MIPS_COP_0_TLB_LO1		# lo1 is loaded
 	COP0_SYNC
+	li		k0, TLBMASK_MASK
+	MTC0		k0, MIPS_COP_0_TLB_PG_MASK
+	COP0_SYNC
 	tlbwr					# write to tlb
 	HAZARD_DELAY
 	eret					# return from exception
diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c
index 35898b4..019b00b 100644
--- a/sys/mips/mips/pmap.c
+++ b/sys/mips/mips/pmap.c
@@ -1302,8 +1302,13 @@ pmap_growkernel(vm_offset_t addr)
 
 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 #ifdef __mips_n64
+#if PAGE_SIZE == 4096
 CTASSERT(_NPCM == 3);
 CTASSERT(_NPCPV == 168);
+#elif PAGE_SIZE == 16384
+CTASSERT(_NPCM == 11);
+CTASSERT(_NPCPV == 677);
+#endif
 #else
 CTASSERT(_NPCM == 11);
 CTASSERT(_NPCPV == 336);
@@ -1319,8 +1324,9 @@ pv_to_chunk(pv_entry_t pv)
 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 
 #ifdef __mips_n64
-#define	PC_FREE0_1	0xfffffffffffffffful
-#define	PC_FREE2	0x000000fffffffffful
+#define PC_FREE_FULL    0xfffffffffffffffful
+#define PC_FREE_PARTIAL	\
+    ((1UL << (_NPCPV - sizeof(u_long) * 8 * (_NPCM - 1))) - 1)
 #else
 #define	PC_FREE0_9	0xfffffffful	/* Free values for index 0 through 9 */
 #define	PC_FREE10	0x0000fffful	/* Free values for index 10 */
@@ -1328,7 +1334,16 @@ pv_to_chunk(pv_entry_t pv)
 
 static const u_long pc_freemask[_NPCM] = {
 #ifdef __mips_n64
-	PC_FREE0_1, PC_FREE0_1, PC_FREE2
+#if PAGE_SIZE == 4096
+	PC_FREE_FULL, PC_FREE_FULL, PC_FREE_PARTIAL
+#elif PAGE_SIZE == 16384
+	PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
+	PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
+	PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
+	PC_FREE_FULL, PC_FREE_PARTIAL
+#else
+#error PAGE_SIZE not either 4K or 16K
+#endif /* PAGE_SIZE */
 #else
 	PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 	PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
diff --git a/sys/mips/mips/swtch.S b/sys/mips/mips/swtch.S
index ae64883..905a4a6 100644
--- a/sys/mips/mips/swtch.S
+++ b/sys/mips/mips/swtch.S
@@ -356,6 +356,9 @@ blocked_loop:
 	PTE_MTC0	zero, MIPS_COP_0_TLB_LO0
 	PTE_MTC0	zero, MIPS_COP_0_TLB_LO1
 	HAZARD_DELAY
+	li	t1, TLBMASK_MASK
+	MTC0	t1, MIPS_COP_0_TLB_PG_MASK
+	HAZARD_DELAY
 	tlbwi
 	HAZARD_DELAY
 	MTC0	v0, MIPS_COP_0_TLB_HI		# set VPN again
@@ -368,6 +371,9 @@ entry0set:
 	HAZARD_DELAY
 	PTE_MTC0	a2, MIPS_COP_0_TLB_LO1		# upte[1]
 	HAZARD_DELAY
+	li		t1, TLBMASK_MASK
+	MTC0		t1, MIPS_COP_0_TLB_PG_MASK
+	HAZARD_DELAY
 	tlbwi					# set TLB entry #0
 	HAZARD_DELAY
 /*
diff --git a/sys/mips/mips/tlb.c b/sys/mips/mips/tlb.c
index 3432b07..55128a5 100644
--- a/sys/mips/mips/tlb.c
+++ b/sys/mips/mips/tlb.c
@@ -109,7 +109,7 @@ tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
 	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
 
 	mips_wr_index(i);
-	mips_wr_pagemask(0);
+	mips_wr_pagemask(TLBMASK_MASK);
 	mips_wr_entryhi(TLBHI_ENTRY(va, 0));
 	mips_wr_entrylo0(pte0);
 	mips_wr_entrylo1(pte1);
@@ -131,7 +131,7 @@ tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
 	s = intr_disable();
 	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
 
-	mips_wr_pagemask(0);
+	mips_wr_pagemask(TLBMASK_MASK);
 	mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
 	tlb_probe();
 	i = mips_rd_index();
@@ -235,7 +235,7 @@ tlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
 		 * within the virtual address range.
 		 */
 		for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) {
-			mips_wr_pagemask(0);
+			mips_wr_pagemask(TLBMASK_MASK);
 			mips_wr_entryhi(hi);
 			tlb_probe();
 			i = mips_rd_index();
@@ -305,7 +305,7 @@ tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
 	s = intr_disable();
 	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
 
-	mips_wr_pagemask(0);
+	mips_wr_pagemask(TLBMASK_MASK);
 	mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
 	tlb_probe();
 	i = mips_rd_index();
@@ -331,7 +331,7 @@ tlb_invalidate_one(unsigned i)
 	mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
 	mips_wr_entrylo0(0);
 	mips_wr_entrylo1(0);
-	mips_wr_pagemask(0);
+	mips_wr_pagemask(TLBMASK_MASK);
 	mips_wr_index(i);
 	tlb_write_indexed();
 }