Index: sys/vm/vm_radix.c =================================================================== --- sys/vm/vm_radix.c (revision 232652) +++ sys/vm/vm_radix.c (working copy) @@ -54,7 +54,22 @@ #include +#ifndef UMA_MD_SMALL_ALLOC +#define VM_RADIX_RNODE_MAP_SCALE (1024 * 1024 / 2) +#define VM_RADIX_WIDTH 4 + +/* + * Bits of height in root. + * The mask of smaller power of 2 containing VM_RADIX_LIMIT. + */ +#define VM_RADIX_HEIGHT 0x1f +#else #define VM_RADIX_WIDTH 5 + +/* See the comment above. */ +#define VM_RADIX_HEIGHT 0xf +#endif + #define VM_RADIX_COUNT (1 << VM_RADIX_WIDTH) #define VM_RADIX_MASK (VM_RADIX_COUNT - 1) #define VM_RADIX_MAXVAL ((vm_pindex_t)-1) @@ -63,9 +78,6 @@ /* Flag bits stored in node pointers. */ #define VM_RADIX_FLAGS 0x3 -/* Bits of height in root. */ -#define VM_RADIX_HEIGHT 0xf - /* Calculates maximum value for a tree of height h. */ #define VM_RADIX_MAX(h) \ ((h) == VM_RADIX_LIMIT ? VM_RADIX_MAXVAL : \ @@ -84,6 +96,8 @@ static uma_zone_t vm_radix_node_zone; #ifndef UMA_MD_SMALL_ALLOC +static vm_map_t rnode_map; + static void * vm_radix_node_zone_allocf(uma_zone_t zone, int size, uint8_t *flags, int wait) { @@ -91,7 +105,7 @@ vm_page_t m; int pflags; - /* Inform UMA that this allocator uses kernel_map. */ + /* Inform UMA that this allocator uses rnode_map. */ *flags = UMA_SLAB_KERNEL; pflags = VM_ALLOC_WIRED | VM_ALLOC_NOOBJ; @@ -104,7 +118,7 @@ VM_ALLOC_SYSTEM; if ((wait & M_ZERO) != 0) pflags |= VM_ALLOC_ZERO; - addr = kmem_alloc_nofault(kernel_map, size); + addr = kmem_alloc_nofault(rnode_map, size); if (addr == 0) return (NULL); @@ -112,7 +126,7 @@ m = vm_page_alloc(NULL, OFF_TO_IDX(addr - VM_MIN_KERNEL_ADDRESS), pflags); if (m == NULL) { - kmem_free(kernel_map, addr, size); + kmem_free(rnode_map, addr, size); return (NULL); } if ((wait & M_ZERO) != 0 && (m->flags & PG_ZERO) == 0) @@ -134,7 +148,7 @@ m = PHYS_TO_VM_PAGE(pmap_kextract(voitem)); pmap_qremove(voitem, 1); vm_page_free(m); - kmem_free(kernel_map, voitem, size); + kmem_free(rnode_map, voitem, size); } static void @@ -196,7 +210,16 @@ void vm_radix_init(void) { +#ifndef UMA_MD_SMALL_ALLOC + vm_offset_t maxaddr, minaddr; + int rnode_map_scale; + rnode_map_scale = VM_RADIX_RNODE_MAP_SCALE; + TUNABLE_INT_FETCH("machdep.rnode_map_scale", &rnode_map_scale); + rnode_map = kmem_suballoc(kernel_map, &minaddr, &maxaddr, + rnode_map_scale * sizeof(struct vm_radix_node), FALSE); +#endif + vm_radix_node_zone = uma_zcreate("RADIX NODE", sizeof(struct vm_radix_node), NULL, #ifdef INVARIANTS Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c (revision 232652) +++ sys/vm/vm_page.c (working copy) @@ -766,6 +766,63 @@ } /* + * vm_page_splay: + * + * Implements Sleator and Tarjan's top-down splay algorithm. Returns + * the vm_page containing the given pindex. If, however, that + * pindex is not found in the vm_object, returns a vm_page that is + * adjacent to the pindex, coming before or after it. + */ +vm_page_t +vm_page_splay(vm_pindex_t pindex, vm_page_t root) +{ + struct vm_page dummy; + vm_page_t lefttreemax, righttreemin, y; + + if (root == NULL) + return (root); + lefttreemax = righttreemin = &dummy; + for (;; root = y) { + if (pindex < root->pindex) { + if ((y = root->left) == NULL) + break; + if (pindex < y->pindex) { + /* Rotate right. */ + root->left = y->right; + y->right = root; + root = y; + if ((y = root->left) == NULL) + break; + } + /* Link into the new root's right tree. */ + righttreemin->left = root; + righttreemin = root; + } else if (pindex > root->pindex) { + if ((y = root->right) == NULL) + break; + if (pindex > y->pindex) { + /* Rotate left. */ + root->right = y->left; + y->left = root; + root = y; + if ((y = root->right) == NULL) + break; + } + /* Link into the new root's left tree. */ + lefttreemax->right = root; + lefttreemax = root; + } else + break; + } + /* Assemble the new root. */ + lefttreemax->right = root->left; + righttreemin->left = root->right; + root->left = dummy.right; + root->right = dummy.left; + return (root); +} + +/* * vm_page_insert: [ internal use only ] * * Inserts the given mem entry into the object and object list. Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h (revision 232652) +++ sys/vm/vm_page.h (working copy) @@ -130,6 +130,8 @@ struct vm_page { TAILQ_ENTRY(vm_page) pageq; /* queue info for FIFO queue or free list (Q) */ TAILQ_ENTRY(vm_page) listq; /* pages in same object (O) */ + struct vm_page *left; /* splay tree link (O) */ + struct vm_page *right; /* splay tree link (O) */ vm_object_t object; /* which object am I in (O,P)*/ vm_pindex_t pindex; /* offset into object (O,P) */ @@ -398,6 +400,7 @@ void vm_page_requeue(vm_page_t m); void vm_page_set_valid_range(vm_page_t m, int base, int size); void vm_page_sleep(vm_page_t m, const char *msg); +vm_page_t vm_page_splay(vm_pindex_t, vm_page_t); vm_offset_t vm_page_startup(vm_offset_t vaddr); void vm_page_unhold_pages(vm_page_t *ma, int count); void vm_page_unwire (vm_page_t, int); Index: sys/i386/include/pmap.h =================================================================== --- sys/i386/include/pmap.h (revision 232652) +++ sys/i386/include/pmap.h (working copy) @@ -426,20 +426,10 @@ struct pv_chunk; struct md_page { - union { - TAILQ_HEAD(,pv_entry) pvi_list; - struct { - vm_page_t pii_left; - vm_page_t pii_right; - } pvi_siters; - } pv_structs; - int pat_mode; + TAILQ_HEAD(,pv_entry) pv_list; + int pat_mode; }; -#define pv_list pv_structs.pvi_list -#define pv_left pv_structs.pvi_siters.pii_left -#define pv_right pv_structs.pvi_siters.pii_right - struct pmap { struct mtx pm_mtx; pd_entry_t *pm_pdir; /* KVA of page directory */ @@ -478,7 +468,7 @@ */ typedef struct pv_entry { vm_offset_t pv_va; /* virtual address for mapping */ - TAILQ_ENTRY(pv_entry) pv_next; + TAILQ_ENTRY(pv_entry) pv_list; } *pv_entry_t; /* Index: sys/i386/i386/pmap.c =================================================================== --- sys/i386/i386/pmap.c (revision 232652) +++ sys/i386/i386/pmap.c (working copy) @@ -327,7 +327,6 @@ static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde); static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde); -static vm_page_t pmap_vmpage_splay(vm_pindex_t pindex, vm_page_t root); static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); @@ -1555,8 +1554,7 @@ while (free != NULL) { m = free; - free = (void *)m->object; - m->object = NULL; + free = m->right; /* Preserve the page's PG_ZERO setting. */ vm_page_free_toq(m); } @@ -1575,7 +1573,7 @@ m->flags |= PG_ZERO; else m->flags &= ~PG_ZERO; - m->object = (void *)*free; + m->right = *free; *free = m; } @@ -1593,20 +1591,20 @@ PMAP_LOCK_ASSERT(pmap, MA_OWNED); root = pmap->pm_root; if (root == NULL) { - mpte->md.pv_left = NULL; - mpte->md.pv_right = NULL; + mpte->left = NULL; + mpte->right = NULL; } else { - root = pmap_vmpage_splay(mpte->pindex, root); + root = vm_page_splay(mpte->pindex, root); if (mpte->pindex < root->pindex) { - mpte->md.pv_left = root->md.pv_left; - mpte->md.pv_right = root; - root->md.pv_left = NULL; + mpte->left = root->left; + mpte->right = root; + root->left = NULL; } else if (mpte->pindex == root->pindex) panic("pmap_insert_pt_page: pindex already inserted"); else { - mpte->md.pv_right = root->md.pv_right; - mpte->md.pv_left = root; - root->md.pv_right = NULL; + mpte->right = root->right; + mpte->left = root; + root->right = NULL; } } pmap->pm_root = mpte; @@ -1625,7 +1623,7 @@ PMAP_LOCK_ASSERT(pmap, MA_OWNED); if ((mpte = pmap->pm_root) != NULL && mpte->pindex != pindex) { - mpte = pmap_vmpage_splay(pindex, mpte); + mpte = vm_page_splay(pindex, mpte); if ((pmap->pm_root = mpte)->pindex != pindex) mpte = NULL; } @@ -1644,20 +1642,14 @@ PMAP_LOCK_ASSERT(pmap, MA_OWNED); if (mpte != pmap->pm_root) - pmap_vmpage_splay(mpte->pindex, pmap->pm_root); - if (mpte->md.pv_left == NULL) - root = mpte->md.pv_right; + vm_page_splay(mpte->pindex, pmap->pm_root); + if (mpte->left == NULL) + root = mpte->right; else { - root = pmap_vmpage_splay(mpte->pindex, mpte->md.pv_left); - root->md.pv_right = mpte->md.pv_right; + root = vm_page_splay(mpte->pindex, mpte->left); + root->right = mpte->right; } pmap->pm_root = root; - - /* - * Reinitialize the pv_list which could be dirty now because of the - * splay tree work. - */ - TAILQ_INIT(&mpte->md.pv_list); } /* @@ -1710,61 +1702,6 @@ } /* - * Implements Sleator and Tarjan's top-down splay algorithm. Returns - * the vm_page containing the given pindex. If, however, that - * pindex is not found in the pmap, returns a vm_page that is - * adjacent to the pindex, coming before or after it. - */ -static vm_page_t -pmap_vmpage_splay(vm_pindex_t pindex, vm_page_t root) -{ - struct vm_page dummy; - vm_page_t lefttreemax, righttreemin, y; - - if (root == NULL) - return (root); - lefttreemax = righttreemin = &dummy; - for (;; root = y) { - if (pindex < root->pindex) { - if ((y = root->md.pv_left) == NULL) - break; - if (pindex < y->pindex) { - /* Rotate right. */ - root->md.pv_left = y->md.pv_right; - y->md.pv_right = root; - root = y; - if ((y = root->md.pv_left) == NULL) - break; - } - /* Link into the new root's right tree. */ - righttreemin->md.pv_left = root; - righttreemin = root; - } else if (pindex > root->pindex) { - if ((y = root->md.pv_right) == NULL) - break; - if (pindex > y->pindex) { - /* Rotate left. */ - root->md.pv_right = y->md.pv_left; - y->md.pv_left = root; - root = y; - if ((y = root->md.pv_right) == NULL) - break; - } - /* Link into the new root's left tree. */ - lefttreemax->md.pv_right = root; - lefttreemax = root; - } else - break; - } - /* Assemble the new root. */ - lefttreemax->md.pv_right = root->md.pv_left; - righttreemin->md.pv_left = root->md.pv_right; - root->md.pv_left = dummy.md.pv_right; - root->md.pv_right = dummy.md.pv_left; - return (root); -} - -/* * After removing a page table entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ @@ -2280,7 +2217,7 @@ TAILQ_FOREACH(m, &vpq->pl, pageq) { if ((m->flags & PG_MARKER) != 0 || m->hold_count || m->busy) continue; - TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_next, next_pv) { + TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); /* Avoid deadlock and lock recursion. */ @@ -2304,7 +2241,7 @@ pmap_unuse_pt(pmap, va, &free); pmap_invalidate_page(pmap, va); pmap_free_zero_pages(free); - TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); free_pv_entry(pmap, pv); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); @@ -2453,9 +2390,9 @@ pv_entry_t pv; mtx_assert(&vm_page_queue_mtx, MA_OWNED); - TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { + TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { - TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); break; } } @@ -2483,7 +2420,7 @@ pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); m = PHYS_TO_VM_PAGE(pa); - TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); /* Instantiate the remaining NPTEPG - 1 pv entries. */ va_last = va + NBPDR - PAGE_SIZE; do { @@ -2519,7 +2456,7 @@ pv = pmap_pvh_remove(&m->md, pmap, va); KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); pvh = pa_to_pvh(pa); - TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list); /* Free the remaining NPTEPG - 1 pv entries. */ va_last = va + NBPDR - PAGE_SIZE; do { @@ -2566,7 +2503,7 @@ mtx_assert(&vm_page_queue_mtx, MA_OWNED); pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; - TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); } /* @@ -2582,7 +2519,7 @@ if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; - TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); return (TRUE); } else return (FALSE); @@ -2602,7 +2539,7 @@ (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; pvh = pa_to_pvh(pa); - TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list); return (TRUE); } else return (FALSE); @@ -3050,7 +2987,7 @@ vm_page_dirty(m); pmap_unuse_pt(pmap, pv->pv_va, &free); pmap_invalidate_page(pmap, pv->pv_va); - TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } @@ -3482,7 +3419,7 @@ if (pv == NULL) pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; - TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); pa |= PG_MANAGED; } else if (pv != NULL) free_pv_entry(pmap, pv); @@ -4188,7 +4125,7 @@ ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; vm_page_lock_queues(); - TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { + TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; @@ -4199,7 +4136,7 @@ } if (!rv && loops < 16) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); - TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { + TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; @@ -4248,7 +4185,7 @@ mtx_assert(&vm_page_queue_mtx, MA_OWNED); sched_pin(); - TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { + TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte = pmap_pte_quick(pmap, pv->pv_va); @@ -4371,7 +4308,7 @@ if ((tpte & PG_PS) != 0) { pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; pvh = pa_to_pvh(tpte & PG_PS_FRAME); - TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) if (TAILQ_EMPTY(&mt->md.pv_list)) @@ -4389,7 +4326,7 @@ } } else { pmap->pm_stats.resident_count--; - TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) @@ -4464,7 +4401,7 @@ mtx_assert(&vm_page_queue_mtx, MA_OWNED); rv = FALSE; sched_pin(); - TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { + TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte = pmap_pte_quick(pmap, pv->pv_va); @@ -4536,7 +4473,7 @@ mtx_assert(&vm_page_queue_mtx, MA_OWNED); rv = FALSE; sched_pin(); - TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { + TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte = pmap_pte_quick(pmap, pv->pv_va); @@ -4577,7 +4514,7 @@ vm_page_lock_queues(); sched_pin(); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); - TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { + TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4586,7 +4523,7 @@ (void)pmap_demote_pde(pmap, pde, va); PMAP_UNLOCK(pmap); } - TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { + TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pde = pmap_pde(pmap, pv->pv_va); @@ -4643,7 +4580,7 @@ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); vm_page_lock_queues(); sched_pin(); - TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, pvn) { + TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4676,9 +4613,9 @@ if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pvf = pv; do { - pvn = TAILQ_NEXT(pv, pv_next); - TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); - TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + pvn = TAILQ_NEXT(pv, pv_list); + TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); + TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pde = pmap_pde(pmap, pv->pv_va); @@ -4730,7 +4667,7 @@ vm_page_lock_queues(); sched_pin(); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); - TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { + TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4766,7 +4703,7 @@ } PMAP_UNLOCK(pmap); } - TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { + TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pde = pmap_pde(pmap, pv->pv_va); @@ -4808,7 +4745,7 @@ vm_page_lock_queues(); sched_pin(); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); - TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { + TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4830,7 +4767,7 @@ } PMAP_UNLOCK(pmap); } - TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { + TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pde = pmap_pde(pmap, pv->pv_va); @@ -5344,7 +5281,7 @@ printf("pa %x", pa); m = PHYS_TO_VM_PAGE(pa); - TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { + TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); printf(" -> pmap %p, va %x", (void *)pmap, pv->pv_va); pads(pmap);