Index: amd64/amd64/uma_machdep.c =================================================================== --- amd64/amd64/uma_machdep.c (.../head/sys) (revision 267054) +++ amd64/amd64/uma_machdep.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -78,7 +78,5 @@ uma_small_free(void *mem, int size, u_int8_t flags pa = DMAP_TO_PHYS((vm_offset_t)mem); dump_drop_page(pa); m = PHYS_TO_VM_PAGE(pa); - m->wire_count--; vm_page_free(m); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); } Index: amd64/amd64/pmap.c =================================================================== --- amd64/amd64/pmap.c (.../head/sys) (revision 267054) +++ amd64/amd64/pmap.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -2012,7 +2012,17 @@ pmap_free_zero_pages(struct spglist *free) while ((m = SLIST_FIRST(free)) != NULL) { SLIST_REMOVE_HEAD(free, plinks.s.ss); - /* Preserve the page's PG_ZERO setting. */ + + /* + * Preserve the page's PG_ZERO setting. + * However, as the pages are unmanaged, fix-up the wired count + * to perform a correct free. + */ + if (m->wire_count != 0) + panic("pmap_free_zero_pages: wrong wire count %u for page %p", + m->wire_count, m); + m->wire_count = 1; + atomic_add_int(&vm_cnt.v_wire_count, 1); vm_page_free_toq(m); } } @@ -2329,8 +2339,6 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, /* Have to allocate a new pdp, recurse */ if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index, lockp) == NULL) { - --m->wire_count; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } @@ -2362,8 +2370,6 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, lockp) == NULL) { - --m->wire_count; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } @@ -2376,9 +2382,6 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, lockp) == NULL) { - --m->wire_count; - atomic_subtract_int(&vm_cnt.v_wire_count, - 1); vm_page_free_zero(m); return (NULL); } @@ -2516,8 +2519,6 @@ pmap_release(pmap_t pmap) pmap->pm_pml4[DMPML4I + i] = 0; pmap->pm_pml4[PML4PML4I] = 0; /* Recursive Mapping */ - m->wire_count--; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); if (pmap->pm_pcid != -1) free_unr(&pcid_unr, pmap->pm_pcid); @@ -2815,6 +2816,9 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock m_pc = SLIST_FIRST(&free); SLIST_REMOVE_HEAD(&free, plinks.s.ss); /* Recycle a freed page table page. */ + KASSERT((m_pc->oflags & VPO_UNMANAGED) != 0, + ("reclaim_pv_chunk: recycled page table page %p not unmanaged", + m_pc)); m_pc->wire_count = 1; atomic_add_int(&vm_cnt.v_wire_count, 1); } @@ -2868,7 +2872,6 @@ free_pv_chunk(struct pv_chunk *pc) /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); dump_drop_page(m->phys_addr); - vm_page_unwire(m, 0); vm_page_free(m); } Index: arm/arm/machdep.c =================================================================== --- arm/arm/machdep.c (.../head/sys) (revision 267054) +++ arm/arm/machdep.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -387,7 +387,8 @@ cpu_startup(void *dummy) pmap_kenter_user(ARM_TP_ADDRESS, ARM_TP_ADDRESS); arm_lock_cache_line(ARM_TP_ADDRESS); #else - m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_ZERO); + m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_ZERO | + VM_ALLOC_WIRED); pmap_kenter_user(ARM_TP_ADDRESS, VM_PAGE_TO_PHYS(m)); #endif *(uint32_t *)ARM_RAS_START = 0; Index: arm/arm/pmap-v6.c =================================================================== --- arm/arm/pmap-v6.c (.../head/sys) (revision 267054) +++ arm/arm/pmap-v6.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -4223,7 +4223,6 @@ pmap_free_pv_chunk(struct pv_chunk *pc) /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); pmap_qremove((vm_offset_t)pc, 1); - vm_page_unwire(m, 0); vm_page_free(m); pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc); Index: powerpc/booke/pmap.c =================================================================== --- powerpc/booke/pmap.c (.../head/sys) (revision 267054) +++ powerpc/booke/pmap.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -648,8 +648,13 @@ ptbl_free(mmu_t mmu, pmap_t pmap, unsigned int pdi va = ((vm_offset_t)ptbl + (i * PAGE_SIZE)); pa = pte_vatopa(mmu, kernel_pmap, va); m = PHYS_TO_VM_PAGE(pa); + + /* Fix-up the wire_count to make free perform correctly. */ + if (m->wire_count != 0) + panic("ptbl_free: invalid wire count %u for page %p", + m->wire_count, m); + ++m->wire_count; vm_page_free_zero(m); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); mmu_booke_kremove(mmu, va); } Index: powerpc/aim/uma_machdep.c =================================================================== --- powerpc/aim/uma_machdep.c (.../head/sys) (revision 267054) +++ powerpc/aim/uma_machdep.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -91,8 +91,6 @@ uma_small_free(void *mem, int size, u_int8_t flags (vm_offset_t)mem + PAGE_SIZE); m = PHYS_TO_VM_PAGE((vm_offset_t)mem); - m->wire_count--; vm_page_free(m); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); atomic_subtract_int(&hw_uma_mdpages, 1); } Index: powerpc/aim/slb.c =================================================================== --- powerpc/aim/slb.c (.../head/sys) (revision 267054) +++ powerpc/aim/slb.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -484,7 +484,7 @@ slb_uma_real_alloc(uma_zone_t zone, int bytes, u_i realmax = platform_real_maxaddr(); *flags = UMA_SLAB_PRIV; - pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED; + pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ; for (;;) { m = vm_page_alloc_contig(NULL, 0, pflags, 1, 0, realmax, Index: sparc64/sparc64/pmap.c =================================================================== --- sparc64/sparc64/pmap.c (.../head/sys) (revision 267054) +++ sparc64/sparc64/pmap.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -1292,8 +1292,6 @@ pmap_release(pmap_t pm) while (!TAILQ_EMPTY(&obj->memq)) { m = TAILQ_FIRST(&obj->memq); m->md.pmap = NULL; - m->wire_count--; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); } VM_OBJECT_WUNLOCK(obj); Index: sparc64/sparc64/vm_machdep.c =================================================================== --- sparc64/sparc64/vm_machdep.c (.../head/sys) (revision 267054) +++ sparc64/sparc64/vm_machdep.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -546,7 +546,5 @@ uma_small_free(void *mem, int size, u_int8_t flags PMAP_STATS_INC(uma_nsmall_free); m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS((vm_offset_t)mem)); - m->wire_count--; vm_page_free(m); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); } Index: kern/kern_sharedpage.c =================================================================== --- kern/kern_sharedpage.c (.../head/sys) (revision 267054) +++ kern/kern_sharedpage.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -108,7 +108,8 @@ shared_page_init(void *dummy __unused) shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE, VM_PROT_DEFAULT, 0, NULL); VM_OBJECT_WLOCK(shared_page_obj); - m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_NOBUSY | VM_ALLOC_ZERO); + m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | + VM_ALLOC_WIRED); m->valid = VM_PAGE_BITS_ALL; VM_OBJECT_WUNLOCK(shared_page_obj); addr = kva_alloc(PAGE_SIZE); Index: kern/vfs_bio.c =================================================================== --- kern/vfs_bio.c (.../head/sys) (revision 267054) +++ kern/vfs_bio.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -4288,9 +4288,7 @@ vm_hold_free_pages(struct buf *bp, int newbsize) if (vm_page_sbusied(p)) printf("vm_hold_free_pages: blkno: %jd, lblkno: %jd\n", (intmax_t)bp->b_blkno, (intmax_t)bp->b_lblkno); - p->wire_count--; vm_page_free(p); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); } bp->b_npages = newnpages; } Index: ia64/ia64/pmap.c =================================================================== --- ia64/ia64/pmap.c (.../head/sys) (revision 267054) +++ ia64/ia64/pmap.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -293,9 +293,8 @@ pmap_alloc_vhpt(void) vm_size_t size; size = 1UL << pmap_vhpt_log2size; - m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | - VM_ALLOC_WIRED, atop(size), 0UL, ~0UL, size, 0UL, - VM_MEMATTR_DEFAULT); + m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ, + atop(size), 0UL, ~0UL, size, 0UL, VM_MEMATTR_DEFAULT); if (m != NULL) { vhpt = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); pmap_initialize_vhpt(vhpt); @@ -932,7 +931,6 @@ free_pv_chunk(struct pv_chunk *pc) PV_STAT(pc_chunk_frees++); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(IA64_RR_MASK((vm_offset_t)pc)); - vm_page_unwire(m, 0); vm_page_free(m); } Index: ia64/ia64/uma_machdep.c =================================================================== --- ia64/ia64/uma_machdep.c (.../head/sys) (revision 267054) +++ ia64/ia64/uma_machdep.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -71,7 +71,5 @@ uma_small_free(void *mem, int size, u_int8_t flags vm_page_t m; m = PHYS_TO_VM_PAGE(IA64_RR_MASK((u_int64_t)mem)); - m->wire_count--; vm_page_free(m); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); } Index: mips/mips/pmap.c =================================================================== --- mips/mips/pmap.c (.../head/sys) (revision 267054) +++ mips/mips/pmap.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -993,9 +993,14 @@ _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_p /* * If the page is finally unwired, simply free it. + * Fix-up the wire_count value to make the function to perform + * the free correctly. */ + if (m->wire_count != 0) + panic("_pmap_unwire_ptp: invalid wire count %u for the page %p", + m->wire_count, m); + ++m->wire_count; vm_page_free_zero(m); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); } /* @@ -1047,8 +1052,7 @@ pmap_alloc_direct_page(unsigned int index, int req { vm_page_t m; - m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_WIRED | - VM_ALLOC_ZERO); + m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_ZERO); if (m == NULL) return (NULL); @@ -1143,8 +1147,6 @@ _pmap_allocpte(pmap_t pmap, unsigned ptepindex, in if (_pmap_allocpte(pmap, NUPDE + segindex, flags) == NULL) { /* alloc failed, release current */ - --m->wire_count; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } @@ -1226,8 +1228,6 @@ pmap_release(pmap_t pmap) ptdva = (vm_offset_t)pmap->pm_segtab; ptdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(ptdva)); - ptdpg->wire_count--; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(ptdpg); } @@ -1535,7 +1535,6 @@ free_pv_chunk(struct pv_chunk *pc) PV_STAT(pc_chunk_frees++); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS((vm_offset_t)pc)); - vm_page_unwire(m, 0); vm_page_free(m); } @@ -1581,8 +1580,7 @@ retry: } } /* No free items, allocate another chunk */ - m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, VM_ALLOC_NORMAL | - VM_ALLOC_WIRED); + m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, VM_ALLOC_NORMAL); if (m == NULL) { if (try) { pv_entry_count--; Index: mips/mips/uma_machdep.c =================================================================== --- mips/mips/uma_machdep.c (.../head/sys) (revision 267054) +++ mips/mips/uma_machdep.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -49,7 +49,7 @@ uma_small_alloc(uma_zone_t zone, int bytes, u_int8 void *va; *flags = UMA_SLAB_PRIV; - pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED; + pflags = malloc2vm_flags(wait); for (;;) { m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, pflags); @@ -77,7 +77,5 @@ uma_small_free(void *mem, int size, u_int8_t flags pa = MIPS_DIRECT_TO_PHYS((vm_offset_t)mem); m = PHYS_TO_VM_PAGE(pa); - m->wire_count--; vm_page_free(m); - atomic_subtract_int(&vm_cnt.v_wire_count, 1); } Index: dev/virtio/balloon/virtio_balloon.c =================================================================== --- dev/virtio/balloon/virtio_balloon.c (.../head/sys) (revision 267054) +++ dev/virtio/balloon/virtio_balloon.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -450,7 +450,6 @@ static void vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) { - vm_page_unwire(m, 0); vm_page_free(m); sc->vtballoon_current_npages--; } Index: dev/drm2/i915/i915_gem_gtt.c =================================================================== --- dev/drm2/i915/i915_gem_gtt.c (.../head/sys) (revision 267054) +++ dev/drm2/i915/i915_gem_gtt.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -205,10 +205,8 @@ i915_gem_cleanup_aliasing_ppgtt(struct drm_device for (i = 0; i < ppgtt->num_pd_entries; i++) { m = ppgtt->pt_pages[i]; - if (m != NULL) { - vm_page_unwire(m, 0); + if (m != NULL) vm_page_free(m); - } } free(ppgtt->pt_pages, DRM_I915_GEM); free(ppgtt, DRM_I915_GEM); Index: dev/drm2/ttm/ttm_page_alloc.c =================================================================== --- dev/drm2/ttm/ttm_page_alloc.c (.../head/sys) (revision 267054) +++ dev/drm2/ttm/ttm_page_alloc.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -139,7 +139,6 @@ ttm_vm_page_free(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m)); m->flags &= ~PG_FICTITIOUS; m->oflags |= VPO_UNMANAGED; - vm_page_unwire(m, 0); vm_page_free(m); } @@ -492,7 +491,7 @@ static int ttm_alloc_new_pages(struct pglist *page unsigned max_cpages = min(count, (unsigned)(PAGE_SIZE/sizeof(vm_page_t))); - aflags = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ | + aflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | ((ttm_alloc_flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? VM_ALLOC_ZERO : 0); @@ -708,7 +707,7 @@ static int ttm_get_pages(vm_page_t *pages, unsigne unsigned count; int r; - aflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | + aflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? VM_ALLOC_ZERO : 0); /* No pool for cached pages */ Index: dev/xen/balloon/balloon.c =================================================================== --- dev/xen/balloon/balloon.c (.../head/sys) (revision 267054) +++ dev/xen/balloon/balloon.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -255,7 +255,6 @@ increase_reservation(unsigned long nr_pages) set_phys_to_machine(pfn, frame_list[i]); - vm_page_unwire(page, 0); vm_page_free(page); } @@ -297,7 +296,6 @@ decrease_reservation(unsigned long nr_pages) set_phys_to_machine(pfn, INVALID_P2M_ENTRY); if (balloon_append(page) != 0) { - vm_page_unwire(page, 0); vm_page_free(page); nr_pages = i; Index: dev/ti/if_ti.c =================================================================== --- dev/ti/if_ti.c (.../head/sys) (revision 267054) +++ dev/ti/if_ti.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -1616,7 +1616,6 @@ ti_newbuf_jumbo(struct ti_softc *sc, int idx, stru } sf[i] = sf_buf_alloc(frame, SFB_NOWAIT); if (sf[i] == NULL) { - vm_page_unwire(frame, 0); vm_page_free(frame); device_printf(sc->ti_dev, "buffer allocation " "failed -- packet dropped!\n"); Index: vm/vm_object.c =================================================================== --- vm/vm_object.c (.../head/sys) (revision 267054) +++ vm/vm_object.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -1951,7 +1951,10 @@ again: if ((options & OBJPR_NOTWIRED) != 0 && wirings != 0) goto next; pmap_remove_all(p); - /* Account for removal of wired mappings. */ + /* + * Account for removal of wired mappings. + * The object will not contain unmanaged pages. + */ if (wirings != 0) { KASSERT(p->wire_count == wirings, ("inconsistent wire count %d %d %p", @@ -2041,8 +2044,12 @@ vm_object_populate(vm_object_t object, vm_pindex_t int rv; VM_OBJECT_ASSERT_WLOCKED(object); + KASSERT((object->flags & OBJ_UNMANAGED) != 0, + ("vm_object_populate: object %p cannot contain unmanaged pages", + object)); for (pindex = start; pindex < end; pindex++) { - m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); + m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | + VM_ALLOC_WIRED); if (m->valid != VM_PAGE_BITS_ALL) { ma[0] = m; rv = vm_pager_get_pages(object, ma, 1, 0); Index: vm/vm_page.c =================================================================== --- vm/vm_page.c (.../head/sys) (revision 267054) +++ vm/vm_page.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -1450,7 +1450,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind struct vnode *vp = NULL; vm_object_t m_object; vm_page_t m, mpred; - int flags, req_class; + int flags, req_class, unmanaged; mpred = 0; /* XXX: pacify gcc */ KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && @@ -1462,6 +1462,10 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind if (object != NULL) VM_OBJECT_ASSERT_WLOCKED(object); + unmanaged = (object == NULL || (object->flags & OBJ_UNMANAGED) != 0); + KASSERT(unmanaged == 0 || (req & VM_ALLOC_WIRED) != 0, + ("vm_page_alloc: unmanaged but unwired request req(%x)", req)); + req_class = req & VM_ALLOC_CLASS_MASK; /* @@ -1585,8 +1589,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind flags |= PG_NODUMP; m->flags = flags; m->aflags = 0; - m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? - VPO_UNMANAGED : 0; + m->oflags = (unmanaged != 0) ? VPO_UNMANAGED : 0; m->busy_lock = VPB_UNBUSIED; if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) m->busy_lock = VPB_SINGLE_EXCLUSIVER; @@ -1608,7 +1611,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind if (vp != NULL) vdrop(vp); pagedaemon_wakeup(); - if (req & VM_ALLOC_WIRED) { + if ((req & VM_ALLOC_WIRED) != 0 && unmanaged == 0) { atomic_subtract_int(&vm_cnt.v_wire_count, 1); m->wire_count = 0; } @@ -1676,6 +1679,8 @@ vm_page_alloc_contig_vdrop(struct spglist *lst) * * The caller must always specify an allocation class. * + * The returned pages will all be wired. + * * allocation classes: * VM_ALLOC_NORMAL normal process request * VM_ALLOC_SYSTEM system *really* needs a page @@ -1686,7 +1691,6 @@ vm_page_alloc_contig_vdrop(struct spglist *lst) * VM_ALLOC_NOOBJ page is not associated with an object and * should not be exclusive busy * VM_ALLOC_SBUSY shared busy the allocated page - * VM_ALLOC_WIRED wire the allocated page * VM_ALLOC_ZERO prefer a zeroed page * * This routine may not sleep. @@ -1708,6 +1712,8 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), ("vm_page_alloc: inconsistent object(%p)/req(%x)", (void *)object, req)); + KASSERT((req & VM_ALLOC_WIRED) == 0, + ("vm_page_alloc_contig: VM_ALLOC_WIRED passed in req (%x)", req)); if (object != NULL) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_PHYS, @@ -1775,8 +1781,7 @@ retry: flags = PG_ZERO; if ((req & VM_ALLOC_NODUMP) != 0) flags |= PG_NODUMP; - if ((req & VM_ALLOC_WIRED) != 0) - atomic_add_int(&vm_cnt.v_wire_count, npages); + atomic_add_int(&vm_cnt.v_wire_count, npages); if (object != NULL) { if (object->memattr != VM_MEMATTR_DEFAULT && memattr == VM_MEMATTR_DEFAULT) @@ -1792,8 +1797,8 @@ retry: if ((req & VM_ALLOC_SBUSY) != 0) m->busy_lock = VPB_SHARERS_WORD(1); } - if ((req & VM_ALLOC_WIRED) != 0) - m->wire_count = 1; + m->wire_count = 1; + /* Unmanaged pages don't use "act_count". */ m->oflags = VPO_UNMANAGED; if (object != NULL) { @@ -1802,13 +1807,10 @@ retry: &deferred_vdrop_list); if (vm_paging_needed()) pagedaemon_wakeup(); - if ((req & VM_ALLOC_WIRED) != 0) - atomic_subtract_int(&vm_cnt.v_wire_count, - npages); for (m_tmp = m, m = m_ret; m < &m_ret[npages]; m++) { - if ((req & VM_ALLOC_WIRED) != 0) - m->wire_count = 0; + m->wire_count = 1; + m->oflags = VPO_UNMANAGED; if (m >= m_tmp) m->object = NULL; vm_page_free(m); @@ -1883,6 +1885,8 @@ vm_page_alloc_init(vm_page_t m) * * The caller must always specify an allocation class. * + * The returned page will be wired. + * * allocation classes: * VM_ALLOC_NORMAL normal process request * VM_ALLOC_SYSTEM system *really* needs a page @@ -1891,7 +1895,6 @@ vm_page_alloc_init(vm_page_t m) * optional allocation flags: * VM_ALLOC_COUNT(number) the number of additional pages that the caller * intends to allocate - * VM_ALLOC_WIRED wire the allocated page * VM_ALLOC_ZERO prefer a zeroed page * * This routine may not sleep. @@ -1904,6 +1907,10 @@ vm_page_alloc_freelist(int flind, int req) u_int flags; int req_class; + KASSERT((req & VM_ALLOC_WIRED) == 0, + ("vm_page_alloc_freelist: VM_ALLOC_WIRED passed in req (%x)", + req)); + req_class = req & VM_ALLOC_CLASS_MASK; /* @@ -1944,14 +1951,14 @@ vm_page_alloc_freelist(int flind, int req) if ((req & VM_ALLOC_ZERO) != 0) flags = PG_ZERO; m->flags &= flags; - if ((req & VM_ALLOC_WIRED) != 0) { - /* - * The page lock is not required for wiring a page that does - * not belong to an object. - */ - atomic_add_int(&vm_cnt.v_wire_count, 1); - m->wire_count = 1; - } + + /* + * The page lock is not required for wiring a page that does + * not belong to an object. + */ + atomic_add_int(&vm_cnt.v_wire_count, 1); + m->wire_count = 1; + /* Unmanaged pages don't use "act_count". */ m->oflags = VPO_UNMANAGED; if (drop != NULL) @@ -2224,9 +2231,15 @@ vm_page_free_toq(vm_page_t m) vm_page_lock_assert(m, MA_OWNED); KASSERT(!pmap_page_is_mapped(m), ("vm_page_free_toq: freeing mapped page %p", m)); - } else + } else { KASSERT(m->queue == PQ_NONE, ("vm_page_free_toq: unmanaged page %p is queued", m)); + KASSERT(m->wire_count == 1, + ("vm_page_free_toq: invalid wired count %u for unmanaged page %p", + m->wire_count, m)); + m->wire_count--; + atomic_subtract_int(&vm_cnt.v_wire_count, 1); + } PCPU_INC(cnt.v_tfree); if (vm_page_sbusied(m)) @@ -2355,10 +2368,10 @@ vm_page_unwire(vm_page_t m, int activate) if (m->wire_count > 0) { m->wire_count--; if (m->wire_count == 0) { + if ((m->oflags & VPO_UNMANAGED) != 0) + panic("vm_page_unwire: unmanaged page %p's wire count is one", + m); atomic_subtract_int(&vm_cnt.v_wire_count, 1); - if ((m->oflags & VPO_UNMANAGED) != 0 || - m->object == NULL) - return; if (!activate) m->flags &= ~PG_WINATCFLS; vm_page_enqueue(activate ? PQ_ACTIVE : PQ_INACTIVE, m); Index: vm/vm_page.h =================================================================== --- vm/vm_page.h (.../head/sys) (revision 267054) +++ vm/vm_page.h (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -171,6 +171,7 @@ struct vm_page { * under PV management cannot be paged out via the * object/vm_page_t because there is no knowledge of their pte * mappings, and such pages are also not on any PQ queue. + * VPO_UNMANAGED pages are also mandatory wired. * */ #define VPO_UNUSED01 0x01 /* --available-- */ Index: vm/uma_core.c =================================================================== --- vm/uma_core.c (.../head/sys) (revision 267054) +++ vm/uma_core.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -1150,10 +1150,8 @@ noobj_alloc(uma_zone_t zone, int bytes, uint8_t *f * Page allocation failed, free intermediate pages and * exit. */ - TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { - vm_page_unwire(p, 0); + TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) vm_page_free(p); - } return (NULL); } *flags = UMA_SLAB_PRIV; Index: vm/vm_kern.c =================================================================== --- vm/vm_kern.c (.../head/sys) (revision 267054) +++ vm/vm_kern.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -169,7 +169,7 @@ kmem_alloc_attr(vmem_t *vmem, vm_size_t size, int if (vmem_alloc(vmem, size, M_BESTFIT | flags, &addr)) return (0); offset = addr - VM_MIN_KERNEL_ADDRESS; - pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; + pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY; VM_OBJECT_WLOCK(object); for (i = 0; i < size; i += PAGE_SIZE) { tries = 0; @@ -193,7 +193,6 @@ retry: i -= PAGE_SIZE; m = vm_page_lookup(object, OFF_TO_IDX(offset + i)); - vm_page_unwire(m, 0); vm_page_free(m); } vmem_free(vmem, addr, size); @@ -232,7 +231,7 @@ kmem_alloc_contig(struct vmem *vmem, vm_size_t siz if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr)) return (0); offset = addr - VM_MIN_KERNEL_ADDRESS; - pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; + pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY; VM_OBJECT_WLOCK(object); tries = 0; retry: @@ -367,7 +366,6 @@ retry: i -= PAGE_SIZE; m = vm_page_lookup(object, OFF_TO_IDX(offset + i)); - vm_page_unwire(m, 0); vm_page_free(m); } VM_OBJECT_WUNLOCK(object); @@ -401,7 +399,6 @@ kmem_unback(vm_object_t object, vm_offset_t addr, VM_OBJECT_WLOCK(object); for (i = 0; i < size; i += PAGE_SIZE) { m = vm_page_lookup(object, OFF_TO_IDX(offset + i)); - vm_page_unwire(m, 0); vm_page_free(m); } VM_OBJECT_WUNLOCK(object); Index: x86/iommu/intel_idpgtbl.c =================================================================== --- x86/iommu/intel_idpgtbl.c (.../head/sys) (revision 267054) +++ x86/iommu/intel_idpgtbl.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -373,17 +373,18 @@ retry: */ m = dmar_pgalloc(ctx->pgtbl_obj, idx, flags | DMAR_PGF_ZERO); - if (m == NULL) - return (NULL); /* - * Prevent potential free while pgtbl_obj is + * If a page is successfully returned, it is assumed + * that the page is properly wired already. This + * prevent potential free while pgtbl_obj is * unlocked in the recursive call to * ctx_pgtbl_map_pte(), if other thread did * pte write and clean while the lock if * dropped. */ - m->wire_count++; + if (m == NULL) + return (NULL); sfp = NULL; ptep = ctx_pgtbl_map_pte(ctx, base, lvl - 1, flags, @@ -391,7 +392,6 @@ retry: if (ptep == NULL) { KASSERT(m->pindex != 0, ("loosing root page %p", ctx)); - m->wire_count--; dmar_pgfree(ctx->pgtbl_obj, m->pindex, flags); return (NULL); } @@ -398,7 +398,6 @@ retry: dmar_pte_store(&ptep->pte, DMAR_PTE_R | DMAR_PTE_W | VM_PAGE_TO_PHYS(m)); sf_buf_page(sfp)->wire_count += 1; - m->wire_count--; dmar_unmap_pgtbl(sfp, DMAR_IS_COHERENT(ctx->dmar)); /* Only executed once. */ goto retry; @@ -573,7 +572,7 @@ ctx_unmap_clear_pte(struct dmar_ctx *ctx, dmar_gad *sf = NULL; } m->wire_count--; - if (m->wire_count != 0) + if (m->wire_count != 1) return; KASSERT(lvl != 0, ("lost reference (lvl) on root pg ctx %p base %jx lvl %d", @@ -685,8 +684,6 @@ ctx_alloc_pgtbl(struct dmar_ctx *ctx) DMAR_CTX_PGLOCK(ctx); m = dmar_pgalloc(ctx->pgtbl_obj, 0, DMAR_PGF_WAITOK | DMAR_PGF_ZERO | DMAR_PGF_OBJL); - /* No implicit free of the top level page table page. */ - m->wire_count = 1; DMAR_CTX_PGUNLOCK(ctx); return (0); } @@ -716,7 +713,7 @@ ctx_free_pgtbl(struct dmar_ctx *ctx) /* Obliterate wire_counts */ VM_OBJECT_ASSERT_WLOCKED(obj); for (m = vm_page_lookup(obj, 0); m != NULL; m = vm_page_next(m)) - m->wire_count = 0; + m->wire_count = 1; VM_OBJECT_WUNLOCK(obj); vm_object_deallocate(obj); } Index: i386/i386/pmap.c =================================================================== --- i386/i386/pmap.c (.../head/sys) (revision 267054) +++ i386/i386/pmap.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -1578,7 +1578,17 @@ pmap_free_zero_pages(struct spglist *free) while ((m = SLIST_FIRST(free)) != NULL) { SLIST_REMOVE_HEAD(free, plinks.s.ss); - /* Preserve the page's PG_ZERO setting. */ + + /* + * Preserve the page's PG_ZERO setting. + * However, as the pages are unmanaged, fix-up the wired count + * to perform a correct free. + */ + if (m->wire_count != 0) + panic("pmap_free_zero_pages: wrong wire count %u for page %p", + m->wire_count, m); + m->wire_count = 1; + atomic_add_int(&vm_cnt.v_wire_count, 1); vm_page_free_toq(m); } } @@ -2049,8 +2059,6 @@ pmap_release(pmap_t pmap) KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME), ("pmap_release: got wrong ptd page")); #endif - m->wire_count--; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); } } @@ -2312,6 +2320,9 @@ out: m_pc = SLIST_FIRST(&free); SLIST_REMOVE_HEAD(&free, plinks.s.ss); /* Recycle a freed page table page. */ + KASSERT((m_pc->oflags & VPO_UNMANAGED) != 0, + ("pmap_pv_reclaim: recycled page table page %p not unmanaged", + m_pc)); m_pc->wire_count = 1; atomic_add_int(&vm_cnt.v_wire_count, 1); } @@ -2368,7 +2379,6 @@ free_pv_chunk(struct pv_chunk *pc) /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); pmap_qremove((vm_offset_t)pc, 1); - vm_page_unwire(m, 0); vm_page_free(m); pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc); } Index: i386/xen/pmap.c =================================================================== --- i386/xen/pmap.c (.../head/sys) (revision 267054) +++ i386/xen/pmap.c (.../user/attilio/rm_vmobj_cache/sys) (revision 267054) @@ -1336,6 +1336,16 @@ pmap_free_zero_pages(vm_page_t free) m = free; free = (void *)m->object; m->object = NULL; + + /* + * As the pages are unmanaged, fix-up the wired count + * to perform a correct free. + */ + if (m->wire_count != 0) + panic("pmap_free_zero_pages: wrong wire count %u for page %p", + m->wire_count, m); + m->wire_count = 1; + atomic_add_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); } } @@ -1812,8 +1822,6 @@ pmap_release(pmap_t pmap) KASSERT(VM_PAGE_TO_MACH(m) == (pmap->pm_pdpt[i] & PG_FRAME), ("pmap_release: got wrong ptd page")); #endif - m->wire_count--; - atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free(m); } #ifdef PAE @@ -2088,6 +2096,9 @@ out: m_pc = free; free = (void *)m_pc->object; /* Recycle a freed page table page. */ + KASSERT((m_pc->oflags & VPO_UNMANAGED) != 0, + ("pmap_pv_reclaim: recycled page table page %p not unmanaged", + m_pc)); m_pc->wire_count = 1; atomic_add_int(&vm_cnt.v_wire_count, 1); } @@ -2144,7 +2155,6 @@ free_pv_chunk(struct pv_chunk *pc) /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); pmap_qremove((vm_offset_t)pc, 1); - vm_page_unwire(m, 0); vm_page_free(m); pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc); }