diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 5389755..f1a6678 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -228,6 +228,10 @@ static vm_offset_t pmap_kmem_choose(vm_o CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t)); CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t)); +struct mtx pv_lock; + +#define PV_ASSERT_LOCKED() mtx_assert(&pv_lock, MA_OWNED) + /* * Move the kernel virtual free pointer to the next * 2MB. This is used to help improve performance @@ -622,6 +626,8 @@ pmap_init(void) pv_entry_max = shpgperproc * maxproc + cnt.v_page_count; TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max); pv_entry_high_water = 9 * (pv_entry_max / 10); + + mtx_init(&pv_lock, "pv lock", "pv lock", MTX_DEF); } SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); @@ -1611,6 +1617,8 @@ pmap_collect(pmap_t locked_pmap, struct vm_offset_t va; vm_page_t m; + PV_ASSERT_LOCKED(); + TAILQ_FOREACH(m, &vpq->pl, pageq) { if (m->hold_count || m->busy) continue; @@ -1628,7 +1636,7 @@ pmap_collect(pmap_t locked_pmap, struct KASSERT((tpte & PG_W) == 0, ("pmap_collect: wired pte %#lx", tpte)); if (tpte & PG_A) - vm_page_flag_set(m, PG_REFERENCED); + vm_page_md_flag_set(m, PG_REFERENCED); if (tpte & PG_M) { KASSERT((tpte & PG_RW), ("pmap_collect: modified page not writable: va: %#lx, pte: %#lx", @@ -1638,7 +1646,7 @@ pmap_collect(pmap_t locked_pmap, struct pmap_invalidate_page(pmap, va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) - vm_page_flag_clear(m, PG_WRITEABLE); + vm_page_md_flag_clear(m, PG_WRITEABLE); m->md.pv_list_count--; pmap_unuse_pt(pmap, va, ptepde); free_pv_entry(pmap, pv); @@ -1659,8 +1667,8 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv struct pv_chunk *pc; int idx, field, bit; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); + PV_ASSERT_LOCKED(); PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); pv_entry_count--; @@ -1701,7 +1709,7 @@ get_pv_entry(pmap_t pmap, int try) vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); PV_STAT(pv_entry_allocs++); pv_entry_count++; if (pv_entry_count > pv_entry_high_water) @@ -1779,7 +1787,7 @@ pmap_remove_entry(pmap_t pmap, vm_page_t pv_entry_t pv; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) break; @@ -1788,7 +1796,7 @@ pmap_remove_entry(pmap_t pmap, vm_page_t TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); m->md.pv_list_count--; if (TAILQ_EMPTY(&m->md.pv_list)) - vm_page_flag_clear(m, PG_WRITEABLE); + vm_page_md_flag_clear(m, PG_WRITEABLE); free_pv_entry(pmap, pv); } @@ -1802,7 +1810,7 @@ pmap_insert_entry(pmap_t pmap, vm_offset pv_entry_t pv; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); @@ -1819,6 +1827,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm PMAP_LOCK_ASSERT(pmap, MA_OWNED); mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; @@ -1858,7 +1867,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t vm_page_dirty(m); } if (oldpte & PG_A) - vm_page_flag_set(m, PG_REFERENCED); + vm_page_md_flag_set(m, PG_REFERENCED); pmap_remove_entry(pmap, m, va); } return (pmap_unuse_pt(pmap, va, ptepde)); @@ -1907,6 +1916,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva anyvalid = 0; vm_page_lock_queues(); + PV_LOCK(); PMAP_LOCK(pmap); /* @@ -1991,6 +2001,7 @@ out: vm_page_unlock_queues(); if (anyvalid) pmap_invalidate_all(pmap); + PV_UNLOCK(); PMAP_UNLOCK(pmap); } @@ -2025,6 +2036,7 @@ pmap_remove_all(vm_page_t m) } #endif mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -2034,7 +2046,7 @@ pmap_remove_all(vm_page_t m) if (tpte & PG_W) pmap->pm_stats.wired_count--; if (tpte & PG_A) - vm_page_flag_set(m, PG_REFERENCED); + vm_page_md_flag_set(m, PG_REFERENCED); /* * Update the vm_page_t clean and reference bits. @@ -2052,7 +2064,7 @@ pmap_remove_all(vm_page_t m) free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } - vm_page_flag_clear(m, PG_WRITEABLE); + vm_page_md_flag_clear(m, PG_WRITEABLE); } /* @@ -2081,6 +2093,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sv anychanged = 0; vm_page_lock_queues(); + PV_LOCK(); PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { @@ -2135,7 +2148,7 @@ retry: m = NULL; if (pbits & PG_A) { m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); - vm_page_flag_set(m, PG_REFERENCED); + vm_page_md_flag_set(m, PG_REFERENCED); pbits &= ~PG_A; } if ((pbits & PG_M) != 0) { @@ -2162,6 +2175,7 @@ retry: } } vm_page_unlock_queues(); + PV_UNLOCK(); if (anychanged) pmap_invalidate_all(pmap); PMAP_UNLOCK(pmap); @@ -2201,7 +2215,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, mpte = NULL; - vm_page_lock_queues(); + PV_LOCK(); PMAP_LOCK(pmap); /* @@ -2315,7 +2329,7 @@ validate: newpte = (pt_entry_t)(pa | PG_V); if ((prot & VM_PROT_WRITE) != 0) { newpte |= PG_RW; - vm_page_flag_set(m, PG_WRITEABLE); + vm_page_md_flag_set(m, PG_WRITEABLE); } if ((prot & VM_PROT_EXECUTE) == 0) newpte |= pg_nx; @@ -2336,7 +2350,7 @@ validate: origpte = pte_load_store(pte, newpte | PG_A); if (origpte & PG_A) { if (origpte & PG_MANAGED) - vm_page_flag_set(om, PG_REFERENCED); + vm_page_md_flag_set(om, PG_REFERENCED); if (opa != VM_PAGE_TO_PHYS(m) || ((origpte & PG_NX) == 0 && (newpte & PG_NX))) invlva = TRUE; @@ -2355,7 +2369,7 @@ validate: } else pte_store(pte, newpte | PG_A); } - vm_page_unlock_queues(); + PV_UNLOCK(); PMAP_UNLOCK(pmap); } @@ -2382,6 +2396,7 @@ pmap_enter_object(pmap_t pmap, vm_offset psize = atop(end - start); mpte = NULL; m = m_start; + PV_LOCK(); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, @@ -2389,6 +2404,7 @@ pmap_enter_object(pmap_t pmap, vm_offset m = TAILQ_NEXT(m, listq); } PMAP_UNLOCK(pmap); + PV_UNLOCK(); } /* @@ -2404,9 +2420,11 @@ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { + PV_LOCK(); PMAP_LOCK(pmap); (void) pmap_enter_quick_locked(pmap, va, m, prot, NULL); PMAP_UNLOCK(pmap); + PV_UNLOCK(); } static vm_page_t @@ -2421,6 +2439,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_ ("pmap_enter_quick_locked: managed mapping within the clean submap")); mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); + PV_ASSERT_LOCKED(); /* * In the case that a page table page is not @@ -2668,6 +2687,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm return; vm_page_lock_queues(); + PV_LOCK(); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); @@ -2764,6 +2784,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm } } vm_page_unlock_queues(); + PV_UNLOCK(); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } @@ -2843,6 +2864,7 @@ pmap_page_exists_quick(pmap_t pmap, vm_p return FALSE; mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { return TRUE; @@ -2878,7 +2900,7 @@ pmap_remove_pages(pmap_t pmap) printf("warning: pmap_remove_pages called with non-current pmap\n"); return; } - vm_page_lock_queues(); + PV_LOCK(); PMAP_LOCK(pmap); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; @@ -2937,7 +2959,7 @@ pmap_remove_pages(pmap_t pmap) m->md.pv_list_count--; TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) - vm_page_flag_clear(m, PG_WRITEABLE); + vm_page_md_flag_clear(m, PG_WRITEABLE); pmap_unuse_pt(pmap, pv->pv_va, *vtopde(pv->pv_va)); } @@ -2952,7 +2974,7 @@ pmap_remove_pages(pmap_t pmap) vm_page_free(m); } } - vm_page_unlock_queues(); + PV_UNLOCK(); pmap_invalidate_all(pmap); PMAP_UNLOCK(pmap); } @@ -2976,6 +2998,7 @@ pmap_is_modified(vm_page_t m) return (rv); mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -3023,9 +3046,10 @@ pmap_remove_write(vm_page_t m) pt_entry_t oldpte, *pte; if ((m->flags & PG_FICTITIOUS) != 0 || - (m->flags & PG_WRITEABLE) == 0) + (m->md_flags & PG_WRITEABLE) == 0) return; mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -3042,7 +3066,7 @@ retry: } PMAP_UNLOCK(pmap); } - vm_page_flag_clear(m, PG_WRITEABLE); + vm_page_md_flag_clear(m, PG_WRITEABLE); } /* @@ -3068,6 +3092,7 @@ pmap_ts_referenced(vm_page_t m) if (m->flags & PG_FICTITIOUS) return (rtval); mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pvf = pv; do { @@ -3103,6 +3128,7 @@ pmap_clear_modify(vm_page_t m) if ((m->flags & PG_FICTITIOUS) != 0) return; mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -3130,6 +3156,7 @@ pmap_clear_reference(vm_page_t m) if ((m->flags & PG_FICTITIOUS) != 0) return; mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PV_ASSERT_LOCKED(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -3138,7 +3165,6 @@ pmap_clear_reference(vm_page_t m) atomic_clear_long(pte, PG_A); pmap_invalidate_page(pmap, pv->pv_va); } - PMAP_UNLOCK(pmap); } } @@ -3354,8 +3380,10 @@ pmap_mincore(pmap_t pmap, vm_offset_t ad * Modified by someone else */ vm_page_lock_queues(); + PV_LOCK(); if (m->dirty || pmap_is_modified(m)) val |= MINCORE_MODIFIED_OTHER; + PV_UNLOCK(); vm_page_unlock_queues(); } /* @@ -3368,11 +3396,13 @@ pmap_mincore(pmap_t pmap, vm_offset_t ad * Referenced by someone else */ vm_page_lock_queues(); - if ((m->flags & PG_REFERENCED) || + PV_LOCK(); + if ((m->md_flags & PG_REFERENCED) || pmap_ts_referenced(m)) { val |= MINCORE_REFERENCED_OTHER; - vm_page_flag_set(m, PG_REFERENCED); + vm_page_md_flag_set(m, PG_REFERENCED); } + PV_UNLOCK(); vm_page_unlock_queues(); } } diff --git a/sys/kern/kern_subr.c b/sys/kern/kern_subr.c index 5e312d8..e911660 100644 --- a/sys/kern/kern_subr.c +++ b/sys/kern/kern_subr.c @@ -108,7 +108,9 @@ retry: if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) goto retry; vm_page_lock_queues(); + PV_LOCK(); pmap_remove_all(user_pg); + PV_UNLOCK(); vm_page_free(user_pg); } else { /* diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 9228c57..8aac692 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -3454,7 +3454,9 @@ retry: * It may not work properly with small-block devices. * We need to find a better way. */ + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); if (clear_modify) vfs_page_set_valid(bp, foff, i, m); else if (m->valid == VM_PAGE_BITS_ALL && diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index e6ab79f..85cb4b3 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -1069,9 +1069,9 @@ swap_pager_getpages(vm_object_t object, VM_OBJECT_LOCK(object); while ((mreq->oflags & VPO_SWAPINPROG) != 0) { mreq->oflags |= VPO_WANTED; - vm_page_lock_queues(); - vm_page_flag_set(mreq, PG_REFERENCED); - vm_page_unlock_queues(); + PV_LOCK(); + vm_page_md_flag_set(mreq, PG_REFERENCED); + PV_UNLOCK(); cnt.v_intrans++; if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) { printf( @@ -1434,7 +1434,9 @@ swp_pager_async_iodone(struct buf *bp) * vm_page_wakeup(). We do not set reqpage's * valid bits here, it is up to the caller. */ + PV_LOCK(); pmap_clear_modify(m); + PV_UNLOCK(); m->valid = VM_PAGE_BITS_ALL; vm_page_undirty(m); @@ -1458,7 +1460,9 @@ swp_pager_async_iodone(struct buf *bp) * status, then finish the I/O ( which decrements the * busy count and possibly wakes waiter's up ). */ + PV_LOCK(); pmap_clear_modify(m); + PV_UNLOCK(); vm_page_undirty(m); vm_page_io_finish(m); if (vm_page_count_severe()) diff --git a/sys/vm/vm_contig.c b/sys/vm/vm_contig.c index da98780..c239381 100644 --- a/sys/vm/vm_contig.c +++ b/sys/vm/vm_contig.c @@ -102,8 +102,11 @@ vm_contig_launder_page(vm_page_t m) return (EBUSY); } vm_page_test_dirty(m); - if (m->dirty == 0 && m->hold_count == 0) + if (m->dirty == 0 && m->hold_count == 0) { + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); + } if (m->dirty) { if ((object->flags & OBJ_DEAD) != 0) { VM_OBJECT_UNLOCK(object); diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index b33fb1a..d4dd66d 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -515,7 +515,9 @@ readrest: mt->hold_count || mt->wire_count) continue; + PV_LOCK(); pmap_remove_all(mt); + PV_UNLOCK(); if (mt->dirty) { vm_page_deactivate(mt); } else { @@ -897,7 +899,9 @@ readrest: } VM_OBJECT_LOCK(fs.object); vm_page_lock_queues(); - vm_page_flag_set(fs.m, PG_REFERENCED); + PV_LOCK(); + vm_page_md_flag_set(fs.m, PG_REFERENCED); + PV_UNLOCK(); /* * If the page is not wired down, then put it where the pageout daemon diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c index 3a8ea06..b5c9794 100644 --- a/sys/vm/vm_mmap.c +++ b/sys/vm/vm_mmap.c @@ -843,14 +843,16 @@ RestartScan: if (m != NULL && m->valid != 0) { mincoreinfo = MINCORE_INCORE; vm_page_lock_queues(); + PV_LOCK(); if (m->dirty || pmap_is_modified(m)) mincoreinfo |= MINCORE_MODIFIED_OTHER; - if ((m->flags & PG_REFERENCED) || + if ((m->md_flags & PG_REFERENCED) || pmap_ts_referenced(m)) { - vm_page_flag_set(m, PG_REFERENCED); + vm_page_md_flag_set(m, PG_REFERENCED); mincoreinfo |= MINCORE_REFERENCED_OTHER; } + PV_UNLOCK(); vm_page_unlock_queues(); } VM_OBJECT_UNLOCK(current->object.vm_object); diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index aaeebcd..c36aa91 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -806,8 +806,11 @@ vm_object_page_clean(vm_object_t object, p->oflags |= VPO_CLEANCHK; if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) clearobjflags = 0; - else + else { + PV_LOCK(); pmap_remove_write(p); + PV_UNLOCK(); + } } if (clearobjflags && (tstart == 0) && (tend == object->size)) { @@ -976,7 +979,9 @@ vm_object_page_collect_flush(vm_object_t vm_pageout_flush(ma, runlen, pagerflags); for (i = 0; i < runlen; i++) { if (ma[i]->valid & ma[i]->dirty) { + PV_LOCK(); pmap_remove_write(ma[i]); + PV_UNLOCK(); ma[i]->oflags |= VPO_CLEANCHK; /* @@ -1152,7 +1157,9 @@ shadowlookup: goto unlock_tobject; } if ((m->oflags & VPO_BUSY) || m->busy) { - vm_page_flag_set(m, PG_REFERENCED); + PV_LOCK(); + vm_page_md_flag_set(m, PG_REFERENCED); + PV_UNLOCK(); vm_page_unlock_queues(); if (object != tobject) VM_OBJECT_UNLOCK(object); @@ -1181,7 +1188,9 @@ shadowlookup: * can without actually taking the step of unmapping * it. */ + PV_LOCK(); pmap_clear_modify(m); + PV_UNLOCK(); m->dirty = 0; m->act_count = 0; vm_page_dontneed(m); @@ -1358,7 +1367,9 @@ retry: * not be changed by this operation. */ if ((m->oflags & VPO_BUSY) || m->busy) { + PV_LOCK(); vm_page_flag_set(m, PG_REFERENCED); + PV_UNLOCK(); vm_page_unlock_queues(); VM_OBJECT_UNLOCK(new_object); m->oflags |= VPO_WANTED; @@ -1489,9 +1500,9 @@ vm_object_backing_scan(vm_object_t objec } } else if (op & OBSC_COLLAPSE_WAIT) { if ((p->oflags & VPO_BUSY) || p->busy) { - vm_page_lock_queues(); - vm_page_flag_set(p, PG_REFERENCED); - vm_page_unlock_queues(); + PV_LOCK(); + vm_page_md_flag_set(p, PG_REFERENCED); + PV_UNLOCK(); VM_OBJECT_UNLOCK(object); p->oflags |= VPO_WANTED; msleep(p, VM_OBJECT_MTX(backing_object), @@ -1836,7 +1847,9 @@ again: next = TAILQ_NEXT(p, listq); if (p->wire_count != 0) { + PV_LOCK(); pmap_remove_all(p); + PV_UNLOCK(); if (!clean_only) p->valid = 0; continue; @@ -1844,11 +1857,15 @@ again: if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) goto again; if (clean_only && p->valid) { + PV_LOCK(); pmap_remove_write(p); + PV_UNLOCK(); if (p->valid & p->dirty) continue; } + PV_LOCK(); pmap_remove_all(p); + PV_UNLOCK(); vm_page_free(p); } vm_page_unlock_queues(); diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 77f91ae..0e0bea6 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -386,6 +386,20 @@ vm_page_flag_clear(vm_page_t m, unsigned } void +vm_page_md_flag_set(vm_page_t m, unsigned short bits) +{ + mtx_assert(&pv_lock, MA_OWNED); + m->md_flags |= bits; +} + +void +vm_page_md_flag_clear(vm_page_t m, unsigned short bits) +{ + mtx_assert(&pv_lock, MA_OWNED); + m->md_flags &= ~bits; +} + +void vm_page_busy(vm_page_t m) { @@ -509,10 +523,9 @@ vm_page_sleep(vm_page_t m, const char *m { VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); - if (!mtx_owned(&vm_page_queue_mtx)) - vm_page_lock_queues(); - vm_page_flag_set(m, PG_REFERENCED); - vm_page_unlock_queues(); + PV_LOCK(); + vm_page_md_flag_set(m, PG_REFERENCED); + PV_UNLOCK(); /* * It's possible that while we sleep, the page will get @@ -666,7 +679,8 @@ vm_page_insert(vm_page_t m, vm_object_t * Since we are inserting a new and possibly dirty page, * update the object's OBJ_MIGHTBEDIRTY flag. */ - if (m->flags & PG_WRITEABLE) + /* XXX pv lock? */ + if (m->md_flags & PG_WRITEABLE) vm_object_set_writeable_dirty(object); } @@ -1312,7 +1326,9 @@ vm_page_try_to_cache(vm_page_t m) (m->oflags & VPO_BUSY) || (m->flags & PG_UNMANAGED)) { return (0); } + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); if (m->dirty) return (0); vm_page_cache(m); @@ -1336,7 +1352,9 @@ vm_page_try_to_free(vm_page_t m) (m->oflags & VPO_BUSY) || (m->flags & PG_UNMANAGED)) { return (0); } + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); if (m->dirty) return (0); vm_page_free(m); @@ -1368,7 +1386,9 @@ vm_page_cache(vm_page_t m) * Remove all pmaps and indicate that the page is not * writeable or mapped. */ + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); if (m->dirty != 0) { panic("vm_page_cache: caching a dirty page, pindex: %ld", (long)m->pindex); @@ -1423,8 +1443,10 @@ vm_page_dontneed(vm_page_t m) return; } + PV_LOCK(); if (m->dirty == 0 && pmap_is_modified(m)) vm_page_dirty(m); + PV_UNLOCK(); if (m->dirty || (dnw & 0x0070) == 0) { /* @@ -1582,7 +1604,9 @@ vm_page_set_validclean(vm_page_t m, int #endif m->dirty &= ~pagebits; if (base == 0 && size == PAGE_SIZE) { + PV_LOCK(); pmap_clear_modify(m); + PV_UNLOCK(); m->oflags &= ~VPO_NOSYNC; } } @@ -1611,8 +1635,11 @@ vm_page_set_invalid(vm_page_t m, int bas VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); bits = vm_page_bits(base, size); mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if (m->valid == VM_PAGE_BITS_ALL && bits != 0) + if (m->valid == VM_PAGE_BITS_ALL && bits != 0) { + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); + } m->valid &= ~bits; m->dirty &= ~bits; m->object->generation++; @@ -1690,9 +1717,11 @@ vm_page_is_valid(vm_page_t m, int base, void vm_page_test_dirty(vm_page_t m) { + PV_LOCK(); if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) { vm_page_dirty(m); } + PV_UNLOCK(); } int so_zerocp_fullpage = 0; @@ -1708,7 +1737,9 @@ vm_page_cowfault(vm_page_t m) pindex = m->pindex; retry_alloc: + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); vm_page_remove(m); mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY); if (mnew == NULL) { @@ -1761,7 +1792,9 @@ vm_page_cowsetup(vm_page_t m) mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->cow++; + PV_LOCK(); pmap_remove_write(m); + PV_UNLOCK(); } #include "opt_ddb.h" diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 55fc904..7dc947d 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -110,6 +110,7 @@ struct vm_page { vm_pindex_t pindex; /* offset into object (O,P) */ vm_paddr_t phys_addr; /* physical address of page */ struct md_page md; /* machine dependant stuff */ + u_int md_flags; u_short queue; /* page queue index */ u_short flags, /* see below */ pc; /* page color */ @@ -282,6 +283,10 @@ extern struct mtx vm_page_queue_mtx; #define vm_page_lock_queues() mtx_lock(&vm_page_queue_mtx) #define vm_page_unlock_queues() mtx_unlock(&vm_page_queue_mtx) +extern struct mtx pv_lock; +#define PV_LOCK() mtx_lock(&pv_lock) +#define PV_UNLOCK() mtx_unlock(&pv_lock) + #if PAGE_SIZE == 4096 #define VM_PAGE_BITS_ALL 0xffu #elif PAGE_SIZE == 8192 @@ -306,6 +311,8 @@ extern struct mtx vm_page_queue_mtx; void vm_page_flag_set(vm_page_t m, unsigned short bits); void vm_page_flag_clear(vm_page_t m, unsigned short bits); +void vm_page_md_flag_set(vm_page_t m, unsigned short bits); +void vm_page_md_flag_clear(vm_page_t m, unsigned short bits); void vm_page_busy(vm_page_t m); void vm_page_flash(vm_page_t m); void vm_page_io_start(vm_page_t m); diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 441aec3..38d63f5 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -435,7 +435,9 @@ vm_pageout_flush(vm_page_t *mc, int coun ("vm_pageout_flush: partially invalid page %p index %d/%d", mc[i], i, count)); vm_page_io_start(mc[i]); + PV_LOCK(); pmap_remove_write(mc[i]); + PV_UNLOCK(); } vm_page_unlock_queues(); vm_object_pip_add(object, count); @@ -448,7 +450,7 @@ vm_pageout_flush(vm_page_t *mc, int coun for (i = 0; i < count; i++) { vm_page_t mt = mc[i]; - KASSERT((mt->flags & PG_WRITEABLE) == 0, + KASSERT((mt->md_flags & PG_WRITEABLE) == 0, ("vm_pageout_flush: page %p is not write protected", mt)); switch (pageout_status[i]) { case VM_PAGER_OK: @@ -461,7 +463,9 @@ vm_pageout_flush(vm_page_t *mc, int coun * essentially lose the changes by pretending it * worked. */ + PV_LOCK(); pmap_clear_modify(mt); + PV_UNLOCK(); vm_page_undirty(mt); break; case VM_PAGER_ERROR: @@ -532,6 +536,7 @@ vm_pageout_object_deactivate_pages(pmap, rcount = object->resident_page_count; p = TAILQ_FIRST(&object->memq); vm_page_lock_queues(); + PV_LOCK(); while (p && (rcount-- > 0)) { if (pmap_resident_count(pmap) <= desired) { vm_page_unlock_queues(); @@ -550,17 +555,17 @@ vm_pageout_object_deactivate_pages(pmap, } actcount = pmap_ts_referenced(p); if (actcount) { - vm_page_flag_set(p, PG_REFERENCED); - } else if (p->flags & PG_REFERENCED) { + vm_page_md_flag_set(p, PG_REFERENCED); + } else if (p->md_flags & PG_REFERENCED) { actcount = 1; } if ((p->queue != PQ_ACTIVE) && - (p->flags & PG_REFERENCED)) { + (p->md_flags & PG_REFERENCED)) { vm_page_activate(p); p->act_count += actcount; - vm_page_flag_clear(p, PG_REFERENCED); + vm_page_md_flag_clear(p, PG_REFERENCED); } else if (p->queue == PQ_ACTIVE) { - if ((p->flags & PG_REFERENCED) == 0) { + if ((p->md_flags & PG_REFERENCED) == 0) { p->act_count -= min(p->act_count, ACT_DECLINE); if (!remove_mode && (vm_pageout_algorithm || (p->act_count == 0))) { pmap_remove_all(p); @@ -570,7 +575,7 @@ vm_pageout_object_deactivate_pages(pmap, } } else { vm_page_activate(p); - vm_page_flag_clear(p, PG_REFERENCED); + vm_page_md_flag_clear(p, PG_REFERENCED); if (p->act_count < (ACT_MAX - ACT_ADVANCE)) p->act_count += ACT_ADVANCE; vm_pageq_requeue(p); @@ -580,6 +585,7 @@ vm_pageout_object_deactivate_pages(pmap, } p = next; } + PV_UNLOCK(); vm_page_unlock_queues(); if ((backing_object = object->backing_object) == NULL) goto unlock_return; @@ -737,6 +743,7 @@ vm_pageout_scan(int pass) if (pass) maxlaunder = 10000; vm_page_lock_queues(); + PV_LOCK(); rescan0: addl_page_shortage = addl_page_shortage_init; maxscan = cnt.v_inactive_count; @@ -790,7 +797,7 @@ rescan0: * references. */ if (object->ref_count == 0) { - vm_page_flag_clear(m, PG_REFERENCED); + vm_page_md_flag_clear(m, PG_REFERENCED); pmap_clear_reference(m); /* @@ -802,7 +809,7 @@ rescan0: * level VM system not knowing anything about existing * references. */ - } else if (((m->flags & PG_REFERENCED) == 0) && + } else if (((m->md_flags & PG_REFERENCED) == 0) && (actcount = pmap_ts_referenced(m))) { vm_page_activate(m); VM_OBJECT_UNLOCK(object); @@ -816,8 +823,8 @@ rescan0: * "activation count" higher than normal so that we will less * likely place pages back onto the inactive queue again. */ - if ((m->flags & PG_REFERENCED) != 0) { - vm_page_flag_clear(m, PG_REFERENCED); + if ((m->md_flags & PG_REFERENCED) != 0) { + vm_page_md_flag_clear(m, PG_REFERENCED); actcount = pmap_ts_referenced(m); vm_page_activate(m); VM_OBJECT_UNLOCK(object); @@ -845,7 +852,7 @@ rescan0: * to the page, removing all access will be cheaper * overall. */ - if ((m->flags & PG_WRITEABLE) != 0) + if ((m->md_flags & PG_WRITEABLE) != 0) pmap_remove_all(m); } else { vm_page_dirty(m); @@ -1096,7 +1103,7 @@ unlock_and_continue: */ actcount = 0; if (object->ref_count != 0) { - if (m->flags & PG_REFERENCED) { + if (m->md_flags & PG_REFERENCED) { actcount += 1; } actcount += pmap_ts_referenced(m); @@ -1110,7 +1117,7 @@ unlock_and_continue: /* * Since we have "tested" this bit, we need to clear it now. */ - vm_page_flag_clear(m, PG_REFERENCED); + vm_page_md_flag_clear(m, PG_REFERENCED); /* * Only if an object is currently being used, do we use the @@ -1177,6 +1184,7 @@ unlock_and_continue: if (m == NULL && cache_first_failure == -1) cache_first_failure = cache_cur; } + PV_UNLOCK(); vm_page_unlock_queues(); #if !defined(NO_SWAPPING) /* @@ -1364,8 +1372,8 @@ vm_pageout_page_stats() } actcount = 0; - if (m->flags & PG_REFERENCED) { - vm_page_flag_clear(m, PG_REFERENCED); + if (m->md_flags & PG_REFERENCED) { + vm_page_md_flag_clear(m, PG_REFERENCED); actcount += 1; } diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index c4d6dce..a068310 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -407,7 +407,9 @@ vnode_pager_setsize(vp, nsize) * dealt with this? */ vm_page_lock_queues(); + PV_LOCK(); pmap_remove_all(m); + PV_UNLOCK(); /* * Clear out partial-page dirty bits. This @@ -562,7 +564,9 @@ vnode_pager_input_smlfs(object, m) } sf_buf_free(sf); vm_page_lock_queues(); + PV_LOCK(); pmap_clear_modify(m); + PV_UNLOCK(); vm_page_unlock_queues(); if (error) { return VM_PAGER_ERROR; @@ -633,7 +637,9 @@ vnode_pager_input_old(object, m) VM_OBJECT_LOCK(object); } vm_page_lock_queues(); + PV_LOCK(); pmap_clear_modify(m); + PV_UNLOCK(); vm_page_undirty(m); vm_page_unlock_queues(); if (!error) @@ -947,7 +953,9 @@ vnode_pager_generic_getpages(vp, m, byte */ mt->valid = VM_PAGE_BITS_ALL; vm_page_undirty(mt); /* should be an assert? XXX */ + PV_LOCK(); pmap_clear_modify(mt); + PV_UNLOCK(); } else { /* * Read did not fill up entire page. Since this