Index: sys/kern/vfs_bio.c =================================================================== --- sys/kern/vfs_bio.c (revision 248247) +++ sys/kern/vfs_bio.c (working copy) @@ -3053,12 +3053,12 @@ m = bp->b_pages[i]; KASSERT(m != bogus_page, ("allocbuf: bogus page found")); - while (vm_page_sleep_if_busy(m, TRUE, - "biodep")) - continue; + do { + vm_page_lock(m); + } while (vm_page_sleep_if_busy(m, TRUE, + "biodep")); bp->b_pages[i] = NULL; - vm_page_lock(m); vm_page_unwire(m, 0); vm_page_unlock(m); } @@ -3581,8 +3581,12 @@ if ((m->oflags & VPO_BUSY) != 0) { for (; last_busied < i; last_busied++) vm_page_busy(bp->b_pages[last_busied]); - while ((m->oflags & VPO_BUSY) != 0) + while ((m->oflags & VPO_BUSY) != 0) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); vm_page_sleep(m, "vbpage"); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); + } } } for (i = 0; i < last_busied; i++) Index: sys/kern/subr_uio.c =================================================================== --- sys/kern/subr_uio.c (revision 248247) +++ sys/kern/subr_uio.c (working copy) @@ -107,9 +107,9 @@ VM_OBJECT_WLOCK(uobject); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { + vm_page_lock(user_pg); if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) goto retry; - vm_page_lock(user_pg); pmap_remove_all(user_pg); vm_page_free(user_pg); vm_page_unlock(user_pg); Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c (revision 248247) +++ sys/kern/uipc_shm.c (working copy) @@ -283,7 +283,10 @@ if (m != NULL) { if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(object); vm_page_sleep(m, "shmtrc"); + VM_OBJECT_WLOCK(object); goto retry; } } else if (vm_pager_has_page(object, idx, NULL, NULL)) { Index: sys/fs/tmpfs/tmpfs_vnops.c =================================================================== --- sys/fs/tmpfs/tmpfs_vnops.c (revision 248247) +++ sys/fs/tmpfs/tmpfs_vnops.c (working copy) @@ -513,8 +513,8 @@ offset = addr & PAGE_MASK; tlen = MIN(PAGE_SIZE - offset, len); +lookupvpg: VM_OBJECT_WLOCK(vobj); -lookupvpg: if (((m = vm_page_lookup(vobj, idx)) != NULL) && vm_page_is_valid(m, offset, tlen)) { if ((m->oflags & VPO_BUSY) != 0) { @@ -523,6 +523,8 @@ * that the page daemon is less likely to reclaim it. */ vm_page_reference(m); + vm_page_lock(m); + VM_OBJECT_WUNLOCK(vobj); vm_page_sleep(m, "tmfsmr"); goto lookupvpg; } @@ -542,6 +544,8 @@ * that the page daemon is less likely to reclaim it. */ vm_page_reference(m); + vm_page_lock(m); + VM_OBJECT_WUNLOCK(vobj); vm_page_sleep(m, "tmfsmr"); goto lookupvpg; } @@ -636,8 +640,8 @@ offset = addr & PAGE_MASK; tlen = MIN(PAGE_SIZE - offset, len); +lookupvpg: VM_OBJECT_WLOCK(vobj); -lookupvpg: if (((vpg = vm_page_lookup(vobj, idx)) != NULL) && vm_page_is_valid(vpg, offset, tlen)) { if ((vpg->oflags & VPO_BUSY) != 0) { @@ -646,6 +650,8 @@ * that the page daemon is less likely to reclaim it. */ vm_page_reference(vpg); + vm_page_lock(vpg); + VM_OBJECT_WUNLOCK(vobj); vm_page_sleep(vpg, "tmfsmw"); goto lookupvpg; } Index: sys/fs/tmpfs/tmpfs_subr.c =================================================================== --- sys/fs/tmpfs/tmpfs_subr.c (revision 248247) +++ sys/fs/tmpfs/tmpfs_subr.c (working copy) @@ -1285,7 +1285,10 @@ if (m != NULL) { if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(uobj); vm_page_sleep(m, "tmfssz"); + VM_OBJECT_WLOCK(uobj); goto retry; } MPASS(m->valid == VM_PAGE_BITS_ALL); Index: sys/fs/fuse/fuse_vnops.c =================================================================== --- sys/fs/fuse/fuse_vnops.c (revision 248247) +++ sys/fs/fuse/fuse_vnops.c (working copy) @@ -1868,15 +1868,12 @@ * now tell them that it is ok to use. */ if (!error) { - if (m->oflags & VPO_WANTED) { - fuse_vm_page_lock(m); + fuse_vm_page_lock(m); + if (m->flags & PG_WANTED) vm_page_activate(m); - fuse_vm_page_unlock(m); - } else { - fuse_vm_page_lock(m); + else vm_page_deactivate(m); - fuse_vm_page_unlock(m); - } + fuse_vm_page_unlock(m); vm_page_wakeup(m); } else { fuse_vm_page_lock(m); Index: sys/dev/drm2/i915/i915_gem.c =================================================================== --- sys/dev/drm2/i915/i915_gem.c (revision 248247) +++ sys/dev/drm2/i915/i915_gem.c (working copy) @@ -1421,7 +1421,10 @@ if ((m->flags & VPO_BUSY) != 0) { DRM_UNLOCK(dev); + vm_page_lock(m); + VM_OBJECT_WUNLOCK(vm_obj); vm_page_sleep(m, "915pbs"); + VM_OBJECT_WLOCK(vm_obj); goto retry; } m->valid = VM_PAGE_BITS_ALL; @@ -2309,14 +2312,17 @@ if (devobj != NULL) { page_count = OFF_TO_IDX(obj->base.size); +retry: VM_OBJECT_WLOCK(devobj); -retry: for (i = 0; i < page_count; i++) { m = vm_page_lookup(devobj, i); if (m == NULL) continue; + vm_page_lock(m); + VM_OBJECT_WUNLOCK(devobj); if (vm_page_sleep_if_busy(m, true, "915unm")) goto retry; + vm_page_unlock(m); cdev_pager_free_page(devobj, m); } VM_OBJECT_WUNLOCK(devobj); Index: sys/dev/drm2/ttm/ttm_bo_vm.c =================================================================== --- sys/dev/drm2/ttm/ttm_bo_vm.c (revision 248247) +++ sys/dev/drm2/ttm/ttm_bo_vm.c (working copy) @@ -215,7 +215,10 @@ VM_OBJECT_WLOCK(vm_obj); if ((m->flags & VPO_BUSY) != 0) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(vm_obj); vm_page_sleep(m, "ttmpbs"); + VM_OBJECT_WLOCK(vm_obj); ttm_mem_io_unlock(man); ttm_bo_unreserve(bo); goto retry; Index: sys/vm/phys_pager.c =================================================================== --- sys/vm/phys_pager.c (revision 248247) +++ sys/vm/phys_pager.c (working copy) @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -152,9 +153,11 @@ KASSERT(m[i]->dirty == 0, ("phys_pager_getpages: dirty page %p", m[i])); /* The requested page must remain busy, the others not. */ - if (i == reqpage) + if (i == reqpage) { + vm_page_lock(m[i]); vm_page_flash(m[i]); - else + vm_page_unlock(m[i]); + } else vm_page_wakeup(m[i]); } return (VM_PAGER_OK); Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c (revision 248247) +++ sys/vm/swap_pager.c (working copy) @@ -1211,13 +1211,15 @@ */ VM_OBJECT_WLOCK(object); while ((mreq->oflags & VPO_SWAPINPROG) != 0) { - mreq->oflags |= VPO_WANTED; PCPU_INC(cnt.v_intrans); - if (VM_OBJECT_SLEEP(object, mreq, PSWP, "swread", hz * 20)) { + vm_page_lock(mreq); + VM_OBJECT_WUNLOCK(object); + if (vm_page_sleep_onpage(mreq, PSWP, "swread", hz * 20)) { printf( "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount); } + VM_OBJECT_WLOCK(object); } /* @@ -1531,8 +1533,11 @@ m->valid = 0; if (i != bp->b_pager.pg_reqpage) swp_pager_free_nrpage(m); - else + else { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } /* * If i == bp->b_pager.pg_reqpage, do not wake * the page up. The caller needs to. @@ -1585,8 +1590,11 @@ vm_page_deactivate(m); vm_page_unlock(m); vm_page_wakeup(m); - } else + } else { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } } else { /* * For write success, clear the dirty Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c (revision 248247) +++ sys/vm/vm_object.c (working copy) @@ -391,7 +391,7 @@ VM_OBJECT_ASSERT_WLOCKED(object); while (object->paging_in_progress) { object->flags |= OBJ_PIPWNT; - VM_OBJECT_SLEEP(object, object, PVM, waitid, 0); + VM_OBJECT_SLEEP(object, object, PVM, waitid); } } @@ -584,7 +584,7 @@ VM_OBJECT_WUNLOCK(robject); object->flags |= OBJ_PIPWNT; VM_OBJECT_SLEEP(object, object, - PDROP | PVM, "objde2", 0); + PDROP | PVM, "objde2"); VM_OBJECT_WLOCK(robject); temp = robject->backing_object; if (object == temp) { @@ -844,6 +844,7 @@ np = TAILQ_NEXT(p, listq); if (p->valid == 0) continue; + vm_page_lock(p); if (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { if (object->generation != curgeneration) { if ((flags & OBJPC_SYNC) != 0) @@ -854,6 +855,7 @@ np = vm_page_find_least(object, pi); continue; } + vm_page_unlock(p); if (!vm_object_page_remove_write(p, flags, &clearobjflags)) continue; @@ -1138,11 +1140,10 @@ */ vm_page_aflag_set(m, PGA_REFERENCED); } - vm_page_unlock(m); if (object != tobject) VM_OBJECT_WUNLOCK(object); - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(tobject, m, PDROP | PVM, "madvpo", 0); + VM_OBJECT_WUNLOCK(tobject); + vm_page_sleep(m, "madvpo"); VM_OBJECT_WLOCK(object); goto relookup; } @@ -1339,8 +1340,10 @@ */ if ((m->oflags & VPO_BUSY) || m->busy) { VM_OBJECT_WUNLOCK(new_object); - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0); + vm_page_lock(m); + VM_OBJECT_WUNLOCK(orig_object); + vm_page_sleep(m, "spltwt"); + VM_OBJECT_WLOCK(orig_object); VM_OBJECT_WLOCK(new_object); goto retry; } @@ -1497,9 +1500,9 @@ } else if (op & OBSC_COLLAPSE_WAIT) { if ((p->oflags & VPO_BUSY) || p->busy) { VM_OBJECT_WUNLOCK(object); - p->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(backing_object, p, - PDROP | PVM, "vmocol", 0); + vm_page_lock(p); + VM_OBJECT_WUNLOCK(backing_object); + vm_page_sleep(p, "vmocol"); VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(backing_object); /* Index: sys/vm/vm_fault.c =================================================================== --- sys/vm/vm_fault.c (revision 248247) +++ sys/vm/vm_fault.c (working copy) @@ -379,8 +379,10 @@ unlock_map(&fs); if (fs.m == vm_page_lookup(fs.object, fs.pindex)) { - vm_page_sleep_if_busy(fs.m, TRUE, - "vmpfw"); + vm_page_lock(fs.m); + if (!vm_page_sleep_if_busy(fs.m, TRUE, + "vmpfw")) + vm_page_unlock(fs.m); } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h (revision 248247) +++ sys/vm/vm_object.h (working copy) @@ -216,8 +216,8 @@ rw_rlock(&(object)->lock) #define VM_OBJECT_RUNLOCK(object) \ rw_runlock(&(object)->lock) -#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg, timo) \ - rw_sleep((wchan), &(object)->lock, (pri), (wmesg), (timo)) +#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg) \ + rw_sleep((wchan), &(object)->lock, (pri), (wmesg), 0) #define VM_OBJECT_TRYRLOCK(object) \ rw_try_rlock(&(object)->lock) #define VM_OBJECT_TRYWLOCK(object) \ Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c (revision 248247) +++ sys/vm/vm_page.c (working copy) @@ -485,9 +485,9 @@ vm_page_flash(vm_page_t m) { - VM_OBJECT_ASSERT_WLOCKED(m->object); - if (m->oflags & VPO_WANTED) { - m->oflags &= ~VPO_WANTED; + vm_page_lock_assert(m, MA_OWNED); + if (m->flags & PG_WANTED) { + m->flags &= ~PG_WANTED; wakeup(m); } } @@ -506,7 +506,9 @@ VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!")); m->oflags &= ~VPO_BUSY; + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); } void @@ -524,8 +526,11 @@ VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(m->busy > 0, ("vm_page_io_finish: page %p is not busy", m)); m->busy--; - if (m->busy == 0) + if (m->busy == 0) { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } } /* @@ -719,15 +724,12 @@ * deactivating the page is usually the best choice, * unless the page is wanted by another thread. */ - if (m->oflags & VPO_WANTED) { - vm_page_lock(m); + vm_page_lock(m); + if (m->flags & PG_WANTED) vm_page_activate(m); - vm_page_unlock(m); - } else { - vm_page_lock(m); + else vm_page_deactivate(m); - vm_page_unlock(m); - } + vm_page_unlock(m); vm_page_wakeup(m); } else { /* @@ -743,29 +745,43 @@ } /* - * vm_page_sleep: + * vm_page_sleep_if_busy: * - * Sleep and release the page lock. + * Sleep and release the page queues lock if VPO_BUSY is set or, + * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the + * thread slept and the page queues lock was released. + * Otherwise, retains the page queues lock and returns FALSE. * - * The object containing the given page must be locked. + * The given page and object containing it must be locked. */ -void -vm_page_sleep(vm_page_t m, const char *msg) +int +vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg) { VM_OBJECT_ASSERT_WLOCKED(m->object); - if (mtx_owned(vm_page_lockptr(m))) - vm_page_unlock(m); + if ((m->oflags & VPO_BUSY) || (also_m_busy && m->busy)) { + VM_OBJECT_WUNLOCK(m->object); + vm_page_sleep(m, msg); + VM_OBJECT_WLOCK(m->object); + return (TRUE); + } + return (FALSE); +} - /* - * It's possible that while we sleep, the page will get - * unbusied and freed. If we are holding the object - * lock, we will assume we hold a reference to the object - * such that even if m->object changes, we can re-lock - * it. - */ - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(m->object, m, PVM, msg, 0); +/* + * vm_page_sleep_onpage: + * + * Sleep and release the page lock, using the page pointer as wchan. + * + * The given page must be locked. + */ +int +vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo) +{ + + vm_page_lock_assert(m, MA_OWNED); + m->flags |= PG_WANTED; + return (msleep(m, vm_page_lockptr(m), pri | PDROP, wmesg, timo)); } /* @@ -2320,6 +2336,7 @@ * likely to reclaim it. */ vm_page_aflag_set(m, PGA_REFERENCED); + vm_page_lock(m); vm_page_sleep(m, "pgrbwt"); goto retrylookup; } else { Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h (revision 248247) +++ sys/vm/vm_page.h (working copy) @@ -166,7 +166,7 @@ * */ #define VPO_BUSY 0x01 /* page is in transit */ -#define VPO_WANTED 0x02 /* someone is waiting for page */ +#define VPO_UNUSED02 0x02 /* --available-- */ #define VPO_UNMANAGED 0x04 /* no PV management for page */ #define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */ #define VPO_NOSYNC 0x10 /* do not collect for syncer */ @@ -270,6 +270,7 @@ #define PG_WINATCFLS 0x0040 /* flush dirty page on inactive q */ #define PG_NODUMP 0x0080 /* don't include this page in a dump */ #define PG_UNHOLDFREE 0x0100 /* delayed free of a held page */ +#define PG_WANTED 0x0200 /* someone is waiting for page */ /* * Misc constants. @@ -401,7 +402,8 @@ void vm_page_requeue(vm_page_t m); void vm_page_requeue_locked(vm_page_t m); void vm_page_set_valid_range(vm_page_t m, int base, int size); -void vm_page_sleep(vm_page_t m, const char *msg); +int vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg); +int vm_page_sleep_onpage(vm_page_t m, int pri, const char *msg, int timo); vm_offset_t vm_page_startup(vm_offset_t vaddr); void vm_page_unhold_pages(vm_page_t *ma, int count); void vm_page_unwire (vm_page_t, int); @@ -527,40 +529,34 @@ } /* - * vm_page_remque: + * vm_page_sleep: * - * If the given page is in a page queue, then remove it from that page - * queue. + * Convenience wrapper around vm_page_sleep_onpage(), passing + * PVM priority and 0 timeout values. Unlocks the page upon return. * * The page must be locked. */ -static inline void -vm_page_remque(vm_page_t m) +static __inline void +vm_page_sleep(vm_page_t m, const char *msg) { - if (m->queue != PQ_NONE) - vm_page_dequeue(m); + vm_page_sleep_onpage(m, PVM, msg, 0); } /* - * vm_page_sleep_if_busy: + * vm_page_remque: * - * Sleep and release the page queues lock if VPO_BUSY is set or, - * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the - * thread slept and the page queues lock was released. - * Otherwise, retains the page queues lock and returns FALSE. + * If the given page is in a page queue, then remove it from that page + * queue. * - * The object containing the given page must be locked. + * The page must be locked. */ -static __inline int -vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg) +static inline void +vm_page_remque(vm_page_t m) { - if ((m->oflags & VPO_BUSY) || (also_m_busy && m->busy)) { - vm_page_sleep(m, msg); - return (TRUE); - } - return (FALSE); + if (m->queue != PQ_NONE) + vm_page_dequeue(m); } /* Index: sys/vm/vnode_pager.c =================================================================== --- sys/vm/vnode_pager.c (revision 248247) +++ sys/vm/vnode_pager.c (working copy) @@ -117,7 +117,7 @@ } VOP_UNLOCK(vp, 0); vm_object_set_flag(object, OBJ_DISCONNECTWNT); - VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead"); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); } @@ -211,7 +211,7 @@ if ((object->flags & OBJ_DEAD) == 0) break; vm_object_set_flag(object, OBJ_DISCONNECTWNT); - VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead"); } if (vp->v_usecount == 0) Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (revision 248247) +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (working copy) @@ -342,7 +342,10 @@ * likely to reclaim it. */ vm_page_reference(pp); + vm_page_lock(pp); + zfs_vmobject_wunlock(obj); vm_page_sleep(pp, "zfsmwb"); + zfs_vmobject_wlock(obj); continue; } } else { @@ -390,7 +393,10 @@ * likely to reclaim it. */ vm_page_reference(pp); + vm_page_lock(pp); + zfs_vmobject_wunlock(obj); vm_page_sleep(pp, "zfsmwb"); + zfs_vmobject_wlock(obj); continue; }