Index: sys/fs/tmpfs/tmpfs_vnops.c =================================================================== --- sys/fs/tmpfs/tmpfs_vnops.c (revision 248205) +++ sys/fs/tmpfs/tmpfs_vnops.c (working copy) @@ -523,6 +523,7 @@ lookupvpg: * that the page daemon is less likely to reclaim it. */ vm_page_reference(m); + vm_page_lock(m); vm_page_sleep(m, "tmfsmr"); goto lookupvpg; } @@ -542,6 +543,7 @@ lookupvpg: * that the page daemon is less likely to reclaim it. */ vm_page_reference(m); + vm_page_lock(m); vm_page_sleep(m, "tmfsmr"); goto lookupvpg; } @@ -646,6 +648,7 @@ lookupvpg: * that the page daemon is less likely to reclaim it. */ vm_page_reference(vpg); + vm_page_lock(vpg); vm_page_sleep(vpg, "tmfsmw"); goto lookupvpg; } Index: sys/fs/tmpfs/tmpfs_subr.c =================================================================== --- sys/fs/tmpfs/tmpfs_subr.c (revision 248205) +++ sys/fs/tmpfs/tmpfs_subr.c (working copy) @@ -1285,6 +1285,7 @@ retry: if (m != NULL) { if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) { + vm_page_lock(m); vm_page_sleep(m, "tmfssz"); goto retry; } Index: sys/fs/fuse/fuse_vnops.c =================================================================== --- sys/fs/fuse/fuse_vnops.c (revision 248205) +++ sys/fs/fuse/fuse_vnops.c (working copy) @@ -1868,15 +1868,12 @@ fuse_vnop_getpages(struct vop_getpages_args *ap) * now tell them that it is ok to use. */ if (!error) { - if (m->oflags & VPO_WANTED) { - fuse_vm_page_lock(m); + fuse_vm_page_lock(m); + if (m->flags & PG_WANTED) vm_page_activate(m); - fuse_vm_page_unlock(m); - } else { - fuse_vm_page_lock(m); + else vm_page_deactivate(m); - fuse_vm_page_unlock(m); - } + fuse_vm_page_unlock(m); vm_page_wakeup(m); } else { fuse_vm_page_lock(m); Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c (revision 248205) +++ sys/vm/swap_pager.c (working copy) @@ -1211,13 +1211,17 @@ swap_pager_getpages(vm_object_t object, vm_page_t */ VM_OBJECT_WLOCK(object); while ((mreq->oflags & VPO_SWAPINPROG) != 0) { - mreq->oflags |= VPO_WANTED; PCPU_INC(cnt.v_intrans); - if (VM_OBJECT_SLEEP(object, mreq, PSWP, "swread", hz * 20)) { + vm_page_lock(mreq); + mreq->flags |= PG_WANTED; + VM_OBJECT_WUNLOCK(object); + if (vm_page_sleep_onpage(mreq, PSWP | PDROP, "swread", + hz * 20)) { printf( "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount); } + VM_OBJECT_WLOCK(object); } /* @@ -1531,8 +1535,11 @@ swp_pager_async_iodone(struct buf *bp) m->valid = 0; if (i != bp->b_pager.pg_reqpage) swp_pager_free_nrpage(m); - else + else { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } /* * If i == bp->b_pager.pg_reqpage, do not wake * the page up. The caller needs to. @@ -1585,8 +1592,11 @@ swp_pager_async_iodone(struct buf *bp) vm_page_deactivate(m); vm_page_unlock(m); vm_page_wakeup(m); - } else + } else { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } } else { /* * For write success, clear the dirty Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c (revision 248205) +++ sys/vm/vm_object.c (working copy) @@ -391,7 +391,7 @@ vm_object_pip_wait(vm_object_t object, char *waiti VM_OBJECT_ASSERT_WLOCKED(object); while (object->paging_in_progress) { object->flags |= OBJ_PIPWNT; - VM_OBJECT_SLEEP(object, object, PVM, waitid, 0); + VM_OBJECT_SLEEP(object, object, PVM, waitid); } } @@ -584,7 +584,7 @@ retry: VM_OBJECT_WUNLOCK(robject); object->flags |= OBJ_PIPWNT; VM_OBJECT_SLEEP(object, object, - PDROP | PVM, "objde2", 0); + PDROP | PVM, "objde2"); VM_OBJECT_WLOCK(robject); temp = robject->backing_object; if (object == temp) { @@ -844,6 +844,7 @@ rescan: np = TAILQ_NEXT(p, listq); if (p->valid == 0) continue; + vm_page_lock(p); if (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { if (object->generation != curgeneration) { if ((flags & OBJPC_SYNC) != 0) @@ -854,6 +855,7 @@ rescan: np = vm_page_find_least(object, pi); continue; } + vm_page_unlock(p); if (!vm_object_page_remove_write(p, flags, &clearobjflags)) continue; @@ -1138,11 +1140,11 @@ shadowlookup: */ vm_page_aflag_set(m, PGA_REFERENCED); } - vm_page_unlock(m); if (object != tobject) VM_OBJECT_WUNLOCK(object); - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(tobject, m, PDROP | PVM, "madvpo", 0); + VM_OBJECT_WUNLOCK(tobject); + m->flags |= PG_WANTED; + vm_page_sleep_onpage(m, PDROP | PVM, "madvpo", 0); VM_OBJECT_WLOCK(object); goto relookup; } @@ -1339,8 +1341,11 @@ retry: */ if ((m->oflags & VPO_BUSY) || m->busy) { VM_OBJECT_WUNLOCK(new_object); - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0); + vm_page_lock(m); + m->flags |= PG_WANTED; + VM_OBJECT_WUNLOCK(orig_object); + vm_page_sleep_onpage(m, PVM | PDROP, "spltwt", 0); + VM_OBJECT_WLOCK(orig_object); VM_OBJECT_WLOCK(new_object); goto retry; } @@ -1497,9 +1502,11 @@ vm_object_backing_scan(vm_object_t object, int op) } else if (op & OBSC_COLLAPSE_WAIT) { if ((p->oflags & VPO_BUSY) || p->busy) { VM_OBJECT_WUNLOCK(object); - p->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(backing_object, p, - PDROP | PVM, "vmocol", 0); + vm_page_lock(p); + p->flags |= PG_WANTED; + VM_OBJECT_WUNLOCK(backing_object); + vm_page_sleep_onpage(p, PDROP | PVM, + "vmocol", 0); VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(backing_object); /* Index: sys/vm/vm_fault.c =================================================================== --- sys/vm/vm_fault.c (revision 248205) +++ sys/vm/vm_fault.c (working copy) @@ -379,8 +379,10 @@ RetryFault:; unlock_map(&fs); if (fs.m == vm_page_lookup(fs.object, fs.pindex)) { - vm_page_sleep_if_busy(fs.m, TRUE, - "vmpfw"); + vm_page_lock(fs.m); + if (!vm_page_sleep_if_busy(fs.m, TRUE, + "vmpfw")) + vm_page_unlock(fs.m); } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h (revision 248205) +++ sys/vm/vm_object.h (working copy) @@ -216,8 +216,8 @@ extern struct vm_object kmem_object_store; rw_rlock(&(object)->lock) #define VM_OBJECT_RUNLOCK(object) \ rw_runlock(&(object)->lock) -#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg, timo) \ - rw_sleep((wchan), &(object)->lock, (pri), (wmesg), (timo)) +#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg) \ + rw_sleep((wchan), &(object)->lock, (pri), (wmesg)) #define VM_OBJECT_TRYRLOCK(object) \ rw_try_rlock(&(object)->lock) #define VM_OBJECT_TRYWLOCK(object) \ Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c (revision 248206) +++ sys/vm/vm_page.c (working copy) @@ -486,8 +486,9 @@ vm_page_flash(vm_page_t m) { VM_OBJECT_ASSERT_WLOCKED(m->object); - if (m->oflags & VPO_WANTED) { - m->oflags &= ~VPO_WANTED; + vm_page_lock_assert(m, MA_OWNED); + if (m->flags & PG_WANTED) { + m->flags &= ~PG_WANTED; wakeup(m); } } @@ -506,7 +507,9 @@ vm_page_wakeup(vm_page_t m) VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!")); m->oflags &= ~VPO_BUSY; + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); } void @@ -524,8 +527,11 @@ vm_page_io_finish(vm_page_t m) VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(m->busy > 0, ("vm_page_io_finish: page %p is not busy", m)); m->busy--; - if (m->busy == 0) + if (m->busy == 0) { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } } /* @@ -719,15 +725,12 @@ vm_page_readahead_finish(vm_page_t m) * deactivating the page is usually the best choice, * unless the page is wanted by another thread. */ - if (m->oflags & VPO_WANTED) { - vm_page_lock(m); + vm_page_lock(m); + if (m->flags & PG_WANTED) vm_page_activate(m); - vm_page_unlock(m); - } else { - vm_page_lock(m); + else vm_page_deactivate(m); - vm_page_unlock(m); - } + vm_page_unlock(m); vm_page_wakeup(m); } else { /* @@ -747,15 +750,14 @@ vm_page_readahead_finish(vm_page_t m) * * Sleep and release the page lock. * - * The object containing the given page must be locked. + * The given page and the object containing it must be locked. */ void vm_page_sleep(vm_page_t m, const char *msg) { VM_OBJECT_ASSERT_WLOCKED(m->object); - if (mtx_owned(vm_page_lockptr(m))) - vm_page_unlock(m); + vm_page_lock_assert(m, MA_OWNED); /* * It's possible that while we sleep, the page will get @@ -764,8 +766,10 @@ vm_page_sleep(vm_page_t m, const char *msg) * such that even if m->object changes, we can re-lock * it. */ - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(m->object, m, PVM, msg, 0); + m->flags |= PG_WANTED; + VM_OBJECT_WUNLOCKED(m->object); + vm_page_sleep_onpage(m, PVM | PDROP, msg, 0); + VM_OBJECT_WLOCKED(m->object); } /* @@ -2320,6 +2324,7 @@ retrylookup: * likely to reclaim it. */ vm_page_aflag_set(m, PGA_REFERENCED); + vm_page_lock(m); vm_page_sleep(m, "pgrbwt"); goto retrylookup; } else { @@ -2691,6 +2696,14 @@ vm_page_trylock_KBI(vm_page_t m, const char *file, return (mtx_trylock_flags_(vm_page_lockptr(m), 0, file, line)); } +int +vm_page_sleep_onpage_KBI(vm_page_t m, int pri, const char *wmsg, int tm, + const char *file, int line) +{ + + return (msleep(m, vm_page_lockptr(m), pri, wmsg, tm)); +} + #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line) Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h (revision 248208) +++ sys/vm/vm_page.h (working copy) @@ -166,7 +166,7 @@ struct vm_page { * */ #define VPO_BUSY 0x01 /* page is in transit */ -#define VPO_WANTED 0x02 /* someone is waiting for page */ +#define VPO_UNUSED02 0x02 /* --available-- */ #define VPO_UNMANAGED 0x04 /* no PV management for page */ #define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */ #define VPO_NOSYNC 0x10 /* do not collect for syncer */ @@ -270,6 +270,7 @@ extern struct mtx_padalign pa_lock[]; #define PG_WINATCFLS 0x0040 /* flush dirty page on inactive q */ #define PG_NODUMP 0x0080 /* don't include this page in a dump */ #define PG_UNHOLDFREE 0x0100 /* delayed free of a held page */ +#define PG_WANTED 0x0200 /* someone is waiting for page */ /* * Misc constants. @@ -564,6 +565,18 @@ vm_page_sleep_if_busy(vm_page_t m, int also_m_busy } /* + * vm_page_sleep_onpage: + * + * XXX + */ +static __inline int +vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo) +{ + + return (msleep(m, vm_page_lockptr(m), pri, wmesg, timo)); +} + +/* * vm_page_undirty: * * Set page to not be dirty. Note: does not clear pmap modify bits Index: sys/vm/phys_pager.c =================================================================== --- sys/vm/phys_pager.c (revision 248205) +++ sys/vm/phys_pager.c (working copy) @@ -152,9 +152,11 @@ phys_pager_getpages(vm_object_t object, vm_page_t KASSERT(m[i]->dirty == 0, ("phys_pager_getpages: dirty page %p", m[i])); /* The requested page must remain busy, the others not. */ - if (i == reqpage) + if (i == reqpage) { + vm_page_lock(m[i]); vm_page_flash(m[i]); - else + vm_page_unlock(m[i]); + } else vm_page_wakeup(m[i]); } return (VM_PAGER_OK); Index: sys/vm/vnode_pager.c =================================================================== --- sys/vm/vnode_pager.c (revision 248205) +++ sys/vm/vnode_pager.c (working copy) @@ -117,7 +117,7 @@ vnode_create_vobject(struct vnode *vp, off_t isize } VOP_UNLOCK(vp, 0); vm_object_set_flag(object, OBJ_DISCONNECTWNT); - VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead"); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); } @@ -211,7 +211,7 @@ retry: if ((object->flags & OBJ_DEAD) == 0) break; vm_object_set_flag(object, OBJ_DISCONNECTWNT); - VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead"); } if (vp->v_usecount == 0) Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (revision 248205) +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (working copy) @@ -342,6 +342,7 @@ page_busy(vnode_t *vp, int64_t start, int64_t off, * likely to reclaim it. */ vm_page_reference(pp); + vm_page_lock(pp); vm_page_sleep(pp, "zfsmwb"); continue; } @@ -390,6 +391,7 @@ page_hold(vnode_t *vp, int64_t start) * likely to reclaim it. */ vm_page_reference(pp); + vm_page_lock(pp); vm_page_sleep(pp, "zfsmwb"); continue; } Index: sys/kern/vfs_bio.c =================================================================== --- sys/kern/vfs_bio.c (revision 248205) +++ sys/kern/vfs_bio.c (working copy) @@ -3053,6 +3053,7 @@ allocbuf(struct buf *bp, int size) m = bp->b_pages[i]; KASSERT(m != bogus_page, ("allocbuf: bogus page found")); + vm_page_lock(m); while (vm_page_sleep_if_busy(m, TRUE, "biodep")) continue; @@ -3581,8 +3582,10 @@ vfs_drain_busy_pages(struct buf *bp) if ((m->oflags & VPO_BUSY) != 0) { for (; last_busied < i; last_busied++) vm_page_busy(bp->b_pages[last_busied]); - while ((m->oflags & VPO_BUSY) != 0) + while ((m->oflags & VPO_BUSY) != 0) { + vm_page_lock(m); vm_page_sleep(m, "vbpage"); + } } } for (i = 0; i < last_busied; i++) Index: sys/kern/subr_uio.c =================================================================== --- sys/kern/subr_uio.c (revision 248205) +++ sys/kern/subr_uio.c (working copy) @@ -107,9 +107,9 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_o VM_OBJECT_WLOCK(uobject); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { + vm_page_lock(user_pg); if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) goto retry; - vm_page_lock(user_pg); pmap_remove_all(user_pg); vm_page_free(user_pg); vm_page_unlock(user_pg); Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c (revision 248205) +++ sys/kern/uipc_shm.c (working copy) @@ -283,6 +283,7 @@ retry: if (m != NULL) { if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) { + vm_page_lock(m); vm_page_sleep(m, "shmtrc"); goto retry; } Index: sys/dev/drm2/i915/i915_gem.c =================================================================== --- sys/dev/drm2/i915/i915_gem.c (revision 248205) +++ sys/dev/drm2/i915/i915_gem.c (working copy) @@ -1421,6 +1421,7 @@ unlocked_vmobj: if ((m->flags & VPO_BUSY) != 0) { DRM_UNLOCK(dev); + vm_page_lock(m); vm_page_sleep(m, "915pbs"); goto retry; } @@ -2315,8 +2316,10 @@ retry: m = vm_page_lookup(devobj, i); if (m == NULL) continue; + vm_page_lock(m); if (vm_page_sleep_if_busy(m, true, "915unm")) goto retry; + vm_page_unlock(m); cdev_pager_free_page(devobj, m); } VM_OBJECT_WUNLOCK(devobj); Index: sys/dev/drm2/ttm/ttm_bo_vm.c =================================================================== --- sys/dev/drm2/ttm/ttm_bo_vm.c (revision 248205) +++ sys/dev/drm2/ttm/ttm_bo_vm.c (working copy) @@ -215,6 +215,7 @@ reserve: VM_OBJECT_WLOCK(vm_obj); if ((m->flags & VPO_BUSY) != 0) { + vm_page_lock(m); vm_page_sleep(m, "ttmpbs"); ttm_mem_io_unlock(man); ttm_bo_unreserve(bo);