Index: sys/fs/tmpfs/tmpfs_subr.c =================================================================== --- sys/fs/tmpfs/tmpfs_subr.c (revision 253430) +++ sys/fs/tmpfs/tmpfs_subr.c (working copy) @@ -1331,7 +1331,8 @@ tmpfs_reg_resize(struct vnode *vp, off_t newsize, retry: m = vm_page_lookup(uobj, idx); if (m != NULL) { - if (vm_page_sleep_if_busy(m, "tmfssz")) + if (vm_page_sleep_if_busy(m, "tmfssz", + VM_ALLOC_NOBUSY, FALSE)) goto retry; MPASS(m->valid == VM_PAGE_BITS_ALL); } else if (vm_pager_has_page(uobj, idx, NULL, NULL)) { Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c (revision 253430) +++ sys/vm/vm_object.c (working copy) @@ -870,7 +870,8 @@ rescan: np = TAILQ_NEXT(p, listq); if (p->valid == 0) continue; - if (vm_page_sleep_if_busy(p, "vpcwai")) { + if (vm_page_sleep_if_busy(p, "vpcwai", VM_ALLOC_NOBUSY, + FALSE)) { if (object->generation != curgeneration) { if ((flags & OBJPC_SYNC) != 0) goto rescan; Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h (revision 253430) +++ sys/vm/vm_object.h (working copy) @@ -226,6 +226,8 @@ extern struct vm_object kmem_object_store; rw_assert(&(object)->lock, RA_WLOCKED) #define VM_OBJECT_LOCK_DOWNGRADE(object) \ rw_downgrade(&(object)->lock) +#define VM_OBJECT_LOCK_TRYUPGRADE(object) \ + rw_try_upgrade(&(object)->lock) #define VM_OBJECT_RLOCK(object) \ rw_rlock(&(object)->lock) #define VM_OBJECT_RUNLOCK(object) \ @@ -238,6 +240,8 @@ extern struct vm_object kmem_object_store; rw_try_wlock(&(object)->lock) #define VM_OBJECT_WLOCK(object) \ rw_wlock(&(object)->lock) +#define VM_OBJECT_WOWNED(object) \ + rw_wowned(&(object)->lock) #define VM_OBJECT_WUNLOCK(object) \ rw_wunlock(&(object)->lock) Index: sys/vm/vm_fault.c =================================================================== --- sys/vm/vm_fault.c (revision 253430) +++ sys/vm/vm_fault.c (working copy) @@ -392,7 +392,8 @@ RetryFault:; unlock_map(&fs); if (fs.m == vm_page_lookup(fs.object, fs.pindex)) { - vm_page_sleep_if_busy(fs.m, "vmpfw"); + vm_page_sleep_if_busy(fs.m, "vmpfw", + VM_ALLOC_NOBUSY, FALSE); } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c (revision 253430) +++ sys/vm/vm_page.c (working copy) @@ -869,22 +869,42 @@ vm_page_readahead_finish(vm_page_t m) * be locked. */ int -vm_page_sleep_if_busy(vm_page_t m, const char *msg) +vm_page_sleep_if_busy(vm_page_t m, const char *msg, int busyflags, + boolean_t pref) { vm_object_t obj; + int cond; vm_page_lock_assert(m, MA_NOTOWNED); - VM_OBJECT_ASSERT_WLOCKED(m->object); - if (vm_page_busy_locked(m)) { + /* + * The page-specific object must be cached because page + * identity can change during the sleep, causing the + * re-lock of a different object. + * It is assumed that a reference to the object is already + * held by the callers. + */ + obj = m->object; + VM_OBJECT_ASSERT_LOCKED(obj); + KASSERT((busyflags & VM_ALLOC_NOBUSY) == 0 || VM_OBJECT_WOWNED(obj), + ("vm_page_sleep_if_busy: VM_ALLOC_NOBUSY with read object lock")); + + if ((busyflags & VM_ALLOC_NOBUSY) != 0) { + cond = (busyflags & VM_ALLOC_IGN_RBUSY) != 0 ? + vm_page_busy_wlocked(m) : vm_page_busy_locked(m); + } else if ((busyflags & VM_ALLOC_RBUSY) != 0) + cond = !vm_page_busy_tryrlock(m); + else + cond = !vm_page_busy_trywlock(m); + if (cond) { + /* - * The page-specific object must be cached because page - * identity can change during the sleep, causing the - * re-lock of a different object. - * It is assumed that a reference to the object is already - * held by the callers. + * Some consumers may want to reference the page before + * unlocking and sleeping so that the page daemon is less + * likely to reclaim it. */ - obj = m->object; + if (pref) + vm_page_aflag_set(m, PGA_REFERENCED); vm_page_lock(m); VM_OBJECT_WUNLOCK(obj); vm_page_busy_sleep(m, msg); @@ -2500,44 +2520,44 @@ vm_page_t vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) { vm_page_t m; - int sleep; + int origwlock; VM_OBJECT_ASSERT_WLOCKED(object); + origwlock = VM_OBJECT_WOWNED(object); KASSERT((allocflags & VM_ALLOC_RETRY) != 0, ("vm_page_grab: VM_ALLOC_RETRY is required")); KASSERT((allocflags & VM_ALLOC_RBUSY) == 0 || (allocflags & VM_ALLOC_IGN_RBUSY) != 0, ("vm_page_grab: VM_ALLOC_RBUSY/VM_ALLOC_IGN_RBUSY mismatch")); + KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || origwlock != 0, + ("vm_page_grab: VM_ALLOC_NOBUSY with object read lock")); retrylookup: if ((m = vm_page_lookup(object, pindex)) != NULL) { - sleep = (allocflags & VM_ALLOC_IGN_RBUSY) != 0 ? - vm_page_busy_wlocked(m) : vm_page_busy_locked(m); - if (sleep) { - /* - * Reference the page before unlocking and - * sleeping so that the page daemon is less - * likely to reclaim it. - */ - vm_page_aflag_set(m, PGA_REFERENCED); - vm_page_lock(m); - VM_OBJECT_WUNLOCK(object); - vm_page_busy_sleep(m, "pgrbwt"); - VM_OBJECT_WLOCK(object); + if (vm_page_sleep_if_busy(m, "pgrbwt", allocflags & + (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY | VM_ALLOC_IGN_RBUSY), + TRUE)) goto retrylookup; - } else { + else { if ((allocflags & VM_ALLOC_WIRED) != 0) { vm_page_lock(m); vm_page_wire(m); vm_page_unlock(m); } - if ((allocflags & - (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY)) == 0) - vm_page_busy_wlock(m); - if ((allocflags & VM_ALLOC_RBUSY) != 0) - vm_page_busy_rlock(m); + + /* + * If the lock state changed in the meanwhile, + * unwind back. + */ + if (VM_OBJECT_WOWNED(object) != origwlock) + VM_OBJECT_LOCK_DOWNGRADE(object); return (m); } } + if (!VM_OBJECT_WOWNED(object) && !VM_OBJECT_LOCK_TRYUPGRADE(object)) { + VM_OBJECT_RUNLOCK(object); + VM_OBJECT_WLOCK(object); + goto retrylookup; + } m = vm_page_alloc(object, pindex, allocflags & ~(VM_ALLOC_RETRY | VM_ALLOC_IGN_RBUSY)); if (m == NULL) { @@ -2545,7 +2565,12 @@ retrylookup: VM_WAIT; VM_OBJECT_WLOCK(object); goto retrylookup; - } else if (m->valid != 0) + } + + /* If the lock state changed in the meanwhile, unwind back. */ + if (VM_OBJECT_WOWNED(object) != origwlock) + VM_OBJECT_LOCK_DOWNGRADE(object); + if (m->valid != 0) return (m); if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h (revision 253430) +++ sys/vm/vm_page.h (working copy) @@ -439,7 +439,8 @@ void vm_page_rename (vm_page_t, vm_object_t, vm_pi void vm_page_requeue(vm_page_t m); void vm_page_requeue_locked(vm_page_t m); void vm_page_set_valid_range(vm_page_t m, int base, int size); -int vm_page_sleep_if_busy(vm_page_t m, const char *msg); +int vm_page_sleep_if_busy(vm_page_t m, const char *msg, int busyflags, + boolean_t pref); vm_offset_t vm_page_startup(vm_offset_t vaddr); void vm_page_unhold_pages(vm_page_t *ma, int count); void vm_page_unwire (vm_page_t, int); Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (revision 253430) +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (working copy) @@ -336,19 +336,9 @@ page_busy(vnode_t *vp, int64_t start, int64_t off, for (;;) { if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL && pp->valid) { - if (vm_page_busy_wlocked(pp)) { - /* - * Reference the page before unlocking and - * sleeping so that the page daemon is less - * likely to reclaim it. - */ - vm_page_reference(pp); - vm_page_lock(pp); - zfs_vmobject_wunlock(obj); - vm_page_busy_sleep(pp, "zfsmwb"); - zfs_vmobject_wlock(obj); + if (vm_page_sleep_if_busy(pp, "zfsmwb", + VM_ALLOC_NOBUSY, TRUE)) continue; - } vm_page_busy_rlock(pp); } else if (pp == NULL) { if (!alloc) Index: sys/kern/vfs_bio.c =================================================================== --- sys/kern/vfs_bio.c (revision 253431) +++ sys/kern/vfs_bio.c (working copy) @@ -3444,7 +3444,7 @@ allocbuf(struct buf *bp, int size) KASSERT(m != bogus_page, ("allocbuf: bogus page found")); while (vm_page_sleep_if_busy(m, - "biodep")) + "biodep", VM_ALLOC_NOBUSY, FALSE)) continue; bp->b_pages[i] = NULL; Index: sys/kern/subr_uio.c =================================================================== --- sys/kern/subr_uio.c (revision 253430) +++ sys/kern/subr_uio.c (working copy) @@ -107,7 +107,8 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_o VM_OBJECT_WLOCK(uobject); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { - if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco")) + if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco", + VM_ALLOC_NOBUSY, FALSE)) goto retry; vm_page_lock(user_pg); pmap_remove_all(user_pg); Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c (revision 253430) +++ sys/kern/uipc_shm.c (working copy) @@ -281,7 +281,8 @@ shm_dotruncate(struct shmfd *shmfd, off_t length) retry: m = vm_page_lookup(object, idx); if (m != NULL) { - if (vm_page_sleep_if_busy(m, "shmtrc")) + if (vm_page_sleep_if_busy(m, "shmtrc", + VM_ALLOC_NOBUSY, FALSE)) goto retry; } else if (vm_pager_has_page(object, idx, NULL, NULL)) { m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL); Index: sys/dev/drm2/i915/i915_gem.c =================================================================== --- sys/dev/drm2/i915/i915_gem.c (revision 253430) +++ sys/dev/drm2/i915/i915_gem.c (working copy) @@ -2333,7 +2333,8 @@ retry: m = vm_page_lookup(devobj, i); if (m == NULL) continue; - if (vm_page_sleep_if_busy(m, "915unm")) + if (vm_page_sleep_if_busy(m, "915unm", VM_ALLOC_NOBUSY, + FALSE)) goto retry; cdev_pager_free_page(devobj, m); }