Index: fs/fuse/fuse_vnops.c =================================================================== --- fs/fuse/fuse_vnops.c (.../vmcontention/sys) (revision 251603) +++ fs/fuse/fuse_vnops.c (.../vmobj-readlock/sys) (revision 251603) @@ -1868,16 +1868,13 @@ fuse_vnop_getpages(struct vop_getpages_args *ap) * now tell them that it is ok to use. */ if (!error) { - if (m->oflags & VPO_WANTED) { - fuse_vm_page_lock(m); + fuse_vm_page_lock(m); + if (m->flags & PG_WANTED) vm_page_activate(m); - fuse_vm_page_unlock(m); - } else { - fuse_vm_page_lock(m); + else vm_page_deactivate(m); - fuse_vm_page_unlock(m); - } - vm_page_wakeup(m); + vm_page_wakeup_locked(m); + fuse_vm_page_unlock(m); } else { fuse_vm_page_lock(m); vm_page_free(m); Index: fs/tmpfs/tmpfs_subr.c =================================================================== --- fs/tmpfs/tmpfs_subr.c (.../vmcontention/sys) (revision 251603) +++ fs/tmpfs/tmpfs_subr.c (.../vmobj-readlock/sys) (revision 251603) @@ -1331,11 +1331,8 @@ tmpfs_reg_resize(struct vnode *vp, off_t newsize, retry: m = vm_page_lookup(uobj, idx); if (m != NULL) { - if ((m->oflags & VPO_BUSY) != 0 || - m->busy != 0) { - vm_page_sleep(m, "tmfssz"); + if (vm_page_sleep_if_busy(m, "tmfssz")) goto retry; - } MPASS(m->valid == VM_PAGE_BITS_ALL); } else if (vm_pager_has_page(uobj, idx, NULL, NULL)) { m = vm_page_alloc(uobj, idx, VM_ALLOC_NORMAL); @@ -1354,8 +1351,8 @@ retry: vm_page_lock(m); if (rv == VM_PAGER_OK) { vm_page_deactivate(m); + vm_page_wakeup_locked(m); vm_page_unlock(m); - vm_page_wakeup(m); } else { vm_page_free(m); vm_page_unlock(m); Index: dev/agp/agp.c =================================================================== --- dev/agp/agp.c (.../vmcontention/sys) (revision 251603) +++ dev/agp/agp.c (.../vmobj-readlock/sys) (revision 251603) @@ -626,9 +626,9 @@ bad: VM_OBJECT_ASSERT_WLOCKED(mem->am_obj); for (k = 0; k < mem->am_size; k += PAGE_SIZE) { m = vm_page_lookup(mem->am_obj, OFF_TO_IDX(k)); + vm_page_lock(m); if (k >= i) - vm_page_wakeup(m); - vm_page_lock(m); + vm_page_wakeup_locked(m); vm_page_unwire(m, 0); vm_page_unlock(m); } Index: dev/md/md.c =================================================================== --- dev/md/md.c (.../vmcontention/sys) (revision 251603) +++ dev/md/md.c (.../vmobj-readlock/sys) (revision 251603) @@ -885,8 +885,8 @@ mdstart_swap(struct md_s *sc, struct bio *bp) } else vm_pager_page_unswapped(m); } - vm_page_wakeup(m); vm_page_lock(m); + vm_page_wakeup_locked(m); if (bp->bio_cmd == BIO_DELETE && len == PAGE_SIZE) vm_page_free(m); else Index: dev/drm2/i915/i915_gem.c =================================================================== --- dev/drm2/i915/i915_gem.c (.../vmcontention/sys) (revision 251603) +++ dev/drm2/i915/i915_gem.c (.../vmobj-readlock/sys) (revision 251603) @@ -1421,7 +1421,10 @@ unlocked_vmobj: if ((m->flags & VPO_BUSY) != 0) { DRM_UNLOCK(dev); + vm_page_lock(m); + VM_OBJECT_WUNLOCK(vm_obj); vm_page_sleep(m, "915pbs"); + VM_OBJECT_WLOCK(vm_obj); goto retry; } m->valid = VM_PAGE_BITS_ALL; @@ -2313,7 +2316,7 @@ retry: m = vm_page_lookup(devobj, i); if (m == NULL) continue; - if (vm_page_sleep_if_busy(m, true, "915unm")) + if (vm_page_sleep_if_busy(m, "915unm")) goto retry; cdev_pager_free_page(devobj, m); } @@ -2511,6 +2514,7 @@ i915_gem_wire_page(vm_object_t object, vm_pindex_t } vm_page_lock(m); vm_page_wire(m); + vm_page_wakeup_locked(m); vm_page_unlock(m); atomic_add_long(&i915_gem_wired_pages_cnt, 1); return (m); Index: dev/drm2/ttm/ttm_bo_vm.c =================================================================== --- dev/drm2/ttm/ttm_bo_vm.c (.../vmcontention/sys) (revision 251603) +++ dev/drm2/ttm/ttm_bo_vm.c (.../vmobj-readlock/sys) (revision 251603) @@ -213,7 +213,10 @@ reserve: VM_OBJECT_WLOCK(vm_obj); if ((m->flags & VPO_BUSY) != 0) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(vm_obj); vm_page_sleep(m, "ttmpbs"); + VM_OBJECT_WLOCK(vm_obj); ttm_mem_io_unlock(man); ttm_bo_unreserve(bo); goto retry; Index: vm/phys_pager.c =================================================================== --- vm/phys_pager.c (.../vmcontention/sys) (revision 251603) +++ vm/phys_pager.c (.../vmobj-readlock/sys) (revision 251603) @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -152,9 +153,11 @@ phys_pager_getpages(vm_object_t object, vm_page_t KASSERT(m[i]->dirty == 0, ("phys_pager_getpages: dirty page %p", m[i])); /* The requested page must remain busy, the others not. */ - if (i == reqpage) + if (i == reqpage) { + vm_page_lock(m[i]); vm_page_flash(m[i]); - else + vm_page_unlock(m[i]); + } else vm_page_wakeup(m[i]); } return (VM_PAGER_OK); Index: vm/swap_pager.c =================================================================== --- vm/swap_pager.c (.../vmcontention/sys) (revision 251603) +++ vm/swap_pager.c (.../vmobj-readlock/sys) (revision 251603) @@ -1219,13 +1219,15 @@ swap_pager_getpages(vm_object_t object, vm_page_t */ VM_OBJECT_WLOCK(object); while ((mreq->oflags & VPO_SWAPINPROG) != 0) { - mreq->oflags |= VPO_WANTED; PCPU_INC(cnt.v_intrans); - if (VM_OBJECT_SLEEP(object, mreq, PSWP, "swread", hz * 20)) { + vm_page_lock(mreq); + VM_OBJECT_WUNLOCK(object); + if (vm_page_sleep_onpage(mreq, PSWP, "swread", hz * 20)) { printf( "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount); } + VM_OBJECT_WLOCK(object); } /* @@ -1542,8 +1544,11 @@ swp_pager_async_iodone(struct buf *bp) m->valid = 0; if (i != bp->b_pager.pg_reqpage) swp_pager_free_nrpage(m); - else + else { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } /* * If i == bp->b_pager.pg_reqpage, do not wake * the page up. The caller needs to. @@ -1594,10 +1599,13 @@ swp_pager_async_iodone(struct buf *bp) if (i != bp->b_pager.pg_reqpage) { vm_page_lock(m); vm_page_deactivate(m); + vm_page_wakeup_locked(m); vm_page_unlock(m); - vm_page_wakeup(m); - } else + } else { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } } else { /* * For write success, clear the dirty @@ -1713,6 +1721,7 @@ swp_pager_force_pagein(vm_object_t object, vm_pind vm_page_dirty(m); vm_page_lock(m); vm_page_activate(m); + vm_page_wakeup_locked(m); vm_page_unlock(m); vm_pager_page_unswapped(m); return; @@ -1725,8 +1734,8 @@ swp_pager_force_pagein(vm_object_t object, vm_pind vm_page_dirty(m); vm_page_lock(m); vm_page_deactivate(m); + vm_page_wakeup_locked(m); vm_page_unlock(m); - vm_page_wakeup(m); vm_pager_page_unswapped(m); } Index: vm/vm_glue.c =================================================================== --- vm/vm_glue.c (.../vmcontention/sys) (revision 251603) +++ vm/vm_glue.c (.../vmobj-readlock/sys) (revision 251603) @@ -261,6 +261,7 @@ vm_imgact_hold_page(vm_object_t object, vm_ooffset } vm_page_lock(m); vm_page_hold(m); + vm_page_wakeup_locked(m); vm_page_unlock(m); out: VM_OBJECT_WUNLOCK(object); Index: vm/vm_object.c =================================================================== --- vm/vm_object.c (.../vmcontention/sys) (revision 251603) +++ vm/vm_object.c (.../vmobj-readlock/sys) (revision 251603) @@ -391,7 +391,7 @@ vm_object_pip_wait(vm_object_t object, char *waiti VM_OBJECT_ASSERT_WLOCKED(object); while (object->paging_in_progress) { object->flags |= OBJ_PIPWNT; - VM_OBJECT_SLEEP(object, object, PVM, waitid, 0); + VM_OBJECT_SLEEP(object, object, PVM, waitid); } } @@ -606,7 +606,7 @@ retry: VM_OBJECT_WUNLOCK(robject); object->flags |= OBJ_PIPWNT; VM_OBJECT_SLEEP(object, object, - PDROP | PVM, "objde2", 0); + PDROP | PVM, "objde2"); VM_OBJECT_WLOCK(robject); temp = robject->backing_object; if (object == temp) { @@ -871,7 +871,7 @@ rescan: np = TAILQ_NEXT(p, listq); if (p->valid == 0) continue; - if (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { + if (vm_page_sleep_if_busy(p, "vpcwai")) { if (object->generation != curgeneration) { if ((flags & OBJPC_SYNC) != 0) goto rescan; @@ -1165,11 +1165,10 @@ shadowlookup: */ vm_page_aflag_set(m, PGA_REFERENCED); } - vm_page_unlock(m); if (object != tobject) VM_OBJECT_WUNLOCK(object); - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(tobject, m, PDROP | PVM, "madvpo", 0); + VM_OBJECT_WUNLOCK(tobject); + vm_page_sleep(m, "madvpo"); VM_OBJECT_WLOCK(object); goto relookup; } @@ -1346,8 +1345,10 @@ retry: */ if ((m->oflags & VPO_BUSY) || m->busy) { VM_OBJECT_WUNLOCK(new_object); - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0); + vm_page_lock(m); + VM_OBJECT_WUNLOCK(orig_object); + vm_page_sleep(m, "spltwt"); + VM_OBJECT_WLOCK(orig_object); VM_OBJECT_WLOCK(new_object); goto retry; } @@ -1505,9 +1506,9 @@ vm_object_backing_scan(vm_object_t object, int op) } else if (op & OBSC_COLLAPSE_WAIT) { if ((p->oflags & VPO_BUSY) || p->busy) { VM_OBJECT_WUNLOCK(object); - p->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(backing_object, p, - PDROP | PVM, "vmocol", 0); + vm_page_lock(p); + VM_OBJECT_WUNLOCK(backing_object); + vm_page_sleep(p, "vmocol"); VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(backing_object); /* @@ -1905,8 +1906,12 @@ again: vm_page_unlock(p); continue; } - if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) + if ((p->oflags & VPO_BUSY) != 0 || p->busy != 0) { + VM_OBJECT_WUNLOCK(object); + vm_page_sleep(p, "vmopar"); + VM_OBJECT_WLOCK(object); goto again; + } KASSERT((p->flags & PG_FICTITIOUS) == 0, ("vm_object_page_remove: page %p is fictitious", p)); if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) { Index: vm/vm_object.h =================================================================== --- vm/vm_object.h (.../vmcontention/sys) (revision 251603) +++ vm/vm_object.h (.../vmobj-readlock/sys) (revision 251603) @@ -229,8 +229,8 @@ extern struct vm_object kmem_object_store; rw_rlock(&(object)->lock) #define VM_OBJECT_RUNLOCK(object) \ rw_runlock(&(object)->lock) -#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg, timo) \ - rw_sleep((wchan), &(object)->lock, (pri), (wmesg), (timo)) +#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg) \ + rw_sleep((wchan), &(object)->lock, (pri), (wmesg), 0) #define VM_OBJECT_TRYRLOCK(object) \ rw_try_rlock(&(object)->lock) #define VM_OBJECT_TRYWLOCK(object) \ Index: vm/vm_fault.c =================================================================== --- vm/vm_fault.c (.../vmcontention/sys) (revision 251603) +++ vm/vm_fault.c (.../vmobj-readlock/sys) (revision 251603) @@ -141,8 +141,8 @@ static inline void release_page(struct faultstate *fs) { - vm_page_wakeup(fs->m); vm_page_lock(fs->m); + vm_page_wakeup_locked(fs->m); vm_page_deactivate(fs->m); vm_page_unlock(fs->m); fs->m = NULL; @@ -379,8 +379,7 @@ RetryFault:; unlock_map(&fs); if (fs.m == vm_page_lookup(fs.object, fs.pindex)) { - vm_page_sleep_if_busy(fs.m, TRUE, - "vmpfw"); + vm_page_sleep_if_busy(fs.m, "vmpfw"); } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); @@ -932,8 +931,8 @@ vnode_locked: *m_hold = fs.m; vm_page_hold(fs.m); } + vm_page_wakeup_locked(fs.m); vm_page_unlock(fs.m); - vm_page_wakeup(fs.m); /* * Unlock everything, and return @@ -1359,13 +1358,14 @@ vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src vm_page_lock(dst_m); vm_page_wire(dst_m); + vm_page_wakeup_locked(dst_m); vm_page_unlock(dst_m); } else { vm_page_lock(dst_m); vm_page_activate(dst_m); + vm_page_wakeup_locked(dst_m); vm_page_unlock(dst_m); } - vm_page_wakeup(dst_m); } VM_OBJECT_WUNLOCK(dst_object); if (upgrade) { Index: vm/vm_page.c =================================================================== --- vm/vm_page.c (.../vmcontention/sys) (revision 251603) +++ vm/vm_page.c (.../vmobj-readlock/sys) (revision 251603) @@ -488,25 +488,44 @@ void vm_page_flash(vm_page_t m) { - VM_OBJECT_ASSERT_WLOCKED(m->object); - if (m->oflags & VPO_WANTED) { - m->oflags &= ~VPO_WANTED; + vm_page_lock_assert(m, MA_OWNED); + if (m->flags & PG_WANTED) { + m->flags &= ~PG_WANTED; wakeup(m); } } /* - * vm_page_wakeup: + * vm_page_wakeup: * + * clear the VPO_BUSY flag and wakeup anyone waiting for the + * page. + * + * The object containing the page must be locked. + */ +void +vm_page_wakeup(vm_page_t m) +{ + + vm_page_lock(m); + vm_page_wakeup_locked(m); + vm_page_unlock(m); +} + +/* + * vm_page_wakeup_locked: + * * clear the VPO_BUSY flag and wakeup anyone waiting for the * page. * + * The page and the object containing the page must be locked. */ void -vm_page_wakeup(vm_page_t m) +vm_page_wakeup_locked(vm_page_t m) { VM_OBJECT_ASSERT_WLOCKED(m->object); + vm_page_lock_assert(m, MA_OWNED); KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!")); m->oflags &= ~VPO_BUSY; vm_page_flash(m); @@ -527,8 +546,11 @@ vm_page_io_finish(vm_page_t m) VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(m->busy > 0, ("vm_page_io_finish: page %p is not busy", m)); m->busy--; - if (m->busy == 0) + if (m->busy == 0) { + vm_page_lock(m); vm_page_flash(m); + vm_page_unlock(m); + } } /* @@ -722,16 +744,13 @@ vm_page_readahead_finish(vm_page_t m) * deactivating the page is usually the best choice, * unless the page is wanted by another thread. */ - if (m->oflags & VPO_WANTED) { - vm_page_lock(m); + vm_page_lock(m); + if (m->flags & PG_WANTED) vm_page_activate(m); - vm_page_unlock(m); - } else { - vm_page_lock(m); + else vm_page_deactivate(m); - vm_page_unlock(m); - } - vm_page_wakeup(m); + vm_page_wakeup_locked(m); + vm_page_unlock(m); } else { /* * Free the completely invalid page. Such page state @@ -746,32 +765,72 @@ vm_page_readahead_finish(vm_page_t m) } /* - * vm_page_sleep: + * _vm_page_sleep_onpage: * - * Sleep and release the page lock. + * Sleep and release the page lock, using the page pointer as wchan. * - * The object containing the given page must be locked. + * The given page must be locked. */ -void -vm_page_sleep(vm_page_t m, const char *msg) +static inline int +_vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo) { + vm_page_lock_assert(m, MA_OWNED); + m->flags |= PG_WANTED; + return (msleep(m, vm_page_lockptr(m), pri | PDROP, wmesg, timo)); +} + +/* + * vm_page_sleep_if_busy: + * + * Sleep and release the page queues lock if VPO_BUSY is set or, + * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the + * thread slept. + * + * The given page must be unlocked and object containing it must + * be locked. + */ +int +vm_page_sleep_if_busy(vm_page_t m, const char *msg) +{ + vm_object_t obj; + + vm_page_lock_assert(m, MA_NOTOWNED); VM_OBJECT_ASSERT_WLOCKED(m->object); - if (mtx_owned(vm_page_lockptr(m))) - vm_page_unlock(m); - /* - * It's possible that while we sleep, the page will get - * unbusied and freed. If we are holding the object - * lock, we will assume we hold a reference to the object - * such that even if m->object changes, we can re-lock - * it. - */ - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(m->object, m, PVM, msg, 0); + if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) { + /* + * The page-specific object must be cached because page + * identity can change during the sleep, causing the + * re-lock of a different object. + * It is assumed that a reference to the object is already + * held by the callers. + */ + obj = m->object; + vm_page_lock(m); + VM_OBJECT_WUNLOCK(obj); + _vm_page_sleep_onpage(m, PVM, msg, 0); + VM_OBJECT_WLOCK(obj); + return (TRUE); + } + return (FALSE); } /* + * vm_page_sleep_onpage: + * + * External version of _vm_page_sleep_onpage(). + * + * Check the inline version for comments. + */ +int +vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo) +{ + + return (_vm_page_sleep_onpage(m, pri, wmesg, timo)); +} + +/* * vm_page_dirty_KBI: [ internal use only ] * * Set all bits in the page's dirty field. @@ -893,6 +952,7 @@ void vm_page_remove(vm_page_t m) { vm_object_t object; + boolean_t lockacq; if ((m->oflags & VPO_UNMANAGED) == 0) vm_page_lock_assert(m, MA_OWNED); @@ -901,7 +961,15 @@ vm_page_remove(vm_page_t m) VM_OBJECT_ASSERT_WLOCKED(object); if (m->oflags & VPO_BUSY) { m->oflags &= ~VPO_BUSY; + lockacq = FALSE; + if ((m->oflags & VPO_UNMANAGED) != 0 && + !mtx_owned(vm_page_lockptr(m))) { + lockacq = TRUE; + vm_page_lock(m); + } vm_page_flash(m); + if (lockacq) + vm_page_unlock(m); } /* @@ -2364,7 +2432,10 @@ retrylookup: * likely to reclaim it. */ vm_page_aflag_set(m, PGA_REFERENCED); + vm_page_lock(m); + VM_OBJECT_WUNLOCK(object); vm_page_sleep(m, "pgrbwt"); + VM_OBJECT_WLOCK(object); goto retrylookup; } else { if ((allocflags & VM_ALLOC_WIRED) != 0) { @@ -2697,7 +2768,7 @@ vm_page_is_valid(vm_page_t m, int base, int size) { vm_page_bits_t bits; - VM_OBJECT_ASSERT_WLOCKED(m->object); + VM_OBJECT_ASSERT_LOCKED(m->object); bits = vm_page_bits(base, size); return (m->valid != 0 && (m->valid & bits) == bits); } Index: vm/vm_page.h =================================================================== --- vm/vm_page.h (.../vmcontention/sys) (revision 251603) +++ vm/vm_page.h (.../vmobj-readlock/sys) (revision 251603) @@ -166,7 +166,7 @@ struct vm_page { * */ #define VPO_BUSY 0x01 /* page is in transit */ -#define VPO_WANTED 0x02 /* someone is waiting for page */ +#define VPO_UNUSED02 0x02 /* --available-- */ #define VPO_UNMANAGED 0x04 /* no PV management for page */ #define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */ #define VPO_NOSYNC 0x10 /* do not collect for syncer */ @@ -271,6 +271,7 @@ extern struct mtx_padalign pa_lock[]; #define PG_WINATCFLS 0x0040 /* flush dirty page on inactive q */ #define PG_NODUMP 0x0080 /* don't include this page in a dump */ #define PG_UNHOLDFREE 0x0100 /* delayed free of a held page */ +#define PG_WANTED 0x0200 /* someone is waiting for page */ /* * Misc constants. @@ -368,6 +369,7 @@ void vm_page_unhold(vm_page_t mem); void vm_page_free(vm_page_t m); void vm_page_free_zero(vm_page_t m); void vm_page_wakeup(vm_page_t m); +void vm_page_wakeup_locked(vm_page_t m); void vm_page_activate (vm_page_t); void vm_page_advise(vm_page_t m, int advice); @@ -402,7 +404,8 @@ void vm_page_rename (vm_page_t, vm_object_t, vm_pi void vm_page_requeue(vm_page_t m); void vm_page_requeue_locked(vm_page_t m); void vm_page_set_valid_range(vm_page_t m, int base, int size); -void vm_page_sleep(vm_page_t m, const char *msg); +int vm_page_sleep_if_busy(vm_page_t m, const char *msg); +int vm_page_sleep_onpage(vm_page_t m, int pri, const char *msg, int timo); vm_offset_t vm_page_startup(vm_offset_t vaddr); void vm_page_unhold_pages(vm_page_t *ma, int count); void vm_page_unwire (vm_page_t, int); @@ -528,40 +531,34 @@ vm_page_dirty(vm_page_t m) } /* - * vm_page_remque: + * vm_page_sleep: * - * If the given page is in a page queue, then remove it from that page - * queue. + * Convenience wrapper around vm_page_sleep_onpage(), passing + * PVM priority and 0 timeout values. Unlocks the page upon return. * * The page must be locked. */ -static inline void -vm_page_remque(vm_page_t m) +static __inline void +vm_page_sleep(vm_page_t m, const char *msg) { - if (m->queue != PQ_NONE) - vm_page_dequeue(m); + vm_page_sleep_onpage(m, PVM, msg, 0); } /* - * vm_page_sleep_if_busy: + * vm_page_remque: * - * Sleep and release the page queues lock if VPO_BUSY is set or, - * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the - * thread slept and the page queues lock was released. - * Otherwise, retains the page queues lock and returns FALSE. + * If the given page is in a page queue, then remove it from that page + * queue. * - * The object containing the given page must be locked. + * The page must be locked. */ -static __inline int -vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg) +static inline void +vm_page_remque(vm_page_t m) { - if ((m->oflags & VPO_BUSY) || (also_m_busy && m->busy)) { - vm_page_sleep(m, msg); - return (TRUE); - } - return (FALSE); + if (m->queue != PQ_NONE) + vm_page_dequeue(m); } /* Index: vm/vnode_pager.c =================================================================== --- vm/vnode_pager.c (.../vmcontention/sys) (revision 251603) +++ vm/vnode_pager.c (.../vmobj-readlock/sys) (revision 251603) @@ -117,7 +117,7 @@ vnode_create_vobject(struct vnode *vp, off_t isize } VOP_UNLOCK(vp, 0); vm_object_set_flag(object, OBJ_DISCONNECTWNT); - VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead"); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); } @@ -211,7 +211,7 @@ retry: if ((object->flags & OBJ_DEAD) == 0) break; vm_object_set_flag(object, OBJ_DISCONNECTWNT); - VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead"); } KASSERT(vp->v_usecount != 0, ("vnode_pager_alloc: no vnode reference")); Index: contrib/dev/acpica/include =================================================================== --- contrib/dev/acpica/include (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/include (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/include ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /vendor-sys/acpica/dist/include:r193249-193331 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/include:r248156-250886 Index: contrib/dev/acpica/components/utilities =================================================================== --- contrib/dev/acpica/components/utilities (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/utilities (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/utilities ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/utilities:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/utilities:r248156-250886 Index: contrib/dev/acpica/components/namespace =================================================================== --- contrib/dev/acpica/components/namespace (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/namespace (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/namespace ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/namespace:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/namespace:r248156-250886 Index: contrib/dev/acpica/components/parser =================================================================== --- contrib/dev/acpica/components/parser (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/parser (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/parser ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/parser:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/parser:r248156-250886 Index: contrib/dev/acpica/components/disassembler =================================================================== --- contrib/dev/acpica/components/disassembler (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/disassembler (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/disassembler ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/disassembler:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/disassembler:r248156-250886 Index: contrib/dev/acpica/components/hardware =================================================================== --- contrib/dev/acpica/components/hardware (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/hardware (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/hardware ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/hardware:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/hardware:r248156-250886 Index: contrib/dev/acpica/components/debugger =================================================================== --- contrib/dev/acpica/components/debugger (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/debugger (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/debugger ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/debugger:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/debugger:r248156-250886 Index: contrib/dev/acpica/components/events =================================================================== --- contrib/dev/acpica/components/events (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/events (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/events ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/events:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/events:r248156-250886 Index: contrib/dev/acpica/components/executer =================================================================== --- contrib/dev/acpica/components/executer (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/executer (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/executer ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/executer:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/executer:r248156-250886 Index: contrib/dev/acpica/components/dispatcher =================================================================== --- contrib/dev/acpica/components/dispatcher (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/dispatcher (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/dispatcher ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/dispatcher:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/dispatcher:r248156-250886 Index: contrib/dev/acpica/components/resources =================================================================== --- contrib/dev/acpica/components/resources (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/resources (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/resources ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/resources:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/resources:r248156-250886 Index: contrib/dev/acpica/components/tables =================================================================== --- contrib/dev/acpica/components/tables (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/components/tables (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/components/tables ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/components/tables:r231844-232160 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/components/tables:r248156-250886 Index: contrib/dev/acpica/changes.txt =================================================================== --- contrib/dev/acpica/changes.txt (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/changes.txt (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/changes.txt ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /vendor-sys/acpica/dist/changes.txt:r193249-193331 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/changes.txt:r248156-250886 Index: contrib/dev/acpica/os_specific =================================================================== --- contrib/dev/acpica/os_specific (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/os_specific (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/os_specific ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/contrib/dev/acpica/os_specific:r228110-228209 Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/os_specific:r248156-250886 Index: contrib/dev/acpica/common =================================================================== --- contrib/dev/acpica/common (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/common (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/common ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/common:r248156-250886 Index: contrib/dev/acpica/compiler =================================================================== --- contrib/dev/acpica/compiler (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica/compiler (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica/compiler ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/contrib/dev/acpica/compiler:r248156-250886 Index: contrib/dev/acpica =================================================================== --- contrib/dev/acpica (.../vmcontention/sys) (revision 251603) +++ contrib/dev/acpica (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/dev/acpica ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/contrib/dev/acpica:r248156-250886 Index: contrib/octeon-sdk =================================================================== --- contrib/octeon-sdk (.../vmcontention/sys) (revision 251603) +++ contrib/octeon-sdk (.../vmobj-readlock/sys) (revision 251603) Property changes on: contrib/octeon-sdk ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/contrib/octeon-sdk:r248156-250635 Index: cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c =================================================================== --- cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (.../vmcontention/sys) (revision 251603) +++ cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c (.../vmobj-readlock/sys) (revision 251603) @@ -342,7 +342,10 @@ page_busy(vnode_t *vp, int64_t start, int64_t off, * likely to reclaim it. */ vm_page_reference(pp); + vm_page_lock(pp); + zfs_vmobject_wunlock(obj); vm_page_sleep(pp, "zfsmwb"); + zfs_vmobject_wlock(obj); continue; } } else { @@ -390,7 +393,10 @@ page_hold(vnode_t *vp, int64_t start) * likely to reclaim it. */ vm_page_reference(pp); + vm_page_lock(pp); + zfs_vmobject_wunlock(obj); vm_page_sleep(pp, "zfsmwb"); + zfs_vmobject_wlock(obj); continue; } Index: cddl/contrib/opensolaris =================================================================== --- cddl/contrib/opensolaris (.../vmcontention/sys) (revision 251603) +++ cddl/contrib/opensolaris (.../vmobj-readlock/sys) (revision 251603) Property changes on: cddl/contrib/opensolaris ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/cddl/contrib/opensolaris:r248156-251602 Index: amd64/include/vmm_instruction_emul.h =================================================================== --- amd64/include/vmm_instruction_emul.h (.../vmcontention/sys) (revision 251603) +++ amd64/include/vmm_instruction_emul.h (.../vmobj-readlock/sys) (revision 251603) Property changes on: amd64/include/vmm_instruction_emul.h ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/amd64/include/vmm_instruction_emul.h:r245652-246294 Merged /user/attilio/vmcontention/sys/amd64/include/vmm_instruction_emul.h:r248156-249197 Index: amd64/include/xen =================================================================== --- amd64/include/xen (.../vmcontention/sys) (revision 251603) +++ amd64/include/xen (.../vmobj-readlock/sys) (revision 251603) Property changes on: amd64/include/xen ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/amd64/include/xen:r248156-250635 Index: amd64/include/vmm.h =================================================================== --- amd64/include/vmm.h (.../vmcontention/sys) (revision 251603) +++ amd64/include/vmm.h (.../vmobj-readlock/sys) (revision 251603) Property changes on: amd64/include/vmm.h ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/amd64/include/vmm.h:r245652-246294 Merged /user/attilio/vmcontention/sys/amd64/include/vmm.h:r248156-250635 Index: amd64/vmm =================================================================== --- amd64/vmm (.../vmcontention/sys) (revision 251603) +++ amd64/vmm (.../vmobj-readlock/sys) (revision 251603) Property changes on: amd64/vmm ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /head/sys/amd64/vmm:r245652-246294 Merged /user/attilio/vmcontention/sys/amd64/vmm:r248156-250635 Index: conf =================================================================== --- conf (.../vmcontention/sys) (revision 251603) +++ conf (.../vmobj-readlock/sys) (revision 251603) Property changes on: conf ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/conf:r248156-251602 Index: kern/subr_uio.c =================================================================== --- kern/subr_uio.c (.../vmcontention/sys) (revision 251603) +++ kern/subr_uio.c (.../vmobj-readlock/sys) (revision 251603) @@ -107,7 +107,7 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_o VM_OBJECT_WLOCK(uobject); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { - if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) + if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco")) goto retry; vm_page_lock(user_pg); pmap_remove_all(user_pg); Index: kern/kern_exec.c =================================================================== --- kern/kern_exec.c (.../vmcontention/sys) (revision 251603) +++ kern/kern_exec.c (.../vmobj-readlock/sys) (revision 251603) @@ -974,6 +974,7 @@ exec_map_first_page(imgp) } vm_page_lock(ma[0]); vm_page_hold(ma[0]); + vm_page_wakeup_locked(ma[0]); vm_page_unlock(ma[0]); VM_OBJECT_WUNLOCK(object); Index: kern/uipc_shm.c =================================================================== --- kern/uipc_shm.c (.../vmcontention/sys) (revision 251603) +++ kern/uipc_shm.c (.../vmobj-readlock/sys) (revision 251603) @@ -281,11 +281,8 @@ shm_dotruncate(struct shmfd *shmfd, off_t length) retry: m = vm_page_lookup(object, idx); if (m != NULL) { - if ((m->oflags & VPO_BUSY) != 0 || - m->busy != 0) { - vm_page_sleep(m, "shmtrc"); + if (vm_page_sleep_if_busy(m, "shmtrc")) goto retry; - } } else if (vm_pager_has_page(object, idx, NULL, NULL)) { m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL); if (m == NULL) { @@ -304,8 +301,8 @@ retry: vm_page_lock(m); if (rv == VM_PAGER_OK) { vm_page_deactivate(m); + vm_page_wakeup_locked(m); vm_page_unlock(m); - vm_page_wakeup(m); } else { vm_page_free(m); vm_page_unlock(m); Index: kern/vfs_bio.c =================================================================== --- kern/vfs_bio.c (.../vmcontention/sys) (revision 251603) +++ kern/vfs_bio.c (.../vmobj-readlock/sys) (revision 251603) @@ -581,7 +581,7 @@ vfs_buf_test_cache(struct buf *bp, vm_page_t m) { - VM_OBJECT_ASSERT_WLOCKED(m->object); + VM_OBJECT_ASSERT_LOCKED(m->object); if (bp->b_flags & B_CACHE) { int base = (foff + off) & PAGE_MASK; if (vm_page_is_valid(m, base, size) == 0) @@ -3454,7 +3454,7 @@ allocbuf(struct buf *bp, int size) m = bp->b_pages[i]; KASSERT(m != bogus_page, ("allocbuf: bogus page found")); - while (vm_page_sleep_if_busy(m, TRUE, + while (vm_page_sleep_if_busy(m, "biodep")) continue; @@ -4007,8 +4007,12 @@ vfs_drain_busy_pages(struct buf *bp) if ((m->oflags & VPO_BUSY) != 0) { for (; last_busied < i; last_busied++) vm_page_busy(bp->b_pages[last_busied]); - while ((m->oflags & VPO_BUSY) != 0) + while ((m->oflags & VPO_BUSY) != 0) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); vm_page_sleep(m, "vbpage"); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); + } } } for (i = 0; i < last_busied; i++) Index: boot =================================================================== --- boot (.../vmcontention/sys) (revision 251603) +++ boot (.../vmobj-readlock/sys) (revision 251603) Property changes on: boot ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys/boot:r248156-251602 Index: . =================================================================== --- . (.../vmcontention/sys) (revision 251603) +++ . (.../vmobj-readlock/sys) (revision 251603) Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /user/attilio/vmcontention/sys:r248156-251602