diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/vnode.c b/sys/cddl/contrib/opensolaris/uts/common/fs/vnode.c index 6d82470d220a..fd0aed4bf34e 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/vnode.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/vnode.c @@ -85,8 +85,7 @@ xva_getxoptattr(xvattr_t *xvap) void vn_rele_async(vnode_t *vp, taskq_t *taskq) { - VERIFY(vp->v_count > 0); - if (refcount_release_if_not_last(&vp->v_usecount)) { + if (vn_ref_release_if_not_last(vp, v_usecount)) { return; } VERIFY(taskq_dispatch((taskq_t *)taskq, diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c index 945a15eb59f8..81f8bd29accd 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c @@ -2079,7 +2079,11 @@ zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting) for (zp = list_head(&zfsvfs->z_all_znodes); zp != NULL; zp = list_next(&zfsvfs->z_all_znodes, zp)) if (zp->z_sa_hdl) { +#ifdef __illumos ASSERT(ZTOV(zp)->v_count >= 0); +#else + VNPASS(vn_ref_read(ZTOV(zp), v_usecount) >= 0, ZTOV(zp)); +#endif zfs_znode_dmu_fini(zp); } mutex_exit(&zfsvfs->z_znodes_lock); diff --git a/sys/fs/devfs/devfs_vnops.c b/sys/fs/devfs/devfs_vnops.c index f038f74e67a0..56cdd5cb4c68 100644 --- a/sys/fs/devfs/devfs_vnops.c +++ b/sys/fs/devfs/devfs_vnops.c @@ -479,7 +479,7 @@ devfs_allocv(struct devfs_dirent *de, struct mount *mp, int lockmode, dev_refl(dev); /* XXX: v_rdev should be protect by vnode lock */ vp->v_rdev = dev; - VNPASS(vp->v_usecount == 1, vp); + VNPASS(vn_ref_read(vp, v_usecount) == 1, vp); dev->si_usecount++; /* Special casing of ttys for deadfs. Probably redundant. */ dsw = dev->si_devsw; @@ -580,7 +580,7 @@ devfs_close(struct vop_close_args *ap) * if the reference count is 2 (this last descriptor * plus the session), release the reference from the session. */ - if (vp->v_usecount == 2 && td != NULL) { + if (vn_ref_read(vp, v_usecount) == 2 && td != NULL) { p = td->td_proc; PROC_LOCK(p); if (vp == p->p_session->s_ttyvp) { @@ -590,7 +590,7 @@ devfs_close(struct vop_close_args *ap) if (vp == p->p_session->s_ttyvp) { SESS_LOCK(p->p_session); VI_LOCK(vp); - if (vp->v_usecount == 2 && vcount(vp) == 1 && + if (vn_ref_read(vp, v_usecount) == 2 && vcount(vp) == 1 && !VN_IS_DOOMED(vp)) { p->p_session->s_ttyvp = NULL; p->p_session->s_ttydp = NULL; @@ -619,7 +619,7 @@ devfs_close(struct vop_close_args *ap) return (ENXIO); dflags = 0; VI_LOCK(vp); - if (vp->v_usecount == 1 && vcount(vp) == 1) + if (vn_ref_read(vp, v_usecount) == 1 && vcount(vp) == 1) dflags |= FLASTCLOSE; if (VN_IS_DOOMED(vp)) { /* Forced close. */ @@ -1431,7 +1431,7 @@ devfs_reclaim_vchr(struct vop_reclaim_args *ap) dev = vp->v_rdev; vp->v_rdev = NULL; if (dev != NULL) - dev->si_usecount -= (vp->v_usecount > 0); + dev->si_usecount -= (vn_ref_read(vp, v_usecount) > 0); dev_unlock(); VI_UNLOCK(vp); if (dev != NULL) diff --git a/sys/fs/ext2fs/ext2_vnops.c b/sys/fs/ext2fs/ext2_vnops.c index 3cd69c847c9c..82ea0896cc4d 100644 --- a/sys/fs/ext2fs/ext2_vnops.c +++ b/sys/fs/ext2fs/ext2_vnops.c @@ -308,7 +308,7 @@ ext2_close(struct vop_close_args *ap) struct vnode *vp = ap->a_vp; VI_LOCK(vp); - if (vp->v_usecount > 1) + if (vn_ref_read(vp, v_usecount) > 1) ext2_itimes_locked(vp); VI_UNLOCK(vp); return (0); @@ -1632,7 +1632,7 @@ ext2fifo_close(struct vop_close_args *ap) struct vnode *vp = ap->a_vp; VI_LOCK(vp); - if (vp->v_usecount > 1) + if (vn_ref_read(vp, v_usecount) > 1) ext2_itimes_locked(vp); VI_UNLOCK(vp); return (fifo_specops.vop_close(ap)); diff --git a/sys/fs/msdosfs/msdosfs_vnops.c b/sys/fs/msdosfs/msdosfs_vnops.c index f625e9368442..b711a8cf7aa9 100644 --- a/sys/fs/msdosfs/msdosfs_vnops.c +++ b/sys/fs/msdosfs/msdosfs_vnops.c @@ -68,6 +68,7 @@ #include #include #include +#include #include #include @@ -213,7 +214,7 @@ msdosfs_close(struct vop_close_args *ap) struct timespec ts; VI_LOCK(vp); - if (vp->v_usecount > 1) { + if (vn_ref_read(vp, v_usecount) > 1) { vfs_timestamp(&ts); DETIMES(dep, &ts, &ts, &ts); } @@ -874,7 +875,8 @@ msdosfs_remove(struct vop_remove_args *ap) else error = removede(ddep, dep); #ifdef MSDOSFS_DEBUG - printf("msdosfs_remove(), dep %p, v_usecount %d\n", dep, ap->a_vp->v_usecount); + printf("msdosfs_remove(), dep %p, v_usecount %d\n", dep, + vn_ref_read(ap->a_vp, v_usecount)); #endif return (error); } diff --git a/sys/fs/nullfs/null_subr.c b/sys/fs/nullfs/null_subr.c index 125b623cb638..54be8878006e 100644 --- a/sys/fs/nullfs/null_subr.c +++ b/sys/fs/nullfs/null_subr.c @@ -45,6 +45,7 @@ #include #include #include +#include #include @@ -209,7 +210,7 @@ null_nodeget(mp, lowervp, vpp) int error; ASSERT_VOP_LOCKED(lowervp, "lowervp"); - VNPASS(lowervp->v_usecount > 0, lowervp); + VNPASS(vn_ref_read(lowervp, v_usecount) > 0, lowervp); /* Lookup the hash firstly. */ *vpp = null_hashget(mp, lowervp); @@ -311,7 +312,7 @@ null_checkvp(vp, fil, lno) panic("null_checkvp %p", vp); } VI_LOCK_FLAGS(a->null_lowervp, MTX_DUPOK); - if (a->null_lowervp->v_usecount < 1) + if (vn_ref_read(a->null_lowervp, v_usecount) < 1) panic ("null with unref'ed lowervp, vp %p lvp %p", vp, a->null_lowervp); VI_UNLOCK(a->null_lowervp); diff --git a/sys/fs/nullfs/null_vfsops.c b/sys/fs/nullfs/null_vfsops.c index c79f4a9f1fec..7c89376f2904 100644 --- a/sys/fs/nullfs/null_vfsops.c +++ b/sys/fs/nullfs/null_vfsops.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include @@ -426,7 +427,7 @@ nullfs_unlink_lowervp(struct mount *mp, struct vnode *lowervp) vhold(vp); vunref(vp); - if (vp->v_usecount == 0) { + if (vn_ref_read(vp, v_usecount) == 0) { /* * If vunref() dropped the last use reference on the * nullfs vnode, it must be reclaimed, and its lock diff --git a/sys/fs/unionfs/union_vnops.c b/sys/fs/unionfs/union_vnops.c index 1ca75543f069..29c8def39e7c 100644 --- a/sys/fs/unionfs/union_vnops.c +++ b/sys/fs/unionfs/union_vnops.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -1706,7 +1707,7 @@ unionfs_getwritemount(struct vop_getwritemount_args *ap) error = VOP_GETWRITEMOUNT(uvp, ap->a_mpp); else { VI_LOCK(vp); - if (vp->v_holdcnt == 0) + if (vn_ref_read(vp, v_holdcnt) == 0) error = EOPNOTSUPP; else error = EACCES; diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c index d928ce763bf0..630fb7b26bcb 100644 --- a/sys/kern/kern_descrip.c +++ b/sys/kern/kern_descrip.c @@ -3254,7 +3254,7 @@ pwd_chdir(struct thread *td, struct vnode *vp) fdp = td->td_proc->p_fd; FILEDESC_XLOCK(fdp); - VNASSERT(vp->v_usecount > 0, vp, + VNASSERT(vn_ref_read(vp, v_usecount) > 0, vp, ("chdir to a vnode with zero usecount")); oldvp = fdp->fd_cdir; fdp->fd_cdir = vp; diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 0d95776676c5..a012a547be1c 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -2859,7 +2859,7 @@ vfs_vmio_iodone(struct buf *bp) REFCOUNT_COUNT(obj->paging_in_progress), bp->b_npages)); vp = bp->b_vp; - VNPASS(vp->v_holdcnt > 0, vp); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); VNPASS(vp->v_object != NULL, vp); foff = bp->b_offset; diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c index 47a5193eebf5..59c3fb8f50d4 100644 --- a/sys/kern/vfs_mount.c +++ b/sys/kern/vfs_mount.c @@ -1394,7 +1394,7 @@ vfs_check_usecounts(struct mount *mp) MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { if ((vp->v_vflag & VV_ROOT) == 0 && vp->v_type != VNON && - vp->v_usecount != 0) { + vn_ref_read(vp, v_usecount) != 0) { VI_UNLOCK(vp); MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); return (EBUSY); diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index f9f1f0a541c6..eb27c19d74f4 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -1125,7 +1125,7 @@ vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) * Also skip free vnodes. We are trying to make space * to expand the free list, not reduce it. */ - if (vp->v_usecount > 0 || vp->v_holdcnt == 0 || + if (vn_ref_read(vp, v_usecount) > 0 || vn_ref_read(vp, v_holdcnt) == 0 || (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src))) goto next_iter; @@ -1135,7 +1135,7 @@ vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) if (!VI_TRYLOCK(vp)) goto next_iter; - if (vp->v_usecount > 0 || vp->v_holdcnt == 0 || + if (vn_ref_read(vp, v_usecount) > 0 || vn_ref_read(vp, v_holdcnt) == 0 || (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) || vp->v_type == VBAD || vp->v_type == VNON || (vp->v_object != NULL && @@ -1160,7 +1160,7 @@ vlrureclaim(bool reclaim_nc_src, int trigger, u_long target) } VI_LOCK(vp); - if (vp->v_usecount > 0 || + if (vn_ref_read(vp, v_usecount) > 0 || (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) || (vp->v_object != NULL && vp->v_object->resident_page_count > trigger)) { @@ -1240,7 +1240,8 @@ vnlru_free_locked(int count, struct vfsops *mnt_op) * Don't recycle if we can't get the interlock without * blocking. */ - if (vp->v_holdcnt > 0 || (mnt_op != NULL && (mp = vp->v_mount) != NULL && + if (vn_ref_read(vp, v_holdcnt) > 0 || + (mnt_op != NULL && (mp = vp->v_mount) != NULL && mp->mnt_op != mnt_op) || !VI_TRYLOCK(vp)) { continue; } @@ -1494,7 +1495,7 @@ vtryrecycle(struct vnode *vp) struct mount *vnmp; CTR2(KTR_VFS, "%s: vp %p", __func__, vp); - VNASSERT(vp->v_holdcnt, vp, + VNASSERT(vn_ref_read(vp, v_holdcnt), vp, ("vtryrecycle: Recycling vp %p without a reference.", vp)); /* * This vnode may found and locked via some other list, if so we @@ -1523,7 +1524,7 @@ vtryrecycle(struct vnode *vp) * will skip over it. */ VI_LOCK(vp); - if (vp->v_usecount) { + if (vn_ref_read(vp, v_usecount)) { VOP_UNLOCK(vp); VI_UNLOCK(vp); vn_finished_write(vnmp); @@ -1757,8 +1758,8 @@ freevnode(struct vnode *vp) CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); bo = &vp->v_bufobj; VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); - VNASSERT(vp->v_holdcnt == 0, vp, ("Non-zero hold count")); - VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); + VNASSERT(vn_ref_read(vp, v_holdcnt) == 0, vp, ("Non-zero hold count")); + VNASSERT(vn_ref_read(vp, v_usecount) == 0, vp, ("Non-zero use count")); VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); @@ -2777,8 +2778,8 @@ v_init_counters(struct vnode *vp) vp, ("%s called for an initialized vnode", __FUNCTION__)); ASSERT_VI_UNLOCKED(vp, __FUNCTION__); - refcount_init(&vp->v_holdcnt, 1); - refcount_init(&vp->v_usecount, 1); + vn_ref_init(vp, v_holdcnt, 1); + vn_ref_init(vp, v_usecount, 1); } /* @@ -2828,7 +2829,7 @@ vget_prep(struct vnode *vp) { enum vgetstate vs; - if (refcount_acquire_if_not_zero(&vp->v_usecount)) { + if (vn_ref_acquire_if_not_zero(vp, v_usecount)) { vs = VGET_USECOUNT; } else { vhold(vp); @@ -2857,29 +2858,19 @@ vget_finish_vchr(struct vnode *vp) /* * See the comment in vget_finish before usecount bump. */ - if (refcount_acquire_if_not_zero(&vp->v_usecount)) { -#ifdef INVARIANTS - int old = atomic_fetchadd_int(&vp->v_holdcnt, -1); - VNASSERT(old > 0, vp, ("%s: wrong hold count %d", __func__, old)); -#else - refcount_release(&vp->v_holdcnt); -#endif + if (vn_ref_acquire_if_not_zero(vp, v_usecount)) { + vn_ref_release(vp, v_holdcnt); return (0); } VI_LOCK(vp); - if (refcount_acquire_if_not_zero(&vp->v_usecount)) { -#ifdef INVARIANTS - int old = atomic_fetchadd_int(&vp->v_holdcnt, -1); - VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); -#else - refcount_release(&vp->v_holdcnt); -#endif + if (vn_ref_acquire_if_not_zero(vp, v_usecount)) { + vn_ref_release(vp, v_holdcnt); VI_UNLOCK(vp); return (0); } v_incr_devcount(vp); - refcount_acquire(&vp->v_usecount); + vn_ref_acquire(vp, v_usecount); VI_UNLOCK(vp); return (0); } @@ -2887,7 +2878,7 @@ vget_finish_vchr(struct vnode *vp) int vget_finish(struct vnode *vp, int flags, enum vgetstate vs) { - int error, old; + int error; VNASSERT((flags & LK_TYPE_MASK) != 0, vp, ("%s: invalid lock operation", __func__)); @@ -2896,11 +2887,11 @@ vget_finish(struct vnode *vp, int flags, enum vgetstate vs) ASSERT_VI_LOCKED(vp, __func__); else ASSERT_VI_UNLOCKED(vp, __func__); - VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); if (vs == VGET_USECOUNT) { - VNASSERT(vp->v_usecount > 0, vp, - ("%s: vnode without usecount when VGET_USECOUNT was passed", - __func__)); + VNASSERT(vn_ref_read(vp, v_usecount) > 0, vp, + ("%s: vnode without usecount when VGET_USECOUNT was passed", + __func__)); } error = vn_lock(vp, flags); @@ -2926,16 +2917,8 @@ vget_finish(struct vnode *vp, int flags, enum vgetstate vs) * the vnode around. Otherwise someone else lended their hold count and * we have to drop ours. */ - old = atomic_fetchadd_int(&vp->v_usecount, 1); - VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old)); - if (old != 0) { -#ifdef INVARIANTS - old = atomic_fetchadd_int(&vp->v_holdcnt, -1); - VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); -#else - refcount_release(&vp->v_holdcnt); -#endif - } + if (vn_ref_acquire_ret(vp, v_usecount) != 0) + vn_ref_release(vp, v_holdcnt); return (0); } @@ -2951,10 +2934,9 @@ vref_vchr(struct vnode *vp, bool interlock) * See the comment in vget_finish before usecount bump. */ if (!interlock) { - if (refcount_acquire_if_not_zero(&vp->v_usecount)) { + if (vn_ref_acquire_if_not_zero(vp, v_usecount)) { VNODE_REFCOUNT_FENCE_ACQ(); - VNASSERT(vp->v_holdcnt > 0, vp, - ("%s: active vnode not held", __func__)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); return; } VI_LOCK(vp); @@ -2971,17 +2953,16 @@ vref_vchr(struct vnode *vp, bool interlock) } } VNASSERT(vp->v_type == VCHR, vp, ("type != VCHR)")); - if (refcount_acquire_if_not_zero(&vp->v_usecount)) { + if (vn_ref_acquire_if_not_zero(vp, v_usecount)) { VNODE_REFCOUNT_FENCE_ACQ(); - VNASSERT(vp->v_holdcnt > 0, vp, - ("%s: active vnode not held", __func__)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); if (!interlock) VI_UNLOCK(vp); return; } vhold(vp); v_incr_devcount(vp); - refcount_acquire(&vp->v_usecount); + vn_ref_acquire(vp, v_usecount); if (!interlock) VI_UNLOCK(vp); return; @@ -2990,7 +2971,6 @@ vref_vchr(struct vnode *vp, bool interlock) void vref(struct vnode *vp) { - int old; CTR2(KTR_VFS, "%s: vp %p", __func__, vp); if (__predict_false(vp->v_type == VCHR)) { @@ -2998,26 +2978,17 @@ vref(struct vnode *vp) return; } - if (refcount_acquire_if_not_zero(&vp->v_usecount)) { + if (vn_ref_acquire_if_not_zero(vp, v_usecount)) { VNODE_REFCOUNT_FENCE_ACQ(); - VNASSERT(vp->v_holdcnt > 0, vp, - ("%s: active vnode not held", __func__)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); return; } vhold(vp); /* * See the comment in vget_finish. */ - old = atomic_fetchadd_int(&vp->v_usecount, 1); - VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old)); - if (old != 0) { -#ifdef INVARIANTS - old = atomic_fetchadd_int(&vp->v_holdcnt, -1); - VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old)); -#else - refcount_release(&vp->v_holdcnt); -#endif - } + if (vn_ref_acquire_ret(vp, v_usecount) != 0) + vn_ref_release(vp, v_holdcnt); } void @@ -3038,12 +3009,7 @@ vrefact(struct vnode *vp) { CTR2(KTR_VFS, "%s: vp %p", __func__, vp); -#ifdef INVARIANTS - int old = atomic_fetchadd_int(&vp->v_usecount, 1); - VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old)); -#else - refcount_acquire(&vp->v_usecount); -#endif + vn_ref_acquire_nz(vp, v_usecount); } void @@ -3051,12 +3017,7 @@ vrefactn(struct vnode *vp, u_int n) { CTR2(KTR_VFS, "%s: vp %p", __func__, vp); -#ifdef INVARIANTS - int old = atomic_fetchadd_int(&vp->v_usecount, n); - VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old)); -#else - atomic_add_int(&vp->v_usecount, n); -#endif + vn_ref_acquiren_nz(vp, v_usecount, n); } /* @@ -3072,7 +3033,7 @@ int vrefcnt(struct vnode *vp) { - return (vp->v_usecount); + return (vn_ref_read(vp, v_usecount)); } void @@ -3080,7 +3041,7 @@ vlazy(struct vnode *vp) { struct mount *mp; - VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); if ((vp->v_mflag & VMP_LAZYLIST) != 0) return; @@ -3128,18 +3089,17 @@ vdefer_inactive(struct vnode *vp) { ASSERT_VI_LOCKED(vp, __func__); - VNASSERT(vp->v_holdcnt > 0, vp, - ("%s: vnode without hold count", __func__)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); if (VN_IS_DOOMED(vp)) { vdropl(vp); return; } if (vp->v_iflag & VI_DEFINACT) { - VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); + VNPASS(vn_ref_read(vp, v_holdcnt) > 1, vp); vdropl(vp); return; } - if (vp->v_usecount > 0) { + if (vn_ref_read(vp, v_usecount) > 0) { vp->v_iflag &= ~VI_OWEINACT; vdropl(vp); return; @@ -3185,8 +3145,8 @@ vputx(struct vnode *vp, enum vputx_op func) else if (func == VPUTX_VPUT) ASSERT_VOP_LOCKED(vp, "vput"); ASSERT_VI_UNLOCKED(vp, __func__); - VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp, - ("%s: wrong ref counts", __func__)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0 && + vn_ref_read(vp, v_usecount) > 0, vp); CTR2(KTR_VFS, "%s: vp %p", __func__, vp); @@ -3199,7 +3159,7 @@ vputx(struct vnode *vp, enum vputx_op func) * count which provides liveness of the vnode, in which case we * have to vdrop. */ - if (!refcount_release(&vp->v_usecount)) { + if (!vn_ref_release(vp, v_usecount)) { if (func == VPUTX_VPUT) VOP_UNLOCK(vp); return; @@ -3210,7 +3170,7 @@ vputx(struct vnode *vp, enum vputx_op func) * By the time we got here someone else might have transitioned * the count back to > 0. */ - if (vp->v_usecount > 0 || vp->v_iflag & VI_DOINGINACT) + if (vn_ref_read(vp, v_usecount) > 0 || vp->v_iflag & VI_DOINGINACT) goto out; /* @@ -3302,12 +3262,9 @@ void vhold(struct vnode *vp) { struct vdbatch *vd; - int old; CTR2(KTR_VFS, "%s: vp %p", __func__, vp); - old = atomic_fetchadd_int(&vp->v_holdcnt, 1); - VNASSERT(old >= 0, vp, ("%s: wrong hold count %d", __func__, old)); - if (old != 0) + if (vn_ref_acquire_ret(vp, v_holdcnt) != 0) return; critical_enter(); vd = DPCPU_PTR(vd); @@ -3329,12 +3286,7 @@ vholdnz(struct vnode *vp) { CTR2(KTR_VFS, "%s: vp %p", __func__, vp); -#ifdef INVARIANTS - int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); - VNASSERT(old > 0, vp, ("%s: wrong hold count %d", __func__, old)); -#else - atomic_add_int(&vp->v_holdcnt, 1); -#endif + vn_ref_acquire_nz(vp, v_holdcnt); } static void __noinline @@ -3475,7 +3427,7 @@ vdrop_deactivate(struct vnode *vp) * vnode to the lazy list and is now responsible for its * removal. */ - if (vp->v_holdcnt == 0) { + if (vn_ref_read(vp, v_holdcnt) == 0) { vp->v_mflag &= ~VMP_LAZYLIST; TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist); mp->mnt_lazyvnodelistsize--; @@ -3491,7 +3443,7 @@ vdrop(struct vnode *vp) ASSERT_VI_UNLOCKED(vp, __func__); CTR2(KTR_VFS, "%s: vp %p", __func__, vp); - if (refcount_release_if_not_last(&vp->v_holdcnt)) + if (vn_ref_release_if_not_last(vp, v_holdcnt)) return; VI_LOCK(vp); vdropl(vp); @@ -3503,7 +3455,7 @@ vdropl(struct vnode *vp) ASSERT_VI_LOCKED(vp, __func__); CTR2(KTR_VFS, "%s: vp %p", __func__, vp); - if (!refcount_release(&vp->v_holdcnt)) { + if (!vn_ref_release(vp, v_holdcnt)) { VI_UNLOCK(vp); return; } @@ -3566,7 +3518,7 @@ vinactive(struct vnode *vp) return; if (vp->v_iflag & VI_DOINGINACT) return; - if (vp->v_usecount > 0) { + if (vn_ref_read(vp, v_usecount) > 0) { vp->v_iflag &= ~VI_OWEINACT; return; } @@ -3674,7 +3626,7 @@ vflush(struct mount *mp, int rootrefs, int flags, struct thread *td) * * If FORCECLOSE is set, forcibly close the vnode. */ - if (vp->v_usecount == 0 || (flags & FORCECLOSE)) { + if (vn_ref_read(vp, v_usecount) == 0 || (flags & FORCECLOSE)) { vgonel(vp); } else { busy++; @@ -3693,10 +3645,10 @@ vflush(struct mount *mp, int rootrefs, int flags, struct thread *td) */ VI_LOCK(rootvp); KASSERT(busy > 0, ("vflush: not busy")); - VNASSERT(rootvp->v_usecount >= rootrefs, rootvp, + VNASSERT(vn_ref_read(rootvp, v_usecount) >= rootrefs, rootvp, ("vflush: usecount %d < rootrefs %d", - rootvp->v_usecount, rootrefs)); - if (busy == 1 && rootvp->v_usecount == rootrefs) { + vn_ref_read(rootvp, v_usecount), rootrefs)); + if (busy == 1 && vn_ref_read(rootvp, v_usecount) == rootrefs) { VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK); vgone(rootvp); VOP_UNLOCK(rootvp); @@ -3740,7 +3692,7 @@ vrecyclel(struct vnode *vp) ASSERT_VI_LOCKED(vp, __func__); CTR2(KTR_VFS, "%s: vp %p", __func__, vp); recycled = 0; - if (vp->v_usecount == 0) { + if (vn_ref_read(vp, v_usecount) == 0) { recycled = 1; vgonel(vp); } @@ -3832,8 +3784,7 @@ vgonel(struct vnode *vp) ASSERT_VOP_ELOCKED(vp, "vgonel"); ASSERT_VI_LOCKED(vp, "vgonel"); - VNASSERT(vp->v_holdcnt, vp, - ("vgonel: vp %p has no reference.", vp)); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); CTR2(KTR_VFS, "%s: vp %p", __func__, vp); td = curthread; @@ -3849,17 +3800,17 @@ vgonel(struct vnode *vp) * Check to see if the vnode is in use. If so, we have to call * VOP_CLOSE() and VOP_INACTIVE(). */ - active = vp->v_usecount > 0; + active = vn_ref_read(vp, v_usecount) > 0; oweinact = (vp->v_iflag & VI_OWEINACT) != 0; /* * If we need to do inactive VI_OWEINACT will be set. */ if (vp->v_iflag & VI_DEFINACT) { - VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count")); + VNPASS(vn_ref_read(vp, v_holdcnt) > 1, vp); vp->v_iflag &= ~VI_DEFINACT; vdropl(vp); } else { - VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count")); + VNPASS(vn_ref_read(vp, v_holdcnt) > 0, vp); VI_UNLOCK(vp); } vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM); @@ -3980,7 +3931,8 @@ vn_printf(struct vnode *vp, const char *fmt, ...) printf("%p: ", (void *)vp); printf("type %s\n", typename[vp->v_type]); printf(" usecount %d, writecount %d, refcount %d", - vp->v_usecount, vp->v_writecount, vp->v_holdcnt); + vn_ref_read(vp, v_usecount), vp->v_writecount, + vn_ref_read(vp, v_holdcnt)); switch (vp->v_type) { case VDIR: printf(" mountedhere %p\n", vp->v_mountedhere); @@ -4081,6 +4033,20 @@ vn_printf(struct vnode *vp, const char *fmt, ...) VOP_PRINT(vp); } +#ifdef INVARIANTS +void +vn_ref_assert(void *obj, long count, const char *file, int line, + const char *func) +{ + struct vnode *vp; + + vp = obj; + + vn_printf(vp, "invalid refcount %ld at %s:%d (%s)\n", count, file, line, func); + panic("invalid refcount %ld at %s:%d (%s)\n", count, file, line, func); +} +#endif + #ifdef DDB /* * List all of the locked vnodes in the system. @@ -4296,7 +4262,7 @@ DB_SHOW_COMMAND(mount, db_show_mount) db_printf("\n\nList of active vnodes\n"); TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { - if (vp->v_type != VMARKER && vp->v_holdcnt > 0) { + if (vp->v_type != VMARKER && vn_ref_read(vp, v_holdcnt) > 0) { vn_printf(vp, "vnode "); if (db_pager_quit) break; @@ -4304,7 +4270,7 @@ DB_SHOW_COMMAND(mount, db_show_mount) } db_printf("\n\nList of inactive vnodes\n"); TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) { - if (vp->v_type != VMARKER && vp->v_holdcnt == 0) { + if (vp->v_type != VMARKER && vn_ref_read(vp, v_holdcnt) == 0) { vn_printf(vp, "vnode "); if (db_pager_quit) break; @@ -6256,7 +6222,7 @@ mnt_vnode_next_lazy_relock(struct vnode *mvp, struct mount *mp, * Since we had a period with no locks held we may be the last * remaining user, in which case there is nothing to do. */ - if (!refcount_release_if_not_last(&vp->v_holdcnt)) + if (!vn_ref_release_if_not_last(vp, v_holdcnt)) goto out_lost; mtx_lock(&mp->mnt_listmtx); return (true); diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c index 20ed0dc8f39d..a009d345a753 100644 --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -4157,7 +4157,7 @@ sys_revoke(struct thread *td, struct revoke_args *uap) if (error != 0) goto out; } - if (vp->v_usecount > 1 || vcount(vp) > 1) + if (vn_ref_read(vp, v_usecount) > 1 || vcount(vp) > 1) VOP_REVOKE(vp, REVOKEALL); out: vput(vp); diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index 963550b9e1eb..d60911105f28 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -1607,7 +1607,7 @@ _vn_lock(struct vnode *vp, int flags, char *file, int line) int error; VNASSERT((flags & LK_TYPE_MASK) != 0, vp, ("vn_lock: no locktype")); - VNASSERT(vp->v_holdcnt != 0, vp, ("vn_lock: zero hold count")); + VNPASS(vn_ref_read(vp, v_holdcnt) != 0, vp); error = VOP_LOCK1(vp, flags, file, line); if (__predict_false(error != 0 || VN_IS_DOOMED(vp))) return (_vn_lock_fallback(vp, flags, file, line, error)); diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h index 19c4930263db..899f62b0f920 100644 --- a/sys/sys/vnode.h +++ b/sys/sys/vnode.h @@ -45,6 +45,7 @@ #include #include #include +#include /* * The vnode is the focus of all file activity in UNIX. There is a @@ -167,8 +168,8 @@ struct vnode { daddr_t v_lastw; /* v last write */ int v_clen; /* v length of cur. cluster */ - u_int v_holdcnt; /* I prevents recycling. */ - u_int v_usecount; /* I ref count of users */ + refcntint_t v_holdcnt; /* I prevents recycling. */ + refcntint_t v_usecount; /* I ref count of users */ u_int v_iflag; /* i vnode flags (see below) */ u_int v_vflag; /* v vnode flags */ u_short v_mflag; /* l mnt-specific vnode flags */ @@ -968,6 +969,39 @@ int vn_dir_check_exec(struct vnode *vp, struct componentname *cnp); _error; \ }) +#ifdef INVARIANTS +void vn_ref_assert(void *, long, const char *, int, const char *); +#else +#define vn_ref_assert NULL +#endif + +#define vn_ref_init(vp, counter, n) \ + refcntint_init(&vp->counter, n) + +#define vn_ref_read(vp, counter) \ + refcntint_read(&vp->counter) + +#define vn_ref_acquire(vp, counter) \ + _refcntint_acquire(&vp->counter, vn_ref_assert, vp) + +#define vn_ref_acquire_ret(vp, counter) \ + _refcntint_acquire_ret(&vp->counter, vn_ref_assert, vp) + +#define vn_ref_acquire_nz(vp, counter) \ + _refcntint_acquire_nz(&vp->counter, vn_ref_assert, vp) + +#define vn_ref_acquiren_nz(vp, counter, n) \ + _refcntint_acquiren_nz(&vp->counter, n, vn_ref_assert, vp) + +#define vn_ref_release(vp, counter) \ + _refcntint_release(&vp->counter, vn_ref_assert, vp) + +#define vn_ref_acquire_if_not_zero(vp, counter) \ + _refcntint_acquire_if_gt(&vp->counter, 0, vn_ref_assert, vp) + +#define vn_ref_release_if_not_last(vp, counter) \ + _refcntint_release_if_gt(&vp->counter, 0, vn_ref_assert, vp) + #include #define VFS_VOP_VECTOR_REGISTER(vnodeops) \ diff --git a/sys/ufs/ffs/ffs_snapshot.c b/sys/ufs/ffs/ffs_snapshot.c index c0ce8f02f6fd..c6b868fa7391 100644 --- a/sys/ufs/ffs/ffs_snapshot.c +++ b/sys/ufs/ffs/ffs_snapshot.c @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include @@ -534,7 +535,7 @@ ffs_snapshot(mp, snapfile) MNT_IUNLOCK(mp); loop: MNT_VNODE_FOREACH_ALL(xvp, mp, mvp) { - if ((xvp->v_usecount == 0 && + if ((vn_ref_read(xvp, v_usecount) == 0 && (xvp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == 0) || xvp->v_type == VNON || IS_SNAPSHOT(VTOI(xvp))) { @@ -556,7 +557,7 @@ ffs_snapshot(mp, snapfile) goto loop; } VI_LOCK(xvp); - if (xvp->v_usecount == 0 && + if (vn_ref_read(xvp, v_usecount) == 0 && (xvp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == 0) { VI_UNLOCK(xvp); VOP_UNLOCK(xvp); @@ -2564,7 +2565,7 @@ process_deferred_inactive(struct mount *mp) */ if (vp->v_type == VNON || ((VTOI(vp)->i_flag & IN_LAZYACCESS) == 0 && - ((vp->v_iflag & VI_OWEINACT) == 0 || vp->v_usecount > 0))) { + ((vp->v_iflag & VI_OWEINACT) == 0 || vn_ref_read(vp, v_usecount) > 0))) { VI_UNLOCK(vp); continue; } diff --git a/sys/ufs/ffs/ffs_vnops.c b/sys/ufs/ffs/ffs_vnops.c index 48121feb1996..895bc41b6685 100644 --- a/sys/ufs/ffs/ffs_vnops.c +++ b/sys/ufs/ffs/ffs_vnops.c @@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -441,7 +442,7 @@ ffs_lock(ap) flags = ap->a_flags; for (;;) { #ifdef DEBUG_VFS_LOCKS - VNPASS(vp->v_holdcnt != 0, vp); + VNPASS(vn_ref_read(vp, v_holdcnt) != 0, vp); #endif lkp = vp->v_vnlock; result = _lockmgr_args(lkp, flags, VI_MTX(vp), diff --git a/sys/ufs/ufs/ufs_gjournal.c b/sys/ufs/ufs/ufs_gjournal.c index a41dc7c04a88..2885b10856f3 100644 --- a/sys/ufs/ufs/ufs_gjournal.c +++ b/sys/ufs/ufs/ufs_gjournal.c @@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -104,7 +105,7 @@ ufs_gjournal_orphan(struct vnode *vp) if (vp->v_mount->mnt_gjprovider == NULL) return; - if (vp->v_usecount < 2 || (vp->v_vflag & VV_DELETED)) + if (vn_ref_read(vp, v_usecount) < 2 || (vp->v_vflag & VV_DELETED)) return; ip = VTOI(vp); if ((vp->v_type == VDIR && ip->i_nlink > 2) || diff --git a/sys/ufs/ufs/ufs_vnops.c b/sys/ufs/ufs/ufs_vnops.c index 4805f205ea22..88965abb6dc0 100644 --- a/sys/ufs/ufs/ufs_vnops.c +++ b/sys/ufs/ufs/ufs_vnops.c @@ -305,7 +305,7 @@ ufs_close(ap) int usecount; VI_LOCK(vp); - usecount = vp->v_usecount; + usecount = vn_ref_read(vp, v_usecount); if (usecount > 1) ufs_itimes_locked(vp); VI_UNLOCK(vp); @@ -2373,7 +2373,7 @@ ufsfifo_close(ap) int usecount; VI_LOCK(vp); - usecount = vp->v_usecount; + usecount = vn_ref_read(vp, v_usecount); if (usecount > 1) ufs_itimes_locked(vp); VI_UNLOCK(vp); diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index 3216ef56579d..73ec37a0f157 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -248,7 +248,7 @@ vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vp = (struct vnode *)handle; ASSERT_VOP_LOCKED(vp, "vnode_pager_alloc"); - VNPASS(vp->v_usecount > 0, vp); + VNPASS(vn_ref_read(vp, v_usecount) > 0, vp); retry: object = vp->v_object;