diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c index 03bafc75acaa..59c778f922b0 100644 --- a/sys/kern/vfs_cache.c +++ b/sys/kern/vfs_cache.c @@ -1416,10 +1416,14 @@ static void cache_zap_locked(struct namecache *ncp) { struct nchashhead *ncpp; + struct vnode *dvp, *vp; + + dvp = ncp->nc_dvp; + vp = ncp->nc_vp; if (!(ncp->nc_flag & NCF_NEGATIVE)) - cache_assert_vnode_locked(ncp->nc_vp); - cache_assert_vnode_locked(ncp->nc_dvp); + cache_assert_vnode_locked(vp); + cache_assert_vnode_locked(dvp); cache_assert_bucket_locked(ncp); cache_ncp_invalidate(ncp); @@ -1427,28 +1431,22 @@ cache_zap_locked(struct namecache *ncp) ncpp = NCP2BUCKET(ncp); CK_SLIST_REMOVE(ncpp, ncp, namecache, nc_hash); if (!(ncp->nc_flag & NCF_NEGATIVE)) { - SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp, - ncp->nc_name, ncp->nc_vp); - TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst); - if (ncp == ncp->nc_vp->v_cache_dd) { - vn_seqc_write_begin_unheld(ncp->nc_vp); - ncp->nc_vp->v_cache_dd = NULL; - vn_seqc_write_end(ncp->nc_vp); + SDT_PROBE3(vfs, namecache, zap, done, dvp, ncp->nc_name, vp); + TAILQ_REMOVE(&vp->v_cache_dst, ncp, nc_dst); + if (ncp == vp->v_cache_dd) { + atomic_store_ptr(&vp->v_cache_dd, NULL); } } else { - SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp, - ncp->nc_name); + SDT_PROBE2(vfs, namecache, zap_negative, done, dvp, ncp->nc_name); cache_neg_remove(ncp); } if (ncp->nc_flag & NCF_ISDOTDOT) { - if (ncp == ncp->nc_dvp->v_cache_dd) { - vn_seqc_write_begin_unheld(ncp->nc_dvp); - ncp->nc_dvp->v_cache_dd = NULL; - vn_seqc_write_end(ncp->nc_dvp); + if (ncp == dvp->v_cache_dd) { + atomic_store_ptr(&dvp->v_cache_dd, NULL); } } else { LIST_REMOVE(ncp, nc_src); - if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) { + if (LIST_EMPTY(&dvp->v_cache_src)) { ncp->nc_flag |= NCF_DVDROP; } } @@ -1602,7 +1600,7 @@ cache_remove_cnp(struct vnode *dvp, struct componentname *cnp) dvlp2 = NULL; mtx_lock(dvlp); retry_dotdot: - ncp = dvp->v_cache_dd; + ncp = atomic_load_ptr(&dvp->v_cache_dd); if (ncp == NULL) { mtx_unlock(dvlp); if (dvlp2 != NULL) @@ -1619,9 +1617,11 @@ cache_remove_cnp(struct vnode *dvp, struct componentname *cnp) mtx_unlock(dvlp2); cache_free(ncp); } else { - vn_seqc_write_begin(dvp); - dvp->v_cache_dd = NULL; - vn_seqc_write_end(dvp); + VI_LOCK(dvp); + vn_seqc_write_begin_locked(dvp); + atomic_store_ptr(&dvp->v_cache_dd, NULL); + vn_seqc_write_end_locked(dvp); + VI_UNLOCK(dvp); mtx_unlock(dvlp); if (dvlp2 != NULL) mtx_unlock(dvlp2); @@ -1720,7 +1720,7 @@ cache_lookup_dotdot(struct vnode *dvp, struct vnode **vpp, struct componentname retry: dvlp = VP2VNODELOCK(dvp); mtx_lock(dvlp); - ncp = dvp->v_cache_dd; + ncp = atomic_load_ptr(&dvp->v_cache_dd); if (ncp == NULL) { SDT_PROBE3(vfs, namecache, lookup, miss, dvp, "..", NULL); mtx_unlock(dvlp); @@ -2148,7 +2148,7 @@ cache_enter_lock(struct celockstate *cel, struct vnode *dvp, struct vnode *vp, cache_lock_vnodes_cel(cel, dvp, vp); if (vp == NULL || vp->v_type != VDIR) break; - ncp = vp->v_cache_dd; + ncp = atomic_load_ptr(&vp->v_cache_dd); if (ncp == NULL) break; if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) @@ -2187,7 +2187,7 @@ cache_enter_lock_dd(struct celockstate *cel, struct vnode *dvp, struct vnode *vp for (;;) { blps[1] = NULL; cache_lock_vnodes_cel(cel, dvp, vp); - ncp = dvp->v_cache_dd; + ncp = atomic_load_ptr(&dvp->v_cache_dd); if (ncp == NULL) break; if ((ncp->nc_flag & NCF_ISDOTDOT) == 0) @@ -2235,14 +2235,14 @@ cache_enter_dotdot_prep(struct vnode *dvp, struct vnode *vp, hash = cache_get_hash(cnp->cn_nameptr, len, dvp); cache_enter_lock_dd(&cel, dvp, vp, hash); vn_seqc_write_begin(dvp); - ncp = dvp->v_cache_dd; + ncp = atomic_load_ptr(&dvp->v_cache_dd); if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT)) { KASSERT(ncp->nc_dvp == dvp, ("wrong isdotdot parent")); cache_zap_locked(ncp); } else { ncp = NULL; } - dvp->v_cache_dd = NULL; + atomic_store_ptr(&dvp->v_cache_dd, NULL); vn_seqc_write_end(dvp); cache_enter_unlock(&cel); if (ncp != NULL) @@ -2380,9 +2380,8 @@ cache_enter_time(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, goto out_unlock_free; KASSERT(vp == NULL || vp->v_type == VDIR, ("wrong vnode type %p", vp)); - vn_seqc_write_begin(dvp); - dvp->v_cache_dd = ncp; - vn_seqc_write_end(dvp); + atomic_thread_fence_rel(); + atomic_store_ptr(&dvp->v_cache_dd, ncp); } if (vp != NULL) { @@ -2391,21 +2390,30 @@ cache_enter_time(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, * For this case, the cache entry maps both the * directory name in it and the name ".." for the * directory's parent. + * + * Don't bother with seqc unless we are redirecting the entry. */ - vn_seqc_write_begin(vp); - if ((ndd = vp->v_cache_dd) != NULL) { - if ((ndd->nc_flag & NCF_ISDOTDOT) != 0) + atomic_thread_fence_rel(); + ndd = vp->v_cache_dd; + if (vp->v_cache_dd == NULL) { + atomic_store_ptr(&vp->v_cache_dd, ncp); + } else { + vn_seqc_write_begin(vp); + if ((ndd->nc_flag & NCF_ISDOTDOT) != 0) { cache_zap_locked(ndd); - else + } else { ndd = NULL; + } + atomic_store_ptr(&vp->v_cache_dd, ncp); + vn_seqc_write_end(vp); } - vp->v_cache_dd = ncp; - vn_seqc_write_end(vp); } else if (vp->v_type != VDIR) { if (vp->v_cache_dd != NULL) { - vn_seqc_write_begin(vp); - vp->v_cache_dd = NULL; - vn_seqc_write_end(vp); + VI_LOCK(vp); + vn_seqc_write_begin_locked(vp); + atomic_store_ptr(&vp->v_cache_dd, NULL); + vn_seqc_write_end_locked(vp); + VI_UNLOCK(vp); } } } @@ -2545,7 +2553,7 @@ cache_vnode_init(struct vnode *vp) LIST_INIT(&vp->v_cache_src); TAILQ_INIT(&vp->v_cache_dst); - vp->v_cache_dd = NULL; + atomic_store_ptr(&vp->v_cache_dd, NULL); cache_prehash(vp); } @@ -2623,7 +2631,7 @@ cache_purge_impl(struct vnode *vp) goto retry; TAILQ_INSERT_TAIL(&batch, ncp, nc_dst); } - ncp = vp->v_cache_dd; + ncp = atomic_load_ptr(&vp->v_cache_dd); if (ncp != NULL) { KASSERT(ncp->nc_flag & NCF_ISDOTDOT, ("lost dotdot link")); @@ -3015,7 +3023,7 @@ vn_vptocnp(struct vnode **vp, char *buf, size_t *buflen) vlp = VP2VNODELOCK(*vp); mtx_lock(vlp); - ncp = (*vp)->v_cache_dd; + ncp = atomic_load_ptr(&(*vp)->v_cache_dd); if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT) == 0) { KASSERT(ncp == vn_dd_from_dst(*vp), ("%s: mismatch for dd entry (%p != %p)", __func__, @@ -3289,10 +3297,6 @@ vn_fullpath_any_smr(struct vnode *vp, struct vnode *rdir, char *buf, cache_rev_failed(&reason); goto out_abort; } - if (!cache_ncp_canuse(ncp)) { - cache_rev_failed(&reason); - goto out_abort; - } if (ncp->nc_nlen >= *buflen) { cache_rev_failed(&reason); error = ENOMEM; @@ -3308,6 +3312,10 @@ vn_fullpath_any_smr(struct vnode *vp, struct vnode *rdir, char *buf, cache_rev_failed(&reason); goto out_abort; } + if (!cache_ncp_canuse(ncp)) { + cache_rev_failed(&reason); + goto out_abort; + } if (!vn_seqc_consistent(vp, vp_seqc)) { cache_rev_failed(&reason); goto out_abort;