diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/arc_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/arc_os.c index 4fc7468bfa47..e063bf5df03c 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/arc_os.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/arc_os.c @@ -51,6 +51,10 @@ #include #include +static struct vnode *arc_vnlru_marker; +static int arc_vnlru_marker_busy; +static long vnlru_running; + extern struct vfsops zfs_vfsops; uint_t zfs_arc_free_target = 0; @@ -153,11 +157,34 @@ arc_default_max(uint64_t min, uint64_t allmem) static void arc_prune_task(void *arg) { + struct vnode *marker; int64_t nr_scan = *(int64_t *)arg; + static bool warned; + + if (atomic_fetchadd_long(&vnlru_running, 1) > 0) { + if (!warned) { + printf("%s: already running!\n", __func__); + warned = true; + } + } arc_reduce_target_size(ptob(nr_scan)); free(arg, M_TEMP); - vnlru_free(nr_scan, &zfs_vfsops); + marker = vnlru_alloc_marker(M_NOWAIT); + if (marker != NULL) { + vnlru_free_vfsops(nr_scan, &zfs_vfsops, marker); + vnlru_free_marker(marker); + atomic_subtract_long(&vnlru_running, 1); + return; + } + + if (atomic_fetchadd_int(&arc_vnlru_marker_busy, 1) > 0) { + atomic_subtract_long(&vnlru_running, 1); + return; + } + vnlru_free_vfsops(nr_scan, &zfs_vfsops, arc_vnlru_marker); + atomic_subtract_int(&arc_vnlru_marker_busy, 1); + atomic_subtract_long(&vnlru_running, 1); } /* @@ -234,7 +261,7 @@ arc_lowmem_init(void) { arc_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, arc_lowmem, NULL, EVENTHANDLER_PRI_FIRST); - + arc_vnlru_marker = vnlru_alloc_marker(M_WAITOK); } void @@ -242,6 +269,8 @@ arc_lowmem_fini(void) { if (arc_event_lowmem != NULL) EVENTHANDLER_DEREGISTER(vm_lowmem, arc_event_lowmem); + if (arc_vnlru_marker != NULL) + vnlru_free_marker(arc_vnlru_marker); } void diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index 37e713ee48ea..7f060daada86 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -508,11 +508,11 @@ PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free, static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker"); static struct vnode * -vn_alloc_marker(struct mount *mp) +vn_alloc_marker(struct mount *mp, int flags) { struct vnode *vp; - vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO); + vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, flags | M_ZERO); vp->v_type = VMARKER; vp->v_mount = mp; @@ -654,9 +654,9 @@ vntblinit(void *dummy __unused) mtx_lock(&vnode_list_mtx); vnlru_recalc(); mtx_unlock(&vnode_list_mtx); - vnode_list_free_marker = vn_alloc_marker(NULL); + vnode_list_free_marker = vn_alloc_marker(NULL, M_WAITOK); TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist); - vnode_list_reclaim_marker = vn_alloc_marker(NULL); + vnode_list_reclaim_marker = vn_alloc_marker(NULL, M_WAITOK); TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist); vnode_zone = uma_zcreate("VNODE", sizeof (struct vnode), NULL, NULL, vnode_init, vnode_fini, UMA_ALIGN_PTR, 0); @@ -1216,9 +1216,9 @@ SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_vnlru_free, * Attempt to reduce the free list by the requested amount. */ static int -vnlru_free_locked(int count, struct vfsops *mnt_op) +vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp) { - struct vnode *vp, *mvp; + struct vnode *vp; struct mount *mp; int ocount; @@ -1226,7 +1226,6 @@ vnlru_free_locked(int count, struct vfsops *mnt_op) if (count > max_vnlru_free) count = max_vnlru_free; ocount = count; - mvp = vnode_list_free_marker; vp = mvp; for (;;) { if (count == 0) { @@ -1268,15 +1267,51 @@ vnlru_free_locked(int count, struct vfsops *mnt_op) return (ocount - count); } +static int +vnlru_free_locked(int count) +{ + + mtx_assert(&vnode_list_mtx, MA_OWNED); + return (vnlru_free_impl(count, NULL, vnode_list_free_marker)); +} + void -vnlru_free(int count, struct vfsops *mnt_op) +vnlru_free(int count) { mtx_lock(&vnode_list_mtx); - vnlru_free_locked(count, mnt_op); + vnlru_free_locked(count); mtx_unlock(&vnode_list_mtx); } +void +vnlru_free_vfsops(int count, struct vfsops *mnt_op, struct vnode *mvp) +{ + + MPASS(mnt_op != NULL); + MPASS(mvp != NULL); + VNPASS(mvp->v_type == VMARKER, mvp); + mtx_lock(&vnode_list_mtx); + TAILQ_INSERT_BEFORE(vnode_list_free_marker, mvp, v_vnodelist); + vnlru_free_impl(count, mnt_op, mvp); + TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); + mtx_unlock(&vnode_list_mtx); +} + +struct vnode * +vnlru_alloc_marker(int flags) +{ + + return (vn_alloc_marker(NULL, flags)); +} + +void +vnlru_free_marker(struct vnode *vp) +{ + + return (vn_free_marker(vp)); +} + static void vnlru_recalc(void) { @@ -1423,7 +1458,7 @@ vnlru_proc(void) * try to reduce it by discarding from the free list. */ if (rnumvnodes > desiredvnodes) { - vnlru_free_locked(rnumvnodes - desiredvnodes, NULL); + vnlru_free_locked(rnumvnodes - desiredvnodes); rnumvnodes = atomic_load_long(&numvnodes); } /* @@ -1615,7 +1650,7 @@ vn_alloc_hard(struct mount *mp) * should be chosen so that we never wait or even reclaim from * the free list to below its target minimum. */ - if (vnlru_free_locked(1, NULL) > 0) + if (vnlru_free_locked(1) > 0) goto alloc; if (mp == NULL || (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { /* @@ -1625,7 +1660,7 @@ vn_alloc_hard(struct mount *mp) msleep(&vnlruproc_sig, &vnode_list_mtx, PVFS, "vlruwk", hz); if (atomic_load_long(&numvnodes) + 1 > desiredvnodes && vnlru_read_freevnodes() > 1) - vnlru_free_locked(1, NULL); + vnlru_free_locked(1); } alloc: rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1; @@ -6534,7 +6569,7 @@ __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp) { struct vnode *vp; - *mvp = vn_alloc_marker(mp); + *mvp = vn_alloc_marker(mp, M_WAITOK); MNT_ILOCK(mp); MNT_REF(mp); @@ -6732,7 +6767,7 @@ __mnt_vnode_first_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist)) return (NULL); - *mvp = vn_alloc_marker(mp); + *mvp = vn_alloc_marker(mp, M_WAITOK); MNT_ILOCK(mp); MNT_REF(mp); MNT_IUNLOCK(mp); diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h index 0e46bea14b64..05cb5d3d8e83 100644 --- a/sys/sys/vnode.h +++ b/sys/sys/vnode.h @@ -826,7 +826,10 @@ void vfs_timestamp(struct timespec *); void vfs_write_resume(struct mount *mp, int flags); int vfs_write_suspend(struct mount *mp, int flags); int vfs_write_suspend_umnt(struct mount *mp); -void vnlru_free(int, struct vfsops *); +struct vnode *vnlru_alloc_marker(int); +void vnlru_free_marker(struct vnode *); +void vnlru_free(int); +void vnlru_free_vfsops(int, struct vfsops *, struct vnode *); int vop_stdbmap(struct vop_bmap_args *); int vop_stdfdatasync_buf(struct vop_fdatasync_args *); int vop_stdfsync(struct vop_fsync_args *);