Index: src/sys/kern/kern_event.c =================================================================== --- src/sys/kern/kern_event.c (revision 179) +++ src/sys/kern/kern_event.c (working copy) @@ -185,12 +185,31 @@ } while (0) #define KN_LIST_LOCK(kn) do { \ if (kn->kn_knlist != NULL) \ - mtx_lock(kn->kn_knlist->kl_lock); \ + kn->kn_knlist->kl_lock(kn->kn_knlist->kl_lockarg); \ } while (0) #define KN_LIST_UNLOCK(kn) do { \ - if (kn->kn_knlist != NULL) \ - mtx_unlock(kn->kn_knlist->kl_lock); \ + if (kn->kn_knlist != NULL) \ + kn->kn_knlist->kl_unlock(kn->kn_knlist->kl_lockarg); \ } while (0) +#define KN_LIST_ASSERT_LOCK(knl, islocked) do { \ + if (islocked) \ + KN_LIST_ASSERT_LOCKED(knl); \ + else \ + KN_LIST_ASSERT_UNLOCKED(knl); \ +} while (0) +#ifdef INVARIANTS +#define KN_LIST_ASSERT_LOCKED(knl) do { \ + if (!knl->kl_locked((knl)->kl_lockarg)) \ + panic("knlist not locked, but should be"); \ +} while (0) +#define KN_LIST_ASSERT_UNLOCKED(knl) do { \ + if (knl->kl_locked((knl)->kl_lockarg)) \ + panic("knlist locked, but should not be"); \ +} while (0) +#else /* !INVARIANTS */ +#define KN_LIST_ASSERT_LOCKED(knl) ; +#define KN_LIST_ASSERT_UNLOCKED(knl) ; +#endif /* INVARIANTS */ #define KN_HASHSIZE 64 /* XXX should be tunable */ #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) @@ -1507,9 +1526,11 @@ if (list == NULL) return; - mtx_assert(list->kl_lock, islocked ? MA_OWNED : MA_NOTOWNED); - if (!islocked) - mtx_lock(list->kl_lock); + KN_LIST_ASSERT_LOCK(list, islocked); + + if (!islocked) + list->kl_lock(list->kl_lockarg); + /* * If we unlock the list lock (and set KN_INFLUX), we can eliminate * the kqueue scheduling, but this will introduce four @@ -1533,7 +1554,7 @@ kq = NULL; } if (!islocked) - mtx_unlock(list->kl_lock); + list->kl_unlock(list->kl_lockarg); } /* @@ -1542,15 +1563,15 @@ void knlist_add(struct knlist *knl, struct knote *kn, int islocked) { - mtx_assert(knl->kl_lock, islocked ? MA_OWNED : MA_NOTOWNED); + KN_LIST_ASSERT_LOCK(knl, islocked); KQ_NOTOWNED(kn->kn_kq); KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) == (KN_INFLUX|KN_DETACHED), ("knote not KN_INFLUX and KN_DETACHED")); if (!islocked) - mtx_lock(knl->kl_lock); + knl->kl_lock(knl->kl_lockarg); SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext); if (!islocked) - mtx_unlock(knl->kl_lock); + knl->kl_unlock(knl->kl_lockarg); KQ_LOCK(kn->kn_kq); kn->kn_knlist = knl; kn->kn_status &= ~KN_DETACHED; @@ -1561,17 +1582,17 @@ knlist_remove_kq(struct knlist *knl, struct knote *kn, int knlislocked, int kqislocked) { KASSERT(!(!!kqislocked && !knlislocked), ("kq locked w/o knl locked")); - mtx_assert(knl->kl_lock, knlislocked ? MA_OWNED : MA_NOTOWNED); + KN_LIST_ASSERT_LOCK(knl, knlislocked); mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED); if (!kqislocked) KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) == KN_INFLUX, ("knlist_remove called w/o knote being KN_INFLUX or already removed")); if (!knlislocked) - mtx_lock(knl->kl_lock); + knl->kl_lock(knl->kl_lockarg); SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext); kn->kn_knlist = NULL; if (!knlislocked) - mtx_unlock(knl->kl_lock); + knl->kl_unlock(knl->kl_lockarg); if (!kqislocked) KQ_LOCK(kn->kn_kq); kn->kn_status |= KN_DETACHED; @@ -1603,24 +1624,48 @@ int knlist_empty(struct knlist *knl) { - - mtx_assert(knl->kl_lock, MA_OWNED); + KN_LIST_ASSERT_LOCKED(knl); return SLIST_EMPTY(&knl->kl_list); } static struct mtx knlist_lock; MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects", MTX_DEF); +static void knlist_mtx_lock(void *arg); +static void knlist_mtx_unlock(void *arg); +static int knlist_mtx_locked(void *arg); +static void +knlist_mtx_lock(void *arg) +{ + mtx_lock(arg); +} + +static void +knlist_mtx_unlock(void *arg) +{ + mtx_unlock(arg); +} + +static int +knlist_mtx_locked(void *arg) +{ + return (mtx_owned((struct mtx *)arg)); +} + void knlist_init(struct knlist *knl, struct mtx *mtx) { if (mtx == NULL) - knl->kl_lock = &knlist_lock; + knl->kl_lockarg = &knlist_lock; else - knl->kl_lock = mtx; + knl->kl_lockarg = mtx; + knl->kl_lock = knlist_mtx_lock; + knl->kl_unlock = knlist_mtx_unlock; + knl->kl_locked = knlist_mtx_locked; + SLIST_INIT(&knl->kl_list); } @@ -1637,7 +1682,7 @@ printf("WARNING: destroying knlist w/ knotes on it!\n"); #endif - knl->kl_lock = NULL; + knl->kl_lockarg = knl->kl_lock = knl->kl_unlock = NULL; SLIST_INIT(&knl->kl_list); } @@ -1652,11 +1697,11 @@ struct kqueue *kq; if (islocked) - mtx_assert(knl->kl_lock, MA_OWNED); + KN_LIST_ASSERT_LOCKED(knl); else { - mtx_assert(knl->kl_lock, MA_NOTOWNED); + KN_LIST_ASSERT_UNLOCKED(knl); again: /* need to reaquire lock since we have dropped it */ - mtx_lock(knl->kl_lock); + knl->kl_lock(knl->kl_lockarg); } SLIST_FOREACH(kn, &knl->kl_list, kn_selnext) { @@ -1686,7 +1731,7 @@ KQ_LOCK(kq); KASSERT(kn->kn_status & KN_INFLUX, ("knote removed w/o list lock")); - mtx_unlock(knl->kl_lock); + knl->kl_unlock(knl->kl_lockarg); kq->kq_state |= KQ_FLUXWAIT; msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqkclr", 0); kq = NULL; @@ -1694,10 +1739,10 @@ } if (islocked) - mtx_assert(knl->kl_lock, MA_OWNED); + KN_LIST_ASSERT_LOCKED(knl); else { - mtx_unlock(knl->kl_lock); - mtx_assert(knl->kl_lock, MA_NOTOWNED); + knl->kl_unlock(knl->kl_lockarg); + KN_LIST_ASSERT_UNLOCKED(knl); } } Index: src/sys/kern/vfs_subr.c =================================================================== --- src/sys/kern/vfs_subr.c (revision 179) +++ src/sys/kern/vfs_subr.c (working copy) @@ -3453,7 +3453,7 @@ struct vop_create_args *a = ap; if (!rc) - VFS_SEND_KNOTE(a->a_dvp, NOTE_WRITE); + VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE); } void @@ -3462,8 +3462,8 @@ struct vop_link_args *a = ap; if (!rc) { - VFS_SEND_KNOTE(a->a_vp, NOTE_LINK); - VFS_SEND_KNOTE(a->a_tdvp, NOTE_WRITE); + VFS_KNOTE_LOCKED(a->a_vp, NOTE_LINK); + VFS_KNOTE_LOCKED(a->a_tdvp, NOTE_WRITE); } } @@ -3473,7 +3473,7 @@ struct vop_mkdir_args *a = ap; if (!rc) - VFS_SEND_KNOTE(a->a_dvp, NOTE_WRITE | NOTE_LINK); + VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK); } void @@ -3482,7 +3482,7 @@ struct vop_mknod_args *a = ap; if (!rc) - VFS_SEND_KNOTE(a->a_dvp, NOTE_WRITE); + VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE); } void @@ -3491,8 +3491,8 @@ struct vop_remove_args *a = ap; if (!rc) { - VFS_SEND_KNOTE(a->a_dvp, NOTE_WRITE); - VFS_SEND_KNOTE(a->a_vp, NOTE_DELETE); + VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE); + VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE); } } @@ -3502,11 +3502,11 @@ struct vop_rename_args *a = ap; if (!rc) { - VFS_SEND_KNOTE(a->a_fdvp, NOTE_WRITE); - VFS_SEND_KNOTE(a->a_tdvp, NOTE_WRITE); - VFS_SEND_KNOTE(a->a_fvp, NOTE_RENAME); + VFS_KNOTE_UNLOCKED(a->a_fdvp, NOTE_WRITE); + VFS_KNOTE_UNLOCKED(a->a_tdvp, NOTE_WRITE); + VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME); if (a->a_tvp) - VFS_SEND_KNOTE(a->a_tvp, NOTE_DELETE); + VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE); } if (a->a_tdvp != a->a_fdvp) vdrop(a->a_fdvp); @@ -3523,8 +3523,8 @@ struct vop_rmdir_args *a = ap; if (!rc) { - VFS_SEND_KNOTE(a->a_dvp, NOTE_WRITE | NOTE_LINK); - VFS_SEND_KNOTE(a->a_vp, NOTE_DELETE); + VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK); + VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE); } } @@ -3534,7 +3534,7 @@ struct vop_setattr_args *a = ap; if (!rc) - VFS_SEND_KNOTE(a->a_vp, NOTE_ATTRIB); + VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB); } void @@ -3543,7 +3543,7 @@ struct vop_symlink_args *a = ap; if (!rc) - VFS_SEND_KNOTE(a->a_dvp, NOTE_WRITE); + VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE); } static struct knlist fs_knlist; @@ -3638,6 +3638,9 @@ static int filt_vfswrite(struct knote *kn, long hint); static int filt_vfsvnode(struct knote *kn, long hint); static void filt_vfsdetach(struct knote *kn); +static void vfs_knllock(void *arg); +static void vfs_knlunlock(void *arg); +static int vfs_knllocked(void *arg); static struct filterops vfsread_filtops = { 1, NULL, filt_vfsdetach, filt_vfsread }; @@ -3646,11 +3649,36 @@ static struct filterops vfsvnode_filtops = { 1, NULL, filt_vfsdetach, filt_vfsvnode }; +static void +vfs_knllock(void *arg) +{ + struct vnode *vp = arg; + + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curthread); +} + +static void +vfs_knlunlock(void *arg) +{ + struct vnode *vp = arg; + + VOP_UNLOCK(vp, 0, curthread); +} + +static int +vfs_knllocked(void *arg) +{ + struct vnode *vp = arg; + + return (VOP_ISLOCKED(vp, curthread) == LK_EXCLUSIVE); +} + int vfs_kqfilter(struct vop_kqfilter_args *ap) { struct vnode *vp = ap->a_vp; struct knote *kn = ap->a_kn; + struct knlist *knl; switch (kn->kn_filter) { case EVFILT_READ: @@ -3672,7 +3700,13 @@ v_addpollinfo(vp); if (vp->v_pollinfo == NULL) return (ENOMEM); - knlist_add(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0); + knl = &vp->v_pollinfo->vpi_selinfo.si_note; + knl->kl_lock = vfs_knllock; + knl->kl_unlock = vfs_knlunlock; + knl->kl_locked = vfs_knllocked; + knl->kl_lockarg = vp; + + knlist_add(knl, kn, 0); return (0); } @@ -3705,11 +3739,8 @@ return (1); } - vn_lock(vp, LK_SHARED | LK_RETRY, curthread); if (VOP_GETATTR(vp, &va, curthread->td_ucred, curthread)) return (0); - if (VOP_UNLOCK(vp, 0, curthread)) - return (0); kn->kn_data = va.va_size - kn->kn_fp->f_offset; return (kn->kn_data != 0); Index: src/sys/sys/mount.h =================================================================== --- src/sys/sys/mount.h (revision 179) +++ src/sys/sys/mount.h (working copy) @@ -550,13 +550,20 @@ mtx_assert(&Giant, MA_OWNED); \ } while (0) -#define VFS_SEND_KNOTE(vp, hint) do \ +#define VFS_KNOTE_LOCKED(vp, hint) do \ { \ if ((vp)->v_mount && \ ((vp)->v_mount->mnt_kern_flag & MNTK_NOKNOTE) == 0) \ - VN_KNOTE_UNLOCKED((vp), (hint)); \ + VN_KNOTE((vp), (hint), 1); \ } while (0) +#define VFS_KNOTE_UNLOCKED(vp, hint) do \ +{ \ + if ((vp)->v_mount && \ + ((vp)->v_mount->mnt_kern_flag & MNTK_NOKNOTE) == 0) \ + VN_KNOTE((vp), (hint), 0); \ +} while (0) + #include /* Index: src/sys/sys/event.h =================================================================== --- src/sys/sys/event.h (revision 179) +++ src/sys/sys/event.h (working copy) @@ -126,8 +126,11 @@ struct kqueue; SLIST_HEAD(kqlist, kqueue); struct knlist { - struct mtx *kl_lock; /* lock to protect kll_list */ struct klist kl_list; + void (*kl_lock)(void *); /* lock function */ + void (*kl_unlock)(void *); + int (*kl_locked)(void *); + void *kl_lockarg; /* argument passed to kl_lockf() */ }; Index: src/sys/sys/vnode.h =================================================================== --- src/sys/sys/vnode.h (revision 179) +++ src/sys/sys/vnode.h (working copy) @@ -231,8 +231,8 @@ if (!VN_KNLIST_EMPTY(vp)) \ KNOTE(&vp->v_pollinfo->vpi_selinfo.si_note, (b), (a)); \ } while (0) -#define VN_KNOTE_LOCKED(vp, b) VN_KNOTE(vp, b, 1) -#define VN_KNOTE_UNLOCKED(vp, b) VN_KNOTE(vp, b, 0) +#define VN_KNOTE_UNLOCKED(vp, b) VN_KNOTE(vp, b, 0) +#define VN_KNOTE_UNLOCKED(vp, b) VN_KNOTE(vp, b, 0) /* * Vnode flags. @@ -703,7 +703,7 @@ #define VOP_WRITE_POST(ap, ret) \ noffset = (ap)->a_uio->uio_offset; \ if (noffset > ooffset && !VN_KNLIST_EMPTY((ap)->a_vp)) { \ - VFS_SEND_KNOTE((ap)->a_vp, NOTE_WRITE \ + VFS_KNOTE_LOCKED((ap)->a_vp, NOTE_WRITE \ | (noffset > osize ? NOTE_EXTEND : 0)); \ }