--- //depot/projects/smpng/sys/kern/kern_rwlock.c 2006/02/03 20:08:50 +++ //depot/user/jhb/lock/kern/kern_rwlock.c 2006/04/11 22:12:39 @@ -55,16 +59,27 @@ struct lock_class lock_class_rw = { "rw", - LC_SLEEPLOCK | LC_RECURSABLE /* | LC_UPGRADABLE */, + LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, #ifdef DDB db_show_rwlock #endif }; -#define rw_owner(rw) \ +/* + * Return a pointer to the owning thread if the lock is write-locked or + * NULL if the lock is unlocked or read-locked. + */ +#define rw_wowner(rw) \ ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ (struct thread *)RW_OWNER((rw)->rw_lock)) +/* + * Return a pointer to the owning thread for this lock who should receive + * any priority lent by threads that block on this lock. Currently this + * is identical to rw_wowner(). + */ +#define rw_owner(rw) rw_wowner(rw) + #ifndef INVARIANTS #define _rw_assert(rw, what, file, line) #endif @@ -76,7 +91,7 @@ rw->rw_lock = RW_UNLOCKED; lock_init(&rw->rw_object, &lock_class_rw, name, NULL, LO_WITNESS | - LO_RECURSABLE /* | LO_UPGRADABLE */); + LO_RECURSABLE | LO_UPGRADABLE); } void @@ -100,7 +115,7 @@ { MPASS(curthread != NULL); - KASSERT(rw_owner(rw) != curthread, + KASSERT(rw_wowner(rw) != curthread, ("%s (%s): wlock already held @ %s:%d", __func__, rw->rw_object.lo_name, file, line)); WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, @@ -124,9 +139,10 @@ void _rw_rlock(struct rwlock *rw, const char *file, int line) { + volatile struct thread *owner; uintptr_t x; - KASSERT(rw_owner(rw) != curthread, + KASSERT(rw_wowner(rw) != curthread, ("%s (%s): wlock already held @ %s:%d", __func__, rw->rw_object.lo_name, file, line)); WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER, file, line); @@ -171,6 +187,7 @@ (void *)(x + RW_ONE_READER)); break; } + cpu_spinwait(); continue; } @@ -189,6 +206,7 @@ x = rw->rw_lock; if (x & RW_LOCK_READ) { turnstile_release(&rw->rw_object); + cpu_spinwait(); continue; } @@ -198,16 +216,36 @@ * it is not set then try to set it. If we fail to set it * drop the turnstile lock and restart the loop. */ - if (!(x & RW_LOCK_READ_WAITERS) && - !atomic_cmpset_ptr(&rw->rw_lock, x, - x | RW_LOCK_READ_WAITERS)) { + if (!(x & RW_LOCK_READ_WAITERS)) { + if (!atomic_cmpset_ptr(&rw->rw_lock, x, + x | RW_LOCK_READ_WAITERS)) { + turnstile_release(&rw->rw_object); + cpu_spinwait(); + continue; + } + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p set read waiters flag", + __func__, rw); + } + +#ifdef SMP + /* + * If the owner is running on another CPU, spin until + * the owner stops running or the state of the lock + * changes. + */ + owner = (struct thread *)RW_OWNER(x); + if (TD_IS_RUNNING(owner)) { turnstile_release(&rw->rw_object); + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR3(KTR_LOCK, "%s: spinning on %p held by %p", + __func__, rw, owner); + while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && + TD_IS_RUNNING(owner)) + cpu_spinwait(); continue; } - if (!(x & RW_LOCK_READ_WAITERS) && - LOCK_LOG_TEST(&rw->rw_object, 0)) - CTR2(KTR_LOCK, "%s: %p set read waiters flag", __func__, - rw); +#endif /* * We were unable to acquire the lock and the read waiters @@ -339,6 +377,7 @@ * release the lock. */ ts = turnstile_lookup(&rw->rw_object); + MPASS(ts != NULL); turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); turnstile_unpend(ts, TS_SHARED_LOCK); break; @@ -353,6 +392,7 @@ void _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) { + volatile struct thread *owner; uintptr_t v; if (LOCK_LOG_TEST(&rw->rw_object, 0)) @@ -402,19 +442,36 @@ * set it. If we fail to set it, then loop back and try * again. */ - if (!(v & RW_LOCK_WRITE_WAITERS) && - !atomic_cmpset_ptr(&rw->rw_lock, v, - v | RW_LOCK_WRITE_WAITERS)) { + if (!(v & RW_LOCK_WRITE_WAITERS)) { + if (!atomic_cmpset_ptr(&rw->rw_lock, v, + v | RW_LOCK_WRITE_WAITERS)) { + turnstile_release(&rw->rw_object); + cpu_spinwait(); + continue; + } + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p set write waiters flag", + __func__, rw); + } + +#ifdef SMP + /* + * If the lock is write locked and the owner is + * running on another CPU, spin until the owner stops + * running or the state of the lock changes. + */ + owner = (struct thread *)RW_OWNER(v); + if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { turnstile_release(&rw->rw_object); - cpu_spinwait(); + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR3(KTR_LOCK, "%s: spinning on %p held by %p", + __func__, rw, owner); + while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && + TD_IS_RUNNING(owner)) + cpu_spinwait(); continue; } - if (!(v & RW_LOCK_WRITE_WAITERS) && - LOCK_LOG_TEST(&rw->rw_object, 0)) - CTR2(KTR_LOCK, "%s: %p set write waiters flag", - __func__, rw); - - /* XXX: Adaptively spin if current wlock owner on another CPU? */ +#endif /* * We were unable to acquire the lock and the write waiters @@ -452,8 +509,22 @@ turnstile_lock(&rw->rw_object); ts = turnstile_lookup(&rw->rw_object); - /* XXX: Adaptive fixup would be required here. */ +#ifdef SMP + /* + * There might not be a turnstile for this lock if all of + * the waiters are adaptively spinning. In that case, just + * reset the lock to the unlocked state and return. + */ + if (ts == NULL) { + atomic_store_rel_ptr(&rw->rw_lock, RW_UNLOCKED); + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p no sleepers", __func__, rw); + turnstile_release(&rw->rw_object); + return; + } +#else MPASS(ts != NULL); +#endif /* * Use the same algo as sx locks for now. Prefer waking up shared @@ -470,24 +541,161 @@ * above. There is probably a potential priority inversion in * there that could be worked around either by waking both queues * of waiters or doing some complicated lock handoff gymnastics. + * + * Note that in the SMP case, if both flags are set, there might + * not be any actual writers on the turnstile as they might all + * be spinning. In that case, we don't want to preserve the + * RW_LOCK_WRITE_WAITERS flag as the turnstile is going to go + * away once we wakeup all the readers. */ + v = RW_UNLOCKED; if (rw->rw_lock & RW_LOCK_READ_WAITERS) { queue = TS_SHARED_QUEUE; - v = RW_UNLOCKED | (rw->rw_lock & RW_LOCK_WRITE_WAITERS); +#ifdef SMP + if (rw->rw_lock & RW_LOCK_WRITE_WAITERS && + !turnstile_empty(ts, TS_EXCLUSIVE_QUEUE)) + v |= RW_LOCK_WRITE_WAITERS; +#else + v |= (rw->rw_lock & RW_LOCK_WRITE_WAITERS); +#endif } else { queue = TS_EXCLUSIVE_QUEUE; v = RW_UNLOCKED; } + +#ifdef SMP + /* + * We have to make sure that we actually have waiters to + * wakeup. If they are all spinning, then we just need to + * disown the turnstile and return. + */ + if (turnstile_empty(ts, queue)) { + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p no sleepers 2", __func__, rw); + atomic_store_rel_ptr(&rw->rw_lock, v); + turnstile_disown(ts); + return; + } +#endif + + /* Wake up all waiters for the specific queue. */ if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, queue == TS_SHARED_QUEUE ? "read" : "write"); - - /* Wake up all waiters for the specific queue. */ turnstile_broadcast(ts, queue); atomic_store_rel_ptr(&rw->rw_lock, v); turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); } +/* + * Attempt to do a non-blocking upgrade from a read lock to a write + * lock. This will only succeed if this thread holds a single read + * lock. Returns true if the upgrade succeeded and false otherwise. + */ +int +_rw_try_upgrade(struct rwlock *rw, const char *file, int line) +{ + uintptr_t v, tid; + int success; + + _rw_assert(rw, RA_RLOCKED, file, line); + + /* + * Attempt to switch from one reader to a writer. If there + * are any write waiters, then we will have to lock the + * turnstile first to prevent races with another writer + * calling turnstile_wait() before we have claimed this + * turnstile. So, do the simple case of no waiters first. + */ + tid = (uintptr_t)curthread; + if (!(rw->rw_lock & RW_LOCK_WRITE_WAITERS)) { + success = atomic_cmpset_acq_ptr(&rw->rw_lock, + RW_READERS_LOCK(1), tid); + goto out; + } + + /* + * Ok, we think we have write waiters, so lock the + * turnstile. + */ + turnstile_lock(&rw->rw_object); + + /* + * Try to switch from one reader to a writer again. This time + * we honor the current state of the RW_LOCK_WRITE_WAITERS + * flag. If we obtain the lock with the flag set, then claim + * ownership of the turnstile. In the SMP case it is possible + * for there to not be an associated turnstile even though there + * are waiters if all of the waiters are spinning. + */ + v = rw->rw_lock & RW_LOCK_WRITE_WAITERS; + success = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, + tid | v); +#ifdef SMP + if (success && v && turnstile_lookup(&rw->rw_object) != NULL) +#else + if (success && v) +#endif + turnstile_claim(&rw->rw_object); + else + turnstile_release(&rw->rw_object); +out: + LOCK_LOG_TRY("WUPGRADE", &rw->rw_object, 0, success, file, line); + if (success) + WITNESS_UPGRADE(&rw->rw_object, LOP_EXCLUSIVE | LOP_TRYLOCK, + file, line); + return (success); +} + +/* + * Downgrade a write lock into a single read lock. + */ +void +_rw_downgrade(struct rwlock *rw, const char *file, int line) +{ + struct turnstile *ts; + uintptr_t tid, v; + + _rw_assert(rw, RA_WLOCKED, file, line); + + WITNESS_DOWNGRADE(&rw->rw_object, 0, file, line); + + /* + * Convert from a writer to a single reader. First we handle + * the easy case with no waiters. If there are any waiters, we + * lock the turnstile, "disown" the lock, and awaken any read + * waiters. + */ + tid = (uintptr_t)curthread; + if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) + goto out; + + /* + * Ok, we think we have waiters, so lock the turnstile so we can + * read the waiter flags without any races. + */ + turnstile_lock(&rw->rw_object); + v = rw->rw_lock; + MPASS(v & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)); + + /* + * Downgrade from a write lock while preserving + * RW_LOCK_WRITE_WAITERS and give up ownership of the + * turnstile. If there are any read waiters, wake them up. + */ + ts = turnstile_lookup(&rw->rw_object); + if (v & RW_LOCK_READ_WAITERS) + turnstile_broadcast(ts, TS_SHARED_QUEUE); + atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | + (v & RW_LOCK_WRITE_WAITERS)); + if (v & RW_LOCK_READ_WAITERS) + turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); + else + turnstile_disown(ts); +out: + LOCK_LOG_LOCK("WDOWNGRADE", &rw->rw_object, 0, 0, file, line); +} + #ifdef INVARIANT_SUPPORT #ifndef INVARIANTS #undef _rw_assert @@ -520,14 +728,14 @@ */ if (rw->rw_lock == RW_UNLOCKED || (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED || - rw_owner(rw) != curthread))) + rw_wowner(rw) != curthread))) panic("Lock %s not %slocked @ %s:%d\n", rw->rw_object.lo_name, (what == RA_RLOCKED) ? "read " : "", file, line); #endif break; case RA_WLOCKED: - if (rw_owner(rw) != curthread) + if (rw_wowner(rw) != curthread) panic("Lock %s not exclusively locked @ %s:%d\n", rw->rw_object.lo_name, file, line); break; @@ -539,7 +747,7 @@ * If we hold a write lock fail. We can't reliably check * to see if we hold a read lock or not. */ - if (rw_owner(rw) == curthread) + if (rw_wowner(rw) == curthread) panic("Lock %s exclusively locked @ %s:%d\n", rw->rw_object.lo_name, file, line); #endif @@ -567,7 +775,7 @@ db_printf("RLOCK: %jd locks\n", (intmax_t)(RW_READERS(rw->rw_lock))); else { - td = rw_owner(rw); + td = rw_wowner(rw); db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); } --- //depot/projects/smpng/sys/kern/subr_turnstile.c 2006/03/29 23:12:18 +++ //depot/user/jhb/lock/kern/subr_turnstile.c 2006/04/11 22:12:39 @@ -879,6 +879,56 @@ } /* + * Give up ownership of a turnstile. This must be called with the + * turnstile chain locked. + */ +void +turnstile_disown(struct turnstile *ts) +{ + struct turnstile_chain *tc; + struct thread *td; + u_char cp, pri; + + MPASS(ts != NULL); + MPASS(ts->ts_owner == curthread); + tc = TC_LOOKUP(ts->ts_lockobj); + mtx_assert(&tc->tc_lock, MA_OWNED); + MPASS(TAILQ_EMPTY(&ts->ts_pending)); + MPASS(!TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) || + !TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])); + + /* + * Remove the turnstile from this thread's list of contested locks + * since this thread doesn't own it anymore. New threads will + * not be blocking on the turnstile until it is claimed by a new + * owner. + */ + mtx_lock_spin(&td_contested_lock); + ts->ts_owner = NULL; + LIST_REMOVE(ts, ts_link); + mtx_unlock_spin(&td_contested_lock); + mtx_unlock_spin(&tc->tc_lock); + + /* + * Adjust the priority of curthread based on other contested + * locks it owns. Don't lower the priority below the base + * priority however. + */ + td = curthread; + pri = PRI_MAX; + mtx_lock_spin(&sched_lock); + mtx_lock_spin(&td_contested_lock); + LIST_FOREACH(ts, &td->td_contested, ts_link) { + cp = turnstile_first_waiter(ts)->td_priority; + if (cp < pri) + pri = cp; + } + mtx_unlock_spin(&td_contested_lock); + sched_unlend_prio(td, pri); + mtx_unlock_spin(&sched_lock); +} + +/* * Return the first thread in a turnstile. */ struct thread * @@ -895,6 +945,23 @@ return (TAILQ_FIRST(&ts->ts_blocked[queue])); } +/* + * Returns true if a sub-queue of a turnstile is empty. + */ +int +turnstile_empty(struct turnstile *ts, int queue) +{ +#ifdef INVARIANTS + struct turnstile_chain *tc; + + MPASS(ts != NULL); + MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); + tc = TC_LOOKUP(ts->ts_lockobj); + mtx_assert(&tc->tc_lock, MA_OWNED); +#endif + return (TAILQ_EMPTY(&ts->ts_blocked[queue])); +} + #ifdef DDB static void print_thread(struct thread *td, const char *prefix) --- //depot/projects/smpng/sys/sys/rwlock.h 2006/02/01 17:14:07 +++ //depot/user/jhb/lock/sys/rwlock.h 2006/04/10 18:40:29 @@ -126,6 +126,8 @@ int line); void _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line); +int _rw_try_upgrade(struct rwlock *rw, const char *file, int line); +void _rw_downgrade(struct rwlock *rw, const char *file, int line); #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) void _rw_assert(struct rwlock *rw, int what, const char *file, int line); #endif @@ -133,7 +135,7 @@ /* * Public interface for lock operations. * - * XXX: Missing try and upgrade/downgrade. + * XXX: Missing try locks. */ #ifndef LOCK_DEBUG @@ -150,6 +152,8 @@ #endif #define rw_rlock(rw) _rw_rlock((rw), LOCK_FILE, LOCK_LINE) #define rw_runlock(rw) _rw_runlock((rw), LOCK_FILE, LOCK_LINE) +#define rw_try_upgrade(rw) _rw_try_upgrade((rw), LOCK_FILE, LOCK_LINE) +#define rw_downgrade(rw) _rw_downgrade((rw), LOCK_FILE, LOCK_LINE) #define rw_initialized(rw) lock_initalized(&(rw)->rw_object) --- //depot/projects/smpng/sys/sys/turnstile.h 2006/01/27 22:52:38 +++ //depot/user/jhb/lock/sys/turnstile.h 2006/04/11 14:34:58 @@ -85,6 +85,8 @@ struct turnstile *turnstile_alloc(void); void turnstile_broadcast(struct turnstile *, int); void turnstile_claim(struct lock_object *); +void turnstile_disown(struct turnstile *); +int turnstile_empty(struct turnstile *ts, int queue); void turnstile_free(struct turnstile *); struct thread *turnstile_head(struct turnstile *, int); void turnstile_lock(struct lock_object *);