diff --git a/lib/libthr/thread/Makefile.inc b/lib/libthr/thread/Makefile.inc index ddde6e9..1e4d5f7 100644 --- a/lib/libthr/thread/Makefile.inc +++ b/lib/libthr/thread/Makefile.inc @@ -55,4 +55,5 @@ SRCS+= \ thr_switch_np.c \ thr_symbols.c \ thr_umtx.c \ - thr_yield.c + thr_yield.c \ + isi_lock_probes.d diff --git a/lib/libthr/thread/isi_lock_probes.d b/lib/libthr/thread/isi_lock_probes.d new file mode 100644 index 0000000..20d1d16 --- /dev/null +++ b/lib/libthr/thread/isi_lock_probes.d @@ -0,0 +1,79 @@ +#include + +provider isilon { + /** + * Hold event probe. + * Fires immediately after a mutex is acquired + * + * is_recursive: a boolean value indicating if the acquisition + * was recursive on a recursive mutex + * cntspin: number of spin iterations that the acquiring + * thread spent on this mutex + * cntyield: number of yield iterations that the acquiring + * thread spend on this mutex + */ + probe mutex__acquire(pthread_mutex_t mutex, int is_recursive, int cntspin, int cntyield); + + /** + * Contention event probe. + * Fires before a thread blocks on a held mutex. Both + * mutex-block and mutex-spin might fire for a single lock + * acquisition. + */ + probe mutex__block(pthread_mutex_t mutex); + + /** + * Contention event probe. + * Fires before a thread begins spinning on a held mutex. Both + * mutex-block and mutex-spin might fire for a single lock + * acquisition. + */ + probe mutex__spin(pthread_mutex_t mutex); + + /** + * Hold event probe. + * Fires immediately after a mutex is released. + * + * is_recursive: a boolean value indicating if the event + * corresponds to a recursive release on a + * recursive mutex. + */ + probe mutex__release(pthread_mutex_t mutex, int is_recursive); + + /** + * Hold event probe. + * Fires immediately after an rwlock is acquired. + */ + probe rwlock__acquire(pthread_rwlock_t rwlock, int is_writer); + + /** + * Contention event probe. + * Fires before a thread blocks while attempting to acquire a + * lock. + */ + probe rwlock__block(pthread_rwlock_t rwlock, int is_writer); + + /** + * Hold event probe. + * Fires immediately after a lock is released. + */ + probe rwlock__release(pthread_rwlock_t rwlock, int is_writer); + + /** + * Hold event probe. + * Fires immediately after a spinlock is acquired + */ + probe spinlock__acquire(pthread_spinlock_t spinlock); + + /** + * Contention event probe. + * Fires before a thread blocks in a spin loop + */ + probe spinlock__block(pthread_spinlock_t spinlock); + + /** + * Hold event probe. + * Fires immediately after a spinlock is released + */ + probe spinlock__release(pthread_spinlock_t spinlock); +}; diff --git a/lib/libthr/thread/isi_lock_profiler.d b/lib/libthr/thread/isi_lock_profiler.d new file mode 100755 index 0000000..0207290 --- /dev/null +++ b/lib/libthr/thread/isi_lock_profiler.d @@ -0,0 +1,260 @@ +#!/usr/sbin/dtrace -s + +/*#pragma D option quiet*/ +#pragma D option aggsortrev +#pragma D option dynvarsize=16m +#pragma D option cleanrate=333hz + +/****************************** + * MUTEX probes + ******************************/ +/* spin */ +isilon$target:::mutex-spin +{ + self->waitstart = timestamp; + @mutex_waits[ustack(10)] = count(); + @stats[" mutex total spins"] = count(); + @stats["mutex total waits"] = count(); +} + +/* block */ +isilon$target:::mutex-block +{ + @stats[" mutex total blocks"] = count(); +} + +isilon$target:::mutex-block +/!self->waitstart/ +{ + self->waitstart = timestamp; + @mutex_waits[ustack(10)] = count(); + @stats["mutex total waits"] = count(); +} + +/* acquire */ +isilon$target:::mutex-acquire +/arg1 == 0/ +{ + /* Use associative array for acquired, to properly handle nested locks */ + self->acquired[arg0] = timestamp; + @stats["mutex total locks"] = count(); +} + +isilon$target:::mutex-acquire +/arg1 == 0 && self->waitstart/ +{ + @mutex_wait_times[ustack(10)] = sum(timestamp - self->waitstart); + self->waitstart = 0; +} + +isilon$target:::mutex-acquire +/arg1 == 1/ +{ + @stats["mutex total recursive locks"] = count(); +} + +/* release */ +isilon$target:::mutex-release +/arg1 == 0 && self->acquired[arg0]/ +{ + @mutex_hold_times[ustack(10)] = sum(timestamp - self->acquired[arg0]); + self->acquired[arg0] = 0; +} + +/* handle errors */ +pid$target::*pthread_mutex_*lock*:return +/arg1/ +{ + @errors[probefunc, arg1] = count(); +} + +/* skip unlock. trylock is fine, since waitstart will be zero there. */ +pid$target::*pthread_mutex_[!u]*ock:return +/arg1 && self->waitstart/ +{ + @mutex_wait_times[ustack(10)] = sum(timestamp - self->waitstart); + self->waitstart = 0; +} + +/****************************** + * RWLOCK probes + ******************************/ +/* block */ +isilon$target:::rwlock-block +{ + this->id = (arg1 == 0) ? "reader" : "writer"; + this->desc = (arg1 == 0) ? "rwlock total waits reader" : "rwlock total waits writer"; + self->waitstart = timestamp; + + @stats[this->desc] = count(); + @rwlock_waits[ustack(10), this->id] = count(); +} + +/* acquire */ +isilon$target:::rwlock-acquire +{ + this->desc = (arg1 == 0) ? "rwlock total locks reader" : "rwlock total locks writer"; + + /* Use associative array for acquired, to properly handle nested locks */ + self->acquired[arg0] = timestamp; + @stats[this->desc] = count(); +} + +isilon$target:::rwlock-acquire +/self->waitstart/ +{ + this->id = (arg1 == 0) ? "reader" : "writer"; + + @rwlock_wait_times[ustack(10), this->id] = sum(timestamp - self->waitstart); + self->waitstart = 0; +} + +/* release */ +isilon$target:::rwlock-release +/self->acquired[arg0]/ +{ + this->id = (arg1 == 0) ? "reader" : "writer"; + + @rwlock_hold_times[ustack(10), this->id] = sum(timestamp - self->acquired[arg0]); + self->acquired[arg0] = 0; +} + +/* handle errors */ +pid$target::*pthread_rwlock_*lock*:return +/arg1/ +{ + @errors[probefunc, arg1] = count(); +} + +/* skip unlock. trylock is fine, since waitstart will be zero there. */ +pid$target::*pthread_rwlock_*[!u]?lock:return +/arg1 && self->waitstart/ +{ + this->id = (arg1 == 0) ? "reader" : "writer"; + + @rwlock_wait_times[ustack(10), this->id] = sum(timestamp - self->waitstart); + self->waitstart = 0; +} + +/****************************** + * SPINLOCK probes + ******************************/ +/* block */ +isilon$target:::spinlock-block +{ + self->waitstart = timestamp; + + @stats["spinlock total waits"] = count(); + @spinlock_waits[ustack(10)] = count(); +} + +/* acquire */ +isilon$target:::spinlock-acquire +{ + /* Use associative array for acquired, to properly handle nested locks */ + self->acquired[arg0] = timestamp; + @stats["spinlock total locks"] = count(); +} + +isilon$target:::spinlock-acquire +/self->waitstart/ +{ + @spinlock_wait_times[ustack(10)] = sum(timestamp - self->waitstart); + self->waitstart = 0; +} + +/* release */ +isilon$target:::spinlock-release +/self->acquired[arg0]/ +{ + @spinlock_hold_times[ustack(10)] = sum(timestamp - self->acquired[arg0]); + self->acquired[arg0] = 0; +} + +/* handle errors */ +pid$target::*pthread_spin_*lock*:return +/arg1/ +{ + @errors[probefunc, arg1] = count(); +} + +pid$target::*pthread_spin_lock:return +/arg1 && self->waitstart/ +{ + @spinlock_wait_times[ustack(10)] = sum(timestamp - self->waitstart); + self->waitstart = 0; +} + +END +{ + trunc(@mutex_hold_times, 10); + trunc(@mutex_wait_times, 10); + trunc(@mutex_waits, 10); + trunc(@rwlock_waits, 10); + trunc(@rwlock_wait_times, 10); + trunc(@rwlock_hold_times, 10); + trunc(@spinlock_waits, 10); + trunc(@spinlock_wait_times, 10); + trunc(@spinlock_hold_times, 10); + normalize(@mutex_hold_times, 1000000); + normalize(@mutex_wait_times, 1000000); + normalize(@rwlock_hold_times, 1000000); + normalize(@rwlock_wait_times, 1000000); + normalize(@spinlock_hold_times, 1000000); + normalize(@spinlock_wait_times, 1000000); + + printf("\n==================================\n"); + printf("MUTEX WAIT COUNT\n"); + printf("==================================\n"); + printa("%-5k\t\t%@d\n", @mutex_waits); + + printf("\n==================================\n"); + printf("MUTEX WAIT TIMES\n"); + printf("==================================\n"); + printa("%-5k\t\t%@dms\n", @mutex_wait_times); + + printf("\n==================================\n"); + printf("MUTEX HOLD TIMES\n"); + printf("==================================\n"); + printa("%-5k\t\t%@dms\n", @mutex_hold_times); + + printf("\n==================================\n"); + printf("RWLOCK WAIT COUNT\n"); + printf("==================================\n"); + printa("%-5k\t%s\t\t%@d\n", @rwlock_waits); + + printf("\n==================================\n"); + printf("RWLOCK WAIT TIMES\n"); + printf("==================================\n"); + printa("%-5k\t%s\t\t%@dms\n", @rwlock_wait_times); + + printf("\n==================================\n"); + printf("RWLOCK HOLD TIMES\n"); + printf("==================================\n"); + printa("%-5k\t%s\t\t%@dms\n", @rwlock_hold_times); + + printf("\n==================================\n"); + printf("SPINLOCK WAIT COUNT\n"); + printf("==================================\n"); + printa("%-5k\t\t%@d\n", @spinlock_waits); + + printf("\n==================================\n"); + printf("SPINLOCK WAIT TIMES\n"); + printf("==================================\n"); + printa("%-5k\t\t%@dms\n", @spinlock_wait_times); + + printf("\n==================================\n"); + printf("SPINLOCK HOLD TIMES\n"); + printf("==================================\n"); + printa("%-5k\t\t%@dms\n", @spinlock_hold_times); + + printf("\n==================================\n"); + printf("STATISTICS\n"); + printf("==================================\n"); + printa("%-30s%10@d\n", @stats); + + printf("\n==================================\n"); + printf("ERRORS\n"); + printf("==================================\n"); + printa("%-30s%10d%10@d\n", @errors); +} diff --git a/lib/libthr/thread/isi_lock_profiler_minimal.d b/lib/libthr/thread/isi_lock_profiler_minimal.d new file mode 100755 index 0000000..7791d82 --- /dev/null +++ b/lib/libthr/thread/isi_lock_profiler_minimal.d @@ -0,0 +1,106 @@ +#!/usr/sbin/dtrace -s + +/*#pragma D option quiet*/ +#pragma D option aggsortrev +#pragma D option dynvarsize=16m +#pragma D option cleanrate=333hz + +/****************************** + * MUTEX probes + ******************************/ +/* spin */ +isilon$target:::mutex-spin +{ + @mutex_spins[ustack(10)] = count(); + @stats["mutex total spins"] = count(); +} + +/* block */ +isilon$target:::mutex-block +{ + @mutex_blocks[ustack(10)] = count(); + @stats["mutex total blocks"] = count(); +} + +/* handle errors */ +pid$target::*pthread_mutex_*lock*:return +/arg1/ +{ + @errors[probefunc, arg1] = count(); +} + +/****************************** + * RWLOCK probes + ******************************/ +/* block */ +isilon$target:::rwlock-block +{ + this->id = (arg1 == 0) ? "reader" : "writer"; + this->desc = (arg1 == 0) ? "rwlock total waits reader" : "rwlock total waits writer"; + + @stats[this->desc] = count(); + @rwlock_waits[ustack(10), this->id] = count(); +} + +/* handle errors */ +pid$target::*pthread_rwlock_*lock*:return +/arg1/ +{ + @errors[probefunc, arg1] = count(); +} + +/****************************** + * SPINLOCK probes + ******************************/ +/* block */ +isilon$target:::spinlock-block +{ + @stats["spinlock total waits"] = count(); + @spinlock_waits[ustack(10)] = count(); +} + +/* handle errors */ +pid$target::*pthread_spin_*lock*:return +/arg1/ +{ + @errors[probefunc, arg1] = count(); +} + + +END +{ + trunc(@mutex_spins, 10); + trunc(@mutex_blocks, 10); + trunc(@rwlock_waits, 10); + trunc(@spinlock_waits, 10); + + printf("\n==================================\n"); + printf("MUTEX SPIN COUNT\n"); + printf("==================================\n"); + printa("%-5k\t\t%@d\n", @mutex_spins); + + printf("\n==================================\n"); + printf("MUTEX BLOCK COUNT\n"); + printf("==================================\n"); + printa("%-5k\t\t%@d\n", @mutex_blocks); + + printf("\n==================================\n"); + printf("RWLOCK WAIT COUNT\n"); + printf("==================================\n"); + printa("%-5k\t%s\t\t%@d\n", @rwlock_waits); + + printf("\n==================================\n"); + printf("SPINLOCK WAIT COUNT\n"); + printf("==================================\n"); + printa("%-5k\t\t%@d\n", @spinlock_waits); + + printf("\n==================================\n"); + printf("STATISTICS\n"); + printf("==================================\n"); + printa("%-30s%10@d\n", @stats); + + printf("\n==================================\n"); + printf("ERRORS\n"); + printf("==================================\n"); + printa("%-30s%10d%10@d\n", @errors); +} diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c index 61ff077..22237ae 100644 --- a/lib/libthr/thread/thr_mutex.c +++ b/lib/libthr/thread/thr_mutex.c @@ -44,6 +44,7 @@ #include "un-namespace.h" #include "thr_private.h" +#include "isi_lock_probes.h" #if defined(_PTHREADS_INVARIANTS) #define MUTEX_INIT_LINK(m) do { \ @@ -71,6 +72,15 @@ */ #define MUTEX_ADAPTIVE_SPINS 2000 +/* Begin Isilon */ +#define _ISILON_MUTEX_ACQUIRED(rv, spinnum, yieldnum) \ + if (__predict_false(ISILON_MUTEX_ACQUIRE_ENABLED() && (!rv))) { \ + int recursive = (PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE && \ + m->m_count > 0); \ + ISILON_MUTEX_ACQUIRE(m, recursive, spinnum, yieldnum); \ + } +/* End Isilon */ + /* * Prototypes */ @@ -94,7 +104,7 @@ static int mutex_self_lock(pthread_mutex_t, const struct timespec *abstime); static int mutex_unlock_common(struct pthread_mutex *, int, int *); static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, - const struct timespec *); + const struct timespec *, int *, int *); __weak_reference(__pthread_mutex_init, pthread_mutex_init); __strong_reference(__pthread_mutex_init, _pthread_mutex_init); @@ -337,20 +347,26 @@ int __pthread_mutex_trylock(pthread_mutex_t *mutex) { struct pthread_mutex *m; + int rv; CHECK_AND_INIT_MUTEX - return (mutex_trylock_common(mutex)); + rv = (mutex_trylock_common(mutex)); + _ISILON_MUTEX_ACQUIRED(rv, 0, 0); + return rv; } static int mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, - const struct timespec *abstime) + const struct timespec *abstime, int *cntspin, int *cntyield) { uint32_t id, owner; int count; int ret; + *cntspin = 0; + *cntyield = 0; + if (m->m_owner == curthread) return mutex_self_lock(m, abstime); @@ -370,7 +386,11 @@ mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, goto yield_loop; count = m->m_spinloops; + if (count) { + ISILON_MUTEX_SPIN(m); + } while (count--) { + ++*cntspin; owner = m->m_lock.m_owner; if ((owner & ~UMUTEX_CONTESTED) == 0) { if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { @@ -384,6 +404,7 @@ mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, yield_loop: count = m->m_yieldloops; while (count--) { + ++*cntyield; _sched_yield(); owner = m->m_lock.m_owner; if ((owner & ~UMUTEX_CONTESTED) == 0) { @@ -395,6 +416,7 @@ yield_loop: } sleep_in_kernel: + ISILON_MUTEX_BLOCK(m); if (abstime == NULL) { ret = __thr_umutex_lock(&m->m_lock, id); } else if (__predict_false( @@ -417,6 +439,7 @@ mutex_lock_common(struct pthread_mutex *m, { struct pthread *curthread = _get_curthread(); int ret; + int cntspin = 0, cntyield = 0; if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE) THR_CRITICAL_ENTER(curthread); @@ -424,10 +447,11 @@ mutex_lock_common(struct pthread_mutex *m, ENQUEUE_MUTEX(curthread, m); ret = 0; } else { - ret = mutex_lock_sleep(curthread, m, abstime); + ret = mutex_lock_sleep(curthread, m, abstime, &cntspin, &cntyield); } if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE) && !cvattach) THR_CRITICAL_LEAVE(curthread); + _ISILON_MUTEX_ACQUIRED(ret, cntspin, cntyield); return (ret); } @@ -652,6 +676,7 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer) PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) { m->m_count--; + ISILON_MUTEX_RELEASE(m, 1); } else { if ((m->m_flags & PMUTEX_FLAG_DEFERED) != 0) { defered = 1; @@ -661,6 +686,7 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer) DEQUEUE_MUTEX(curthread, m); _thr_umutex_unlock2(&m->m_lock, id, mtx_defer); + ISILON_MUTEX_RELEASE(m, 0); if (mtx_defer == NULL && defered) { _thr_wake_all(curthread->defer_waiters, diff --git a/lib/libthr/thread/thr_pspinlock.c b/lib/libthr/thread/thr_pspinlock.c index 9e1f96e..4a4aec1 100644 --- a/lib/libthr/thread/thr_pspinlock.c +++ b/lib/libthr/thread/thr_pspinlock.c @@ -34,6 +34,8 @@ #include "thr_private.h" +#include "isi_lock_probes.h" + #define SPIN_COUNT 100000 __weak_reference(_pthread_spin_init, pthread_spin_init); @@ -88,6 +90,8 @@ _pthread_spin_trylock(pthread_spinlock_t *lock) ret = EINVAL; else ret = THR_UMUTEX_TRYLOCK(curthread, &lck->s_lock); + if (ret == 0) + ISILON_SPINLOCK_ACQUIRE(lck); return (ret); } @@ -97,12 +101,17 @@ _pthread_spin_lock(pthread_spinlock_t *lock) struct pthread *curthread = _get_curthread(); struct pthread_spinlock *lck; int ret, count; + int block_fired = 0; if (lock == NULL || (lck = *lock) == NULL) ret = EINVAL; else { count = SPIN_COUNT; while ((ret = THR_UMUTEX_TRYLOCK(curthread, &lck->s_lock)) != 0) { + if (!block_fired) { + ISILON_SPINLOCK_BLOCK(lck); + block_fired = 1; + } while (lck->s_lock.m_owner) { if (!_thr_is_smp) { _pthread_yield(); @@ -119,6 +128,8 @@ _pthread_spin_lock(pthread_spinlock_t *lock) ret = 0; } + if (ret == 0) + ISILON_SPINLOCK_ACQUIRE(lck); return (ret); } @@ -134,5 +145,7 @@ _pthread_spin_unlock(pthread_spinlock_t *lock) else { ret = THR_UMUTEX_UNLOCK(curthread, &lck->s_lock); } + if (ret == 0) + ISILON_SPINLOCK_RELEASE(lck); return (ret); } diff --git a/lib/libthr/thread/thr_rwlock.c b/lib/libthr/thread/thr_rwlock.c index 397663e..3ae9750 100644 --- a/lib/libthr/thread/thr_rwlock.c +++ b/lib/libthr/thread/thr_rwlock.c @@ -35,6 +35,8 @@ #include "un-namespace.h" #include "thr_private.h" +#include "isi_lock_probes.h" + __weak_reference(_pthread_rwlock_destroy, pthread_rwlock_destroy); __weak_reference(_pthread_rwlock_init, pthread_rwlock_init); __weak_reference(_pthread_rwlock_rdlock, pthread_rwlock_rdlock); @@ -125,6 +127,7 @@ rwlock_rdlock_common(pthread_rwlock_t *rwlock, const struct timespec *abstime) pthread_rwlock_t prwlock; int flags; int ret; + int block_fired = 0; CHECK_AND_INIT_RWLOCK @@ -151,16 +154,22 @@ rwlock_rdlock_common(pthread_rwlock_t *rwlock, const struct timespec *abstime) * not be checked if the lock can be immediately acquired. */ ret = _thr_rwlock_tryrdlock(&prwlock->lock, flags); - if (ret == 0) { - curthread->rdlock_count++; - return (ret); - } + if (ret == 0) + goto done; if (__predict_false(abstime && - (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0))) - return (EINVAL); + (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0))) { + ret = EINVAL; + goto done; + } for (;;) { + if (!block_fired) { + /* Try-lock was unsuccessful, assume we are going to block */ + ISILON_RWLOCK_BLOCK(prwlock, 0); + block_fired = 1; + } + /* goto kernel and lock it */ ret = __thr_rwlock_rdlock(&prwlock->lock, flags, abstime); if (ret != EINTR) @@ -172,8 +181,11 @@ rwlock_rdlock_common(pthread_rwlock_t *rwlock, const struct timespec *abstime) break; } } - if (ret == 0) +done: + if (ret == 0) { curthread->rdlock_count++; + ISILON_RWLOCK_ACQUIRE(prwlock, 0); + } return (ret); } @@ -219,8 +231,10 @@ _pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) } ret = _thr_rwlock_tryrdlock(&prwlock->lock, flags); - if (ret == 0) + if (ret == 0) { curthread->rdlock_count++; + ISILON_RWLOCK_ACQUIRE(prwlock, 0); + } return (ret); } @@ -234,8 +248,10 @@ _pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock) CHECK_AND_INIT_RWLOCK ret = _thr_rwlock_trywrlock(&prwlock->lock); - if (ret == 0) + if (ret == 0) { prwlock->owner = curthread; + ISILON_RWLOCK_ACQUIRE(prwlock, 1); + } return (ret); } @@ -245,6 +261,7 @@ rwlock_wrlock_common (pthread_rwlock_t *rwlock, const struct timespec *abstime) struct pthread *curthread = _get_curthread(); pthread_rwlock_t prwlock; int ret; + int block_fired = 0; CHECK_AND_INIT_RWLOCK @@ -253,33 +270,37 @@ rwlock_wrlock_common (pthread_rwlock_t *rwlock, const struct timespec *abstime) * not be checked if the lock can be immediately acquired. */ ret = _thr_rwlock_trywrlock(&prwlock->lock); - if (ret == 0) { - prwlock->owner = curthread; - return (ret); - } + if (ret == 0) + goto done; if (__predict_false(abstime && - (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0))) - return (EINVAL); + (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0))) { + ret = EINVAL; + goto done; + } for (;;) { + if (!block_fired) { + /* Try-lock was unsuccessful, assume we are going to block */ + ISILON_RWLOCK_BLOCK(prwlock, 1); + block_fired = 1; + } /* goto kernel and lock it */ ret = __thr_rwlock_wrlock(&prwlock->lock, abstime); - if (ret == 0) { - prwlock->owner = curthread; - break; - } - - if (ret != EINTR) + if (ret == 0 || ret != EINTR) break; /* if interrupted, try to lock it in userland again. */ if (_thr_rwlock_trywrlock(&prwlock->lock) == 0) { ret = 0; - prwlock->owner = curthread; break; } } +done: + if (ret == 0) { + prwlock->owner = curthread; + ISILON_RWLOCK_ACQUIRE(prwlock, 1); + } return (ret); } @@ -317,8 +338,11 @@ _pthread_rwlock_unlock (pthread_rwlock_t *rwlock) } ret = _thr_rwlock_unlock(&prwlock->lock); - if (ret == 0 && (state & URWLOCK_WRITE_OWNER) == 0) - curthread->rdlock_count--; + if (ret == 0) { + if ((state & URWLOCK_WRITE_OWNER) == 0) + curthread->rdlock_count--; + ISILON_RWLOCK_RELEASE(prwlock, (state & URWLOCK_WRITE_OWNER) != 0); + } return (ret); }