Index: sys/sys/umtx.h =================================================================== --- sys/sys/umtx.h (revision 233067) +++ sys/sys/umtx.h (working copy) @@ -55,7 +55,9 @@ #define URWLOCK_READER_COUNT(c) ((c) & URWLOCK_MAX_READERS) /* _usem flags */ -#define SEM_NAMED 0x0002 +#define SEM_NAMED 0x00000002 +#define SEM_VER2 0x00000004 +#define SEM_WAITERS 0x80000000 /* op code for _umtx_op */ #define UMTX_OP_LOCK 0 Index: sys/kern/kern_umtx.c =================================================================== --- sys/kern/kern_umtx.c (revision 233067) +++ sys/kern/kern_umtx.c (working copy) @@ -2804,24 +2804,42 @@ umtxq_insert(uq); umtxq_unlock(&uq->uq_key); - if (fuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters)) == 0) + if (__predict_false((flags & SEM_VER2) == 0)) { casuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters), 0, 1); - - count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count)); - if (count != 0) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_remove(uq); - umtxq_unlock(&uq->uq_key); - umtx_key_release(&uq->uq_key); - return (0); + rmb(); + count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count)); + if (count != 0) { + umtxq_lock(&uq->uq_key); + umtxq_unbusy(&uq->uq_key); + umtxq_remove(uq); + umtxq_unlock(&uq->uq_key); + umtx_key_release(&uq->uq_key); + return (0); + } + } else { + count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count)); + for (;;) { + if ((count & ~SEM_WAITERS) != 0) { + umtxq_lock(&uq->uq_key); + umtxq_unbusy(&uq->uq_key); + umtxq_remove(uq); + umtxq_unlock(&uq->uq_key); + umtx_key_release(&uq->uq_key); + return (0); + } + if ((count & SEM_WAITERS) == 0) { + int old = casuword32(__DEVOLATILE(uint32_t *, + &sem->_count), count, count | SEM_WAITERS); + if (old == count) + break; + count = old; + } else + break; + } } - umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); - umtxq_lock(&uq->uq_key); if (timeout == NULL) { error = umtxq_sleep(uq, "usem", 0); } else { @@ -2862,22 +2880,42 @@ do_sem_wake(struct thread *td, struct _usem *sem) { struct umtx_key key; - int error, cnt, nwake; - uint32_t flags; + int error, nwait; + uint32_t flags, count; flags = fuword32(&sem->_flags); if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); - cnt = umtxq_count(&key); - nwake = umtxq_signal(&key, 1); - if (cnt <= nwake) { - umtxq_unlock(&key); - error = suword32( - __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0); - umtxq_lock(&key); + nwait = umtxq_count(&key); + umtxq_unlock(&key); + if ((flags & SEM_VER2) != 0) { + error = 0; + count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count)); + for (;;) { + if (__predict_false((count & ~SEM_WAITERS) == __INT_MAX) ) { + umtxq_lock(&key); + umtxq_unbusy(&key); + umtxq_unlock(&key); + return (ERANGE); + } + int newval = count + 1; + if (nwait <= 1) + newval &= ~SEM_WAITERS; + int old = casuword32(__DEVOLATILE(uint32_t *, + &sem->_count), count, newval); + if (old == count) + break; + count = old; + } + } else { + if (nwait <= 1) + error = suword32( + __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0); } + umtxq_lock(&key); + umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); Index: lib/libc/gen/sem_new.c =================================================================== --- lib/libc/gen/sem_new.c (revision 233067) +++ lib/libc/gen/sem_new.c (working copy) @@ -46,6 +46,7 @@ #include #include #include +#include #include "un-namespace.h" #include "libc_private.h" @@ -61,7 +62,8 @@ __weak_reference(_sem_wait, sem_wait); #define SEM_PREFIX "/tmp/SEMD" -#define SEM_MAGIC ((u_int32_t)0x73656d31) +#define SEM_MAGIC1 ((u_int32_t)0x73656d31) +#define SEM_MAGIC ((u_int32_t)0x73656d32) struct sem_nameinfo { int open_count; @@ -109,7 +111,7 @@ sem_check_validity(sem_t *sem) { - if (sem->_magic == SEM_MAGIC) + if (sem->_magic == SEM_MAGIC || sem->_magic == SEM_MAGIC1) return (0); else { errno = EINVAL; @@ -130,7 +132,7 @@ sem->_magic = SEM_MAGIC; sem->_kern._count = (u_int32_t)value; sem->_kern._has_waiters = 0; - sem->_kern._flags = pshared ? USYNC_PROCESS_SHARED : 0; + sem->_kern._flags = (pshared ? USYNC_PROCESS_SHARED : 0) | SEM_VER2; return (0); } @@ -207,7 +209,7 @@ tmp._magic = SEM_MAGIC; tmp._kern._has_waiters = 0; tmp._kern._count = value; - tmp._kern._flags = USYNC_PROCESS_SHARED | SEM_NAMED; + tmp._kern._flags = USYNC_PROCESS_SHARED | SEM_NAMED | SEM_VER2; if (_write(fd, &tmp, sizeof(tmp)) != sizeof(tmp)) { flock(fd, LOCK_UN); goto error; @@ -325,19 +327,11 @@ if (sem_check_validity(sem) != 0) return (-1); - *sval = (int)sem->_kern._count; + *sval = (int)sem->_kern._count & ~SEM_WAITERS; return (0); } static __inline int -usem_wake(struct _usem *sem) -{ - if (!sem->_has_waiters) - return (0); - return _umtx_op(sem, UMTX_OP_SEM_WAKE, 0, NULL, NULL); -} - -static __inline int usem_wait(struct _usem *sem, const struct timespec *timeout) { if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && @@ -349,19 +343,36 @@ __DECONST(void*, timeout)); } +static inline int +_trywait(sem_t *sem) +{ + int val; + + if ((sem->_kern._flags & SEM_VER2) != 0) { + while (((val = sem->_kern._count) & ~SEM_WAITERS) > 0) { + if (atomic_cmpset_acq_int(&sem->_kern._count, val, val - 1)) + return (0); + } + } else { + while ((val = sem->_kern._count) > 0) { + if (atomic_cmpset_acq_int(&sem->_kern._count, val, val - 1)) + return (0); + } + } + return (EAGAIN); +} + int _sem_trywait(sem_t *sem) { - int val; + int status; if (sem_check_validity(sem) != 0) return (-1); - - while ((val = sem->_kern._count) > 0) { - if (atomic_cmpset_acq_int(&sem->_kern._count, val, val - 1)) - return (0); - } - errno = EAGAIN; + status = _trywait(sem); + if ((status = _trywait(sem)) == 0) + return (0); + errno = status; return (-1); } @@ -381,17 +392,15 @@ const struct timespec * __restrict abstime) { struct timespec ts, ts2; - int val, retval; + int retval; if (sem_check_validity(sem) != 0) return (-1); retval = 0; for (;;) { - while ((val = sem->_kern._count) > 0) { - if (atomic_cmpset_acq_int(&sem->_kern._count, val, val - 1)) - return (0); - } + if (_trywait(sem) == 0) + return (0); if (retval) { _pthread_testcancel(); @@ -432,10 +441,36 @@ int _sem_post(sem_t *sem) { + int count; if (sem_check_validity(sem) != 0) return (-1); - atomic_add_rel_int(&sem->_kern._count, 1); - return usem_wake(&sem->_kern); + if ((sem->_kern._flags & SEM_VER2) != 0) { + for (;;) { + count = sem->_kern._count; + if ((count & SEM_WAITERS) == 0) { + if (__predict_false(count == SEM_VALUE_MAX)) { + errno = ERANGE; + return (-1); + } + if (atomic_cmpset_rel_int(&sem->_kern._count, count, count+1)) + return 0; + } else { + return _umtx_op(&sem->_kern, UMTX_OP_SEM_WAKE, 0, NULL, NULL); + } + } + } else { + do { + count = sem->_kern._count; + if (__predict_false(count == SEM_VALUE_MAX)) { + errno = ERANGE; + return (-1); + } + } while (!atomic_cmpset_rel_int(&sem->_kern._count, count, count+1)); + rmb(); + if (!sem->_kern._has_waiters) + return (0); + return _umtx_op(sem, UMTX_OP_SEM_WAKE, 0, NULL, NULL); + } } Index: lib/libthr/thread/thr_umtx.c =================================================================== --- lib/libthr/thread/thr_umtx.c (revision 233067) +++ lib/libthr/thread/thr_umtx.c (working copy) @@ -152,13 +152,6 @@ int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) { -#ifndef __ia64__ - /* XXX this logic has a race-condition on ia64. */ - if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { - atomic_cmpset_rel_32(&mtx->m_owner, id | UMUTEX_CONTESTED, UMUTEX_CONTESTED); - return _umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE, 0, 0, 0); - } -#endif /* __ia64__ */ return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0); }