Index: include/pthread_np.h =================================================================== --- include/pthread_np.h (revision 190196) +++ include/pthread_np.h (working copy) @@ -10,10 +10,7 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by John Birrell. - * 4. Neither the name of the author nor the names of any co-contributors + * 3. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * @@ -34,6 +31,9 @@ #ifndef _PTHREAD_NP_H_ #define _PTHREAD_NP_H_ +#include +#include + /* * Non-POSIX type definitions: */ @@ -45,6 +45,9 @@ __BEGIN_DECLS int pthread_attr_setcreatesuspend_np(pthread_attr_t *); int pthread_attr_get_np(pthread_t, pthread_attr_t *); +int pthread_attr_getaffinity_np(const pthread_attr_t *, size_t, cpuset_t *); +int pthread_attr_setaffinity_np(pthread_attr_t *, size_t, const cpuset_t *); +int pthread_getaffinity_np(pthread_t, size_t, cpuset_t *); int pthread_main_np(void); int pthread_multi_np(void); int pthread_mutexattr_getkind_np(pthread_mutexattr_t); @@ -52,6 +55,12 @@ void pthread_resume_all_np(void); int pthread_resume_np(pthread_t); void pthread_set_name_np(pthread_t, const char *); +int pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count); +int pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); +int pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count); +int pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); +int pthread_mutex_isowned_np(pthread_mutex_t *mutex); +int pthread_setaffinity_np(pthread_t, size_t, const cpuset_t *); int pthread_single_np(void); void pthread_suspend_all_np(void); int pthread_suspend_np(pthread_t); Index: include/pthread.h =================================================================== --- include/pthread.h (revision 190196) +++ include/pthread.h (working copy) @@ -135,6 +135,10 @@ #define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_ERRORCHECK +struct _pthread_cleanup_info { + __uintptr_t pthread_cleanup_pad[8]; +}; + /* * Thread function prototype definitions: */ @@ -185,6 +189,7 @@ int pthread_equal(pthread_t, pthread_t); void pthread_exit(void *) __dead2; void *pthread_getspecific(pthread_key_t); +int pthread_getcpuclockid(pthread_t, clockid_t *); int pthread_join(pthread_t, void **); int pthread_key_create(pthread_key_t *, void (*) (void *)); @@ -267,6 +272,10 @@ const struct sched_param *); int pthread_getconcurrency(void); int pthread_setconcurrency(int); + +void __pthread_cleanup_push_imp(void (*)(void *), void *, + struct _pthread_cleanup_info *); +void __pthread_cleanup_pop_imp(int); __END_DECLS #endif Index: lib/libc/stdlib/Symbol.map =================================================================== --- lib/libc/stdlib/Symbol.map (revision 190196) +++ lib/libc/stdlib/Symbol.map (working copy) @@ -95,6 +95,7 @@ FBSDprivate_1.0 { __use_pts; + _malloc_thread_cleanup; _malloc_prefork; _malloc_postfork; __system; Index: lib/libc/stdlib/malloc.c =================================================================== --- lib/libc/stdlib/malloc.c (revision 190196) +++ lib/libc/stdlib/malloc.c (working copy) @@ -4693,6 +4693,17 @@ /* * End non-standard functions. */ + +/* + * We provide an unpublished interface in order to receive notifications from + * the pthreads library whenever a thread exits. This allows us to clean up + * thread caches. + */ +void +_malloc_thread_cleanup(void) +{ +} + /******************************************************************************/ /* * Begin library-private functions, used by threading libraries for protection Index: lib/libc/gen/Symbol.map =================================================================== --- lib/libc/gen/Symbol.map (revision 190196) +++ lib/libc/gen/Symbol.map (working copy) @@ -403,6 +403,8 @@ _spinlock; _spinlock_debug; _spinunlock; + _rtld_atfork_pre; + _rtld_atfork_post; _rtld_error; /* for private use */ _rtld_thread_init; /* for private use */ _err; Index: lib/libc/gen/dlfcn.c =================================================================== --- lib/libc/gen/dlfcn.c (revision 190196) +++ lib/libc/gen/dlfcn.c (working copy) @@ -137,3 +137,15 @@ _rtld_error(sorry); return 0; } + +#pragma weak _rtld_atfork_pre +void +_rtld_atfork_pre(int *locks) +{ +} + +#pragma weak _rtld_atfork_post +void +_rtld_atfork_post(int *locks) +{ +} Index: lib/libc/include/libc_private.h =================================================================== --- lib/libc/include/libc_private.h (revision 190196) +++ lib/libc/include/libc_private.h (working copy) @@ -158,6 +158,12 @@ extern const char *__progname; /* + * This function is used by the threading libraries to notify malloc that a + * thread is exiting. + */ +void _malloc_thread_cleanup(void); + +/* * These functions are used by the threading libraries in order to protect * malloc across fork(). */ Index: lib/libthr/pthread.map =================================================================== --- lib/libthr/pthread.map (revision 190196) +++ lib/libthr/pthread.map (working copy) @@ -6,7 +6,6 @@ * Use the same naming scheme as libc. */ FBSD_1.0 { -global: __error; accept; aio_suspend; @@ -118,8 +117,8 @@ pthread_rwlockattr_getpshared; pthread_rwlockattr_init; pthread_rwlockattr_setpshared; + pthread_set_name_np; pthread_self; - pthread_set_name_np; pthread_setcancelstate; pthread_setcanceltype; pthread_setconcurrency; @@ -165,15 +164,12 @@ system; tcdrain; usleep; - vfork; wait; wait3; wait4; waitpid; write; writev; -local: - *; }; /* @@ -181,7 +177,6 @@ * These are not part of our application ABI. */ FBSDprivate_1.0 { -global: ___creat; ___pause; ___pselect; @@ -233,6 +228,7 @@ _pthread_barrierattr_setpshared; _pthread_attr_destroy; _pthread_attr_get_np; + _pthread_attr_getaffinity_np; _pthread_attr_getdetachstate; _pthread_attr_getguardsize; _pthread_attr_getinheritsched; @@ -243,6 +239,7 @@ _pthread_attr_getstackaddr; _pthread_attr_getstacksize; _pthread_attr_init; + _pthread_attr_setaffinity_np; _pthread_attr_setcreatesuspend_np; _pthread_attr_setdetachstate; _pthread_attr_setguardsize; @@ -272,7 +269,9 @@ _pthread_detach; _pthread_equal; _pthread_exit; + _pthread_getaffinity_np; _pthread_getconcurrency; + _pthread_getcpuclockid; _pthread_getprio; _pthread_getschedparam; _pthread_getspecific; @@ -284,10 +283,15 @@ _pthread_multi_np; _pthread_mutex_destroy; _pthread_mutex_getprioceiling; + _pthread_mutex_getspinloops_np; + _pthread_mutex_getyieldloops_np; _pthread_mutex_init; _pthread_mutex_init_calloc_cb; + _pthread_mutex_isowned_np; _pthread_mutex_lock; _pthread_mutex_setprioceiling; + _pthread_mutex_setspinloops_np; + _pthread_mutex_setyieldloops_np; _pthread_mutex_timedlock; _pthread_mutex_trylock; _pthread_mutex_unlock; @@ -321,6 +325,7 @@ _pthread_rwlockattr_setpshared; _pthread_self; _pthread_set_name_np; + _pthread_setaffinity_np; _pthread_setcancelstate; _pthread_setcanceltype; _pthread_setconcurrency; @@ -358,7 +363,6 @@ _spinlock; _spinlock_debug; _spinunlock; - _vfork; /* Debugger needs these. */ _libthr_debug; @@ -386,6 +390,19 @@ _thread_size_key; _thread_state_running; _thread_state_zoombie; -local: - *; }; + +FBSD_1.1 { + __pthread_cleanup_pop_imp; + __pthread_cleanup_push_imp; + pthread_attr_getaffinity_np; + pthread_attr_setaffinity_np; + pthread_getaffinity_np; + pthread_getcpuclockid; + pthread_setaffinity_np; + pthread_mutex_getspinloops_np; + pthread_mutex_getyieldloops_np; + pthread_mutex_isowned_np; + pthread_mutex_setspinloops_np; + pthread_mutex_setyieldloops_np; +}; Index: lib/libthr/thread/thr_event.c =================================================================== --- lib/libthr/thread/thr_event.c (revision 190196) +++ lib/libthr/thread/thr_event.c (working copy) @@ -42,7 +42,7 @@ _thr_report_creation(struct pthread *curthread, struct pthread *newthread) { curthread->event_buf.event = TD_CREATE; - curthread->event_buf.th_p = (td_thrhandle_t *)newthread; + curthread->event_buf.th_p = (uintptr_t)newthread; curthread->event_buf.data = 0; THR_UMUTEX_LOCK(curthread, &_thr_event_lock); _thread_last_event = curthread; @@ -55,7 +55,7 @@ _thr_report_death(struct pthread *curthread) { curthread->event_buf.event = TD_DEATH; - curthread->event_buf.th_p = (td_thrhandle_t *)curthread; + curthread->event_buf.th_p = (uintptr_t)curthread; curthread->event_buf.data = 0; THR_UMUTEX_LOCK(curthread, &_thr_event_lock); _thread_last_event = curthread; Index: lib/libthr/thread/thr_clean.c =================================================================== --- lib/libthr/thread/thr_clean.c (revision 190196) +++ lib/libthr/thread/thr_clean.c (working copy) @@ -38,38 +38,61 @@ #include "thr_private.h" +#undef pthread_cleanup_push +#undef pthread_cleanup_pop + +/* old binary compatible interfaces */ __weak_reference(_pthread_cleanup_push, pthread_cleanup_push); __weak_reference(_pthread_cleanup_pop, pthread_cleanup_pop); void -_pthread_cleanup_push(void (*routine) (void *), void *routine_arg) +__pthread_cleanup_push_imp(void (*routine)(void *), void *arg, + struct _pthread_cleanup_info *info) { struct pthread *curthread = _get_curthread(); - struct pthread_cleanup *new; + struct pthread_cleanup *newbuf; - if ((new = (struct pthread_cleanup *) - malloc(sizeof(struct pthread_cleanup))) != NULL) { - new->routine = routine; - new->routine_arg = routine_arg; - new->onstack = 0; - new->next = curthread->cleanup; - - curthread->cleanup = new; - } + newbuf = (void *)info; + newbuf->routine = routine; + newbuf->routine_arg = arg; + newbuf->onheap = 0; + newbuf->prev = curthread->cleanup; + curthread->cleanup = newbuf; } void -_pthread_cleanup_pop(int execute) +__pthread_cleanup_pop_imp(int execute) { struct pthread *curthread = _get_curthread(); struct pthread_cleanup *old; if ((old = curthread->cleanup) != NULL) { - curthread->cleanup = old->next; - if (execute) { + curthread->cleanup = old->prev; + if (execute) old->routine(old->routine_arg); - } - if (old->onstack == 0) + if (old->onheap) free(old); } } + +void +_pthread_cleanup_push(void (*routine) (void *), void *arg) +{ + struct pthread *curthread = _get_curthread(); + struct pthread_cleanup *newbuf; + + if ((newbuf = (struct pthread_cleanup *) + malloc(sizeof(struct _pthread_cleanup_info))) != NULL) { + newbuf->routine = routine; + newbuf->routine_arg = arg; + newbuf->onheap = 1; + newbuf->prev = curthread->cleanup; + curthread->cleanup = newbuf; + } +} + +void +_pthread_cleanup_pop(int execute) +{ + __pthread_cleanup_pop_imp(execute); +} Index: lib/libthr/thread/thr_sig.c =================================================================== --- lib/libthr/thread/thr_sig.c (revision 190196) +++ lib/libthr/thread/thr_sig.c (working copy) @@ -52,9 +52,15 @@ int _raise(int); int __sigtimedwait(const sigset_t *set, siginfo_t *info, const struct timespec * timeout); +int _sigtimedwait(const sigset_t *set, siginfo_t *info, + const struct timespec * timeout); int __sigwaitinfo(const sigset_t *set, siginfo_t *info); +int _sigwaitinfo(const sigset_t *set, siginfo_t *info); int __sigwait(const sigset_t *set, int *sig); +int _sigwait(const sigset_t *set, int *sig); +int __sigsuspend(const sigset_t *sigmask); + static void sigcancel_handler(int sig __unused, siginfo_t *info __unused, ucontext_t *ucp __unused) @@ -83,9 +89,12 @@ void _thr_suspend_check(struct pthread *curthread) { - long cycle; + uint32_t cycle; int err; + if (curthread->force_exit) + return; + err = errno; /* * Blocks SIGCANCEL which other threads must send. @@ -105,7 +114,7 @@ cycle = curthread->cycle; /* Wake the thread suspending us. */ - _thr_umtx_wake(&curthread->cycle, INT_MAX); + _thr_umtx_wake(&curthread->cycle, INT_MAX, 0); /* * if we are from pthread_exit, we don't want to @@ -115,7 +124,7 @@ break; curthread->flags |= THR_FLAGS_SUSPENDED; THR_UMUTEX_UNLOCK(curthread, &(curthread)->lock); - _thr_umtx_wait(&curthread->cycle, cycle, NULL); + _thr_umtx_wait_uint(&curthread->cycle, cycle, NULL, 0); THR_UMUTEX_LOCK(curthread, &(curthread)->lock); curthread->flags &= ~THR_FLAGS_SUSPENDED; } Index: lib/libthr/thread/thr_spinlock.c =================================================================== --- lib/libthr/thread/thr_spinlock.c (revision 190196) +++ lib/libthr/thread/thr_spinlock.c (working copy) @@ -63,16 +63,16 @@ void _spinunlock(spinlock_t *lck) { - struct spinlock_extra *extra; + struct spinlock_extra *_extra; - extra = (struct spinlock_extra *)lck->fname; - THR_UMUTEX_UNLOCK(_get_curthread(), &extra->lock); + _extra = (struct spinlock_extra *)lck->fname; + THR_UMUTEX_UNLOCK(_get_curthread(), &_extra->lock); } void _spinlock(spinlock_t *lck) { - struct spinlock_extra *extra; + struct spinlock_extra *_extra; if (!__isthreaded) PANIC("Spinlock called when not threaded."); @@ -80,8 +80,8 @@ PANIC("Spinlocks not initialized."); if (lck->fname == NULL) init_spinlock(lck); - extra = (struct spinlock_extra *)lck->fname; - THR_UMUTEX_LOCK(_get_curthread(), &extra->lock); + _extra = (struct spinlock_extra *)lck->fname; + THR_UMUTEX_LOCK(_get_curthread(), &_extra->lock); } void Index: lib/libthr/thread/thr_sem.c =================================================================== --- lib/libthr/thread/thr_sem.c (revision 190196) +++ lib/libthr/thread/thr_sem.c (working copy) @@ -172,6 +172,16 @@ return (-1); } +static void +sem_cancel_handler(void *arg) +{ + sem_t *sem = arg; + + atomic_add_int(&(*sem)->nwaiters, -1); + if ((*sem)->nwaiters && (*sem)->count) + _thr_umtx_wake(&(*sem)->count, 1, 0); +} + int _sem_wait(sem_t *sem) { @@ -195,9 +205,13 @@ if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1)) return (0); } + atomic_add_int(&(*sem)->nwaiters, 1); + THR_CLEANUP_PUSH(curthread, sem_cancel_handler, sem); _thr_cancel_enter(curthread); - retval = _thr_umtx_wait((long *)&(*sem)->count, 0, NULL); + retval = _thr_umtx_wait_uint(&(*sem)->count, 0, NULL, 0); _thr_cancel_leave(curthread); + THR_CLEANUP_POP(curthread, 0); + atomic_add_int(&(*sem)->nwaiters, -1); } while (retval == 0); errno = retval; return (-1); @@ -238,18 +252,27 @@ } clock_gettime(CLOCK_REALTIME, &ts); TIMESPEC_SUB(&ts2, abstime, &ts); + atomic_add_int(&(*sem)->nwaiters, 1); + THR_CLEANUP_PUSH(curthread, sem_cancel_handler, sem); _thr_cancel_enter(curthread); - retval = _thr_umtx_wait((long *)&(*sem)->count, 0, &ts2); + retval = _thr_umtx_wait_uint((uint32_t*)&(*sem)->count, 0, &ts2, 0); _thr_cancel_leave(curthread); + THR_CLEANUP_POP(curthread, 0); + atomic_add_int(&(*sem)->nwaiters, -1); } while (retval == 0); errno = retval; return (-1); } +/* + * sem_post() is required to be safe to call from within + * signal handlers, these code should work as that. + */ + int _sem_post(sem_t *sem) { - int val, retval; + int retval = 0; if (sem_check_validity(sem) != 0) return (-1); @@ -257,15 +280,12 @@ if ((*sem)->syssem != 0) return (ksem_post((*sem)->semid)); - /* - * sem_post() is required to be safe to call from within - * signal handlers, these code should work as that. - */ - do { - val = (*sem)->count; - } while (!atomic_cmpset_acq_int(&(*sem)->count, val, val + 1)); - retval = _thr_umtx_wake((long *)&(*sem)->count, val + 1); - if (retval > 0) - retval = 0; + atomic_add_rel_int(&(*sem)->count, 1); + + if ((*sem)->nwaiters) { + retval = _thr_umtx_wake(&(*sem)->count, 1, 0); + if (retval != 0) + retval = -1; + } return (retval); } Index: lib/libthr/thread/Makefile.inc =================================================================== --- lib/libthr/thread/Makefile.inc (revision 190196) +++ lib/libthr/thread/Makefile.inc (working copy) @@ -4,6 +4,7 @@ .PATH: ${.CURDIR}/thread SRCS+= \ + thr_affinity.c \ thr_attr.c \ thr_barrier.c \ thr_barrierattr.c \ @@ -19,6 +20,7 @@ thr_exit.c \ thr_fork.c \ thr_getprio.c \ + thr_getcpuclockid.c \ thr_getschedparam.c \ thr_info.c \ thr_init.c \ Index: lib/libthr/thread/thr_resume_np.c =================================================================== --- lib/libthr/thread/thr_resume_np.c (revision 190196) +++ lib/libthr/thread/thr_resume_np.c (working copy) @@ -87,5 +87,5 @@ /* Clear the suspend flag: */ thread->flags &= ~THR_FLAGS_NEED_SUSPEND; thread->cycle++; - _thr_umtx_wake(&thread->cycle, 1); + _thr_umtx_wake(&thread->cycle, 1, 0); } Index: lib/libthr/thread/thr_syscalls.c =================================================================== --- lib/libthr/thread/thr_syscalls.c (revision 190196) +++ lib/libthr/thread/thr_syscalls.c (working copy) @@ -132,7 +132,9 @@ int __close(int); int __connect(int, const struct sockaddr *, socklen_t); int __fcntl(int, int,...); +#ifdef SYSCALL_COMPAT extern int __fcntl_compat(int, int,...); +#endif int __fsync(int); int __msync(void *, size_t, int); int __nanosleep(const struct timespec *, struct timespec *); @@ -150,7 +152,6 @@ pid_t __wait4(pid_t, int *, int, struct rusage *); ssize_t __write(int, const void *, size_t); ssize_t __writev(int, const struct iovec *, int); -int _vfork(void); __weak_reference(__accept, accept); @@ -254,7 +255,11 @@ ret = __sys_fcntl(fd, cmd); break; default: +#ifdef SYSCALL_COMPAT ret = __fcntl_compat(fd, cmd, va_arg(ap, void *)); +#else + ret = __sys_fcntl(fd, cmd, va_arg(ap, void *)); +#endif } va_end(ap); @@ -530,14 +535,6 @@ return (ret); } -__weak_reference(_vfork, vfork); - -int -_vfork(void) -{ - return (fork()); -} - __weak_reference(___wait, wait); pid_t Index: lib/libthr/thread/thr_mutex.c =================================================================== --- lib/libthr/thread/thr_mutex.c (revision 190196) +++ lib/libthr/thread/thr_mutex.c (working copy) @@ -40,6 +40,7 @@ #include #include #include +#include #include "un-namespace.h" #include "thr_private.h" @@ -50,12 +51,12 @@ (m)->m_qe.tqe_next = NULL; \ } while (0) #define MUTEX_ASSERT_IS_OWNED(m) do { \ - if ((m)->m_qe.tqe_prev == NULL) \ + if (__predict_false((m)->m_qe.tqe_prev == NULL))\ PANIC("mutex is not on list"); \ } while (0) #define MUTEX_ASSERT_NOT_OWNED(m) do { \ - if (((m)->m_qe.tqe_prev != NULL) || \ - ((m)->m_qe.tqe_next != NULL)) \ + if (__predict_false((m)->m_qe.tqe_prev != NULL || \ + (m)->m_qe.tqe_next != NULL)) \ PANIC("mutex is on list"); \ } while (0) #else @@ -68,7 +69,7 @@ * For adaptive mutexes, how many times to spin doing trylock2 * before entering the kernel to block */ -#define MUTEX_ADAPTIVE_SPINS 200 +#define MUTEX_ADAPTIVE_SPINS 2000 /* * Prototypes @@ -79,16 +80,30 @@ int __pthread_mutex_lock(pthread_mutex_t *mutex); int __pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abstime); +int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, + void *(calloc_cb)(size_t, size_t)); +int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count); +int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); +int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); +int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); +int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count); +int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); static int mutex_self_trylock(pthread_mutex_t); static int mutex_self_lock(pthread_mutex_t, const struct timespec *abstime); static int mutex_unlock_common(pthread_mutex_t *); +static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, + const struct timespec *); __weak_reference(__pthread_mutex_init, pthread_mutex_init); +__strong_reference(__pthread_mutex_init, _pthread_mutex_init); __weak_reference(__pthread_mutex_lock, pthread_mutex_lock); +__strong_reference(__pthread_mutex_lock, _pthread_mutex_lock); __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); +__strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock); __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); +__strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock); /* Single underscore versions provided for libc internal usage: */ /* No difference between libc and application usage of these: */ @@ -98,9 +113,18 @@ __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling); __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling); +__weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np); +__strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np); +__weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np); + +__weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np); +__strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np); +__weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np); +__weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np); + static int mutex_init(pthread_mutex_t *mutex, - const pthread_mutexattr_t *mutex_attr, int private, + const pthread_mutexattr_t *mutex_attr, void *(calloc_cb)(size_t, size_t)) { const struct pthread_mutex_attr *attr; @@ -123,11 +147,10 @@ pmutex->m_type = attr->m_type; pmutex->m_owner = NULL; - pmutex->m_flags = attr->m_flags | MUTEX_FLAGS_INITED; - if (private) - pmutex->m_flags |= MUTEX_FLAGS_PRIVATE; pmutex->m_count = 0; pmutex->m_refcount = 0; + pmutex->m_spinloops = 0; + pmutex->m_yieldloops = 0; MUTEX_INIT_LINK(pmutex); switch(attr->m_protocol) { case PTHREAD_PRIO_INHERIT: @@ -143,6 +166,13 @@ pmutex->m_lock.m_owner = UMUTEX_UNOWNED; pmutex->m_lock.m_flags = 0; } + + if (pmutex->m_type == PTHREAD_MUTEX_ADAPTIVE_NP) { + pmutex->m_spinloops = + _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS; + pmutex->m_yieldloops = _thr_yieldloops; + } + *mutex = pmutex; return (0); } @@ -155,7 +185,7 @@ THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); if (*mutex == NULL) - ret = mutex_init(mutex, NULL, 0, calloc); + ret = mutex_init(mutex, NULL, calloc); else ret = 0; @@ -164,23 +194,6 @@ return (ret); } -static int -init_static_private(struct pthread *thread, pthread_mutex_t *mutex) -{ - int ret; - - THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); - - if (*mutex == NULL) - ret = mutex_init(mutex, NULL, 1, calloc); - else - ret = 0; - - THR_LOCK_RELEASE(thread, &_mutex_static_lock); - - return (ret); -} - static void set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) { @@ -194,17 +207,10 @@ } int -_pthread_mutex_init(pthread_mutex_t *mutex, - const pthread_mutexattr_t *mutex_attr) -{ - return mutex_init(mutex, mutex_attr, 1, calloc); -} - -int __pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *mutex_attr) { - return mutex_init(mutex, mutex_attr, 0, calloc); + return mutex_init(mutex, mutex_attr, calloc); } /* This function is used internally by malloc. */ @@ -215,12 +221,11 @@ static const struct pthread_mutex_attr attr = { .m_type = PTHREAD_MUTEX_NORMAL, .m_protocol = PTHREAD_PRIO_NONE, - .m_ceiling = 0, - .m_flags = 0 + .m_ceiling = 0 }; static const struct pthread_mutex_attr *pattr = &attr; - return mutex_init(mutex, (pthread_mutexattr_t *)&pattr, 0, calloc_cb); + return mutex_init(mutex, (pthread_mutexattr_t *)&pattr, calloc_cb); } void @@ -294,7 +299,6 @@ return (ret); } - #define ENQUEUE_MUTEX(curthread, m) \ do { \ (m)->m_owner = curthread; \ @@ -343,142 +347,90 @@ return (mutex_trylock_common(curthread, mutex)); } -int -_pthread_mutex_trylock(pthread_mutex_t *mutex) +static int +mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, + const struct timespec *abstime) { - struct pthread *curthread = _get_curthread(); + uint32_t id, owner; + int count; int ret; + if (m->m_owner == curthread) + return mutex_self_lock(m, abstime); + + id = TID(curthread); /* - * If the mutex is statically initialized, perform the dynamic - * initialization marking the mutex private (delete safe): + * For adaptive mutexes, spin for a bit in the expectation + * that if the application requests this mutex type then + * the lock is likely to be released quickly and it is + * faster than entering the kernel */ - if (__predict_false(*mutex == NULL)) { - ret = init_static_private(curthread, mutex); - if (__predict_false(ret)) - return (ret); - } - return (mutex_trylock_common(curthread, mutex)); -} + if (m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) + goto sleep_in_kernel; -static int -mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex, - const struct timespec * abstime) -{ - struct timespec ts, ts2; - struct pthread_mutex *m; - uint32_t id; - int ret; - int count; + if (!_thr_is_smp) + goto yield_loop; - id = TID(curthread); - m = *mutex; - ret = _thr_umutex_trylock2(&m->m_lock, id); - if (ret == 0) { - ENQUEUE_MUTEX(curthread, m); - } else if (m->m_owner == curthread) { - ret = mutex_self_lock(m, abstime); - } else { - /* - * For adaptive mutexes, spin for a bit in the expectation - * that if the application requests this mutex type then - * the lock is likely to be released quickly and it is - * faster than entering the kernel - */ - if (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) - goto sleep_in_kernel; - - if (!_thr_is_smp) - goto yield_loop; - - if (m->m_type == PTHREAD_MUTEX_ADAPTIVE_NP) { - count = MUTEX_ADAPTIVE_SPINS; - - while (count--) { - ret = _thr_umutex_trylock2(&m->m_lock, id); - if (ret == 0) - break; - CPU_SPINWAIT; - } - if (ret == 0) + count = m->m_spinloops; + while (count--) { + owner = m->m_lock.m_owner; + if ((owner & ~UMUTEX_CONTESTED) == 0) { + if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { + ret = 0; goto done; - } else { - if (_thr_spinloops != 0) { - count = _thr_spinloops; - while (count) { - if (m->m_lock.m_owner == UMUTEX_UNOWNED) { - ret = _thr_umutex_trylock2(&m->m_lock, id); - if (ret == 0) - goto done; - } - CPU_SPINWAIT; - count--; - } } } + CPU_SPINWAIT; + } yield_loop: - if (_thr_yieldloops != 0) { - count = _thr_yieldloops; - while (count--) { - _sched_yield(); - ret = _thr_umutex_trylock2(&m->m_lock, id); - if (ret == 0) - goto done; + count = m->m_yieldloops; + while (count--) { + _sched_yield(); + owner = m->m_lock.m_owner; + if ((owner & ~UMUTEX_CONTESTED) == 0) { + if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { + ret = 0; + goto done; } } + } sleep_in_kernel: - if (abstime == NULL) { - ret = __thr_umutex_lock(&m->m_lock); - } else if (__predict_false( - abstime->tv_sec < 0 || abstime->tv_nsec < 0 || - abstime->tv_nsec >= 1000000000)) { - ret = EINVAL; - } else { - clock_gettime(CLOCK_REALTIME, &ts); - TIMESPEC_SUB(&ts2, abstime, &ts); - ret = __thr_umutex_timedlock(&m->m_lock, &ts2); - /* - * Timed out wait is not restarted if - * it was interrupted, not worth to do it. - */ - if (ret == EINTR) - ret = ETIMEDOUT; - } -done: - if (ret == 0) - ENQUEUE_MUTEX(curthread, m); + if (abstime == NULL) { + ret = __thr_umutex_lock(&m->m_lock, id); + } else if (__predict_false( + abstime->tv_nsec < 0 || + abstime->tv_nsec >= 1000000000)) { + ret = EINVAL; + } else { + ret = __thr_umutex_timedlock(&m->m_lock, id, abstime); } +done: + if (ret == 0) + ENQUEUE_MUTEX(curthread, m); + return (ret); } -int -__pthread_mutex_lock(pthread_mutex_t *m) +static inline int +mutex_lock_common(struct pthread *curthread, struct pthread_mutex *m, + const struct timespec *abstime) { - struct pthread *curthread; - int ret; - _thr_check_init(); - - curthread = _get_curthread(); - - /* - * If the mutex is statically initialized, perform the dynamic - * initialization: - */ - if (__predict_false(*m == NULL)) { - ret = init_static(curthread, m); - if (__predict_false(ret)) - return (ret); + if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) { + ENQUEUE_MUTEX(curthread, m); + return (0); } - return (mutex_lock_common(curthread, m, NULL)); + + return (mutex_lock_sleep(curthread, m, abstime)); } int -_pthread_mutex_lock(pthread_mutex_t *m) +__pthread_mutex_lock(pthread_mutex_t *mutex) { struct pthread *curthread; + struct pthread_mutex *m; int ret; _thr_check_init(); @@ -487,20 +439,23 @@ /* * If the mutex is statically initialized, perform the dynamic - * initialization marking it private (delete safe): + * initialization: */ - if (__predict_false(*m == NULL)) { - ret = init_static_private(curthread, m); + if (__predict_false((m = *mutex) == NULL)) { + ret = init_static(curthread, mutex); if (__predict_false(ret)) return (ret); + m = *mutex; } + return (mutex_lock_common(curthread, m, NULL)); } int -__pthread_mutex_timedlock(pthread_mutex_t *m, const struct timespec *abstime) +__pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abstime) { struct pthread *curthread; + struct pthread_mutex *m; int ret; _thr_check_init(); @@ -511,37 +466,16 @@ * If the mutex is statically initialized, perform the dynamic * initialization: */ - if (__predict_false(*m == NULL)) { - ret = init_static(curthread, m); + if (__predict_false((m = *mutex) == NULL)) { + ret = init_static(curthread, mutex); if (__predict_false(ret)) return (ret); + m = *mutex; } return (mutex_lock_common(curthread, m, abstime)); } int -_pthread_mutex_timedlock(pthread_mutex_t *m, const struct timespec *abstime) -{ - struct pthread *curthread; - int ret; - - _thr_check_init(); - - curthread = _get_curthread(); - - /* - * If the mutex is statically initialized, perform the dynamic - * initialization marking it private (delete safe): - */ - if (__predict_false(*m == NULL)) { - ret = init_static_private(curthread, m); - if (__predict_false(ret)) - return (ret); - } - return (mutex_lock_common(curthread, m, abstime)); -} - -int _pthread_mutex_unlock(pthread_mutex_t *m) { return (mutex_unlock_common(m)); @@ -552,7 +486,7 @@ { int ret; - ret = mutex_lock_common(_get_curthread(), m, NULL); + ret = mutex_lock_common(_get_curthread(), *m, NULL); if (ret == 0) { (*m)->m_refcount--; (*m)->m_count += count; @@ -598,10 +532,15 @@ case PTHREAD_MUTEX_ERRORCHECK: case PTHREAD_MUTEX_ADAPTIVE_NP: if (abstime) { - clock_gettime(CLOCK_REALTIME, &ts1); - TIMESPEC_SUB(&ts2, abstime, &ts1); - __sys_nanosleep(&ts2, NULL); - ret = ETIMEDOUT; + if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || + abstime->tv_nsec >= 1000000000) { + ret = EINVAL; + } else { + clock_gettime(CLOCK_REALTIME, &ts1); + TIMESPEC_SUB(&ts2, abstime, &ts1); + __sys_nanosleep(&ts2, NULL); + ret = ETIMEDOUT; + } } else { /* * POSIX specifies that mutexes should return @@ -618,10 +557,15 @@ */ ret = 0; if (abstime) { - clock_gettime(CLOCK_REALTIME, &ts1); - TIMESPEC_SUB(&ts2, abstime, &ts1); - __sys_nanosleep(&ts2, NULL); - ret = ETIMEDOUT; + if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || + abstime->tv_nsec >= 1000000000) { + ret = EINVAL; + } else { + clock_gettime(CLOCK_REALTIME, &ts1); + TIMESPEC_SUB(&ts2, abstime, &ts1); + __sys_nanosleep(&ts2, NULL); + ret = ETIMEDOUT; + } } else { ts1.tv_sec = 30; ts1.tv_nsec = 0; @@ -719,17 +663,6 @@ return (0); } -void -_mutex_unlock_private(pthread_t pthread) -{ - struct pthread_mutex *m, *m_next; - - TAILQ_FOREACH_SAFE(m, &pthread->mutexq, m_qe, m_next) { - if ((m->m_flags & MUTEX_FLAGS_PRIVATE) != 0) - _pthread_mutex_unlock(&m); - } -} - int _pthread_mutex_getprioceiling(pthread_mutex_t *mutex, int *prioceiling) @@ -768,11 +701,11 @@ MUTEX_ASSERT_IS_OWNED(m); m1 = TAILQ_PREV(m, mutex_queue, m_qe); m2 = TAILQ_NEXT(m, m_qe); - if ((m1 != NULL && m1->m_lock.m_ceilings[0] > ceiling) || - (m2 != NULL && m2->m_lock.m_ceilings[0] < ceiling)) { + if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) || + (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) { TAILQ_REMOVE(&curthread->pp_mutexq, m, m_qe); TAILQ_FOREACH(m2, &curthread->pp_mutexq, m_qe) { - if (m2->m_lock.m_ceilings[0] > ceiling) { + if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) { TAILQ_INSERT_BEFORE(m2, m, m_qe); return (0); } @@ -782,3 +715,65 @@ } return (0); } + +int +_pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count) +{ + if (*mutex == NULL) + return (EINVAL); + *count = (*mutex)->m_spinloops; + return (0); +} + +int +__pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count) +{ + struct pthread *curthread = _get_curthread(); + int ret; + + if (__predict_false(*mutex == NULL)) { + ret = init_static(curthread, mutex); + if (__predict_false(ret)) + return (ret); + } + (*mutex)->m_spinloops = count; + return (0); +} + +int +_pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count) +{ + if (*mutex == NULL) + return (EINVAL); + *count = (*mutex)->m_yieldloops; + return (0); +} + +int +__pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count) +{ + struct pthread *curthread = _get_curthread(); + int ret; + + if (__predict_false(*mutex == NULL)) { + ret = init_static(curthread, mutex); + if (__predict_false(ret)) + return (ret); + } + (*mutex)->m_yieldloops = count; + return (0); +} + +int +_pthread_mutex_isowned_np(pthread_mutex_t *mutex) +{ + struct pthread *curthread = _get_curthread(); + int ret; + + if (__predict_false(*mutex == NULL)) { + ret = init_static(curthread, mutex); + if (__predict_false(ret)) + return (ret); + } + return ((*mutex)->m_owner == curthread); +} Index: lib/libthr/thread/thr_pspinlock.c =================================================================== --- lib/libthr/thread/thr_pspinlock.c (revision 190196) +++ lib/libthr/thread/thr_pspinlock.c (working copy) @@ -104,7 +104,7 @@ count = SPIN_COUNT; while ((ret = THR_UMUTEX_TRYLOCK(curthread, &lck->s_lock)) != 0) { while (lck->s_lock.m_owner) { - if (_thr_is_smp) { + if (!_thr_is_smp) { _pthread_yield(); } else { CPU_SPINWAIT; Index: lib/libthr/thread/thr_attr.c =================================================================== --- lib/libthr/thread/thr_attr.c (revision 190196) +++ lib/libthr/thread/thr_attr.c (working copy) @@ -99,6 +99,7 @@ #include #include #include +#include #include "un-namespace.h" #include "thr_private.h" @@ -148,7 +149,9 @@ attr.flags |= PTHREAD_DETACHED; _thr_ref_delete(curthread, pid); memcpy(*dst, &attr, sizeof(struct pthread_attr)); - + /* XXX */ + (*dst)->cpuset = NULL; + (*dst)->cpusetsize = 0; return (0); } @@ -543,3 +546,92 @@ } return(ret); } + +static size_t +_get_kern_cpuset_size(void) +{ + static int kern_cpuset_size = 0; + + if (kern_cpuset_size == 0) { + size_t len; + + len = sizeof(kern_cpuset_size); + if (sysctlbyname("kern.smp.maxcpus", &kern_cpuset_size, + &len, NULL, 0)) + PANIC("failed to get sysctl kern.smp.maxcpus"); + + kern_cpuset_size = (kern_cpuset_size + 7) / 8; + } + + return (kern_cpuset_size); +} + +__weak_reference(_pthread_attr_setaffinity_np, pthread_attr_setaffinity_np); +int +_pthread_attr_setaffinity_np(pthread_attr_t *pattr, size_t cpusetsize, + const cpuset_t *cpusetp) +{ + pthread_attr_t attr; + int ret; + + if (pattr == NULL || (attr = (*pattr)) == NULL) + ret = EINVAL; + else { + if (cpusetsize == 0 || cpusetp == NULL) { + if (attr->cpuset != NULL) { + free(attr->cpuset); + attr->cpuset = NULL; + attr->cpusetsize = 0; + } + return (0); + } + + if (cpusetsize > attr->cpusetsize) { + size_t kern_size = _get_kern_cpuset_size(); + if (cpusetsize > kern_size) { + size_t i; + for (i = kern_size; i < cpusetsize; ++i) { + if (((char *)cpusetp)[i]) + return (EINVAL); + } + } + void *newset = realloc(attr->cpuset, cpusetsize); + if (newset == NULL) + return (ENOMEM); + attr->cpuset = newset; + attr->cpusetsize = cpusetsize; + } else { + memset(((char *)attr->cpuset) + cpusetsize, 0, + attr->cpusetsize - cpusetsize); + attr->cpusetsize = cpusetsize; + } + memcpy(attr->cpuset, cpusetp, cpusetsize); + ret = 0; + } + return (ret); +} + +__weak_reference(_pthread_attr_getaffinity_np, pthread_attr_getaffinity_np); +int +_pthread_attr_getaffinity_np(const pthread_attr_t *pattr, size_t cpusetsize, + cpuset_t *cpusetp) +{ + pthread_attr_t attr; + int ret = 0; + + if (pattr == NULL || (attr = (*pattr)) == NULL) + ret = EINVAL; + else if (attr->cpuset != NULL) { + memcpy(cpusetp, attr->cpuset, MIN(cpusetsize, attr->cpusetsize)); + if (cpusetsize > attr->cpusetsize) + memset(((char *)cpusetp) + attr->cpusetsize, 0, + cpusetsize - attr->cpusetsize); + } else { + size_t kern_size = _get_kern_cpuset_size(); + memset(cpusetp, -1, MIN(cpusetsize, kern_size)); + if (cpusetsize > kern_size) + memset(((char *)cpusetp) + kern_size, 0, + cpusetsize - kern_size); + } + return (ret); +} Index: lib/libthr/thread/thr_suspend_np.c =================================================================== --- lib/libthr/thread/thr_suspend_np.c (revision 190196) +++ lib/libthr/thread/thr_suspend_np.c (working copy) @@ -130,7 +130,7 @@ THR_THREAD_UNLOCK(curthread, thread); _thr_send_sig(thread, SIGCANCEL); if (waitok) { - _thr_umtx_wait(&thread->cycle, tmp, NULL); + _thr_umtx_wait_uint(&thread->cycle, tmp, NULL, 0); THR_THREAD_LOCK(curthread, thread); } else { THR_THREAD_LOCK(curthread, thread); Index: lib/libthr/thread/thr_private.h =================================================================== --- lib/libthr/thread/thr_private.h (revision 190196) +++ lib/libthr/thread/thr_private.h (working copy) @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include #include #include @@ -116,27 +118,20 @@ struct umutex m_lock; enum pthread_mutextype m_type; struct pthread *m_owner; - int m_flags; int m_count; int m_refcount; + int m_spinloops; + int m_yieldloops; /* * Link for all mutexes a thread currently owns. */ TAILQ_ENTRY(pthread_mutex) m_qe; }; -/* - * Flags for mutexes. - */ -#define MUTEX_FLAGS_PRIVATE 0x01 -#define MUTEX_FLAGS_INITED 0x02 -#define MUTEX_FLAGS_BUSY 0x04 - struct pthread_mutex_attr { enum pthread_mutextype m_type; int m_protocol; int m_ceiling; - int m_flags; }; #define PTHREAD_MUTEXATTR_STATIC_INITIALIZER \ @@ -181,10 +176,10 @@ * Cleanup definitions. */ struct pthread_cleanup { - struct pthread_cleanup *next; - void (*routine)(void *args); + struct pthread_cleanup *prev; + void (*routine)(void *); void *routine_arg; - int onstack; + int onheap; }; #define THR_CLEANUP_PUSH(td, func, arg) { \ @@ -192,12 +187,12 @@ \ __cup.routine = func; \ __cup.routine_arg = arg; \ - __cup.onstack = 1; \ - __cup.next = (td)->cleanup; \ + __cup.onheap = 0; \ + __cup.prev = (td)->cleanup; \ (td)->cleanup = &__cup; #define THR_CLEANUP_POP(td, exec) \ - (td)->cleanup = __cup.next; \ + (td)->cleanup = __cup.prev; \ if ((exec) != 0) \ __cup.routine(__cup.routine_arg); \ } @@ -219,6 +214,8 @@ void *stackaddr_attr; size_t stacksize_attr; size_t guardsize_attr; + cpuset_t *cpuset; + size_t cpusetsize; }; /* @@ -309,7 +306,7 @@ struct umutex lock; /* Internal condition variable cycle number. */ - long cycle; + uint32_t cycle; /* How many low level locks the thread held. */ int locklevel; @@ -370,11 +367,14 @@ sigset_t sigmask; /* Thread is in SIGCANCEL handler. */ - int in_sigcancel_handler; + int in_sigcancel_handler; /* New thread should unblock SIGCANCEL. */ - int unblock_sigcancel; + int unblock_sigcancel; + /* Force new thread to exit. */ + int force_exit; + /* Thread state: */ enum pthread_state state; @@ -451,8 +451,10 @@ (thrd)->critical_count++ #define THR_CRITICAL_LEAVE(thrd) \ - (thrd)->critical_count--; \ - _thr_ast(thrd); + do { \ + (thrd)->critical_count--; \ + _thr_ast(thrd); \ + } while (0) #define THR_UMUTEX_TRYLOCK(thrd, lck) \ _thr_umutex_trylock((lck), TID(thrd)) @@ -608,7 +610,6 @@ int _mutex_cv_unlock(pthread_mutex_t *, int *count) __hidden; int _mutex_reinit(pthread_mutex_t *) __hidden; void _mutex_fork(struct pthread *curthread) __hidden; -void _mutex_unlock_private(struct pthread *) __hidden; void _libpthread_init(struct pthread *) __hidden; struct pthread *_thr_alloc(struct pthread *) __hidden; void _thread_exit(const char *, int, const char *) __hidden __dead2; @@ -660,6 +661,9 @@ void _thread_bp_death(void); int _sched_yield(void); +void _pthread_cleanup_push(void (*)(void *), void *); +void _pthread_cleanup_pop(int); + /* #include */ #ifdef _SYS_FCNTL_H_ int __sys_fcntl(int, int, ...); Index: lib/libthr/thread/thr_getcpuclockid.c =================================================================== --- lib/libthr/thread/thr_getcpuclockid.c (revision 0) +++ lib/libthr/thread/thr_getcpuclockid.c (revision 0) @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2008 David Xu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "namespace.h" +#include +#include +#include +#include "un-namespace.h" + +#include "thr_private.h" + +__weak_reference(_pthread_getcpuclockid, pthread_getcpuclockid); + +int +_pthread_getcpuclockid(pthread_t pthread, clockid_t *clock_id) +{ + if (pthread == NULL) + return (EINVAL); + + *clock_id = CLOCK_THREAD_CPUTIME_ID; + return (0); +} Index: lib/libthr/thread/thr_once.c =================================================================== --- lib/libthr/thread/thr_once.c (revision 190196) +++ lib/libthr/thread/thr_once.c (working copy) @@ -38,11 +38,8 @@ #define ONCE_NEVER_DONE PTHREAD_NEEDS_INIT #define ONCE_DONE PTHREAD_DONE_INIT #define ONCE_IN_PROGRESS 0x02 -#define ONCE_MASK 0x03 +#define ONCE_WAIT 0x03 -static pthread_mutex_t _thr_once_lock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t _thr_once_cv = PTHREAD_COND_INITIALIZER; - /* * POSIX: * The pthread_once() function is not a cancellation point. However, @@ -55,47 +52,46 @@ { pthread_once_t *once_control = arg; - _pthread_mutex_lock(&_thr_once_lock); - once_control->state = ONCE_NEVER_DONE; - _pthread_mutex_unlock(&_thr_once_lock); - _pthread_cond_broadcast(&_thr_once_cv); + if (atomic_cmpset_rel_int(&once_control->state, ONCE_IN_PROGRESS, ONCE_NEVER_DONE)) + return; + atomic_store_rel_int(&once_control->state, ONCE_NEVER_DONE); + _thr_umtx_wake(&once_control->state, INT_MAX, 0); } int _pthread_once(pthread_once_t *once_control, void (*init_routine) (void)) { struct pthread *curthread; - int wakeup = 0; + int state; - if (once_control->state == ONCE_DONE) + for (;;) { + state = once_control->state; + if (state == ONCE_DONE) + return (0); + if (state == ONCE_NEVER_DONE) { + if (atomic_cmpset_acq_int(&once_control->state, state, ONCE_IN_PROGRESS)) + break; + } else if (state == ONCE_IN_PROGRESS) { + if (atomic_cmpset_acq_int(&once_control->state, state, ONCE_WAIT)) + _thr_umtx_wait_uint(&once_control->state, ONCE_WAIT, NULL, 0); + } else if (state == ONCE_WAIT) { + _thr_umtx_wait_uint(&once_control->state, state, NULL, 0); + } else + return (EINVAL); + } + + curthread = _get_curthread(); + THR_CLEANUP_PUSH(curthread, once_cancel_handler, once_control); + init_routine(); + THR_CLEANUP_POP(curthread, 0); + if (atomic_cmpset_rel_int(&once_control->state, ONCE_IN_PROGRESS, ONCE_DONE)) return (0); - _pthread_mutex_lock(&_thr_once_lock); - while (*(volatile int *)&(once_control->state) == ONCE_IN_PROGRESS) - _pthread_cond_wait(&_thr_once_cv, &_thr_once_lock); - /* - * If previous thread was canceled, then the state still - * could be ONCE_NEVER_DONE, we need to check it again. - */ - if (*(volatile int *)&(once_control->state) == ONCE_NEVER_DONE) { - once_control->state = ONCE_IN_PROGRESS; - _pthread_mutex_unlock(&_thr_once_lock); - curthread = _get_curthread(); - THR_CLEANUP_PUSH(curthread, once_cancel_handler, once_control); - init_routine(); - THR_CLEANUP_POP(curthread, 0); - _pthread_mutex_lock(&_thr_once_lock); - once_control->state = ONCE_DONE; - wakeup = 1; - } - _pthread_mutex_unlock(&_thr_once_lock); - if (wakeup) - _pthread_cond_broadcast(&_thr_once_cv); + atomic_store_rel_int(&once_control->state, ONCE_DONE); + _thr_umtx_wake(&once_control->state, INT_MAX, 0); return (0); } void _thr_once_init() { - _thr_once_lock = PTHREAD_MUTEX_INITIALIZER; - _thr_once_cv = PTHREAD_COND_INITIALIZER; } Index: lib/libthr/thread/thr_umtx.c =================================================================== --- lib/libthr/thread/thr_umtx.c (revision 190196) +++ lib/libthr/thread/thr_umtx.c (working copy) @@ -48,26 +48,74 @@ } int -__thr_umutex_lock(struct umutex *mtx) +__thr_umutex_lock(struct umutex *mtx, uint32_t id) { - return _umtx_op(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); + uint32_t owner; + + if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { + for (;;) { + /* wait in kernel */ + _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); + + owner = mtx->m_owner; + if ((owner & ~UMUTEX_CONTESTED) == 0 && + atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) + return (0); + } + } + + return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); } int -__thr_umutex_timedlock(struct umutex *mtx, - const struct timespec *timeout) +__thr_umutex_timedlock(struct umutex *mtx, uint32_t id, + const struct timespec *ets) { - if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && - timeout->tv_nsec <= 0))) + struct timespec timo, cts; + uint32_t owner; + int ret; + + clock_gettime(CLOCK_REALTIME, &cts); + TIMESPEC_SUB(&timo, ets, &cts); + + if (timo.tv_sec < 0) return (ETIMEDOUT); - return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, - __DECONST(void *, timeout)); + for (;;) { + if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { + + /* wait in kernel */ + ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, &timo); + + /* now try to lock it */ + owner = mtx->m_owner; + if ((owner & ~UMUTEX_CONTESTED) == 0 && + atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) + return (0); + } else { + ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, &timo); + if (ret == 0) + break; + } + if (ret == ETIMEDOUT) + break; + clock_gettime(CLOCK_REALTIME, &cts); + TIMESPEC_SUB(&timo, ets, &cts); + if (timo.tv_sec < 0 || (timo.tv_sec == 0 && timo.tv_nsec == 0)) { + ret = ETIMEDOUT; + break; + } + } + return (ret); } int -__thr_umutex_unlock(struct umutex *mtx) +__thr_umutex_unlock(struct umutex *mtx, uint32_t id) { + if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { + atomic_cmpset_rel_32(&mtx->m_owner, id | UMUTEX_CONTESTED, UMUTEX_CONTESTED); + return _umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE, 0, 0, 0); + } return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0); } @@ -95,19 +143,20 @@ } int -_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout) +_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout, int shared) { if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && timeout->tv_nsec <= 0))) return (ETIMEDOUT); - return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAIT_UINT, id, 0, - __DECONST(void*, timeout)); + return _umtx_op_err(__DEVOLATILE(void *, mtx), + shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 0, + __DECONST(void*, timeout)); } int -_thr_umtx_wake(volatile long *mtx, int nr_wakeup) +_thr_umtx_wake(volatile void *mtx, int nr_wakeup, int shared) { - return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAKE, + return _umtx_op_err(__DEVOLATILE(void *, mtx), shared ? UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE, nr_wakeup, 0, 0); } @@ -123,7 +172,8 @@ { if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && timeout->tv_nsec <= 0))) { - __thr_umutex_unlock(m); + struct pthread *curthread = _get_curthread(); + _thr_umutex_unlock(m, TID(curthread)); return (ETIMEDOUT); } return _umtx_op_err(cv, UMTX_OP_CV_WAIT, Index: lib/libthr/thread/thr_fork.c =================================================================== --- lib/libthr/thread/thr_fork.c (revision 190196) +++ lib/libthr/thread/thr_fork.c (working copy) @@ -67,6 +67,7 @@ #include "un-namespace.h" #include "libc_private.h" +#include "rtld_lock.h" #include "thr_private.h" __weak_reference(_pthread_atfork, pthread_atfork); @@ -105,6 +106,7 @@ pid_t ret; int errsave; int unlock_malloc; + int rtld_locks[MAX_RTLD_LOCKS]; if (!_thr_is_inited()) return (__sys_fork()); @@ -127,6 +129,7 @@ if (_thr_isthreaded() != 0) { unlock_malloc = 1; _malloc_prefork(); + _rtld_atfork_pre(rtld_locks); } else { unlock_malloc = 0; } @@ -155,6 +158,9 @@ /* clear other threads locked us. */ _thr_umutex_init(&curthread->lock); _thr_umutex_init(&_thr_atfork_lock); + + if (unlock_malloc) + _rtld_atfork_post(rtld_locks); _thr_setthreaded(0); /* reinitialize libc spinlocks. */ @@ -167,6 +173,12 @@ /* Ready to continue, unblock signals. */ _thr_signal_unblock(curthread); + if (unlock_malloc) { + __isthreaded = 1; + _malloc_postfork(); + __isthreaded = 0; + } + /* Run down atfork child handlers. */ TAILQ_FOREACH(af, &_thr_atfork_list, qe) { if (af->child != NULL) @@ -179,8 +191,10 @@ /* Ready to continue, unblock signals. */ _thr_signal_unblock(curthread); - if (unlock_malloc) + if (unlock_malloc) { + _rtld_atfork_post(rtld_locks); _malloc_postfork(); + } /* Run down atfork parent handlers. */ TAILQ_FOREACH(af, &_thr_atfork_list, qe) { Index: lib/libthr/thread/thr_umtx.h =================================================================== --- lib/libthr/thread/thr_umtx.h (revision 190196) +++ lib/libthr/thread/thr_umtx.h (working copy) @@ -32,13 +32,12 @@ #include #include -#define DEFAULT_UMUTEX {0} +#define DEFAULT_UMUTEX {0,0, {0,0},{0,0,0,0}} - -int __thr_umutex_lock(struct umutex *mtx) __hidden; -int __thr_umutex_timedlock(struct umutex *mtx, +int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden; +int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, const struct timespec *timeout) __hidden; -int __thr_umutex_unlock(struct umutex *mtx) __hidden; +int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) __hidden; int __thr_umutex_trylock(struct umutex *mtx) __hidden; int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, uint32_t *oldceiling) __hidden; @@ -46,7 +45,9 @@ void _thr_umutex_init(struct umutex *mtx) __hidden; int _thr_umtx_wait(volatile long *mtx, long exp, const struct timespec *timeout) __hidden; -int _thr_umtx_wake(volatile long *mtx, int count) __hidden; +int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp, + const struct timespec *timeout, int shared) __hidden; +int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden; int _thr_ucond_wait(struct ucond *cv, struct umutex *m, const struct timespec *timeout, int check_unpaking) __hidden; void _thr_ucond_init(struct ucond *cv) __hidden; @@ -70,26 +71,30 @@ static inline int _thr_umutex_trylock2(struct umutex *mtx, uint32_t id) { - if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) + if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0) return (0); + if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED && + __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0)) + if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED)) + return (0); return (EBUSY); } static inline int _thr_umutex_lock(struct umutex *mtx, uint32_t id) { - if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) + if (_thr_umutex_trylock2(mtx, id) == 0) return (0); - return (__thr_umutex_lock(mtx)); + return (__thr_umutex_lock(mtx, id)); } static inline int _thr_umutex_timedlock(struct umutex *mtx, uint32_t id, const struct timespec *timeout) { - if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) + if (_thr_umutex_trylock2(mtx, id) == 0) return (0); - return (__thr_umutex_timedlock(mtx, timeout)); + return (__thr_umutex_timedlock(mtx, id, timeout)); } static inline int @@ -97,7 +102,7 @@ { if (atomic_cmpset_rel_32(&mtx->m_owner, id, UMUTEX_UNOWNED)) return (0); - return (__thr_umutex_unlock(mtx)); + return (__thr_umutex_unlock(mtx, id)); } static inline int Index: lib/libthr/thread/thr_init.c =================================================================== --- lib/libthr/thread/thr_init.c (revision 190196) +++ lib/libthr/thread/thr_init.c (working copy) @@ -75,20 +75,21 @@ struct pthread_attr _pthread_attr_default = { .sched_policy = SCHED_OTHER, - .sched_inherit = 0, + .sched_inherit = PTHREAD_INHERIT_SCHED, .prio = 0, .suspend = THR_CREATE_RUNNING, .flags = PTHREAD_SCOPE_SYSTEM, .stackaddr_attr = NULL, .stacksize_attr = THR_STACK_DEFAULT, - .guardsize_attr = 0 + .guardsize_attr = 0, + .cpusetsize = 0, + .cpuset = NULL }; struct pthread_mutex_attr _pthread_mutexattr_default = { .m_type = PTHREAD_MUTEX_DEFAULT, .m_protocol = PTHREAD_PRIO_NONE, - .m_ceiling = 0, - .m_flags = 0 + .m_ceiling = 0 }; /* Default condition variable attributes: */ @@ -158,7 +159,6 @@ STATIC_LIB_REQUIRE(_spinlock_debug); STATIC_LIB_REQUIRE(_spinunlock); STATIC_LIB_REQUIRE(_thread_init_hack); -STATIC_LIB_REQUIRE(_vfork); /* * These are needed when linking statically. All references within Index: lib/libthr/thread/thr_create.c =================================================================== --- lib/libthr/thread/thr_create.c (revision 190196) +++ lib/libthr/thread/thr_create.c (working copy) @@ -36,6 +36,7 @@ #include #include #include +#include #include "un-namespace.h" #include "thr_private.h" @@ -55,6 +56,8 @@ struct rtprio rtp; int ret = 0, locked, create_suspended; sigset_t set, oset; + cpuset_t *cpusetp = NULL; + int cpusetsize = 0; _thr_check_init(); @@ -73,8 +76,13 @@ if (attr == NULL || *attr == NULL) /* Use the default thread attributes: */ new_thread->attr = _pthread_attr_default; - else + else { new_thread->attr = *(*attr); + cpusetp = new_thread->attr.cpuset; + cpusetsize = new_thread->attr.cpusetsize; + new_thread->attr.cpuset = NULL; + new_thread->attr.cpusetsize = 0; + } if (new_thread->attr.sched_inherit == PTHREAD_INHERIT_SCHED) { /* inherit scheduling contention scope */ if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) @@ -129,7 +137,7 @@ _thr_link(curthread, new_thread); /* Return thread pointer eariler so that new thread can use it. */ (*thread) = new_thread; - if (SHOULD_REPORT_EVENT(curthread, TD_CREATE)) { + if (SHOULD_REPORT_EVENT(curthread, TD_CREATE) || cpusetp != NULL) { THR_THREAD_LOCK(curthread, new_thread); locked = 1; } else @@ -147,7 +155,7 @@ param.flags |= THR_SYSTEM_SCOPE; if (new_thread->attr.sched_inherit == PTHREAD_INHERIT_SCHED) param.rtp = NULL; - else { + else { sched_param.sched_priority = new_thread->attr.prio; _schedparam_to_rtp(new_thread->attr.sched_policy, &sched_param, &rtp); @@ -160,6 +168,7 @@ SIGDELSET(set, SIGTRAP); __sys_sigprocmask(SIG_SETMASK, &set, &oset); new_thread->sigmask = oset; + SIGDELSET(new_thread->sigmask, SIGCANCEL); } ret = thr_new(¶m, sizeof(param)); @@ -183,7 +192,7 @@ new_thread->tid = TID_TERMINATED; if (new_thread->flags & THR_FLAGS_NEED_SUSPEND) { new_thread->cycle++; - _thr_umtx_wake(&new_thread->cycle, INT_MAX); + _thr_umtx_wake(&new_thread->cycle, INT_MAX, 0); } THR_THREAD_UNLOCK(curthread, new_thread); THREAD_LIST_LOCK(curthread); @@ -191,11 +200,31 @@ new_thread->tlflags |= TLFLAGS_DETACHED; _thr_ref_delete_unlocked(curthread, new_thread); THREAD_LIST_UNLOCK(curthread); - (*thread) = 0; } else if (locked) { + if (cpusetp != NULL) { + if (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, + TID(new_thread), cpusetsize, cpusetp)) { + ret = errno; + /* kill the new thread */ + new_thread->force_exit = 1; + THR_THREAD_UNLOCK(curthread, new_thread); + goto out; + } + } + _thr_report_creation(curthread, new_thread); THR_THREAD_UNLOCK(curthread, new_thread); +out: + if (ret) { + THREAD_LIST_LOCK(curthread); + new_thread->tlflags |= TLFLAGS_DETACHED; + THR_GCLIST_ADD(new_thread); + THREAD_LIST_UNLOCK(curthread); + } } + + if (ret) + (*thread) = 0; return (ret); } @@ -231,6 +260,9 @@ THR_LOCK(curthread); THR_UNLOCK(curthread); + if (curthread->force_exit) + _pthread_exit(PTHREAD_CANCELED); + if (curthread->unblock_sigcancel) { sigset_t set1; Index: lib/libthr/thread/thr_rtld.c =================================================================== --- lib/libthr/thread/thr_rtld.c (revision 190196) +++ lib/libthr/thread/thr_rtld.c (working copy) @@ -40,8 +40,6 @@ extern int errno; #define CACHE_LINE_SIZE 64 -#define WAFLAG 0x1 -#define RC_INCR 0x2 static int _thr_rtld_clr_flag(int); static void *_thr_rtld_lock_create(void); @@ -52,43 +50,42 @@ static void _thr_rtld_wlock_acquire(void *); struct rtld_lock { - volatile int lock; - volatile int rd_waiters; - volatile int wr_waiters; - volatile long rd_cv; - volatile long wr_cv; - void *base; + struct urwlock lock; + char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)]; }; +static struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE); +static int busy_places; + static void * _thr_rtld_lock_create(void) { - void *base; - char *p; - uintptr_t r; - struct rtld_lock *l; + int locki; + struct rtld_lock *l; + static const char fail[] = "_thr_rtld_lock_create failed\n"; - THR_ASSERT(sizeof(struct rtld_lock) <= CACHE_LINE_SIZE, - "rtld_lock too large"); - base = calloc(1, CACHE_LINE_SIZE); - p = (char *)base; - if ((uintptr_t)p % CACHE_LINE_SIZE != 0) { - free(base); - base = calloc(1, 2 * CACHE_LINE_SIZE); - p = (char *)base; - if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0) - p += CACHE_LINE_SIZE - r; + for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) { + if ((busy_places & (1 << locki)) == 0) + break; } - l = (struct rtld_lock *)p; - l->base = base; + if (locki == MAX_RTLD_LOCKS) { + write(2, fail, sizeof(fail) - 1); + return (NULL); + } + busy_places |= (1 << locki); + + l = &lock_place[locki]; + l->lock.rw_flags = URWLOCK_PREFER_READER; return (l); } static void _thr_rtld_lock_destroy(void *lock) { - struct rtld_lock *l = (struct rtld_lock *)lock; - free(l->base); + int locki; + + locki = (struct rtld_lock *)lock - &lock_place[0]; + busy_places &= ~(1 << locki); } #define SAVE_ERRNO() { \ @@ -110,7 +107,6 @@ { struct pthread *curthread; struct rtld_lock *l; - long v; int errsave; curthread = _get_curthread(); @@ -118,18 +114,8 @@ l = (struct rtld_lock *)lock; THR_CRITICAL_ENTER(curthread); - atomic_add_acq_int(&l->lock, RC_INCR); - if (!(l->lock & WAFLAG)) { - RESTORE_ERRNO(); - return; - } - v = l->rd_cv; - atomic_add_int(&l->rd_waiters, 1); - while (l->lock & WAFLAG) { - _thr_umtx_wait(&l->rd_cv, v, NULL); - v = l->rd_cv; - } - atomic_add_int(&l->rd_waiters, -1); + while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0) + ; RESTORE_ERRNO(); } @@ -138,7 +124,6 @@ { struct pthread *curthread; struct rtld_lock *l; - long v; int errsave; curthread = _get_curthread(); @@ -146,19 +131,9 @@ l = (struct rtld_lock *)lock; _thr_signal_block(curthread); - for (;;) { - if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) { - RESTORE_ERRNO(); - return; - } - v = l->wr_cv; - atomic_add_int(&l->wr_waiters, 1); - while (l->lock != 0) { - _thr_umtx_wait(&l->wr_cv, v, NULL); - v = l->wr_cv; - } - atomic_add_int(&l->wr_waiters, -1); - } + while (_thr_rwlock_wrlock(&l->lock, NULL) != 0) + ; + RESTORE_ERRNO(); } static void @@ -166,29 +141,20 @@ { struct pthread *curthread; struct rtld_lock *l; + int32_t state; int errsave; curthread = _get_curthread(); SAVE_ERRNO(); l = (struct rtld_lock *)lock; - if ((l->lock & WAFLAG) == 0) { - atomic_add_rel_int(&l->lock, -RC_INCR); - if (l->lock == 0 && l->wr_waiters) { - atomic_add_long(&l->wr_cv, 1); - _thr_umtx_wake(&l->wr_cv, l->wr_waiters); + state = l->lock.rw_state; + if (_thr_rwlock_unlock(&l->lock) == 0) { + if ((state & URWLOCK_WRITE_OWNER) == 0) { + THR_CRITICAL_LEAVE(curthread); + } else { + _thr_signal_unblock(curthread); } - THR_CRITICAL_LEAVE(curthread); - } else { - atomic_add_rel_int(&l->lock, -WAFLAG); - if (l->lock == 0 && l->wr_waiters) { - atomic_add_long(&l->wr_cv, 1); - _thr_umtx_wake(&l->wr_cv, l->wr_waiters); - } else if (l->rd_waiters) { - atomic_add_long(&l->rd_cv, 1); - _thr_umtx_wake(&l->rd_cv, l->rd_waiters); - } - _thr_signal_unblock(curthread); } RESTORE_ERRNO(); } Index: lib/libthr/thread/thr_exit.c =================================================================== --- lib/libthr/thread/thr_exit.c (revision 190196) +++ lib/libthr/thread/thr_exit.c (working copy) @@ -29,11 +29,14 @@ * $FreeBSD$ */ +#include "namespace.h" #include #include #include #include +#include "un-namespace.h" +#include "libc_private.h" #include "thr_private.h" void _pthread_exit(void *status); @@ -60,22 +63,6 @@ void _thr_exit_cleanup(void) { - struct pthread *curthread = _get_curthread(); - - /* - * POSIX states that cancellation/termination of a thread should - * not release any visible resources (such as mutexes) and that - * it is the applications responsibility. Resources that are - * internal to the threads library, including file and fd locks, - * are not visible to the application and need to be released. - */ - /* Unlock all private mutexes: */ - _mutex_unlock_private(curthread); - - /* - * This still isn't quite correct because we don't account - * for held spinlocks (see libc/stdlib/malloc.c). - */ } void @@ -100,7 +87,7 @@ /* Save the return value: */ curthread->ret = status; while (curthread->cleanup != NULL) { - pthread_cleanup_pop(1); + _pthread_cleanup_pop(1); } /* Check if there is thread specific data: */ @@ -119,8 +106,18 @@ exit(0); /* Never reach! */ } + THREAD_LIST_UNLOCK(curthread); + + /* Tell malloc that the thread is exiting. */ + _malloc_thread_cleanup(); + + THREAD_LIST_LOCK(curthread); THR_LOCK(curthread); curthread->state = PS_DEAD; + if (curthread->flags & THR_FLAGS_NEED_SUSPEND) { + curthread->cycle++; + _thr_umtx_wake(&curthread->cycle, INT_MAX, 0); + } THR_UNLOCK(curthread); /* * Thread was created with initial refcount 1, we drop the @@ -130,7 +127,7 @@ if (curthread->tlflags & TLFLAGS_DETACHED) THR_GCLIST_ADD(curthread); THREAD_LIST_UNLOCK(curthread); - if (SHOULD_REPORT_EVENT(curthread, TD_DEATH)) + if (!curthread->force_exit && SHOULD_REPORT_EVENT(curthread, TD_DEATH)) _thr_report_death(curthread); /* Index: lib/libthr/thread/thr_affinity.c =================================================================== --- lib/libthr/thread/thr_affinity.c (revision 0) +++ lib/libthr/thread/thr_affinity.c (revision 0) @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2008, David Xu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#include "namespace.h" +#include +#include +#include +#include "un-namespace.h" + +#include "thr_private.h" + +__weak_reference(_pthread_getaffinity_np, pthread_getaffinity_np); +__weak_reference(_pthread_setaffinity_np, pthread_setaffinity_np); + +int +_pthread_setaffinity_np(pthread_t td, size_t cpusetsize, const cpuset_t *cpusetp) +{ + struct pthread *curthread = _get_curthread(); + lwpid_t tid; + int error; + + if (td == curthread) { + error = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, + -1, cpusetsize, cpusetp); + if (error == -1) + error = errno; + } else { + THR_THREAD_LOCK(curthread, td); + if (td->state == PS_DEAD) { + THR_THREAD_UNLOCK(curthread, td); + return (EINVAL); + } + tid = TID(td); + error = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, tid, + cpusetsize, cpusetp); + if (error == -1) + error = errno; + THR_THREAD_UNLOCK(curthread, td); + } + return (error); +} + +int +_pthread_getaffinity_np(pthread_t td, size_t cpusetsize, cpuset_t *cpusetp) +{ + struct pthread *curthread = _get_curthread(); + lwpid_t tid; + int error; + + tid = TID(td); + error = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, + (td == curthread) ? -1 : tid, cpusetsize, cpusetp); + if (error == -1) + error = errno; + return (error); +} Index: lib/libthr/arch/i386/Makefile.inc =================================================================== --- lib/libthr/arch/i386/Makefile.inc (revision 190196) +++ lib/libthr/arch/i386/Makefile.inc (working copy) @@ -2,4 +2,4 @@ .PATH: ${.CURDIR}/arch/${MACHINE_ARCH}/${MACHINE_ARCH} -SRCS+= pthread_md.c _umtx_op_err.S +SRCS+= pthread_md.c _umtx_op_err.S Index: lib/libthr/arch/amd64/include/pthread_md.h =================================================================== --- lib/libthr/arch/amd64/include/pthread_md.h (revision 190196) +++ lib/libthr/arch/amd64/include/pthread_md.h (working copy) @@ -98,6 +98,6 @@ return (TCB_GET64(tcb_thread)); } -#define HAS__UMTX_OP_ERR 1 +#define HAS__UMTX_OP_ERR 1 #endif Index: lib/libthr/Makefile =================================================================== --- lib/libthr/Makefile (revision 190196) +++ lib/libthr/Makefile (working copy) @@ -8,16 +8,14 @@ # (for system call stubs) to CFLAGS below. -DSYSLIBC_SCCS affects just the # system call stubs. +SHLIBDIR?= /lib + .include +MK_SSP= no -.if (${DEFAULT_THREAD_LIB} == "libthr" || ${MK_LIBKSE} == "no") && \ - ${SHLIBDIR} == "/usr/lib" -SHLIBDIR= /lib -.endif - LIB=thr SHLIB_MAJOR= 3 -WARNS?= 2 +WARNS?= 3 CFLAGS+=-DPTHREAD_KERNEL CFLAGS+=-I${.CURDIR}/../libc/include -I${.CURDIR}/thread \ -I${.CURDIR}/../../include @@ -28,10 +26,9 @@ CFLAGS+=-I${.CURDIR}/../libthread_db CFLAGS+=-Winline -# CFLAGS+=-DSYSTEM_SCOPE_ONLY +VERSION_DEF=${.CURDIR}/../libc/Versions.def +SYMBOL_MAPS=${.CURDIR}/pthread.map -VERSION_MAP=${.CURDIR}/pthread.map - MAN= libthr.3 # enable extra internal consistancy checks @@ -44,14 +41,18 @@ .include "${.CURDIR}/sys/Makefile.inc" .include "${.CURDIR}/thread/Makefile.inc" -.if ${DEFAULT_THREAD_LIB} == "libthr" || ${MK_LIBKSE} == "no" +.if ${MK_INSTALLLIB} != "no" SYMLINKS+=lib${LIB}.a ${LIBDIR}/libpthread.a +.endif .if !defined(NO_PIC) SYMLINKS+=lib${LIB}.so ${LIBDIR}/libpthread.so .endif .if ${MK_PROFILE} != "no" SYMLINKS+=lib${LIB}_p.a ${LIBDIR}/libpthread_p.a .endif + +.if !defined(WITHOUT_SYSCALL_COMPAT) +CFLAGS+=-DSYSCALL_COMPAT .endif .include Index: libexec/rtld-elf/Symbol.map =================================================================== --- libexec/rtld-elf/Symbol.map (revision 190196) +++ libexec/rtld-elf/Symbol.map (working copy) @@ -21,4 +21,6 @@ _rtld_thread_init; _rtld_allocate_tls; _rtld_free_tls; + _rtld_atfork_pre; + _rtld_atfork_post; }; Index: libexec/rtld-elf/rtld.c =================================================================== --- libexec/rtld-elf/rtld.c (revision 190196) +++ libexec/rtld-elf/rtld.c (working copy) @@ -206,6 +206,8 @@ (func_ptr_type) &_rtld_allocate_tls, (func_ptr_type) &_rtld_free_tls, (func_ptr_type) &dl_iterate_phdr, + (func_ptr_type) &_rtld_atfork_pre, + (func_ptr_type) &_rtld_atfork_post, NULL }; Index: libexec/rtld-elf/rtld_lock.c =================================================================== --- libexec/rtld-elf/rtld_lock.c (revision 190196) +++ libexec/rtld-elf/rtld_lock.c (working copy) @@ -316,3 +316,19 @@ thread_mask_set(flags); dbg("_rtld_thread_init: done"); } + +void +_rtld_atfork_pre(int *locks) +{ + + locks[2] = wlock_acquire(rtld_phdr_lock); + locks[0] = rlock_acquire(rtld_bind_lock); +} + +void +_rtld_atfork_post(int *locks) +{ + + rlock_release(rtld_bind_lock, locks[0]); + wlock_release(rtld_phdr_lock, locks[2]); +} Index: libexec/rtld-elf/rtld_lock.h =================================================================== --- libexec/rtld-elf/rtld_lock.h (revision 190196) +++ libexec/rtld-elf/rtld_lock.h (working copy) @@ -29,6 +29,7 @@ #define _RTLD_LOCK_H_ #define RTLI_VERSION 0x01 +#define MAX_RTLD_LOCKS 8 struct RtldLockInfo { @@ -44,6 +45,8 @@ }; extern void _rtld_thread_init(struct RtldLockInfo *); +extern void _rtld_atfork_pre(int *); +extern void _rtld_atfork_post(int *); #ifdef IN_RTLD Index: sys/kern/kern_thr.c =================================================================== --- sys/kern/kern_thr.c (revision 190196) +++ sys/kern/kern_thr.c (working copy) @@ -276,7 +276,7 @@ /* Signal userland that it can free the stack. */ if ((void *)uap->state != NULL) { suword_lwpid(uap->state, 1); - kern_umtx_wake(td, uap->state, INT_MAX); + kern_umtx_wake(td, uap->state, INT_MAX, 0); } PROC_LOCK(p); Index: sys/kern/kern_umtx.c =================================================================== --- sys/kern/kern_umtx.c (revision 190196) +++ sys/kern/kern_umtx.c (working copy) @@ -58,14 +58,17 @@ #include #endif -#define TYPE_SIMPLE_LOCK 0 -#define TYPE_SIMPLE_WAIT 1 -#define TYPE_NORMAL_UMUTEX 2 -#define TYPE_PI_UMUTEX 3 -#define TYPE_PP_UMUTEX 4 -#define TYPE_CV 5 +#define TYPE_SIMPLE_WAIT 0 +#define TYPE_CV 1 +#define TYPE_SIMPLE_LOCK 2 +#define TYPE_NORMAL_UMUTEX 3 +#define TYPE_PI_UMUTEX 4 +#define TYPE_PP_UMUTEX 5 #define TYPE_RWLOCK 6 +#define _UMUTEX_TRY 1 +#define _UMUTEX_WAIT 2 + /* Key to represent a unique userland synchronous object */ struct umtx_key { int hash; @@ -163,6 +166,7 @@ }; #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) +#define UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy")) /* * Don't propagate time-sharing priority, there is a security reason, @@ -191,7 +195,7 @@ #define BUSY_SPINS 200 static uma_zone_t umtx_pi_zone; -static struct umtxq_chain umtxq_chains[UMTX_CHAINS]; +static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); static int umtx_pi_allocated; @@ -232,18 +236,20 @@ static void umtxq_sysinit(void *arg __unused) { - int i; + int i, j; umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); - for (i = 0; i < UMTX_CHAINS; ++i) { - mtx_init(&umtxq_chains[i].uc_lock, "umtxql", NULL, - MTX_DEF | MTX_DUPOK); - TAILQ_INIT(&umtxq_chains[i].uc_queue[0]); - TAILQ_INIT(&umtxq_chains[i].uc_queue[1]); - TAILQ_INIT(&umtxq_chains[i].uc_pi_list); - umtxq_chains[i].uc_busy = 0; - umtxq_chains[i].uc_waiters = 0; + for (i = 0; i < 2; ++i) { + for (j = 0; j < UMTX_CHAINS; ++j) { + mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL, + MTX_DEF | MTX_DUPOK); + TAILQ_INIT(&umtxq_chains[i][j].uc_queue[0]); + TAILQ_INIT(&umtxq_chains[i][j].uc_queue[1]); + TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list); + umtxq_chains[i][j].uc_busy = 0; + umtxq_chains[i][j].uc_waiters = 0; + } } mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN); EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, @@ -285,7 +291,9 @@ static inline struct umtxq_chain * umtxq_getchain(struct umtx_key *key) { - return (&umtxq_chains[key->hash]); + if (key->type <= TYPE_CV) + return (&umtxq_chains[1][key->hash]); + return (&umtxq_chains[0][key->hash]); } /* @@ -324,14 +332,18 @@ uc = umtxq_getchain(key); mtx_assert(&uc->uc_lock, MA_OWNED); if (uc->uc_busy) { - int count = BUSY_SPINS; - if (count > 0) { - umtxq_unlock(key); - while (uc->uc_busy && --count > 0) - cpu_spinwait(); - umtxq_lock(key); +#ifdef SMP + if (smp_cpus > 1) { + int count = BUSY_SPINS; + if (count > 0) { + umtxq_unlock(key); + while (uc->uc_busy && --count > 0) + cpu_spinwait(); + umtxq_lock(key); + } } - while (uc->uc_busy != 0) { +#endif + while (uc->uc_busy) { uc->uc_waiters++; msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); uc->uc_waiters--; @@ -944,7 +956,7 @@ */ static int do_wait(struct thread *td, void *addr, u_long id, - struct timespec *timeout, int compat32) + struct timespec *timeout, int compat32, int is_private) { struct umtx_q *uq; struct timespec ts, ts2, ts3; @@ -953,8 +965,8 @@ int error = 0; uq = td->td_umtxq; - if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, AUTO_SHARE, - &uq->uq_key)) != 0) + if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, + is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) return (error); umtxq_lock(&uq->uq_key); @@ -1009,13 +1021,13 @@ * Wake up threads sleeping on the specified address. */ int -kern_umtx_wake(struct thread *td, void *uaddr, int n_wake) +kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) { struct umtx_key key; int ret; - if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, AUTO_SHARE, - &key)) != 0) + if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, + is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) return (ret); umtxq_lock(&key); ret = umtxq_signal(&key, n_wake); @@ -1029,7 +1041,7 @@ */ static int _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo, - int try) + int mode) { struct umtx_q *uq; uint32_t owner, old, id; @@ -1043,40 +1055,46 @@ * can fault on any access. */ for (;;) { - /* - * Try the uncontested case. This should be done in userland. - */ - owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); + owner = fuword32(__DEVOLATILE(void *, &m->m_owner)); + if (mode == _UMUTEX_WAIT) { + if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED) + return (0); + } else { + /* + * Try the uncontested case. This should be done in userland. + */ + owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); - /* The acquire succeeded. */ - if (owner == UMUTEX_UNOWNED) - return (0); - - /* The address was invalid. */ - if (owner == -1) - return (EFAULT); - - /* If no one owns it but it is contested try to acquire it. */ - if (owner == UMUTEX_CONTESTED) { - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); - - if (owner == UMUTEX_CONTESTED) + /* The acquire succeeded. */ + if (owner == UMUTEX_UNOWNED) return (0); /* The address was invalid. */ if (owner == -1) return (EFAULT); - /* If this failed the lock has changed, restart. */ - continue; + /* If no one owns it but it is contested try to acquire it. */ + if (owner == UMUTEX_CONTESTED) { + owner = casuword32(&m->m_owner, + UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + + if (owner == UMUTEX_CONTESTED) + return (0); + + /* The address was invalid. */ + if (owner == -1) + return (EFAULT); + + /* If this failed the lock has changed, restart. */ + continue; + } } if ((flags & UMUTEX_ERROR_CHECK) != 0 && (owner & ~UMUTEX_CONTESTED) == id) return (EDEADLK); - if (try != 0) + if (mode == _UMUTEX_TRY) return (EBUSY); /* @@ -1093,7 +1111,6 @@ umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); - umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); /* @@ -1108,6 +1125,7 @@ if (old == -1) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); + umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (EFAULT); @@ -1119,6 +1137,7 @@ * unlocking the umtx. */ umtxq_lock(&uq->uq_key); + umtxq_unbusy(&uq->uq_key); if (old == owner) error = umtxq_sleep(uq, "umtxn", timo); umtxq_remove(uq); @@ -1154,7 +1173,6 @@ if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); - /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); if (old == -1) @@ -1193,6 +1211,50 @@ return (0); } +/* + * Check if the mutex is available and wake up a waiter, + * only for simple mutex. + */ +static int +do_wake_umutex(struct thread *td, struct umutex *m) +{ + struct umtx_key key; + uint32_t owner; + uint32_t flags; + int error; + int count; + + owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); + if (owner == -1) + return (EFAULT); + + if ((owner & ~UMUTEX_CONTESTED) != 0) + return (0); + + flags = fuword32(&m->m_flags); + + /* We should only ever be in here for contested locks */ + if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), + &key)) != 0) + return (error); + + umtxq_lock(&key); + umtxq_busy(&key); + count = umtxq_count(&key); + umtxq_unlock(&key); + + if (count <= 1) + owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED); + + umtxq_lock(&key); + if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) + umtxq_signal(&key, 1); + umtxq_unbusy(&key); + umtxq_unlock(&key); + umtx_key_release(&key); + return (0); +} + static inline struct umtx_pi * umtx_pi_alloc(int flags) { @@ -1331,7 +1393,8 @@ oldpri = pi->pi_owner->td_user_pri; sched_unlend_user_prio(pi->pi_owner, pri); thread_unlock(pi->pi_owner); - umtx_pi_adjust_locked(pi->pi_owner, oldpri); + if (uq_owner->uq_pi_blocked != NULL) + umtx_pi_adjust_locked(pi->pi_owner, oldpri); pi = uq_owner->uq_pi_blocked; } } @@ -1452,7 +1515,9 @@ KASSERT(td == curthread, ("inconsistent uq_thread")); uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); + UMTXQ_BUSY_ASSERT(uc); umtxq_insert(uq); + mtx_lock_spin(&umtx_lock); if (pi->pi_owner == NULL) { /* XXX * Current, We only support process private PI-mutex, @@ -1463,6 +1528,7 @@ * For process private PI-mutex, we can find owner * thread and boost its priority safely. */ + mtx_unlock_spin(&umtx_lock); PROC_LOCK(curproc); td1 = thread_find(curproc, owner); mtx_lock_spin(&umtx_lock); @@ -1471,8 +1537,6 @@ umtx_pi_setowner(pi, td1); } PROC_UNLOCK(curproc); - } else { - mtx_lock_spin(&umtx_lock); } TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { @@ -1490,26 +1554,18 @@ thread_lock(td); td->td_flags |= TDF_UPIBLOCKED; thread_unlock(td); - mtx_unlock_spin(&umtx_lock); - umtxq_unlock(&uq->uq_key); - - mtx_lock_spin(&umtx_lock); umtx_propagate_priority(td); mtx_unlock_spin(&umtx_lock); + umtxq_unbusy(&uq->uq_key); - umtxq_lock(&uq->uq_key); if (uq->uq_flags & UQF_UMTXQ) { error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); if (error == EWOULDBLOCK) error = ETIMEDOUT; if (uq->uq_flags & UQF_UMTXQ) { - umtxq_busy(&uq->uq_key); umtxq_remove(uq); - umtxq_unbusy(&uq->uq_key); } } - umtxq_unlock(&uq->uq_key); - mtx_lock_spin(&umtx_lock); uq->uq_pi_blocked = NULL; thread_lock(td); @@ -1518,9 +1574,8 @@ TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); umtx_unpropagate_priority(pi); mtx_unlock_spin(&umtx_lock); + umtxq_unlock(&uq->uq_key); - umtxq_lock(&uq->uq_key); - return (error); } @@ -1545,7 +1600,6 @@ umtx_pi_unref(struct umtx_pi *pi) { struct umtxq_chain *uc; - int free = 0; uc = umtxq_getchain(&pi->pi_key); UMTXQ_LOCKED_ASSERT(uc); @@ -1561,10 +1615,8 @@ ("blocked queue not empty")); mtx_unlock_spin(&umtx_lock); TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); - free = 1; + umtx_pi_free(pi); } - if (free) - umtx_pi_free(pi); } /* @@ -1625,7 +1677,6 @@ if (new_pi == NULL) { umtxq_unlock(&uq->uq_key); new_pi = umtx_pi_alloc(M_WAITOK); - new_pi->pi_key = uq->uq_key; umtxq_lock(&uq->uq_key); pi = umtx_pi_lookup(&uq->uq_key); if (pi != NULL) { @@ -1671,7 +1722,9 @@ if (owner == UMUTEX_CONTESTED) { umtxq_lock(&uq->uq_key); + umtxq_busy(&uq->uq_key); error = umtx_pi_claim(pi, td); + umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); break; } @@ -1726,7 +1779,6 @@ } umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread @@ -1735,7 +1787,10 @@ if (old == owner) error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, "umtxpi", timo); - umtxq_unlock(&uq->uq_key); + else { + umtxq_unbusy(&uq->uq_key); + umtxq_unlock(&uq->uq_key); + } } umtxq_lock(&uq->uq_key); @@ -1790,18 +1845,26 @@ umtxq_busy(&key); count = umtxq_count_pi(&key, &uq_first); if (uq_first != NULL) { + mtx_lock_spin(&umtx_lock); pi = uq_first->uq_pi_blocked; + KASSERT(pi != NULL, ("pi == NULL?")); if (pi->pi_owner != curthread) { + mtx_unlock_spin(&umtx_lock); umtxq_unbusy(&key); umtxq_unlock(&key); + umtx_key_release(&key); /* userland messed the mutex */ return (EPERM); } uq_me = curthread->td_umtxq; - mtx_lock_spin(&umtx_lock); pi->pi_owner = NULL; TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link); + /* get highest priority thread which is still sleeping. */ uq_first = TAILQ_FIRST(&pi->pi_blocked); + while (uq_first != NULL && + (uq_first->uq_flags & UQF_UMTXQ) == 0) { + uq_first = TAILQ_NEXT(uq_first, uq_lockq); + } pri = PRI_MAX; TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); @@ -1814,6 +1877,8 @@ sched_unlend_user_prio(curthread, pri); thread_unlock(curthread); mtx_unlock_spin(&umtx_lock); + if (uq_first) + umtxq_signal_thread(uq_first); } umtxq_unlock(&key); @@ -1826,8 +1891,6 @@ count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); umtxq_lock(&key); - if (uq_first != NULL) - umtxq_signal_thread(uq_first); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); @@ -2136,15 +2199,15 @@ static int _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo, - int try) + int mode) { switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: - return (_do_lock_normal(td, m, flags, timo, try)); + return (_do_lock_normal(td, m, flags, timo, mode)); case UMUTEX_PRIO_INHERIT: - return (_do_lock_pi(td, m, flags, timo, try)); + return (_do_lock_pi(td, m, flags, timo, mode)); case UMUTEX_PRIO_PROTECT: - return (_do_lock_pp(td, m, flags, timo, try)); + return (_do_lock_pp(td, m, flags, timo, mode)); } return (EINVAL); } @@ -2154,7 +2217,7 @@ */ static int do_lock_umutex(struct thread *td, struct umutex *m, - struct timespec *timeout, int try) + struct timespec *timeout, int mode) { struct timespec ts, ts2, ts3; struct timeval tv; @@ -2166,16 +2229,16 @@ return (EFAULT); if (timeout == NULL) { - error = _do_lock_umutex(td, m, flags, 0, try); + error = _do_lock_umutex(td, m, flags, 0, mode); /* Mutex locking is restarted if it is interrupted. */ - if (error == EINTR) + if (error == EINTR && mode != _UMUTEX_WAIT) error = ERESTART; } else { getnanouptime(&ts); timespecadd(&ts, timeout); TIMESPEC_TO_TIMEVAL(&tv, timeout); for (;;) { - error = _do_lock_umutex(td, m, flags, tvtohz(&tv), try); + error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode); if (error != ETIMEDOUT) break; getnanouptime(&ts2); @@ -2604,7 +2667,7 @@ } static int -do_rwlock_unlock(struct thread *td, struct urwlock *rwlock) +do_rw_unlock(struct thread *td, struct urwlock *rwlock) { struct umtx_q *uq; uint32_t flags; @@ -2741,7 +2804,7 @@ return (EINVAL); ts = &timeout; } - return do_wait(td, uap->obj, uap->val, ts, 0); + return do_wait(td, uap->obj, uap->val, ts, 0, 0); } static int @@ -2761,16 +2824,42 @@ return (EINVAL); ts = &timeout; } - return do_wait(td, uap->obj, uap->val, ts, 1); + return do_wait(td, uap->obj, uap->val, ts, 1, 0); } static int +__umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) + return (EINVAL); + ts = &timeout; + } + return do_wait(td, uap->obj, uap->val, ts, 1, 1); +} + +static int __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) { - return (kern_umtx_wake(td, uap->obj, uap->val)); + return (kern_umtx_wake(td, uap->obj, uap->val, 0)); } static int +__umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) +{ + return (kern_umtx_wake(td, uap->obj, uap->val, 1)); +} + +static int __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) { struct timespec *ts, timeout; @@ -2796,10 +2885,39 @@ static int __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) { - return do_lock_umutex(td, uap->obj, NULL, 1); + return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY); } static int +__umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + /* Allow a null timespec (wait forever). */ + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin(uap->uaddr2, &timeout, + sizeof(timeout)); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) { + return (EINVAL); + } + ts = &timeout; + } + return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT); +} + +static int +__umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) +{ + return do_wake_umutex(td, uap->obj); +} + +static int __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) { return do_unlock_umutex(td, uap->obj); @@ -2896,7 +3014,7 @@ static int __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) { - return do_rwlock_unlock(td, uap->obj); + return do_rw_unlock(td, uap->obj); } typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); @@ -2916,7 +3034,11 @@ __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */ __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */ __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */ - __umtx_op_rw_unlock /* UMTX_OP_RW_UNLOCK */ + __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ + __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */ + __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ + __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */ + __umtx_op_wake_umutex /* UMTX_OP_UMUTEX_WAKE */ }; int @@ -3006,7 +3128,7 @@ return (EINVAL); ts = &timeout; } - return do_wait(td, uap->obj, uap->val, ts, 1); + return do_wait(td, uap->obj, uap->val, ts, 1, 0); } static int @@ -3031,6 +3153,27 @@ } static int +__umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + /* Allow a null timespec (wait forever). */ + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin_timeout32(uap->uaddr2, &timeout); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) + return (EINVAL); + ts = &timeout; + } + return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT); +} + +static int __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) { struct timespec *ts, timeout; @@ -3097,6 +3240,26 @@ return (error); } +static int +__umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin_timeout32(uap->uaddr2, &timeout); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) + return (EINVAL); + ts = &timeout; + } + return do_wait(td, uap->obj, uap->val, ts, 1, 1); +} + static _umtx_op_func op_table_compat32[] = { __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */ __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */ @@ -3112,7 +3275,11 @@ __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */ __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */ __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */ - __umtx_op_rw_unlock /* UMTX_OP_RW_UNLOCK */ + __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ + __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */ + __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ + __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */ + __umtx_op_wake_umutex /* UMTX_OP_UMUTEX_WAKE */ }; int Index: sys/sys/umtx.h =================================================================== --- sys/sys/umtx.h (revision 190196) +++ sys/sys/umtx.h (working copy) @@ -99,7 +99,11 @@ #define UMTX_OP_RW_RDLOCK 12 #define UMTX_OP_RW_WRLOCK 13 #define UMTX_OP_RW_UNLOCK 14 -#define UMTX_OP_MAX 15 +#define UMTX_OP_WAIT_UINT_PRIVATE 15 +#define UMTX_OP_WAKE_PRIVATE 16 +#define UMTX_OP_MUTEX_WAIT 17 +#define UMTX_OP_MUTEX_WAKE 18 +#define UMTX_OP_MAX 19 /* flags for UMTX_OP_CV_WAIT */ #define UMTX_CHECK_UNPARKING 0x01 @@ -190,11 +194,11 @@ struct umtx_q *umtxq_alloc(void); void umtxq_free(struct umtx_q *); -int kern_umtx_wake(struct thread *td, void *uaddr, int n_wake); -void umtx_pi_adjust(struct thread *td, u_char oldpri); -void umtx_thread_init(struct thread *td); -void umtx_thread_fini(struct thread *td); -void umtx_thread_alloc(struct thread *td); -void umtx_thread_exit(struct thread *td); +int kern_umtx_wake(struct thread *, void *, int, int); +void umtx_pi_adjust(struct thread *, u_char); +void umtx_thread_init(struct thread *); +void umtx_thread_fini(struct thread *); +void umtx_thread_alloc(struct thread *); +void umtx_thread_exit(struct thread *); #endif /* !_KERNEL */ #endif /* !_SYS_UMTX_H_ */