diff -ru /cvs/sys_old/i386/include/mutex.h /usr/src/sys/i386/include/mutex.h --- /cvs/sys_old/i386/include/mutex.h Mon Jan 22 15:12:16 2001 +++ /usr/src/sys/i386/include/mutex.h Mon Jan 22 20:33:28 2001 @@ -267,7 +267,7 @@ pushl $0 ; /* dummy __FILE__ */ \ pushl $type ; \ pushl $lck ; \ - call _mtx_enter ; \ + call _mtx_lock ; \ addl $16,%esp #define MTX_EXIT(lck, type) \ @@ -275,7 +275,7 @@ pushl $0 ; /* dummy __FILE__ */ \ pushl $type ; \ pushl $lck ; \ - call _mtx_exit ; \ + call _mtx_unlock ; \ addl $16,%esp #endif /* !LOCORE */ diff -ru /cvs/sys_old/kern/kern_mutex.c /usr/src/sys/kern/kern_mutex.c --- /cvs/sys_old/kern/kern_mutex.c Mon Jan 22 15:12:24 2001 +++ /usr/src/sys/kern/kern_mutex.c Mon Jan 22 19:47:50 2001 @@ -53,10 +53,7 @@ #include "opt_ddb.h" #include "opt_witness.h" -/* - * Cause non-inlined mtx_*() to be compiled. - * Must be defined early because other system headers may include mutex.h. - */ +/* XXX: HACK for now (for STR crap) */ #define _KERN_MUTEX_C_ #include @@ -104,10 +101,10 @@ #endif /* WITNESS */ /* - * Assembly macros - *------------------------------------------------------------------------------ + * Assembly macro(s) + ********** + * REVISIT */ - #define _V(x) __STRING(x) /* @@ -132,86 +129,6 @@ atomic_store_rel_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED) #endif -#ifndef _getlock_sleep -/* Get a sleep lock, deal with recursion inline. */ -#define _getlock_sleep(mp, tid, type) do { \ - if (!_obtain_lock(mp, tid)) { \ - if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\ - mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \ - else { \ - atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED); \ - (mp)->mtx_recurse++; \ - } \ - } \ -} while (0) -#endif - -#ifndef _getlock_spin_block -/* Get a spin lock, handle recursion inline (as the less common case) */ -#define _getlock_spin_block(mp, tid, type) do { \ - u_int _mtx_intr = save_intr(); \ - disable_intr(); \ - if (!_obtain_lock(mp, tid)) \ - mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_intr); \ - else \ - (mp)->mtx_saveintr = _mtx_intr; \ -} while (0) -#endif - -#ifndef _getlock_norecurse -/* - * Get a lock without any recursion handling. Calls the hard enter function if - * we can't get it inline. - */ -#define _getlock_norecurse(mp, tid, type) do { \ - if (!_obtain_lock(mp, tid)) \ - mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0); \ -} while (0) -#endif - -#ifndef _exitlock_norecurse -/* - * Release a sleep lock assuming we haven't recursed on it, recursion is handled - * in the hard function. - */ -#define _exitlock_norecurse(mp, tid, type) do { \ - if (!_release_lock(mp, tid)) \ - mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ -} while (0) -#endif - -#ifndef _exitlock -/* - * Release a sleep lock when its likely we recursed (the code to - * deal with simple recursion is inline). - */ -#define _exitlock(mp, tid, type) do { \ - if (!_release_lock(mp, tid)) { \ - if ((mp)->mtx_lock & MTX_RECURSED) { \ - if (--((mp)->mtx_recurse) == 0) \ - atomic_clear_ptr(&(mp)->mtx_lock, \ - MTX_RECURSED); \ - } else { \ - mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ - } \ - } \ -} while (0) -#endif - -#ifndef _exitlock_spin -/* Release a spin lock (with possible recursion). */ -#define _exitlock_spin(mp) do { \ - if (!mtx_recursed((mp))) { \ - int _mtx_intr = (mp)->mtx_saveintr; \ - \ - _release_lock_quick(mp); \ - restore_intr(_mtx_intr); \ - } else { \ - (mp)->mtx_recurse--; \ - } \ -} while (0) -#endif - #ifdef WITNESS static void witness_init(struct mtx *, int flag); static void witness_destroy(struct mtx *); @@ -243,16 +160,13 @@ #define witness_try_enter(m, t, f, l) #endif /* WITNESS */ +/************** +***** UGLY GLOBALS; revisit +**/ static int mtx_cur_cnt; static int mtx_max_cnt; static void propagate_priority(struct proc *); -static void mtx_enter_hard(struct mtx *, int type, int saveintr); -static void mtx_exit_hard(struct mtx *, int type); - -#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) -#define mtx_owner(m) (mtx_unowned(m) ? NULL \ - : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) #define RETIP(x) *(((uintptr_t *)(&x)) - 1) #define SET_PRIO(p, pri) (p)->p_priority = (pri) @@ -315,7 +229,7 @@ * quit. */ if (p->p_stat == SRUN) { - printf("XXX: moving process %d(%s) to a new run queue\n", + printf("XXX: moving proc %d(%s) to a new run queue\n", p->p_pid, p->p_comm); MPASS(p->p_blocked == NULL); remrunqueue(p); @@ -350,7 +264,7 @@ p1 = TAILQ_PREV(p, rq, p_procq); if (p1->p_priority <= pri) { printf( - "XXX: previous process %d(%s) has higher priority\n", + "XXX: previous proc %d(%s) has higher priority\n", p->p_pid, p->p_comm); continue; } @@ -377,161 +291,61 @@ } /* - * Get lock 'm', the macro handles the easy (and most common cases) and leaves - * the slow stuff to the mtx_enter_hard() function. - * - * Note: since type is usually a constant much of this code is optimized out. - */ -void -_mtx_enter(struct mtx *mtxp, int type, const char *file, int line) -{ - struct mtx *mpp = mtxp; - - /* bits only valid on mtx_exit() */ - MPASS4(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0, - STR_mtx_bad_type, file, line); - - if ((type) & MTX_SPIN) { - /* - * Easy cases of spin locks: - * - * 1) We already own the lock and will simply recurse on it (if - * RLIKELY) - * - * 2) The lock is free, we just get it - */ - if ((type) & MTX_RLIKELY) { - /* - * Check for recursion, if we already have this - * lock we just bump the recursion count. - */ - if (mpp->mtx_lock == (uintptr_t)CURTHD) { - mpp->mtx_recurse++; - goto done; - } - } - - if (((type) & MTX_TOPHALF) == 0) { - /* - * If an interrupt thread uses this we must block - * interrupts here. - */ - if ((type) & MTX_FIRST) { - ASS_IEN; - disable_intr(); - _getlock_norecurse(mpp, CURTHD, - (type) & MTX_HARDOPTS); - } else { - _getlock_spin_block(mpp, CURTHD, - (type) & MTX_HARDOPTS); - } - } else - _getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS); - } else { - /* Sleep locks */ - if ((type) & MTX_RLIKELY) - _getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS); - else - _getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS); - } -done: - WITNESS_ENTER(mpp, type, file, line); - if (((type) & MTX_QUIET) == 0) - CTR5(KTR_LOCK, STR_mtx_enter_fmt, - mpp->mtx_description, mpp, mpp->mtx_recurse, file, line); - -} - -/* - * Attempt to get MTX_DEF lock, return non-zero if lock acquired. - * - * XXX DOES NOT HANDLE RECURSION + * The important part of mtx_try_lock() + * Try acquiring lock `mtxp.' We do NOT handle recursion here; we assume that + * if we're called, it's because we know we don't already own this lock. */ int -_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line) +_mtx_try_lock(struct mtx *mtxp, int opts, const char *file, int line) { - struct mtx *const mpp = mtxp; int rval; - rval = _obtain_lock(mpp, CURTHD); + rval = _obtain_lock(mtxp, CURTHD); + #ifdef WITNESS - if (rval && mpp->mtx_witness != NULL) { - MPASS(mpp->mtx_recurse == 0); - witness_try_enter(mpp, type, file, line); + if (rval && mtxp->mtx_witness != NULL) { + MPASS(mtxp->mtx_recurse == 0); + witness_try_enter(mtxp, opts, file, line); } #endif /* WITNESS */ - if (((type) & MTX_QUIET) == 0) + + if (((opts) & MTX_QUIET) == 0) CTR5(KTR_LOCK, STR_mtx_try_enter_fmt, - mpp->mtx_description, mpp, rval, file, line); + mtxp->mtx_description, mtxp, rval, file, line); return rval; } /* - * Release lock m. + * The important part of mtx_lock(). Here we actually acquire the lock `m' + * regardless of whether it is a spin or sleep lock. We take care to handle + * recursion. */ void -_mtx_exit(struct mtx *mtxp, int type, const char *file, int line) -{ - struct mtx *const mpp = mtxp; - - MPASS4(mtx_owned(mpp), STR_mtx_owned, file, line); - WITNESS_EXIT(mpp, type, file, line); - if (((type) & MTX_QUIET) == 0) - CTR5(KTR_LOCK, STR_mtx_exit_fmt, - mpp->mtx_description, mpp, mpp->mtx_recurse, file, line); - if ((type) & MTX_SPIN) { - if ((type) & MTX_NORECURSE) { - int mtx_intr = mpp->mtx_saveintr; - - MPASS4(mpp->mtx_recurse == 0, STR_mtx_recurse, - file, line); - _release_lock_quick(mpp); - if (((type) & MTX_TOPHALF) == 0) { - if ((type) & MTX_FIRST) { - ASS_IDIS; - enable_intr(); - } else - restore_intr(mtx_intr); - } - } else { - if (((type & MTX_TOPHALF) == 0) && - (type & MTX_FIRST)) { - ASS_IDIS; - ASS_SIEN(mpp); - } - _exitlock_spin(mpp); - } - } else { - /* Handle sleep locks */ - if ((type) & MTX_RLIKELY) - _exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS); - else { - _exitlock_norecurse(mpp, CURTHD, - (type) & MTX_HARDOPTS); - } - } -} - -void -mtx_enter_hard(struct mtx *m, int type, int saveintr) +_mtx_lock(struct mtx *m, int opts, const char *file, int line) { struct proc *p = CURPROC; - KASSERT(p != NULL, ("curproc is NULL in mutex")); + KASSERT(p != NULL, ("curproc is NULL in _mtx_lock")); + + switch (m->mtx_flags & MTX_TYPES) { - switch (type) { case MTX_DEF: + { + if (_obtain_lock(m, p)) + goto got_lock; + if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { m->mtx_recurse++; atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); - if ((type & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "mtx_enter: 0x%p recurse", m); - return; + if ((opts & MTX_QUIET) == 0) + CTR1(KTR_LOCK, "mtx_lock: 0x%p recurse", m); + goto got_lock; } - if ((type & MTX_QUIET) == 0) + + if ((opts & MTX_QUIET) == 0) CTR3(KTR_LOCK, - "mtx_enter: 0x%p contested (lock=%p) [0x%p]", + "mtx_lock: 0x%p contested (lock=%p) [0x%p]", m, (void *)m->mtx_lock, (void *)RETIP(m)); /* @@ -549,30 +363,34 @@ uintptr_t v; struct proc *p1; - mtx_enter(&sched_lock, MTX_SPIN | MTX_RLIKELY); + mtx_lock(&sched_lock); /* * check if the lock has been released while * waiting for the schedlock. */ if ((v = m->mtx_lock) == MTX_UNOWNED) { - mtx_exit(&sched_lock, MTX_SPIN); + mtx_unlock(&sched_lock); continue; } + /* * The mutex was marked contested on release. This * means that there are processes blocked on it. */ if (v == MTX_CONTESTED) { p1 = TAILQ_FIRST(&m->mtx_blocked); - KASSERT(p1 != NULL, ("contested mutex has no contesters")); - KASSERT(p != NULL, ("curproc is NULL for contested mutex")); + KASSERT(p1 != NULL, + ("contested mutex has no contesters")); + KASSERT(p != NULL, + ("curproc is NULL for contested mutex")); m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; if (p1->p_priority < p->p_priority) { SET_PRIO(p, p1->p_priority); } - mtx_exit(&sched_lock, MTX_SPIN); - return; + mtx_unlock(&sched_lock); + goto got_lock; } + /* * If the mutex isn't already contested and * a failure occurs setting the contested bit the @@ -582,7 +400,7 @@ if ((v & MTX_CONTESTED) == 0 && !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, (void *)(v | MTX_CONTESTED))) { - mtx_exit(&sched_lock, MTX_SPIN); + mtx_unlock(&sched_lock); continue; } @@ -598,9 +416,9 @@ ithd_t *it = (ithd_t *)p; if (it->it_interrupted) { - if ((type & MTX_QUIET) == 0) + if ((opts & MTX_QUIET) == 0) CTR2(KTR_LOCK, - "mtx_enter: 0x%x interrupted 0x%x", + "mtx_lock: 0x%x interrupted 0x%x", it, it->it_interrupted); intr_thd_fixup(it); } @@ -625,39 +443,54 @@ p_procq); } - p->p_blocked = m; /* Who we're blocked on */ + /* Save who we're blocked on. */ + p->p_blocked = m; p->p_mtxname = m->mtx_description; p->p_stat = SMTX; #if 0 propagate_priority(p); #endif - if ((type & MTX_QUIET) == 0) + if ((opts & MTX_QUIET) == 0) CTR3(KTR_LOCK, - "mtx_enter: p 0x%p blocked on [0x%p] %s", + "mtx_lock: p 0x%p blocked on [0x%p] %s", p, m, m->mtx_description); + mi_switch(); - if ((type & MTX_QUIET) == 0) + + if ((opts & MTX_QUIET) == 0) CTR3(KTR_LOCK, - "mtx_enter: p 0x%p free from blocked on [0x%p] %s", + "mtx_lock: p 0x%p free from blocked on [0x%p] %s", p, m, m->mtx_description); - mtx_exit(&sched_lock, MTX_SPIN); + + mtx_unlock(&sched_lock); } - return; + + break; + } + case MTX_SPIN: - case MTX_SPIN | MTX_FIRST: - case MTX_SPIN | MTX_TOPHALF: { int i = 0; + u_int mtx_intr = save_intr(); + + disable_intr(); + if (_obtain_lock(m, p)) { + m->mtx_saveintr = mtx_intr; + goto got_lock; + } if (m->mtx_lock == (uintptr_t)p) { m->mtx_recurse++; - return; + goto got_lock; } - if ((type & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "mtx_enter: %p spinning", m); + + if ((opts & MTX_QUIET) == 0) + CTR1(KTR_LOCK, "mtx_lock: %p spinning", m); + for (;;) { if (_obtain_lock(m, p)) break; + while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 1000000) continue; @@ -675,123 +508,159 @@ } } -#ifdef MUTEX_DEBUG - if (type != MTX_SPIN) - m->mtx_saveintr = 0xbeefface; - else -#endif - m->mtx_saveintr = saveintr; - if ((type & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "mtx_enter: 0x%p spin done", m); - return; + m->mtx_saveintr = mtx_intr; + break; } + + default: + panic("mtx_lock: unsupported mutex type: 0x%x\n", + m->mtx_flags & MTX_TYPES); } + +got_lock: +/****** +**** CHANGE SECOND ARGUMENT +**/ + WITNESS_ENTER(m, opts | m->mtx_flags, file, line); + + if ((opts & MTX_QUIET) == 0) { + if (m->mtx_flags & MTX_DEF) + CTR1(KTR_LOCK, "mtx_lock: 0x%p sleep done", m); + else + CTR1(KTR_LOCK, "mtx_lock: 0x%p spin done", m); + } + + return; } +/* + * The important part of mtx_unlock(). Here we actually release the lock `m' + * regardless of whether it is a spin or sleep lock. We take care to handle + * recursion. + */ void -mtx_exit_hard(struct mtx *m, int type) +_mtx_unlock(struct mtx *m, int opts, const char *file, int line) { struct proc *p, *p1; struct mtx *m1; int pri; p = CURPROC; - switch (type) { + KASSERT(p != NULL, ("curproc is NULL in _mtx_unlock")); + MPASS4(mtx_owned(m), STR_mtx_owned, file, line); + WITNESS_EXIT(m, opts | m->mtx_flags, file, line); + + if ((opts & MTX_QUIET) == 0) + CTR5(KTR_LOCK, STR_mtx_exit_fmt, m->mtx_description, m, + m->mtx_recurse, file, line); + + switch (m->mtx_flags & MTX_TYPES) { + case MTX_DEF: - case MTX_DEF | MTX_NOSWITCH: + { + if (_release_lock(m, p)) + return; + if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); - if ((type & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "mtx_exit: 0x%p unrecurse", m); + if ((opts & MTX_QUIET) == 0) + CTR1(KTR_LOCK, "mtx_unlock: 0x%p unrecurse", m); return; } - mtx_enter(&sched_lock, MTX_SPIN); - if ((type & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "mtx_exit: 0x%p contested", m); + + mtx_lock(&sched_lock); + if ((opts & MTX_QUIET) == 0) + CTR1(KTR_LOCK, "mtx_unlock: 0x%p contested", m); p1 = TAILQ_FIRST(&m->mtx_blocked); MPASS(p->p_magic == P_MAGIC); MPASS(p1->p_magic == P_MAGIC); TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); + if (TAILQ_EMPTY(&m->mtx_blocked)) { LIST_REMOVE(m, mtx_contested); _release_lock_quick(m); - if ((type & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "mtx_exit: 0x%p not held", m); + if ((opts & MTX_QUIET) == 0) + CTR1(KTR_LOCK, "mtx_unlock: 0x%p not held", m); } else atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); + pri = MAXPRI; LIST_FOREACH(m1, &p->p_contested, mtx_contested) { int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_priority; if (cp < pri) pri = cp; } + if (pri > p->p_nativepri) pri = p->p_nativepri; SET_PRIO(p, pri); - if ((type & MTX_QUIET) == 0) + + if ((opts & MTX_QUIET) == 0) CTR2(KTR_LOCK, - "mtx_exit: 0x%p contested setrunqueue 0x%p", m, p1); + "mtx_unlock: 0x%p contested setrunqueue 0x%p", m, p1); + p1->p_blocked = NULL; p1->p_mtxname = NULL; p1->p_stat = SRUN; setrunqueue(p1); - if ((type & MTX_NOSWITCH) == 0 && p1->p_priority < pri) { + + if ((opts & MTX_NOSWITCH) == 0 && p1->p_priority < pri) { #ifdef notyet if (p->p_flag & (P_ITHD | P_SITHD)) { ithd_t *it = (ithd_t *)p; if (it->it_interrupted) { - if ((type & MTX_QUIET) == 0) + if ((opts & MTX_QUIET) == 0) CTR2(KTR_LOCK, - "mtx_exit: 0x%x interruped 0x%x", + "mtx_unlock: 0x%x interruped 0x%x", it, it->it_interrupted); intr_thd_fixup(it); } } #endif setrunqueue(p); - if ((type & MTX_QUIET) == 0) + if ((opts & MTX_QUIET) == 0) CTR2(KTR_LOCK, - "mtx_exit: 0x%p switching out lock=0x%p", + "mtx_unlock: 0x%p switching out lock=0x%p", m, (void *)m->mtx_lock); + mi_switch(); - if ((type & MTX_QUIET) == 0) + + if ((opts & MTX_QUIET) == 0) CTR2(KTR_LOCK, - "mtx_exit: 0x%p resuming lock=0x%p", + "mtx_unlock: 0x%p resuming lock=0x%p", m, (void *)m->mtx_lock); } - mtx_exit(&sched_lock, MTX_SPIN); + mtx_unlock(&sched_lock); break; + } + case MTX_SPIN: - case MTX_SPIN | MTX_FIRST: + { if (mtx_recursed(m)) { m->mtx_recurse--; return; } - MPASS(mtx_owned(m)); - _release_lock_quick(m); - if (type & MTX_FIRST) - enable_intr(); /* XXX is this kosher? */ - else { - MPASS(m->mtx_saveintr != 0xbeefface); - restore_intr(m->mtx_saveintr); - } - break; - case MTX_SPIN | MTX_TOPHALF: - if (mtx_recursed(m)) { - m->mtx_recurse--; - return; - } - MPASS(mtx_owned(m)); + _release_lock_quick(m); + MPASS(m->mtx_saveintr != 0); + restore_intr(m->mtx_saveintr); break; + } default: - panic("mtx_exit_hard: unsupported type 0x%x\n", type); + panic("mtx_unlock: unsupported type 0x%x\n", m->mtx_flags & + MTX_TYPES); } + + return; } +/****************************************************************************** +* mtx_assert() is INVARIANTS ; revisit and cleanup later. +******************************************************************************/ + #ifdef INVARIANTS void _mtx_assert(struct mtx *m, int what, const char *file, int line) @@ -826,6 +695,10 @@ #define MV_DESTROY 0 /* validate before destory */ #define MV_INIT 1 /* validate before init */ +/****************************************************************************** +* mtx_validate() is MUTEX_DEBUG ; cleanup later. +******************************************************************************/ + #ifdef MUTEX_DEBUG int mtx_validate __P((struct mtx *, int)); @@ -844,7 +717,7 @@ if (m == &all_mtx || cold) return 0; - mtx_enter(&all_mtx, MTX_DEF); + mtx_lock(&all_mtx); /* * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly * we can re-enable the kernacc() checks. @@ -888,16 +761,20 @@ retval = 1; } } - mtx_exit(&all_mtx, MTX_DEF); + mtx_unlock(&all_mtx); return (retval); } #endif +/* + * Initialize mutex lock `m' + */ void -mtx_init(struct mtx *m, const char *t, int flag) +mtx_init(struct mtx *m, const char *description, int flag) { if ((flag & MTX_QUIET) == 0) - CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t); + CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, description); + #ifdef MUTEX_DEBUG if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ return; @@ -905,6 +782,7 @@ bzero((void *)m, sizeof *m); TAILQ_INIT(&m->mtx_blocked); + #ifdef WITNESS if (!witness_cold) { /* XXX - should not use DEVBUF */ @@ -912,29 +790,31 @@ M_DEVBUF, M_NOWAIT | M_ZERO); MPASS(m->mtx_union.mtxu_debug != NULL); - m->mtx_description = t; + m->mtx_description = description; } else { /* * Save a pointer to the description so that witness_fixup() * can properly initialize this mutex later on. */ - m->mtx_union.mtxu_description = t; + m->mtx_union.mtxu_description = description; } #else - m->mtx_description = t; + m->mtx_description = description; #endif m->mtx_flags = flag; m->mtx_lock = MTX_UNOWNED; + /* Put on all mutex queue */ - mtx_enter(&all_mtx, MTX_DEF); + mtx_lock(&all_mtx); m->mtx_next = &all_mtx; m->mtx_prev = all_mtx.mtx_prev; m->mtx_prev->mtx_next = m; all_mtx.mtx_prev = m; if (++mtx_cur_cnt > mtx_max_cnt) mtx_max_cnt = mtx_cur_cnt; - mtx_exit(&all_mtx, MTX_DEF); + mtx_unlock(&all_mtx); + #ifdef WITNESS if (!witness_cold) witness_init(m, flag); @@ -949,7 +829,9 @@ KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", __FUNCTION__)); #endif + CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description); + #ifdef MUTEX_DEBUG if (m->mtx_next == NULL) panic("mtx_destroy: %p (%s) already destroyed", @@ -961,7 +843,7 @@ MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); } mtx_validate(m, MV_DESTROY); /* diagnostic */ -#endif +#endif /* MUTEX_DEBUG */ #ifdef WITNESS if (m->mtx_witness) @@ -969,18 +851,21 @@ #endif /* WITNESS */ /* Remove from the all mutex queue */ - mtx_enter(&all_mtx, MTX_DEF); + mtx_lock(&all_mtx); m->mtx_next->mtx_prev = m->mtx_prev; m->mtx_prev->mtx_next = m->mtx_next; + #ifdef MUTEX_DEBUG m->mtx_next = m->mtx_prev = NULL; #endif + #ifdef WITNESS free(m->mtx_union.mtxu_debug, M_DEVBUF); m->mtx_union.mtxu_debug = NULL; #endif + mtx_cur_cnt--; - mtx_exit(&all_mtx, MTX_DEF); + mtx_unlock(&all_mtx); } static void @@ -1010,9 +895,14 @@ } SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) +/****************************************************************************** +******************************************************************************* +* BEGINNING OF EVIL WITNESS CODE WRITTEN BY JEHOVAH'S WITNESSES * +******************************************************************************* +******************************************************************************/ + /* - * The non-inlined versions of the mtx_*() functions are always built (above), - * but the witness code depends on the WITNESS kernel option being specified. + * The witness code depends on the WITNESS kernel option being specified. */ #ifdef WITNESS @@ -1209,17 +1099,17 @@ file, line); return; } - mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_lock_opts(&w_mtx, MTX_QUIET); i = PCPU_GET(witness_spin_check); if (i != 0 && w->w_level < i) { - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" " %s:%d already holding %s:%x", m->mtx_description, w->w_level, file, line, spin_order_list[ffs(i)-1], i); } PCPU_SET(witness_spin_check, i | w->w_level); - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1243,7 +1133,7 @@ goto out; if (!mtx_legal2block()) - panic("blockable mtx_enter() of %s when not legal @ %s:%d", + panic("blockable mtx_lock() of %s when not legal @ %s:%d", m->mtx_description, file, line); /* * Is this the first mutex acquired @@ -1265,16 +1155,16 @@ goto out; } MPASS(!mtx_owned(&w_mtx)); - mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_lock_opts(&w_mtx, MTX_QUIET); /* * If we have a known higher number just say ok */ if (witness_watch > 1 && w->w_level > w1->w_level) { - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); goto out; } if (isitmydescendant(m1->mtx_witness, w)) { - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); goto out; } for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { @@ -1282,7 +1172,7 @@ MPASS(i < 200); w1 = m1->mtx_witness; if (isitmydescendant(w, w1)) { - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); if (blessed(w, w1)) goto out; if (m1 == &Giant) { @@ -1311,7 +1201,7 @@ } m1 = LIST_FIRST(&p->p_heldmtx); if (!itismychild(m1->mtx_witness, w)) - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); out: #ifdef DDB @@ -1354,10 +1244,10 @@ m->mtx_description, file, line); return; } - mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_lock_opts(&w_mtx, MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) | w->w_level); - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1405,10 +1295,10 @@ file, line); return; } - mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_lock_opts(&w_mtx, MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) & ~w->w_level); - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); return; } if (w->w_spin) @@ -1424,7 +1314,7 @@ } if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) - panic("switchable mtx_exit() of %s when not legal @ %s:%d", + panic("switchable mtx_unlock() of %s when not legal @ %s:%d", m->mtx_description, file, line); LIST_REMOVE(m, mtx_held); m->mtx_held.le_prev = NULL; @@ -1495,10 +1385,10 @@ } if ((flag & MTX_SPIN) && witness_skipspin) return (NULL); - mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_lock_opts(&w_mtx, MTX_QUIET); for (w = w_all; w; w = w->w_next) { if (strcmp(description, w->w_description) == 0) { - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); return (w); } } @@ -1507,7 +1397,7 @@ w->w_next = w_all; w_all = w; w->w_description = description; - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); if (flag & MTX_SPIN) { w->w_spin = 1; @@ -1733,7 +1623,7 @@ if ((w = w_free) == NULL) { witness_dead = 1; - mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET); + mtx_unlock_opts(&w_mtx, MTX_QUIET); printf("witness exhausted\n"); return (NULL); } diff -ru /cvs/sys_old/sys/mutex.h /usr/src/sys/sys/mutex.h --- /cvs/sys_old/sys/mutex.h Mon Jan 22 15:12:59 2001 +++ /usr/src/sys/sys/mutex.h Mon Jan 22 20:23:41 2001 @@ -41,6 +41,7 @@ #include #include #endif /* _KERNEL_ */ + #endif /* !LOCORE */ #include @@ -48,31 +49,36 @@ #ifdef _KERNEL /* - * Mutex flags - * - * Types + * Mutex types and options stored in mutex->mtx_flags + */ +#define MTX_DEF 0x00000000 /* DEFAULT sleep lock */ +#define MTX_SPIN 0x00000001 /* Spin lock */ +#define MTX_RECURSE 0x00000002 /* Lock allowed to recurse */ + +/* + * Mask out all non-type bits in mutex->mtx_flags such that we can fetch + * `the type of lock' by looking at (mutex->mtx_flags & MTX_TYPES) + */ +#define MTX_TYPES (MTX_DEF | MTX_SPIN) + +/* + * Internal options for mtx_lock() and/or mtx_unlock() + * (passed as `opts' parameter by wrapper macros) */ -#define MTX_DEF 0x0 /* Default (spin/sleep) */ -#define MTX_SPIN 0x1 /* Spin only lock */ +/***************** +******* REVISIT: Right now, we don't interfere with above bits because of +******* WITNESS, this will CHANGE +**/ +#define MTX_NOSWITCH 0x00000004 /* Do not switch on release */ +#define MTX_QUIET 0x00000008 /* Don't log mutex events */ -/* Options */ -#define MTX_RECURSE 0x2 /* Recursive lock (for mtx_init) */ -#define MTX_RLIKELY 0x4 /* Recursion likely */ -#define MTX_NORECURSE 0x8 /* No recursion possible */ -#define MTX_NOSPIN 0x10 /* Don't spin before sleeping */ -#define MTX_NOSWITCH 0x20 /* Do not switch on release */ -#define MTX_FIRST 0x40 /* First spin lock holder */ -#define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */ -#define MTX_QUIET 0x100 /* Don't log a mutex event */ - -/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */ -#define MTX_HARDOPTS (MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH) - -/* Flags/value used in mtx_lock */ -#define MTX_RECURSED 0x01 /* (non-spin) lock held recursively */ -#define MTX_CONTESTED 0x02 /* (non-spin) lock contested */ +/* + * State bits kept in mutex->mtx_lock. + */ +#define MTX_RECURSED 0x00000001 /* Lock being held recursively */ +#define MTX_CONTESTED 0x00000002 /* Lock is contested */ +#define MTX_UNOWNED 0x00000004 /* Cookie for free mutex */ #define MTX_FLAGMASK ~(MTX_RECURSED | MTX_CONTESTED) -#define MTX_UNOWNED 0x8 /* Cookie for free mutex */ #endif /* _KERNEL */ @@ -101,47 +107,96 @@ #define mp_fixme(string) #ifdef _KERNEL -/* Prototypes */ + +/* + * Prototypes + * + * NOTE: Functions prepended with `_' (underscore) are exported to other + * parts of the kernel with the help of macros, thus allowing us to + * use the cpp __FILE__ and __LINE__ directives. + */ void mtx_init(struct mtx *m, const char *description, int flag); void mtx_destroy(struct mtx *m); +void _mtx_lock(struct mtx *m, int opts, const char *file, int line); +void _mtx_unlock(struct mtx *m, int opts, const char *file, int line); +int _mtx_try_lock(struct mtx *mtxp, int opts, const char *file, int line); + +/***** +*** HACKS; remove when done. +**/ + +#define mtx_enter(lock, type) \ + mtx_lock_opts((lock), (type)) + +#define mtx_exit(lock, type) \ + mtx_unlock_opts((lock), (type)) + +#define mtx_try_enter(lock, type) \ + mtx_try_lock_opts((lock), (type)) /* - * Wrap the following functions with cpp macros so that filenames and line - * numbers are embedded in the code correctly. + * Exported lock manipulation interface. + * + * mtx_{lock, unlock}_opts allows us to pass options such as MTX_NOSWITCH + * and MTX_QUIET to the routines, while at the same time not making them + * necessary under normal conditions. */ -void _mtx_enter(struct mtx *mtxp, int type, const char *file, int line); -int _mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line); -void _mtx_exit(struct mtx *mtxp, int type, const char *file, int line); +#define mtx_lock(mtxp) \ + _mtx_lock((mtxp), 0, __FILE__, __LINE__) + +#define mtx_lock_opts(mtxp, opts) \ + _mtx_lock((mtxp), (opts), __FILE__, __LINE__) + +#define mtx_unlock(mtxp) \ + _mtx_unlock((mtxp), 0, __FILE__, __LINE__) + +#define mtx_unlock_opts(mtxp, opts) \ + _mtx_unlock((mtxp), (opts), __FILE__, __LINE__) -#define mtx_enter(mtxp, type) \ - _mtx_enter((mtxp), (type), __FILE__, __LINE__) +#define mtx_try_lock(mtxp) \ + _mtx_try_lock((mtxp), 0, __FILE__, __LINE__) -#define mtx_try_enter(mtxp, type) \ - _mtx_try_enter((mtxp), (type), __FILE__, __LINE__) +#define mtx_try_lock_opts(mtxp, opts) \ + _mtx_try_lock((mtxp), (opts), __FILE__, __LINE__) + +/* Returns non-zero if lock `m' is owned by presently executing thread */ +#define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == (uintptr_t)CURTHD) + +/* Returns non-zero if lock `m' is not owned */ +#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) + +/* Returns the owner of the lock `m' */ +#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ + : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) + +/* Returns non-zero if lock `m' is presently recursed */ +#define mtx_recursed(m) ((m)->mtx_recurse != 0) -#define mtx_exit(mtxp, type) \ - _mtx_exit((mtxp), (type), __FILE__, __LINE__) -/* Global locks */ +/* + * Global locks + */ extern struct mtx sched_lock; extern struct mtx Giant; /* + * Giant lock manipulation and clean exit macros. * Used to replace return with an exit Giant and return. */ #define EGAR(a) \ do { \ - mtx_exit(&Giant, MTX_DEF); \ + mtx_unlock(&Giant); \ return (a); \ } while (0) #define VEGAR \ do { \ - mtx_exit(&Giant, MTX_DEF); \ + mtx_unlock(&Giant); \ return; \ } while (0) +/* NOTE: DROP_GIANT*() needs to be paired with PICKUP_GIANT() */ #define DROP_GIANT_NOSWITCH() \ do { \ int _giantcnt; \ @@ -150,7 +205,7 @@ if (mtx_owned(&Giant)) \ WITNESS_SAVE(&Giant, Giant); \ for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ - mtx_exit(&Giant, MTX_DEF | MTX_NOSWITCH) + mtx_unlock_opts(&Giant, MTX_NOSWITCH) #define DROP_GIANT() \ do { \ @@ -160,12 +215,12 @@ if (mtx_owned(&Giant)) \ WITNESS_SAVE(&Giant, Giant); \ for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ - mtx_exit(&Giant, MTX_DEF) + mtx_unlock(&Giant) #define PICKUP_GIANT() \ mtx_assert(&Giant, MA_NOTOWNED); \ while (_giantcnt--) \ - mtx_enter(&Giant, MTX_DEF); \ + mtx_lock(&Giant); \ if (mtx_owned(&Giant)) \ WITNESS_RESTORE(&Giant, Giant); \ } while (0) @@ -173,24 +228,37 @@ #define PARTIAL_PICKUP_GIANT() \ mtx_assert(&Giant, MA_NOTOWNED); \ while (_giantcnt--) \ - mtx_enter(&Giant, MTX_DEF); \ + mtx_lock(&Giant); \ if (mtx_owned(&Giant)) \ WITNESS_RESTORE(&Giant, Giant) /* - * Debugging + * Debugging options and defines + */ + +/* + * INVARIANTS enables: + * mtx_assert() + * */ #ifdef INVARIANTS #define MA_OWNED 1 #define MA_NOTOWNED 2 #define MA_RECURSED 4 #define MA_NOTRECURSED 8 + void _mtx_assert(struct mtx *m, int what, const char *file, int line); #define mtx_assert(m, what) _mtx_assert((m), (what), __FILE__, __LINE__) + #else /* INVARIANTS */ + #define mtx_assert(m, what) #endif /* INVARIANTS */ +/* + * MUTEX_DEBUG enables: + * MPASS*() sanity-check macros + */ #ifdef MUTEX_DEBUG #define MPASS(ex) \ if (!(ex)) \ @@ -211,21 +279,9 @@ #define MPASS4(ex, what, file, line) #endif /* MUTEX_DEBUG */ -/* - * Externally visible mutex functions. - *------------------------------------------------------------------------------ - */ - -/* - * Return non-zero if a mutex is already owned by the current thread. - */ -#define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == (uintptr_t)CURTHD) - -/* - * Return non-zero if a mutex has been recursively acquired. - */ -#define mtx_recursed(m) ((m)->mtx_recurse != 0) - +/******* +*** THIS NEEDS TO GO! +**/ /* Common strings */ #ifdef _KERN_MUTEX_C_ char STR_mtx_enter_fmt[] = "GOT %s [%p] r=%d at %s:%d"; @@ -242,6 +298,29 @@ extern char STR_mtx_recurse[]; extern char STR_mtx_try_enter_fmt[]; #endif /* _KERN_MUTEX_C_ */ + +/* + * WITNESS enables: + * witness_save() + * witness_restore() + * witness_enter() + * witness_try_enter() + * witness_exit() + * witness_list() + * witness_sleep() + * witness_enter() - WITNESS_ENTER() + * witness_exit() - WITNESS_EXIT() + * witness_sleep() - WITNESS_SLEEP() + * witness_save() - WITNESS_SAVE() - WITNESS_SAVE_DECL() + * witness_restore() - WITNESS_RESTORE() + * + * WITNESS also enables, but it's prototyped in kern_mutex.c: + * witness_init() + * witness_destroy() + * witness_display() + * + * define mtx_debug + */ #ifdef WITNESS void witness_save(struct mtx *, const char **, int *);