--- /usr/src/sys/kern/kern_sx.c 2008-03-12 14:27:05.000000000 +0100 +++ src/sys/sys/kern_sx.c 2008-05-14 16:15:34.000000000 +0200 @@ -101,6 +101,7 @@ * Returns true if an exclusive lock is recursed. It assumes * curthread currently has an exclusive lock. */ +#define sx_recurse lock_object.lo_data #define sx_recursed(sx) ((sx)->sx_recurse != 0) static void assert_sx(struct lock_object *lock, int what); --- /usr/src/sys/kern/subr_lock.c 2008-02-06 01:04:09.000000000 +0100 +++ src/sys/kern/subr_lock.c 2008-05-15 01:55:32.000000000 +0200 @@ -89,10 +89,9 @@ /* Initialize the lock object. */ lock->lo_name = name; - lock->lo_type = type != NULL ? type : name; lock->lo_flags |= flags | LO_INITIALIZED; LOCK_LOG_INIT(lock, 0); - WITNESS_INIT(lock); + WITNESS_INIT(lock, (type != NULL) ? type : name); } void @@ -121,8 +120,6 @@ class = LOCK_CLASS(lock); db_printf(" class: %s\n", class->lc_name); db_printf(" name: %s\n", lock->lo_name); - if (lock->lo_type && lock->lo_type != lock->lo_name) - db_printf(" type: %s\n", lock->lo_type); class->lc_ddb_show(lock); } #endif --- /usr/src/sys/kern/subr_witness.c 2008-05-07 23:41:36.000000000 +0200 +++ src/sys/kern/subr_witness.c 2008-05-15 01:56:51.000000000 +0200 @@ -113,10 +113,6 @@ #define KTR_WITNESS 0 #endif -/* Easier to stay with the old names. */ -#define lo_list lo_witness_data.lod_list -#define lo_witness lo_witness_data.lod_witness - #define LI_RECURSEMASK 0x0000ffff /* Recursion depth of lock instance. */ #define LI_EXCLUSIVE 0x00010000 /* Exclusive lock instance. */ @@ -126,6 +122,7 @@ #define WITNESS_COUNT 1024 #define WITNESS_CHILDCOUNT (WITNESS_COUNT * 4) #define WITNESS_SBUFSIZE 32768 +#define WITNESS_PENDLIST 512 /* * XXX: This is somewhat bogus, as we assume here that at most 1024 threads * will hold LOCK_NCHILDREN * 2 locks. We handle failure ok, and we should @@ -204,10 +201,15 @@ struct lock_class *w_class; }; +struct witness_pendhelp { + const char *wh_type; + struct lock_object *wh_lock; +}; + #ifdef BLESSING static int blessed(struct witness *, struct witness *); #endif -static int depart(struct witness *w); +static void depart(struct witness *w); static struct witness *enroll(const char *description, struct lock_class *lock_class); static int insertchild(struct witness *parent, struct witness *child); @@ -218,7 +220,7 @@ static int sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS); static int sysctl_debug_witness_graphs(SYSCTL_HANDLER_ARGS); static const char *fixup_filename(const char *file); -static void witness_addgraph(struct sbuf *sb, struct witness *w); +static void witness_addgraph(struct sbuf *sb, struct witness *parent); static struct witness *witness_get(void); static void witness_free(struct witness *m); static struct witness_child_list_entry *witness_child_get(void); @@ -296,6 +298,8 @@ static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep); static struct witness_child_list_entry *w_child_free = NULL; static struct lock_list_entry *w_lock_list_free = NULL; +static struct witness_pendhelp pending_locks[WITNESS_PENDLIST]; +static u_int pending_cnt; static int w_free_cnt, w_spin_cnt, w_sleep_cnt, w_child_free_cnt, w_child_cnt; SYSCTL_INT(_debug_witness, OID_AUTO, free_cnt, CTLFLAG_RD, &w_free_cnt, 0, ""); @@ -525,13 +529,6 @@ #endif /* - * List of locks initialized prior to witness being initialized whose - * enrollment is currently deferred. - */ -STAILQ_HEAD(, lock_object) pending_locks = - STAILQ_HEAD_INITIALIZER(pending_locks); - -/* * This global is set to 0 once it becomes safe to use the witness code. */ static int witness_cold = 1; @@ -591,13 +588,13 @@ witness_spin_warn = 1; /* Iterate through all locks and add them to witness. */ - while (!STAILQ_EMPTY(&pending_locks)) { - lock = STAILQ_FIRST(&pending_locks); - STAILQ_REMOVE_HEAD(&pending_locks, lo_list); + for (i = 0; pending_locks[i].wh_lock != NULL; i++) { + lock = pending_locks[i].wh_lock; KASSERT(lock->lo_flags & LO_WITNESS, ("%s: lock %s is on pending list but not LO_WITNESS", __func__, lock->lo_name)); - lock->lo_witness = enroll(lock->lo_type, LOCK_CLASS(lock)); + lock->lo_witness = enroll(pending_locks[i].wh_type, + LOCK_CLASS(lock)); } /* Mark the witness code as being ready for use. */ @@ -659,7 +656,7 @@ } void -witness_init(struct lock_object *lock) +witness_init(struct lock_object *lock, const char *type) { struct lock_class *class; @@ -689,10 +686,13 @@ (lock->lo_flags & LO_WITNESS) == 0) lock->lo_witness = NULL; else if (witness_cold) { - STAILQ_INSERT_TAIL(&pending_locks, lock, lo_list); - lock->lo_flags |= LO_ENROLLPEND; + pending_locks[pending_cnt].wh_lock = lock; + pending_locks[pending_cnt++].wh_type = type; + if (pending_cnt > WITNESS_PENDLIST) + panic("%s: pending locks list is too small, bump it\n", + __func__); } else - lock->lo_witness = enroll(lock->lo_type, class); + lock->lo_witness = enroll(type, class); } void @@ -707,28 +707,15 @@ class->lc_name, lock->lo_name); /* XXX: need to verify that no one holds the lock */ - if ((lock->lo_flags & (LO_WITNESS | LO_ENROLLPEND)) == LO_WITNESS && - lock->lo_witness != NULL) { + if ((lock->lo_flags & LO_WITNESS) && lock->lo_witness != NULL) { w = lock->lo_witness; mtx_lock_spin(&w_mtx); MPASS(w->w_refcount > 0); w->w_refcount--; - /* - * Lock is already released if we have an allocation failure - * and depart() fails. - */ - if (w->w_refcount != 0 || depart(w)) - mtx_unlock_spin(&w_mtx); - } - - /* - * If this lock is destroyed before witness is up and running, - * remove it from the pending list. - */ - if (lock->lo_flags & LO_ENROLLPEND) { - STAILQ_REMOVE(&pending_locks, lock, lock_object, lo_list); - lock->lo_flags &= ~LO_ENROLLPEND; + if (w->w_refcount == 0) + depart(w); + mtx_unlock_spin(&w_mtx); } } @@ -925,7 +912,7 @@ /* Try to add the new order. */ CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__, - lock2->lo_type, lock1->lo_type); + lock2->lo_witness->w_name, lock1->lo_witness->w_name); if (!itismychild(lock1->lo_witness, lock2->lo_witness)) return (ENOMEM); mtx_unlock_spin(&w_mtx); @@ -1034,7 +1021,7 @@ return; w->w_same_squawked = 1; printf("acquiring duplicate lock of same type: \"%s\"\n", - lock->lo_type); + w->w_name); printf(" 1st %s @ %s:%d\n", lock1->li_lock->lo_name, lock1->li_file, lock1->li_line); printf(" 2nd %s @ %s:%d\n", lock->lo_name, file, line); @@ -1164,21 +1151,19 @@ if (i < 0) { printf(" 1st %p %s (%s) @ %s:%d\n", lock1->li_lock, lock1->li_lock->lo_name, - lock1->li_lock->lo_type, lock1->li_file, - lock1->li_line); + w1->w_name, lock1->li_file, lock1->li_line); printf(" 2nd %p %s (%s) @ %s:%d\n", lock, - lock->lo_name, lock->lo_type, file, line); + lock->lo_name, w->w_name, file, line); } else { printf(" 1st %p %s (%s) @ %s:%d\n", lock2->li_lock, lock2->li_lock->lo_name, - lock2->li_lock->lo_type, lock2->li_file, - lock2->li_line); + lock2->li_lock->lo_witness->w_name, + lock2->li_file, lock2->li_line); printf(" 2nd %p %s (%s) @ %s:%d\n", lock1->li_lock, lock1->li_lock->lo_name, - lock1->li_lock->lo_type, lock1->li_file, - lock1->li_line); + w1->w_name, lock1->li_file, lock1->li_line); printf(" 3rd %p %s (%s) @ %s:%d\n", lock, - lock->lo_name, lock->lo_type, file, line); + lock->lo_name, w->w_name, file, line); } #ifdef KDB goto debugger; @@ -1198,7 +1183,7 @@ !(lock1->li_lock == &Giant.lock_object && (lock->lo_flags & LO_SLEEPABLE) != 0)) { CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__, - lock->lo_type, lock1->li_lock->lo_type); + w->w_name, lock1->li_lock->lo_witness->w_name); if (!itismychild(lock1->li_lock->lo_witness, w)) /* Witness is dead. */ return; @@ -1533,8 +1518,10 @@ return (w); } } - if ((w = witness_get()) == NULL) + if ((w = witness_get()) == NULL) { + printf("WITNESS: unable to allocate a new witness object\n"); goto out; + } w->w_name = description; w->w_class = lock_class; w->w_refcount = 1; @@ -1567,7 +1554,7 @@ } /* Don't let the door bang you on the way out... */ -static int +static void depart(struct witness *w) { struct witness_child_list_entry *wcl, *nwcl; @@ -1610,8 +1597,6 @@ STAILQ_REMOVE(list, w, witness, w_typelist); STAILQ_REMOVE(&w_all, w, witness, w_list); witness_free(w); - - return (1); } /* @@ -1646,7 +1631,6 @@ static int itismychild(struct witness *parent, struct witness *child) { - struct witness_list *list; MPASS(child != NULL && parent != NULL); if ((parent->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) != @@ -1656,14 +1640,7 @@ __func__, parent->w_class->lc_name, child->w_class->lc_name); - if (!insertchild(parent, child)) - return (0); - - if (parent->w_class->lc_flags & LC_SLEEPLOCK) - list = &w_sleep; - else - list = &w_spin; - return (1); + return (insertchild(parent, child)); } static void @@ -1864,8 +1841,8 @@ lock = instance->li_lock; printf("%s %s %s", (instance->li_flags & LI_EXCLUSIVE) != 0 ? "exclusive" : "shared", LOCK_CLASS(lock)->lc_name, lock->lo_name); - if (lock->lo_type != lock->lo_name) - printf(" (%s)", lock->lo_type); + if (lock->lo_witness->w_name != lock->lo_name) + printf(" (%s)", lock->lo_witness->w_name); printf(" r = %d (%p) locked @ %s:%d\n", instance->li_flags & LI_RECURSEMASK, lock, instance->li_file, instance->li_line); --- /usr/src/sys/sys/_lock.h 2007-12-27 20:47:40.000000000 +0100 +++ src/sys/sys/_lock.h 2008-05-15 01:56:29.000000000 +0200 @@ -33,12 +33,9 @@ struct lock_object { const char *lo_name; /* Individual lock name. */ - const char *lo_type; /* General lock type. */ u_int lo_flags; - union { /* Data for witness. */ - STAILQ_ENTRY(lock_object) lod_list; - struct witness *lod_witness; - } lo_witness_data; + u_int lo_data; /* General class specific data. */ + struct witness *lo_witness; /* Data for witness. */ }; #endif /* !_SYS__LOCK_H_ */ --- /usr/src/sys/sys/_lockmgr.h 2008-04-06 22:08:50.000000000 +0200 +++ src/sys/sys/_lockmgr.h 2008-05-14 15:59:06.000000000 +0200 @@ -38,7 +38,6 @@ struct lock { struct lock_object lock_object; volatile uintptr_t lk_lock; - volatile unsigned lk_recurse; int lk_timo; int lk_pri; #ifdef DEBUG_LOCKS --- /usr/src/sys/sys/_mutex.h 2007-03-21 22:20:51.000000000 +0100 +++ src/sys/sys/_mutex.h 2008-05-14 16:04:20.000000000 +0200 @@ -37,7 +37,6 @@ struct mtx { struct lock_object lock_object; /* Common lock properties. */ volatile uintptr_t mtx_lock; /* Owner and flags. */ - volatile u_int mtx_recurse; /* Number of recursive holds. */ }; #endif /* !_SYS__MUTEX_H_ */ --- /usr/src/sys/sys/_rwlock.h 2007-06-26 23:31:56.000000000 +0200 +++ src/sys/sys/_rwlock.h 2008-05-14 15:59:18.000000000 +0200 @@ -38,7 +38,6 @@ struct rwlock { struct lock_object lock_object; volatile uintptr_t rw_lock; - volatile unsigned rw_recurse; }; #endif /* !_SYS__RWLOCK_H_ */ --- /usr/src/sys/sys/_sx.h 2007-04-01 01:23:42.000000000 +0200 +++ src/sys/sys/_sx.h 2008-05-14 15:59:25.000000000 +0200 @@ -37,7 +37,6 @@ struct sx { struct lock_object lock_object; volatile uintptr_t sx_lock; - volatile unsigned sx_recurse; }; #endif /* !_SYS__SX_H_ */ --- /usr/src/sys/sys/lock.h 2008-05-04 14:49:55.000000000 +0200 +++ src/sys/sys/lock.h 2008-05-15 01:54:26.000000000 +0200 @@ -78,7 +78,6 @@ #define LO_SLEEPABLE 0x00100000 /* Lock may be held while sleeping. */ #define LO_UPGRADABLE 0x00200000 /* Lock may be upgraded/downgraded. */ #define LO_DUPOK 0x00400000 /* Don't check for duplicate acquires */ -#define LO_ENROLLPEND 0x00800000 /* On the pending enroll list. */ #define LO_CLASSMASK 0x0f000000 /* Class index bitmask. */ #define LO_NOPROFILE 0x10000000 /* Don't profile this lock */ @@ -201,7 +200,7 @@ void lock_destroy(struct lock_object *); void spinlock_enter(void); void spinlock_exit(void); -void witness_init(struct lock_object *); +void witness_init(struct lock_object *, const char *); void witness_destroy(struct lock_object *); int witness_defineorder(struct lock_object *, struct lock_object *); void witness_checkorder(struct lock_object *, int, const char *, int); @@ -225,8 +224,8 @@ #define WARN_PANIC 0x02 /* Panic if check fails. */ #define WARN_SLEEPOK 0x04 /* Sleepable locks are exempt from check. */ -#define WITNESS_INIT(lock) \ - witness_init((lock)) +#define WITNESS_INIT(lock, type) \ + witness_init((lock), (type)) #define WITNESS_DESTROY(lock) \ witness_destroy(lock) @@ -273,7 +272,7 @@ witness_line(lock) #else /* WITNESS */ -#define WITNESS_INIT(lock) +#define WITNESS_INIT(lock, type) #define WITNESS_DESTROY(lock) #define WITNESS_DEFINEORDER(lock1, lock2) 0 #define WITNESS_CHECKORDER(lock, flags, file, line) --- /usr/src/sys/sys/lockmgr.h 2008-04-07 16:46:38.000000000 +0200 +++ src/sys/sys/lockmgr.h 2008-05-14 22:30:52.000000000 +0200 @@ -59,6 +59,7 @@ #endif struct thread; +#define lk_recurse lock_object.lo_data /* * Function prototipes. Routines that start with an underscore are not part --- /usr/src/sys/sys/mutex.h 2007-11-24 04:19:48.000000000 +0100 +++ src/sys/sys/mutex.h 2008-05-14 16:07:40.000000000 +0200 @@ -134,6 +134,8 @@ #define thread_unlock(tdp) \ mtx_unlock_spin((tdp)->td_lock) +#define mtx_recurse lock_object.lo_data + /* * We define our machine-independent (unoptimized) mutex micro-operations * here, if they are not already defined in the machine-dependent mutex.h --- /usr/src/sys/sys/rwlock.h 2008-04-12 14:50:34.000000000 +0200 +++ src/sys/sys/rwlock.h 2008-05-14 16:07:32.000000000 +0200 @@ -83,6 +83,8 @@ #ifdef _KERNEL +#define rw_recurse lock_object.lo_data + /* Very simple operations on rw_lock. */ /* Try to obtain a write lock once. */