Index: tree/sys/sys/_callout.h =================================================================== --- tree/sys/sys/_callout.h (revision 235056) +++ tree/sys/sys/_callout.h (working copy) @@ -56,6 +56,7 @@ struct lock_object *c_lock; /* lock to handle */ int c_flags; /* state of this entry */ volatile int c_cpu; /* CPU we're scheduled on */ + int c_index; /* callout heap index */ }; #endif Index: tree/sys/kern/kern_timeout.c =================================================================== --- tree/sys/kern/kern_timeout.c (revision 235056) +++ tree/sys/kern/kern_timeout.c (working copy) @@ -80,12 +80,10 @@ static int avg_mpcalls; SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls, CTLFLAG_RD, &avg_mpcalls, 0, "Average number of MP callouts made per softclock call. Units = 1/1000"); -/* - * TODO: - * allocate more timeout table slots when table overflows. - */ -int callwheelsize, callwheelbits, callwheelmask; +static int heapsize = 10000; +TUNABLE_INT("kern.heapsize", &heapsize); + /* * The callout cpu migration entity represents informations necessary for * describing the migrating callout to the new callout cpu. @@ -118,11 +116,17 @@ * cc_ticks is also used in callout_reset_cpu() to determine * when the callout should be served. */ + +struct heap_elm { + struct callout *c; + int c_time; +}; + struct callout_cpu { struct cc_mig_ent cc_migrating_entity; struct mtx cc_lock; struct callout *cc_callout; - struct callout_tailq *cc_callwheel; + struct callout *cc_heap; struct callout_list cc_callfree; struct callout *cc_next; struct callout *cc_curr; @@ -132,6 +136,7 @@ int cc_cancel; int cc_waiting; int cc_firsttick; + int heap_nelements; }; #ifdef SMP @@ -153,6 +158,10 @@ #define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock) #define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED) +#define CC_HEAP_PARENT(ndx) (((ndx) - 1) >> 1) +#define CC_HEAP_RIGHT(ndx) (((ndx) + 1) << 1) +#define CC_HEAP_LEFT(ndx) ((((ndx) + 1) << 1) - 1) + static int timeout_cpu; void (*callout_new_inserted)(int cpu, int ticks) = NULL; @@ -217,19 +226,11 @@ timeout_cpu = PCPU_GET(cpuid); cc = CC_CPU(timeout_cpu); - /* - * Calculate callout wheel size - */ - for (callwheelsize = 1, callwheelbits = 0; - callwheelsize < ncallout; - callwheelsize <<= 1, ++callwheelbits) - ; - callwheelmask = callwheelsize - 1; - + cc->cc_callout = (struct callout *)v; v = (caddr_t)(cc->cc_callout + ncallout); - cc->cc_callwheel = (struct callout_tailq *)v; - v = (caddr_t)(cc->cc_callwheel + callwheelsize); + cc->cc_heap = (callout *)v; + v = (caddr_t)(cc->cc_heap + heapsize); return(v); } @@ -241,8 +242,8 @@ mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE); SLIST_INIT(&cc->cc_callfree); - for (i = 0; i < callwheelsize; i++) { - TAILQ_INIT(&cc->cc_callwheel[i]); + for (i = 0; i < heapsize; i++) { + cc->cc_heap[i] = NULL; } cc_cme_cleanup(cc); if (cc->cc_callout == NULL) @@ -322,8 +323,8 @@ INTR_MPSAFE, &cc->cc_cookie)) panic("died while creating standard software ithreads"); cc->cc_callout = NULL; /* Only cpu0 handles timeout(). */ - cc->cc_callwheel = malloc( - sizeof(struct callout_tailq) * callwheelsize, M_CALLOUT, + cc->cc_heap = malloc( + sizeof(struct heap_elm) * heapsize, M_CALLOUT, M_WAITOK); callout_cpu_init(cc); } @@ -347,9 +348,9 @@ cc = CC_SELF(); mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); cc->cc_firsttick = cc->cc_ticks = ticks; - for (; (cc->cc_softticks - cc->cc_ticks) <= 0; cc->cc_softticks++) { - bucket = cc->cc_softticks & callwheelmask; - if (!TAILQ_EMPTY(&cc->cc_callwheel[bucket])) { + if (heap[0] != NULL) { + /* Overflow */ + if (heap[0]->c_time - cc->cc_ticks <= 0) need_softclock = 1; break; } @@ -368,24 +369,20 @@ { struct callout_cpu *cc; struct callout *c; - struct callout_tailq *sc; int curticks; int skip = 1; cc = CC_SELF(); mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); + c = cc->heap[0]; + curticks = cc->cc_ticks; - while( skip < ncallout && skip < limit ) { - sc = &cc->cc_callwheel[ (curticks+skip) & callwheelmask ]; - /* search scanning ticks */ - TAILQ_FOREACH( c, sc, c_links.tqe ){ - if (c->c_time - curticks <= ncallout) - goto out; - } - skip++; - } -out: - cc->cc_firsttick = curticks + skip; + + if (c != NULL) + skip = imin(c->c_time - curticks, limit); + else + skip = limit; + mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); return (skip); } @@ -414,7 +411,80 @@ return (cc); } +static void +callout_heap_add(struct callout *c,struct callout_cpu *cc) +{ + int ndx, parent, current; + struct heap_elm *h_curr, *h_par, h_tmp; + + mtx_lock_spin(&cc->cc_lock); + cc->heap[cc->heap_nelements].c = c; + cc->heap[cc->heap_nelements].c_time = c->c_time; + cc->heap_nelements++; + ndx = heap_nelements; + + /* + * Perform the upheap operation + */ + + if (heap_current == 0) + goto done; + + parent = CC_HEAP_PARENT(cc->heap_nelements); + current = ndx; + + for (;;) { + h_curr = &cc->heap[current]; + h_par = &cc->heap[parent]; + + /* + * Our expiration time later than your parent. + */ + if (h_curr->time >= h_par->time) + break; + /* + * Otherwhise we need to switch and continue. + */ + h_tmp = cc->heap[parent]; + cc->heap[parent] = *h_curr; + cc->heap[current] = h_tmp; + /* + * If that's the root, we're done. + */ + if (parent == 0) + break; + + current = parent; + parent = CC_HEAP_PARENT(current); + } +done: + mtx_unlock_spin(&cc->cc_lock); +} + static void +callout_heap_remove(struct callout *c, struct callout_cpu *cc) +{ + int ndx; + + mtx_lock_spin(&cc->cc_lock); + + for (i = 0; i < cc->heap_nelements; ++i) { + if((cc->heap[i]).c == c) + break; + } + + if (i == heap_nelements) + panic("Removing a not-existing callout"); + + cc->nelements = cc->nelements--; + + if(nelements == 0) + return; + + mtx_unlock_spin(&cc->cc_lock); +} + +static void callout_cc_add(struct callout *c, struct callout_cpu *cc, int to_ticks, void (*func)(void *), void *arg, int cpu) { @@ -427,8 +497,7 @@ c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING); c->c_func = func; c->c_time = ticks + to_ticks; - TAILQ_INSERT_TAIL(&cc->cc_callwheel[c->c_time & callwheelmask], - c, c_links.tqe); + callout_heap_add(c, cpu); if ((c->c_time - cc->cc_firsttick) < 0 && callout_new_inserted != NULL) { cc->cc_firsttick = c->c_time; @@ -1024,10 +1093,8 @@ CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p", c, c->c_func, c->c_arg); - TAILQ_REMOVE(&cc->cc_callwheel[c->c_time & callwheelmask], c, - c_links.tqe); + callout_heap_remove(c, cc); callout_cc_del(c, cc); - CC_UNLOCK(cc); return (1); }