Index: tree/sys/sys/_callout.h =================================================================== --- tree/sys/sys/_callout.h (revision 235056) +++ tree/sys/sys/_callout.h (working copy) @@ -50,7 +50,8 @@ SLIST_ENTRY(callout) sle; TAILQ_ENTRY(callout) tqe; } c_links; - int c_time; /* ticks to the event */ + TAILQ_ENTRY(callout) c_staiter; + struct bintime c_time; /* ticks to the event */ void *c_arg; /* function argument */ void (*c_func)(void *); /* function to call */ struct lock_object *c_lock; /* lock to handle */ Index: tree/sys/kern/kern_timeout.c =================================================================== --- tree/sys/kern/kern_timeout.c (revision 235056) +++ tree/sys/kern/kern_timeout.c (working copy) @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -94,10 +93,10 @@ */ struct cc_mig_ent { #ifdef SMP - void (*ce_migration_func)(void *); - void *ce_migration_arg; - int ce_migration_cpu; - int ce_migration_ticks; + void (*ce_migration_func)(void *); + void *ce_migration_arg; + int ce_migration_cpu; + struct bintime ce_migration_time; #endif }; @@ -127,18 +126,19 @@ struct callout *cc_next; struct callout *cc_curr; void *cc_cookie; - int cc_ticks; - int cc_softticks; + struct bintime cc_ticks; + struct bintime cc_softticks; int cc_cancel; int cc_waiting; - int cc_firsttick; + struct bintime cc_firsttick; + struct callout_tailq *cc_localexp; }; #ifdef SMP #define cc_migration_func cc_migrating_entity.ce_migration_func #define cc_migration_arg cc_migrating_entity.ce_migration_arg #define cc_migration_cpu cc_migrating_entity.ce_migration_cpu -#define cc_migration_ticks cc_migrating_entity.ce_migration_ticks +#define cc_migration_time cc_migrating_entity.ce_migration_time struct callout_cpu cc_cpu[MAXCPU]; #define CPUBLOCK MAXCPU @@ -160,19 +160,19 @@ /** * Locked by cc_lock: - * cc_curr - If a callout is in progress, it is curr_callout. - * If curr_callout is non-NULL, threads waiting in + * cc_curr - If a callout is in progress, it is cc_curr. + * If cc_curr is non-NULL, threads waiting in * callout_drain() will be woken up as soon as the * relevant callout completes. - * cc_cancel - Changing to 1 with both callout_lock and c_lock held + * cc_cancel - Changing to 1 with both callout_lock and cc_lock held * guarantees that the current callout will not run. * The softclock() function sets this to 0 before it - * drops callout_lock to acquire c_lock, and it calls + * drops callout_lock to acquire cc_lock, and it calls * the handler only if curr_cancelled is still 0 after - * c_lock is successfully acquired. + * cc_lock is successfully acquired. * cc_waiting - If a thread is waiting in callout_drain(), then * callout_wait is nonzero. Set only when - * curr_callout is non-NULL. + * cc_curr is non-NULL. */ /* @@ -184,7 +184,8 @@ #ifdef SMP cc->cc_migration_cpu = CPUBLOCK; - cc->cc_migration_ticks = 0; + cc->cc_migration_time.sec = 0; + cc->cc_migration_time.frac = 0; cc->cc_migration_func = NULL; cc->cc_migration_arg = NULL; #endif @@ -235,6 +236,7 @@ static void callout_cpu_init(struct callout_cpu *cc) + { struct callout *c; int i; @@ -332,12 +334,25 @@ SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL); +static int +get_bucket(struct bintime *bt) +{ + time_t sec; + uint64_t frac; + sec = bt->sec; + frac = bt->frac; + return (int) (((sec<<10)+(frac>>54)) & callwheelmask); +} + void callout_tick(void) { + struct callout tmp; struct callout_cpu *cc; + struct bintime bt; int need_softclock; int bucket; + int first_bucket; /* * Process callouts at a very low cpu priority, so we don't keep the @@ -346,12 +361,18 @@ need_softclock = 0; cc = CC_SELF(); mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); - cc->cc_firsttick = cc->cc_ticks = ticks; - for (; (cc->cc_softticks - cc->cc_ticks) <= 0; cc->cc_softticks++) { - bucket = cc->cc_softticks & callwheelmask; - if (!TAILQ_EMPTY(&cc->cc_callwheel[bucket])) { - need_softclock = 1; - break; + getbintime(&bt); + cc->cc_firsttick = cc->cc_ticks = bt; + first_bucket = get_bucket(cc->cc_ticks); + bucket = get_bucket(cc->softticks); + for (; first_bucket != bucket; ((bucket++) & callwheelsize)) { + TAILQ_FOREACH(tmp, cc->cc_callwheel[bucket], c_staiter) { + if (bintime_cmp(tmp.c_time,cc->cc_firsttick, <=)) { + TAILQ_INSERT(tmp,cc->localexp,c_staiter); + TAILQ_REMOVE(tmp,cc->cc_callwheel[bucket],c_links.tque); + need_softclock = 1; + break; + } } } mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); @@ -363,29 +384,32 @@ swi_sched(cc->cc_cookie, 0); } -int +struct bintime callout_tickstofirst(int limit) { struct callout_cpu *cc; struct callout *c; struct callout_tailq *sc; - int curticks; - int skip = 1; + struct bintime curticks; + struct bintime skip; + int now; cc = CC_SELF(); mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); curticks = cc->cc_ticks; - while( skip < ncallout && skip < limit ) { - sc = &cc->cc_callwheel[ (curticks+skip) & callwheelmask ]; + now = get_bucket(curticks); + while( bintime_cmp(skip, ncallout, <) && bintime_cmp(skip, limit, <)) { + sc = &cc->cc_callwheel[(now+skip) & callwheelmask ]; /* search scanning ticks */ TAILQ_FOREACH( c, sc, c_links.tqe ){ - if (c->c_time - curticks <= ncallout) + if (bintime_cmp(bintime_sub(c->c_time, curticks), ncallout, <=)) { + skip = bintime_sub(c->c_time, curticks); goto out; + } } - skip++; } out: - cc->cc_firsttick = curticks + skip; + cc->cc_firsttick = bintime_add(curticks, skip); mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); return (skip); } @@ -415,25 +439,37 @@ } static void -callout_cc_add(struct callout *c, struct callout_cpu *cc, int to_ticks, - void (*func)(void *), void *arg, int cpu) +callout_cc_add(struct callout *c, struct callout_cpu *cc, + struct bintime to_bintime, void (*func)(void *), void *arg, int cpu) { + struct bintime bt; + int bucket; CC_LOCK_ASSERT(cc); - if (to_ticks <= 0) - to_ticks = 1; + if (bintime_cmp(to_bintime, cc->cc_softticks, <)) { + to_bintime = cc->cc_softticks; + } c->c_arg = arg; c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING); c->c_func = func; - c->c_time = ticks + to_ticks; - TAILQ_INSERT_TAIL(&cc->cc_callwheel[c->c_time & callwheelmask], + getbintime(&bt); + c->c_time = to_bintime; + bucket = get_bucket(c->c_time); + TAILQ_INSERT_TAIL(&cc->cc_callwheel[bucket], c, c_links.tqe); - if ((c->c_time - cc->cc_firsttick) < 0 && + + /* + * cc->cc_firsttick keeps track of the time at which the + * nearest event in the future should be fired. + * We don't need to call callout_new_inserted if the time + * of the callout we're inserting is greater than this value. + */ + if (bintime_cmp(c->c_time - cc->cc_firsttick, 0, <) && callout_new_inserted != NULL) { cc->cc_firsttick = c->c_time; (*callout_new_inserted)(cpu, - to_ticks + (ticks - cc->cc_ticks)); + bintime_add(to_ticks, bintime_sub(ticks, cc->cc_ticks))); } } @@ -462,7 +498,8 @@ struct callout_cpu *new_cc; void (*new_func)(void *); void *new_arg; - int new_cpu, new_ticks; + int new_cpu; + struct bintime new_time; #endif #ifdef DIAGNOSTIC struct bintime bt1, bt2; @@ -493,7 +530,7 @@ */ if (cc->cc_cancel) { class->lc_unlock(c_lock); - goto skip; + oto skip; } /* The callout cannot be stopped now. */ cc->cc_cancel = 1; @@ -574,7 +611,7 @@ * migration just perform it now. */ new_cpu = cc->cc_migration_cpu; - new_ticks = cc->cc_migration_ticks; + new_time = cc->cc_migration_time; new_func = cc->cc_migration_func; new_arg = cc->cc_migration_arg; cc_cme_cleanup(cc); @@ -598,7 +635,7 @@ * is not easy. */ new_cc = callout_cpu_switch(c, cc, new_cpu); - callout_cc_add(c, new_cc, new_ticks, new_func, new_arg, + callout_cc_add(c, new_cc, new_time, new_func, new_arg, new_cpu); CC_UNLOCK(new_cc); CC_LOCK(cc); @@ -633,10 +670,10 @@ { struct callout_cpu *cc; struct callout *c; + struct callout tmp; struct callout_tailq *bucket; - int curticks; + struct bintime curticks; int steps; /* #steps since we last allowed interrupts */ - int depth; int mpcalls; int lockcalls; int gcalls; @@ -652,37 +689,28 @@ steps = 0; cc = (struct callout_cpu *)arg; CC_LOCK(cc); - while (cc->cc_softticks - 1 != cc->cc_ticks) { - /* - * cc_softticks may be modified by hard clock, so cache - * it while we work on a given bucket. - */ - curticks = cc->cc_softticks; - cc->cc_softticks++; - bucket = &cc->cc_callwheel[curticks & callwheelmask]; - c = TAILQ_FIRST(bucket); - while (c != NULL) { - depth++; - if (c->c_time != curticks) { - c = TAILQ_NEXT(c, c_links.tqe); - ++steps; - if (steps >= MAX_SOFTCLOCK_STEPS) { - cc->cc_next = c; - /* Give interrupts a chance. */ - CC_UNLOCK(cc); - ; /* nothing */ - CC_LOCK(cc); - c = cc->cc_next; - steps = 0; - } - } else { - TAILQ_REMOVE(bucket, c, c_links.tqe); - c = softclock_call_cc(c, cc, &mpcalls, - &lockcalls, &gcalls); - steps = 0; - } + + + c = TAILQ_FIRST(cc->cc_localexp); + while (c != NULL) { + ++steps; + if (steps >= MAX_SOFTCLOCK_STEPS) { + cc->cc_next = c; + /* Give interrupts a chance. */ + CC_UNLOCK(cc); + ; /* nothing */ + CC_LOCK(cc); + c = cc->next; + steps = 0; } + else { + TAILQ_REMOVE(cc->cc_localexp, c, c_staiter); + c = softclock_call_cc(c, cc, &mpcalls, + &lockcalls, &gcalls); + steps = 0; + } } + avg_depth += (depth * 1000 - avg_depth) >> 8; avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8; avg_lockcalls += (lockcalls * 1000 - avg_lockcalls) >> 8; @@ -1015,7 +1043,7 @@ c, c->c_func, c->c_arg); CC_UNLOCK(cc); KASSERT(!sq_locked, ("sleepqueue chain still locked")); - return (0); + +return (0); } if (sq_locked) sleepq_release(&cc->cc_waiting); Index: tree/sys/kern/kern_clocksource.c =================================================================== --- tree/sys/kern/kern_clocksource.c (revision 235056) +++ tree/sys/kern/kern_clocksource.c (working copy) @@ -269,7 +269,7 @@ { struct bintime tmp; struct pcpu_state *state; - int skip; + struct bintime skip; state = DPCPU_PTR(timerstate); /* Handle hardclock() events. */ @@ -279,12 +279,16 @@ skip = idle ? 4 : (stathz / 2); if (curcpu == CPU_FIRST() && tc_min_ticktock_freq > skip) skip = tc_min_ticktock_freq; - skip = callout_tickstofirst(hz / skip) - 1; CTR2(KTR_SPARE2, "skip at %d: %d", curcpu, skip); tmp = hardperiod; bintime_mul(&tmp, skip); bintime_add(event, &tmp); } + + tmp = callout_tickstofirst(hz/4); + if (bintime_cmp(event, &tmp, >)) + *event = tmp; + if (!idle) { /* If CPU is active - handle other types of events. */ if (bintime_cmp(event, &state->nextstat, >)) *event = state->nextstat;