Index: sys/kern/kern_tc.c =================================================================== --- sys/kern/kern_tc.c (revision 246071) +++ sys/kern/kern_tc.c (working copy) @@ -122,7 +122,10 @@ SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnin struct bintime bt_timethreshold; struct bintime bt_tickthreshold; +dur_t dur_timethreshold; +dur_t dur_tickthreshold; struct bintime tc_tick_bt; +dur_t tc_tick_dur; int tc_timeexp; int tc_timepercentage = TC_DEFAULTPERC; TUNABLE_INT("kern.timecounter.alloweddeviation", &tc_timepercentage); @@ -1736,6 +1739,8 @@ tc_adjprecision(void) bt_timethreshold.frac = ~(uint64_t)0; bt_tickthreshold = bt_timethreshold; } + dur_timethreshold = bintime_shrink(bt_timethreshold); + dur_tickthreshold = bintime_shrink(bt_tickthreshold); } static int @@ -1772,8 +1777,10 @@ inittimecounter(void *dummy) tc_tick = 1; tc_adjprecision(); FREQ2BT(hz, &tick_bt); + tick_dur = bintime_shrink(tick_bt); tick_rate = hz / tc_tick; FREQ2BT(tick_rate, &tc_tick_bt); + tc_tick_dur = bintime_shrink(tc_tick_bt); p = (tc_tick * 1000000) / hz; printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); Index: sys/kern/kern_timeout.c =================================================================== --- sys/kern/kern_timeout.c (revision 246035) +++ sys/kern/kern_timeout.c (working copy) @@ -121,7 +121,7 @@ struct cc_exec { void (*ce_migration_func)(void *); void *ce_migration_arg; int ce_migration_cpu; - struct bintime ce_migration_time; + dur_t ce_migration_time; #endif int cc_cancel; int cc_waiting; @@ -138,8 +138,8 @@ struct callout_cpu { struct callout_tailq *cc_callwheel; struct callout_tailq cc_expireq; struct callout_list cc_callfree; - struct bintime cc_firstevent; - struct bintime cc_lastscan; + dur_t cc_firstevent; + dur_t cc_lastscan; void *cc_cookie; }; @@ -217,7 +217,7 @@ cc_cme_cleanup(struct callout_cpu *cc, int direct) cc->cc_exec_entity[direct].cc_waiting = 0; #ifdef SMP cc->cc_exec_entity[direct].ce_migration_cpu = CPUBLOCK; - bintime_clear(&cc->cc_exec_entity[direct].ce_migration_time); + cc->cc_exec_entity[direct].ce_migration_time = 0; cc->cc_exec_entity[direct].ce_migration_func = NULL; cc->cc_exec_entity[direct].ce_migration_arg = NULL; #endif @@ -368,30 +368,29 @@ SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER #define CC_HASH_SHIFT 10 static inline int -callout_hash(struct bintime *bt) +callout_hash(dur_t dur) { - - return (int) ((bt->sec << CC_HASH_SHIFT) + - (bt->frac >> (64 - CC_HASH_SHIFT))); + + return (int)(dur >> (32 - CC_HASH_SHIFT)); } static inline int -get_bucket(struct bintime *bt) +callout_get_bucket(dur_t dur) { - return callout_hash(bt) & callwheelmask; + return callout_hash(dur) & callwheelmask; } void callout_process(struct bintime *now) { - struct bintime first, last, max, tmp_max; + struct bintime first_bt, last_bt; struct callout *tmp, *tmpn; struct callout_cpu *cc; struct callout_tailq *sc; - uint64_t lookahead; - int depth_dir, firstb, mpcalls_dir, lastb, nowb, lockcalls_dir, - need_softclock, exit_allowed, exit_wanted; + dur_t first, last, max, now2, tmp_max; + int depth_dir, firstb, lastb, mpcalls_dir, nowb, lookahead, + lockcalls_dir, need_softclock, exit_allowed, exit_wanted; need_softclock = 0; depth_dir = 0; @@ -401,22 +400,22 @@ callout_process(struct bintime *now) mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); /* Compute the buckets of the last scan and present times. */ - firstb = callout_hash(&cc->cc_lastscan); - cc->cc_lastscan = *now; - nowb = callout_hash(now); + firstb = callout_hash(cc->cc_lastscan); + now2 = bintime_shrink(*now); + cc->cc_lastscan = now2; + nowb = callout_hash(now2); /* Compute the last bucket and minimum time of the bucket after it. */ if (nowb == firstb) - lookahead = 1LLU << 60; /* 1/16s */ + lookahead = 1 << 29; /* 1/16s */ else if (nowb - firstb == 1) - lookahead = 1LLU << 61; /* 1/8s */ + lookahead = 1 << 30; /* 1/8s */ else - lookahead = 1LLU << 63; /* 1/2s */ - first = last = *now; - bintime_addx(&first, lookahead / 2); - bintime_addx(&last, lookahead); - last.frac &= (0xffffffffffffffffLLU << (64 - CC_HASH_SHIFT)); - lastb = callout_hash(&last) - 1; + lookahead = 1 << 31; /* 1/2s */ + first = last = now2; + first += (lookahead / 2); + last += lookahead; + lastb = callout_hash(last) - 1; max = last; /* @@ -438,7 +437,7 @@ callout_process(struct bintime *now) tmp = TAILQ_FIRST(sc); while (tmp != NULL) { /* Run the callout if present time within allowed. */ - if (bintime_cmp(&tmp->c_time, now, <=)) { + if (tmp->c_time <= now2) { /* * Consumer told us the callout may be run * directly from hardware interrupt context. @@ -464,22 +463,22 @@ callout_process(struct bintime *now) continue; } /* Skip events from distant future. */ - if (bintime_cmp(&tmp->c_time, &max, >=)) + if (tmp->c_time >= max) goto next; /* * Event minimal time is bigger than present maximal * time, so it cannot be aggregated. */ - if (bintime_cmp(&tmp->c_time, &last, >)) { + if (tmp->c_time > last) { exit_wanted = 1; goto next; } /* Update first and last time, respecting this event. */ - if (bintime_cmp(&tmp->c_time, &first, <)) + if (tmp->c_time < first) first = tmp->c_time; tmp_max = tmp->c_time; - bintime_add(&tmp_max, &tmp->c_precision); - if (bintime_cmp(&tmp_max, &last, <)) + tmp_max += tmp->c_precision; + if (tmp_max < last) last = tmp_max; next: tmp = TAILQ_NEXT(tmp, c_links.tqe); @@ -499,8 +498,10 @@ next: firstb = (firstb + 1) & callwheelmask; } cc->cc_exec_next_dir = NULL; + last_bt = bintime_expand(last); + first_bt = bintime_expand(first); if (callout_new_inserted != NULL) - (*callout_new_inserted)(curcpu, last, first); + (*callout_new_inserted)(curcpu, last_bt, first_bt); cc->cc_firstevent = last; #ifdef CALLOUT_PROFILING avg_depth_dir += (depth_dir * 1000 - avg_depth_dir) >> 8; @@ -542,39 +543,40 @@ callout_lock(struct callout *c) static void callout_cc_add(struct callout *c, struct callout_cpu *cc, - struct bintime to_bintime, struct bintime precision, void (*func)(void *), + dur_t to_time, dur_t precision, void (*func)(void *), void *arg, int cpu, int flags) { - struct bintime last; + dur_t last; + struct bintime ctime, prec; int bucket; CC_LOCK_ASSERT(cc); - if (bintime_cmp(&to_bintime, &cc->cc_lastscan, <)) - to_bintime = cc->cc_lastscan; + if (to_time < cc->cc_lastscan) + to_time = cc->cc_lastscan; c->c_arg = arg; c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING); if (flags & C_DIRECT_EXEC) c->c_flags |= CALLOUT_DIRECT; c->c_flags &= ~CALLOUT_PROCESSED; c->c_func = func; - c->c_time = to_bintime; + c->c_time = to_time; c->c_precision = precision; - CTR4(KTR_CALLOUT, "precision set for %p: %d.%08x%08x", - c, c->c_precision.sec, (u_int) (c->c_precision.frac >> 32), - (u_int) (c->c_precision.frac & 0xffffffff)); - bucket = get_bucket(&c->c_time); + bucket = callout_get_bucket(c->c_time); + CTR3(KTR_CALLOUT, "precision set for %p: %d.%08x", + c, (int)(c->c_precision >> 32), + (u_int)(c->c_precision & 0xffffffff)); TAILQ_INSERT_TAIL(&cc->cc_callwheel[bucket], c, c_links.tqe); /* * Inform the eventtimers(4) subsystem there's a new callout * that has been inserted, but only if really required. */ - last = c->c_time; - bintime_add(&last, &c->c_precision); - if (callout_new_inserted != NULL && - (bintime_cmp(&last, &cc->cc_firstevent, <) || - !bintime_isset(&cc->cc_firstevent))) { + last = c->c_time + c->c_precision; + if (callout_new_inserted != NULL && ((last < cc->cc_firstevent) || + (cc->cc_firstevent == 0))) { cc->cc_firstevent = last; - (*callout_new_inserted)(cpu, last, c->c_time); + ctime = bintime_expand(c->c_time); + prec = bintime_expand(last); + (*callout_new_inserted)(cpu, prec, ctime); } } @@ -602,7 +604,7 @@ softclock_call_cc(struct callout *c, struct callou void (*new_func)(void *); void *new_arg; int flags, new_cpu; - struct bintime new_time; + dur_t new_time; #endif #ifdef DIAGNOSTIC struct bintime bt1, bt2; @@ -896,27 +898,32 @@ DPCPU_DECLARE(struct bintime, hardclocktime); * callout_deactivate() - marks the callout as having been serviced */ int -callout_reset_bt_on(struct callout *c, struct bintime bt, struct bintime pr, +callout_reset_bt_on(struct callout *c, struct bintime bt, struct bintime precis, void (*ftn)(void *), void *arg, int cpu, int flags) { + dur_t d_time, d_prec; + dur_t to_time, pr; struct bintime to_bt, pr1; struct callout_cpu *cc; int bucket, cancelled, direct; + d_time = bintime_shrink(bt); + d_prec = bintime_shrink(precis); cancelled = 0; if (flags & C_ABSOLUTE) { - to_bt = bt; + to_time = d_time; } else { - if ((flags & C_HARDCLOCK) && bintime_cmp(&bt, &tick_bt, <)) + if ((flags & C_HARDCLOCK) && (d_time < tick_dur)) bt = tick_bt; if ((flags & C_HARDCLOCK) || #ifdef NO_EVENTTIMERS - bintime_cmp(&bt, &bt_timethreshold, >=)) { + d_time >= dur_timethreshold) { getbinuptime(&to_bt); + to_time = bintime_shrink(to_bt); /* Add safety belt for the case of hz > 1000. */ - bintime_addx(&to_bt, tc_tick_bt.frac - tick_bt.frac); + to_time += (tc_tick_dur - tick_dur); #else - bintime_cmp(&bt, &bt_tickthreshold, >=)) { + d_time >= dur_tickthreshold) { /* * Obtain the time of the last hardclock() call on * this CPU directly from the kern_clocksource.c. @@ -925,20 +932,24 @@ int */ spinlock_enter(); to_bt = DPCPU_GET(hardclocktime); + to_time = bintime_shrink(to_bt); spinlock_exit(); #endif if ((flags & C_HARDCLOCK) == 0) - bintime_addx(&to_bt, tick_bt.frac); - } else + to_time += tick_dur; + } else { binuptime(&to_bt); - bintime_add(&to_bt, &bt); + to_time = bintime_shrink(to_bt); + } + to_time += d_time; pr1 = bt; if (C_PRELGET(flags) < 0) bintime_shift(&pr1, -tc_timeexp); else bintime_shift(&pr1, -C_PRELGET(flags)); - if (bintime_cmp(&pr1, &pr, >)) - pr = pr1; + pr = bintime_shrink(pr1); + if (pr > d_prec) + d_prec = pr; } /* * Don't allow migration of pre-allocated callouts lest they @@ -975,7 +986,7 @@ int if (cc->cc_exec_next_dir == c) cc->cc_exec_next_dir = TAILQ_NEXT(c, c_links.tqe); - bucket = get_bucket(&c->c_time); + bucket = callout_get_bucket(c->c_time); TAILQ_REMOVE(&cc->cc_callwheel[bucket], c, c_links.tqe); } else @@ -994,14 +1005,14 @@ int if (cc->cc_exec_entity[direct].cc_curr == c) { cc->cc_exec_entity[direct].ce_migration_cpu = cpu; cc->cc_exec_entity[direct].ce_migration_time - = to_bt; + = to_time; cc->cc_exec_entity[direct].ce_migration_func = ftn; cc->cc_exec_entity[direct].ce_migration_arg = arg; c->c_flags |= CALLOUT_DFRMIGRATION; CTR6(KTR_CALLOUT, "migration of %p func %p arg %p in %d.%08x to %u deferred", - c, c->c_func, c->c_arg, (int)(to_bt.sec), - (u_int)(to_bt.frac >> 32), cpu); + c, c->c_func, c->c_arg, (int)(to_time >> 32), + (u_int)(to_time & 0xffffffff), cpu); CC_UNLOCK(cc); return (cancelled); } @@ -1009,10 +1020,10 @@ int } #endif - callout_cc_add(c, cc, to_bt, pr, ftn, arg, cpu, flags); + callout_cc_add(c, cc, to_time, pr, ftn, arg, cpu, flags); CTR6(KTR_CALLOUT, "%sscheduled %p func %p arg %p in %d.%08x", - cancelled ? "re" : "", c, c->c_func, c->c_arg, (int)(to_bt.sec), - (u_int)(to_bt.frac >> 32)); + cancelled ? "re" : "", c, c->c_func, c->c_arg,(int)(to_time >> 32), + (u_int)(to_time & 0xffffffff)); CC_UNLOCK(cc); return (cancelled); @@ -1197,7 +1208,7 @@ again: if ((c->c_flags & CALLOUT_PROCESSED) == 0) { if (cc->cc_exec_next_dir == c) cc->cc_exec_next_dir = TAILQ_NEXT(c, c_links.tqe); - bucket = get_bucket(&c->c_time); + bucket = callout_get_bucket(c->c_time); TAILQ_REMOVE(&cc->cc_callwheel[bucket], c, c_links.tqe); } else Index: sys/kern/subr_param.c =================================================================== --- sys/kern/subr_param.c (revision 246071) +++ sys/kern/subr_param.c (working copy) @@ -84,7 +84,8 @@ static int sysctl_kern_vm_guest(SYSCTL_HANDLER_ARG int hz; /* system clock's frequency */ int tick; /* usec per tick (1000000 / hz) */ struct bintime tick_bt; /* bintime per tick (1s / hz) */ -struct bintime zero_bt = { 0, 0 }; /* bintime per tick (1s / hz) */ +struct bintime zero_bt = { 0, 0 }; /* bintime per tick (1s / hz) */ +dur_t tick_dur; int maxusers; /* base tunable */ int maxproc; /* maximum # of processes */ int maxprocperuid; /* max # of procs per user */ @@ -224,6 +225,7 @@ init_param1(void) hz = vm_guest > VM_GUEST_NO ? HZ_VM : HZ; tick = 1000000 / hz; FREQ2BT(hz, &tick_bt); + tick_dur = bintime_shrink(tick_bt); #ifdef VM_SWZONE_SIZE_MAX maxswzone = VM_SWZONE_SIZE_MAX; Index: sys/netinet/tcp_timer.c =================================================================== --- sys/netinet/tcp_timer.c (revision 246071) +++ sys/netinet/tcp_timer.c (working copy) @@ -719,9 +719,11 @@ tcp_timer_active(struct tcpcb *tp, int timer_type) #define ticks_to_msecs(t) (1000*(t) / hz) static int -delta_bintime_in_msecs(struct bintime bt, struct bintime now) +delta_bintime_in_msecs(dur_t dt, struct bintime now) { + struct bintime bt; + bt = bintime_expand(dt); bintime_sub(&bt, &now); return (((uint64_t)1000 * (uint64_t)(bt.frac >> 32)) >> 32) + (bt.sec * 1000); Index: sys/sys/_callout.h =================================================================== --- sys/sys/_callout.h (revision 246035) +++ sys/sys/_callout.h (working copy) @@ -51,8 +51,8 @@ struct callout { SLIST_ENTRY(callout) sle; TAILQ_ENTRY(callout) tqe; } c_links; - struct bintime c_time; /* ticks to the event */ - struct bintime c_precision; /* delta allowed wrt opt */ + dur_t c_time; /* ticks to the event */ + dur_t c_precision; /* delta allowed wrt opt */ void *c_arg; /* function argument */ void (*c_func)(void *); /* function to call */ struct lock_object *c_lock; /* lock to handle */ Index: sys/sys/systm.h =================================================================== --- sys/sys/systm.h (revision 246035) +++ sys/sys/systm.h (working copy) @@ -345,7 +345,7 @@ static __inline void splx(intrmask_t ipl __unused * less often. */ int _sleep(void *chan, struct lock_object *lock, int pri, const char *wmesg, - struct bintime bt, struct bintime pr, int flags) __nonnull(1); + struct bintime bt, struct bintime pr, int flags) __nonnull(1); #define msleep(chan, mtx, pri, wmesg, timo) \ _sleep((chan), &(mtx)->lock_object, (pri), (wmesg), \ ticks2bintime(timo), zero_bt, C_HARDCLOCK) Index: sys/sys/time.h =================================================================== --- sys/sys/time.h (revision 246071) +++ sys/sys/time.h (working copy) @@ -50,6 +50,19 @@ struct timezone { #define DST_CAN 6 /* Canada */ #if __BSD_VISIBLE +#ifdef _KERNEL + +typedef int64_t dur_t; /* signed for bug catching */ +#define DURSEC ((dur_t)1 << 32) +#define DURMIN (DURSEC * 60) +#define DURMSEC (DURSEC / 1000) +#define DURUSEC (DURSEC / 1000000) +#define DURNSEC (DURSEC / 1000000000) + +#endif /* _KERNEL */ +#endif /* __BSD_VISIBLE */ + +#if __BSD_VISIBLE struct bintime { time_t sec; uint64_t frac; @@ -125,8 +138,9 @@ bintime_shift(struct bintime *bt, int exp) ((a)->sec cmp (b)->sec)) #ifdef _KERNEL + extern struct bintime tick_bt; -extern struct bintime zero_bt; +extern dur_t tick_dur; static __inline struct bintime ticks2bintime(u_int ticks) @@ -140,8 +154,42 @@ ticks2bintime(u_int ticks) bt.frac = (p2 << 32) | (p1 & 0xffffffff); return (bt); } -#endif +static __inline dur_t +bintime_shrink(struct bintime bt) +{ + dur_t res; + int64_t sec, frac; + + sec = (bt.sec & 0xffffffff); + frac = ((bt.frac >> 32) & 0xffffffff); + res = (sec << 32) | (frac); + return (res); +} + +static __inline struct bintime +bintime_expand(dur_t dur) +{ + struct bintime bt; + + bt.sec = dur >> 32; + bt.frac = dur & 0xffffffff; + return bt; +} + +static __inline dur_t +ticks2durt(u_int ticks) +{ + struct bintime bt; + dur_t res; + + bt = ticks2bintime(ticks); + res = bintime_shrink(bt); + return res; +} + +#endif /* KERNEL */ + /*- * Background information: * @@ -324,11 +372,14 @@ extern volatile time_t time_second; extern volatile time_t time_uptime; extern struct bintime boottimebin; extern struct bintime tc_tick_bt; +extern struct bintime zero_bt; extern struct timeval boottime; extern int tc_timeexp; extern int tc_timepercentage; extern struct bintime bt_timethreshold; extern struct bintime bt_tickthreshold; +extern dur_t dur_timethreshold; +extern dur_t dur_tickthreshold; /* * Functions for looking at our clock: [get]{bin,nano,micro}[up]time()