Index: sys/x86/isa/clock.c =================================================================== --- sys/x86/isa/clock.c (revision 219697) +++ sys/x86/isa/clock.c (working copy) @@ -62,6 +62,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -245,40 +246,85 @@ getit(void) return ((high << 8) | low); } -static __inline void -delay_tsc(int n) +#ifdef __i386__ +static __inline uint64_t +atomic_fetch_quad_i386(uint64_t *p) { - uint64_t start, end, now; + uint64_t v; - sched_pin(); - start = rdtsc(); - end = start + (tsc_freq * n) / 1000000; - do { - cpu_spinwait(); - now = rdtsc(); - } while (now < end || (now > start && end < start)); - sched_unpin(); + /* i486 does not support SMP. */ + __asm __volatile( + " pushfl ; " + " cli ; " + " movl (%1), %%eax ; " + " movl 4(%1), %%edx ; " + " popfl" + : "=&A" (v) : "r" (p)); + return (v); } -static __inline void -delay_timecounter(struct timecounter *tc, int n) +static __inline uint64_t +atomic_fetch_quad_i586(uint64_t *p) { - uint64_t end, now; + uint64_t v; + + __asm __volatile( + " movl %%ebx,%%eax ; " + " movl %%ecx,%%edx ; " + " " MPLOCKED " cmpxchg8b (%1)" + : "=&A" (v) : "r" (p) : "cc"); + return (v); +} + +static __inline uint64_t +_fetch_frequency(uint64_t *p) +{ + + if (cpu_class == CPUCLASS_486) + return (atomic_fetch_quad_i386(p)); + return (atomic_fetch_quad_i586(p)); +} +#else +#define _fetch_frequency(p) (*(p)) +#endif + +static __inline int +_delay(int n) +{ + struct timecounter *tc; + uint64_t end, freq, now; u_int last, mask, u; + int use_tsc; - mask = tc->tc_counter_mask; - last = tc->tc_get_timecount(tc) & mask; - end = tc->tc_frequency * n / 1000000; + tc = timecounter; + freq = _fetch_frequency(&tsc_freq); + use_tsc = tsc_is_invariant && freq != 0; + if (use_tsc) { + mask = ~0u; + sched_pin(); + last = rdtsc(); + } else { + if (tc->tc_quality <= 0) + return (0); + freq = _fetch_frequency(&tc->tc_frequency); + mask = tc->tc_counter_mask; + last = tc->tc_get_timecount(tc); + } + last &= mask; + end = freq * n / 1000000; now = 0; do { cpu_spinwait(); - u = tc->tc_get_timecount(tc) & mask; + u = (use_tsc ? rdtsc() : tc->tc_get_timecount(tc)) & mask; if (u < last) now += mask - last + u + 1; else now += u - last; last = u; } while (now < end); + if (use_tsc) + sched_unpin(); + return (1); } /* @@ -289,7 +335,6 @@ getit(void) void DELAY(int n) { - struct timecounter *tc; int delta, prev_tick, tick, ticks_left; #ifdef DELAYDEBUG @@ -298,15 +343,8 @@ DELAY(int n) static int state = 0; #endif - if (tsc_freq != 0) { - delay_tsc(n); + if (_delay(n)) return; - } - tc = timecounter; - if (tc->tc_quality > 0) { - delay_timecounter(tc, n); - return; - } #ifdef DELAYDEBUG if (state == 0) { state = 1;