diff -ruNp --exclude compile sys.prev2/conf/files.powerpc sys/conf/files.powerpc --- sys.prev2/conf/files.powerpc 2010-09-03 02:15:40.000000000 +0300 +++ sys/conf/files.powerpc 2010-09-05 09:05:35.000000000 +0300 @@ -52,6 +52,7 @@ dev/syscons/scvtb.c optional sc dev/tsec/if_tsec.c optional tsec dev/tsec/if_tsec_fdt.c optional tsec fdt dev/uart/uart_cpu_powerpc.c optional uart aim +kern/kern_clocksource.c standard kern/syscalls.c optional ktr libkern/ashldi3.c optional powerpc libkern/ashrdi3.c optional powerpc diff -ruNp --exclude compile sys.prev2/kern/kern_et.c sys/kern/kern_et.c diff -ruNp --exclude compile sys.prev2/powerpc/aim/clock.c sys/powerpc/aim/clock.c --- sys.prev2/powerpc/aim/clock.c 2010-09-03 02:15:35.000000000 +0300 +++ sys/powerpc/aim/clock.c 2010-09-05 18:08:22.000000000 +0300 @@ -65,107 +65,195 @@ __FBSDID("$FreeBSD: head/sys/powerpc/aim #include #include #include +#include #include #include #include #include +#include #include #include /* * Initially we assume a processor with a bus frequency of 12.5 MHz. */ -u_long ns_per_tick = 80; +static int initialized = 0; +static u_long ns_per_tick = 80; static u_long ticks_per_sec = 12500000; -static long ticks_per_intr; +static u_long *decr_counts[MAXCPU]; +static int decr_et_start(struct eventtimer *et, + struct bintime *first, struct bintime *period); +static int decr_et_stop(struct eventtimer *et); static timecounter_get_t decr_get_timecount; -static struct timecounter decr_timecounter = { +struct decr_state { + int mode; /* 0 - off, 1 - periodic, 2 - one-shot. */ + int32_t div; /* Periodic divisor. */ +}; +static DPCPU_DEFINE(struct decr_state, decr_state); + +static struct eventtimer decr_et; +static struct timecounter decr_tc = { decr_get_timecount, /* get_timecount */ 0, /* no poll_pps */ ~0u, /* counter_mask */ 0, /* frequency */ - "decrementer" /* name */ + "timebase" /* name */ }; +/* + * Decrementor interrupt handler. + */ void decr_intr(struct trapframe *frame) { - int32_t tick, nticks; + struct decr_state *s = DPCPU_PTR(decr_state); + int nticks = 0; + int32_t val; - /* - * Check whether we are initialized. - */ - if (!ticks_per_intr) + if (!initialized) return; - /* - * Based on the actual time delay since the last decrementer reload, - * we arrange for earlier interrupt next time. - */ - __asm ("mfdec %0" : "=r"(tick)); - for (nticks = 0; tick < 0; nticks++) - tick += ticks_per_intr; - mtdec(tick); + (*decr_counts[curcpu])++; + + if (s->mode == 1) { + /* + * Based on the actual time delay since the last decrementer + * reload, we arrange for earlier interrupt next time. + */ + __asm ("mfdec %0" : "=r"(val)); + while (val < 0) { + val += s->div; + nticks++; + } + mtdec(val); + } else if (s->mode == 2) { + nticks = 1; + decr_et_stop(NULL); + } while (nticks-- > 0) { - if (PCPU_GET(cpuid) == 0) - hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); - else - hardclock_cpu(TRAPF_USERMODE(frame)); - - statclock(TRAPF_USERMODE(frame)); - if (profprocs != 0) - profclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); + if (decr_et.et_active) + decr_et.et_event_cb(&decr_et, decr_et.et_arg); } } +/* + * BSP early initialization. + */ void decr_init(void) { struct cpuref cpu; - register_t msr; + char buf[32]; /* * Check the BSP's timebase frequency. Sometimes we can't find the BSP, so fall * back to the first CPU in this case. */ - if (platform_smp_get_bsp(&cpu) != 0) platform_smp_first_cpu(&cpu); - ticks_per_sec = platform_timebase_freq(&cpu); - - msr = mfmsr(); - mtmsr(msr & ~PSL_EE); - ns_per_tick = 1000000000 / ticks_per_sec; - ticks_per_intr = ticks_per_sec / hz; - mtdec(ticks_per_intr); set_cputicker(mftb, ticks_per_sec, 0); - - mtmsr(msr); + snprintf(buf, sizeof(buf), "cpu%d:decrementer", curcpu); + intrcnt_add(buf, &decr_counts[curcpu]); + decr_et_stop(NULL); + initialized = 1; } #ifdef SMP +/* + * AP early initialization. + */ void decr_ap_init(void) { + char buf[32]; + snprintf(buf, sizeof(buf), "cpu%d:decrementer", curcpu); + intrcnt_add(buf, &decr_counts[curcpu]); + decr_et_stop(NULL); } #endif +/* + * Final initialization. + */ void decr_tc_init(void) { - decr_timecounter.tc_frequency = ticks_per_sec; - tc_init(&decr_timecounter); + + decr_tc.tc_frequency = ticks_per_sec; + tc_init(&decr_tc); + decr_et.et_name = "decrementer"; + decr_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | + ET_FLAGS_PERCPU; + decr_et.et_quality = 1000; + decr_et.et_frequency = ticks_per_sec; + decr_et.et_min_period.sec = 0; + decr_et.et_min_period.frac = + ((0x00000002LLU << 32) / ticks_per_sec) << 32; + decr_et.et_max_period.sec = 0x7fffffffLLU / ticks_per_sec; + decr_et.et_max_period.frac = + ((0x7fffffffLLU << 32) / ticks_per_sec) << 32; + decr_et.et_start = decr_et_start; + decr_et.et_stop = decr_et_stop; + decr_et.et_priv = NULL; + et_register(&decr_et); +} + +/* + * Event timer start method. + */ +static int +decr_et_start(struct eventtimer *et, + struct bintime *first, struct bintime *period) +{ + struct decr_state *s = DPCPU_PTR(decr_state); + uint32_t fdiv; + + if (period != NULL) { + s->mode = 1; + s->div = (decr_et.et_frequency * (period->frac >> 32)) >> 32; + if (period->sec != 0) + s->div += decr_et.et_frequency * period->sec; + } else { + s->mode = 2; + s->div = 0x7fffffff; + } + if (first != NULL) { + fdiv = (decr_et.et_frequency * (first->frac >> 32)) >> 32; + if (first->sec != 0) + fdiv += decr_et.et_frequency * first->sec; + } else + fdiv = s->div; + + mtdec(fdiv); + return (0); +} + +/* + * Event timer stop method. + */ +static int +decr_et_stop(struct eventtimer *et) +{ + struct decr_state *s = DPCPU_PTR(decr_state); + + s->mode = 0; + s->div = 0x7fffffff; + mtdec(s->div); + return (0); } +/* + * Timecounter get method. + */ static unsigned decr_get_timecount(struct timecounter *tc) { @@ -189,17 +277,3 @@ DELAY(int n) tb = mftb(); } -/* - * Nothing to do. - */ -void -cpu_startprofclock(void) -{ - - /* Do nothing */ -} - -void -cpu_stopprofclock(void) -{ -} diff -ruNp --exclude compile sys.prev2/powerpc/aim/interrupt.c sys/powerpc/aim/interrupt.c --- sys.prev2/powerpc/aim/interrupt.c 2010-09-03 02:15:35.000000000 +0300 +++ sys/powerpc/aim/interrupt.c 2010-09-05 10:08:23.000000000 +0300 @@ -72,6 +72,7 @@ void powerpc_interrupt(struct trapframe *framep) { struct thread *td; + struct trapframe *oldframe; register_t ee; td = curthread; @@ -88,8 +89,11 @@ powerpc_interrupt(struct trapframe *fram case EXC_DECR: critical_enter(); atomic_add_int(&td->td_intr_nesting_level, 1); + oldframe = td->td_intr_frame; + td->td_intr_frame = framep; decr_intr(framep); - atomic_subtract_int(&td->td_intr_nesting_level, 1); + td->td_intr_frame = oldframe; + atomic_subtract_int(&td->td_intr_nesting_level, 1); critical_exit(); break; diff -ruNp --exclude compile sys.prev2/powerpc/aim/machdep.c sys/powerpc/aim/machdep.c --- sys.prev2/powerpc/aim/machdep.c 2010-09-03 02:15:35.000000000 +0300 +++ sys/powerpc/aim/machdep.c 2010-09-05 10:34:45.000000000 +0300 @@ -609,8 +609,7 @@ cpu_initclocks(void) { decr_tc_init(); - stathz = hz; - profhz = hz; + cpu_initclocks_bsp(); } /* diff -ruNp --exclude compile sys.prev2/powerpc/booke/clock.c sys/powerpc/booke/clock.c --- sys.prev2/powerpc/booke/clock.c 2010-09-03 02:15:34.000000000 +0300 +++ sys/powerpc/booke/clock.c 2010-09-05 18:07:57.000000000 +0300 @@ -63,12 +63,14 @@ __FBSDID("$FreeBSD: head/sys/powerpc/boo #include #include #include +#include #include #include +#include #include -#include #include +#include #include #include #include @@ -78,33 +80,46 @@ __FBSDID("$FreeBSD: head/sys/powerpc/boo /* * Initially we assume a processor with a bus frequency of 12.5 MHz. */ -u_int tickspending; -u_long ns_per_tick = 80; -static u_long ticks_per_sec = 12500000; -static long ticks_per_intr; +static int initialized = 0; +static u_long ns_per_tick = 80; +static u_long ticks_per_sec = 12500000; +static u_long *decr_counts[MAXCPU]; #define DIFF19041970 2082844800 +static int decr_et_start(struct eventtimer *et, + struct bintime *first, struct bintime *period); +static int decr_et_stop(struct eventtimer *et); static timecounter_get_t decr_get_timecount; +struct decr_state { + int mode; /* 0 - off, 1 - periodic, 2 - one-shot. */ + int32_t div; /* Periodic divisor. */ +}; +static DPCPU_DEFINE(struct decr_state, decr_state); + +static struct eventtimer decr_et; static struct timecounter decr_timecounter = { decr_get_timecount, /* get_timecount */ 0, /* no poll_pps */ ~0u, /* counter_mask */ 0, /* frequency */ - "decrementer" /* name */ + "timebase" /* name */ }; +/* + * Decrementor interrupt handler. + */ void decr_intr(struct trapframe *frame) { + struct decr_state *s = DPCPU_PTR(decr_state); - /* - * Check whether we are initialized. - */ - if (!ticks_per_intr) + if (!initialized) return; + (*decr_counts[curcpu])++; + /* * Interrupt handler must reset DIS to avoid getting another * interrupt once EE is enabled. @@ -113,14 +128,11 @@ decr_intr(struct trapframe *frame) CTR1(KTR_INTR, "%s: DEC interrupt", __func__); - if (PCPU_GET(cpuid) == 0) - hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); - else - hardclock_cpu(TRAPF_USERMODE(frame)); - - statclock(TRAPF_USERMODE(frame)); - if (profprocs != 0) - profclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); + if (s->mode == 2) + decr_et_stop(NULL); + + if (decr_et.et_active) + decr_et.et_event_cb(&decr_et, decr_et.et_arg); } void @@ -128,57 +140,129 @@ cpu_initclocks(void) { decr_tc_init(); - stathz = hz; - profhz = hz; + cpu_initclocks_bsp(); } +/* + * BSP early initialization. + */ void decr_init(void) { struct cpuref cpu; - unsigned int msr; + char buf[32]; if (platform_smp_get_bsp(&cpu) != 0) platform_smp_first_cpu(&cpu); ticks_per_sec = platform_timebase_freq(&cpu); - - msr = mfmsr(); - mtmsr(msr & ~(PSL_EE)); - ns_per_tick = 1000000000 / ticks_per_sec; - ticks_per_intr = ticks_per_sec / hz; - - mtdec(ticks_per_intr); - - mtspr(SPR_DECAR, ticks_per_intr); - mtspr(SPR_TCR, mfspr(SPR_TCR) | TCR_DIE | TCR_ARE); set_cputicker(mftb, ticks_per_sec, 0); - - mtmsr(msr); + snprintf(buf, sizeof(buf), "cpu%d:decrementer", curcpu); + intrcnt_add(buf, &decr_counts[curcpu]); + decr_et_stop(NULL); + initialized = 1; } #ifdef SMP +/* + * AP early initialization. + */ void decr_ap_init(void) { + char buf[32]; - /* Set auto-reload value and enable DEC interrupts in TCR */ - mtspr(SPR_DECAR, ticks_per_intr); - mtspr(SPR_TCR, mfspr(SPR_TCR) | TCR_DIE | TCR_ARE); - - CTR2(KTR_INTR, "%s: set TCR=%p", __func__, mfspr(SPR_TCR)); + snprintf(buf, sizeof(buf), "cpu%d:decrementer", curcpu); + intrcnt_add(buf, &decr_counts[curcpu]); + decr_et_stop(NULL); } #endif +/* + * Final initialization. + */ void decr_tc_init(void) { decr_timecounter.tc_frequency = ticks_per_sec; tc_init(&decr_timecounter); + decr_et.et_name = "decrementer"; + decr_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | + ET_FLAGS_PERCPU; + decr_et.et_quality = 1000; + decr_et.et_frequency = ticks_per_sec; + decr_et.et_min_period.sec = 0; + decr_et.et_min_period.frac = + ((0x00000002LLU << 32) / ticks_per_sec) << 32; + decr_et.et_max_period.sec = 0xfffffffeLLU / ticks_per_sec; + decr_et.et_max_period.frac = + ((0xfffffffeLLU << 32) / ticks_per_sec) << 32; + decr_et.et_start = decr_et_start; + decr_et.et_stop = decr_et_stop; + decr_et.et_priv = NULL; + et_register(&decr_et); } +/* + * Event timer start method. + */ +static int +decr_et_start(struct eventtimer *et, + struct bintime *first, struct bintime *period) +{ + struct decr_state *s = DPCPU_PTR(decr_state); + uint32_t fdiv, tcr; + + if (period != NULL) { + s->mode = 1; + s->div = (decr_et.et_frequency * (period->frac >> 32)) >> 32; + if (period->sec != 0) + s->div += decr_et.et_frequency * period->sec; + } else { + s->mode = 2; + s->div = 0xffffffff; + } + if (first != NULL) { + fdiv = (decr_et.et_frequency * (first->frac >> 32)) >> 32; + if (first->sec != 0) + fdiv += decr_et.et_frequency * first->sec; + } else + fdiv = s->div; + + tcr = mfspr(SPR_TCR); + tcr |= TCR_DIE; + if (s->mode == 1) { + mtspr(SPR_DECAR, s->div); + tcr |= TCR_ARE; + } else + tcr &= ~TCR_ARE; + mtdec(fdiv); + mtspr(SPR_TCR, tcr); + return (0); +} + +/* + * Event timer stop method. + */ +static int +decr_et_stop(struct eventtimer *et) +{ + struct decr_state *s = DPCPU_PTR(decr_state); + uint32_t tcr; + + s->mode = 0; + s->div = 0xffffffff; + tcr = mfspr(SPR_TCR); + tcr &= ~(TCR_DIE | TCR_ARE); + mtspr(SPR_TCR, tcr); + return (0); +} + +/* + * Timecounter get method. + */ static unsigned decr_get_timecount(struct timecounter *tc) { @@ -203,18 +287,3 @@ DELAY(int n) } while (now < end || (now > start && end < start)); } -/* - * Nothing to do. - */ -void -cpu_startprofclock(void) -{ - - /* Do nothing */ -} - -void -cpu_stopprofclock(void) -{ - -} diff -ruNp --exclude compile sys.prev2/powerpc/booke/interrupt.c sys/powerpc/booke/interrupt.c --- sys.prev2/powerpc/booke/interrupt.c 2010-09-03 02:15:34.000000000 +0300 +++ sys/powerpc/booke/interrupt.c 2010-09-05 18:00:56.000000000 +0300 @@ -116,11 +116,15 @@ void powerpc_decr_interrupt(struct trapframe *framep) { struct thread *td; + struct trapframe *oldframe; td = PCPU_GET(curthread); critical_enter(); atomic_add_int(&td->td_intr_nesting_level, 1); + oldframe = td->td_intr_frame; + td->td_intr_frame = framep; decr_intr(framep); + td->td_intr_frame = oldframe; atomic_subtract_int(&td->td_intr_nesting_level, 1); critical_exit(); framep->srr1 &= ~PSL_WE; diff -ruNp --exclude compile sys.prev2/powerpc/booke/platform_bare.c sys/powerpc/booke/platform_bare.c --- sys.prev2/powerpc/booke/platform_bare.c 2010-09-03 02:15:34.000000000 +0300 +++ sys/powerpc/booke/platform_bare.c 2010-09-05 00:15:30.000000000 +0300 @@ -82,7 +82,7 @@ static platform_method_t bare_methods[] PLATFORMMETHOD(platform_smp_get_bsp, bare_smp_get_bsp), PLATFORMMETHOD(platform_smp_start_cpu, bare_smp_start_cpu), - PLATFORMMETHOD(platform_reset, e500_reset); + PLATFORMMETHOD(platform_reset, e500_reset), { 0, 0 } }; diff -ruNp --exclude compile sys.prev2/powerpc/include/intr_machdep.h sys/powerpc/include/intr_machdep.h --- sys.prev2/powerpc/include/intr_machdep.h 2010-09-03 02:15:35.000000000 +0300 +++ sys/powerpc/include/intr_machdep.h 2010-09-04 23:57:40.000000000 +0300 @@ -48,6 +48,8 @@ struct trapframe; driver_filter_t powerpc_ipi_handler; +void intrcnt_add(const char *name, u_long **countp); + void powerpc_register_pic(device_t, u_int); int powerpc_ign_lookup(uint32_t pic_id); diff -ruNp --exclude compile sys.prev2/powerpc/include/md_var.h sys/powerpc/include/md_var.h --- sys.prev2/powerpc/include/md_var.h 2010-09-03 02:15:35.000000000 +0300 +++ sys/powerpc/include/md_var.h 2010-09-05 10:21:18.000000000 +0300 @@ -47,8 +47,6 @@ extern int busdma_swi_pending; extern vm_offset_t kstack0; extern vm_offset_t kstack0_phys; -extern u_long ns_per_tick; - extern int powerpc_pow_enabled; extern int cacheline_size; extern int hw_direct_map; diff -ruNp --exclude compile sys.prev2/powerpc/include/smp.h sys/powerpc/include/smp.h --- sys.prev2/powerpc/include/smp.h 2010-09-03 02:15:35.000000000 +0300 +++ sys/powerpc/include/smp.h 2010-09-05 17:36:26.000000000 +0300 @@ -36,6 +36,7 @@ #define IPI_RENDEZVOUS 2 #define IPI_STOP 3 #define IPI_STOP_HARD 3 +#define IPI_HARDCLOCK 4 #ifndef LOCORE diff -ruNp --exclude compile sys.prev2/powerpc/powerpc/intr_machdep.c sys/powerpc/powerpc/intr_machdep.c --- sys.prev2/powerpc/powerpc/intr_machdep.c 2010-09-03 02:15:34.000000000 +0300 +++ sys/powerpc/powerpc/intr_machdep.c 2010-09-05 08:55:05.000000000 +0300 @@ -95,6 +95,7 @@ struct powerpc_intr { device_t pic; u_int intline; u_int vector; + u_int cntindex; cpumask_t cpu; enum intr_trigger trig; enum intr_polarity pol; @@ -106,6 +107,7 @@ struct pic { int ipi_irq; }; +static u_int intrcnt_index = 0; static struct mtx intr_table_lock; static struct powerpc_intr *powerpc_intrs[INTR_VECTORS]; static struct pic piclist[MAX_PICS]; @@ -152,6 +154,16 @@ intrcnt_setname(const char *name, int in MAXCOMLEN, name); } +void +intrcnt_add(const char *name, u_long **countp) +{ + int idx; + + idx = atomic_fetchadd_int(&intrcnt_index, 1); + *countp = &intrcnt[idx]; + intrcnt_setname(name, idx); +} + static struct powerpc_intr * intr_lookup(u_int irq) { @@ -200,8 +212,10 @@ intr_lookup(u_int irq) if (iscan == NULL && i->vector != -1) { powerpc_intrs[i->vector] = i; + i->cntindex = atomic_fetchadd_int(&intrcnt_index, 1); + i->cntp = &intrcnt[i->cntindex]; sprintf(intrname, "irq%u:", i->irq); - intrcnt_setname(intrname, i->vector); + intrcnt_setname(intrname, i->cntindex); nvectors++; } mtx_unlock(&intr_table_lock); @@ -384,8 +398,6 @@ powerpc_setup_intr(const char *name, u_i if (error) return (error); - i->cntp = &intrcnt[i->vector]; - enable = 1; } @@ -393,7 +405,7 @@ powerpc_setup_intr(const char *name, u_i intr_priority(flags), flags, cookiep); mtx_lock(&intr_table_lock); - intrcnt_setname(i->event->ie_fullname, i->vector); + intrcnt_setname(i->event->ie_fullname, i->cntindex); mtx_unlock(&intr_table_lock); if (!cold) { diff -ruNp --exclude compile sys.prev2/powerpc/powerpc/mp_machdep.c sys/powerpc/powerpc/mp_machdep.c --- sys.prev2/powerpc/powerpc/mp_machdep.c 2010-09-03 02:15:34.000000000 +0300 +++ sys/powerpc/powerpc/mp_machdep.c 2010-09-05 17:45:23.000000000 +0300 @@ -90,6 +90,9 @@ machdep_ap_bootstrap(void) /* Let the DEC and external interrupts go */ mtmsr(mfmsr() | PSL_EE); + /* Start per-CPU event timers. */ + cpu_initclocks_ap(); + /* Announce ourselves awake, and enter the scheduler */ sched_throw(NULL); } @@ -305,6 +308,10 @@ powerpc_ipi_handler(void *arg) atomic_clear_int(&stopped_cpus, self); CTR1(KTR_SMP, "%s: IPI_STOP (restart)", __func__); break; + case IPI_HARDCLOCK: + CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); + hardclockintr(curthread->td_intr_frame); + break; } }