commit 14d9e56122d8f215f93c25c5b51c6a83eb1dde92 Author: Andriy Gapon Date: Tue Dec 4 08:56:12 2012 +0200 adapt kib's patch for corrupted callwheel to calloutng diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index c30e1a2..00b1c3f 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -498,7 +498,7 @@ proc0_init(void *dummy __unused) strncpy(p->p_comm, "kernel", sizeof (p->p_comm)); strncpy(td->td_name, "swapper", sizeof (td->td_name)); - callout_init(&p->p_itcallout, CALLOUT_MPSAFE); + callout_init_mtx(&p->p_itcallout, &p->p_mtx, 0); callout_init_mtx(&p->p_limco, &p->p_mtx, 0); callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 5dc43ca..b8a4825 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -591,7 +591,7 @@ do_fork(struct thread *td, int flags, struct proc *p2, struct thread *td2, LIST_INIT(&p2->p_children); LIST_INIT(&p2->p_orphans); - callout_init(&p2->p_itcallout, CALLOUT_MPSAFE); + callout_init_mtx(&p2->p_itcallout, &p2->p_mtx, 0); /* * If PF_FORK is set, the child process inherits the diff --git a/sys/kern/kern_time.c b/sys/kern/kern_time.c index d46c969..c20da58 100644 --- a/sys/kern/kern_time.c +++ b/sys/kern/kern_time.c @@ -787,13 +787,11 @@ realitexpire(void *arg) struct timeval ctv, ntv; p = (struct proc *)arg; - PROC_LOCK(p); kern_psignal(p, SIGALRM); if (!timevalisset(&p->p_realtimer.it_interval)) { timevalclear(&p->p_realtimer.it_value); if (p->p_flag & P_WEXIT) wakeup(&p->p_itcallout); - PROC_UNLOCK(p); return; } for (;;) { @@ -805,7 +803,6 @@ realitexpire(void *arg) timevalsub(&ntv, &ctv); callout_reset(&p->p_itcallout, tvtohz(&ntv) - 1, realitexpire, p); - PROC_UNLOCK(p); return; } } diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c index 1f9fe76..0ac1915 100644 --- a/sys/kern/kern_timeout.c +++ b/sys/kern/kern_timeout.c @@ -178,7 +178,7 @@ struct callout_cpu cc_cpu; static int timeout_cpu; void (*callout_new_inserted)(int cpu, struct bintime bt, struct bintime bt_opt) = NULL; -static struct callout * +static void softclock_call_cc(struct callout *c, struct callout_cpu *cc, int *mpcalls, int *lockcalls, int *gcalls, int direct); @@ -447,9 +447,11 @@ callout_process(struct bintime *now) if (tmp->c_flags & CALLOUT_DIRECT) { ++depth_dir; TAILQ_REMOVE(sc, tmp, c_links.tqe); - tmp = softclock_call_cc(tmp, cc, + cc->cc_exec_next_dir = TAILQ_NEXT(tmp, c_links.tqe); + softclock_call_cc(tmp, cc, &mpcalls_dir, &lockcalls_dir, NULL, 1); + tmp = cc->cc_exec_next_dir; } else { TAILQ_INSERT_TAIL(&cc->cc_expireq, tmp, c_staiter); @@ -606,20 +608,16 @@ callout_cc_add(struct callout *c, struct callout_cpu *cc, } static void -callout_cc_del(struct callout *c, struct callout_cpu *cc, int direct) +callout_cc_del(struct callout *c, struct callout_cpu *cc) { - - if (cc->cc_exec_next_dir == c) - cc->cc_exec_next_dir = TAILQ_NEXT(c, c_links.tqe); - else if (cc->cc_exec_next == c) - cc->cc_exec_next = TAILQ_NEXT(c, c_staiter); - if (c->c_flags & CALLOUT_LOCAL_ALLOC) { - c->c_func = NULL; - SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); - } + + if ((c->c_flags & CALLOUT_LOCAL_ALLOC) == 0) + return; + c->c_func = NULL; + SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); } -static struct callout * +static void softclock_call_cc(struct callout *c, struct callout_cpu *cc, int *mpcalls, int *lockcalls, int *gcalls, int direct) { @@ -642,10 +640,9 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc, int *mpcalls, static timeout_t *lastfunc; #endif - if (direct) - cc->cc_exec_next_dir = TAILQ_NEXT(c, c_links.tqe); - else - cc->cc_exec_next = TAILQ_NEXT(c, c_staiter); + KASSERT((c->c_flags & (CALLOUT_PENDING | CALLOUT_ACTIVE)) == + (CALLOUT_PENDING | CALLOUT_ACTIVE), + ("softclock_call_cc: pend|act %p %x", c, c->c_flags)); class = (c->c_lock != NULL) ? LOCK_CLASS(c->c_lock) : NULL; sharedlock = (c->c_flags & CALLOUT_SHAREDLOCK) ? 0 : 1; c_lock = c->c_lock; @@ -719,20 +716,7 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc, int *mpcalls, class->lc_unlock(c_lock); skip: CC_LOCK(cc); - /* - * If the current callout is locally allocated (from - * timeout(9)) then put it on the freelist. - * - * Note: we need to check the cached copy of c_flags because - * if it was not local, then it's not safe to deref the - * callout pointer. - */ - if (c_flags & CALLOUT_LOCAL_ALLOC) { - KASSERT(c->c_flags == CALLOUT_LOCAL_ALLOC, - ("corrupted callout")); - c->c_func = NULL; - SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); - } + KASSERT(cc->cc_exec_entity[direct].cc_curr == c, ("Mishandled cc_curr")); cc->cc_exec_entity[direct].cc_curr = NULL; if (cc->cc_exec_entity[direct].cc_waiting) { /* @@ -741,8 +725,11 @@ skip: * If the callout was scheduled for * migration just cancel it. */ - if (cc_cme_migrating(cc, direct)) + if (cc_cme_migrating(cc, direct)) { cc_cme_cleanup(cc, direct); + if ((c->c_flags & CALLOUT_DFRMIGRATION) != 0) + c->c_flags &= ~CALLOUT_DFRMIGRATION; + } cc->cc_exec_entity[direct].cc_waiting = 0; CC_UNLOCK(cc); wakeup(&cc->cc_exec_entity[direct].cc_waiting); @@ -766,8 +753,8 @@ skip: CTR3(KTR_CALLOUT, "deferred cancelled %p func %p arg %p", c, new_func, new_arg); - callout_cc_del(c, cc, direct); - goto nextc; + callout_cc_del(c, cc); + return; } c->c_flags &= ~CALLOUT_DFRMIGRATION; @@ -786,11 +773,19 @@ skip: #else panic("migration should not happen"); #endif + /* + * If the current callout is locally allocated (from + * timeout(9)) then put it on the freelist. + * + * Note: we need to check the cached copy of c_flags because + * if it was not local, then it's not safe to deref the + * callout pointer. + */ + } else if (c_flags & CALLOUT_LOCAL_ALLOC) { + KASSERT(c->c_flags == CALLOUT_LOCAL_ALLOC, + ("corrupted callout")); + callout_cc_del(c, cc); } -#ifdef SMP -nextc: -#endif - return cc->cc_exec_entity[direct].cc_next; } /* @@ -825,9 +820,11 @@ softclock(void *arg) c = TAILQ_FIRST(&cc->cc_expireq); while (c != NULL) { ++depth; + cc->cc_exec_next = TAILQ_NEXT(c, c_staiter); TAILQ_REMOVE(&cc->cc_expireq, c, c_staiter); - c = softclock_call_cc(c, cc, &mpcalls, + softclock_call_cc(c, cc, &mpcalls, &lockcalls, &gcalls, 0); + c = cc->cc_exec_next; } #ifdef CALLOUT_PROFILING avg_depth += (depth * 1000 - avg_depth) >> 8; @@ -1198,13 +1195,17 @@ again: CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p", c, c->c_func, c->c_arg); + if (cc->cc_exec_next_dir == c) + cc->cc_exec_next_dir = TAILQ_NEXT(c, c_links.tqe); + else if (cc->cc_exec_next == c) + cc->cc_exec_next = TAILQ_NEXT(c, c_staiter); if ((c->c_flags & CALLOUT_PROCESSED) == 0) { bucket = get_bucket(&c->c_time); TAILQ_REMOVE(&cc->cc_callwheel[bucket], c, c_links.tqe); } else TAILQ_REMOVE(&cc->cc_expireq, c, c_staiter); - callout_cc_del(c, cc, direct); + callout_cc_del(c, cc); CC_UNLOCK(cc); return (1);