Index: sys/sys/sdt.h =================================================================== --- sys/sys/sdt.h (revision 234420) +++ sys/sys/sdt.h (working copy) @@ -92,6 +92,7 @@ #define SDT_PROBE_DEFINE4(prov, mod, func, name, sname, arg0, arg1, arg2, arg3) #define SDT_PROBE_DEFINE5(prov, mod, func, name, sname, arg0, arg1, arg2, arg3, arg4) +#define SDT_PROBE0(prov, mod, func, name) #define SDT_PROBE1(prov, mod, func, name, arg0) #define SDT_PROBE2(prov, mod, func, name, arg0, arg1) #define SDT_PROBE3(prov, mod, func, name, arg0, arg1, arg2) @@ -231,6 +232,8 @@ SDT_PROBE_ARGTYPE(prov, mod, func, name, 3, arg3); \ SDT_PROBE_ARGTYPE(prov, mod, func, name, 4, arg4) +#define SDT_PROBE0(prov, mod, func, name) \ + SDT_PROBE(prov, mod, func, name, 0, 0, 0, 0, 0) #define SDT_PROBE1(prov, mod, func, name, arg0) \ SDT_PROBE(prov, mod, func, name, arg0, 0, 0, 0, 0) #define SDT_PROBE2(prov, mod, func, name, arg0, arg1) \ Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c (revision 234420) +++ sys/kern/kern_thread.c (working copy) @@ -27,6 +27,7 @@ */ #include "opt_witness.h" +#include "opt_kdtrace.h" #include "opt_hwpmc_hooks.h" #include @@ -39,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +61,10 @@ #include #include +SDT_PROVIDER_DECLARE(proc); +SDT_PROBE_DEFINE(proc, , , lwp_exit, lwp-exit); + + /* * thread related storage. */ Index: sys/kern/sched_ule.c =================================================================== --- sys/kern/sched_ule.c (revision 234420) +++ sys/kern/sched_ule.c (working copy) @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -327,6 +328,24 @@ SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, NULL); +SDT_PROVIDER_DEFINE(sched); + +SDT_PROBE_DEFINE3(sched, , , change_pri, change-pri, "struct thread *", + "struct proc *", "uint8_t"); +SDT_PROBE_DEFINE3(sched, , , dequeue, dequeue, "struct thread *", + "struct proc *", "void *"); +SDT_PROBE_DEFINE4(sched, , , enqueue, enqueue, "struct thread *", + "struct proc *", "void *", "int"); +SDT_PROBE_DEFINE4(sched, , , lend_pri, lend-pri, "struct thread *", + "struct proc *", "uint8_t", "struct thread *"); +SDT_PROBE_DEFINE2(sched, , , load_change, load-change, "int", "int"); +SDT_PROBE_DEFINE2(sched, , , off_cpu, off-cpu, "struct thread *", + "struct proc *"); +SDT_PROBE_DEFINE(sched, , , on_cpu, on-cpu); +SDT_PROBE_DEFINE(sched, , , remain_cpu, remain-cpu); +SDT_PROBE_DEFINE2(sched, , , surrender, surrender, "struct thread *", + "struct proc *"); + /* * Print the threads waiting on a run-queue. */ @@ -509,6 +528,7 @@ if ((td->td_flags & TDF_NOLOAD) == 0) tdq->tdq_sysload++; KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load); + SDT_PROBE2(sched, , , load_change, (int)TDQ_ID(tdq), tdq->tdq_load); } /* @@ -528,6 +548,7 @@ if ((td->td_flags & TDF_NOLOAD) == 0) tdq->tdq_sysload--; KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load); + SDT_PROBE2(sched, , , load_change, (int)TDQ_ID(tdq), tdq->tdq_load); } /* @@ -1625,10 +1646,13 @@ KTR_POINT3(KTR_SCHED, "thread", sched_tdname(td), "prio", "prio:%d", td->td_priority, "new prio:%d", prio, KTR_ATTR_LINKED, sched_tdname(curthread)); + SDT_PROBE3(sched, , , change_pri, td, td->td_proc, prio); if (td != curthread && prio > td->td_priority) { KTR_POINT3(KTR_SCHED, "thread", sched_tdname(curthread), "lend prio", "prio:%d", td->td_priority, "new prio:%d", prio, KTR_ATTR_LINKED, sched_tdname(td)); + SDT_PROBE4(sched, , , lend_pri, td, td->td_proc, prio, + curthread); } ts = td->td_sched; THREAD_LOCK_ASSERT(td, MA_OWNED); @@ -1879,6 +1903,7 @@ if (PMC_PROC_IS_USING_PMCS(td->td_proc)) PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); #endif + SDT_PROBE2(sched, , , off_cpu, td, td->td_proc); lock_profile_release_lock(&TDQ_LOCKPTR(tdq)->lock_object); TDQ_LOCKPTR(tdq)->mtx_lock = (uintptr_t)newtd; sched_pctcpu_update(newtd->td_sched, 0); @@ -1903,12 +1928,16 @@ tdq = TDQ_CPU(cpuid); lock_profile_obtain_lock_success( &TDQ_LOCKPTR(tdq)->lock_object, 0, 0, __FILE__, __LINE__); + + SDT_PROBE0(sched, , , on_cpu); #ifdef HWPMC_HOOKS if (PMC_PROC_IS_USING_PMCS(td->td_proc)) PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN); #endif - } else + } else { thread_unblock_switch(td, mtx); + SDT_PROBE0(sched, , , remain_cpu); + } /* * Assert that all went well and return. */ @@ -2102,6 +2131,8 @@ { struct tdq *tdq; + SDT_PROBE2(sched, , , surrender, td, td->td_proc); + thread_lock(td); tdq = TDQ_SELF(); TDQ_LOCK_ASSERT(tdq, MA_OWNED); @@ -2330,6 +2361,8 @@ sched_tdname(curthread)); KTR_POINT1(KTR_SCHED, "thread", sched_tdname(curthread), "wokeup", KTR_ATTR_LINKED, sched_tdname(td)); + SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL, + flags & SRQ_PREEMPTED); THREAD_LOCK_ASSERT(td, MA_OWNED); /* * Recalculate the priority before we select the target cpu or @@ -2375,6 +2408,7 @@ KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "runq rem", "prio:%d", td->td_priority); + SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL); tdq = TDQ_CPU(td->td_sched->ts_cpu); TDQ_LOCK_ASSERT(tdq, MA_OWNED); MPASS(td->td_lock == TDQ_LOCKPTR(tdq)); Index: sys/kern/kern_clock.c =================================================================== --- sys/kern/kern_clock.c (revision 234420) +++ sys/kern/kern_clock.c (working copy) @@ -40,6 +40,7 @@ #include "opt_kdb.h" #include "opt_device_polling.h" #include "opt_hwpmc_hooks.h" +#include "opt_kdtrace.h" #include "opt_ntp.h" #include "opt_watchdog.h" @@ -56,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -88,6 +90,9 @@ /* Spin-lock protecting profiling statistics. */ static struct mtx time_lock; +SDT_PROVIDER_DECLARE(sched); +SDT_PROBE_DEFINE2(sched, , , tick, tick, "struct thread *", "struct proc *"); + static int sysctl_kern_cp_time(SYSCTL_HANDLER_ARGS) { @@ -760,6 +765,7 @@ ru->ru_maxrss = rss; KTR_POINT2(KTR_SCHED, "thread", sched_tdname(td), "statclock", "prio:%d", td->td_priority, "stathz:%d", (stathz)?stathz:hz); + SDT_PROBE2(sched, , , tick, td, td->td_proc); thread_lock_flags(td, MTX_QUIET); for ( ; cnt > 0; cnt--) sched_clock(td); Index: sys/kern/sched_4bsd.c =================================================================== --- sys/kern/sched_4bsd.c (revision 234420) +++ sys/kern/sched_4bsd.c (working copy) @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -244,12 +245,31 @@ "allow threads to share a quantum"); #endif +SDT_PROVIDER_DEFINE(sched); + +SDT_PROBE_DEFINE3(sched, , , change_pri, change-pri, "struct thread *", + "struct proc *", "uint8_t"); +SDT_PROBE_DEFINE3(sched, , , dequeue, dequeue, "struct thread *", + "struct proc *", "void *"); +SDT_PROBE_DEFINE4(sched, , , enqueue, enqueue, "struct thread *", + "struct proc *", "void *", "int"); +SDT_PROBE_DEFINE4(sched, , , lend_pri, lend-pri, "struct thread *", + "struct proc *", "uint8_t", "struct thread *"); +SDT_PROBE_DEFINE2(sched, , , load_change, load-change, "int", "int"); +SDT_PROBE_DEFINE2(sched, , , off_cpu, off-cpu, "struct thread *", + "struct proc *"); +SDT_PROBE_DEFINE(sched, , , on_cpu, on-cpu); +SDT_PROBE_DEFINE(sched, , , remain_cpu, remain-cpu); +SDT_PROBE_DEFINE2(sched, , , surrender, surrender, "struct thread *", + "struct proc *"); + static __inline void sched_load_add(void) { sched_tdcnt++; KTR_COUNTER0(KTR_SCHED, "load", "global load", sched_tdcnt); + SDT_PROBE2(sched, , , load_change, NOCPU, sched_tdcnt); } static __inline void @@ -258,6 +278,7 @@ sched_tdcnt--; KTR_COUNTER0(KTR_SCHED, "load", "global load", sched_tdcnt); + SDT_PROBE2(sched, , , load_change, NOCPU, sched_tdcnt); } /* * Arrange to reschedule if necessary, taking the priorities and @@ -795,10 +816,13 @@ KTR_POINT3(KTR_SCHED, "thread", sched_tdname(td), "priority change", "prio:%d", td->td_priority, "new prio:%d", prio, KTR_ATTR_LINKED, sched_tdname(curthread)); + SDT_PROBE3(sched, , , change_pri, td, td->td_proc, prio); if (td != curthread && prio > td->td_priority) { KTR_POINT3(KTR_SCHED, "thread", sched_tdname(curthread), "lend prio", "prio:%d", td->td_priority, "new prio:%d", prio, KTR_ATTR_LINKED, sched_tdname(td)); + SDT_PROBE4(sched, , , lend_pri, td, td->td_proc, prio, + curthread); } THREAD_LOCK_ASSERT(td, MA_OWNED); if (td->td_priority == prio) @@ -987,6 +1011,9 @@ if (PMC_PROC_IS_USING_PMCS(td->td_proc)) PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); #endif + + SDT_PROBE2(sched, , , off_cpu, td, td->td_proc); + /* I feel sleepy */ lock_profile_release_lock(&sched_lock.lock_object); #ifdef KDTRACE_HOOKS @@ -1018,11 +1045,14 @@ * needed to, or the thread_wait() or wait() will * need to reap it. */ + + SDT_PROBE0(sched, , , on_cpu); #ifdef HWPMC_HOOKS if (PMC_PROC_IS_USING_PMCS(td->td_proc)) PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN); #endif - } + } else + SDT_PROBE0(sched, , , remain_cpu); #ifdef SMP if (td->td_flags & TDF_IDLETD) @@ -1223,6 +1253,8 @@ sched_tdname(curthread)); KTR_POINT1(KTR_SCHED, "thread", sched_tdname(curthread), "wokeup", KTR_ATTR_LINKED, sched_tdname(td)); + SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL, + flags & SRQ_PREEMPTED); /* @@ -1315,6 +1347,8 @@ sched_tdname(curthread)); KTR_POINT1(KTR_SCHED, "thread", sched_tdname(curthread), "wokeup", KTR_ATTR_LINKED, sched_tdname(td)); + STD_PROBE4(sched, , , enqueue, td, td->td_proc, NULL, + flags & SRQ_PREEMPTED); /* * Now that the thread is moving to the run-queue, set the lock @@ -1362,6 +1396,7 @@ KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq rem", "prio:%d", td->td_priority, KTR_ATTR_LINKED, sched_tdname(curthread)); + SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL); if ((td->td_flags & TDF_NOLOAD) == 0) sched_load_rem(); @@ -1425,6 +1460,8 @@ void sched_preempt(struct thread *td) { + + SDT_PROBE2(sched, , , surrender, td, td->td_proc); thread_lock(td); if (td->td_critnest > 1) td->td_owepreempt = 1; Index: sys/kern/subr_sleepqueue.c =================================================================== --- sys/kern/subr_sleepqueue.c (revision 234420) +++ sys/kern/subr_sleepqueue.c (working copy) @@ -64,6 +64,7 @@ #include "opt_sleepqueue_profiling.h" #include "opt_ddb.h" +#include "opt_kdtrace.h" #include "opt_sched.h" #include @@ -75,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -166,6 +168,9 @@ static void sleepq_switch(void *wchan, int pri); static void sleepq_timeout(void *arg); +SDT_PROBE_DECLARE(sched, , , sleep); +SDT_PROBE_DECLARE(sched, , , wakeup); + /* * Early initialization of sleep queues that is called from the sleepinit() * SYSINIT. @@ -534,6 +539,7 @@ MPASS(td->td_sleepqueue == NULL); sched_sleep(td, pri); thread_lock_set(td, &sc->sc_lock); + SDT_PROBE0(sched, , , sleep); TD_SET_SLEEPING(td); mi_switch(SW_VOL | SWT_SLEEPQ, NULL); KASSERT(TD_IS_RUNNING(td), ("running but not TDS_RUNNING")); @@ -715,6 +721,8 @@ sc = SC_LOOKUP(sq->sq_wchan); mtx_assert(&sc->sc_lock, MA_OWNED); + SDT_PROBE2(sched, , , wakeup, td, td->td_proc); + /* Remove the thread from the queue. */ sq->sq_blockedcnt[td->td_sqqueue]--; TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq); Index: sys/kern/kern_synch.c =================================================================== --- sys/kern/kern_synch.c (revision 234420) +++ sys/kern/kern_synch.c (working copy) @@ -37,6 +37,7 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_kdtrace.h" #include "opt_ktrace.h" #include "opt_sched.h" @@ -51,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -105,6 +107,20 @@ static void loadav(void *arg); +SDT_PROVIDER_DECLARE(sched); +SDT_PROBE_DEFINE(sched, , , preempt, preempt); + +/* + * These probes reference Solaris features that are not implemented in FreeBSD. + * Create the probes anyway for compatibility with existing D scripts.; they'll + * just never fire. + */ +SDT_PROBE_DEFINE(sched, , , cpucaps_sleep, cpucaps-sleep); +SDT_PROBE_DEFINE(sched, , , cpucaps_wakeup, cpucaps-wakeup); +SDT_PROBE_DEFINE(sched, , , schedctl_nopreempt, schedctl-nopreempt); +SDT_PROBE_DEFINE(sched, , , schedctl_preempt, schedctl-preempt); +SDT_PROBE_DEFINE(sched, , , schedctl_yield, schedctl-yield); + void sleepinit(void) { @@ -462,6 +478,7 @@ "prio:%d", td->td_priority, "wmesg:\"%s\"", td->td_wmesg, "lockname:\"%s\"", td->td_lockname); #endif + SDT_PROBE0(sched, , , preempt); #ifdef XEN PT_UPDATES_FLUSH(); #endif Index: sys/kern/subr_turnstile.c =================================================================== --- sys/kern/subr_turnstile.c (revision 234420) +++ sys/kern/subr_turnstile.c (working copy) @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include "opt_ddb.h" +#include "opt_kdtrace.h" #include "opt_turnstile_profiling.h" #include "opt_sched.h" @@ -73,6 +74,7 @@ #include #include #include +#include #include #include @@ -167,6 +169,11 @@ static int turnstile_init(void *mem, int size, int flags); static void turnstile_fini(void *mem, int size); +SDT_PROVIDER_DECLARE(sched); +SDT_PROBE_DEFINE(sched, , , sleep, sleep); +SDT_PROBE_DEFINE2(sched, , , wakeup, wakeup, "struct thread *", + "struct proc *"); + /* * Walks the chain of turnstiles and their owners to propagate the priority * of the thread being blocked to all the threads holding locks that have to @@ -740,6 +747,8 @@ CTR4(KTR_LOCK, "%s: td %d blocked on [%p] %s", __func__, td->td_tid, lock, lock->lo_name); + SDT_PROBE0(sched, , , sleep); + THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock); mi_switch(SW_VOL | SWT_TURNSTILE, NULL); @@ -916,6 +925,7 @@ while (!TAILQ_EMPTY(&pending_threads)) { td = TAILQ_FIRST(&pending_threads); TAILQ_REMOVE(&pending_threads, td, td_lockq); + SDT_PROBE2(sched, , , wakeup, td, td->td_proc); thread_lock(td); THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock); MPASS(td->td_proc->p_magic == P_MAGIC);