diff --git a/sys/cddl/contrib/opensolaris/uts/common/ctf/ctf_mod.c b/sys/cddl/contrib/opensolaris/uts/common/ctf/ctf_mod.c index b34cf40..b991f24 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/ctf/ctf_mod.c +++ b/sys/cddl/contrib/opensolaris/uts/common/ctf/ctf_mod.c @@ -154,7 +154,9 @@ ctf_modopen(struct module *mp, int *error) strsect.cts_entsize = 1; strsect.cts_offset = 0; +#if defined(sun) ASSERT(MUTEX_HELD(&mod_lock)); +#endif if ((fp = ctf_bufopen(&ctfsect, &symsect, &strsect, error)) == NULL) return (NULL); diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c index 729a2cb..5bb3b9a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c +++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c @@ -278,8 +278,6 @@ static kmutex_t dtrace_meta_lock; /* meta-provider state lock */ #if !defined(sun) /* XXX FreeBSD hacks. */ -static kmutex_t mod_lock; - #define cr_suid cr_svuid #define cr_sgid cr_svgid #define ipaddr_t in_addr_t @@ -7678,7 +7676,9 @@ dtrace_unregister(dtrace_provider_id_t id) } } else { mutex_enter(&dtrace_provider_lock); +#if defined(sun) mutex_enter(&mod_lock); +#endif mutex_enter(&dtrace_lock); } @@ -7692,7 +7692,9 @@ dtrace_unregister(dtrace_provider_id_t id) dtrace_anon.dta_state->dts_necbs > 0))) { if (!self) { mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); } return (EBUSY); @@ -7726,7 +7728,9 @@ dtrace_unregister(dtrace_provider_id_t id) if (!self) { mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); } @@ -7808,7 +7812,9 @@ dtrace_unregister(dtrace_provider_id_t id) if (!self) { mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); } @@ -8092,19 +8098,6 @@ dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp) (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1); } -#if !defined(sun) -static int -dtrace_probe_provide_cb(linker_file_t lf, void *arg) -{ - dtrace_provider_t *prv = (dtrace_provider_t *) arg; - - prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, lf); - - return(0); -} -#endif - - /* * Called to indicate that a probe -- or probes -- should be provided by a * specfied provider. If the specified description is NULL, the provider will @@ -8141,6 +8134,7 @@ dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) */ prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc); +#if defined(sun) /* * Now call the per-module provide operation. We will grab * mod_lock to prevent the list from being modified. Note @@ -8149,7 +8143,6 @@ dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) */ mutex_enter(&mod_lock); -#if defined(sun) ctl = &modules; do { if (ctl->mod_busy || ctl->mod_mp == NULL) @@ -8158,11 +8151,9 @@ dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); } while ((ctl = ctl->mod_next) != &modules); -#else - (void) linker_file_foreach(dtrace_probe_provide_cb, prv); -#endif mutex_exit(&mod_lock); +#endif } while (all && (prv = prv->dtpv_next) != NULL); } @@ -15146,28 +15137,33 @@ dtrace_helpers_duplicate(proc_t *from, proc_t *to) dtrace_helper_provider_register(to, newhelp, NULL); } -#if defined(sun) /* * DTrace Hook Functions */ static void -dtrace_module_loaded(modctl_t *ctl) +dtrace_module_loaded(struct linker_file *lf) { dtrace_provider_t *prv; mutex_enter(&dtrace_provider_lock); +#if defined(sun) mutex_enter(&mod_lock); +#endif +#if defined(sun) ASSERT(ctl->mod_busy); +#endif /* * We're going to call each providers per-module provide operation * specifying only this module. */ for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next) - prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); + prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, lf); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); /* @@ -15204,15 +15200,30 @@ dtrace_module_loaded(modctl_t *ctl) } static void -dtrace_module_unloaded(modctl_t *ctl) +dtrace_module_unloaded(struct linker_file *lf) { dtrace_probe_t template, *probe, *first, *next; dtrace_provider_t *prov; +#if !defined(sun) + char modname[DTRACE_MODNAMELEN]; + size_t len; +#endif +#if defined(sun) template.dtpr_mod = ctl->mod_modname; +#else + /* Handle the fact that lf->filename may end in ".ko". */ + strlcpy(modname, lf->filename, sizeof(modname)); + len = strlen(lf->filename); + if (len > 3 && strcmp(modname + len - 3, ".ko") == 0) + modname[len - 3] = '\0'; + template.dtpr_mod = modname; +#endif mutex_enter(&dtrace_provider_lock); +#if defined(sun) mutex_enter(&mod_lock); +#endif mutex_enter(&dtrace_lock); if (dtrace_bymod == NULL) { @@ -15221,7 +15232,9 @@ dtrace_module_unloaded(modctl_t *ctl) * we don't have any work to do. */ mutex_exit(&dtrace_provider_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_lock); return; } @@ -15230,7 +15243,9 @@ dtrace_module_unloaded(modctl_t *ctl) probe != NULL; probe = probe->dtpr_nextmod) { if (probe->dtpr_ecb != NULL) { mutex_exit(&dtrace_provider_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_lock); /* @@ -15245,7 +15260,7 @@ dtrace_module_unloaded(modctl_t *ctl) */ if (dtrace_err_verbose) { cmn_err(CE_WARN, "unloaded module '%s' had " - "enabled probes", ctl->mod_modname); + "enabled probes", modname); } return; @@ -15288,15 +15303,22 @@ dtrace_module_unloaded(modctl_t *ctl) kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); +#if defined(sun) vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1); +#else + free_unr(dtrace_arena, probe->dtpr_id); +#endif kmem_free(probe, sizeof (dtrace_probe_t)); } mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); } +#if defined(sun) static void dtrace_suspend(void) { @@ -16254,19 +16276,25 @@ dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) return (EINVAL); mutex_enter(&dtrace_provider_lock); +#if defined(sun) mutex_enter(&mod_lock); +#endif mutex_enter(&dtrace_lock); if (desc.dtargd_id > dtrace_nprobes) { mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); return (EINVAL); } if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) { mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); return (EINVAL); } @@ -16290,7 +16318,9 @@ dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) probe->dtpr_id, probe->dtpr_arg, &desc); } +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace.h b/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace.h index 4595c2e..d60a088 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace.h +++ b/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace.h @@ -2312,8 +2312,8 @@ extern void dtrace_membar_producer(void); extern void dtrace_membar_consumer(void); extern void (*dtrace_cpu_init)(processorid_t); -extern void (*dtrace_modload)(modctl_t *); -extern void (*dtrace_modunload)(modctl_t *); +extern void (*dtrace_modload)(struct linker_file *); +extern void (*dtrace_modunload)(struct linker_file *); extern void (*dtrace_helpers_cleanup)(void); extern void (*dtrace_helpers_fork)(proc_t *parent, proc_t *child); extern void (*dtrace_cpustart_init)(void); diff --git a/sys/cddl/dev/dtrace/dtrace_ioctl.c b/sys/cddl/dev/dtrace/dtrace_ioctl.c index bb03ffd..87da9a7 100644 --- a/sys/cddl/dev/dtrace/dtrace_ioctl.c +++ b/sys/cddl/dev/dtrace/dtrace_ioctl.c @@ -580,19 +580,25 @@ dtrace_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, return (EINVAL); mutex_enter(&dtrace_provider_lock); +#if defined(sun) mutex_enter(&mod_lock); +#endif mutex_enter(&dtrace_lock); if (desc->dtargd_id > dtrace_nprobes) { mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); return (EINVAL); } if ((probe = dtrace_probes[desc->dtargd_id - 1]) == NULL) { mutex_exit(&dtrace_lock); +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); return (EINVAL); } @@ -616,7 +622,9 @@ dtrace_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, probe->dtpr_id, probe->dtpr_arg, desc); } +#if defined(sun) mutex_exit(&mod_lock); +#endif mutex_exit(&dtrace_provider_lock); return (0); diff --git a/sys/cddl/dev/dtrace/dtrace_load.c b/sys/cddl/dev/dtrace/dtrace_load.c index 5926725..ec28a65 100644 --- a/sys/cddl/dev/dtrace/dtrace_load.c +++ b/sys/cddl/dev/dtrace/dtrace_load.c @@ -56,11 +56,9 @@ dtrace_load(void *dummy) /* Hang our hook for exceptions. */ dtrace_invop_init(); - /* - * XXX This is a short term hack to avoid having to comment - * out lots and lots of lock/unlock calls. - */ - mutex_init(&mod_lock,"XXX mod_lock hack", MUTEX_DEFAULT, NULL); + /* Register callbacks for module load and unload events. */ + dtrace_modload = dtrace_module_loaded; + dtrace_modunload = dtrace_module_unloaded; /* * Initialise the mutexes without 'witness' because the dtrace diff --git a/sys/cddl/dev/dtrace/dtrace_unload.c b/sys/cddl/dev/dtrace/dtrace_unload.c index 247cbb1..2ebb52d 100644 --- a/sys/cddl/dev/dtrace/dtrace_unload.c +++ b/sys/cddl/dev/dtrace/dtrace_unload.c @@ -123,9 +123,6 @@ dtrace_unload() mutex_destroy(&dtrace_lock); mutex_destroy(&dtrace_errlock); - /* XXX Hack */ - mutex_destroy(&mod_lock); - /* Reset our hook for exceptions. */ dtrace_invop_uninit(); diff --git a/sys/cddl/dev/fbt/fbt.c b/sys/cddl/dev/fbt/fbt.c index b828163..7b278f7 100644 --- a/sys/cddl/dev/fbt/fbt.c +++ b/sys/cddl/dev/fbt/fbt.c @@ -1335,6 +1335,15 @@ fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_a return; } +static int +fbt_linker_file_cb(linker_file_t lf, void *arg) +{ + + fbt_provide_module(arg, lf); + + return (0); +} + static void fbt_load(void *dummy) { @@ -1359,8 +1368,10 @@ fbt_load(void *dummy) if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER, NULL, &fbt_pops, NULL, &fbt_id) != 0) return; -} + /* Create probes for the kernel and already-loaded modules. */ + linker_file_foreach(fbt_linker_file_cb, NULL); +} static int fbt_unload() diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c index 96ab550..f1db72d 100644 --- a/sys/cddl/dev/sdt/sdt.c +++ b/sys/cddl/dev/sdt/sdt.c @@ -31,29 +31,37 @@ #include #include #include + #include #include #include -#include #include +#include +#include +#include #include #include +#include #include +#include #include -#define SDT_ADDR2NDX(addr) (((uintptr_t)(addr)) >> 4) - static d_open_t sdt_open; static int sdt_unload(void); static void sdt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); static void sdt_provide_probes(void *, dtrace_probedesc_t *); +static void sdt_provide_module(void *, struct linker_file *); static void sdt_destroy(void *, dtrace_id_t, void *); static void sdt_enable(void *, dtrace_id_t, void *); static void sdt_disable(void *, dtrace_id_t, void *); static void sdt_load(void *); -static int sdt_provider_unreg_callback(struct sdt_provider *prov, - void *arg); +static void sdt_create_provider(struct sdt_provider *); +static void sdt_create_probe(struct sdt_probe *); +static void sdt_modload(struct linker_file *); +static void sdt_modunload(struct linker_file *); + +static MALLOC_DEFINE(M_SDT, "SDT", "DTrace SDT providers"); static struct cdevsw sdt_cdevsw = { .d_version = D_VERSION, @@ -70,8 +78,8 @@ static dtrace_pattr_t sdt_attr = { }; static dtrace_pops_t sdt_pops = { - sdt_provide_probes, NULL, + sdt_provide_module, sdt_enable, sdt_disable, NULL, @@ -82,138 +90,272 @@ static dtrace_pops_t sdt_pops = { sdt_destroy }; -static struct cdev *sdt_cdev; - -static int -sdt_argtype_callback(struct sdt_argtype *argtype, void *arg) -{ - dtrace_argdesc_t *desc = arg; +static dtrace_pops_t sdt_child_pops = { + sdt_provide_probes, + NULL, + sdt_enable, + sdt_disable, + NULL, + NULL, + sdt_getargdesc, + NULL, + NULL, + sdt_destroy, +}; - if (desc->dtargd_ndx == argtype->ndx) { - desc->dtargd_mapping = desc->dtargd_ndx; /* XXX */ - strlcpy(desc->dtargd_native, argtype->type, - sizeof(desc->dtargd_native)); - desc->dtargd_xlate[0] = '\0'; /* XXX */ - } +static struct cdev *sdt_cdev; - return (0); -} +static TAILQ_HEAD(, sdt_provider) sdt_prov_list; static void -sdt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) +sdt_create_provider(struct sdt_provider *prov) { - struct sdt_probe *probe = parg; + struct sdt_provider *curr, *newprov; - if (desc->dtargd_ndx < probe->n_args) - (void) (sdt_argtype_listall(probe, sdt_argtype_callback, desc)); - else - desc->dtargd_ndx = DTRACE_ARGNONE; + TAILQ_FOREACH(curr, &sdt_prov_list, prov_entry) + if (strcmp(prov->name, curr->name) == 0) { + printf("provider '%s' is already registered\n", + prov->name); + /* The provider has already been defined. */ + curr->sdt_refs++; + return; + } + + /* + * Make a copy of prov so that we don't lose fields if its module is + * unloaded but the provider isn't destroyed. This could happen with + * a provider that spans multiple modules. + */ + newprov = malloc(sizeof(*newprov), M_SDT, M_WAITOK | M_ZERO); + newprov->name = strdup(prov->name, M_SDT); + prov->sdt_refs = newprov->sdt_refs = 1; + TAILQ_INIT(&newprov->probe_list); + + TAILQ_INSERT_TAIL(&sdt_prov_list, newprov, prov_entry); - return; + (void)dtrace_register(newprov->name, &sdt_attr, DTRACE_PRIV_USER, NULL, + &sdt_child_pops, NULL, (dtrace_provider_id_t *)&newprov->id); + prov->id = newprov->id; } -static int -sdt_probe_callback(struct sdt_probe *probe, void *arg __unused) +static void +sdt_create_probe(struct sdt_probe *probe) { - struct sdt_provider *prov = probe->prov; - char mod[64]; - char func[64]; - char name[64]; + struct sdt_provider *prov; + char mod[DTRACE_MODNAMELEN]; + char func[DTRACE_FUNCNAMELEN]; + char name[DTRACE_NAMELEN]; + size_t len; + + TAILQ_FOREACH(prov, &sdt_prov_list, prov_entry) + if (strcmp(prov->name, probe->prov->name) == 0) + break; + + KASSERT(prov != NULL, ("probe defined without a provider")); + + /* If no module name was specified, use the module filename. */ + if (*probe->mod == 0) { + strlcpy(mod, probe->sdtp_lf->filename, sizeof(mod)); + len = strlen(mod); + if (len > 3 && strcmp(mod + len - 3, ".ko") == 0) + mod[len - 3] = '\0'; + } else + strlcpy(mod, probe->mod, sizeof(mod)); /* * Unfortunately this is necessary because the Solaris DTrace * code mixes consts and non-consts with casts to override * the incompatibilies. On FreeBSD, we use strict warnings - * in gcc, so we have to respect const vs non-const. + * in the C compiler, so we have to respect const vs non-const. */ - strlcpy(mod, probe->mod, sizeof(mod)); strlcpy(func, probe->func, sizeof(func)); strlcpy(name, probe->name, sizeof(name)); - if (dtrace_probe_lookup(prov->id, mod, func, name) != 0) - return (0); + if (dtrace_probe_lookup(prov->id, mod, func, name) != DTRACE_IDNONE) + return; - (void) dtrace_probe_create(prov->id, probe->mod, probe->func, - probe->name, 1, probe); + probe->prov = prov; + TAILQ_INSERT_TAIL(&prov->probe_list, probe, probe_entry); - return (0); + /* Let DTrace know this probe exists. */ + (void)dtrace_probe_create(prov->id, mod, func, name, 1, probe); } -static int -sdt_provider_entry(struct sdt_provider *prov, void *arg) +static void +sdt_provide_probes(void *arg, dtrace_probedesc_t *desc) { - return (sdt_probe_listall(prov, sdt_probe_callback, NULL)); } static void -sdt_provide_probes(void *arg, dtrace_probedesc_t *desc) +sdt_provide_module(void *arg __unused, struct linker_file *lf) { - if (desc != NULL) + struct sdt_probe *probe, **probe_curr, **probe_first, **probe_last; + struct sdt_argtype **argtype, **argtype_first, **argtype_last; + + if (linker_file_lookup_set(lf, "sdt_probes_set", &probe_first, + &probe_last, NULL)) return; + for (probe_curr = probe_first; probe_curr < probe_last; probe_curr++) { + probe = *probe_curr; + probe->sdtp_lf = lf; + sdt_create_probe(probe); + TAILQ_INIT(&probe->argtype_list); + } - (void) sdt_provider_listall(sdt_provider_entry, NULL); + if (linker_file_lookup_set(lf, "sdt_argtypes_set", &argtype_first, + &argtype_last, NULL)) + return; + for (argtype = argtype_first; argtype < argtype_last; argtype++) { + (*argtype)->probe->n_args++; + TAILQ_INSERT_TAIL(&(*argtype)->probe->argtype_list, *argtype, + argtype_entry); + } } static void -sdt_destroy(void *arg, dtrace_id_t id, void *parg) +sdt_enable(void *arg __unused, dtrace_id_t id, void *parg) { - /* Nothing to do here. */ + struct sdt_probe *probe = parg; + + probe->id = id; + /* Make sure the module doesn't go away. */ + probe->sdtp_lf->nenabled++; } static void -sdt_enable(void *arg, dtrace_id_t id, void *parg) +sdt_disable(void *arg __unused, dtrace_id_t id, void *parg) { struct sdt_probe *probe = parg; - probe->id = id; + probe->id = 0; + probe->sdtp_lf->nenabled--; } static void -sdt_disable(void *arg, dtrace_id_t id, void *parg) +sdt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) { + struct sdt_argtype *argtype; struct sdt_probe *probe = parg; - probe->id = 0; + if (desc->dtargd_ndx < probe->n_args) { + TAILQ_FOREACH(argtype, &probe->argtype_list, argtype_entry) { + if (desc->dtargd_ndx == argtype->ndx) { + desc->dtargd_mapping = desc->dtargd_ndx; /* XXX */ + strlcpy(desc->dtargd_native, argtype->type, + sizeof(desc->dtargd_native)); + desc->dtargd_xlate[0] = '\0'; /* XXX */ + } + } + } else + desc->dtargd_ndx = DTRACE_ARGNONE; +} + +static void +sdt_destroy(void *arg, dtrace_id_t id, void *parg) +{ + struct sdt_probe *probe; + + probe = parg; + TAILQ_REMOVE(&probe->prov->probe_list, probe, probe_entry); +} + +/* + * Called from the kernel linker when a module is loaded, before + * dtrace_module_loaded() is called. This is done so that it's possible to + * register new providers when module load. We cannot do this in the + * provide_module method since it's called with the provider lock held + * and dtrace_register() will try to acquire it again. + */ +static void +sdt_modload(struct linker_file *lf) +{ + struct sdt_provider **prov, **prov_first, **prov_last; + + if (linker_file_lookup_set(lf, "sdt_providers_set", &prov_first, + &prov_last, NULL)) + return; + for (prov = prov_first; prov < prov_last; prov++) + sdt_create_provider(*prov); +} + +static void +sdt_modunload(struct linker_file *lf) +{ + struct sdt_provider *prov, **prov_curr, **prov_first, **prov_last, *tmp; + + if (linker_file_lookup_set(lf, "sdt_providers_set", &prov_first, + &prov_last, NULL)) + return; + + for (prov_curr = prov_first; prov_curr < prov_last; prov_curr++) { + TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) { + if (strcmp(prov->name, (*prov_curr)->name) == 0) { + if (prov->sdt_refs == 1) { + TAILQ_REMOVE(&sdt_prov_list, prov, prov_entry); + dtrace_unregister(prov->id); + free(prov, M_SDT); + } else + prov->sdt_refs--; + break; + } + } + } } static int -sdt_provider_reg_callback(struct sdt_provider *prov, void *arg __unused) +sdt_linker_file_cb(linker_file_t lf, void *arg) { - return (dtrace_register(prov->name, &sdt_attr, DTRACE_PRIV_USER, - NULL, &sdt_pops, NULL, (dtrace_provider_id_t *) &prov->id)); + + sdt_modload(lf); + sdt_provide_module(arg, lf); + + return (0); } static void sdt_load(void *dummy) { + static dtrace_provider_id_t id; + int error; + + TAILQ_INIT(&sdt_prov_list); + /* Create the /dev/dtrace/sdt entry. */ sdt_cdev = make_dev(&sdt_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "dtrace/sdt"); sdt_probe_func = dtrace_probe; - sdt_register_callbacks(sdt_provider_reg_callback, NULL, - sdt_provider_unreg_callback, NULL, sdt_probe_callback, NULL); -} + dtrace_sdt_modload = sdt_modload; + dtrace_sdt_modunload = sdt_modunload; -static int -sdt_provider_unreg_callback(struct sdt_provider *prov, void *arg __unused) -{ - return (dtrace_unregister(prov->id)); + /* + * Create a dummy SDT provider so that we get called when a module is + * loaded. The provide_module method is responsible for creating probes + * defined in the module being loaded. + */ + error = dtrace_register("sdt", &sdt_attr, DTRACE_PRIV_USER, NULL, + &sdt_pops, NULL, &id); + if (error != 0) { + printf("sdt_load: failed to register SDT provider\n"); + return; + } + + /* Pick up probes from the kernel and already-loaded modules. */ + linker_file_foreach(sdt_linker_file_cb, &id); } static int sdt_unload() { - int error = 0; + + dtrace_sdt_modload = NULL; sdt_probe_func = sdt_probe_stub; - sdt_deregister_callbacks(); - destroy_dev(sdt_cdev); - return (error); + return (0); } /* ARGSUSED */ @@ -235,7 +377,6 @@ sdt_modevent(module_t mod __unused, int type, void *data __unused) default: error = EOPNOTSUPP; break; - } return (error); @@ -243,8 +384,10 @@ sdt_modevent(module_t mod __unused, int type, void *data __unused) /* ARGSUSED */ static int -sdt_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) +sdt_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, + struct thread *td __unused) { + return (0); } diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c index b3ab4df..72ef199 100644 --- a/sys/kern/kern_linker.c +++ b/sys/kern/kern_linker.c @@ -28,8 +28,9 @@ __FBSDID("$FreeBSD$"); #include "opt_ddb.h" -#include "opt_kld.h" #include "opt_hwpmc_hooks.h" +#include "opt_kdtrace.h" +#include "opt_kld.h" #include #include @@ -63,6 +64,14 @@ __FBSDID("$FreeBSD$"); #include #endif +#ifdef KDTRACE_HOOKS +#include +dtrace_modevt_t dtrace_modload; +dtrace_modevt_t dtrace_modunload; +dtrace_modevt_t dtrace_sdt_modload; +dtrace_modevt_t dtrace_sdt_modunload; +#endif + #ifdef KLD_DEBUG int kld_debug = 0; SYSCTL_INT(_debug, OID_AUTO, kld_debug, CTLFLAG_RW | CTLFLAG_TUN, @@ -582,8 +591,6 @@ linker_make_file(const char *pathname, linker_class_t lc) lf->ndeps = 0; lf->deps = NULL; lf->loadcnt = ++loadcnt; - lf->sdt_probes = NULL; - lf->sdt_nprobes = 0; STAILQ_INIT(&lf->common); TAILQ_INIT(&lf->modules); TAILQ_INSERT_TAIL(&linker_files, lf, link); @@ -1046,6 +1053,13 @@ kern_kldload(struct thread *td, const char *file, int *fileid) lf->userrefs++; if (fileid != NULL) *fileid = lf->id; +#ifdef KDTRACE_HOOKS + /* Make it possible for the SDT provider to register new providers. */ + if (dtrace_sdt_modload != NULL) + dtrace_sdt_modload(lf); + if (dtrace_modload != NULL) + dtrace_modload(lf); +#endif #ifdef HWPMC_HOOKS KLD_DOWNGRADE(); pkm.pm_file = lf->filename; @@ -1119,6 +1133,12 @@ kern_kldunload(struct thread *td, int fileid, int flags) pkm.pm_address = (uintptr_t) lf->address; pkm.pm_size = lf->size; #endif +#ifdef KDTRACE_HOOKS + if (dtrace_modunload != NULL) + dtrace_modunload(lf); + if (dtrace_sdt_modunload != NULL) + dtrace_sdt_modunload(lf); +#endif lf->userrefs--; error = linker_file_unload(lf, flags); if (error) diff --git a/sys/sys/dtrace_bsd.h b/sys/sys/dtrace_bsd.h index 2198b26..0c13321 100644 --- a/sys/sys/dtrace_bsd.h +++ b/sys/sys/dtrace_bsd.h @@ -40,6 +40,7 @@ struct vnode; struct reg; struct devstat; struct bio; +struct linker_file; /* * Cyclic clock function type definition used to hook the cyclic @@ -100,6 +101,20 @@ extern dtrace_execexit_func_t dtrace_fasttrap_exit; typedef void (*dtrace_malloc_probe_func_t)(u_int32_t, uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4); +/* Module load and unload handlers. */ +typedef void (*dtrace_modevt_t)(struct linker_file *); +extern dtrace_modevt_t dtrace_modload; +extern dtrace_modevt_t dtrace_modunload; +/* + * We have a separate handler for SDT so that the SDT provider can register new + * providers when a module is loaded. This cannot be done from SDT's + * module_provider method because of locking problems associated with calling + * back into the DTrace framework. Similarly, we cannot deregister a provider + * from say, the probe destroy method, so we use our own handler. + */ +extern dtrace_modevt_t dtrace_sdt_modload; +extern dtrace_modevt_t dtrace_sdt_modunload; + extern dtrace_malloc_probe_func_t dtrace_malloc_probe; /* dtnfsclient NFSv[34] access cache provider hooks. */ diff --git a/sys/sys/linker.h b/sys/sys/linker.h index 8ea140f..b2942f2 100644 --- a/sys/sys/linker.h +++ b/sys/sys/linker.h @@ -92,10 +92,6 @@ struct linker_file { */ int nenabled; /* number of enabled probes. */ int fbt_nentries; /* number of fbt entries created. */ - void *sdt_probes; - int sdt_nentries; - size_t sdt_nprobes; - size_t sdt_size; }; /* diff --git a/sys/sys/param.h b/sys/sys/param.h index 72bacce..b33049e 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -58,7 +58,7 @@ * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1000035 /* Master, propagated to newvers */ +#define __FreeBSD_version 1000036 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h index 21edd53..b67f33e 100644 --- a/sys/sys/sdt.h +++ b/sys/sys/sdt.h @@ -77,6 +77,9 @@ #else /* _KERNEL */ +#include +#include + #ifndef KDTRACE_HOOKS #define SDT_PROVIDER_DEFINE(prov) @@ -129,6 +132,7 @@ typedef enum { struct sdt_probe; struct sdt_provider; +struct linker_file; struct sdt_argtype { int ndx; /* Argument index. */ @@ -142,8 +146,7 @@ struct sdt_argtype { struct sdt_probe { int version; /* Set to sizeof(struct sdt_ref). */ sdt_state_t state; - struct sdt_provider - *prov; /* Ptr to the provider structure. */ + struct sdt_provider *prov; /* Ptr to the provider structure. */ TAILQ_ENTRY(sdt_probe) probe_entry; /* SDT probe list entry. */ TAILQ_HEAD(argtype_list_head, sdt_argtype) argtype_list; @@ -152,6 +155,7 @@ struct sdt_probe { const char *name; id_t id; /* DTrace probe ID. */ int n_args; /* Number of arguments. */ + struct linker_file *sdtp_lf; /* Module in which we're defined. */ }; struct sdt_provider { @@ -160,18 +164,15 @@ struct sdt_provider { prov_entry; /* SDT provider list entry. */ TAILQ_HEAD(probe_list_head, sdt_probe) probe_list; uintptr_t id; /* DTrace provider ID. */ + int sdt_refs; /* Number of module references. */ }; #define SDT_PROVIDER_DEFINE(prov) \ struct sdt_provider sdt_provider_##prov[1] = { \ - { #prov, { NULL, NULL }, { NULL, NULL } } \ + { #prov, { NULL, NULL }, { NULL, NULL }, 0 } \ }; \ - SYSINIT(sdt_provider_##prov##_init, SI_SUB_KDTRACE, \ - SI_ORDER_SECOND, sdt_provider_register, \ - sdt_provider_##prov ); \ - SYSUNINIT(sdt_provider_##prov##_uninit, SI_SUB_KDTRACE, \ - SI_ORDER_SECOND, sdt_provider_deregister, \ - sdt_provider_##prov ) + SET_DECLARE(sdt_providers_set, struct sdt_provider); \ + DATA_SET(sdt_providers_set, sdt_provider_##prov); #define SDT_PROVIDER_DECLARE(prov) \ extern struct sdt_provider sdt_provider_##prov[1] @@ -181,12 +182,8 @@ struct sdt_provider { { sizeof(struct sdt_probe), 0, sdt_provider_##prov, \ { NULL, NULL }, { NULL, NULL }, #mod, #func, #sname, 0, 0 } \ }; \ - SYSINIT(sdt_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ - SI_ORDER_SECOND + 1, sdt_probe_register, \ - sdt_##prov##_##mod##_##func##_##name ); \ - SYSUNINIT(sdt_##prov##_##mod##_##func##_##name##_uninit, \ - SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, sdt_probe_deregister, \ - sdt_##prov##_##mod##_##func##_##name ) + SET_DECLARE(sdt_probes_set, struct sdt_probe); \ + DATA_SET(sdt_probes_set, sdt_##prov##_##mod##_##func##_##name); #define SDT_PROBE_DECLARE(prov, mod, func, name) \ extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1] @@ -203,12 +200,8 @@ struct sdt_provider { = { { num, type, { NULL, NULL }, \ sdt_##prov##_##mod##_##func##_##name } \ }; \ - SYSINIT(sdt_##prov##_##mod##_##func##_##name##num##_init, \ - SI_SUB_KDTRACE, SI_ORDER_SECOND + 2, sdt_argtype_register, \ - sdt_##prov##_##mod##_##func##_##name##num ); \ - SYSUNINIT(sdt_##prov##_##mod##_##func##_##name##num##_uninit, \ - SI_SUB_KDTRACE, SI_ORDER_SECOND + 2, sdt_argtype_deregister, \ - sdt_##prov##_##mod##_##func##_##name##num ) + SET_DECLARE(sdt_argtypes_set, struct sdt_argtype); \ + DATA_SET(sdt_argtypes_set, sdt_##prov##_##mod##_##func##_##name##num); #define SDT_PROBE_DEFINE1(prov, mod, func, name, sname, arg0) \ SDT_PROBE_DEFINE(prov, mod, func, name, sname); \