diff --git a/share/man/man9/EVENTHANDLER.9 b/share/man/man9/EVENTHANDLER.9 index c3b646d..d6e7556 100644 --- a/share/man/man9/EVENTHANDLER.9 +++ b/share/man/man9/EVENTHANDLER.9 @@ -202,6 +202,8 @@ Callbacks invoked when a BPF listener attaches to/detaches from network interfac .It Vt kld_load Callbacks invoked after a linker file has been loaded. .It Vt kld_unload +Callbacks invoked after a linker file has been successfully unloaded. +.It Vt kld_unload_try Callbacks invoked before a linker file is about to be unloaded. These callbacks may be used to return an error and prevent the unload from proceeding. diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c index 9c007e7..babc42c4 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c +++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c @@ -242,7 +242,7 @@ int dtrace_in_probe; /* non-zero if executing a probe */ uintptr_t dtrace_in_probe_addr; /* Address of invop when already in probe */ #endif static eventhandler_tag dtrace_kld_load_tag; -static eventhandler_tag dtrace_kld_unload_tag; +static eventhandler_tag dtrace_kld_unload_try_tag; #endif /* @@ -15351,7 +15351,7 @@ dtrace_kld_load(void *arg __unused, linker_file_t lf) } static void -dtrace_kld_unload(void *arg __unused, linker_file_t lf, int *error) +dtrace_kld_unload_try(void *arg __unused, linker_file_t lf, int *error) { if (*error != 0) diff --git a/sys/cddl/dev/dtrace/dtrace_load.c b/sys/cddl/dev/dtrace/dtrace_load.c index f37d593..05436fe 100644 --- a/sys/cddl/dev/dtrace/dtrace_load.c +++ b/sys/cddl/dev/dtrace/dtrace_load.c @@ -59,8 +59,8 @@ dtrace_load(void *dummy) /* Register callbacks for linker file load and unload events. */ dtrace_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, dtrace_kld_load, NULL, EVENTHANDLER_PRI_ANY); - dtrace_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, - dtrace_kld_unload, NULL, EVENTHANDLER_PRI_ANY); + dtrace_kld_unload_try_tag = EVENTHANDLER_REGISTER(kld_unload_try, + dtrace_kld_unload_try, NULL, EVENTHANDLER_PRI_ANY); /* * Initialise the mutexes without 'witness' because the dtrace diff --git a/sys/cddl/dev/dtrace/dtrace_unload.c b/sys/cddl/dev/dtrace/dtrace_unload.c index 9e37114..e9bd53c 100644 --- a/sys/cddl/dev/dtrace/dtrace_unload.c +++ b/sys/cddl/dev/dtrace/dtrace_unload.c @@ -68,7 +68,7 @@ dtrace_unload() dtrace_provider = NULL; EVENTHANDLER_DEREGISTER(kld_load, dtrace_kld_load_tag); - EVENTHANDLER_DEREGISTER(kld_unload, dtrace_kld_unload_tag); + EVENTHANDLER_DEREGISTER(kld_unload_try, dtrace_kld_unload_try_tag); if ((state = dtrace_anon_grab()) != NULL) { /* diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c index a1cd1e2..80f676f 100644 --- a/sys/cddl/dev/sdt/sdt.c +++ b/sys/cddl/dev/sdt/sdt.c @@ -59,7 +59,7 @@ static int sdt_unload(void *); static void sdt_create_provider(struct sdt_provider *); static void sdt_create_probe(struct sdt_probe *); static void sdt_kld_load(void *, struct linker_file *); -static void sdt_kld_unload(void *, struct linker_file *, int *); +static void sdt_kld_unload_try(void *, struct linker_file *, int *); static MALLOC_DEFINE(M_SDT, "SDT", "DTrace SDT providers"); @@ -95,7 +95,7 @@ static struct cdev *sdt_cdev; static TAILQ_HEAD(, sdt_provider) sdt_prov_list; eventhandler_tag sdt_kld_load_tag; -eventhandler_tag sdt_kld_unload_tag; +eventhandler_tag sdt_kld_unload_try_tag; static void sdt_create_provider(struct sdt_provider *prov) @@ -260,7 +260,7 @@ sdt_kld_load(void *arg __unused, struct linker_file *lf) } static void -sdt_kld_unload(void *arg __unused, struct linker_file *lf, int *error __unused) +sdt_kld_unload_try(void *arg __unused, struct linker_file *lf, int *error __unused) { struct sdt_provider *prov, **curr, **begin, **end, *tmp; @@ -315,8 +315,8 @@ sdt_load(void *arg __unused) sdt_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, sdt_kld_load, NULL, EVENTHANDLER_PRI_ANY); - sdt_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, sdt_kld_unload, - NULL, EVENTHANDLER_PRI_ANY); + sdt_kld_unload_try_tag = EVENTHANDLER_REGISTER(kld_unload_try, + sdt_kld_unload_try, NULL, EVENTHANDLER_PRI_ANY); /* Pick up probes from the kernel and already-loaded linker files. */ linker_file_foreach(sdt_linker_file_cb, NULL); @@ -328,7 +328,7 @@ sdt_unload(void *arg __unused) struct sdt_provider *prov, *tmp; EVENTHANDLER_DEREGISTER(kld_load, sdt_kld_load_tag); - EVENTHANDLER_DEREGISTER(kld_unload, sdt_kld_unload_tag); + EVENTHANDLER_DEREGISTER(kld_unload_try, sdt_kld_unload_try_tag); sdt_probe_func = sdt_probe_stub; diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c index 150c69a..86242d9 100644 --- a/sys/dev/hwpmc/hwpmc_mod.c +++ b/sys/dev/hwpmc/hwpmc_mod.c @@ -132,7 +132,8 @@ static int *pmc_pmcdisp; /* PMC row dispositions */ /* various event handlers */ -static eventhandler_tag pmc_exit_tag, pmc_fork_tag; +static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag, + pmc_kld_unload_tag; /* Module statistics */ struct pmc_op_getdriverstats pmc_stats; @@ -1476,50 +1477,6 @@ pmc_process_csw_out(struct thread *td) } /* - * Log a KLD operation. - */ - -static void -pmc_process_kld_load(struct pmckern_map_in *pkm) -{ - struct pmc_owner *po; - - sx_assert(&pmc_sx, SX_LOCKED); - - /* - * Notify owners of system sampling PMCs about KLD operations. - */ - - LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) - if (po->po_flags & PMC_PO_OWNS_LOGFILE) - pmclog_process_map_in(po, (pid_t) -1, pkm->pm_address, - (char *) pkm->pm_file); - - /* - * TODO: Notify owners of (all) process-sampling PMCs too. - */ - - return; -} - -static void -pmc_process_kld_unload(struct pmckern_map_out *pkm) -{ - struct pmc_owner *po; - - sx_assert(&pmc_sx, SX_LOCKED); - - LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) - if (po->po_flags & PMC_PO_OWNS_LOGFILE) - pmclog_process_map_out(po, (pid_t) -1, - pkm->pm_address, pkm->pm_address + pkm->pm_size); - - /* - * TODO: Notify owners of process-sampling PMCs. - */ -} - -/* * A mapping change for a process. */ @@ -1833,8 +1790,8 @@ const char *pmc_hooknames[] = { "CSW-IN", "CSW-OUT", "SAMPLE", - "KLDLOAD", - "KLDUNLOAD", + "UNUSED1", + "UNUSED2", "MMAP", "MUNMAP", "CALLCHAIN-NMI", @@ -2002,17 +1959,6 @@ pmc_hook_handler(struct thread *td, int function, void *arg) pmc_process_samples(PCPU_GET(cpuid), PMC_SR); break; - - case PMC_FN_KLD_LOAD: - sx_assert(&pmc_sx, SX_LOCKED); - pmc_process_kld_load((struct pmckern_map_in *) arg); - break; - - case PMC_FN_KLD_UNLOAD: - sx_assert(&pmc_sx, SX_LOCKED); - pmc_process_kld_unload((struct pmckern_map_out *) arg); - break; - case PMC_FN_MMAP: sx_assert(&pmc_sx, SX_LOCKED); pmc_process_mmap(td, (struct pmckern_map_in *) arg); @@ -4644,6 +4590,47 @@ pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc, sx_xunlock(&pmc_sx); } +static void +pmc_kld_load(void *arg __unused, linker_file_t lf) +{ + struct pmc_owner *po; + + sx_slock(&pmc_sx); + + /* + * Notify owners of system sampling PMCs about KLD operations. + */ + LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) + if (po->po_flags & PMC_PO_OWNS_LOGFILE) + pmclog_process_map_in(po, (pid_t) -1, + (uintfptr_t) lf->address, lf->filename); + + /* + * TODO: Notify owners of (all) process-sampling PMCs too. + */ + + sx_sunlock(&pmc_sx); +} + +static void +pmc_kld_unload(void *arg __unused, const char *filename __unused, + caddr_t address, size_t size) +{ + struct pmc_owner *po; + + sx_slock(&pmc_sx); + + LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) + if (po->po_flags & PMC_PO_OWNS_LOGFILE) + pmclog_process_map_out(po, (pid_t) -1, + (uintfptr_t) address, (uintfptr_t) address + size); + + /* + * TODO: Notify owners of process-sampling PMCs. + */ + + sx_sunlock(&pmc_sx); +} /* * initialization @@ -4913,6 +4900,12 @@ pmc_initialize(void) pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork, pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY); + /* register kld event handlers */ + pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load, + NULL, EVENTHANDLER_PRI_ANY); + pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload, + NULL, EVENTHANDLER_PRI_ANY); + /* initialize logging */ pmclog_initialize(); @@ -4970,6 +4963,8 @@ pmc_cleanup(void) /* deregister event handlers */ EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag); EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag); + EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag); + EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag); /* send SIGBUS to all owner threads, free up allocations */ if (pmc_ownerhash) diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c index 3e500f3..0ddd21b 100644 --- a/sys/kern/kern_linker.c +++ b/sys/kern/kern_linker.c @@ -81,6 +81,10 @@ TUNABLE_INT("debug.kld_debug", &kld_debug); if (!cold) \ sx_assert(&kld_sx, SX_XLOCKED); \ } while (0) +#define KLD_LOCK_READ_ASSERT() do { \ + if (!cold) \ + sx_assert(&kld_sx, SX_LOCKED); \ +} while (0) /* * static char *linker_search_path(const char *name, struct mod_depend @@ -208,6 +212,8 @@ linker_file_sysinit(linker_file_t lf) KLD_DPF(FILE, ("linker_file_sysinit: calling SYSINITs for %s\n", lf->filename)); + KLD_LOCK_ASSERT(); + if (linker_file_lookup_set(lf, "sysinit_set", &start, &stop, NULL) != 0) return; /* @@ -233,6 +239,7 @@ linker_file_sysinit(linker_file_t lf) * Traverse the (now) ordered list of system initialization tasks. * Perform each task, and continue on to the next task. */ + KLD_UNLOCK(); mtx_lock(&Giant); for (sipp = start; sipp < stop; sipp++) { if ((*sipp)->subsystem == SI_SUB_DUMMY) @@ -242,6 +249,7 @@ linker_file_sysinit(linker_file_t lf) (*((*sipp)->func)) ((*sipp)->udata); } mtx_unlock(&Giant); + KLD_LOCK(); } static void @@ -252,6 +260,8 @@ linker_file_sysuninit(linker_file_t lf) KLD_DPF(FILE, ("linker_file_sysuninit: calling SYSUNINITs for %s\n", lf->filename)); + KLD_LOCK_ASSERT(); + if (linker_file_lookup_set(lf, "sysuninit_set", &start, &stop, NULL) != 0) return; @@ -279,6 +289,7 @@ linker_file_sysuninit(linker_file_t lf) * Traverse the (now) ordered list of system initialization tasks. * Perform each task, and continue on to the next task. */ + KLD_UNLOCK(); mtx_lock(&Giant); for (sipp = start; sipp < stop; sipp++) { if ((*sipp)->subsystem == SI_SUB_DUMMY) @@ -288,6 +299,7 @@ linker_file_sysuninit(linker_file_t lf) (*((*sipp)->func)) ((*sipp)->udata); } mtx_unlock(&Giant); + KLD_LOCK(); } static void @@ -299,13 +311,17 @@ linker_file_register_sysctls(linker_file_t lf) ("linker_file_register_sysctls: registering SYSCTLs for %s\n", lf->filename)); + KLD_LOCK_ASSERT(); + if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0) return; + KLD_UNLOCK(); sysctl_lock(); for (oidp = start; oidp < stop; oidp++) sysctl_register_oid(*oidp); sysctl_unlock(); + KLD_LOCK(); } static void @@ -316,13 +332,17 @@ linker_file_unregister_sysctls(linker_file_t lf) KLD_DPF(FILE, ("linker_file_unregister_sysctls: unregistering SYSCTLs" " for %s\n", lf->filename)); + KLD_LOCK_ASSERT(); + if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0) return; + KLD_UNLOCK(); sysctl_lock(); for (oidp = start; oidp < stop; oidp++) sysctl_unregister_oid(*oidp); sysctl_unlock(); + KLD_LOCK(); } static int @@ -335,6 +355,8 @@ linker_file_register_modules(linker_file_t lf) KLD_DPF(FILE, ("linker_file_register_modules: registering modules" " in %s\n", lf->filename)); + KLD_LOCK_ASSERT(); + if (linker_file_lookup_set(lf, "modmetadata_set", &start, &stop, NULL) != 0) { /* @@ -421,10 +443,8 @@ linker_load_file(const char *filename, linker_file_t *result) return (error); } modules = !TAILQ_EMPTY(&lf->modules); - KLD_UNLOCK(); linker_file_register_sysctls(lf); linker_file_sysinit(lf); - KLD_LOCK(); lf->flags |= LINKER_FILE_LINKED; /* @@ -674,10 +694,8 @@ linker_file_unload(linker_file_t file, int flags) */ if (file->flags & LINKER_FILE_LINKED) { file->flags &= ~LINKER_FILE_LINKED; - KLD_UNLOCK(); linker_file_sysuninit(file); linker_file_unregister_sysctls(file); - KLD_LOCK(); } TAILQ_REMOVE(&linker_files, file, link); @@ -746,15 +764,9 @@ int linker_file_lookup_set(linker_file_t file, const char *name, void *firstp, void *lastp, int *countp) { - int error, locked; - locked = KLD_LOCKED(); - if (!locked) - KLD_LOCK(); - error = LINKER_LOOKUP_SET(file, name, firstp, lastp, countp); - if (!locked) - KLD_UNLOCK(); - return (error); + KLD_LOCK_READ_ASSERT(); + return (LINKER_LOOKUP_SET(file, name, firstp, lastp, countp)); } /* @@ -1004,9 +1016,6 @@ linker_search_symbol_name(caddr_t value, char *buf, u_int buflen, int kern_kldload(struct thread *td, const char *file, int *fileid) { -#ifdef HWPMC_HOOKS - struct pmckern_map_in pkm; -#endif const char *kldname, *modname; linker_file_t lf; int error; @@ -1046,17 +1055,9 @@ kern_kldload(struct thread *td, const char *file, int *fileid) if (fileid != NULL) *fileid = lf->id; - EVENTHANDLER_INVOKE(kld_load, lf); - -#ifdef HWPMC_HOOKS KLD_DOWNGRADE(); - pkm.pm_file = lf->filename; - pkm.pm_address = (uintptr_t) lf->address; - PMC_CALL_HOOK(td, PMC_FN_KLD_LOAD, (void *) &pkm); + EVENTHANDLER_INVOKE(kld_load, lf); KLD_UNLOCK_READ(); -#else - KLD_UNLOCK(); -#endif done: CURVNET_RESTORE(); @@ -1085,10 +1086,10 @@ sys_kldload(struct thread *td, struct kldload_args *uap) int kern_kldunload(struct thread *td, int fileid, int flags) { -#ifdef HWPMC_HOOKS - struct pmckern_map_out pkm; -#endif linker_file_t lf; + char *filename = NULL; + caddr_t address; + size_t size; int error = 0; if ((error = securelevel_gt(td->td_ucred, 0)) != 0) @@ -1103,7 +1104,7 @@ kern_kldunload(struct thread *td, int fileid, int flags) if (lf) { KLD_DPF(FILE, ("kldunload: lf->userrefs=%d\n", lf->userrefs)); - EVENTHANDLER_INVOKE(kld_unload, lf, &error); + EVENTHANDLER_INVOKE(kld_unload_try, lf, &error); if (error != 0) error = EBUSY; else if (lf->userrefs == 0) { @@ -1114,11 +1115,11 @@ kern_kldunload(struct thread *td, int fileid, int flags) " loaded by the kernel\n"); error = EBUSY; } else { -#ifdef HWPMC_HOOKS - /* Save data needed by hwpmc(4) before unloading. */ - pkm.pm_address = (uintptr_t) lf->address; - pkm.pm_size = lf->size; -#endif + /* Save data needed for the kld_unload callbacks. */ + filename = strdup(lf->filename, M_TEMP); + address = lf->address; + size = lf->size; + lf->userrefs--; error = linker_file_unload(lf, flags); if (error) @@ -1127,16 +1128,14 @@ kern_kldunload(struct thread *td, int fileid, int flags) } else error = ENOENT; -#ifdef HWPMC_HOOKS if (error == 0) { KLD_DOWNGRADE(); - PMC_CALL_HOOK(td, PMC_FN_KLD_UNLOAD, (void *) &pkm); + EVENTHANDLER_INVOKE(kld_unload, filename, address, size); KLD_UNLOCK_READ(); } else KLD_UNLOCK(); -#else - KLD_UNLOCK(); -#endif + free(filename, M_TEMP); + CURVNET_RESTORE(); return (error); } @@ -1648,11 +1647,13 @@ restart: lf->filename); goto fail; } + KLD_LOCK(); linker_file_register_modules(lf); if (linker_file_lookup_set(lf, "sysinit_set", &si_start, &si_stop, NULL) == 0) sysinit_add(si_start, si_stop); linker_file_register_sysctls(lf); + KLD_UNLOCK(); lf->flags |= LINKER_FILE_LINKED; continue; fail: diff --git a/sys/sys/eventhandler.h b/sys/sys/eventhandler.h index a1f9c55..2656877 100644 --- a/sys/sys/eventhandler.h +++ b/sys/sys/eventhandler.h @@ -269,8 +269,11 @@ EVENTHANDLER_DECLARE(maxsockets_change, uma_zone_chfn); /* Kernel linker file load and unload events */ struct linker_file; typedef void (*kld_load_fn)(void *, struct linker_file *); -typedef void (*kld_unload_fn)(void *, struct linker_file *, int *); +typedef void (*kld_unload_fn)(void *, const char *filename, caddr_t addr, + size_t size); +typedef void (*kld_unload_try_fn)(void *, struct linker_file *, int *); EVENTHANDLER_DECLARE(kld_load, kld_load_fn); EVENTHANDLER_DECLARE(kld_unload, kld_unload_fn); +EVENTHANDLER_DECLARE(kld_unload_try, kld_unload_try_fn); #endif /* SYS_EVENTHANDLER_H */ diff --git a/sys/sys/pmckern.h b/sys/sys/pmckern.h index e3e18a6..70b9b3f 100644 --- a/sys/sys/pmckern.h +++ b/sys/sys/pmckern.h @@ -51,8 +51,8 @@ #define PMC_FN_CSW_IN 2 #define PMC_FN_CSW_OUT 3 #define PMC_FN_DO_SAMPLES 4 -#define PMC_FN_KLD_LOAD 5 -#define PMC_FN_KLD_UNLOAD 6 +#define PMC_FN_UNUSED1 5 +#define PMC_FN_UNUSED2 6 #define PMC_FN_MMAP 7 #define PMC_FN_MUNMAP 8 #define PMC_FN_USER_CALLCHAIN 9