diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index 67146240ba58..6f6dbd49211f 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -90,6 +90,7 @@ PMC_SOFT_DEFINE( , , page_fault, write); #ifdef SMP #include #endif +#include #include #include #include @@ -248,19 +249,8 @@ trap(struct trapframe *frame) return; } - if (type == T_NMI) { -#ifdef HWPMC_HOOKS - /* - * CPU PMCs interrupt using an NMI. If the PMC module is - * active, pass the 'rip' value to the PMC module's interrupt - * handler. A non-zero return value from the handler means that - * the NMI was consumed by it and we can return immediately. - */ - if (pmc_intr != NULL && - (*pmc_intr)(frame) != 0) - return; -#endif - } + if (type == T_NMI && nmi_handle_pcint(frame)) + return; if ((frame->tf_rflags & PSL_I) == 0) { /* diff --git a/sys/dev/hwpmc/hwpmc_x86.c b/sys/dev/hwpmc/hwpmc_x86.c index 1d04a6610674..04f6dbc0e72a 100644 --- a/sys/dev/hwpmc/hwpmc_x86.c +++ b/sys/dev/hwpmc/hwpmc_x86.c @@ -242,12 +242,13 @@ pmc_md_initialize(void) return (NULL); /* disallow sampling if we do not have an LAPIC */ - if (md != NULL && !lapic_enable_pmc()) + if (md != NULL && !lapic_enable_pcint()) for (i = 0; i < md->pmd_nclass; i++) { if (i == PMC_CLASS_INDEX_SOFT) continue; md->pmd_classdep[i].pcd_caps &= ~PMC_CAP_INTERRUPT; } + nmi_register_pcint_handler(md->pmd_intr); return (md); } @@ -256,7 +257,8 @@ void pmc_md_finalize(struct pmc_mdep *md) { - lapic_disable_pmc(); + lapic_disable_pcint(); + nmi_remove_pcint_handler(md->pmd_intr); if (cpu_vendor_id == CPU_VENDOR_AMD || cpu_vendor_id == CPU_VENDOR_HYGON) pmc_amd_finalize(md); diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h index f19c557e270b..6ac8309dc741 100644 --- a/sys/x86/include/x86_var.h +++ b/sys/x86/include/x86_var.h @@ -147,6 +147,9 @@ void zenbleed_sanitize_enable(void); void zenbleed_check_and_apply(bool all_cpus); void nmi_call_kdb(u_int cpu, u_int type, struct trapframe *frame); void nmi_call_kdb_smp(u_int type, struct trapframe *frame); +void nmi_register_pcint_handler(int (*handler)(struct trapframe *)); +void nmi_remove_pcint_handler(int (*handler)(struct trapframe *)); +bool nmi_handle_pcint(struct trapframe *frame); void nmi_handle_intr(u_int type, struct trapframe *frame); void pagecopy(void *from, void *to); void printcpuinfo(void); diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c index 5f6cc35dda6f..3904dfac5ffc 100644 --- a/sys/x86/x86/cpu_machdep.c +++ b/sys/x86/x86/cpu_machdep.c @@ -87,6 +87,7 @@ #include #include +#include #include #include #include @@ -898,6 +899,75 @@ nmi_handle_intr(u_int type, struct trapframe *frame) nmi_call_kdb(PCPU_GET(cpuid), type, frame); } +/* + * Dynamically registered PCINT NMI handlers. + */ +static struct nmi_pcint_handler { + int (*func)(struct trapframe *); + struct nmi_pcint_handler *next; +} pcint_handlers_head; +static uma_zone_t pcint_handler_zone; + +void +nmi_register_pcint_handler(int (*handler)(struct trapframe *)) +{ + struct nmi_pcint_handler *hp; + + for (hp = &pcint_handlers_head; hp != NULL; hp = hp->next) { + if (atomic_load_ptr(&hp->func) == handler || + atomic_cmpset_ptr((volatile uintptr_t *)&hp->func, + (uintptr_t)NULL, (uintptr_t)handler) != 0) + return; + } + hp = uma_zalloc(pcint_handler_zone, M_WAITOK | M_ZERO); + hp->func = handler; + do { + hp->next = pcint_handlers_head.next; + } while (atomic_cmpset_ptr((volatile u_long *)&pcint_handlers_head.next, + (uintptr_t)hp->next, (uintptr_t)hp) == 0); +} +void +nmi_remove_pcint_handler(int (*handler)(struct trapframe *)) +{ + struct nmi_pcint_handler *hp; + + for (hp = &pcint_handlers_head; hp != NULL; hp = hp->next) { + if (atomic_cmpset_ptr((volatile u_long *)&hp->func, + (uintptr_t)handler, (uintptr_t)NULL) != 0) + return; + } + + panic("%s: attempting to remove an unregistered PCINT handler %p\n", + __func__, handler); +} + +bool +nmi_handle_pcint(struct trapframe *frame) +{ + bool handled; + struct nmi_pcint_handler *hp; + + lapic_toggle_pcint(false); + handled = false; + for (hp = &pcint_handlers_head; hp != NULL; hp = hp->next) { + if (hp->func != NULL && (*hp->func)(frame) != 0) + handled = true; + } + lapic_toggle_pcint(true); + + return (handled); +} + +static void +cpu_init_pcint_handlers(void *arg __unused) +{ + pcint_handler_zone = uma_zcreate("Dynamically registered PCINT handlers", + sizeof(struct nmi_pcint_handler), NULL, NULL, NULL, NULL, 0, + UMA_ZONE_NOFREE); +} +SYSINIT(cpu_init_pcint_handlers, SI_SUB_CPU, SI_ORDER_ANY, cpu_init_pcint_handlers, + NULL); + static int hw_ibrs_active; int hw_ibrs_ibpb_active; int hw_ibrs_disable = 1;