diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 778d07689ff0..0e3e755473a4 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -11682,6 +11682,35 @@ pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) return (error); } +void +pmap_qenter_zcond(pmap_t pmap, vm_page_t m, vm_offset_t sva) { + pt_entry_t oldpte, pa; + pt_entry_t *pte; + int cache_bits; + + oldpte = 0; + pte = pmap_pte(pmap, sva); + + cache_bits = pmap_cache_bits(pmap, m->md.pat_mode, false); + pa = VM_PAGE_TO_PHYS(m) | cache_bits; + if ((*pte & (PG_FRAME | X86_PG_PTE_CACHE)) != pa) { + oldpte |= *pte; + pte_store(pte, pa | pg_g | pg_nx | X86_PG_A | + X86_PG_M | X86_PG_RW | X86_PG_V); + } + + if (__predict_false((oldpte & X86_PG_V) != 0)) + pmap_invalidate_range(kernel_pmap, sva, sva + PAGE_SIZE); +} + +void +pmap_qremove_zcond(pmap_t pmap, vm_offset_t sva) { + pt_entry_t *pte; + + pte = pmap_pte(pmap, sva); + pte_clear(pte); +} + #if defined(KASAN) || defined(KMSAN) /* diff --git a/sys/amd64/amd64/zcond_machdep.c b/sys/amd64/amd64/zcond_machdep.c index 4c6898a67bc0..1683e5fe7157 100644 --- a/sys/amd64/amd64/zcond_machdep.c +++ b/sys/amd64/amd64/zcond_machdep.c @@ -22,7 +22,7 @@ zcond_after_patch(void) { // restore_wp(wp); // load_cr3(cr3); - invltlb(); + mfence(); } void diff --git a/sys/kern/kern_zcond.c b/sys/kern/kern_zcond.c index becdb50e2a99..8dbc940fa3c1 100644 --- a/sys/kern/kern_zcond.c +++ b/sys/kern/kern_zcond.c @@ -138,24 +138,22 @@ __zcond_set_enabled(struct zcond *cond, bool new_state) //KASSERT(INKERNEL(p->patch_addr), ("inspection point patch address outside of kernel: %#08lx", p->patch_addr)); p->mirror_address = kva_alloc(PAGE_SIZE); patch_page = PHYS_TO_VM_PAGE(vtophys(p->patch_addr)); - KASSERT(patch_page != NULL, ("patch page is NULL")); + KASSERT(patch_page != NULL, ("patch page is NULL")); - pmap_enter(&zcond_patching_pmap, p->mirror_address, patch_page, - VM_PROT_WRITE, PMAP_ENTER_WIRED, 0); - pmap_invalidate_page(kernel_pmap, p->patch_addr & (~PAGE_MASK)); - printf("patch_point %#08lx mapped to %#08lx\n", p->patch_addr, - p->mirror_address); + pmap_qenter_zcond(&zcond_patching_pmap, patch_page, p->mirror_address); + pmap_invalidate_page(kernel_pmap, p->patch_addr & (~PAGE_MASK)); + printf("patch_point %#08lx mapped to %#08lx\n", p->patch_addr, + p->mirror_address); } zcond_before_rendezvous(); smp_rendezvous(NULL, rendezvous_cb, NULL, &arg); zcond_after_rendezvous(); - + + invltlb(); SLIST_FOREACH(p, &cond->ins_points, next) { - pmap_qremove(p->mirror_address, 1); + pmap_qremove_zcond(&zcond_patching_pmap, p->mirror_address); kva_free(p->mirror_address, PAGE_SIZE); - - pmap_invalidate_page(kernel_pmap, p->patch_addr & (!PAGE_MASK)); } } diff --git a/sys/sys/zcond.h b/sys/sys/zcond.h index 02284ce071a4..d2b9f6d53479 100644 --- a/sys/sys/zcond.h +++ b/sys/sys/zcond.h @@ -182,5 +182,6 @@ void zcond_after_rendezvous(void); */ void zcond_get_patch_insn(struct ins_point *ins_p, unsigned char insn[], size_t *size); + #endif #endif diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h index 9101201287b2..f4f4483f8fc7 100644 --- a/sys/vm/pmap.h +++ b/sys/vm/pmap.h @@ -166,6 +166,8 @@ int pmap_ts_referenced(vm_page_t m); void pmap_unwire(pmap_t pmap, vm_offset_t start, vm_offset_t end); void pmap_zero_page(vm_page_t); void pmap_zero_page_area(vm_page_t, int off, int size); +void pmap_qenter_zcond(pmap_t pmap, vm_page_t m, vm_offset_t va); +void pmap_qremove_zcond(pmap_t pmap, vm_offset_t va); #define pmap_resident_count(pm) ((pm)->pm_stats.resident_count) #define pmap_wired_count(pm) ((pm)->pm_stats.wired_count)