diff -urN -x WARP9* /usr/src/sys/amd64/amd64/bootpatch.c src/sys/amd64/amd64/bootpatch.c --- /usr/src/sys/amd64/amd64/bootpatch.c Thu Jan 1 01:00:00 1970 +++ src/sys/amd64/amd64/bootpatch.c Sun May 28 00:13:41 2006 @@ -0,0 +1,81 @@ +/*- + * Copyright (c) 2006 Suleiman Souhlal + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD $"); + +#include +#include +#include +#include +#include + +#include +#include +#include + +static void bootpatch_do(struct bootpatch *bp); + +static void +bootpatch_do(struct bootpatch *bp) +{ + + KASSERT(bp->oldlen >= bp->newlen, ("Old instruction shorter than new " + "at %p", bp->oldp)); + /* + * If we wanted to undo the patching, we could swap the old and new + * instructions instead of doing a regular overwrite. + */ + memcpy(bp->oldp, bp->newp, bp->newlen); + + /* Put NOPs over the remaining bytes */ + if (bp->oldlen > bp->newlen) + memset(bp->oldp + bp->newlen, 0x90, bp->oldlen - bp->newlen); + + /* + * Since we do all the patching before turning on other cpus, + * We don't have to worry about invalidating their caches. + */ +} + +void +bootpatch_cpuid(void) +{ + struct bootpatch *bp; + + printf("Considering %d patches\n", + (int)((struct bootpatch *)bootpatchesend + - (struct bootpatch *)bootpatches)); + + for (bp = (struct bootpatch *)bootpatches; + bp < (struct bootpatch *)bootpatchesend; bp++) { + if (((bp->type == BP_CPUID) && (cpu_feature & bp->tag)) || + ((bp->type == BP_CPUID2) && (cpu_feature2 & bp->tag))) { + bootpatch_do(bp); + } + } +} + diff -urN -x WARP9* /usr/src/sys/amd64/amd64/machdep.c src/sys/amd64/amd64/machdep.c --- /usr/src/sys/amd64/amd64/machdep.c Fri May 26 13:10:45 2006 +++ src/sys/amd64/amd64/machdep.c Sat May 27 23:45:15 2006 @@ -123,6 +123,7 @@ #ifdef SMP #include #endif +#include #include #include @@ -226,6 +227,8 @@ vm_pager_bufferinit(); cpu_setregs(); + + bootpatch_cpuid(); } /* diff -urN -x WARP9* /usr/src/sys/amd64/include/bootpatch.h src/sys/amd64/include/bootpatch.h --- /usr/src/sys/amd64/include/bootpatch.h Thu Jan 1 01:00:00 1970 +++ src/sys/amd64/include/bootpatch.h Sun May 28 00:10:44 2006 @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2006 Suleiman Souhlal + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD $ + */ + +#ifndef _MACHINE_BOOTPATCH_H +#define _MACHINE_BOOTPATCH_H + +struct bootpatch { + char *oldp __packed; + char *newp __packed; + long tag __packed; + char oldlen __packed; + char newlen __packed; + char type __packed; + char pad[5]; +}; + +#define BP_CPUID 0 +#define BP_CPUID2 1 +#define BP_BOOTARG 2 + +#define BOOTPATCH(old, new, type, tag) \ + __asm __volatile("666: " old " 667:" \ + ".section .bootpatch_new, \"a\"; 668: " new " 669: .previous;" \ + ".section .bootpatch, \"a\";" \ + ".align 8;" \ + ".quad 666b;" \ + ".quad 668b;" \ + ".quad (%c1);" \ + ".byte 667b - 666b;" \ + ".byte 669b - 668b;" \ + ".byte (%c0);" \ + ".previous;" \ + : : ) + +#define BOOTPATCH_INPUT(old, new, type, tag, input...) \ + __asm __volatile("666:\n " old "\n 667:" \ + ".section .bootpatch_new, \"a\"; 668: " new " 669: .previous;" \ + ".section .bootpatch, \"a\";" \ + ".align 8;" \ + ".quad 666b;" \ + ".quad 668b;" \ + ".quad (%c1);" \ + ".byte 667b - 666b;" \ + ".byte 669b - 668b;" \ + ".byte (%c0);" \ + ".previous;" \ + : : "i" (type), "i" (tag), input) + +extern char bootpatches[]; +extern char bootpatchesend[]; + +void bootpatch_cpuid(void); + +#endif diff -urN -x WARP9* /usr/src/sys/amd64/include/cpufunc.h src/sys/amd64/include/cpufunc.h --- /usr/src/sys/amd64/include/cpufunc.h Fri May 13 02:05:55 2005 +++ src/sys/amd64/include/cpufunc.h Sat May 27 23:57:22 2006 @@ -515,6 +515,17 @@ } #ifdef _KERNEL +#include +#include + +static __inline int +prefetch(void *p) +{ + BOOTPATCH_INPUT("prefetchnta (%2);", "prefetch (%2);", BP_CPUID, + CPUID_SSE2, "r" (p)); + return (1); +} + /* This is defined in but is too painful to get to */ #ifndef MSR_FSBASE #define MSR_FSBASE 0xc0000100 diff -urN -x WARP9* /usr/src/sys/conf/files.amd64 src/sys/conf/files.amd64 --- /usr/src/sys/conf/files.amd64 Fri May 26 13:10:51 2006 +++ src/sys/conf/files.amd64 Sat May 27 17:59:20 2006 @@ -86,6 +86,7 @@ amd64/amd64/atomic.c standard amd64/amd64/autoconf.c standard amd64/amd64/bios.c standard +amd64/amd64/bootpatch.c standard amd64/amd64/bpf_jit_machdep.c optional bpf_jitter amd64/amd64/busdma_machdep.c standard amd64/amd64/cpu_switch.S standard diff -urN -x WARP9* /usr/src/sys/conf/ldscript.amd64 src/sys/conf/ldscript.amd64 --- /usr/src/sys/conf/ldscript.amd64 Sat May 29 03:09:00 2004 +++ src/sys/conf/ldscript.amd64 Sat May 27 23:10:39 2006 @@ -56,6 +56,11 @@ PROVIDE (__etext = .); PROVIDE (_etext = .); PROVIDE (etext = .); + . = ALIGN(8); + PROVIDE (bootpatches = .); + .bootpatch : { *(.bootpatch) } + PROVIDE (bootpatchesend = .); + .bootpatch_new : { *(.bootpatch_new) } .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } .rodata1 : { *(.rodata1) } .eh_frame_hdr : { *(.eh_frame_hdr) } diff -urN -x WARP9* /usr/src/sys/sys/queue.h src/sys/sys/queue.h --- /usr/src/sys/sys/queue.h Fri May 26 13:11:15 2006 +++ src/sys/sys/queue.h Sat May 27 23:53:44 2006 @@ -34,6 +34,7 @@ #define _SYS_QUEUE_H_ #include +#include /* * This file defines four types of data structures: singly-linked lists, @@ -159,7 +160,7 @@ #define SLIST_FOREACH(var, head, field) \ for ((var) = SLIST_FIRST((head)); \ (var); \ - (var) = SLIST_NEXT((var), field)) + (var) = SLIST_NEXT((var), field), (var) && prefetch(SLIST_NEXT((var), field))) #define SLIST_FOREACH_SAFE(var, head, field, tvar) \ for ((var) = SLIST_FIRST((head)); \ @@ -240,7 +241,7 @@ #define STAILQ_FOREACH(var, head, field) \ for((var) = STAILQ_FIRST((head)); \ (var); \ - (var) = STAILQ_NEXT((var), field)) + (var) = STAILQ_NEXT((var), field), (var) && prefetch(STAILQ_NEXT((var), field))) #define STAILQ_FOREACH_SAFE(var, head, field, tvar) \ @@ -358,7 +359,7 @@ #define LIST_FOREACH(var, head, field) \ for ((var) = LIST_FIRST((head)); \ (var); \ - (var) = LIST_NEXT((var), field)) + (var) = LIST_NEXT((var), field), (var) && prefetch(LIST_NEXT((var), field))) #define LIST_FOREACH_SAFE(var, head, field, tvar) \ for ((var) = LIST_FIRST((head)); \ @@ -448,7 +449,7 @@ #define TAILQ_FOREACH(var, head, field) \ for ((var) = TAILQ_FIRST((head)); \ (var); \ - (var) = TAILQ_NEXT((var), field)) + (var) = TAILQ_NEXT((var), field), (var) && prefetch(TAILQ_NEXT((var), field))) #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ for ((var) = TAILQ_FIRST((head)); \