Index: sys/amd64/include/atomic.h =================================================================== --- sys/amd64/include/atomic.h (revision 253861) +++ sys/amd64/include/atomic.h (working copy) @@ -54,6 +54,7 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) @@ -60,6 +61,7 @@ * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -80,6 +82,8 @@ int atomic_cmpset_int(volatile u_int *dst, u_int e int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); +int atomic_testandset_int(volatile u_int *p, int v); +int atomic_testandset_long(volatile u_long *p, int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -138,14 +142,13 @@ atomic_cmpset_int(volatile u_int *dst, u_int expec __asm __volatile( " " MPLOCKED " " - " cmpxchgl %2,%1 ; " + " cmpxchgl %3, %1 ; " " sete %0 ; " - "1: " "# atomic_cmpset_int" - : "=a" (res), /* 0 */ - "=m" (*dst) /* 1 */ - : "r" (src), /* 2 */ - "a" (expect), /* 3 */ + : "=r" (res), /* 0 */ + "=m" (*dst), /* 1 */ + "+a" (expect) /* 2 */ + : "r" (src), /* 3 */ "m" (*dst) /* 4 */ : "memory", "cc"); @@ -159,14 +162,13 @@ atomic_cmpset_long(volatile u_long *dst, u_long ex __asm __volatile( " " MPLOCKED " " - " cmpxchgq %2,%1 ; " + " cmpxchgq %3, %1 ; " " sete %0 ; " - "1: " "# atomic_cmpset_long" - : "=a" (res), /* 0 */ - "=m" (*dst) /* 1 */ - : "r" (src), /* 2 */ - "a" (expect), /* 3 */ + : "=r" (res), /* 0 */ + "=m" (*dst), /* 1 */ + "+a" (expect) /* 2 */ + : "r" (src), /* 3 */ "m" (*dst) /* 4 */ : "memory", "cc"); @@ -211,6 +213,42 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2, %1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=r" (res), /* 0 */ + "=m" (*p) /* 1 */ + : "r" (v), /* 2 */ + "m" (*p) /* 3 */ + : "cc"); + return (res); +} + +static __inline int +atomic_testandset_long(volatile u_long *p, int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsq %2, %1 ; " + " setc %0 ; " + "# atomic_testandset_long" + : "=r" (res), /* 0 */ + "=m" (*p) /* 1 */ + : "r" ((u_long)v), /* 2 */ + "m" (*p) /* 3 */ + : "cc"); + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -303,45 +341,39 @@ ATOMIC_STORE(long); #ifndef WANT_FUNCTIONS -/* Read the current value and store a zero in the destination. */ +/* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int -atomic_readandclear_int(volatile u_int *addr) +atomic_swap_int(volatile u_int *p, u_int v) { - u_int res; - res = 0; __asm __volatile( - " xchgl %1,%0 ; " - "# atomic_readandclear_int" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); - - return (res); + " xchgl %1, %0 ; " + "# atomic_swap_int" + : "+r" (v), /* 0 */ + "=m" (*p) /* 1 */ + : "m" (*p)); + return (v); } static __inline u_long -atomic_readandclear_long(volatile u_long *addr) +atomic_swap_long(volatile u_long *p, u_long v) { - u_long res; - res = 0; __asm __volatile( - " xchgq %1,%0 ; " - "# atomic_readandclear_long" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); - - return (res); + " xchgq %1, %0 ; " + "# atomic_swap_long" + : "+r" (v), /* 0 */ + "=m" (*p) /* 1 */ + : "m" (*p)); + return (v); } #else /* !__GNUCLIKE_ASM */ -u_int atomic_readandclear_int(volatile u_int *addr); -u_long atomic_readandclear_long(volatile u_long *addr); +u_int atomic_swap_int(volatile u_int *, u_int); +u_long atomic_swap_long(volatile u_long *, u_long); #endif /* __GNUCLIKE_ASM */ @@ -385,6 +417,9 @@ static __inline u_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_readandclear_int(p) atomic_swap_int(p, 0) +#define atomic_readandclear_long(p) atomic_swap_long(p, 0) + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char @@ -435,8 +470,10 @@ static __inline u_long #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long @@ -456,7 +493,9 @@ static __inline u_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long +#define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long +#define atomic_testandset_64 atomic_testandset_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long @@ -476,6 +515,7 @@ static __inline u_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long +#define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ Index: sys/amd64/include/pmap.h =================================================================== --- sys/amd64/include/pmap.h (revision 253861) +++ sys/amd64/include/pmap.h (working copy) @@ -185,42 +185,14 @@ extern u_int64_t KPML4phys; /* physical address of pt_entry_t *vtopte(vm_offset_t); #define vtophys(va) pmap_kextract(((vm_offset_t) (va))) -static __inline pt_entry_t -pte_load(pt_entry_t *ptep) -{ - pt_entry_t r; +#define pte_load(ptep) atomic_load_acq_long(ptep) +#define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) +#define pte_load_clear(pte) atomic_swap_long(pte, 0) +#define pte_store(ptep, pte) atomic_store_rel_long(ptep, pte) +#define pte_clear(ptep) atomic_store_rel_long(ptep, 0) - r = *ptep; - return (r); -} +#define pde_store(pdep, pde) atomic_store_rel_long(pdep, pde) -static __inline pt_entry_t -pte_load_store(pt_entry_t *ptep, pt_entry_t pte) -{ - pt_entry_t r; - - __asm __volatile( - "xchgq %0,%1" - : "=m" (*ptep), - "=r" (r) - : "1" (pte), - "m" (*ptep)); - return (r); -} - -#define pte_load_clear(pte) atomic_readandclear_long(pte) - -static __inline void -pte_store(pt_entry_t *ptep, pt_entry_t pte) -{ - - *ptep = pte; -} - -#define pte_clear(ptep) pte_store((ptep), (pt_entry_t)0ULL) - -#define pde_store(pdep, pde) pte_store((pdep), (pde)) - extern pt_entry_t pg_nx; #endif /* _KERNEL */ Index: sys/i386/i386/machdep.c =================================================================== --- sys/i386/i386/machdep.c (revision 253861) +++ sys/i386/i386/machdep.c (working copy) @@ -1548,10 +1548,16 @@ idle_sysctl(SYSCTL_HANDLER_ARGS) SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0, idle_sysctl, "A", "currently selected idle function"); +int (*atomic_cmpset_64)(volatile uint64_t *, uint64_t, uint64_t) = + atomic_cmpset_64_i386; uint64_t (*atomic_load_acq_64)(volatile uint64_t *) = atomic_load_acq_64_i386; void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t) = atomic_store_rel_64_i386; +uint64_t (*atomic_swap_64)(volatile uint64_t *, uint64_t) = + atomic_swap_64_i386; +int (*atomic_testandset_64)(volatile uint64_t *, int) = + atomic_testandset_64_i386; static void cpu_probe_cmpxchg8b(void) @@ -1559,8 +1565,11 @@ cpu_probe_cmpxchg8b(void) if ((cpu_feature & CPUID_CX8) != 0 || cpu_vendor_id == CPU_VENDOR_RISE) { + atomic_cmpset_64 = atomic_cmpset_64_i586; atomic_load_acq_64 = atomic_load_acq_64_i586; atomic_store_rel_64 = atomic_store_rel_64_i586; + atomic_swap_64 = atomic_swap_64_i586; + atomic_testandset_64 = atomic_testandset_64_i586; } } Index: sys/i386/include/atomic.h =================================================================== --- sys/i386/include/atomic.h (revision 253861) +++ sys/i386/include/atomic.h (working copy) @@ -54,6 +54,7 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) @@ -60,6 +61,7 @@ * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -78,6 +80,7 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); +int atomic_testandset_int(volatile u_int *p, int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -121,28 +124,40 @@ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, } \ struct __hack +#define ATOMIC_LOCK_I386(f) \ + __asm __volatile("pushfl; popl %0; cli" : "=r" (f)) +#define ATOMIC_UNLOCK_I386(f) \ + __asm __volatile("pushl %0; popfl" : : "r" (f)) + #if defined(_KERNEL) && !defined(WANT_FUNCTIONS) /* I486 does not support SMP or CMPXCHG8B. */ + +static __inline int +atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) +{ + int res; + register_t lock; + + res = 0; + ATOMIC_LOCK_I386(lock); + if (*dst == expect) { + *dst = src; + res = 1; + } + ATOMIC_UNLOCK_I386(lock); + return (res); +} + static __inline uint64_t atomic_load_acq_64_i386(volatile uint64_t *p) { - volatile uint32_t *high, *low; uint64_t res; + register_t lock; - low = (volatile uint32_t *)p; - high = (volatile uint32_t *)p + 1; - __asm __volatile( - " pushfl ; " - " cli ; " - " movl %1,%%eax ; " - " movl %2,%%edx ; " - " popfl" - : "=&A" (res) /* 0 */ - : "m" (*low), /* 1 */ - "m" (*high) /* 2 */ - : "memory"); - + ATOMIC_LOCK_I386(lock); + res = *p; + ATOMIC_UNLOCK_I386(lock); return (res); } @@ -149,20 +164,57 @@ atomic_load_acq_64_i386(volatile uint64_t *p) static __inline void atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) { - volatile uint32_t *high, *low; + register_t lock; - low = (volatile uint32_t *)p; - high = (volatile uint32_t *)p + 1; + ATOMIC_LOCK_I386(lock); + *p = v; + ATOMIC_UNLOCK_I386(lock); +} + +static __inline uint64_t +atomic_swap_64_i386(volatile uint64_t *p, uint64_t v) +{ + uint64_t res; + register_t lock; + + ATOMIC_LOCK_I386(lock); + res = *p; + *p = v; + ATOMIC_UNLOCK_I386(lock); + return (res); +} + +static __inline int +atomic_testandset_64_i386(volatile uint64_t *p, int v) +{ + const uint64_t s = 1ULL << v % 64; + int res; + register_t lock; + + ATOMIC_LOCK_I386(lock); + res = (*p & s) != 0; + *p |= s; + ATOMIC_UNLOCK_I386(lock); + return (res); +} + +static __inline int +atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) +{ + u_char res; + __asm __volatile( - " pushfl ; " - " cli ; " - " movl %%eax,%0 ; " - " movl %%edx,%1 ; " - " popfl" - : "=m" (*low), /* 0 */ - "=m" (*high) /* 1 */ - : "A" (v) /* 2 */ - : "memory"); + " " MPLOCKED " " + " cmpxchg8b %1 ; " + " sete %0" + : "=r" (res), /* 0 */ + "=m" (*dst), /* 1 */ + "+A" (expect) /* 2 */ + : "b" ((uint32_t)src), /* 3 */ + "c" ((uint32_t)(src >> 32)), /* 4 */ + "m" (*dst) /* 5 */ + : "memory", "cc"); + return (res); } static __inline uint64_t @@ -171,15 +223,14 @@ atomic_load_acq_64_i586(volatile uint64_t *p) uint64_t res; __asm __volatile( - " movl %%ebx,%%eax ; " - " movl %%ecx,%%edx ; " + " movl %%ebx, %%eax ; " + " movl %%ecx, %%edx ; " " " MPLOCKED " " - " cmpxchg8b %2" + " cmpxchg8b %1" : "=&A" (res), /* 0 */ "=m" (*p) /* 1 */ : "m" (*p) /* 2 */ : "memory", "cc"); - return (res); } @@ -188,11 +239,11 @@ atomic_store_rel_64_i586(volatile uint64_t *p, uin { __asm __volatile( - " movl %%eax,%%ebx ; " - " movl %%edx,%%ecx ; " + " movl %%eax, %%ebx ; " + " movl %%edx, %%ecx ; " "1: " " " MPLOCKED " " - " cmpxchg8b %2 ; " + " cmpxchg8b %0 ; " " jne 1b" : "=m" (*p), /* 0 */ "+A" (v) /* 1 */ @@ -200,6 +251,36 @@ atomic_store_rel_64_i586(volatile uint64_t *p, uin : "ebx", "ecx", "memory", "cc"); } +static __inline uint64_t +atomic_swap_64_i586(volatile uint64_t *p, uint64_t v) +{ + + __asm __volatile( + " movl %%eax, %%ebx ; " + " movl %%edx, %%ecx ; " + "1: " + " " MPLOCKED " " + " cmpxchg8b %0 ; " + " jne 1b" + : "=m" (*p), /* 0 */ + "+A" (v) /* 1 */ + : "m" (*p) /* 2 */ + : "ebx", "ecx", "memory", "cc"); + return (v); +} + +static __inline int +atomic_testandset_64_i586(volatile uint64_t *p, int v) +{ + const uint64_t s = 1ULL << v % 64; + uint64_t n; + + do { + n = *p; + } while (!atomic_cmpset_64_i586(p, n, n | s)); + return ((n & s) != 0); +} + #endif /* _KERNEL && !WANT_FUNCTIONS */ /* @@ -215,25 +296,16 @@ atomic_store_rel_64_i586(volatile uint64_t *p, uin static __inline int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) { - u_char res; + int res; + register_t lock; - __asm __volatile( - " pushfl ; " - " cli ; " - " cmpl %3,%4 ; " - " jne 1f ; " - " movl %2,%1 ; " - "1: " - " sete %0 ; " - " popfl ; " - "# atomic_cmpset_int" - : "=q" (res), /* 0 */ - "=m" (*dst) /* 1 */ - : "r" (src), /* 2 */ - "r" (expect), /* 3 */ - "m" (*dst) /* 4 */ - : "memory"); - + res = 0; + ATOMIC_LOCK_I386(lock); + if (*dst == expect) { + *dst = src; + res = 1; + } + ATOMIC_UNLOCK_I386(lock); return (res); } @@ -246,22 +318,23 @@ atomic_cmpset_int(volatile u_int *dst, u_int expec __asm __volatile( " " MPLOCKED " " - " cmpxchgl %2,%1 ; " - " sete %0 ; " - "1: " + " cmpxchgl %3, %1 ; " + " sete %0 ; " "# atomic_cmpset_int" - : "=a" (res), /* 0 */ - "=m" (*dst) /* 1 */ - : "r" (src), /* 2 */ - "a" (expect), /* 3 */ + : "=r" (res), /* 0 */ + "=m" (*dst), /* 1 */ + "+a" (expect) /* 2 */ + : "r" (src), /* 3 */ "m" (*dst) /* 4 */ : "memory", "cc"); - return (res); } #endif /* CPU_DISABLE_CMPXCHG */ +#undef ATOMIC_LOCK_I386 +#undef ATOMIC_UNLOCK_I386 + /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. @@ -281,6 +354,24 @@ atomic_fetchadd_int(volatile u_int *p, u_int v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2, %1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=r" (res), /* 0 (result) */ + "=m" (*p) /* 1 */ + : "r" (v), /* 2 */ + "m" (*p) /* 3 */ + : "cc"); + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -374,8 +465,11 @@ ATOMIC_STORE(long); #ifndef WANT_FUNCTIONS #ifdef _KERNEL +extern int (*atomic_cmpset_64)(volatile uint64_t *, uint64_t, uint64_t); extern uint64_t (*atomic_load_acq_64)(volatile uint64_t *); extern void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t); +extern uint64_t (*atomic_swap_64)(volatile uint64_t *, uint64_t); +extern int (*atomic_testandset_64)(volatile uint64_t *, int); #endif static __inline int @@ -393,45 +487,40 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); } -/* Read the current value and store a zero in the destination. */ +static __inline int +atomic_testandset_long(volatile u_long *p, int v) +{ + + return (atomic_testandset_int((volatile u_int *)p, v)); +} + +/* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int -atomic_readandclear_int(volatile u_int *addr) +atomic_swap_int(volatile u_int *p, u_int v) { - u_int res; - res = 0; __asm __volatile( - " xchgl %1,%0 ; " - "# atomic_readandclear_int" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); - - return (res); + " xchgl %1, %0 ; " + "# atomic_swap_int" + : "+r" (v), /* 0 */ + "=m" (*p) /* 1 */ + : "m" (*p)); + return (v); } static __inline u_long -atomic_readandclear_long(volatile u_long *addr) +atomic_swap_long(volatile u_long *p, u_long v) { - u_long res; - res = 0; - __asm __volatile( - " xchgl %1,%0 ; " - "# atomic_readandclear_long" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); - - return (res); + return (atomic_swap_int((volatile u_int *)p, (u_int)v)); } #else /* !__GNUCLIKE_ASM */ -u_int atomic_readandclear_int(volatile u_int *addr); -u_long atomic_readandclear_long(volatile u_long *addr); +u_int atomic_swap_int(volatile u_int *, u_int); +u_long atomic_swap_long(volatile u_long *, u_long); #endif /* __GNUCLIKE_ASM */ @@ -475,6 +564,9 @@ static __inline u_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_readandclear_int(p) atomic_swap_int(p, 0) +#define atomic_readandclear_long(p) atomic_swap_long(p, 0) + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char @@ -525,8 +617,10 @@ static __inline u_long #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on pointers. */ #define atomic_set_ptr(p, v) \ @@ -565,6 +659,8 @@ static __inline u_long #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) +#define atomic_swap_ptr(p, v) \ + atomic_swap_int((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_ptr(p) \ atomic_readandclear_int((volatile u_int *)(p)) Index: sys/i386/include/pmap.h =================================================================== --- sys/i386/include/pmap.h (revision 253861) +++ sys/i386/include/pmap.h (working copy) @@ -326,98 +326,27 @@ pmap_kextract(vm_offset_t va) #if defined(PAE) && !defined(XEN) -#define pde_cmpset(pdep, old, new) \ - atomic_cmpset_64((pdep), (old), (new)) +#define pde_cmpset(pdep, old, new) atomic_cmpset_64_i586(pdep, old, new) +#define pte_load(ptep) atomic_load_acq_64_i586(ptep) +#define pte_load_store(ptep, pte) atomic_swap_64_i586(ptep, pte) +#define pte_load_clear(ptep) atomic_swap_64_i586(ptep, 0) +#define pte_store(ptep, pte) atomic_store_rel_64_i586(ptep, pte) -static __inline pt_entry_t -pte_load(pt_entry_t *ptep) -{ - pt_entry_t r; - - __asm __volatile( - "lock; cmpxchg8b %1" - : "=A" (r) - : "m" (*ptep), "a" (0), "d" (0), "b" (0), "c" (0)); - return (r); -} - -static __inline pt_entry_t -pte_load_store(pt_entry_t *ptep, pt_entry_t v) -{ - pt_entry_t r; - - r = *ptep; - __asm __volatile( - "1:\n" - "\tlock; cmpxchg8b %1\n" - "\tjnz 1b" - : "+A" (r) - : "m" (*ptep), "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32))); - return (r); -} - -/* XXXRU move to atomic.h? */ -static __inline int -atomic_cmpset_64(volatile uint64_t *dst, uint64_t exp, uint64_t src) -{ - int64_t res = exp; - - __asm __volatile ( - " lock ; " - " cmpxchg8b %2 ; " - " setz %%al ; " - " movzbl %%al,%0 ; " - "# atomic_cmpset_64" - : "+A" (res), /* 0 (result) */ - "=m" (*dst) /* 1 */ - : "m" (*dst), /* 2 */ - "b" ((uint32_t)src), - "c" ((uint32_t)(src >> 32))); - - return (res); -} - -#define pte_load_clear(ptep) pte_load_store((ptep), (pt_entry_t)0ULL) - -#define pte_store(ptep, pte) pte_load_store((ptep), (pt_entry_t)pte) - extern pt_entry_t pg_nx; -#elif !defined(PAE) && !defined (XEN) +#elif !defined(PAE) && !defined(XEN) -#define pde_cmpset(pdep, old, new) \ - atomic_cmpset_int((pdep), (old), (new)) +#define pde_cmpset(pdep, old, new) atomic_cmpset_int(pdep, old, new) +#define pte_load(ptep) atomic_load_acq_int(ptep) +#define pte_load_store(ptep, pte) atomic_swap_int(ptep, pte) +#define pte_load_clear(pte) atomic_swap_int(pte, 0) +#define pte_store(ptep, pte) atomic_store_rel_int(ptep, pte) -static __inline pt_entry_t -pte_load(pt_entry_t *ptep) -{ - pt_entry_t r; - - r = *ptep; - return (r); -} - -static __inline pt_entry_t -pte_load_store(pt_entry_t *ptep, pt_entry_t pte) -{ - __asm volatile("xchgl %0, %1" : "+m" (*ptep), "+r" (pte)); - return (pte); -} - -#define pte_load_clear(pte) atomic_readandclear_int(pte) - -static __inline void -pte_store(pt_entry_t *ptep, pt_entry_t pte) -{ - - *ptep = pte; -} - #endif /* PAE */ -#define pte_clear(ptep) pte_store((ptep), (pt_entry_t)0ULL) +#define pte_clear(ptep) pte_store(ptep, 0) -#define pde_store(pdep, pde) pte_store((pdep), (pde)) +#define pde_store(pdep, pde) pte_store(pdep, pde) #endif /* _KERNEL */