Index: sys/powerpc/include/atomic.h =================================================================== --- sys/powerpc/include/atomic.h (revision 221608) +++ sys/powerpc/include/atomic.h (working copy) @@ -48,13 +48,7 @@ * { *p += v; } */ -#define __ATOMIC_ADD_8(p, v, t) \ - 8-bit atomic_add not implemented - -#define __ATOMIC_ADD_16(p, v, t) \ - 16-bit atomic_add not implemented - -#define __ATOMIC_ADD_32(p, v, t) \ +#define __atomic_add_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ @@ -63,10 +57,10 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_ADD_32 */ + /* __atomic_add_int */ #ifdef __powerpc64__ -#define __ATOMIC_ADD_64(p, v, t) \ +#define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ @@ -75,69 +69,78 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_ADD_64 */ + /* __atomic_add_long */ #else -#define __ATOMIC_ADD_64(p, v, t) \ - 64-bit atomic_add not implemented +#define __atomic_add_long(p, v, t) \ + long atomic_add not implemented #endif -#define _ATOMIC_ADD(width, suffix, type) \ +#define _ATOMIC_ADD(type) \ static __inline void \ - atomic_add_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_ADD_##width(p, v, t); \ + atomic_add_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_add_##type(p, v, t); \ } \ \ static __inline void \ - atomic_add_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_ADD_##width(p, v, t); \ + atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_add_##type(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ - atomic_add_rel_##suffix(volatile type *p, type v) { \ - type t; \ + atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ __ATOMIC_BARRIER; \ - __ATOMIC_ADD_##width(p, v, t); \ + __atomic_add_##type(p, v, t); \ } \ /* _ATOMIC_ADD */ -#if 0 -_ATOMIC_ADD(8, 8, uint8_t) -_ATOMIC_ADD(8, char, u_char) -_ATOMIC_ADD(16, 16, uint16_t) -_ATOMIC_ADD(16, short, u_short) -#endif -_ATOMIC_ADD(32, 32, uint32_t) -_ATOMIC_ADD(32, int, u_int) +_ATOMIC_ADD(int) + +#define atomic_add_32 atomic_add_int +#define atomic_add_acq_32 atomic_add_acq_int +#define atomic_add_rel_32 atomic_add_rel_int + #ifdef __powerpc64__ -_ATOMIC_ADD(64, 64, uint64_t) -_ATOMIC_ADD(64, long, u_long) -_ATOMIC_ADD(64, ptr, uintptr_t) +_ATOMIC_ADD(long) + +#define atomic_add_64 atomic_add_long +#define atomic_add_acq_64 atomic_add_acq_long +#define atomic_add_rel_64 atomic_add_rel_long + +#define atomic_add_ptr(p, v) \ + atomic_add_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_add_acq_ptr(p, v) \ + atomic_add_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_add_rel_ptr(p, v) \ + atomic_add_rel_long((volatile u_long *)(p), (u_long)(v)) #else -_ATOMIC_ADD(32, long, u_long) -_ATOMIC_ADD(32, ptr, uintptr_t) +#define atomic_add_long(p, v) \ + atomic_add_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_acq_long(p, v) \ + atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_rel_long(p, v) \ + atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_add_ptr(p, v) \ + atomic_add_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_acq_ptr(p, v) \ + atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_rel_ptr(p, v) \ + atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) #endif - #undef _ATOMIC_ADD -#undef __ATOMIC_ADD_64 -#undef __ATOMIC_ADD_32 -#undef __ATOMIC_ADD_16 -#undef __ATOMIC_ADD_8 +#undef __atomic_add_long +#undef __atomic_add_int /* * atomic_clear(p, v) * { *p &= ~v; } */ -#define __ATOMIC_CLEAR_8(p, v, t) \ - 8-bit atomic_clear not implemented - -#define __ATOMIC_CLEAR_16(p, v, t) \ - 16-bit atomic_clear not implemented - -#define __ATOMIC_CLEAR_32(p, v, t) \ +#define __atomic_clear_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ @@ -146,10 +149,10 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_CLEAR_32 */ + /* __atomic_clear_int */ #ifdef __powerpc64__ -#define __ATOMIC_CLEAR_64(p, v, t) \ +#define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ @@ -158,57 +161,73 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_CLEAR_64 */ + /* __atomic_clear_long */ #else -#define __ATOMIC_CLEAR_64(p, v, t) \ - 64-bit atomic_clear not implemented +#define __atomic_clear_long(p, v, t) \ + long atomic_clear not implemented #endif -#define _ATOMIC_CLEAR(width, suffix, type) \ +#define _ATOMIC_CLEAR(type) \ static __inline void \ - atomic_clear_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_CLEAR_##width(p, v, t); \ + atomic_clear_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_clear_##type(p, v, t); \ } \ \ static __inline void \ - atomic_clear_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_CLEAR_##width(p, v, t); \ + atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_clear_##type(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ - atomic_clear_rel_##suffix(volatile type *p, type v) { \ - type t; \ + atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ __ATOMIC_BARRIER; \ - __ATOMIC_CLEAR_##width(p, v, t); \ + __atomic_clear_##type(p, v, t); \ } \ /* _ATOMIC_CLEAR */ -#if 0 -_ATOMIC_CLEAR(8, 8, uint8_t) -_ATOMIC_CLEAR(8, char, u_char) -_ATOMIC_CLEAR(16, 16, uint16_t) -_ATOMIC_CLEAR(16, short, u_short) -#endif -_ATOMIC_CLEAR(32, 32, uint32_t) -_ATOMIC_CLEAR(32, int, u_int) + +_ATOMIC_CLEAR(int) + +#define atomic_clear_32 atomic_clear_int +#define atomic_clear_acq_32 atomic_clear_acq_int +#define atomic_clear_rel_32 atomic_clear_rel_int + #ifdef __powerpc64__ -_ATOMIC_CLEAR(64, 64, uint64_t) -_ATOMIC_CLEAR(64, long, u_long) -_ATOMIC_CLEAR(64, ptr, uintptr_t) +_ATOMIC_CLEAR(long) + +#define atomic_clear_64 atomic_clear_long +#define atomic_clear_acq_64 atomic_clear_acq_long +#define atomic_clear_rel_64 atomic_clear_rel_long + +#define atomic_clear_ptr(p, v) \ + atomic_clear_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_clear_acq_ptr(p, v) \ + atomic_clear_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_clear_rel_ptr(p, v) \ + atomic_clear_rel_long((volatile u_long *)(p), (u_long)(v)) #else -_ATOMIC_CLEAR(32, long, u_long) -_ATOMIC_CLEAR(32, ptr, uintptr_t) +#define atomic_clear_long(p, v) \ + atomic_clear_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_acq_long(p, v) \ + atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_rel_long(p, v) \ + atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_clear_ptr(p, v) \ + atomic_clear_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_acq_ptr(p, v) \ + atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_rel_ptr(p, v) \ + atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) #endif +#undef _ATOMIC_ADD +#undef __atomic_clear_long +#undef __atomic_clear_int -#undef _ATOMIC_CLEAR -#undef __ATOMIC_CLEAR_64 -#undef __ATOMIC_CLEAR_32 -#undef __ATOMIC_CLEAR_16 -#undef __ATOMIC_CLEAR_8 - /* * atomic_cmpset(p, o, n) */ @@ -229,13 +248,7 @@ * { *p |= v; } */ -#define __ATOMIC_SET_8(p, v, t) \ - 8-bit atomic_set not implemented - -#define __ATOMIC_SET_16(p, v, t) \ - 16-bit atomic_set not implemented - -#define __ATOMIC_SET_32(p, v, t) \ +#define __atomic_set_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ @@ -244,10 +257,10 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SET_32 */ + /* __atomic_set_int */ #ifdef __powerpc64__ -#define __ATOMIC_SET_64(p, v, t) \ +#define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ @@ -256,69 +269,78 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SET_64 */ + /* __atomic_set_long */ #else -#define __ATOMIC_SET_64(p, v, t) \ - 64-bit atomic_set not implemented +#define __atomic_set_long(p, v, t) \ + long atomic_set not implemented #endif -#define _ATOMIC_SET(width, suffix, type) \ +#define _ATOMIC_SET(type) \ static __inline void \ - atomic_set_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SET_##width(p, v, t); \ + atomic_set_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_set_##type(p, v, t); \ } \ \ static __inline void \ - atomic_set_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SET_##width(p, v, t); \ + atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_set_##type(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ - atomic_set_rel_##suffix(volatile type *p, type v) { \ - type t; \ + atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ __ATOMIC_BARRIER; \ - __ATOMIC_SET_##width(p, v, t); \ + __atomic_set_##type(p, v, t); \ } \ /* _ATOMIC_SET */ -#if 0 -_ATOMIC_SET(8, 8, uint8_t) -_ATOMIC_SET(8, char, u_char) -_ATOMIC_SET(16, 16, uint16_t) -_ATOMIC_SET(16, short, u_short) -#endif -_ATOMIC_SET(32, 32, uint32_t) -_ATOMIC_SET(32, int, u_int) +_ATOMIC_SET(int) + +#define atomic_set_32 atomic_set_int +#define atomic_set_acq_32 atomic_set_acq_int +#define atomic_set_rel_32 atomic_set_rel_int + #ifdef __powerpc64__ -_ATOMIC_SET(64, 64, uint64_t) -_ATOMIC_SET(64, long, u_long) -_ATOMIC_SET(64, ptr, uintptr_t) +_ATOMIC_SET(long) + +#define atomic_set_64 atomic_set_long +#define atomic_set_acq_64 atomic_set_acq_long +#define atomic_set_rel_64 atomic_set_rel_long + +#define atomic_set_ptr(p, v) \ + atomic_set_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_set_acq_ptr(p, v) \ + atomic_set_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_set_rel_ptr(p, v) \ + atomic_set_rel_long((volatile u_long *)(p), (u_long)(v)) #else -_ATOMIC_SET(32, long, u_long) -_ATOMIC_SET(32, ptr, uintptr_t) +#define atomic_set_long(p, v) \ + atomic_set_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_acq_long(p, v) \ + atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_rel_long(p, v) \ + atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_set_ptr(p, v) \ + atomic_set_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_acq_ptr(p, v) \ + atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_rel_ptr(p, v) \ + atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) #endif - #undef _ATOMIC_SET -#undef __ATOMIC_SET_64 -#undef __ATOMIC_SET_32 -#undef __ATOMIC_SET_16 -#undef __ATOMIC_SET_8 +#undef __atomic_set_long +#undef __atomic_set_int /* * atomic_subtract(p, v) * { *p -= v; } */ -#define __ATOMIC_SUBTRACT_8(p, v, t) \ - 8-bit atomic_subtract not implemented - -#define __ATOMIC_SUBTRACT_16(p, v, t) \ - 16-bit atomic_subtract not implemented - -#define __ATOMIC_SUBTRACT_32(p, v, t) \ +#define __atomic_subtract_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ @@ -327,10 +349,10 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SUBTRACT_32 */ + /* __atomic_subtract_int */ #ifdef __powerpc64__ -#define __ATOMIC_SUBTRACT_64(p, v, t) \ +#define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ @@ -339,56 +361,71 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SUBTRACT_64 */ + /* __atomic_subtract_long */ #else -#define __ATOMIC_SUBTRACT_64(p, v, t) \ - 64-bit atomic_subtract not implemented +#define __atomic_subtract_long(p, v, t) \ + long atomic_subtract not implemented #endif -#define _ATOMIC_SUBTRACT(width, suffix, type) \ - static __inline void \ - atomic_subtract_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SUBTRACT_##width(p, v, t); \ - } \ - \ - static __inline void \ - atomic_subtract_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SUBTRACT_##width(p, v, t); \ - __ATOMIC_BARRIER; \ - } \ - \ - static __inline void \ - atomic_subtract_rel_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_BARRIER; \ - __ATOMIC_SUBTRACT_##width(p, v, t); \ - } \ +#define _ATOMIC_SUBTRACT(type) \ + static __inline void \ + atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_subtract_##type(p, v, t); \ + } \ + \ + static __inline void \ + atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_subtract_##type(p, v, t); \ + __ATOMIC_BARRIER; \ + } \ + \ + static __inline void \ + atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __ATOMIC_BARRIER; \ + __atomic_subtract_##type(p, v, t); \ + } \ /* _ATOMIC_SUBTRACT */ -#if 0 -_ATOMIC_SUBTRACT(8, 8, uint8_t) -_ATOMIC_SUBTRACT(8, char, u_char) -_ATOMIC_SUBTRACT(16, 16, uint16_t) -_ATOMIC_SUBTRACT(16, short, u_short) -#endif -_ATOMIC_SUBTRACT(32, 32, uint32_t) -_ATOMIC_SUBTRACT(32, int, u_int) +_ATOMIC_SUBTRACT(int) + +#define atomic_subtract_32 atomic_subtract_int +#define atomic_subtract_acq_32 atomic_subtract_acq_int +#define atomic_subtract_rel_32 atomic_subtract_rel_int + #ifdef __powerpc64__ -_ATOMIC_SUBTRACT(64, 64, uint64_t) -_ATOMIC_SUBTRACT(64, long, u_long) -_ATOMIC_SUBTRACT(64, ptr, uintptr_t) +_ATOMIC_SUBTRACT(long) + +#define atomic_subtract_64 atomic_subtract_long +#define atomic_subtract_acq_64 atomic_subract_acq_long +#define atomic_subtract_rel_64 atomic_subtract_rel_long + +#define atomic_subtract_ptr(p, v) \ + atomic_subtract_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_subtract_acq_ptr(p, v) \ + atomic_subtract_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_subtract_rel_ptr(p, v) \ + atomic_subtract_rel_long((volatile u_long *)(p), (u_long)(v)) #else -_ATOMIC_SUBTRACT(32, long, u_long) -_ATOMIC_SUBTRACT(32, ptr, uintptr_t) +#define atomic_subtract_long(p, v) \ + atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_acq_long(p, v) \ + atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_rel_long(p, v) \ + atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_subtract_ptr(p, v) \ + atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_acq_ptr(p, v) \ + atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_rel_ptr(p, v) \ + atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) #endif - #undef _ATOMIC_SUBTRACT -#undef __ATOMIC_SUBTRACT_64 -#undef __ATOMIC_SUBTRACT_32 -#undef __ATOMIC_SUBTRACT_16 -#undef __ATOMIC_SUBTRACT_8 +#undef __atomic_subtract_long +#undef __atomic_subtract_int /* * atomic_store_rel(p, v) @@ -399,10 +436,10 @@ * Old/original implementations that still need revisiting. */ -static __inline uint32_t -atomic_readandclear_32(volatile uint32_t *addr) +static __inline u_int +atomic_readandclear_int(volatile u_int *addr) { - uint32_t result,temp; + u_int result,temp; #ifdef __GNUCLIKE_ASM __asm __volatile ( @@ -420,10 +457,10 @@ } #ifdef __powerpc64__ -static __inline uint64_t -atomic_readandclear_64(volatile uint64_t *addr) +static __inline u_long +atomic_readandclear_long(volatile u_long *addr) { - uint64_t result,temp; + u_long result,temp; #ifdef __GNUCLIKE_ASM __asm __volatile ( @@ -441,38 +478,26 @@ } #endif -#define atomic_readandclear_int atomic_readandclear_32 +#define atomic_readandclear_32 atomic_readandclear_int #ifdef __powerpc64__ -#define atomic_readandclear_long atomic_readandclear_64 -#define atomic_readandclear_ptr atomic_readandclear_64 +#define atomic_readandclear_64 atomic_readandclear_long + +#define atomic_readandclear_ptr(p) \ + atomic_readandclear_long((volatile u_long *)(p)) #else -#define atomic_readandclear_long atomic_readandclear_32 -#define atomic_readandclear_ptr atomic_readandclear_32 +#define atomic_readandclear_long(p) \ + atomic_readandclear_int((volatile u_int *)(p)) + +#define atomic_readandclear_ptr(p) \ + atomic_readandclear_int((volatile u_int *)(p)) #endif /* * We assume that a = b will do atomic loads and stores. */ -#define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ +#define ATOMIC_STORE_LOAD(TYPE) \ static __inline u_##TYPE \ -atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ -{ \ - u_##TYPE v; \ - \ - v = *p; \ - __ATOMIC_BARRIER; \ - return (v); \ -} \ - \ -static __inline void \ -atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ -{ \ - __ATOMIC_BARRIER; \ - *p = v; \ -} \ - \ -static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE v; \ @@ -489,25 +514,32 @@ *p = v; \ } -ATOMIC_STORE_LOAD(char, 8) -ATOMIC_STORE_LOAD(short, 16) -ATOMIC_STORE_LOAD(int, 32) +ATOMIC_STORE_LOAD(int) + +#define atomic_load_acq_32 atomic_load_acq_int +#define atomic_store_rel_32 atomic_store_rel_int + #ifdef __powerpc64__ -ATOMIC_STORE_LOAD(long, 64) -#endif +ATOMIC_STORE_LOAD(long) -#ifdef __powerpc64__ -#define atomic_load_acq_long atomic_load_acq_64 -#define atomic_store_rel_long atomic_store_rel_64 -#define atomic_load_acq_ptr atomic_load_acq_64 -#define atomic_store_rel_ptr atomic_store_rel_64 +#define atomic_load_acq_64 atomic_load_acq_long +#define atomic_store_rel_64 atomic_store_rel_long + +#define atomic_load_acq_ptr(p) \ + atomic_load_acq_long((volatile u_long *)(p)) +#define atomic_store_rel_ptr(p, v) \ + atomic_store_rel_long((volatile u_long *)(p), (u_long)(v)) #else -#define atomic_load_acq_long atomic_load_acq_32 -#define atomic_store_rel_long atomic_store_rel_32 -#define atomic_load_acq_ptr atomic_load_acq_32 -#define atomic_store_rel_ptr atomic_store_rel_32 +#define atomic_load_acq_long(p) \ + atomic_load_acq_int((volatile u_int *)(p)) +#define atomic_store_rel_long(p, v) \ + atomic_store_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_load_acq_ptr(p) \ + atomic_load_acq_int((volatile u_int *)(p)) +#define atomic_store_rel_ptr(p, v) \ + atomic_store_rel_int((volatile u_int *)(p), (u_int)(v)) #endif - #undef ATOMIC_STORE_LOAD /* @@ -516,7 +548,7 @@ * zero if the compare failed, nonzero otherwise. */ static __inline int -atomic_cmpset_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) +atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { int ret; @@ -542,32 +574,43 @@ } static __inline int +atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) +{ + int retval; + + retval = atomic_cmpset_int(p, cmpval, newval); + __ATOMIC_BARRIER; + return (retval); +} + +static __inline int +atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) +{ + __ATOMIC_BARRIER; + return (atomic_cmpset_int(p, cmpval, newval)); +} + +#define atomic_cmpset_32 atomic_cmpset_int +#define atomic_cmpset_acq_32 atomic_cmpset_acq_int +#define atomic_cmpset_rel_32 atomic_cmpset_rel_int + +#ifdef __powerpc64__ +static __inline int atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) { int ret; #ifdef __GNUCLIKE_ASM __asm __volatile ( - #ifdef __powerpc64__ "1:\tldarx %0, 0, %2\n\t" /* load old value */ "cmpld %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stdcx. %4, 0, %2\n\t" /* attempt to store */ - #else - "1:\tlwarx %0, 0, %2\n\t" /* load old value */ - "cmplw %3, %0\n\t" /* compare */ - "bne 2f\n\t" /* exit if not equal */ - "stwcx. %4, 0, %2\n\t" /* attempt to store */ - #endif "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" - #ifdef __powerpc64__ "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ - #else - "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ - #endif "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) @@ -578,34 +621,7 @@ return (ret); } -#define atomic_cmpset_int atomic_cmpset_32 - -#ifdef __powerpc64__ -#define atomic_cmpset_ptr(dst, old, new) \ - atomic_cmpset_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) -#else -#define atomic_cmpset_ptr(dst, old, new) \ - atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) -#endif - static __inline int -atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) -{ - int retval; - - retval = atomic_cmpset_32(p, cmpval, newval); - __ATOMIC_BARRIER; - return (retval); -} - -static __inline int -atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) -{ - __ATOMIC_BARRIER; - return (atomic_cmpset_32(p, cmpval, newval)); -} - -static __inline int atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) { u_long retval; @@ -622,39 +638,59 @@ return (atomic_cmpset_long(p, cmpval, newval)); } -#define atomic_cmpset_acq_int atomic_cmpset_acq_32 -#define atomic_cmpset_rel_int atomic_cmpset_rel_32 +#define atomic_cmpset_64 atomic_cmpset_long +#define atomic_cmpset_acq_64 atomic_cmpset_acq_long +#define atomic_cmpset_rel_64 atomic_cmpset_rel_long -#ifdef __powerpc64__ -#define atomic_cmpset_acq_ptr(dst, old, new) \ - atomic_cmpset_acq_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) -#define atomic_cmpset_rel_ptr(dst, old, new) \ - atomic_cmpset_rel_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) +#define atomic_cmpset_ptr(dst, old, new) \ + atomic_cmpset_long((volatile u_long *)(dst), (u_long)(old), \ + (u_long)(new)) +#define atomic_cmpset_acq_ptr(dst, old, new) \ + atomic_cmpset_acq_long((volatile u_long *)(dst), (u_long)(old), \ + (u_long)(new)) +#define atomic_cmpset_rel_ptr(dst, old, new) \ + atomic_cmpset_rel_long((volatile u_long *)(dst), (u_long)(old), \ + (u_long)(new)) #else -#define atomic_cmpset_acq_ptr(dst, old, new) \ - atomic_cmpset_acq_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) -#define atomic_cmpset_rel_ptr(dst, old, new) \ - atomic_cmpset_rel_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) +#define atomic_cmpset_long(dst, old, new) \ + atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_acq_long(dst, old, new) \ + atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_rel_long(dst, old, new) \ + atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) + +#define atomic_cmpset_ptr(dst, old, new) \ + atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_acq_ptr(dst, old, new) \ + atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_rel_ptr(dst, old, new) \ + atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) #endif -static __inline uint32_t -atomic_fetchadd_32(volatile uint32_t *p, uint32_t v) +static __inline u_int +atomic_fetchadd_int(volatile u_int *p, u_int v) { - uint32_t value; + u_int value; do { value = *p; - } while (!atomic_cmpset_32(p, value, value + v)); + } while (!atomic_cmpset_int(p, value, value + v)); return (value); } -#define atomic_fetchadd_int atomic_fetchadd_32 +#define atomic_fetchadd_32 atomic_fetchadd_int #ifdef __powerpc64__ -static __inline uint64_t -atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) +static __inline u_long +atomic_fetchadd_long(volatile u_long *p, u_long v) { - uint64_t value; + u_long value; do { value = *p; @@ -662,10 +698,10 @@ return (value); } -#define atomic_fetchadd_long atomic_fetchadd_64 +#define atomic_fetchadd_64 atomic_fetchadd_long #else -#define atomic_fetchadd_long(p, v) \ - (u_long)atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v)) +#define atomic_fetchadd_long(p, v) \ + (u_long)atomic_fetchadd_int((volatile u_int *)(p), (u_int)(v)) #endif #endif /* ! _MACHINE_ATOMIC_H_ */