Index: arm/include/atomic.h =================================================================== --- arm/include/atomic.h (revision 309656) +++ arm/include/atomic.h (working copy) @@ -82,6 +82,9 @@ #define atomic_clear_ptr atomic_clear_32 #define atomic_set_ptr atomic_set_32 +#define atomic_cas_ptr atomic_cas_32 +#define atomic_cas_rel_ptr atomic_cas_rel_32 +#define atomic_cas_acq_ptr atomic_cas_acq_32 #define atomic_cmpset_ptr atomic_cmpset_32 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_32 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 @@ -100,6 +103,9 @@ #define atomic_set_int atomic_set_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_set_rel_int atomic_set_rel_32 +#define atomic_cas_int atomic_cas_32 +#define atomic_cas_acq_int atomic_cas_acq_32 +#define atomic_cas_rel_int atomic_cas_rel_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 Index: arm/include/atomic-v4.h =================================================================== --- arm/include/atomic-v4.h (revision 309656) +++ arm/include/atomic-v4.h (working copy) @@ -113,6 +113,35 @@ } static __inline u_int32_t +atomic_cas_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) +{ + u_int32_t ret; + + __with_interrupts_disabled( + { + ret = *p; + if (*p == cmpval) + *p = newval; + ret = 1; + }); + return (ret); +} + +static __inline u_int64_t +atomic_cas_64(volatile u_int64_t *p, volatile u_int64_t cmpval, volatile u_int64_t newval) +{ + u_int64_t ret; + + __with_interrupts_disabled( + { + ret = *p; + if (*p == cmpval) + ret = 1; + }); + return (ret); +} + +static __inline u_int32_t atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) { int ret; @@ -370,6 +399,12 @@ return (__swp(v, p)); } +#define atomic_cas_rel_32 atomic_cas_32 +#define atomic_cas_acq_32 atomic_cas_32 +#define atomic_cas_rel_64 atomic_cas_64 +#define atomic_cas_acq_64 atomic_cas_64 +#define atomic_cas_acq_long atomic_cas_long +#define atomic_cas_rel_long atomic_cas_long #define atomic_cmpset_rel_32 atomic_cmpset_32 #define atomic_cmpset_acq_32 atomic_cmpset_32 #define atomic_cmpset_rel_64 atomic_cmpset_64 @@ -421,6 +456,13 @@ } static __inline u_long +atomic_cas_long(volatile u_long *dst, u_long old, u_long newe) +{ + + return (atomic_cas_32((volatile uint32_t *)dst, old, newe)); +} + +static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { Index: arm/include/atomic-v6.h =================================================================== --- arm/include/atomic-v6.h (revision 309656) +++ arm/include/atomic-v6.h (working copy) @@ -191,6 +191,115 @@ ATOMIC_ACQ_REL_LONG(clear) static __inline uint32_t +atomic_cas_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) +{ + uint32_t ret, tmp; + + __asm __volatile( + "1: ldrex %0, [%2] \n" + " cmp %0, %3 \n" + " it ne \n" + " bne 2f \n" + " strex %1, %4, [%2] \n" + " cmp %1, #0 \n" + " it ne \n" + " bne 1b \n" + "2:" + : "=&r" (ret), "=&r" (tmp), "+r" (p), "+r" (cmpval), "+r" (newval) + : : "cc", "memory"); + return (ret); +} + +static __inline uint64_t +atomic_cas_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) +{ + uint64_t ret; + uint32_t tmp; + + __asm __volatile( + "1: \n" + " ldrexd %Q[ret], %R[ret], [%[ptr]] \n" + " teq %Q[ret], %Q[cmpval] \n" + " itee eq \n" + " teqeq %R[ret], %R[cmpval] \n" + " bne 2f \n" + " strexd %[tmp], %Q[newval], %R[newval], [%[ptr]]\n" + " teq %[tmp], #0 \n" + " it ne \n" + " bne 1b \n" + "2: \n" + : [ret] "=&r" (ret), + [tmp] "=&r" (tmp) + : [ptr] "r" (p), + [cmpval] "r" (cmpval), + [newval] "r" (newval) + : "cc", "memory"); + return (ret); +} + +static __inline u_long +atomic_cas_long(volatile u_long *p, u_long cmpval, u_long newval) +{ + + return (atomic_cas_32((volatile uint32_t *)p, cmpval, newval)); +} + +static __inline uint64_t +atomic_cas_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) +{ + uint64_t ret; + + ret = atomic_cas_64(p, cmpval, newval); + dmb(); + return (ret); +} + +static __inline u_long +atomic_cas_acq_long(volatile u_long *p, u_long cmpval, u_long newval) +{ + u_long ret; + + ret = atomic_cas_long(p, cmpval, newval); + dmb(); + return (ret); +} + +static __inline uint32_t +atomic_cas_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) +{ + + uint32_t ret; + + ret = atomic_cas_32(p, cmpval, newval); + dmb(); + return (ret); +} + +static __inline uint32_t +atomic_cas_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) +{ + + dmb(); + return (atomic_cas_32(p, cmpval, newval)); +} + +static __inline uint64_t +atomic_cas_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) +{ + + dmb(); + return (atomic_cas_64(p, cmpval, newval)); +} + +static __inline u_long +atomic_cas_rel_long(volatile u_long *p, u_long cmpval, u_long newval) +{ + + dmb(); + return (atomic_cas_long(p, cmpval, newval)); +} + +static __inline uint32_t atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t ret; Index: arm64/include/atomic.h =================================================================== --- arm64/include/atomic.h (revision 309656) +++ arm64/include/atomic.h (working copy) @@ -98,6 +98,57 @@ ATOMIC(set, orr) ATOMIC(subtract, sub) +#define ATOMIC_CAS(bar, a, l) \ +static __inline uint32_t \ +atomic_cas_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ + uint32_t newval) \ +{ \ + uint32_t res; \ + int tmp; \ + \ + __asm __volatile( \ + "1: ld"#a"xr %w0, [%2] \n" \ + " cmp %w0, %w3 \n" \ + " b.ne 2f \n" \ + " st"#l"xr %w1, %w4, [%2] \n" \ + " cbnz %w1, 1b \n" \ + "2:" \ + : "=&r"(res), "=&r"(tmp) \ + : "r" (p), "r" (cmpval), "r" (newval) \ + : "cc", "memory" \ + ); \ + \ + return (res); \ +} \ + \ +static __inline uint64_t \ +atomic_cas_##bar##64(volatile uint64_t *p, uint64_t cmpval, \ + uint64_t newval) \ +{ \ + uint64_t res; \ + int tmp; \ + \ + __asm __volatile( \ + "1: ld"#a"xr %0, [%2] \n" \ + " cmp %0, %3 \n" \ + " b.ne 2f \n" \ + " st"#l"xr %w1, %4, [%2] \n" \ + " cbnz %w1, 1b \n" \ + "2:" \ + : "=&r"(res), "=&r"(tmp) \ + : "r" (p), "r" (cmpval), "r" (newval) \ + : "cc", "memory" \ + ); \ + \ + return (res); \ +} + +ATOMIC_CAS( , , ) +ATOMIC_CAS(acq_, a, ) +ATOMIC_CAS(rel_, ,l) + +#undef ATOMIC_CAS + #define ATOMIC_CMPSET(bar, a, l) \ static __inline int \ atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ @@ -311,6 +362,7 @@ #define atomic_add_int atomic_add_32 +#define atomic_cas_int atomic_cas_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 @@ -320,6 +372,7 @@ #define atomic_subtract_int atomic_subtract_32 #define atomic_add_acq_int atomic_add_acq_32 +#define atomic_cas_acq_int atomic_cas_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 @@ -327,6 +380,7 @@ #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_add_rel_int atomic_add_rel_32 +#define atomic_cas_rel_int atomic_cas_rel_32 #define atomic_clear_rel_int atomic_add_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 @@ -334,6 +388,7 @@ #define atomic_store_rel_int atomic_store_rel_32 #define atomic_add_long atomic_add_64 +#define atomic_cas_long atomic_cas_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 @@ -343,6 +398,7 @@ #define atomic_subtract_long atomic_subtract_64 #define atomic_add_ptr atomic_add_64 +#define atomic_cas_ptr atomic_cas_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 @@ -352,6 +408,7 @@ #define atomic_subtract_ptr atomic_subtract_64 #define atomic_add_acq_long atomic_add_acq_64 +#define atomic_cas_acq_long atomic_cas_acq_64 #define atomic_clear_acq_long atomic_add_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 @@ -359,6 +416,7 @@ #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 +#define atomic_cas_acq_ptr atomic_cas_acq_64 #define atomic_clear_acq_ptr atomic_add_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 @@ -366,6 +424,7 @@ #define atomic_subtract_acq_ptr atomic_subtract_acq_64 #define atomic_add_rel_long atomic_add_rel_64 +#define atomic_cas_rel_long atomic_cas_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 @@ -373,6 +432,7 @@ #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 +#define atomic_cas_rel_ptr atomic_cas_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64