diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 91c33e6..a8dd2a6 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -54,12 +54,14 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -80,6 +82,8 @@ int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); +int atomic_testandset_int(volatile u_int *p, int v); +int atomic_testandset_long(volatile u_long *p, int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -211,6 +215,44 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2, %1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=r" (res), /* 0 */ + "=m" (*p) /* 1 */ + : "r" (v), /* 2 */ + "m" (*p) + : "cc"); + + return (res); +} + +static __inline int +atomic_testandset_long(volatile u_long *p, int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsq %2, %1 ; " + " setc %0 ; " + "# atomic_testandset_long" + : "=r" (res), /* 0 */ + "=m" (*p) /* 1 */ + : "r" ((u_long)v), /* 2 */ + "m" (*p) + : "cc"); + + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -303,45 +345,41 @@ ATOMIC_STORE(long); #ifndef WANT_FUNCTIONS -/* Read the current value and store a zero in the destination. */ +/* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int -atomic_readandclear_int(volatile u_int *addr) +atomic_swap_int(volatile u_int *p, u_int v) { - u_int res; - res = 0; __asm __volatile( - " xchgl %1,%0 ; " - "# atomic_readandclear_int" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); + " xchgl %1, %0 ; " + "# atomic_swap_int" + : "+r" (v), /* 0 */ + "=m" (*p) /* 1 */ + : "m" (*p)); - return (res); + return (v); } static __inline u_long -atomic_readandclear_long(volatile u_long *addr) +atomic_swap_long(volatile u_long *p, u_long v) { - u_long res; - res = 0; __asm __volatile( - " xchgq %1,%0 ; " - "# atomic_readandclear_long" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); + " xchgq %1, %0 ; " + "# atomic_swap_long" + : "+r" (v), /* 0 */ + "=m" (*p) /* 1 */ + : "m" (*p)); - return (res); + return (v); } #else /* !__GNUCLIKE_ASM */ -u_int atomic_readandclear_int(volatile u_int *addr); -u_long atomic_readandclear_long(volatile u_long *addr); +u_int atomic_swap_int(volatile u_int *, u_int); +u_long atomic_swap_long(volatile u_long *, u_long); #endif /* __GNUCLIKE_ASM */ @@ -385,6 +423,9 @@ u_long atomic_readandclear_long(volatile u_long *addr); #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_readandclear_int(p) atomic_swap_int(p, 0) +#define atomic_readandclear_long(p) atomic_swap_long(p, 0) + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char @@ -435,8 +476,10 @@ u_long atomic_readandclear_long(volatile u_long *addr); #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long @@ -456,7 +499,9 @@ u_long atomic_readandclear_long(volatile u_long *addr); #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long +#define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long +#define atomic_testandset_64 atomic_testandset_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long @@ -476,6 +521,7 @@ u_long atomic_readandclear_long(volatile u_long *addr); #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long +#define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ diff --git a/sys/dev/drm2/drm_atomic.h b/sys/dev/drm2/drm_atomic.h index 0f11f0b..b901341 100644 --- a/sys/dev/drm2/drm_atomic.h +++ b/sys/dev/drm2/drm_atomic.h @@ -32,110 +32,51 @@ #include __FBSDID("$FreeBSD$"); -/* Many of these implementations are rather fake, but good enough. */ +typedef uint32_t atomic_t; +typedef uint64_t atomic64_t; -typedef u_int32_t atomic_t; -typedef u_int64_t atomic64_t; +#define BITS_TO_LONGS(x) howmany(x, sizeof(long) * NBBY) -#define atomic_set(p, v) (*(p) = (v)) -#define atomic_read(p) (*(p)) -#define atomic_inc(p) atomic_add_int(p, 1) -#define atomic_dec(p) atomic_subtract_int(p, 1) -#define atomic_add(n, p) atomic_add_int(p, n) -#define atomic_sub(n, p) atomic_subtract_int(p, n) +#define atomic_set(p, v) atomic_store_rel_int(p, v) +#define atomic_read(p) atomic_load_acq_int(p) -static __inline atomic_t -test_and_set_bit(int b, volatile void *p) -{ - long bit, val; +#define atomic_add(v, p) atomic_add_int(p, v) +#define atomic_sub(v, p) atomic_subtract_int(p, v) +#define atomic_inc(p) atomic_add(1, p) +#define atomic_dec(p) atomic_sub(1, p) - bit = 1 << b; - do { - val = *(volatile long *)p; - } while (atomic_cmpset_long(p, val, val | bit) == 0); +#define atomic_add_return(v, p) (atomic_fetchadd_int(p, v) + (v)) +#define atomic_sub_return(v, p) (atomic_fetchadd_int(p, -(v)) - (v)) +#define atomic_inc_return(p) atomic_add_return(1, p) +#define atomic_dec_return(p) atomic_sub_return(1, p) - return ((val & bit) != 0); -} +#define atomic_add_and_test(v, p) (atomic_add_return(v, p) == 0) +#define atomic_sub_and_test(v, p) (atomic_sub_return(v, p) == 0) +#define atomic_inc_and_test(p) (atomic_inc_return(p) == 0) +#define atomic_dec_and_test(p) (atomic_dec_return(p) == 0) -static __inline void -clear_bit(int b, volatile void *p) -{ - atomic_clear_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); -} +#define atomic_xchg(p, v) atomic_swap_int(p, v) +#define atomic64_xchg(p, v) atomic_swap_64(p, v) -static __inline void -set_bit(int b, volatile void *p) -{ - atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); -} - -static __inline int -test_bit(int b, volatile void *p) -{ - return ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f)); -} +#define clear_bit(b, p) \ + atomic_clear_int((volatile u_int *)(p) + (b) / 32, 1 << (b) % 32) +#define set_bit(b, p) \ + atomic_set_int((volatile u_int *)(p) + (b) / 32, 1 << (b) % 32) +#define test_bit(b, p) \ + (atomic_load_acq_int((volatile u_int *)(p) + (b) / 32) & (1 << (b) % 32)) +#define test_and_set_bit(b, p) \ + atomic_testandset_int((volatile u_int *)(p) + (b) / 32, b) static __inline int find_first_zero_bit(volatile void *p, int max) { - int b; - volatile int *ptr = (volatile int *)p; + volatile int *np = p; + int i, n; - for (b = 0; b < max; b += 32) { - if (ptr[b >> 5] != ~0) { - for (;;) { - if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0) - return b; - b++; - } - } + for (i = 0; i < max / (NBBY * sizeof(int)); i++) { + n = ~np[i]; + if (n != 0) + return (i * NBBY * sizeof(int) + ffs(n) - 1); } - return max; -} - -static __inline int -atomic_xchg(volatile int *p, int new) -{ - int old; - - do { - old = *p; - } while(!atomic_cmpset_int(p, old, new)); - - return (old); -} - -static __inline uint64_t -atomic64_xchg(volatile uint64_t *p, uint64_t new) -{ - uint64_t old; - - do { - old = *p; - } while(!atomic_cmpset_64(p, old, new)); - - return (old); -} - -static __inline int -atomic_add_return(int i, atomic_t *p) -{ - - return i + atomic_fetchadd_int(p, i); + return (max); } - -static __inline int -atomic_sub_return(int i, atomic_t *p) -{ - - return atomic_fetchadd_int(p, -i) - i; -} - -#define atomic_inc_return(v) atomic_add_return(1, (v)) -#define atomic_dec_return(v) atomic_sub_return(1, (v)) - -#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) -#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - -#define BITS_TO_LONGS(x) (howmany((x), NBBY * sizeof(long))) diff --git a/sys/dev/drm2/drm_gem.c b/sys/dev/drm2/drm_gem.c index 2ddea98..b780e50 100644 --- a/sys/dev/drm2/drm_gem.c +++ b/sys/dev/drm2/drm_gem.c @@ -121,7 +121,7 @@ drm_gem_private_object_init(struct drm_device *dev, struct drm_gem_object *obj, obj->vm_obj = NULL; obj->refcount = 1; - atomic_set(&obj->handle_count, 0); + atomic_store_rel_int(&obj->handle_count, 0); obj->size = size; return (0); diff --git a/sys/dev/drm2/drm_irq.c b/sys/dev/drm2/drm_irq.c index 0324e8c..130c20f 100644 --- a/sys/dev/drm2/drm_irq.c +++ b/sys/dev/drm2/drm_irq.c @@ -786,7 +786,7 @@ int drm_vblank_get(struct drm_device *dev, int crtc) mtx_lock(&dev->vbl_lock); /* Going from 0->1 means we have to enable interrupts again */ - if (atomic_fetchadd_int(&dev->vblank_refcount[crtc], 1) == 0) { + if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) { mtx_lock(&dev->vblank_time_lock); if (!dev->vblank_enabled[crtc]) { /* Enable vblank irqs under vblank_time_lock protection. @@ -831,7 +831,7 @@ void drm_vblank_put(struct drm_device *dev, int crtc) ("Too many drm_vblank_put for crtc %d", crtc)); /* Last user schedules interrupt disable */ - if (atomic_fetchadd_int(&dev->vblank_refcount[crtc], -1) == 1 && + if (atomic_dec_and_test(&dev->vblank_refcount[crtc]) && (drm_vblank_offdelay > 0)) callout_reset(&dev->vblank_disable_callout, (drm_vblank_offdelay * DRM_HZ) / 1000, diff --git a/sys/dev/drm2/ttm/ttm_bo.c b/sys/dev/drm2/ttm/ttm_bo.c index 3b13540..f598748 100644 --- a/sys/dev/drm2/ttm/ttm_bo.c +++ b/sys/dev/drm2/ttm/ttm_bo.c @@ -1723,7 +1723,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, if (driver->sync_obj_signaled(bo->sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); + atomic_clear_long(&bo->priv_flags, + 1UL << TTM_BO_PRIV_FLAG_MOVING); mtx_unlock(&bdev->fence_lock); driver->sync_obj_unref(&tmp_obj); mtx_lock(&bdev->fence_lock); @@ -1746,8 +1747,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, if (likely(bo->sync_obj == sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, - &bo->priv_flags); + atomic_clear_long(&bo->priv_flags, + 1UL << TTM_BO_PRIV_FLAG_MOVING); mtx_unlock(&bdev->fence_lock); driver->sync_obj_unref(&sync_obj); driver->sync_obj_unref(&tmp_obj); diff --git a/sys/dev/drm2/ttm/ttm_bo_util.c b/sys/dev/drm2/ttm/ttm_bo_util.c index bca0ad3..98d4a5f 100644 --- a/sys/dev/drm2/ttm/ttm_bo_util.c +++ b/sys/dev/drm2/ttm/ttm_bo_util.c @@ -638,7 +638,8 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, * operation has completed. */ - set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); + atomic_set_long(&bo->priv_flags, + 1UL << TTM_BO_PRIV_FLAG_MOVING); mtx_unlock(&bdev->fence_lock); if (tmp_obj) driver->sync_obj_unref(&tmp_obj); diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c index 6b4e86e..fdff8cb 100644 --- a/sys/dev/drm2/ttm/ttm_bo_vm.c +++ b/sys/dev/drm2/ttm/ttm_bo_vm.c @@ -152,7 +152,8 @@ reserve: */ mtx_lock(&bdev->fence_lock); - if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { + if ((atomic_load_acq_long(&bo->priv_flags) & + (1UL << TTM_BO_PRIV_FLAG_MOVING)) != 0) { /* * Here, the behavior differs between Linux and FreeBSD. * diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index 3b9d001..7a01829 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -54,12 +54,14 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -78,6 +80,7 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); +int atomic_testandset_int(volatile u_int *p, int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -281,6 +284,25 @@ atomic_fetchadd_int(volatile u_int *p, u_int v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2, %1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=r" (res), /* 0 */ + "=m" (*p) /* 1 */ + : "r" (v), /* 2 */ + "m" (*p) + : "cc"); + + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -393,45 +415,41 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); } -/* Read the current value and store a zero in the destination. */ +static __inline int +atomic_testandset_long(volatile u_long *p, int v) +{ + + return (atomic_testandset_int((volatile u_int *)p, v)); +} + +/* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int -atomic_readandclear_int(volatile u_int *addr) +atomic_swap_int(volatile u_int *p, u_int v) { - u_int res; - res = 0; __asm __volatile( - " xchgl %1,%0 ; " - "# atomic_readandclear_int" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); + " xchgl %1, %0 ; " + "# atomic_swap_int" + : "+r" (v), /* 0 */ + "=m" (*p) /* 1 */ + : "m" (*p)); - return (res); + return (v); } static __inline u_long -atomic_readandclear_long(volatile u_long *addr) +atomic_swap_long(volatile u_long *p, u_long v) { - u_long res; - - res = 0; - __asm __volatile( - " xchgl %1,%0 ; " - "# atomic_readandclear_long" - : "+r" (res), /* 0 */ - "=m" (*addr) /* 1 */ - : "m" (*addr)); - return (res); + return (atomic_swap_int((volatile u_int *)p, (u_int)v)); } #else /* !__GNUCLIKE_ASM */ -u_int atomic_readandclear_int(volatile u_int *addr); -u_long atomic_readandclear_long(volatile u_long *addr); +u_int atomic_swap_int(volatile u_int *, u_int); +u_long atomic_swap_long(volatile u_long *, u_long); #endif /* __GNUCLIKE_ASM */ @@ -475,6 +493,9 @@ u_long atomic_readandclear_long(volatile u_long *addr); #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_readandclear_int(p) atomic_swap_int(p, 0) +#define atomic_readandclear_long(p) atomic_swap_long(p, 0) + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char @@ -525,8 +546,10 @@ u_long atomic_readandclear_long(volatile u_long *addr); #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on pointers. */ #define atomic_set_ptr(p, v) \ @@ -565,6 +588,8 @@ u_long atomic_readandclear_long(volatile u_long *addr); #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) +#define atomic_swap_ptr(p, v) \ + atomic_swap_int((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_ptr(p) \ atomic_readandclear_int((volatile u_int *)(p))