/*-
 * Copyright (c) 1998 Doug Rabson
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/amd64/include/atomic.h 254618 2013-08-21 22:05:58Z jkim $
 */
#ifndef __LWBASE_ATOMIC_LOCK_H__
#define __LWBASE_ATOMIC_LOCK_H__

extern pthread_mutex_t gAtomicMutex;
extern int pthread_mutex_lock(pthread_mutex_t *mutex);
extern int pthread_mutex_unlock(pthread_mutex_t *mutex);

/*
 * Various simple operations on memory, each of which is atomic in the
 * presence of interrupts and multiple processors.
 *
 * lw_atomic_set_char(P, V)		(*(unsigned char *)(P) |= (V))
 * lw_atomic_clear_char(P, V)		(*(unsigned char *)(P) &= ~(V))
 * lw_atomic_add_char(P, V)		(*(unsigned char *)(P) += (V))
 * lw_atomic_subtract_char(P, V)	(*(unsigned char *)(P) -= (V))
 *
 * lw_atomic_set_short(P, V)		(*(unsigned short *)(P) |= (V))
 * lw_atomic_clear_short(P, V)		(*(unsigned short *)(P) &= ~(V))
 * lw_atomic_add_short(P, V)		(*(unsigned short *)(P) += (V))
 * lw_atomic_subtract_short(P, V)	(*(unsigned short *)(P) -= (V))
 *
 * lw_atomic_set_int(P, V)		(*(unsigned int *)(P) |= (V))
 * lw_atomic_clear_int(P, V)		(*(unsigned int *)(P) &= ~(V))
 * lw_atomic_add_int(P, V)		(*(unsigned int *)(P) += (V))
 * lw_atomic_subtract_int(P, V)		(*(unsigned int *)(P) -= (V))
 * lw_atomic_swap_int(P, V)		(return (*(unsigned int *)(P));
 * 				 	 *(usigned int *)(P) = (V);)
 * lw_atomic_readandclear_int(P)	(return (*(usigned int *)(P));
 *					 *(unsigned int *)(P) = 0;)
 *
 * lw_atomic_set_long(P, V)		(*(unsigned long *)(P) |= (V))
 * lw_atomic_clear_long(P, V)		(*(unsigned long *)(P) &= ~(V))
 * lw_atomic_add_long(P, V)		(*(unsigned long *)(P) += (V))
 * lw_atomic_subtract_long(P, V)	(*(unsigned long *)(P) -= (V))
 * lw_atomic_swap_long(P, V)		(return (*(unsigned long *)(P));
 *					 *(unsigned long *)(P) = (V);)
 * lw_atomic_readandclear_long(P)	(return (*(unsigned long *)(P));
 *					 *(unsigned long *)(P) = 0;)
 */

#define	ATOMIC_FUNC(NAME, TNAME, TYPE, OP)			\
static inline void						\
lw_atomic_##NAME##_##TNAME(volatile TYPE *p, TYPE v)		\
{								\
								\
	pthread_mutex_lock(&gAtomicMutex);			\
	OP;							\
	pthread_mutex_unlock(&gAtomicMutex);			\
}								\
								\
static inline void						\
lw_atomic_##NAME##_barr_##TNAME(volatile TYPE *p, TYPE v)	\
{								\
								\
	lw_atomic_##NAME##_##TNAME(p, v);			\
}

#define ATOMIC_CTYPE_FUNC(NAME, TYPE, OP)		\
	ATOMIC_FUNC(NAME, TYPE, TYPE, OP)		\
	ATOMIC_FUNC(NAME, u##TYPE, unsigned TYPE, OP)

ATOMIC_CTYPE_FUNC(set,      char,  *p |= v)
ATOMIC_CTYPE_FUNC(clear,    char,  *p &= ~v)
ATOMIC_CTYPE_FUNC(add,      char,  *p += v)
ATOMIC_CTYPE_FUNC(subtract, char,  *p -= v)

ATOMIC_CTYPE_FUNC(set,      short, *p |= v)
ATOMIC_CTYPE_FUNC(clear,    short, *p &= ~v)
ATOMIC_CTYPE_FUNC(add,      short, *p += v)
ATOMIC_CTYPE_FUNC(subtract, short, *p -= v)

ATOMIC_CTYPE_FUNC(set,      int,   *p |= v)
ATOMIC_CTYPE_FUNC(clear,    int,   *p &= ~v)
ATOMIC_CTYPE_FUNC(add,      int,   *p += v)
ATOMIC_CTYPE_FUNC(subtract, int,   *p -= v)

ATOMIC_CTYPE_FUNC(set,      long,  *p |= v)
ATOMIC_CTYPE_FUNC(clear,    long,  *p &= ~v)
ATOMIC_CTYPE_FUNC(add,      long,  *p += v)
ATOMIC_CTYPE_FUNC(subtract, long,  *p -= v)
#undef ATOMIC_CTYPE_FUNC

#define ATOMIC_STDINT_FUNC(NAME, TYPE, OP)		\
	ATOMIC_FUNC(NAME, TYPE, TYPE, OP)		\
	ATOMIC_FUNC(NAME, u##TYPE, u##TYPE, OP)

ATOMIC_STDINT_FUNC(set,      int8_t,  *p |= v)
ATOMIC_STDINT_FUNC(clear,    int8_t,  *p &= ~v)
ATOMIC_STDINT_FUNC(add,      int8_t,  *p += v)
ATOMIC_STDINT_FUNC(subtract, int8_t,  *p -= v)

ATOMIC_STDINT_FUNC(set,      int16_t, *p |= v)
ATOMIC_STDINT_FUNC(clear,    int16_t, *p &= ~v)
ATOMIC_STDINT_FUNC(add,      int16_t, *p += v)
ATOMIC_STDINT_FUNC(subtract, int16_t, *p -= v)

ATOMIC_STDINT_FUNC(set,      int32_t, *p |= v)
ATOMIC_STDINT_FUNC(clear,    int32_t, *p &= ~v)
ATOMIC_STDINT_FUNC(add,      int32_t, *p += v)
ATOMIC_STDINT_FUNC(subtract, int32_t, *p -= v)

ATOMIC_STDINT_FUNC(set,      int64_t, *p |= v)
ATOMIC_STDINT_FUNC(clear,    int64_t, *p &= ~v)
ATOMIC_STDINT_FUNC(add,      int64_t, *p += v)
ATOMIC_STDINT_FUNC(subtract, int64_t, *p -= v)
#undef ATOMIC_STDINT_FUNC
#undef ATOMIC_FUNC

/*
 * Atomic compare and set, used by the mutex functions
 *
 * if (*dst == expect) *dst = src (all 32 bit words)
 *
 * Returns 0 on failure, non-zero on success
 */
#define	ATOMIC_CMPSET(TNAME, TYPE)					\
static inline unsigned int						\
lw_atomic_cmpset_##TNAME(volatile TYPE *dst, TYPE expect, TYPE src)	\
{									\
	unsigned char res;						\
									\
	res = 0;							\
	pthread_mutex_lock(&gAtomicMutex);				\
	if (*dst == expect) {						\
		*dst = src;						\
		res = 1;						\
	}								\
	pthread_mutex_unlock(&gAtomicMutex);				\
	return (res);							\
}

#define	ATOMIC_CTYPE_CMPSET(TYPE)		\
	ATOMIC_CMPSET(TYPE, TYPE)		\
	ATOMIC_CMPSET(u##TYPE, unsigned TYPE)

ATOMIC_CTYPE_CMPSET(int)
ATOMIC_CTYPE_CMPSET(long)

#define	ATOMIC_STDINT_CMPSET(TYPE)		\
	ATOMIC_CMPSET(TYPE, TYPE)		\
	ATOMIC_CMPSET(u##TYPE, u##TYPE)

ATOMIC_STDINT_CMPSET(int32_t)
ATOMIC_STDINT_CMPSET(int64_t)
ATOMIC_STDINT_CMPSET(intptr_t)
#undef ATOMIC_STDINT_CMPSET
#undef ATOMIC_CMPSET

/*
 * Atomically add the value of v to the integer pointed to by p and return
 * the previous value of *p.
 */
#define	ATOMIC_FETCHADD(TNAME, TYPE)					\
static inline TYPE							\
lw_atomic_fetchadd_##TNAME(volatile TYPE *p, TYPE v)			\
{									\
	TYPE previous;							\
									\
	pthread_mutex_lock(&gAtomicMutex);				\
	previous = *p;							\
	*p += v;							\
	pthread_mutex_unlock(&gAtomicMutex);				\
	return (previous);						\
}

#define	ATOMIC_CTYPE_FETCHADD(TYPE)		\
	ATOMIC_FETCHADD(TYPE, TYPE)		\
	ATOMIC_FETCHADD(u##TYPE, unsigned TYPE)

ATOMIC_CTYPE_FETCHADD(int)
ATOMIC_CTYPE_FETCHADD(long)
#undef ATOMIC_CTYPE_FETCHADD

#define	ATOMIC_STDINT_FETCHADD(TYPE)		\
	ATOMIC_FETCHADD(TYPE, TYPE)		\
	ATOMIC_FETCHADD(u##TYPE, u##TYPE)

ATOMIC_STDINT_FETCHADD(int32_t)
ATOMIC_STDINT_FETCHADD(int64_t)
ATOMIC_STDINT_FETCHADD(intptr_t)
#undef ATOMIC_STDINT_FETCHADD
#undef ATOMIC_FETCHADD

#define	ATOMIC_TESTANDSET(TNAME, TYPE)					\
static inline int							\
lw_atomic_testandset_##TNAME(volatile TYPE *p, unsigned int bit_idx)	\
{									\
	int res;							\
									\
	pthread_mutex_lock(&gAtomicMutex);				\
	res = (*p & ((TYPE)0x1 << bit_idx)) != 0;			\
	*p |= 0x1 << bit_idx;						\
	pthread_mutex_unlock(&gAtomicMutex);				\
	return (res);							\
}

#define	ATOMIC_CTYPE_TESTANDSET(TYPE)			\
	ATOMIC_TESTANDSET(TYPE, TYPE)			\
	ATOMIC_TESTANDSET(u##TYPE, unsigned TYPE)

ATOMIC_CTYPE_TESTANDSET(int)
ATOMIC_CTYPE_TESTANDSET(long)
#undef	ATOMIC_CTYPE_TESTANDSET

#define	ATOMIC_STDINT_TESTANDSET(TYPE)			\
	ATOMIC_TESTANDSET(TYPE, TYPE)			\
	ATOMIC_TESTANDSET(u##TYPE, u##TYPE)

ATOMIC_STDINT_TESTANDSET(int32_t)
ATOMIC_STDINT_TESTANDSET(int64_t)
ATOMIC_STDINT_TESTANDSET(intptr_t)
#undef ATOMIC_STDINT_TESTANDSET
#undef ATOMIC_TESTANDSET

#define	ATOMIC_STORE(TNAME, TYPE)			\
static inline void					\
lw_atomic_store_rel_##TNAME(volatile TYPE *p, TYPE v)	\
{							\
	pthread_mutex_lock(&gAtomicMutex);		\
	*p = v;						\
	pthread_mutex_unlock(&gAtomicMutex);		\
}

#define	ATOMIC_CTYPE_STORE(TYPE)			\
	ATOMIC_STORE(TYPE, TYPE)			\
	ATOMIC_STORE(u##TYPE, unsigned TYPE)

ATOMIC_CTYPE_STORE(char)
ATOMIC_CTYPE_STORE(short)
ATOMIC_CTYPE_STORE(int)
ATOMIC_CTYPE_STORE(long)
#undef	ATOMIC_CTYPE_STORE

#define	ATOMIC_STDINT_STORE(TYPE)			\
	ATOMIC_STORE(TYPE, TYPE)			\
	ATOMIC_STORE(u##TYPE, u##TYPE)

ATOMIC_STDINT_STORE(int8_t)
ATOMIC_STDINT_STORE(int16_t)
ATOMIC_STDINT_STORE(int32_t)
ATOMIC_STDINT_STORE(int64_t)
#undef ATOMIC_STDINT_STORE
#undef ATOMIC_STORE

#define	ATOMIC_LOAD(TNAME, TYPE)		\
static inline TYPE				\
lw_atomic_load_acq_##TNAME(volatile TYPE *p)	\
{						\
	TYPE res;				\
						\
	pthread_mutex_lock(&gAtomicMutex);	\
	res = *p;				\
	pthread_mutex_unlock(&gAtomicMutex);	\
	return (res);				\
}

#define	ATOMIC_CTYPE_LOAD(TYPE)			\
	ATOMIC_LOAD(TYPE, TYPE)			\
	ATOMIC_LOAD(u##TYPE, unsigned TYPE)

ATOMIC_CTYPE_LOAD(char)
ATOMIC_CTYPE_LOAD(short)
ATOMIC_CTYPE_LOAD(int)
ATOMIC_CTYPE_LOAD(long)
#undef ATOMIC_CTYPE_LOAD

#define	ATOMIC_STDINT_LOAD(TYPE)		\
	ATOMIC_LOAD(TYPE, TYPE)			\
	ATOMIC_LOAD(u##TYPE, u##TYPE)

ATOMIC_STDINT_LOAD(int8_t)
ATOMIC_STDINT_LOAD(int16_t)
ATOMIC_STDINT_LOAD(int32_t)
ATOMIC_STDINT_LOAD(int64_t)
#undef ATOMIC_STDINT_LOAD
#undef ATOMIC_LOAD

#define	ATOMIC_SWAP(TNAME, TYPE)			\
static inline TYPE					\
lw_atomic_swap_##TNAME(volatile TYPE *p, TYPE v)	\
{							\
	TYPE previous;					\
	pthread_mutex_lock(&gAtomicMutex);		\
	previous = *p;					\
	*p = v;						\
	pthread_mutex_unlock(&gAtomicMutex);		\
	return (previous);				\
}

#define	ATOMIC_CTYPE_SWAP(TYPE)		\
	ATOMIC_SWAP(TYPE, TYPE)		\
	ATOMIC_SWAP(u##TYPE, unsigned TYPE)

ATOMIC_CTYPE_SWAP(int)
ATOMIC_CTYPE_SWAP(long)
#undef ATOMIC_CTYPE_SWAP

#define	ATOMIC_STDINT_SWAP(TYPE)	\
	ATOMIC_SWAP(TYPE, TYPE)		\
	ATOMIC_SWAP(u##TYPE, u##TYPE)

ATOMIC_STDINT_SWAP(int32_t)
ATOMIC_STDINT_SWAP(int64_t)
#undef ATOMIC_STDINT_SWAP
#undef ATOMIC_SWAP

#define	lw_atomic_set_acq_char		lw_atomic_set_barr_char
#define	lw_atomic_set_rel_char		lw_atomic_set_barr_char
#define	lw_atomic_clear_acq_char	lw_atomic_clear_barr_char
#define	lw_atomic_clear_rel_char	lw_atomic_clear_barr_char
#define	lw_atomic_add_acq_char		lw_atomic_add_barr_char
#define	lw_atomic_add_rel_char		lw_atomic_add_barr_char
#define	lw_atomic_subtract_acq_char	lw_atomic_subtract_barr_char
#define	lw_atomic_subtract_rel_char	lw_atomic_subtract_barr_char
#define	lw_atomic_set_acq_uchar		lw_atomic_set_barr_uchar
#define	lw_atomic_set_rel_uchar		lw_atomic_set_barr_uchar
#define	lw_atomic_clear_acq_uchar	lw_atomic_clear_barr_uchar
#define	lw_atomic_clear_rel_uchar	lw_atomic_clear_barr_uchar
#define	lw_atomic_add_acq_uchar		lw_atomic_add_barr_uchar
#define	lw_atomic_add_rel_uchar		lw_atomic_add_barr_uchar
#define	lw_atomic_subtract_acq_uchar	lw_atomic_subtract_barr_uchar
#define	lw_atomic_subtract_rel_uchar	lw_atomic_subtract_barr_uchar

#define	lw_atomic_set_acq_short		lw_atomic_set_barr_short
#define	lw_atomic_set_rel_short		lw_atomic_set_barr_short
#define	lw_atomic_clear_acq_short	lw_atomic_clear_barr_short
#define	lw_atomic_clear_rel_short	lw_atomic_clear_barr_short
#define	lw_atomic_add_acq_short		lw_atomic_add_barr_short
#define	lw_atomic_add_rel_short		lw_atomic_add_barr_short
#define	lw_atomic_subtract_acq_short	lw_atomic_subtract_barr_short
#define	lw_atomic_subtract_rel_short	lw_atomic_subtract_barr_short
#define	lw_atomic_set_acq_ushort		lw_atomic_set_barr_ushort
#define	lw_atomic_set_rel_ushort		lw_atomic_set_barr_ushort
#define	lw_atomic_clear_acq_ushort	lw_atomic_clear_barr_ushort
#define	lw_atomic_clear_rel_ushort	lw_atomic_clear_barr_ushort
#define	lw_atomic_add_acq_ushort		lw_atomic_add_barr_ushort
#define	lw_atomic_add_rel_ushort		lw_atomic_add_barr_ushort
#define	lw_atomic_subtract_acq_ushort	lw_atomic_subtract_barr_ushort
#define	lw_atomic_subtract_rel_ushort	lw_atomic_subtract_barr_ushort

#define	lw_atomic_set_acq_int		lw_atomic_set_barr_int
#define	lw_atomic_set_rel_int		lw_atomic_set_barr_int
#define	lw_atomic_clear_acq_int		lw_atomic_clear_barr_int
#define	lw_atomic_clear_rel_int		lw_atomic_clear_barr_int
#define	lw_atomic_add_acq_int		lw_atomic_add_barr_int
#define	lw_atomic_add_rel_int		lw_atomic_add_barr_int
#define	lw_atomic_subtract_acq_int	lw_atomic_subtract_barr_int
#define	lw_atomic_subtract_rel_int	lw_atomic_subtract_barr_int
#define	lw_atomic_cmpset_acq_int	lw_atomic_cmpset_int
#define	lw_atomic_cmpset_rel_int	lw_atomic_cmpset_int
#define	lw_atomic_set_acq_uint		lw_atomic_set_barr_uint
#define	lw_atomic_set_rel_uint		lw_atomic_set_barr_uint
#define	lw_atomic_clear_acq_uint	lw_atomic_clear_barr_uint
#define	lw_atomic_clear_rel_uint	lw_atomic_clear_barr_uint
#define	lw_atomic_add_acq_uint		lw_atomic_add_barr_uint
#define	lw_atomic_add_rel_uint		lw_atomic_add_barr_uint
#define	lw_atomic_subtract_acq_uint	lw_atomic_subtract_barr_uint
#define	lw_atomic_subtract_rel_uint	lw_atomic_subtract_barr_uint
#define	lw_atomic_cmpset_acq_uint	lw_atomic_cmpset_uint
#define	lw_atomic_cmpset_rel_uint	lw_atomic_cmpset_uint

#define	lw_atomic_set_acq_long		lw_atomic_set_barr_long
#define	lw_atomic_set_rel_long		lw_atomic_set_barr_long
#define	lw_atomic_clear_acq_long	lw_atomic_clear_barr_long
#define	lw_atomic_clear_rel_long	lw_atomic_clear_barr_long
#define	lw_atomic_add_acq_long		lw_atomic_add_barr_long
#define	lw_atomic_add_rel_long		lw_atomic_add_barr_long
#define	lw_atomic_subtract_acq_long	lw_atomic_subtract_barr_long
#define	lw_atomic_subtract_rel_long	lw_atomic_subtract_barr_long
#define	lw_atomic_cmpset_acq_long	lw_atomic_cmpset_long
#define	lw_atomic_cmpset_rel_long	lw_atomic_cmpset_long
#define	lw_atomic_set_acq_ulong		lw_atomic_set_barr_ulong
#define	lw_atomic_set_rel_ulong		lw_atomic_set_barr_ulong
#define	lw_atomic_clear_acq_ulong	lw_atomic_clear_barr_ulong
#define	lw_atomic_clear_rel_ulong	lw_atomic_clear_barr_ulong
#define	lw_atomic_add_acq_ulong		lw_atomic_add_barr_ulong
#define	lw_atomic_add_rel_ulong		lw_atomic_add_barr_ulong
#define	lw_atomic_subtract_acq_ulong	lw_atomic_subtract_barr_ulong
#define	lw_atomic_subtract_rel_ulong	lw_atomic_subtract_barr_ulong
#define	lw_atomic_cmpset_acq_ulong	lw_atomic_cmpset_ulong
#define	lw_atomic_cmpset_rel_ulong	lw_atomic_cmpset_ulong

#define	lw_atomic_readandclear_int(p)	lw_atomic_swap_int(p, 0)
#define	lw_atomic_readandclear_uint(p)	lw_atomic_swap_uint(p, 0)

#define	lw_atomic_readandclear_long(p)	lw_atomic_swap_long(p, 0)
#define	lw_atomic_readandclear_ulong(p)	lw_atomic_swap_ulong(p, 0)

/* Operations on 8-bit bytes. */
#define	lw_atomic_set_8			lw_atomic_set_int8_t
#define	lw_atomic_set_acq_8		lw_atomic_set_acq_int8_t
#define	lw_atomic_set_rel_8		lw_atomic_set_rel_int8_t
#define	lw_atomic_clear_8		lw_atomic_clear_int8_t
#define	lw_atomic_clear_acq_8		lw_atomic_clear_acq_int8_t
#define	lw_atomic_clear_rel_8		lw_atomic_clear_rel_int8_t
#define	lw_atomic_add_8			lw_atomic_add_int8_t
#define	lw_atomic_add_acq_8		lw_atomic_add_acq_int8_t
#define	lw_atomic_add_rel_8		lw_atomic_add_rel_int8_t
#define	lw_atomic_subtract_8		lw_atomic_subtract_int8_t
#define	lw_atomic_subtract_acq_8	lw_atomic_subtract_acq_int8_t
#define	lw_atomic_subtract_rel_8	lw_atomic_subtract_rel_int8_t
#define	lw_atomic_load_acq_8		lw_atomic_load_acq_int8_t
#define	lw_atomic_store_rel_8		lw_atomic_store_rel_int8_t
#define	lw_atomic_set_u8		lw_atomic_set_uint8_t
#define	lw_atomic_set_acq_u8		lw_atomic_set_acq_uint8_t
#define	lw_atomic_set_rel_u8		lw_atomic_set_rel_uint8_t
#define	lw_atomic_clear_u8		lw_atomic_clear_uint8_t
#define	lw_atomic_clear_acq_u8		lw_atomic_clear_acq_uint8_t
#define	lw_atomic_clear_rel_u8		lw_atomic_clear_rel_uint8_t
#define	lw_atomic_add_u8		lw_atomic_add_uint8_t
#define	lw_atomic_add_acq_u8		lw_atomic_add_acq_uint8_t
#define	lw_atomic_add_rel_u8		lw_atomic_add_rel_uint8_t
#define	lw_atomic_subtract_u8		lw_atomic_subtract_uint8_t
#define	lw_atomic_subtract_acq_u8	lw_atomic_subtract_acq_uint8_t
#define	lw_atomic_subtract_rel_u8	lw_atomic_subtract_rel_uint8_t
#define	lw_atomic_load_acq_u8		lw_atomic_load_acq_uint8_t
#define	lw_atomic_store_rel_u8		lw_atomic_store_rel_uint8_t

/* Operations on 16-bit words. */
#define	lw_atomic_set_16		lw_atomic_set_int16_t
#define	lw_atomic_set_acq_16		lw_atomic_set_acq_int16_t
#define	lw_atomic_set_rel_16		lw_atomic_set_rel_int16_t
#define	lw_atomic_clear_16		lw_atomic_clear_int16_t
#define	lw_atomic_clear_acq_16		lw_atomic_clear_acq_int16_t
#define	lw_atomic_clear_rel_16		lw_atomic_clear_rel_int16_t
#define	lw_atomic_add_16		lw_atomic_add_int16_t
#define	lw_atomic_add_acq_16		lw_atomic_add_acq_int16_t
#define	lw_atomic_add_rel_16		lw_atomic_add_rel_int16_t
#define	lw_atomic_subtract_16		lw_atomic_subtract_int16_t
#define	lw_atomic_subtract_acq_16	lw_atomic_subtract_acq_int16_t
#define	lw_atomic_subtract_rel_16	lw_atomic_subtract_rel_int16_t
#define	lw_atomic_load_acq_16		lw_atomic_load_acq_int16_t
#define	lw_atomic_store_rel_16		lw_atomic_store_rel_int16_t
#define	lw_atomic_set_u16		lw_atomic_set_uint16_t
#define	lw_atomic_set_acq_u16		lw_atomic_set_acq_uint16_t
#define	lw_atomic_set_rel_u16		lw_atomic_set_rel_uint16_t
#define	lw_atomic_clear_u16		lw_atomic_clear_uint16_t
#define	lw_atomic_clear_acq_u16		lw_atomic_clear_acq_uint16_t
#define	lw_atomic_clear_rel_u16		lw_atomic_clear_rel_uint16_t
#define	lw_atomic_add_u16		lw_atomic_add_uint16_t
#define	lw_atomic_add_acq_u16		lw_atomic_add_acq_uint16_t
#define	lw_atomic_add_rel_u16		lw_atomic_add_rel_uint16_t
#define	lw_atomic_subtract_u16		lw_atomic_subtract_uint16_t
#define	lw_atomic_subtract_acq_u16	lw_atomic_subtract_acq_uint16_t
#define	lw_atomic_subtract_rel_u16	lw_atomic_subtract_rel_uint16_t
#define	lw_atomic_load_acq_u16		lw_atomic_load_acq_uint16_t
#define	lw_atomic_store_rel_u16		lw_atomic_store_rel_uint16_t

/* Operations on 32-bit double words. */
#define	lw_atomic_set_32		lw_atomic_set_int32_t
#define	lw_atomic_set_acq_32		lw_atomic_set_acq_int32_t
#define	lw_atomic_set_rel_32		lw_atomic_set_rel_int32_t
#define	lw_atomic_clear_32		lw_atomic_clear_int32_t
#define	lw_atomic_clear_acq_32		lw_atomic_clear_acq_int32_t
#define	lw_atomic_clear_rel_32		lw_atomic_clear_rel_int32_t
#define	lw_atomic_add_32		lw_atomic_add_int32_t
#define	lw_atomic_add_acq_32		lw_atomic_add_acq_int32_t
#define	lw_atomic_add_rel_32		lw_atomic_add_rel_int32_t
#define	lw_atomic_subtract_32		lw_atomic_subtract_int32_t
#define	lw_atomic_subtract_acq_32	lw_atomic_subtract_acq_int32_t
#define	lw_atomic_subtract_rel_32	lw_atomic_subtract_rel_int32_t
#define	lw_atomic_load_acq_32		lw_atomic_load_acq_int32_t
#define	lw_atomic_store_rel_32		lw_atomic_store_rel_int32_t
#define	lw_atomic_cmpset_32		lw_atomic_cmpset_int32_t
#define	lw_atomic_cmpset_acq_32		lw_atomic_cmpset_acq_int32_t
#define	lw_atomic_cmpset_rel_32		lw_atomic_cmpset_rel_int32_t
#define	lw_atomic_swap_32		lw_atomic_swap_int32_t
#define	lw_atomic_readandclear_32	lw_atomic_readandclear_int32_t
#define	lw_atomic_fetchadd_32		lw_atomic_fetchadd_int32_t
#define	lw_atomic_testandset_32		lw_atomic_testandset_int32_t
#define	lw_atomic_set_u32		lw_atomic_set_uint32_t
#define	lw_atomic_set_acq_u32		lw_atomic_set_acq_uint32_t
#define	lw_atomic_set_rel_u32		lw_atomic_set_rel_uint32_t
#define	lw_atomic_clear_u32		lw_atomic_clear_uint32_t
#define	lw_atomic_clear_acq_u32		lw_atomic_clear_acq_uint32_t
#define	lw_atomic_clear_rel_u32		lw_atomic_clear_rel_uint32_t
#define	lw_atomic_add_u32		lw_atomic_add_uint32_t
#define	lw_atomic_add_acq_u32		lw_atomic_add_acq_uint32_t
#define	lw_atomic_add_rel_u32		lw_atomic_add_rel_uint32_t
#define	lw_atomic_subtract_u32		lw_atomic_subtract_uint32_t
#define	lw_atomic_subtract_acq_u32	lw_atomic_subtract_acq_uint32_t
#define	lw_atomic_subtract_rel_u32	lw_atomic_subtract_rel_uint32_t
#define	lw_atomic_load_acq_u32		lw_atomic_load_acq_uint32_t
#define	lw_atomic_store_rel_u32		lw_atomic_store_rel_uint32_t
#define	lw_atomic_cmpset_u32		lw_atomic_cmpset_uint32_t
#define	lw_atomic_cmpset_acq_u32	lw_atomic_cmpset_acq_uint32_t
#define	lw_atomic_cmpset_rel_u32	lw_atomic_cmpset_rel_uint32_t
#define	lw_atomic_swap_u32		lw_atomic_swap_uint32_t
#define	lw_atomic_readandclear_u32	lw_atomic_readandclear_uint32_t
#define	lw_atomic_fetchadd_u32		lw_atomic_fetchadd_uint32_t
#define	lw_atomic_testandset_u32	lw_atomic_testandset_uint32_t

/* Operations on 64-bit quad words. */
#define	lw_atomic_set_64		lw_atomic_set_int64_t
#define	lw_atomic_set_acq_64		lw_atomic_set_acq_int64_t
#define	lw_atomic_set_rel_64		lw_atomic_set_rel_int64_t
#define	lw_atomic_clear_64		lw_atomic_clear_int64_t
#define	lw_atomic_clear_acq_64		lw_atomic_clear_acq_int64_t
#define	lw_atomic_clear_rel_64		lw_atomic_clear_rel_int64_t
#define	lw_atomic_add_64		lw_atomic_add_int64_t
#define	lw_atomic_add_acq_64		lw_atomic_add_acq_int64_t
#define	lw_atomic_add_rel_64		lw_atomic_add_rel_int64_t
#define	lw_atomic_subtract_64		lw_atomic_subtract_int64_t
#define	lw_atomic_subtract_acq_64	lw_atomic_subtract_acq_int64_t
#define	lw_atomic_subtract_rel_64	lw_atomic_subtract_rel_int64_t
#define	lw_atomic_load_acq_64		lw_atomic_load_acq_int64_t
#define	lw_atomic_store_rel_64		lw_atomic_store_rel_int64_t
#define	lw_atomic_cmpset_64		lw_atomic_cmpset_int64_t
#define	lw_atomic_cmpset_acq_64		lw_atomic_cmpset_acq_int64_t
#define	lw_atomic_cmpset_rel_64		lw_atomic_cmpset_rel_int64_t
#define	lw_atomic_swap_64		lw_atomic_swap_int64_t
#define	lw_atomic_readandclear_64	lw_atomic_readandclear_int64_t
#define	lw_atomic_testandset_64		lw_atomic_testandset_int64_t
#define	lw_atomic_set_u64		lw_atomic_set_uint64_t
#define	lw_atomic_set_acq_u64		lw_atomic_set_acq_uint64_t
#define	lw_atomic_set_rel_u64		lw_atomic_set_rel_uint64_t
#define	lw_atomic_clear_u64		lw_atomic_clear_uint64_t
#define	lw_atomic_clear_acq_u64		lw_atomic_clear_acq_uint64_t
#define	lw_atomic_clear_rel_u64		lw_atomic_clear_rel_uint64_t
#define	lw_atomic_add_u64		lw_atomic_add_uint64_t
#define	lw_atomic_add_acq_u64		lw_atomic_add_acq_uint64_t
#define	lw_atomic_add_rel_u64		lw_atomic_add_rel_uint64_t
#define	lw_atomic_subtract_u64		lw_atomic_subtract_uint64_t
#define	lw_atomic_subtract_acq_u64	lw_atomic_subtract_acq_uint64_t
#define	lw_atomic_subtract_rel_u64	lw_atomic_subtract_rel_uint64_t
#define	lw_atomic_load_acq_u64		lw_atomic_load_acq_uint64_t
#define	lw_atomic_store_rel_u64		lw_atomic_store_rel_uint64_t
#define	lw_atomic_cmpset_u64		lw_atomic_cmpset_uint64_t
#define	lw_atomic_cmpset_acq_u64	lw_atomic_cmpset_acq_uint64_t
#define	lw_atomic_cmpset_rel_u64	lw_atomic_cmpset_rel_uint64_t
#define	lw_atomic_swap_u64		lw_atomic_swap_uint64_t
#define	lw_atomic_readandclear_u64	lw_atomic_readandclear_uint64_t
#define	lw_atomic_testandset_u64	lw_atomic_testandset_uint64_t

#endif /* __LWBASE_ATOMIC_POSIX_H__ */
