/*-
 * Copyright (c) 1998 Doug Rabson
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/i386/include/atomic.h 254620 2013-08-21 22:30:11Z jkim $
 */
#ifndef __LWBASE_ATOMIC_X86_H__
#define __LWBASE_ATOMIC_X86_H__

#define lw_compiler_membar()	__asm__ __volatile(" " : : : "memory")

#define	lw_mb() \
	__asm__ __volatile__("lock; addl $0,(%%esp)" : : : "memory", "cc")
#define	lw_wmb() \
	__asm__ __volatile__("lock; addl $0,(%%esp)" : : : "memory", "cc")
#define	lw_rmb() \
	__asm__ __volatile__("lock; addl $0,(%%esp)" : : : "memory", "cc")

/*
 * Various simple operations on memory, each of which is atomic in the
 * presence of interrupts and multiple processors.
 *
 * lw_atomic_set_char(P, V)		(*(unsigned char *)(P) |= (V))
 * lw_atomic_clear_char(P, V)		(*(unsigned char *)(P) &= ~(V))
 * lw_atomic_add_char(P, V)		(*(unsigned char *)(P) += (V))
 * lw_atomic_subtract_char(P, V)	(*(unsigned char *)(P) -= (V))
 *
 * lw_atomic_set_short(P, V)		(*(unsigned short *)(P) |= (V))
 * lw_atomic_clear_short(P, V)		(*(unsigned short *)(P) &= ~(V))
 * lw_atomic_add_short(P, V)		(*(unsigned short *)(P) += (V))
 * lw_atomic_subtract_short(P, V)	(*(unsigned short *)(P) -= (V))
 *
 * lw_atomic_set_int(P, V)		(*(unsigned int *)(P) |= (V))
 * lw_atomic_clear_int(P, V)		(*(unsigned int *)(P) &= ~(V))
 * lw_atomic_add_int(P, V)		(*(unsigned int *)(P) += (V))
 * lw_atomic_subtract_int(P, V)		(*(unsigned int *)(P) -= (V))
 * lw_atomic_swap_int(P, V)		(return (*(unsigned int *)(P));
 * 				 	 *(usigned int *)(P) = (V);)
 * lw_atomic_readandclear_int(P)	(return (*(usigned int *)(P));
 *					 *(unsigned int *)(P) = 0;)
 *
 * lw_atomic_set_long(P, V)		(*(unsigned long *)(P) |= (V))
 * lw_atomic_clear_long(P, V)		(*(unsigned long *)(P) &= ~(V))
 * lw_atomic_add_long(P, V)		(*(unsigned long *)(P) += (V))
 * lw_atomic_subtract_long(P, V)	(*(unsigned long *)(P) -= (V))
 * lw_atomic_swap_long(P, V)		(return (*(unsigned long *)(P));
 *					 *(unsigned long *)(P) = (V);)
 * lw_atomic_readandclear_long(P)	(return (*(unsigned long *)(P));
 *					 *(unsigned long *)(P) = 0;)
 */

#define	MPLOCKED	"lock ; "

/*
 * The assembly is volatilized to avoid code chunk removal by the compiler.
 * GCC aggressively reorders operations and memory clobbering is necessary
 * in order to avoid that for memory barriers.
 */
#define	ATOMIC_ASM(NAME, TYPE, OP, CONS, V)				   \
static inline void							   \
lw_atomic_##NAME##_u##TYPE(volatile unsigned TYPE *p, unsigned TYPE v)     \
{									   \
	__asm__ __volatile__(MPLOCKED OP				   \
	: "+m" (*p)							   \
	: CONS (V)							   \
	: "cc");							   \
}									   \
									   \
static inline void							   \
lw_atomic_##NAME##_barr_u##TYPE(volatile unsigned TYPE *p, unsigned TYPE v)\
{									   \
	__asm__ __volatile__(MPLOCKED OP				   \
	: "+m" (*p)							   \
	: CONS (V)							   \
	: "memory", "cc");						   \
}									   \
									   \
static inline void							   \
lw_atomic_##NAME##_##TYPE(volatile TYPE *p, TYPE v)			   \
{									   \
	__asm__ __volatile__(MPLOCKED OP				   \
	: "+m" (*p)							   \
	: CONS (V)							   \
	: "cc");							   \
}									   \
									   \
static inline void							   \
lw_atomic_##NAME##_barr_##TYPE(volatile TYPE *p, TYPE v)		   \
{									   \
	__asm__ __volatile__(MPLOCKED OP				   \
	: "+m" (*p)							   \
	: CONS (V)							   \
	: "memory", "cc");						   \
}

/*
 * Atomic compare and set.
 *
 * if (*dst == expect) *dst = src (all 32 bit words)
 *
 * Returns 0 on failure, non-zero on success
 */

static inline int
lw_atomic_cmpset_uint(volatile unsigned int *dst, unsigned int expect,
    unsigned int src)
{
	unsigned char res;

	__asm__ __volatile__(
	"	" MPLOCKED "		"
	"	cmpxchgl %3,%1 ;	"
	"       sete	%0 ;		"
	"# lw_atomic_cmpset_uint"
	: "=q" (res),			/* 0 */
	  "+m" (*dst),			/* 1 */
	  "+a" (expect)			/* 2 */
	: "r" (src)			/* 3 */
	: "memory", "cc");
	return (res);
}

static inline int
lw_atomic_cmpset_int(volatile int *dst, int expect, int src)
{
	unsigned char res;

	__asm__ __volatile__(
	"	" MPLOCKED "		"
	"	cmpxchgl %3,%1 ;	"
	"       sete	%0 ;		"
	"# lw_atomic_cmpset_int"
	: "=q" (res),			/* 0 */
	  "+m" (*dst),			/* 1 */
	  "+a" (expect)			/* 2 */
	: "r" (src)			/* 3 */
	: "memory", "cc");
	return (res);
}

/*
 * Atomically add the value of v to the integer pointed to by p and return
 * the previous value of *p.
 */
static inline unsigned int
lw_atomic_fetchadd_uint(volatile unsigned int *p, unsigned int v)
{

	__asm__ __volatile__(
	"	" MPLOCKED "		"
	"	xaddl	%0,%1 ;		"
	"# lw_atomic_fetchadd_uint"
	: "+r" (v),			/* 0 */
	  "+m" (*p)			/* 1 */
	: : "cc");
	return (v);
}

static inline int
lw_atomic_fetchadd_int(volatile int *p, int v)
{

	__asm__ __volatile__(
	"	" MPLOCKED "		"
	"	xaddl	%0,%1 ;		"
	"# lw_atomic_fetchadd_int"
	: "+r" (v),			/* 0 */
	  "+m" (*p)			/* 1 */
	: : "cc");
	return (v);
}

static inline int
lw_atomic_testandset_uint(volatile unsigned int *p, unsigned int v)
{
	unsigned char res;

	__asm__ __volatile__(
	"	" MPLOCKED "		"
	"	btsl	%2,%1 ;		"
	"	setc	%0 ;		"
	"# lw_atomic_testandset_uint"
	: "=q" (res),			/* 0 */
	  "+m" (*p)			/* 1 */
	: "Ir" (v & 0x1f)		/* 2 */
	: "cc");
	return (res);
}

static inline int
lw_atomic_testandset_int(volatile int *p, int v)
{
	unsigned char res;

	__asm__ __volatile__(
	"	" MPLOCKED "		"
	"	btsl	%2,%1 ;		"
	"	setc	%0 ;		"
	"# lw_atomic_testandset_int"
	: "=q" (res),			/* 0 */
	  "+m" (*p)			/* 1 */
	: "Ir" (v & 0x1f)		/* 2 */
	: "cc");
	return (res);
}

/*
 * We assume that a = b will do atomic loads and stores.  Due to the
 * IA32 memory model, a simple store guarantees release semantics.
 *
 * However, loads may pass stores, so for lw_atomic_load_acq we have to
 * ensure a Store/Load barrier to do the load in SMP kernels.  We use
 * "lock cmpxchg" as recommended by the AMD Software Optimization
 * Guide, and not mfence.  For UP kernels, however, the cache of the
 * single processor is always consistent, so we only need to take care
 * of the compiler.
 */
#define	ATOMIC_STORE(TYPE)					       \
static inline void						       \
lw_atomic_store_rel_u##TYPE(volatile unsigned TYPE *p, unsigned TYPE v)\
{								       \
	lw_compiler_membar();					       \
	*p = v;							       \
}								       \
								       \
static inline void						       \
lw_atomic_store_rel_##TYPE(volatile TYPE *p, TYPE v)		       \
{								       \
	lw_compiler_membar();					       \
	*p = v;							       \
}

#define	ATOMIC_LOAD(TYPE, LOP)				\
static inline unsigned TYPE				\
lw_atomic_load_acq_u##TYPE(volatile unsigned TYPE *p)	\
{							\
	unsigned TYPE res;				\
							\
	__asm__ __volatile__(MPLOCKED LOP		\
	: "=a" (res),			/* 0 */		\
	  "+m" (*p)			/* 1 */		\
	: : "memory", "cc");				\
	return (res);					\
}							\
							\
static inline TYPE					\
lw_atomic_load_acq_##TYPE(volatile TYPE *p)		\
{							\
	unsigned TYPE res;				\
							\
	__asm__ __volatile__(MPLOCKED LOP		\
	: "=a" (res),			/* 0 */		\
	  "+m" (*p)			/* 1 */		\
	: : "memory", "cc");				\
	return (res);					\
}

ATOMIC_ASM(set,	     char,  "orb %b1,%0",  "iq",  v)
ATOMIC_ASM(clear,    char,  "andb %b1,%0", "iq", ~v)
ATOMIC_ASM(add,	     char,  "addb %b1,%0", "iq",  v)
ATOMIC_ASM(subtract, char,  "subb %b1,%0", "iq",  v)

ATOMIC_ASM(set,	     short, "orw %w1,%0",  "ir",  v)
ATOMIC_ASM(clear,    short, "andw %w1,%0", "ir", ~v)
ATOMIC_ASM(add,	     short, "addw %w1,%0", "ir",  v)
ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir",  v)

ATOMIC_ASM(set,	     int,   "orl %1,%0",   "ir",  v)
ATOMIC_ASM(clear,    int,   "andl %1,%0",  "ir", ~v)
ATOMIC_ASM(add,	     int,   "addl %1,%0",  "ir",  v)
ATOMIC_ASM(subtract, int,   "subl %1,%0",  "ir",  v)

ATOMIC_ASM(set,	     long,  "orl %1,%0",   "ir",  v)
ATOMIC_ASM(clear,    long,  "andl %1,%0",  "ir", ~v)
ATOMIC_ASM(add,	     long,  "addl %1,%0",  "ir",  v)
ATOMIC_ASM(subtract, long,  "subl %1,%0",  "ir",  v)

ATOMIC_LOAD(char,  "cmpxchgb %b0,%1")
ATOMIC_LOAD(short, "cmpxchgw %w0,%1")
ATOMIC_LOAD(int,   "cmpxchgl %0,%1")
ATOMIC_LOAD(long,  "cmpxchgl %0,%1")

ATOMIC_STORE(char)
ATOMIC_STORE(short)
ATOMIC_STORE(int)
ATOMIC_STORE(long)

#undef ATOMIC_ASM
#undef ATOMIC_LOAD
#undef ATOMIC_STORE

static inline int
lw_atomic_cmpset_ulong(volatile unsigned long *dst, unsigned long expect,
    unsigned long src)
{

	return (lw_atomic_cmpset_uint((volatile unsigned int *)dst,
	    (unsigned int)expect, (unsigned int)src));
}

static inline int
lw_atomic_cmpset_long(volatile long *dst, long expect, long src)
{

	return (lw_atomic_cmpset_int((volatile int *)dst, (int)expect,
	    (int)src));
}

static inline unsigned long
lw_atomic_fetchadd_ulong(volatile unsigned long *p, unsigned long v)
{

	return (lw_atomic_fetchadd_uint((volatile unsigned int *)p,
	    (unsigned int)v));
}

static inline long
lw_atomic_fetchadd_long(volatile long *p, long v)
{

	return (lw_atomic_fetchadd_int((volatile int *)p, (int)v));
}

static inline int
lw_atomic_testandset_ulong(volatile unsigned long *p, unsigned int v)
{

	return (lw_atomic_testandset_int((volatile unsigned int *)p, v));
}

static inline int
lw_atomic_testandset_long(volatile long *p, unsigned int v)
{

	return (lw_atomic_testandset_int((volatile int *)p, v));
}

/* Read the current value and store a new value in the destination. */
static inline unsigned int
lw_atomic_swap_uint(volatile unsigned int *p, unsigned int v)
{

	__asm__ __volatile__(
	"	xchgl	%1,%0 ;		"
	"# lw_atomic_swap_int"
	: "+r" (v),			/* 0 */
	  "+m" (*p));			/* 1 */
	return (v);
}

static inline int
lw_atomic_swap_int(volatile int *p, int v)
{

	__asm__ __volatile__(
	"	xchgl	%1,%0 ;		"
	"# lw_atomic_swap_int"
	: "+r" (v),			/* 0 */
	  "+m" (*p));			/* 1 */
	return (v);
}

static inline unsigned long
lw_atomic_swap_ulong(volatile unsigned long *p, unsigned long v)
{

	return (lw_atomic_swap_uint((volatile unsigned int *)p,
	    (unsigned int)v));
}

static inline long
lw_atomic_swap_long(volatile long *p, long v)
{

	return (lw_atomic_swap_int((volatile int *)p, (int)v));
}

#define	lw_atomic_set_acq_char		lw_atomic_set_barr_char
#define	lw_atomic_set_rel_char		lw_atomic_set_barr_char
#define	lw_atomic_clear_acq_char	lw_atomic_clear_barr_char
#define	lw_atomic_clear_rel_char	lw_atomic_clear_barr_char
#define	lw_atomic_add_acq_char		lw_atomic_add_barr_char
#define	lw_atomic_add_rel_char		lw_atomic_add_barr_char
#define	lw_atomic_subtract_acq_char	lw_atomic_subtract_barr_char
#define	lw_atomic_subtract_rel_char	lw_atomic_subtract_barr_char
#define	lw_atomic_set_acq_uchar		lw_atomic_set_barr_uchar
#define	lw_atomic_set_rel_uchar		lw_atomic_set_barr_uchar
#define	lw_atomic_clear_acq_uchar	lw_atomic_clear_barr_uchar
#define	lw_atomic_clear_rel_uchar	lw_atomic_clear_barr_uchar
#define	lw_atomic_add_acq_uchar		lw_atomic_add_barr_uchar
#define	lw_atomic_add_rel_uchar		lw_atomic_add_barr_uchar
#define	lw_atomic_subtract_acq_uchar	lw_atomic_subtract_barr_uchar
#define	lw_atomic_subtract_rel_uchar	lw_atomic_subtract_barr_uchar

#define	lw_atomic_set_acq_short		lw_atomic_set_barr_short
#define	lw_atomic_set_rel_short		lw_atomic_set_barr_short
#define	lw_atomic_clear_acq_short	lw_atomic_clear_barr_short
#define	lw_atomic_clear_rel_short	lw_atomic_clear_barr_short
#define	lw_atomic_add_acq_short		lw_atomic_add_barr_short
#define	lw_atomic_add_rel_short		lw_atomic_add_barr_short
#define	lw_atomic_subtract_acq_short	lw_atomic_subtract_barr_short
#define	lw_atomic_subtract_rel_short	lw_atomic_subtract_barr_short
#define	lw_atomic_set_acq_ushort	lw_atomic_set_barr_ushort
#define	lw_atomic_set_rel_ushort	lw_atomic_set_barr_ushort
#define	lw_atomic_clear_acq_ushort	lw_atomic_clear_barr_ushort
#define	lw_atomic_clear_rel_ushort	lw_atomic_clear_barr_ushort
#define	lw_atomic_add_acq_ushort	lw_atomic_add_barr_ushort
#define	lw_atomic_add_rel_ushort	lw_atomic_add_barr_ushort
#define	lw_atomic_subtract_acq_ushort	lw_atomic_subtract_barr_ushort
#define	lw_atomic_subtract_rel_ushort	lw_atomic_subtract_barr_ushort

#define	lw_atomic_set_acq_int		lw_atomic_set_barr_int
#define	lw_atomic_set_rel_int		lw_atomic_set_barr_int
#define	lw_atomic_clear_acq_int		lw_atomic_clear_barr_int
#define	lw_atomic_clear_rel_int		lw_atomic_clear_barr_int
#define	lw_atomic_add_acq_int		lw_atomic_add_barr_int
#define	lw_atomic_add_rel_int		lw_atomic_add_barr_int
#define	lw_atomic_subtract_acq_int	lw_atomic_subtract_barr_int
#define	lw_atomic_subtract_rel_int	lw_atomic_subtract_barr_int
#define	lw_atomic_cmpset_acq_int	lw_atomic_cmpset_int
#define	lw_atomic_cmpset_rel_int	lw_atomic_cmpset_int
#define	lw_atomic_set_acq_uint		lw_atomic_set_barr_uint
#define	lw_atomic_set_rel_uint		lw_atomic_set_barr_uint
#define	lw_atomic_clear_acq_uint	lw_atomic_clear_barr_uint
#define	lw_atomic_clear_rel_uint	lw_atomic_clear_barr_uint
#define	lw_atomic_add_acq_uint		lw_atomic_add_barr_uint
#define	lw_atomic_add_rel_uint		lw_atomic_add_barr_uint
#define	lw_atomic_subtract_acq_uint	lw_atomic_subtract_barr_uint
#define	lw_atomic_subtract_rel_uint	lw_atomic_subtract_barr_uint
#define	lw_atomic_cmpset_acq_uint	lw_atomic_cmpset_uint
#define	lw_atomic_cmpset_rel_uint	lw_atomic_cmpset_uint

#define	lw_atomic_set_acq_long		lw_atomic_set_barr_long
#define	lw_atomic_set_rel_long		lw_atomic_set_barr_long
#define	lw_atomic_clear_acq_long	lw_atomic_clear_barr_long
#define	lw_atomic_clear_rel_long	lw_atomic_clear_barr_long
#define	lw_atomic_add_acq_long		lw_atomic_add_barr_long
#define	lw_atomic_add_rel_long		lw_atomic_add_barr_long
#define	lw_atomic_subtract_acq_long	lw_atomic_subtract_barr_long
#define	lw_atomic_subtract_rel_long	lw_atomic_subtract_barr_long
#define	lw_atomic_cmpset_acq_long	lw_atomic_cmpset_long
#define	lw_atomic_cmpset_rel_long	lw_atomic_cmpset_long
#define	lw_atomic_set_acq_ulong		lw_atomic_set_barr_ulong
#define	lw_atomic_set_rel_ulong		lw_atomic_set_barr_ulong
#define	lw_atomic_clear_acq_ulong	lw_atomic_clear_barr_ulong
#define	lw_atomic_clear_rel_ulong	lw_atomic_clear_barr_ulong
#define	lw_atomic_add_acq_ulong		lw_atomic_add_barr_ulong
#define	lw_atomic_add_rel_ulong		lw_atomic_add_barr_ulong
#define	lw_atomic_subtract_acq_ulong	lw_atomic_subtract_barr_ulong
#define	lw_atomic_subtract_rel_ulong	lw_atomic_subtract_barr_ulong
#define	lw_atomic_cmpset_acq_ulong	lw_atomic_cmpset_ulong
#define	lw_atomic_cmpset_rel_ulong	lw_atomic_cmpset_ulong

#define	lw_atomic_readandclear_int(p)	lw_atomic_swap_int(p, 0)
#define	lw_atomic_readandclear_uint(p)	lw_atomic_swap_uint(p, 0)
#define	lw_atomic_readandclear_long(p)	lw_atomic_swap_long(p, 0)
#define	lw_atomic_readandclear_ulong(p)	lw_atomic_swap_ulong(p, 0)

/* Operations on 8-bit bytes. */
#define	lw_atomic_set_8			lw_atomic_set_char
#define	lw_atomic_set_acq_8		lw_atomic_set_acq_char
#define	lw_atomic_set_rel_8		lw_atomic_set_rel_char
#define	lw_atomic_clear_8		lw_atomic_clear_char
#define	lw_atomic_clear_acq_8		lw_atomic_clear_acq_char
#define	lw_atomic_clear_rel_8		lw_atomic_clear_rel_char
#define	lw_atomic_add_8			lw_atomic_add_char
#define	lw_atomic_add_acq_8		lw_atomic_add_acq_char
#define	lw_atomic_add_rel_8		lw_atomic_add_rel_char
#define	lw_atomic_subtract_8		lw_atomic_subtract_char
#define	lw_atomic_subtract_acq_8	lw_atomic_subtract_acq_char
#define	lw_atomic_subtract_rel_8	lw_atomic_subtract_rel_char
#define	lw_atomic_load_acq_8		lw_atomic_load_acq_char
#define	lw_atomic_store_rel_8		lw_atomic_store_rel_char
#define	lw_atomic_set_u8		lw_atomic_set_uchar
#define	lw_atomic_set_acq_u8		lw_atomic_set_acq_uchar
#define	lw_atomic_set_rel_u8		lw_atomic_set_rel_uchar
#define	lw_atomic_clear_u8		lw_atomic_clear_uchar
#define	lw_atomic_clear_acq_u8		lw_atomic_clear_acq_uchar
#define	lw_atomic_clear_rel_u8		lw_atomic_clear_rel_uchar
#define	lw_atomic_add_u8		lw_atomic_add_uchar
#define	lw_atomic_add_acq_u8		lw_atomic_add_acq_uchar
#define	lw_atomic_add_rel_u8		lw_atomic_add_rel_uchar
#define	lw_atomic_subtract_u8		lw_atomic_subtract_uchar
#define	lw_atomic_subtract_acq_u8	lw_atomic_subtract_acq_uchar
#define	lw_atomic_subtract_rel_u8	lw_atomic_subtract_rel_uchar
#define	lw_atomic_load_acq_u8		lw_atomic_load_acq_uchar
#define	lw_atomic_store_rel_u8		lw_atomic_store_rel_uchar
#define	lw_atomic_set_int8_t		lw_atomic_set_char
#define	lw_atomic_set_acq_int8_t	lw_atomic_set_acq_char
#define	lw_atomic_set_rel_int8_t	lw_atomic_set_rel_char
#define	lw_atomic_clear_int8_t		lw_atomic_clear_char
#define	lw_atomic_clear_acq_int8_t	lw_atomic_clear_acq_char
#define	lw_atomic_clear_rel_int8_t	lw_atomic_clear_rel_char
#define	lw_atomic_add_int8_t		lw_atomic_add_char
#define	lw_atomic_add_acq_int8_t	lw_atomic_add_acq_char
#define	lw_atomic_add_rel_int8_t	lw_atomic_add_rel_char
#define	lw_atomic_subtract_int8_t	lw_atomic_subtract_char
#define	lw_atomic_subtract_acq_int8_t	lw_atomic_subtract_acq_char
#define	lw_atomic_subtract_rel_int8_t	lw_atomic_subtract_rel_char
#define	lw_atomic_load_acq_int8_t	lw_atomic_load_acq_char
#define	lw_atomic_store_rel_int8_t	lw_atomic_store_rel_char
#define	lw_atomic_set_uint8_t		lw_atomic_set_uchar
#define	lw_atomic_set_acq_uint8_t	lw_atomic_set_acq_uchar
#define	lw_atomic_set_rel_uint8_t	lw_atomic_set_rel_uchar
#define	lw_atomic_clear_uint8_t		lw_atomic_clear_uchar
#define	lw_atomic_clear_acq_uint8_t	lw_atomic_clear_acq_uchar
#define	lw_atomic_clear_rel_uint8_t	lw_atomic_clear_rel_uchar
#define	lw_atomic_add_uint8_t		lw_atomic_add_uchar
#define	lw_atomic_add_acq_uint8_t	lw_atomic_add_acq_uchar
#define	lw_atomic_add_rel_uint8_t	lw_atomic_add_rel_uchar
#define	lw_atomic_subtract_uint8_t	lw_atomic_subtract_uchar
#define	lw_atomic_subtract_acq_uint8_t	lw_atomic_subtract_acq_uchar
#define	lw_atomic_subtract_rel_uint8_t	lw_atomic_subtract_rel_uchar
#define	lw_atomic_load_acq_uint8_t	lw_atomic_load_acq_uchar
#define	lw_atomic_store_rel_uint8_t	lw_atomic_store_rel_uchar

/* Operations on 16-bit words. */
#define	lw_atomic_set_16		lw_atomic_set_short
#define	lw_atomic_set_acq_16		lw_atomic_set_acq_short
#define	lw_atomic_set_rel_16		lw_atomic_set_rel_short
#define	lw_atomic_clear_16		lw_atomic_clear_short
#define	lw_atomic_clear_acq_16		lw_atomic_clear_acq_short
#define	lw_atomic_clear_rel_16		lw_atomic_clear_rel_short
#define	lw_atomic_add_16		lw_atomic_add_short
#define	lw_atomic_add_acq_16		lw_atomic_add_acq_short
#define	lw_atomic_add_rel_16		lw_atomic_add_rel_short
#define	lw_atomic_subtract_16		lw_atomic_subtract_short
#define	lw_atomic_subtract_acq_16	lw_atomic_subtract_acq_short
#define	lw_atomic_subtract_rel_16	lw_atomic_subtract_rel_short
#define	lw_atomic_load_acq_16		lw_atomic_load_acq_short
#define	lw_atomic_store_rel_16		lw_atomic_store_rel_short
#define	lw_atomic_set_u16		lw_atomic_set_ushort
#define	lw_atomic_set_acq_u16		lw_atomic_set_acq_ushort
#define	lw_atomic_set_rel_u16		lw_atomic_set_rel_ushort
#define	lw_atomic_clear_u16		lw_atomic_clear_ushort
#define	lw_atomic_clear_acq_u16		lw_atomic_clear_acq_ushort
#define	lw_atomic_clear_rel_u16		lw_atomic_clear_rel_ushort
#define	lw_atomic_add_u16		lw_atomic_add_ushort
#define	lw_atomic_add_acq_u16		lw_atomic_add_acq_ushort
#define	lw_atomic_add_rel_u16		lw_atomic_add_rel_ushort
#define	lw_atomic_subtract_u16		lw_atomic_subtract_ushort
#define	lw_atomic_subtract_acq_u16	lw_atomic_subtract_acq_ushort
#define	lw_atomic_subtract_rel_u16	lw_atomic_subtract_rel_ushort
#define	lw_atomic_load_acq_u16		lw_atomic_load_acq_ushort
#define	lw_atomic_store_rel_u16		lw_atomic_store_rel_ushort
#define	lw_atomic_set_int16_t		lw_atomic_set_short
#define	lw_atomic_set_acq_int16_t	lw_atomic_set_acq_short
#define	lw_atomic_set_rel_int16_t	lw_atomic_set_rel_short
#define	lw_atomic_clear_int16_t		lw_atomic_clear_short
#define	lw_atomic_clear_acq_int16_t	lw_atomic_clear_acq_short
#define	lw_atomic_clear_rel_int16_t	lw_atomic_clear_rel_short
#define	lw_atomic_add_int16_t		lw_atomic_add_short
#define	lw_atomic_add_acq_int16_t	lw_atomic_add_acq_short
#define	lw_atomic_add_rel_int16_t	lw_atomic_add_rel_short
#define	lw_atomic_subtract_int16_t	lw_atomic_subtract_short
#define	lw_atomic_subtract_acq_int16_t	lw_atomic_subtract_acq_short
#define	lw_atomic_subtract_rel_int16_t	lw_atomic_subtract_rel_short
#define	lw_atomic_load_acq_int16_t	lw_atomic_load_acq_short
#define	lw_atomic_store_rel_int16_t	lw_atomic_store_rel_short
#define	lw_atomic_set_uint16_t		lw_atomic_set_ushort
#define	lw_atomic_set_acq_uint16_t	lw_atomic_set_acq_ushort
#define	lw_atomic_set_rel_uint16_t	lw_atomic_set_rel_ushort
#define	lw_atomic_clear_uint16_t	lw_atomic_clear_ushort
#define	lw_atomic_clear_acq_uint16_t	lw_atomic_clear_acq_ushort
#define	lw_atomic_clear_rel_uint16_t	lw_atomic_clear_rel_ushort
#define	lw_atomic_add_uint16_t		lw_atomic_add_ushort
#define	lw_atomic_add_acq_uint16_t	lw_atomic_add_acq_ushort
#define	lw_atomic_add_rel_uint16_t	lw_atomic_add_rel_ushort
#define	lw_atomic_subtract_uint16_t	lw_atomic_subtract_ushort
#define	lw_atomic_subtract_acq_uint16_t	lw_atomic_subtract_acq_ushort
#define	lw_atomic_subtract_rel_uint16_t	lw_atomic_subtract_rel_ushort
#define	lw_atomic_load_acq_uint16_t	lw_atomic_load_acq_ushort
#define	lw_atomic_store_rel_uint16_t	lw_atomic_store_rel_ushort

/* Operations on 32-bit double words. */
#define	lw_atomic_set_32		lw_atomic_set_int
#define	lw_atomic_set_acq_32		lw_atomic_set_acq_int
#define	lw_atomic_set_rel_32		lw_atomic_set_rel_int
#define	lw_atomic_clear_32		lw_atomic_clear_int
#define	lw_atomic_clear_acq_32		lw_atomic_clear_acq_int
#define	lw_atomic_clear_rel_32		lw_atomic_clear_rel_int
#define	lw_atomic_add_32		lw_atomic_add_int
#define	lw_atomic_add_acq_32		lw_atomic_add_acq_int
#define	lw_atomic_add_rel_32		lw_atomic_add_rel_int
#define	lw_atomic_subtract_32		lw_atomic_subtract_int
#define	lw_atomic_subtract_acq_32	lw_atomic_subtract_acq_int
#define	lw_atomic_subtract_rel_32	lw_atomic_subtract_rel_int
#define	lw_atomic_load_acq_32		lw_atomic_load_acq_int
#define	lw_atomic_store_rel_32		lw_atomic_store_rel_int
#define	lw_atomic_cmpset_32		lw_atomic_cmpset_int
#define	lw_atomic_cmpset_acq_32		lw_atomic_cmpset_acq_int
#define	lw_atomic_cmpset_rel_32		lw_atomic_cmpset_rel_int
#define	lw_atomic_swap_32		lw_atomic_swap_int
#define	lw_atomic_readandclear_32	lw_atomic_readandclear_int
#define	lw_atomic_fetchadd_32		lw_atomic_fetchadd_int
#define	lw_atomic_testandset_32		lw_atomic_testandset_int
#define	lw_atomic_set_u32		lw_atomic_set_uint
#define	lw_atomic_set_acq_u32		lw_atomic_set_acq_uint
#define	lw_atomic_set_rel_u32		lw_atomic_set_rel_uint
#define	lw_atomic_clear_u32		lw_atomic_clear_uint
#define	lw_atomic_clear_acq_u32		lw_atomic_clear_acq_uint
#define	lw_atomic_clear_rel_u32		lw_atomic_clear_rel_uint
#define	lw_atomic_add_u32		lw_atomic_add_uint
#define	lw_atomic_add_acq_u32		lw_atomic_add_acq_uint
#define	lw_atomic_add_rel_u32		lw_atomic_add_rel_uint
#define	lw_atomic_subtract_u32		lw_atomic_subtract_uint
#define	lw_atomic_subtract_acq_u32	lw_atomic_subtract_acq_uint
#define	lw_atomic_subtract_rel_u32	lw_atomic_subtract_rel_uint
#define	lw_atomic_load_acq_u32		lw_atomic_load_acq_uint
#define	lw_atomic_store_rel_u32		lw_atomic_store_rel_uint
#define	lw_atomic_cmpset_u32		lw_atomic_cmpset_uint
#define	lw_atomic_cmpset_acq_u32	lw_atomic_cmpset_acq_uint
#define	lw_atomic_cmpset_rel_u32	lw_atomic_cmpset_rel_uint
#define	lw_atomic_swap_u32		lw_atomic_swap_uint
#define	lw_atomic_readandclear_u32	lw_atomic_readandclear_uint
#define	lw_atomic_fetchadd_u32		lw_atomic_fetchadd_uint
#define	lw_atomic_testandset_u32	lw_atomic_testandset_uint
#define	lw_atomic_set_int32_t		lw_atomic_set_int
#define	lw_atomic_set_acq_int32_t	lw_atomic_set_acq_int
#define	lw_atomic_set_rel_int32_t	lw_atomic_set_rel_int
#define	lw_atomic_clear_int32_t		lw_atomic_clear_int
#define	lw_atomic_clear_acq_int32_t	lw_atomic_clear_acq_int
#define	lw_atomic_clear_rel_int32_t	lw_atomic_clear_rel_int
#define	lw_atomic_add_int32_t		lw_atomic_add_int
#define	lw_atomic_add_acq_int32_t	lw_atomic_add_acq_int
#define	lw_atomic_add_rel_int32_t	lw_atomic_add_rel_int
#define	lw_atomic_subtract_int32_t	lw_atomic_subtract_int
#define	lw_atomic_subtract_acq_int32_t	lw_atomic_subtract_acq_int
#define	lw_atomic_subtract_rel_int32_t	lw_atomic_subtract_rel_int
#define	lw_atomic_load_acq_int32_t	lw_atomic_load_acq_int
#define	lw_atomic_store_rel_int32_t	lw_atomic_store_rel_int
#define	lw_atomic_cmpset_int32_t	lw_atomic_cmpset_int
#define	lw_atomic_cmpset_acq_int32_t	lw_atomic_cmpset_acq_int
#define	lw_atomic_cmpset_rel_int32_t	lw_atomic_cmpset_rel_int
#define	lw_atomic_swap_int32_t		lw_atomic_swap_int
#define	lw_atomic_readandclear_int32_t	lw_atomic_readandclear_int
#define	lw_atomic_fetchadd_int32_t	lw_atomic_fetchadd_int
#define	lw_atomic_testandset_int32_t	lw_atomic_testandset_int
#define	lw_atomic_set_uint32_t		lw_atomic_set_uint
#define	lw_atomic_set_acq_uint32_t	lw_atomic_set_acq_uint
#define	lw_atomic_set_rel_uint32_t	lw_atomic_set_rel_uint
#define	lw_atomic_clear_uint32_t	lw_atomic_clear_uint
#define	lw_atomic_clear_acq_uint32_t	lw_atomic_clear_acq_uint
#define	lw_atomic_clear_rel_uint32_t	lw_atomic_clear_rel_uint
#define	lw_atomic_add_uint32_t		lw_atomic_add_uint
#define	lw_atomic_add_acq_uint32_t	lw_atomic_add_acq_uint
#define	lw_atomic_add_rel_uint32_t	lw_atomic_add_rel_uint
#define	lw_atomic_subtract_uint32_t	lw_atomic_subtract_uint
#define	lw_atomic_subtract_acq_uint32_t	lw_atomic_subtract_acq_uint
#define	lw_atomic_subtract_rel_uint32_t	lw_atomic_subtract_rel_uint
#define	lw_atomic_load_acq_uint32_t	lw_atomic_load_acq_uint
#define	lw_atomic_store_rel_uint32_t	lw_atomic_store_rel_uint
#define	lw_atomic_cmpset_uint32_t	lw_atomic_cmpset_uint
#define	lw_atomic_cmpset_acq_uint32_t	lw_atomic_cmpset_acq_uint
#define	lw_atomic_cmpset_rel_uint32_t	lw_atomic_cmpset_rel_uint
#define	lw_atomic_swap_uint32_t		lw_atomic_swap_uint
#define	lw_atomic_readandclear_uint32_t	lw_atomic_readandclear_uint
#define	lw_atomic_fetchadd_uint32_t	lw_atomic_fetchadd_uint
#define	lw_atomic_testandset_uint32_t	lw_atomic_testandset_uint

/* Operations on intptr_t. */
#define	lw_atomic_set_acq_intptr_t		lw_atomic_set_acq_long
#define	lw_atomic_set_rel_intptr_t		lw_atomic_set_rel_long
#define	lw_atomic_clear_intptr_t		lw_atomic_clear_long
#define	lw_atomic_clear_acq_intptr_t		lw_atomic_clear_acq_long
#define	lw_atomic_clear_rel_intptr_t		lw_atomic_clear_rel_long
#define	lw_atomic_add_intptr_t			lw_atomic_add_long
#define	lw_atomic_add_acq_intptr_t		lw_atomic_add_acq_long
#define	lw_atomic_add_rel_intptr_t		lw_atomic_add_rel_long
#define	lw_atomic_subtract_intptr_t		lw_atomic_subtract_long
#define	lw_atomic_subtract_acq_intptr_t		lw_atomic_subtract_acq_long
#define	lw_atomic_subtract_rel_intptr_t		lw_atomic_subtract_rel_long
#define	lw_atomic_load_acq_intptr_t		lw_atomic_load_acq_long
#define	lw_atomic_store_rel_intptr_t		lw_atomic_store_rel_long
#define	lw_atomic_cmpset_intptr_t		lw_atomic_cmpset_long
#define	lw_atomic_cmpset_acq_intptr_t		lw_atomic_cmpset_acq_long
#define	lw_atomic_cmpset_rel_intptr_t		lw_atomic_cmpset_rel_long
#define	lw_atomic_swap_intptr_t			lw_atomic_swap_long
#define	lw_atomic_readandclear_intptr_t		lw_atomic_readandclear_long
#define	lw_atomic_testandset_intptr_t		lw_atomic_testandset_long
#define	lw_atomic_set_uintptr_t			lw_atomic_set_ulong
#define	lw_atomic_set_acq_uintptr_t		lw_atomic_set_acq_ulong
#define	lw_atomic_set_rel_uintptr_t		lw_atomic_set_rel_ulong
#define	lw_atomic_clear_uintptr_t		lw_atomic_clear_ulong
#define	lw_atomic_clear_acq_uintptr_t		lw_atomic_clear_acq_ulong
#define	lw_atomic_clear_rel_uintptr_t		lw_atomic_clear_rel_ulong
#define	lw_atomic_add_uintptr_t			lw_atomic_add_ulong
#define	lw_atomic_add_acq_uintptr_t		lw_atomic_add_acq_ulong
#define	lw_atomic_add_rel_uintptr_t		lw_atomic_add_rel_ulong
#define	lw_atomic_subtract_uintptr_t		lw_atomic_subtract_ulong
#define	lw_atomic_subtract_acq_uintptr_t	lw_atomic_subtract_acq_ulong
#define	lw_atomic_subtract_rel_uintptr_t	lw_atomic_subtract_rel_ulong
#define	lw_atomic_load_acq_uintptr_t		lw_atomic_load_acq_ulong
#define	lw_atomic_store_rel_uintptr_t		lw_atomic_store_rel_ulong
#define	lw_atomic_cmpset_uintptr_t		lw_atomic_cmpset_ulong
#define	lw_atomic_cmpset_acq_uintptr_t		lw_atomic_cmpset_acq_ulong
#define	lw_atomic_cmpset_rel_uintptr_t		lw_atomic_cmpset_rel_ulong
#define	lw_atomic_swap_uintptr_t		lw_atomic_swap_ulong
#define	lw_atomic_readandclear_uintptr_t	lw_atomic_readandclear_ulong
#define	lw_atomic_testandset_uintptr_t		lw_atomic_testandset_ulong
#endif /* __LWBASE_ATOMIC_X86_H__ */
