/arch/x86/include/asm/cmpxchg_32.h

  1#ifndef _ASM_X86_CMPXCHG_32_H
  2#define _ASM_X86_CMPXCHG_32_H
  3
  4#include <linux/bitops.h> /* for LOCK_PREFIX */
  5
  6/*
  7 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
  8 *       you need to test for the feature in boot_cpu_data.
  9 */
 10
 11extern void __xchg_wrong_size(void);
 12
 13/*
 14 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
 15 * Note 2: xchg has side effect, so that attribute volatile is necessary,
 16 *	  but generally the primitive is invalid, *ptr is output argument. --ANK
 17 */
 18
 19struct __xchg_dummy {
 20	unsigned long a[100];
 21};
 22#define __xg(x) ((struct __xchg_dummy *)(x))
 23
 24#define __xchg(x, ptr, size)						\
 25({									\
 26	__typeof(*(ptr)) __x = (x);					\
 27	switch (size) {							\
 28	case 1:								\
 29		asm volatile("xchgb %b0,%1"				\
 30			     : "=q" (__x), "+m" (*__xg(ptr))		\
 31			     : "0" (__x)				\
 32			     : "memory");				\
 33		break;							\
 34	case 2:								\
 35		asm volatile("xchgw %w0,%1"				\
 36			     : "=r" (__x), "+m" (*__xg(ptr))		\
 37			     : "0" (__x)				\
 38			     : "memory");				\
 39		break;							\
 40	case 4:								\
 41		asm volatile("xchgl %0,%1"				\
 42			     : "=r" (__x), "+m" (*__xg(ptr))		\
 43			     : "0" (__x)				\
 44			     : "memory");				\
 45		break;							\
 46	default:							\
 47		__xchg_wrong_size();					\
 48	}								\
 49	__x;								\
 50})
 51
 52#define xchg(ptr, v)							\
 53	__xchg((v), (ptr), sizeof(*ptr))
 54
 55/*
 56 * CMPXCHG8B only writes to the target if we had the previous
 57 * value in registers, otherwise it acts as a read and gives us the
 58 * "new previous" value.  That is why there is a loop.  Preloading
 59 * EDX:EAX is a performance optimization: in the common case it means
 60 * we need only one locked operation.
 61 *
 62 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
 63 * least an FPU save and/or %cr0.ts manipulation.
 64 *
 65 * cmpxchg8b must be used with the lock prefix here to allow the
 66 * instruction to be executed atomically.  We need to have the reader
 67 * side to see the coherent 64bit value.
 68 */
 69static inline void set_64bit(volatile u64 *ptr, u64 value)
 70{
 71	u32 low  = value;
 72	u32 high = value >> 32;
 73	u64 prev = *ptr;
 74
 75	asm volatile("\n1:\t"
 76		     LOCK_PREFIX "cmpxchg8b %0\n\t"
 77		     "jnz 1b"
 78		     : "=m" (*ptr), "+A" (prev)
 79		     : "b" (low), "c" (high)
 80		     : "memory");
 81}
 82
 83extern void __cmpxchg_wrong_size(void);
 84
 85/*
 86 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 87 * store NEW in MEM.  Return the initial value in MEM.  Success is
 88 * indicated by comparing RETURN with OLD.
 89 */
 90#define __raw_cmpxchg(ptr, old, new, size, lock)			\
 91({									\
 92	__typeof__(*(ptr)) __ret;					\
 93	__typeof__(*(ptr)) __old = (old);				\
 94	__typeof__(*(ptr)) __new = (new);				\
 95	switch (size) {							\
 96	case 1:								\
 97		asm volatile(lock "cmpxchgb %b2,%1"			\
 98			     : "=a" (__ret), "+m" (*__xg(ptr))		\
 99			     : "q" (__new), "0" (__old)			\
100			     : "memory");				\
101		break;							\
102	case 2:								\
103		asm volatile(lock "cmpxchgw %w2,%1"			\
104			     : "=a" (__ret), "+m" (*__xg(ptr))		\
105			     : "r" (__new), "0" (__old)			\
106			     : "memory");				\
107		break;							\
108	case 4:								\
109		asm volatile(lock "cmpxchgl %2,%1"			\
110			     : "=a" (__ret), "+m" (*__xg(ptr))		\
111			     : "r" (__new), "0" (__old)			\
112			     : "memory");				\
113		break;							\
114	default:							\
115		__cmpxchg_wrong_size();					\
116	}								\
117	__ret;								\
118})
119
120#define __cmpxchg(ptr, old, new, size)					\
121	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
122
123#define __sync_cmpxchg(ptr, old, new, size)				\
124	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
125
126#define __cmpxchg_local(ptr, old, new, size)				\
127	__raw_cmpxchg((ptr), (old), (new), (size), "")
128
129#ifdef CONFIG_X86_CMPXCHG
130#define __HAVE_ARCH_CMPXCHG 1
131
132#define cmpxchg(ptr, old, new)						\
133	__cmpxchg((ptr), (old), (new), sizeof(*ptr))
134
135#define sync_cmpxchg(ptr, old, new)					\
136	__sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
137
138#define cmpxchg_local(ptr, old, new)					\
139	__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
140#endif
141
142#ifdef CONFIG_X86_CMPXCHG64
143#define cmpxchg64(ptr, o, n)						\
144	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
145					 (unsigned long long)(n)))
146#define cmpxchg64_local(ptr, o, n)					\
147	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
148					       (unsigned long long)(n)))
149#endif
150
151static inline unsigned long long __cmpxchg64(volatile void *ptr,
152					     unsigned long long old,
153					     unsigned long long new)
154{
155	unsigned long long prev;
156	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
157		     : "=A" (prev),
158		       "+m" (*__xg(ptr))
159		     : "b" ((unsigned long)new),
160		       "c" ((unsigned long)(new >> 32)),
161		       "0" (old)
162		     : "memory");
163	return prev;
164}
165
166static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
167						   unsigned long long old,
168						   unsigned long long new)
169{
170	unsigned long long prev;
171	asm volatile("cmpxchg8b %1"
172		     : "=A" (prev),
173		       "+m" (*__xg(ptr))
174		     : "b" ((unsigned long)new),
175		       "c" ((unsigned long)(new >> 32)),
176		       "0" (old)
177		     : "memory");
178	return prev;
179}
180
181#ifndef CONFIG_X86_CMPXCHG
182/*
183 * Building a kernel capable running on 80386. It may be necessary to
184 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
185 * a function for each of the sizes we support.
186 */
187
188extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
189extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
190extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
191
192static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
193					unsigned long new, int size)
194{
195	switch (size) {
196	case 1:
197		return cmpxchg_386_u8(ptr, old, new);
198	case 2:
199		return cmpxchg_386_u16(ptr, old, new);
200	case 4:
201		return cmpxchg_386_u32(ptr, old, new);
202	}
203	return old;
204}
205
206#define cmpxchg(ptr, o, n)						\
207({									\
208	__typeof__(*(ptr)) __ret;					\
209	if (likely(boot_cpu_data.x86 > 3))				\
210		__ret = (__typeof__(*(ptr)))__cmpxchg((ptr),		\
211				(unsigned long)(o), (unsigned long)(n),	\
212				sizeof(*(ptr)));			\
213	else								\
214		__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),		\
215				(unsigned long)(o), (unsigned long)(n),	\
216				sizeof(*(ptr)));			\
217	__ret;								\
218})
219#define cmpxchg_local(ptr, o, n)					\
220({									\
221	__typeof__(*(ptr)) __ret;					\
222	if (likely(boot_cpu_data.x86 > 3))				\
223		__ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr),	\
224				(unsigned long)(o), (unsigned long)(n),	\
225				sizeof(*(ptr)));			\
226	else								\
227		__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),		\
228				(unsigned long)(o), (unsigned long)(n),	\
229				sizeof(*(ptr)));			\
230	__ret;								\
231})
232#endif
233
234#ifndef CONFIG_X86_CMPXCHG64
235/*
236 * Building a kernel capable running on 80386 and 80486. It may be necessary
237 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
238 */
239
240extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
241
242#define cmpxchg64(ptr, o, n)					\
243({								\
244	__typeof__(*(ptr)) __ret;				\
245	__typeof__(*(ptr)) __old = (o);				\
246	__typeof__(*(ptr)) __new = (n);				\
247	alternative_io(LOCK_PREFIX_HERE				\
248			"call cmpxchg8b_emu",			\
249			"lock; cmpxchg8b (%%esi)" ,		\
250		       X86_FEATURE_CX8,				\
251		       "=A" (__ret),				\
252		       "S" ((ptr)), "0" (__old),		\
253		       "b" ((unsigned int)__new),		\
254		       "c" ((unsigned int)(__new>>32))		\
255		       : "memory");				\
256	__ret; })
257
258
259
260#define cmpxchg64_local(ptr, o, n)					\
261({									\
262	__typeof__(*(ptr)) __ret;					\
263	if (likely(boot_cpu_data.x86 > 4))				\
264		__ret = (__typeof__(*(ptr)))__cmpxchg64_local((ptr),	\
265				(unsigned long long)(o),		\
266				(unsigned long long)(n));		\
267	else								\
268		__ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr),	\
269				(unsigned long long)(o),		\
270				(unsigned long long)(n));		\
271	__ret;								\
272})
273
274#endif
275
276#endif /* _ASM_X86_CMPXCHG_32_H */