/arch/mips/include/asm/spinlock.h

https://github.com/kernel-digger/linux · C Header · 428 lines · 342 code · 42 blank · 44 comment · 19 complexity · bb8af404d86432230b7898cf69d9282f MD5 · raw file 

    

  1. /*
    2. 

  2.  * This file is subject to the terms and conditions of the GNU General Public
    3. 

  3.  * License.  See the file "COPYING" in the main directory of this archive
    4. 

  4.  * for more details.
    5. 

  5.  *
    6. 

  6.  * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
    7. 

  7.  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
    8. 

  8.  */
    9. 

  9. #ifndef _ASM_SPINLOCK_H
    10. 

  10. #define _ASM_SPINLOCK_H
    11. 

  11. 

  12. #include <linux/compiler.h>
    13. 

  13. 

  14. #include <asm/barrier.h>
    15. 

  15. #include <asm/compiler.h>
    16. 

  16. #include <asm/war.h>
    17. 

  17. 

  18. /*
    19. 

  19.  * Your basic SMP spinlocks, allowing only a single CPU anywhere
    20. 

  20.  *
    21. 

  21.  * Simple spin lock operations.	 There are two variants, one clears IRQ's
    22. 

  22.  * on the local processor, one does not.
    23. 

  23.  *
    24. 

  24.  * These are fair FIFO ticket locks
    25. 

  25.  *
    26. 

  26.  * (the type definitions are in asm/spinlock_types.h)
    27. 

  27.  */
    28. 

  28. 

  29. 

  30. /*
    31. 

  31.  * Ticket locks are conceptually two parts, one indicating the current head of
    32. 

  32.  * the queue, and the other indicating the current tail. The lock is acquired
    33. 

  33.  * by atomically noting the tail and incrementing it by one (thus adding
    34. 

  34.  * ourself to the queue and noting our position), then waiting until the head
    35. 

  35.  * becomes equal to the the initial value of the tail.
    36. 

  36.  */
    37. 

  37. 

  38. static inline int arch_spin_is_locked(arch_spinlock_t *lock)
    39. 

  39. {
    40. 

  40. 	u32 counters = ACCESS_ONCE(lock->lock);
    41. 

  41. 

  42. 	return ((counters >> 16) ^ counters) & 0xffff;
    43. 

  43. }
    44. 

  44. 

  45. #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
    46. 

  46. #define arch_spin_unlock_wait(x) \
    47. 

  47. 	while (arch_spin_is_locked(x)) { cpu_relax(); }
    48. 

  48. 

  49. static inline int arch_spin_is_contended(arch_spinlock_t *lock)
    50. 

  50. {
    51. 

  51. 	u32 counters = ACCESS_ONCE(lock->lock);
    52. 

  52. 

  53. 	return (((counters >> 16) - counters) & 0xffff) > 1;
    54. 

  54. }
    55. 

  55. #define arch_spin_is_contended	arch_spin_is_contended
    56. 

  56. 

  57. static inline void arch_spin_lock(arch_spinlock_t *lock)
    58. 

  58. {
    59. 

  59. 	int my_ticket;
    60. 

  60. 	int tmp;
    61. 

  61. 	int inc = 0x10000;
    62. 

  62. 

  63. 	if (R10000_LLSC_WAR) {
    64. 

  64. 		__asm__ __volatile__ (
    65. 

  65. 		"	.set push		# arch_spin_lock	\n"
    66. 

  66. 		"	.set noreorder					\n"
    67. 

  67. 		"							\n"
    68. 

  68. 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
    69. 

  69. 		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
    70. 

  70. 		"	sc	%[my_ticket], %[ticket_ptr]		\n"
    71. 

  71. 		"	beqzl	%[my_ticket], 1b			\n"
    72. 

  72. 		"	 nop						\n"
    73. 

  73. 		"	srl	%[my_ticket], %[ticket], 16		\n"
    74. 

  74. 		"	andi	%[ticket], %[ticket], 0xffff		\n"
    75. 

  75. 		"	bne	%[ticket], %[my_ticket], 4f		\n"
    76. 

  76. 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
    77. 

  77. 		"2:							\n"
    78. 

  78. 		"	.subsection 2					\n"
    79. 

  79. 		"4:	andi	%[ticket], %[ticket], 0xffff		\n"
    80. 

  80. 		"	sll	%[ticket], 5				\n"
    81. 

  81. 		"							\n"
    82. 

  82. 		"6:	bnez	%[ticket], 6b				\n"
    83. 

  83. 		"	 subu	%[ticket], 1				\n"
    84. 

  84. 		"							\n"
    85. 

  85. 		"	lhu	%[ticket], %[serving_now_ptr]		\n"
    86. 

  86. 		"	beq	%[ticket], %[my_ticket], 2b		\n"
    87. 

  87. 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
    88. 

  88. 		"	b	4b					\n"
    89. 

  89. 		"	 subu	%[ticket], %[ticket], 1			\n"
    90. 

  90. 		"	.previous					\n"
    91. 

  91. 		"	.set pop					\n"
    92. 

  92. 		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
    93. 

  93. 		  [serving_now_ptr] "+m" (lock->h.serving_now),
    94. 

  94. 		  [ticket] "=&r" (tmp),
    95. 

  95. 		  [my_ticket] "=&r" (my_ticket)
    96. 

  96. 		: [inc] "r" (inc));
    97. 

  97. 	} else {
    98. 

  98. 		__asm__ __volatile__ (
    99. 

  99. 		"	.set push		# arch_spin_lock	\n"
    100. 

  100. 		"	.set noreorder					\n"
    101. 

  101. 		"							\n"
    102. 

  102. 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
    103. 

  103. 		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
    104. 

  104. 		"	sc	%[my_ticket], %[ticket_ptr]		\n"
    105. 

  105. 		"	beqz	%[my_ticket], 1b			\n"
    106. 

  106. 		"	 srl	%[my_ticket], %[ticket], 16		\n"
    107. 

  107. 		"	andi	%[ticket], %[ticket], 0xffff		\n"
    108. 

  108. 		"	bne	%[ticket], %[my_ticket], 4f		\n"
    109. 

  109. 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
    110. 

  110. 		"2:							\n"
    111. 

  111. 		"	.subsection 2					\n"
    112. 

  112. 		"4:	andi	%[ticket], %[ticket], 0x1fff		\n"
    113. 

  113. 		"	sll	%[ticket], 5				\n"
    114. 

  114. 		"							\n"
    115. 

  115. 		"6:	bnez	%[ticket], 6b				\n"
    116. 

  116. 		"	 subu	%[ticket], 1				\n"
    117. 

  117. 		"							\n"
    118. 

  118. 		"	lhu	%[ticket], %[serving_now_ptr]		\n"
    119. 

  119. 		"	beq	%[ticket], %[my_ticket], 2b		\n"
    120. 

  120. 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
    121. 

  121. 		"	b	4b					\n"
    122. 

  122. 		"	 subu	%[ticket], %[ticket], 1			\n"
    123. 

  123. 		"	.previous					\n"
    124. 

  124. 		"	.set pop					\n"
    125. 

  125. 		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
    126. 

  126. 		  [serving_now_ptr] "+m" (lock->h.serving_now),
    127. 

  127. 		  [ticket] "=&r" (tmp),
    128. 

  128. 		  [my_ticket] "=&r" (my_ticket)
    129. 

  129. 		: [inc] "r" (inc));
    130. 

  130. 	}
    131. 

  131. 

  132. 	smp_llsc_mb();
    133. 

  133. }
    134. 

  134. 

  135. static inline void arch_spin_unlock(arch_spinlock_t *lock)
    136. 

  136. {
    137. 

  137. 	unsigned int serving_now = lock->h.serving_now + 1;
    138. 

  138. 	wmb();
    139. 

  139. 	lock->h.serving_now = (u16)serving_now;
    140. 

  140. 	nudge_writes();
    141. 

  141. }
    142. 

  142. 

  143. static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
    144. 

  144. {
    145. 

  145. 	int tmp, tmp2, tmp3;
    146. 

  146. 	int inc = 0x10000;
    147. 

  147. 

  148. 	if (R10000_LLSC_WAR) {
    149. 

  149. 		__asm__ __volatile__ (
    150. 

  150. 		"	.set push		# arch_spin_trylock	\n"
    151. 

  151. 		"	.set noreorder					\n"
    152. 

  152. 		"							\n"
    153. 

  153. 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
    154. 

  154. 		"	srl	%[my_ticket], %[ticket], 16		\n"
    155. 

  155. 		"	andi	%[now_serving], %[ticket], 0xffff	\n"
    156. 

  156. 		"	bne	%[my_ticket], %[now_serving], 3f	\n"
    157. 

  157. 		"	 addu	%[ticket], %[ticket], %[inc]		\n"
    158. 

  158. 		"	sc	%[ticket], %[ticket_ptr]		\n"
    159. 

  159. 		"	beqzl	%[ticket], 1b				\n"
    160. 

  160. 		"	 li	%[ticket], 1				\n"
    161. 

  161. 		"2:							\n"
    162. 

  162. 		"	.subsection 2					\n"
    163. 

  163. 		"3:	b	2b					\n"
    164. 

  164. 		"	 li	%[ticket], 0				\n"
    165. 

  165. 		"	.previous					\n"
    166. 

  166. 		"	.set pop					\n"
    167. 

  167. 		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
    168. 

  168. 		  [ticket] "=&r" (tmp),
    169. 

  169. 		  [my_ticket] "=&r" (tmp2),
    170. 

  170. 		  [now_serving] "=&r" (tmp3)
    171. 

  171. 		: [inc] "r" (inc));
    172. 

  172. 	} else {
    173. 

  173. 		__asm__ __volatile__ (
    174. 

  174. 		"	.set push		# arch_spin_trylock	\n"
    175. 

  175. 		"	.set noreorder					\n"
    176. 

  176. 		"							\n"
    177. 

  177. 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
    178. 

  178. 		"	srl	%[my_ticket], %[ticket], 16		\n"
    179. 

  179. 		"	andi	%[now_serving], %[ticket], 0xffff	\n"
    180. 

  180. 		"	bne	%[my_ticket], %[now_serving], 3f	\n"
    181. 

  181. 		"	 addu	%[ticket], %[ticket], %[inc]		\n"
    182. 

  182. 		"	sc	%[ticket], %[ticket_ptr]		\n"
    183. 

  183. 		"	beqz	%[ticket], 1b				\n"
    184. 

  184. 		"	 li	%[ticket], 1				\n"
    185. 

  185. 		"2:							\n"
    186. 

  186. 		"	.subsection 2					\n"
    187. 

  187. 		"3:	b	2b					\n"
    188. 

  188. 		"	 li	%[ticket], 0				\n"
    189. 

  189. 		"	.previous					\n"
    190. 

  190. 		"	.set pop					\n"
    191. 

  191. 		: [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
    192. 

  192. 		  [ticket] "=&r" (tmp),
    193. 

  193. 		  [my_ticket] "=&r" (tmp2),
    194. 

  194. 		  [now_serving] "=&r" (tmp3)
    195. 

  195. 		: [inc] "r" (inc));
    196. 

  196. 	}
    197. 

  197. 

  198. 	smp_llsc_mb();
    199. 

  199. 

  200. 	return tmp;
    201. 

  201. }
    202. 

  202. 

  203. /*
    204. 

  204.  * Read-write spinlocks, allowing multiple readers but only one writer.
    205. 

  205.  *
    206. 

  206.  * NOTE! it is quite common to have readers in interrupts but no interrupt
    207. 

  207.  * writers. For those circumstances we can "mix" irq-safe locks - any writer
    208. 

  208.  * needs to get a irq-safe write-lock, but readers can get non-irqsafe
    209. 

  209.  * read-locks.
    210. 

  210.  */
    211. 

  211. 

  212. /*
    213. 

  213.  * read_can_lock - would read_trylock() succeed?
    214. 

  214.  * @lock: the rwlock in question.
    215. 

  215.  */
    216. 

  216. #define arch_read_can_lock(rw)	((rw)->lock >= 0)
    217. 

  217. 

  218. /*
    219. 

  219.  * write_can_lock - would write_trylock() succeed?
    220. 

  220.  * @lock: the rwlock in question.
    221. 

  221.  */
    222. 

  222. #define arch_write_can_lock(rw) (!(rw)->lock)
    223. 

  223. 

  224. static inline void arch_read_lock(arch_rwlock_t *rw)
    225. 

  225. {
    226. 

  226. 	unsigned int tmp;
    227. 

  227. 

  228. 	if (R10000_LLSC_WAR) {
    229. 

  229. 		__asm__ __volatile__(
    230. 

  230. 		"	.set	noreorder	# arch_read_lock	\n"
    231. 

  231. 		"1:	ll	%1, %2					\n"
    232. 

  232. 		"	bltz	%1, 1b					\n"
    233. 

  233. 		"	 addu	%1, 1					\n"
    234. 

  234. 		"	sc	%1, %0					\n"
    235. 

  235. 		"	beqzl	%1, 1b					\n"
    236. 

  236. 		"	 nop						\n"
    237. 

  237. 		"	.set	reorder					\n"
    238. 

  238. 		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
    239. 

  239. 		: GCC_OFF12_ASM() (rw->lock)
    240. 

  240. 		: "memory");
    241. 

  241. 	} else {
    242. 

  242. 		do {
    243. 

  243. 			__asm__ __volatile__(
    244. 

  244. 			"1:	ll	%1, %2	# arch_read_lock	\n"
    245. 

  245. 			"	bltz	%1, 1b				\n"
    246. 

  246. 			"	 addu	%1, 1				\n"
    247. 

  247. 			"2:	sc	%1, %0				\n"
    248. 

  248. 			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
    249. 

  249. 			: GCC_OFF12_ASM() (rw->lock)
    250. 

  250. 			: "memory");
    251. 

  251. 		} while (unlikely(!tmp));
    252. 

  252. 	}
    253. 

  253. 

  254. 	smp_llsc_mb();
    255. 

  255. }
    256. 

  256. 

  257. /* Note the use of sub, not subu which will make the kernel die with an
    258. 

  258.    overflow exception if we ever try to unlock an rwlock that is already
    259. 

  259.    unlocked or is being held by a writer.  */
    260. 

  260. static inline void arch_read_unlock(arch_rwlock_t *rw)
    261. 

  261. {
    262. 

  262. 	unsigned int tmp;
    263. 

  263. 

  264. 	smp_mb__before_llsc();
    265. 

  265. 

  266. 	if (R10000_LLSC_WAR) {
    267. 

  267. 		__asm__ __volatile__(
    268. 

  268. 		"1:	ll	%1, %2		# arch_read_unlock	\n"
    269. 

  269. 		"	sub	%1, 1					\n"
    270. 

  270. 		"	sc	%1, %0					\n"
    271. 

  271. 		"	beqzl	%1, 1b					\n"
    272. 

  272. 		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
    273. 

  273. 		: GCC_OFF12_ASM() (rw->lock)
    274. 

  274. 		: "memory");
    275. 

  275. 	} else {
    276. 

  276. 		do {
    277. 

  277. 			__asm__ __volatile__(
    278. 

  278. 			"1:	ll	%1, %2	# arch_read_unlock	\n"
    279. 

  279. 			"	sub	%1, 1				\n"
    280. 

  280. 			"	sc	%1, %0				\n"
    281. 

  281. 			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
    282. 

  282. 			: GCC_OFF12_ASM() (rw->lock)
    283. 

  283. 			: "memory");
    284. 

  284. 		} while (unlikely(!tmp));
    285. 

  285. 	}
    286. 

  286. }
    287. 

  287. 

  288. static inline void arch_write_lock(arch_rwlock_t *rw)
    289. 

  289. {
    290. 

  290. 	unsigned int tmp;
    291. 

  291. 

  292. 	if (R10000_LLSC_WAR) {
    293. 

  293. 		__asm__ __volatile__(
    294. 

  294. 		"	.set	noreorder	# arch_write_lock	\n"
    295. 

  295. 		"1:	ll	%1, %2					\n"
    296. 

  296. 		"	bnez	%1, 1b					\n"
    297. 

  297. 		"	 lui	%1, 0x8000				\n"
    298. 

  298. 		"	sc	%1, %0					\n"
    299. 

  299. 		"	beqzl	%1, 1b					\n"
    300. 

  300. 		"	 nop						\n"
    301. 

  301. 		"	.set	reorder					\n"
    302. 

  302. 		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
    303. 

  303. 		: GCC_OFF12_ASM() (rw->lock)
    304. 

  304. 		: "memory");
    305. 

  305. 	} else {
    306. 

  306. 		do {
    307. 

  307. 			__asm__ __volatile__(
    308. 

  308. 			"1:	ll	%1, %2	# arch_write_lock	\n"
    309. 

  309. 			"	bnez	%1, 1b				\n"
    310. 

  310. 			"	 lui	%1, 0x8000			\n"
    311. 

  311. 			"2:	sc	%1, %0				\n"
    312. 

  312. 			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
    313. 

  313. 			: GCC_OFF12_ASM() (rw->lock)
    314. 

  314. 			: "memory");
    315. 

  315. 		} while (unlikely(!tmp));
    316. 

  316. 	}
    317. 

  317. 

  318. 	smp_llsc_mb();
    319. 

  319. }
    320. 

  320. 

  321. static inline void arch_write_unlock(arch_rwlock_t *rw)
    322. 

  322. {
    323. 

  323. 	smp_mb();
    324. 

  324. 

  325. 	__asm__ __volatile__(
    326. 

  326. 	"				# arch_write_unlock	\n"
    327. 

  327. 	"	sw	$0, %0					\n"
    328. 

  328. 	: "=m" (rw->lock)
    329. 

  329. 	: "m" (rw->lock)
    330. 

  330. 	: "memory");
    331. 

  331. }
    332. 

  332. 

  333. static inline int arch_read_trylock(arch_rwlock_t *rw)
    334. 

  334. {
    335. 

  335. 	unsigned int tmp;
    336. 

  336. 	int ret;
    337. 

  337. 

  338. 	if (R10000_LLSC_WAR) {
    339. 

  339. 		__asm__ __volatile__(
    340. 

  340. 		"	.set	noreorder	# arch_read_trylock	\n"
    341. 

  341. 		"	li	%2, 0					\n"
    342. 

  342. 		"1:	ll	%1, %3					\n"
    343. 

  343. 		"	bltz	%1, 2f					\n"
    344. 

  344. 		"	 addu	%1, 1					\n"
    345. 

  345. 		"	sc	%1, %0					\n"
    346. 

  346. 		"	.set	reorder					\n"
    347. 

  347. 		"	beqzl	%1, 1b					\n"
    348. 

  348. 		"	 nop						\n"
    349. 

  349. 		__WEAK_LLSC_MB
    350. 

  350. 		"	li	%2, 1					\n"
    351. 

  351. 		"2:							\n"
    352. 

  352. 		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
    353. 

  353. 		: GCC_OFF12_ASM() (rw->lock)
    354. 

  354. 		: "memory");
    355. 

  355. 	} else {
    356. 

  356. 		__asm__ __volatile__(
    357. 

  357. 		"	.set	noreorder	# arch_read_trylock	\n"
    358. 

  358. 		"	li	%2, 0					\n"
    359. 

  359. 		"1:	ll	%1, %3					\n"
    360. 

  360. 		"	bltz	%1, 2f					\n"
    361. 

  361. 		"	 addu	%1, 1					\n"
    362. 

  362. 		"	sc	%1, %0					\n"
    363. 

  363. 		"	beqz	%1, 1b					\n"
    364. 

  364. 		"	 nop						\n"
    365. 

  365. 		"	.set	reorder					\n"
    366. 

  366. 		__WEAK_LLSC_MB
    367. 

  367. 		"	li	%2, 1					\n"
    368. 

  368. 		"2:							\n"
    369. 

  369. 		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
    370. 

  370. 		: GCC_OFF12_ASM() (rw->lock)
    371. 

  371. 		: "memory");
    372. 

  372. 	}
    373. 

  373. 

  374. 	return ret;
    375. 

  375. }
    376. 

  376. 

  377. static inline int arch_write_trylock(arch_rwlock_t *rw)
    378. 

  378. {
    379. 

  379. 	unsigned int tmp;
    380. 

  380. 	int ret;
    381. 

  381. 

  382. 	if (R10000_LLSC_WAR) {
    383. 

  383. 		__asm__ __volatile__(
    384. 

  384. 		"	.set	noreorder	# arch_write_trylock	\n"
    385. 

  385. 		"	li	%2, 0					\n"
    386. 

  386. 		"1:	ll	%1, %3					\n"
    387. 

  387. 		"	bnez	%1, 2f					\n"
    388. 

  388. 		"	 lui	%1, 0x8000				\n"
    389. 

  389. 		"	sc	%1, %0					\n"
    390. 

  390. 		"	beqzl	%1, 1b					\n"
    391. 

  391. 		"	 nop						\n"
    392. 

  392. 		__WEAK_LLSC_MB
    393. 

  393. 		"	li	%2, 1					\n"
    394. 

  394. 		"	.set	reorder					\n"
    395. 

  395. 		"2:							\n"
    396. 

  396. 		: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
    397. 

  397. 		: GCC_OFF12_ASM() (rw->lock)
    398. 

  398. 		: "memory");
    399. 

  399. 	} else {
    400. 

  400. 		do {
    401. 

  401. 			__asm__ __volatile__(
    402. 

  402. 			"	ll	%1, %3	# arch_write_trylock	\n"
    403. 

  403. 			"	li	%2, 0				\n"
    404. 

  404. 			"	bnez	%1, 2f				\n"
    405. 

  405. 			"	lui	%1, 0x8000			\n"
    406. 

  406. 			"	sc	%1, %0				\n"
    407. 

  407. 			"	li	%2, 1				\n"
    408. 

  408. 			"2:						\n"
    409. 

  409. 			: "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp),
    410. 

  410. 			  "=&r" (ret)
    411. 

  411. 			: GCC_OFF12_ASM() (rw->lock)
    412. 

  412. 			: "memory");
    413. 

  413. 		} while (unlikely(!tmp));
    414. 

  414. 

  415. 		smp_llsc_mb();
    416. 

  416. 	}
    417. 

  417. 

  418. 	return ret;
    419. 

  419. }
    420. 

  420. 

  421. #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
    422. 

  422. #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
    423. 

  423. 

  424. #define arch_spin_relax(lock)	cpu_relax()
    425. 

  425. #define arch_read_relax(lock)	cpu_relax()
    426. 

  426. #define arch_write_relax(lock)	cpu_relax()
    427. 

  427. 

  428. #endif /* _ASM_SPINLOCK_H */
    429.