cmpxchg.h 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. /*
  2. * Based on arch/arm/include/asm/cmpxchg.h
  3. *
  4. * Copyright (C) 2012 ARM Ltd.
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. */
  18. #ifndef __ASM_CMPXCHG_H
  19. #define __ASM_CMPXCHG_H
  20. #include <linux/bug.h>
  21. #include <linux/mmdebug.h>
  22. #include <asm/atomic.h>
  23. #include <asm/barrier.h>
  24. #include <asm/lse.h>
  25. static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
  26. {
  27. unsigned long ret, tmp;
  28. switch (size) {
  29. case 1:
  30. asm volatile(ARM64_LSE_ATOMIC_INSN(
  31. /* LL/SC */
  32. " prfm pstl1strm, %2\n"
  33. "1: ldxrb %w0, %2\n"
  34. " stlxrb %w1, %w3, %2\n"
  35. " cbnz %w1, 1b\n"
  36. " dmb ish",
  37. /* LSE atomics */
  38. " nop\n"
  39. " nop\n"
  40. " swpalb %w3, %w0, %2\n"
  41. " nop\n"
  42. " nop")
  43. : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
  44. : "r" (x)
  45. : "memory");
  46. break;
  47. case 2:
  48. asm volatile(ARM64_LSE_ATOMIC_INSN(
  49. /* LL/SC */
  50. " prfm pstl1strm, %2\n"
  51. "1: ldxrh %w0, %2\n"
  52. " stlxrh %w1, %w3, %2\n"
  53. " cbnz %w1, 1b\n"
  54. " dmb ish",
  55. /* LSE atomics */
  56. " nop\n"
  57. " nop\n"
  58. " swpalh %w3, %w0, %2\n"
  59. " nop\n"
  60. " nop")
  61. : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
  62. : "r" (x)
  63. : "memory");
  64. break;
  65. case 4:
  66. asm volatile(ARM64_LSE_ATOMIC_INSN(
  67. /* LL/SC */
  68. " prfm pstl1strm, %2\n"
  69. "1: ldxr %w0, %2\n"
  70. " stlxr %w1, %w3, %2\n"
  71. " cbnz %w1, 1b\n"
  72. " dmb ish",
  73. /* LSE atomics */
  74. " nop\n"
  75. " nop\n"
  76. " swpal %w3, %w0, %2\n"
  77. " nop\n"
  78. " nop")
  79. : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
  80. : "r" (x)
  81. : "memory");
  82. break;
  83. case 8:
  84. asm volatile(ARM64_LSE_ATOMIC_INSN(
  85. /* LL/SC */
  86. " prfm pstl1strm, %2\n"
  87. "1: ldxr %0, %2\n"
  88. " stlxr %w1, %3, %2\n"
  89. " cbnz %w1, 1b\n"
  90. " dmb ish",
  91. /* LSE atomics */
  92. " nop\n"
  93. " nop\n"
  94. " swpal %3, %0, %2\n"
  95. " nop\n"
  96. " nop")
  97. : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
  98. : "r" (x)
  99. : "memory");
  100. break;
  101. default:
  102. BUILD_BUG();
  103. }
  104. return ret;
  105. }
  106. #define xchg(ptr,x) \
  107. ({ \
  108. __typeof__(*(ptr)) __ret; \
  109. __ret = (__typeof__(*(ptr))) \
  110. __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
  111. __ret; \
  112. })
  113. static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
  114. unsigned long new, int size)
  115. {
  116. switch (size) {
  117. case 1:
  118. return __cmpxchg_case_1(ptr, (u8)old, new);
  119. case 2:
  120. return __cmpxchg_case_2(ptr, (u16)old, new);
  121. case 4:
  122. return __cmpxchg_case_4(ptr, old, new);
  123. case 8:
  124. return __cmpxchg_case_8(ptr, old, new);
  125. default:
  126. BUILD_BUG();
  127. }
  128. unreachable();
  129. }
  130. static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
  131. unsigned long new, int size)
  132. {
  133. switch (size) {
  134. case 1:
  135. return __cmpxchg_case_mb_1(ptr, (u8)old, new);
  136. case 2:
  137. return __cmpxchg_case_mb_2(ptr, (u16)old, new);
  138. case 4:
  139. return __cmpxchg_case_mb_4(ptr, old, new);
  140. case 8:
  141. return __cmpxchg_case_mb_8(ptr, old, new);
  142. default:
  143. BUILD_BUG();
  144. }
  145. unreachable();
  146. }
  147. #define cmpxchg(ptr, o, n) \
  148. ({ \
  149. __typeof__(*(ptr)) __ret; \
  150. __ret = (__typeof__(*(ptr))) \
  151. __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
  152. sizeof(*(ptr))); \
  153. __ret; \
  154. })
  155. #define cmpxchg_local(ptr, o, n) \
  156. ({ \
  157. __typeof__(*(ptr)) __ret; \
  158. __ret = (__typeof__(*(ptr))) \
  159. __cmpxchg((ptr), (unsigned long)(o), \
  160. (unsigned long)(n), sizeof(*(ptr))); \
  161. __ret; \
  162. })
  163. #define system_has_cmpxchg_double() 1
  164. #define __cmpxchg_double_check(ptr1, ptr2) \
  165. ({ \
  166. if (sizeof(*(ptr1)) != 8) \
  167. BUILD_BUG(); \
  168. VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
  169. })
  170. #define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
  171. ({\
  172. int __ret;\
  173. __cmpxchg_double_check(ptr1, ptr2); \
  174. __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
  175. (unsigned long)(n1), (unsigned long)(n2), \
  176. ptr1); \
  177. __ret; \
  178. })
  179. #define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
  180. ({\
  181. int __ret;\
  182. __cmpxchg_double_check(ptr1, ptr2); \
  183. __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
  184. (unsigned long)(n1), (unsigned long)(n2), \
  185. ptr1); \
  186. __ret; \
  187. })
  188. #define _protect_cmpxchg_local(pcp, o, n) \
  189. ({ \
  190. typeof(*raw_cpu_ptr(&(pcp))) __ret; \
  191. preempt_disable(); \
  192. __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
  193. preempt_enable(); \
  194. __ret; \
  195. })
  196. #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
  197. #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
  198. #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
  199. #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
  200. #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
  201. ({ \
  202. int __ret; \
  203. preempt_disable(); \
  204. __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
  205. raw_cpu_ptr(&(ptr2)), \
  206. o1, o2, n1, n2); \
  207. preempt_enable(); \
  208. __ret; \
  209. })
  210. #define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
  211. #define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
  212. #define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n))
  213. #endif /* __ASM_CMPXCHG_H */