1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
|
/*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999-2003 by Hewlett-Packard Company. All rights reserved.
*
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*
*/
#include "../read_ordered.h"
#include "../test_and_set_t_is_ao_t.h" /* Probably suboptimal */
/* NEC LE-IT: ARMv6 is the first architecture providing support for */
/* simple LL/SC. A data memory barrier must be raised via CP15 command */
/* (see documentation). */
/* ARMv7 is compatible to ARMv6 but has a simpler command for issuing */
/* a memory barrier (DMB). Raising it via CP15 should still work as */
/* told me by the support engineers. If it turns out to be much quicker */
/* than we should implement custom code for ARMv7 using the asm { dmb } */
/* instruction. */
/* If only a single processor is used, we can define AO_UNIPROCESSOR */
/* and do not need to access CP15 for ensuring a DMB. */
#if defined(__thumb__) && !defined(__thumb2__)
/* Thumb One mode does not have ARM "mcr", "swp" and some load/store */
/* instructions, so we temporarily switch to ARM mode and go back */
/* afterwards (clobbering "r3" register). */
# define AO_THUMB_GO_ARM \
" adr r3, 4f\n" \
" bx r3\n" \
" .align\n" \
" .arm\n" \
"4:\n"
# define AO_THUMB_RESTORE_MODE \
" adr r3, 5f + 1\n" \
" bx r3\n" \
" .thumb\n" \
"5:\n"
# define AO_THUMB_SWITCH_CLOBBERS "r3",
#else
# define AO_THUMB_GO_ARM /* empty */
# define AO_THUMB_RESTORE_MODE /* empty */
# define AO_THUMB_SWITCH_CLOBBERS /* empty */
#endif /* !__thumb__ */
/* NEC LE-IT: gcc has no way to easily check the arm architecture */
/* but it defines only one of __ARM_ARCH_x__ to be true. */
#if !defined(__ARM_ARCH_2__) && !defined(__ARM_ARCH_3__) \
&& !defined(__ARM_ARCH_3M__) && !defined(__ARM_ARCH_4__) \
&& !defined(__ARM_ARCH_4T__) && !defined(__ARM_ARCH_5__) \
&& !defined(__ARM_ARCH_5E__) && !defined(__ARM_ARCH_5T__) \
&& !defined(__ARM_ARCH_5TE__) && !defined(__ARM_ARCH_5TEJ__) \
&& !defined(__ARM_ARCH_6M__)
#include "../standard_ao_double_t.h"
AO_INLINE void
AO_nop_full(void)
{
# ifndef AO_UNIPROCESSOR
unsigned dest = 0;
/* Issue a data memory barrier (keeps ordering of memory */
/* transactions before and after this operation). */
__asm__ __volatile__("@AO_nop_full\n"
AO_THUMB_GO_ARM
" mcr p15,0,%0,c7,c10,5\n"
AO_THUMB_RESTORE_MODE
: "=&r"(dest)
: /* empty */
: AO_THUMB_SWITCH_CLOBBERS "memory");
# endif
}
#define AO_HAVE_nop_full
/* NEC LE-IT: AO_t load is simple reading */
AO_INLINE AO_t
AO_load(const volatile AO_t *addr)
{
/* Cast away the volatile for architectures like IA64 where */
/* volatile adds barrier semantics. */
return (*(const AO_t *)addr);
}
#define AO_HAVE_load
/* NEC LE-IT: atomic "store" - according to ARM documentation this is
* the only safe way to set variables also used in LL/SC environment.
* A direct write won't be recognized by the LL/SC construct on the _same_ CPU.
*
* Support engineers response for behaviour of ARMv6:
*
Core1 Core2 SUCCESS
===================================
LDREX(x)
STREX(x) Yes
-----------------------------------
LDREX(x)
STR(x)
STREX(x) No
-----------------------------------
LDREX(x)
STR(x)
STREX(x) Yes
-----------------------------------
*
* ARMv7 behaves similar, see documentation CortexA8 TRM, point 8.5
*
* HB: I think this is only a problem if interrupt handlers do not clear
* the reservation, as they almost certainly should. Probably change this back
* in a while?
*/
AO_INLINE void AO_store(volatile AO_t *addr, AO_t value)
{
AO_t flag;
__asm__ __volatile__("@AO_store\n"
AO_THUMB_GO_ARM
"1: ldrex %0, [%2]\n"
" strex %0, %3, [%2]\n"
" teq %0, #0\n"
" bne 1b\n"
AO_THUMB_RESTORE_MODE
: "=&r"(flag), "+m"(*addr)
: "r" (addr), "r"(value)
: AO_THUMB_SWITCH_CLOBBERS "cc");
}
#define AO_HAVE_store
/* NEC LE-IT: replace the SWAP as recommended by ARM:
"Applies to: ARM11 Cores
Though the SWP instruction will still work with ARM V6 cores, it is
recommended to use the new V6 synchronization instructions. The SWP
instruction produces 'locked' read and write accesses which are atomic,
i.e. another operation cannot be done between these locked accesses which
ties up external bus (AHB, AXI) bandwidth and can increase worst case
interrupt latencies. LDREX, STREX are more flexible, other instructions
can be done between the LDREX and STREX accesses."
*/
#if !defined(AO_FORCE_USE_SWP) || defined(__thumb2__)
/* But, on the other hand, there could be a considerable performance */
/* degradation in case of a race. Eg., test_atomic.c executing */
/* test_and_set test on a dual-core ARMv7 processor using LDREX/STREX */
/* showed around 35 times lower performance than that using SWP. */
/* To force use of SWP instruction, use -D AO_FORCE_USE_SWP option */
/* (this is ignored in the Thumb-2 mode as SWP is missing there). */
AO_INLINE AO_TS_VAL_t
AO_test_and_set(volatile AO_TS_t *addr)
{
AO_TS_VAL_t oldval;
unsigned long flag;
__asm__ __volatile__("@AO_test_and_set\n"
AO_THUMB_GO_ARM
"1: ldrex %0, [%3]\n"
" strex %1, %4, [%3]\n"
" teq %1, #0\n"
" bne 1b\n"
AO_THUMB_RESTORE_MODE
: "=&r"(oldval), "=&r"(flag), "+m"(*addr)
: "r"(addr), "r"(1)
: AO_THUMB_SWITCH_CLOBBERS "cc");
return oldval;
}
# define AO_HAVE_test_and_set
#endif /* !AO_FORCE_USE_SWP */
/* NEC LE-IT: fetch and add for ARMv6 */
AO_INLINE AO_t
AO_fetch_and_add(volatile AO_t *p, AO_t incr)
{
unsigned long flag, tmp;
AO_t result;
__asm__ __volatile__("@AO_fetch_and_add\n"
AO_THUMB_GO_ARM
"1: ldrex %0, [%5]\n" /* get original */
" add %2, %0, %4\n" /* sum up in incr */
" strex %1, %2, [%5]\n" /* store them */
" teq %1, #0\n"
" bne 1b\n"
AO_THUMB_RESTORE_MODE
: "=&r"(result), "=&r"(flag), "=&r"(tmp), "+m"(*p) /* 0..3 */
: "r"(incr), "r"(p) /* 4..5 */
: AO_THUMB_SWITCH_CLOBBERS "cc");
return result;
}
#define AO_HAVE_fetch_and_add
/* NEC LE-IT: fetch and add1 for ARMv6 */
AO_INLINE AO_t
AO_fetch_and_add1(volatile AO_t *p)
{
unsigned long flag, tmp;
AO_t result;
__asm__ __volatile__("@AO_fetch_and_add1\n"
AO_THUMB_GO_ARM
"1: ldrex %0, [%4]\n" /* get original */
" add %1, %0, #1\n" /* increment */
" strex %2, %1, [%4]\n" /* store them */
" teq %2, #0\n"
" bne 1b\n"
AO_THUMB_RESTORE_MODE
: "=&r"(result), "=&r"(tmp), "=&r"(flag), "+m"(*p)
: "r"(p)
: AO_THUMB_SWITCH_CLOBBERS "cc");
return result;
}
#define AO_HAVE_fetch_and_add1
/* NEC LE-IT: fetch and sub for ARMv6 */
AO_INLINE AO_t
AO_fetch_and_sub1(volatile AO_t *p)
{
unsigned long flag, tmp;
AO_t result;
__asm__ __volatile__("@AO_fetch_and_sub1\n"
AO_THUMB_GO_ARM
"1: ldrex %0, [%4]\n" /* get original */
" sub %1, %0, #1\n" /* decrement */
" strex %2, %1, [%4]\n" /* store them */
" teq %2, #0\n"
" bne 1b\n"
AO_THUMB_RESTORE_MODE
: "=&r"(result), "=&r"(tmp), "=&r"(flag), "+m"(*p)
: "r"(p)
: AO_THUMB_SWITCH_CLOBBERS "cc");
return result;
}
#define AO_HAVE_fetch_and_sub1
/* NEC LE-IT: compare and swap */
/* Returns nonzero if the comparison succeeded. */
AO_INLINE int
AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
{
AO_t result, tmp;
__asm__ __volatile__("@AO_compare_and_swap\n"
AO_THUMB_GO_ARM
"1: mov %0, #2\n" /* store a flag */
" ldrex %1, [%3]\n" /* get original */
" teq %1, %4\n" /* see if match */
# ifdef __thumb2__
" it eq\n"
# endif
" strexeq %0, %5, [%3]\n" /* store new one if matched */
" teq %0, #1\n"
" beq 1b\n" /* if update failed, repeat */
AO_THUMB_RESTORE_MODE
: "=&r"(result), "=&r"(tmp), "+m"(*addr)
: "r"(addr), "r"(old_val), "r"(new_val)
: AO_THUMB_SWITCH_CLOBBERS "cc");
return !(result&2); /* if succeded, return 1, else 0 */
}
#define AO_HAVE_compare_and_swap
#if !defined(__ARM_ARCH_6__) && !defined(__ARM_ARCH_6J__) \
&& !defined(__ARM_ARCH_6T2__) && !defined(__ARM_ARCH_6Z__) \
&& !defined(__ARM_ARCH_6ZT2__) && (!defined(__thumb__) \
|| (defined(__thumb2__) && !defined(__ARM_ARCH_7__) \
&& !defined(__ARM_ARCH_7M__) && !defined(__ARM_ARCH_7EM__)))
/* LDREXD/STREXD present in ARMv6K/M+ (see gas/config/tc-arm.c) */
/* In the Thumb mode, this works only starting from ARMv7 (except for */
/* the base and 'M' models). */
AO_INLINE int
AO_compare_double_and_swap_double(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
double_ptr_storage old_val =
((double_ptr_storage)old_val2 << 32) | old_val1;
double_ptr_storage new_val =
((double_ptr_storage)new_val2 << 32) | new_val1;
double_ptr_storage tmp;
int result = 1;
do {
__asm__ __volatile__("@AO_compare_double_and_swap_double\n"
" ldrexd %0, [%1]\n" /* get original to r1 & r2 */
: "=&r"(tmp)
: "r"(addr)
: "cc");
if (tmp != old_val)
break;
__asm__ __volatile__(
" strexd %0, %2, [%3]\n" /* store new one if matched */
: "=&r"(result), "+m"(*addr)
: "r"(new_val), "r"(addr)
: "cc");
} while (result);
return !result; /* if succeded, return 1 else 0 */
}
# define AO_HAVE_compare_double_and_swap_double
#endif
#else
/* pre ARMv6 architectures ... */
/* I found a slide set that, if I read it correctly, claims that */
/* Loads followed by either a Load or Store are ordered, but nothing */
/* else is. */
/* It appears that SWP is the only simple memory barrier. */
#include "../all_atomic_load_store.h"
/* The code should run correctly on a multi-core ARMv6+ as well. */
/* There is only a single concern related to AO_store (defined in */
/* atomic_load_store.h file): */
/* HB: Based on subsequent discussion, I think it would be OK to use an */
/* ordinary store here if we knew that interrupt handlers always */
/* cleared the reservation. They should, but there is some doubt that */
/* this is currently always the case, e.g., for Linux. */
/* ARMv6M does not support ARM mode. */
#endif /* __ARM_ARCH_x */
#if !defined(AO_HAVE_test_and_set_full) && !defined(AO_HAVE_test_and_set) \
&& !defined(__ARM_ARCH_2__) && !defined(__ARM_ARCH_6M__)
AO_INLINE AO_TS_VAL_t
AO_test_and_set_full(volatile AO_TS_t *addr)
{
AO_TS_VAL_t oldval;
/* SWP on ARM is very similar to XCHG on x86. */
/* The first operand is the result, the second the value */
/* to be stored. Both registers must be different from addr. */
/* Make the address operand an early clobber output so it */
/* doesn't overlap with the other operands. The early clobber */
/* on oldval is necessary to prevent the compiler allocating */
/* them to the same register if they are both unused. */
__asm__ __volatile__("@AO_test_and_set_full\n"
AO_THUMB_GO_ARM
" swp %0, %2, [%3]\n"
/* Ignore GCC "SWP is deprecated for this architecture" */
/* warning here (for ARMv6+). */
AO_THUMB_RESTORE_MODE
: "=&r"(oldval), "=&r"(addr)
: "r"(1), "1"(addr)
: AO_THUMB_SWITCH_CLOBBERS "memory");
return oldval;
}
# define AO_HAVE_test_and_set_full
#endif /* !AO_HAVE_test_and_set[_full] */
|
