1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
|
/* Copyright (C) 2012-2013 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Atomic Library (libatomic).
Libatomic is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.
Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
/* This file contains data types and function declarations that are
private to the implementation of libatomic. */
#ifndef LIBATOMIC_H
#define LIBATOMIC_H 1
#include "auto-config.h"
#include <stdbool.h>
#include <stdint.h>
#include <stddef.h>
#include <limits.h>
#include <string.h>
/* Symbol concatenation macros. */
#define C2_(X,Y) X ## Y
#define C2(X,Y) C2_(X,Y)
#define C3_(X,Y,Z) X ## Y ## Z
#define C3(X,Y,Z) C3_(X,Y,Z)
#define C4_(W,X,Y,Z) W ## X ## Y ## Z
#define C4(W,X,Y,Z) C4_(W,X,Y,Z)
/* Stringification macros. */
#define S2(X) #X
#define S(X) S2(X)
/* All of the primitive types on which we operate. */
typedef unsigned U_1 __attribute__((mode(QI)));
#if HAVE_INT2
typedef unsigned U_2 __attribute__((mode(HI)));
#endif
#if HAVE_INT4
typedef unsigned U_4 __attribute__((mode(SI)));
#endif
#if HAVE_INT8
typedef unsigned U_8 __attribute__((mode(DI)));
#endif
#if HAVE_INT16
typedef unsigned U_16 __attribute__((mode(TI)));
#endif
/* The widest type that we support. */
#if HAVE_INT16
# define MAX_SIZE 16
#elif HAVE_INT8
# define MAX_SIZE 8
#elif HAVE_INT4
# define MAX_SIZE 4
#elif HAVE_INT2
# define MAX_SIZE 2
#else
# define MAX_SIZE 1
#endif
typedef C2(U_,MAX_SIZE) U_MAX;
/* Provide dummy fallback types so that stuff is syntactically correct
without having to overdo the ifdefs. The code using these should
always be protected with the HAVE_INT{n} macros. */
#if !HAVE_INT2
typedef U_MAX U_2;
#endif
#if !HAVE_INT4
typedef U_MAX U_4;
#endif
#if !HAVE_INT8
typedef U_MAX U_8;
#endif
#if !HAVE_INT16
typedef U_MAX U_16;
#endif
union max_size_u
{
U_1 b[MAX_SIZE];
U_2 i2;
U_4 i4;
U_8 i8;
U_16 i16;
};
/* The "word" size of the machine. */
typedef unsigned UWORD __attribute__((mode(word)));
/* Macros for handing sub-word sized quantities. */
#define MASK_1 ((UWORD)0xff)
#define MASK_2 ((UWORD)0xffff)
#define MASK_4 ((UWORD)0xffffffff)
#define INVERT_MASK_1 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT))
#define INVERT_MASK_2 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT))
#define INVERT_MASK_4 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT))
/* Most of the files in this library are compiled multiple times with
N defined to be a power of 2 between 1 and 16. The SIZE macro is
then used to append _N to the symbol being manipulated. */
#define SIZE(X) C3(X,_,N)
#define WSIZE(X) C3(X,_,WORDSIZE)
#define PTR(N,X) ((C2(U_,N) *)X)
/* And thus, the type on which this compilation will be operating. */
#define ITYPE SIZE(I)
#define UTYPE SIZE(U)
/* Utility macros for GCC attributes. */
#define UNUSED __attribute__((unused))
#ifdef HAVE_ATTRIBUTE_VISIBILITY
# define HIDDEN __attribute__((visibility("hidden")))
#else
# define HIDDEN
#endif
/* Occasionally we have to play games with internal and external symbol
names, in order to work around builtin functions of the same name.
This macro sets the external name of the function appropriately. */
#define ASMNAME(X) __asm__(S(C2(__USER_LABEL_PREFIX__,X)))
/* Locking for a "small" operation. In the bare-metal single processor
cases this could be implemented by disabling interrupts. Thus the extra
word passed between the two functions, saving the interrupt level.
It is assumed that the object being locked does not cross the locking
granularity.
Not actually declared here so that they can be defined static inline
in a target-specfic <host-config.h>.
UWORD protect_start (void *ptr);
void protect_end (void *ptr, UWORD);
*/
/* Locking for a "large' operation. This should always be some sort of
test-and-set operation, as we assume that the interrupt latency would
be unreasonably large. */
void libat_lock_n (void *ptr, size_t n);
void libat_unlock_n (void *ptr, size_t n);
/* We'll need to declare all of the sized functions a few times... */
#define DECLARE_ALL_SIZED(N) DECLARE_ALL_SIZED_(N,C2(U_,N))
#define DECLARE_ALL_SIZED_(N,T) \
DECLARE_1(T, C2(load_,N), (T *mptr, int)); \
DECLARE_1(void, C2(store_,N), (T *mptr, T val, int)); \
DECLARE_1(T, C2(exchange_,N), (T *mptr, T, int)); \
DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \
DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int)); \
DECLARE_1(T, C2(fetch_add_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(fetch_sub_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(fetch_and_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(fetch_xor_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(fetch_or_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(fetch_nand_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(add_fetch_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(sub_fetch_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(and_fetch_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(xor_fetch_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(or_fetch_,N), (T *mptr, T, int)); \
DECLARE_1(T, C2(nand_fetch_,N), (T *mptr, T, int))
/* All sized operations are implemented in hidden functions prefixed with
"libat_". These are either renamed or aliased to the expected prefix
of "__atomic". Some amount of renaming is required to avoid hiding or
conflicting with the builtins of the same name, but this additional
use of hidden symbols (where appropriate) avoids unnecessary PLT entries
on relevant targets. */
#if IFUNC_ALT
# define MAN(X) ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN
#elif defined(HAVE_ATTRIBUTE_ALIAS)
# define MAN(X) HIDDEN
#else
# define MAN(X) ASMNAME(C2(__atomic_,X))
#endif
#if !defined(N) && HAVE_IFUNC
# define DECLARE_1(RET,NAME,ARGS) \
RET C2(libat_,NAME) ARGS MAN(NAME); \
RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME))
#else
# define DECLARE_1(RET,NAME,ARGS) RET C2(libat_,NAME) ARGS MAN(NAME)
#endif
/* Prefix to use when calling internal, possibly ifunc'ed functions. */
#if HAVE_IFUNC
# define local_ ifunc_
#else
# define local_ libat_
#endif
DECLARE_ALL_SIZED(1);
DECLARE_ALL_SIZED(2);
DECLARE_ALL_SIZED(4);
DECLARE_ALL_SIZED(8);
DECLARE_ALL_SIZED(16);
#undef DECLARE_1
#undef DECLARE_ALL_SIZED
#undef DECLARE_ALL_SIZED_
/* And the generic sized versions. */
void libat_load (size_t, void *, void *, int) MAN(load);
void libat_store (size_t, void *, void *, int) MAN(store);
void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange);
bool libat_compare_exchange (size_t, void *, void *, void *, int, int)
MAN(compare_exchange);
bool libat_is_lock_free (size_t, void *) MAN(is_lock_free);
#undef MAN
#include <host-config.h>
/* We don't have IFUNC_NCOND until after host-config.h. */
#if !HAVE_IFUNC
# define IFUNC_NCOND(N) 0
#endif
#if IFUNC_ALT
# define EXPORT_ALIAS(X) /* exported symbol in non-alternate file */
#elif defined(N) && IFUNC_NCOND(N)
# if IFUNC_NCOND(N) == 1
# define GEN_SELECTOR(X) \
extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
static void * C2(select_,X) (void) \
{ \
if (IFUNC_COND_1) \
return C3(libat_,X,_i1); \
return C2(libat_,X); \
}
# elif IFUNC_NCOND(N) == 2
# define GEN_SELECTOR(X) \
extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
static void * C2(select_,X) (void) \
{ \
if (IFUNC_COND_1) \
return C3(libat_,X,_i1); \
if (IFUNC_COND_2) \
return C3(libat_,X,_i2); \
return C2(libat_,X); \
}
# elif IFUNC_NCOND(N) == 3
# define GEN_SELECTOR(X) \
extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN; \
static void * C2(select_,X) (void) \
{ \
if (IFUNC_COND_1) \
return C3(libat_,X,_i1); \
if (IFUNC_COND_2) \
return C3(libat_,X,_i2); \
if (IFUNC_COND_3) \
return C3(libat_,X,_i3); \
return C2(libat_,X); \
}
# else
# error "Unsupported number of ifunc alternatives."
# endif
# define EXPORT_ALIAS(X) \
GEN_SELECTOR(X) \
typeof(C2(libat_,X)) C2(ifunc_,X) \
ASMNAME(C2(__atomic_,X)) \
__attribute__((ifunc(S(C2(select_,X)))))
#elif defined(HAVE_ATTRIBUTE_ALIAS)
# define EXPORT_ALIAS(X) \
extern typeof(C2(libat_,X)) C2(export_,X) \
ASMNAME(C2(__atomic_,X)) \
__attribute__((alias(S(C2(libat_,X)))))
#else
# define EXPORT_ALIAS(X) /* original symbol is exported */
#endif
#endif /* LIBATOMIC_H */
|
