1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
//===-- Implementation using the __builtin_XXX_inline ---------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file provides generic C++ building blocks to compose memory functions.
// They rely on the compiler to generate the best possible code through the use
// of the `__builtin_XXX_inline` builtins. These builtins are currently only
// available in Clang.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_BUILTIN_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_BUILTIN_H
#include "src/string/memory_utils/utils.h"
namespace __llvm_libc::builtin {
///////////////////////////////////////////////////////////////////////////////
// Memcpy
template <size_t Size> struct Memcpy {
static constexpr size_t SIZE = Size;
LIBC_INLINE static void block(Ptr __restrict dst, CPtr __restrict src) {
#ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
return __builtin_memcpy_inline(dst, src, SIZE);
#else
// The codegen may be suboptimal.
for (size_t i = 0; i < Size; ++i)
dst[i] = src[i];
#endif
}
LIBC_INLINE static void tail(Ptr __restrict dst, CPtr __restrict src,
size_t count) {
block(dst + count - SIZE, src + count - SIZE);
}
LIBC_INLINE static void head_tail(Ptr __restrict dst, CPtr __restrict src,
size_t count) {
block(dst, src);
tail(dst, src, count);
}
LIBC_INLINE static void loop_and_tail(Ptr __restrict dst, CPtr __restrict src,
size_t count) {
static_assert(Size > 1, "a loop of size 1 does not need tail");
size_t offset = 0;
do {
block(dst + offset, src + offset);
offset += SIZE;
} while (offset < count - SIZE);
tail(dst, src, count);
}
};
///////////////////////////////////////////////////////////////////////////////
// Memset
template <size_t Size> struct Memset {
using ME = Memset;
static constexpr size_t SIZE = Size;
LIBC_INLINE static void block(Ptr dst, uint8_t value) {
#ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE
__builtin_memset_inline(dst, value, Size);
#else
deferred_static_assert("Missing __builtin_memset_inline");
(void)dst;
(void)value;
#endif
}
LIBC_INLINE static void tail(Ptr dst, uint8_t value, size_t count) {
block(dst + count - SIZE, value);
}
LIBC_INLINE static void head_tail(Ptr dst, uint8_t value, size_t count) {
block(dst, value);
tail(dst, value, count);
}
LIBC_INLINE static void loop_and_tail(Ptr dst, uint8_t value, size_t count) {
static_assert(Size > 1, "a loop of size 1 does not need tail");
size_t offset = 0;
do {
block(dst + offset, value);
offset += SIZE;
} while (offset < count - SIZE);
tail(dst, value, count);
}
};
///////////////////////////////////////////////////////////////////////////////
// Bcmp
template <size_t Size> struct Bcmp {
using ME = Bcmp;
static constexpr size_t SIZE = Size;
LIBC_INLINE static BcmpReturnType block(CPtr, CPtr) {
deferred_static_assert("Missing __builtin_memcmp_inline");
return BcmpReturnType::ZERO();
}
LIBC_INLINE static BcmpReturnType tail(CPtr, CPtr, size_t) {
deferred_static_assert("Not implemented");
return BcmpReturnType::ZERO();
}
LIBC_INLINE static BcmpReturnType head_tail(CPtr, CPtr, size_t) {
deferred_static_assert("Not implemented");
return BcmpReturnType::ZERO();
}
LIBC_INLINE static BcmpReturnType loop_and_tail(CPtr, CPtr, size_t) {
deferred_static_assert("Not implemented");
return BcmpReturnType::ZERO();
}
};
///////////////////////////////////////////////////////////////////////////////
// Memcmp
template <size_t Size> struct Memcmp {
using ME = Memcmp;
static constexpr size_t SIZE = Size;
LIBC_INLINE static MemcmpReturnType block(CPtr, CPtr) {
deferred_static_assert("Missing __builtin_memcmp_inline");
return MemcmpReturnType::ZERO();
}
LIBC_INLINE static MemcmpReturnType tail(CPtr, CPtr, size_t) {
deferred_static_assert("Not implemented");
return MemcmpReturnType::ZERO();
}
LIBC_INLINE static MemcmpReturnType head_tail(CPtr, CPtr, size_t) {
deferred_static_assert("Not implemented");
return MemcmpReturnType::ZERO();
}
LIBC_INLINE static MemcmpReturnType loop_and_tail(CPtr, CPtr, size_t) {
deferred_static_assert("Not implemented");
return MemcmpReturnType::ZERO();
}
};
} // namespace __llvm_libc::builtin
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_BUILTIN_H
|
