1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
/* fp16i.c
*
* Copyright 2021 Red Hat, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* SPDX-License-Identifier: LGPL-2.1-or-later
*/
#include "config.h"
#include "fp16private.h"
#ifdef HAVE_F16C
#include <immintrin.h>
#if defined(_MSC_VER) && !defined(__clang__)
#define CAST_M128I_P(a) (__m128i const *) a
#else
#define CAST_M128I_P(a) (__m128i_u const *) a
#endif
void
float_to_half4_f16c (const float f[4],
guint16 h[4])
{
__m128 s = _mm_loadu_ps (f);
__m128i i = _mm_cvtps_ph (s, 0);
_mm_storel_epi64 ((__m128i*)h, i);
}
void
half_to_float4_f16c (const guint16 h[4],
float f[4])
{
__m128i i = _mm_loadl_epi64 (CAST_M128I_P (h));
__m128 s = _mm_cvtph_ps (i);
_mm_store_ps (f, s);
}
#define ALIGNED(p, n) (GPOINTER_TO_UINT(p) % n == 0)
void
float_to_half_f16c (const float *f,
guint16 *h,
int n)
{
__m128 s;
__m128i i;
int j;
const float *ff = f;
guint16 *hh = h;
for (j = 0; j < n; j++)
{
if (ALIGNED (ff, 16) && ALIGNED (hh, 16))
break;
ff++;
hh++;
}
float_to_half_c (f, h, j);
for (; j + 4 < n; j += 4)
{
s = _mm_loadu_ps (ff);
i = _mm_cvtps_ph (s, 0);
_mm_storel_epi64 ((__m128i*)hh, i);
ff += 4;
hh += 4;
}
if (j < n)
float_to_half_c (ff, hh, n - j);
}
void
half_to_float_f16c (const guint16 *h,
float *f,
int n)
{
__m128i i;
__m128 s;
int j;
const guint16 *hh = h;
float *ff = f;
for (j = 0; j < n; j++)
{
if (ALIGNED (ff, 16) && ALIGNED (hh, 16))
break;
ff++;
hh++;
}
half_to_float_c (h, f, j);
for (; j + 4 < n; j += 4)
{
i = _mm_loadl_epi64 (CAST_M128I_P (hh));
s = _mm_cvtph_ps (i);
_mm_store_ps (ff, s);
hh += 4;
ff += 4;
}
if (j < n)
half_to_float_c (hh, ff, n - j);
}
#endif /* HAVE_F16C */
|
