1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
|
%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2000-2013. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%%
%% %CopyrightEnd%
%%
%% This program is used to generate a header file with data for
%% normalizing denormalized unicode.
%% The C header is generated from a text file containing tuples in the
%% following format:
%% {RevList,Translation}
%% Where 'RevList' is a reversed list of the denormalized repressentation of
%% the character 'Translation'. An example would be the swedish character
%% 'ö', which would be represented in the file as:
%% {[776,111],246}, as the denormalized representation of codepoint 246
%% is [111,776] (i.e an 'o' followed by the "double dot accent character 776),
%% while 'ä' instead is represented as {[776,97],228}, as the denormalized
%% form would be [97,776] (same accent but an 'a' instead).
%% The datafile is generated from the table on Apple's developer connection
%% http://developer.apple.com/library/mac/#technotes/tn/tn1150table.html
%% The generating is done whenever new data is present (i.e. dec.dat has
%% to be changed) and not for every build. The product (the C header) is copied
%% to $ERL_TOP/erts/beam after generation and checked in.
%% The program and the data file is included for reference.
-module(dec).
-compile(export_all).
-define(HASH_SIZE_FACTOR,2).
-define(BIG_PREFIX_SIZE,392).
-define(INPUT_FILE_NAME,"dec.dat").
-define(OUTPUT_FILE_NAME,"erl_unicode_normalize.h").
read(FName) ->
{ok,L} = file:consult(FName),
[{A,B} || {A,B} <- L,
length(A) > 1% , hd(A) < 769
].
dec() ->
L = read(?INPUT_FILE_NAME),
G = group(L),
{ok,Out} = file:open(?OUTPUT_FILE_NAME,[write]),
io:format
(Out,
"/*~n"
"* %CopyrightBegin%~n"
"*~n"
"* Copyright Ericsson AB 1999-2010. All Rights Reserved.~n"
"*~n"
"* The contents of this file are subject to the Erlang Public License,~n"
"* Version 1.1, (the \"License\"); you may not use this file except in~n"
"* compliance with the License. You should have received a copy of the~n"
"* Erlang Public License along with this software. If not, it can be~n"
"* retrieved online at http://www.erlang.org/.~n"
"*~n"
"* Software distributed under the License is distributed on an "
"\"AS IS\"~n"
"* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See~n"
"* the License for the specific language governing rights and "
"limitations~n"
"* under the License.~n"
"*~n"
"* %CopyrightEnd%~n"
"*/~n"
"/*~n"
"* This file is automatically generated by ~p.erl, "
"do not edit manually~n"
"*/~n",
[?MODULE]),
io:format(Out,
"#define HASH_SIZE_FACTOR ~w~n"
"typedef struct _compose_entry {~n"
" Uint16 c;~n"
" Uint16 res;~n"
" Uint16 num_subs;~n"
" struct _compose_entry *subs;~n"
" int *hash;~n"
"} CompEntry;~n~n"
"static int compose_tab_size = ~p;~n",
[?HASH_SIZE_FACTOR,length(G)]),
d(Out,G,[],0),
PreTab = tuple_to_list(make_prefix_table(G,erlang:make_tuple(102,0))),
dump_prefixes(Out,PreTab),
%% Using this cuts down on the searching in the
%% actual implementation, but wastes memory with little real gain..
%% LL = lists:flatten([PartList || {PartList,_} <- L]),
%% BigPreTab = tuple_to_list(
%% make_big_prefixes(LL,
%% erlang:make_tuple(?BIG_PREFIX_SIZE,0))),
%% dump_big_prefixes(Out,BigPreTab),
file:close(Out),
ok.
d(Out,List,D,C) ->
d_sub(Out,List,D,C),
d_top_hash(Out,List,D,C),
d_top(Out,List,D,C).
d_sub(_Out,[],_D,_C) ->
ok;
d_sub(Out,[{_CP,[],_Res}|T],D,C) ->
d_sub(Out,T,D,C+1);
d_sub(Out,[{_CP,Subs,_Res0}|T],D,C) ->
d(Out,Subs,[C|D],0),
d_sub(Out,T,D,C+1).
d_top(Out,L,D,C) ->
io:format(Out,"static CompEntry ~s[] = {~n",[format_depth(D)]),
d_top_1(Out,L,D,C),
io:format(Out,"}; /* ~s */ ~n",[format_depth(D)]).
d_top_1(_Out,[],_D,_C) ->
ok;
d_top_1(Out,[{CP,[],Res}|T],D,C) ->
io:format(Out,
"{~w, ~w, 0, NULL, NULL}",[CP,Res]),
if
T =:= [] ->
io:format(Out,"~n",[]);
true ->
io:format(Out,",~n",[])
end,
d_top_1(Out,T,D,C+1);
d_top_1(Out,[{CP,Subs,_Res}|T],D,C) ->
io:format(Out,
"{~w, 0, ~w, ~s, ~s}",[CP,length(Subs),
format_depth([C|D]),
"hash_"++format_depth([C|D])]),
if
T =:= [] ->
io:format(Out,"~n",[]);
true ->
io:format(Out,",~n",[])
end,
d_top_1(Out,T,D,C+1).
d_top_hash(Out,List,D,_C) ->
HSize = length(List)*?HASH_SIZE_FACTOR,
io:format(Out,"static int ~s[~p] = ~n",["hash_"++format_depth(D),HSize]),
Tup = d_top_hash_1(List,0,erlang:make_tuple(HSize,-1),HSize),
io:format(Out,"~p; /* ~s */ ~n",[Tup,"hash_"++format_depth(D)]).
d_top_hash_1([],_,Hash,_HSize) ->
Hash;
d_top_hash_1([{CP,_,_}|T],Index,Hash,HSize) ->
Bucket = hash_search(Hash,HSize,CP rem HSize),
d_top_hash_1(T,Index+1,erlang:setelement(Bucket+1,Hash,Index),HSize).
hash_search(Hash,_HSize,Bucket) when element(Bucket+1,Hash) =:= -1 ->
Bucket;
hash_search(Hash,HSize,Bucket) ->
hash_search(Hash,HSize,(Bucket + 1) rem HSize).
format_depth(D) ->
lists:reverse(tl(lists:reverse(lists:flatten(["compose_tab_",[ integer_to_list(X) ++ "_" || X <- lists:reverse(D) ]])))).
make_prefix_table([],Table) ->
Table;
make_prefix_table([{C,_,_}|T],Table) when C =< 4023 ->
Index = (C div 32) + 1 - 24,
Pos = C rem 32,
X = element(Index,Table),
Y = X bor (1 bsl Pos),
NewTab = setelement(Index,Table,Y),
make_prefix_table(T,NewTab);
make_prefix_table([_|T],Tab) ->
make_prefix_table(T,Tab).
dump_prefixes(Out,L) ->
io:format(Out,"#define COMP_CANDIDATE_MAP_OFFSET 24~n",[]),
io:format(Out,"static Uint32 comp_candidate_map[] = {~n",[]),
dump_prefixes_1(Out,L).
dump_prefixes_1(Out,[H]) ->
io:format(Out," 0x~8.16.0BU~n",[H]),
io:format(Out,"};~n",[]);
dump_prefixes_1(Out,[H|T]) ->
io:format(Out," 0x~8.16.0BU,~n",[H]),
dump_prefixes_1(Out,T).
%% make_big_prefixes([],Table) ->
%% Table;
%% make_big_prefixes([C|T],Table) ->
%% Index = (C div 32) + 1,
%% Pos = C rem 32,
%% X = element(Index,Table),
%% Y = X bor (1 bsl Pos),
%% NewTab = setelement(Index,Table,Y),
%% make_big_prefixes(T,NewTab).
%% dump_big_prefixes(Out,L) ->
%% io:format(Out,"#define BIG_COMP_CANDIDATE_SIZE ~w~n", [?BIG_PREFIX_SIZE]),
%% io:format(Out,"static Uint32 big_comp_candidate_map[] = {~n",[]),
%% dump_prefixes_1(Out,L).
pick([],_,Acc) ->
{lists:reverse(Acc),[]};
pick([{[H|TT],N}|T],H,Acc) ->
pick(T,H,[{TT,N}|Acc]);
pick([{[H|_],_}|_]=L,M,Acc) when H =/= M ->
{lists:reverse(Acc),L}.
group([]) ->
[];
group([{[H],N}|T]) ->
{Part,Rest} = pick(T,H,[]),
[{H,group(Part),N}| group(Rest)];
group([{[H|_],_}|_]=L) ->
{Part,Rest} = pick(L,H,[]),
[{H,group(Part),0}| group(Rest)].
|
