1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
--TEST--
Test of ASCII and JIS X 0201/0208/0212 support in ISO-2022-JP and JIS7/8 encodings
--SKIPIF--
<?php
extension_loaded('mbstring') or die('skip mbstring not available');
if (getenv("SKIP_SLOW_TESTS")) die("skip slow test");
?>
--FILE--
<?php
include('encoding_tests.inc');
mb_substitute_character(0x25); // '%'
/* Read in table of all characters in JISX-0212 charset */
readConversionTable(__DIR__ . '/data/JISX0212.txt', $jisx0212Chars, $unused);
/* Read in table of all characters in JISX-0208 charset */
$jisx0208Chars = array(); /* JISX0208 -> UTF-16BE */
$fp = fopen(__DIR__ . '/data/JISX0208.txt', 'r+');
while ($line = fgets($fp, 256)) {
if ($line[0] == '#')
continue;
if (sscanf($line, "0x%x\t0x%x\t0x%x", $shiftJIS, $jis0208Code, $unicodeCP) == 3) {
$jisx0208Chars[pack('n', $jis0208Code)] = pack('n', $unicodeCP);
}
}
/* Read in table of all characters in JISX-0201 charset */
readConversionTable(__DIR__ . '/data/JISX0201.txt', $jisx0201Chars, $unused);
/* The JIS X 0208 character set does not have a single, straightforward
* mapping to the Unicode character set
* mbstring converts one character differently from the mappings in
* ../docs/JISX0208.txt, which comes from the Unicode Consortium */
/* 0x2140 is a backslash; this can be mapped to 0x005C for an ordinary
* backslash, or 0xFF3C for a _fullwidth_ one */
$jisx0208Chars["\x21\x40"] = "\xFF\x3C";
function testValid($from, $to, $encoding, $bothWays = true) {
identifyValidString($from, $encoding);
convertValidString($from, $to, $encoding, 'UTF-16BE', false);
if ($bothWays) {
/* An 0xF at the beginning of a JIS7 string is redundant; it switches
* to ASCII mode, but ASCII mode is default */
if ($from[0] == "\x0F")
$from = substr($from, 1, strlen($from) - 1);
/* ESC ( B at the beginning is redundant, since ASCII mode is the default */
if (substr($from, 0, 3) == "\x1B(B")
$from = substr($from, 3, strlen($from) - 3);
/* If the string switches to a different charset, it should switch back to
* ASCII at the end */
if (strpos($from, "\x1B\$B") !== false || strpos($from, "\x1B(J") !== false)
$from .= "\x1B(B";
convertValidString($to, $from, 'UTF-16BE', $encoding, false);
}
}
function testInvalid($from, $to, $encoding) {
testInvalidString($from, $to, $encoding, 'UTF-16BE');
}
for ($i = 0; $i < 0x80; $i++) {
if ($i == 0xE || $i == 0xF || $i == 0x1B)
continue;
testValid(chr($i), "\x00" . chr($i), 'JIS');
testValid("\x0F" . chr($i), "\x00" . chr($i), 'JIS'); /* 0xF is 'Shift Out' code */
testValid("\x1B(B" . chr($i), "\x00" . chr($i), 'JIS');
testValid(chr($i), "\x00" . chr($i), 'ISO-2022-JP');
testValid("\x1B(B" . chr($i), "\x00" . chr($i), 'ISO-2022-JP');
}
for ($i = 0x80; $i < 256; $i++) {
if ($i >= 0xA1 && $i <= 0xDF) // We convert single bytes from 0xA1-0xDF as JIS X 0201 kana
continue;
testInvalid(chr($i), "\x00%", 'JIS');
testInvalid("\x0F" . chr($i), "\x00%", 'JIS');
testInvalid("\x1B(B" . chr($i), "\x00%", 'JIS');
testInvalid(chr($i), "\x00%", 'ISO-2022-JP');
testInvalid("\x1B(B" . chr($i), "\x00%", 'ISO-2022-JP');
}
echo "ASCII support OK\n";
/* All valid JIS X 0201 characters
* Those with a 1 in the high bit are JIS X 0201 kana; JIS7 encodes those
* with a 0 in the high bit and treats them as a separate charset
* (We don't test ISO-2022-JP here, as it does not support the JIS X 0201 charset) */
foreach ($jisx0201Chars as $jisx0201 => $utf16BE) {
if (ord($jisx0201) >= 128) {
$kana = chr(ord($jisx0201) - 128);
testValid("\x1B(I" . $kana, $utf16BE, 'JIS', false);
testValid("\x0E" . $kana, $utf16BE, 'JIS', false); /* 0xE is 'Shift In' code */
testValid($jisx0201, $utf16BE, 'JIS', false);
} else {
testValid("\x1B(J" . $jisx0201, $utf16BE, 'JIS', $utf16BE > "\x00\x80");
}
}
for ($i = 0x80; $i < 256; $i++) {
if ($i >= 0xA1 && $i <= 0xDF)
continue;
testInvalid("\x1B(I" . chr($i), "\x00%", 'JIS');
testInvalid("\x1B(J" . chr($i), "\x00%", 'JIS');
}
echo "JIS X 0201 support OK\n";
/* All valid JISX0208 characters */
foreach ($jisx0208Chars as $jisx0208 => $utf16BE) {
testValid("\x1B\$B" . $jisx0208, $utf16BE, 'JIS');
testValid("\x1B\$B" . $jisx0208, $utf16BE, 'ISO-2022-JP');
}
/* All invalid 2-byte JISX0208 characters */
for ($i = 0x21; $i <= 0x7E; $i++) {
for ($j = 0; $j < 256; $j++) {
$testString = chr($i) . chr($j);
if (!isset($jisx0208Chars[$testString])) {
testInvalid("\x1B\$B" . $testString, "\x00%", 'JIS');
testInvalid("\x1B\$B" . $testString, "\x00%", 'ISO-2022-JP');
}
}
}
/* Try truncated JISX0208 characters */
for ($i = 0x21; $i <= 0x7E; $i++) {
testInvalid("\x1B\$B" . chr($i), "\x00%", 'JIS');
testInvalid("\x1B\$B" . chr($i), "\x00%", 'ISO-2022-JP');
}
echo "JIS X 0208 support OK\n";
/* JIS7 supports escape to switch to JIS X 0212 charset, but ISO-2022-JP does not */
/* All valid JISX0212 characters */
foreach ($jisx0212Chars as $jisx0212 => $utf16BE) {
testValid("\x1B\$(D" . $jisx0212, $utf16BE, 'JIS', false);
}
/* All invalid 2-byte JISX0212 characters */
for ($i = 0x21; $i <= 0x7E; $i++) {
for ($j = 0; $j < 256; $j++) {
$testString = chr($i) . chr($j);
if (!isset($jisx0212Chars[$testString])) {
testInvalid("\x1B\$(D" . $testString, "\x00%", 'JIS');
}
}
}
/* Try truncated JISX0212 characters */
for ($i = 0x21; $i <= 0x7E; $i++) {
testInvalid("\x1B\$(D" . chr($i), "\x00%", 'JIS');
}
echo "JIS X 0212 support OK\n";
/* All possible escape sequences */
$validEscapes = ["\x1B\$@" => true, "\x1B\$B" => true, "\x1B\$(@" => true, "\x1B\$(B" => true, "\x1B\$(D" => true, "\x1B(B" => true, "\x1B(H" => true, "\x1B(J" => true, "\x1B(I" => true];
for ($i = 0; $i <= 0xFF; $i++) {
for ($j = 0; $j <= 0xFF; $j++) {
$escapeSequence = "\x1B" . chr($i) . chr($j);
if ($escapeSequence === "\x1B\$(")
continue;
if (isset($validEscapes[$escapeSequence])) {
testValid($escapeSequence, "", 'JIS', false);
testValid($escapeSequence, "", 'ISO-2022-JP', false);
} else {
identifyInvalidString($escapeSequence, 'JIS');
identifyInvalidString($escapeSequence, 'ISO-2022-JP');
}
}
}
for ($i = 0; $i <= 0xFF; $i++) {
$escapeSequence = "\x1B\$(" . chr($i);
if (isset($validEscapes[$escapeSequence])) {
testValid($escapeSequence, "", 'JIS', false);
testValid($escapeSequence, "", 'ISO-2022-JP', false);
} else {
identifyInvalidString($escapeSequence, 'JIS');
identifyInvalidString($escapeSequence, 'ISO-2022-JP');
}
}
echo "All escape sequences work as expected\n";
?>
--EXPECT--
ASCII support OK
JIS X 0201 support OK
JIS X 0208 support OK
JIS X 0212 support OK
All escape sequences work as expected
|
