Complete UTF-8 and UTF-16 decoders.

Reader is mostly done (though untested).


git-svn-id: http://svn.pyyaml.org/libyaml/trunk@181 18f92427-320e-0410-9341-c67f048884a3

4 files changed, 386 insertions, 108 deletions

diff --git a/include/yaml/yaml.h b/include/yaml/yaml.h
index 9ec2c7a..2f32e52 100644
--- a/include/yaml/yaml.h
+++ b/include/yaml/yaml.h
@@ -291,8 +291,15 @@ typedef struct {
      * @{
      */
 
+    /** Error type. */
     yaml_error_type_t error;
 
+    /** Error description. */
+    const char *problem;
+
+    /** The byte about which the problem occured. */
+    size_t problem_offset;
+
     /**
      * @}
      */
@@ -302,7 +309,7 @@ typedef struct {
      * @{
      */
 
-    /** Read handler */
+    /** Read handler. */
     yaml_read_handler_t *read_handler;
 
     /** A pointer for passing to the read handler. */
@@ -488,6 +495,18 @@ yaml_free(void *ptr);
 
 #define YAML_BUFFER_SIZE    (YAML_RAW_BUFFER_SIZE*3)
 
+/**
+ * Ensure that the buffer contains at least @a length characters.
+ *
+ * @param[in]   parser  A parser object.
+ * @param[in]   length  The number of characters in the buffer.
+ *
+ * @returns @c 1 on success, @c 0 on error.
+ */
+
+int
+yaml_parser_update_buffer(yaml_parser_t *parser, size_t length);
+
 /** @} */
 
 
diff --git a/src/reader.c b/src/reader.c
index 440a88f..5ee8c43 100644
--- a/src/reader.c
+++ b/src/reader.c
@@ -7,51 +7,20 @@
 
 #include <assert.h>
 
-/* Check for the UTF-16-BE BOM. */
-#define IS_UTF16BE_BOM(pointer) ((pointer)[0] == 0xFE && (pointer)[1] == 0xFF)
-
-/* Check for the UTF-16-LE BOM. */
-#define IS_UTF16LE_BOM(pointer) ((pointer)[0] == 0xFF && (pointer)[1] == 0xFE)
-
-/* Get a UTF-16-BE character. */
-#define UTF16BE_CHAR(pointer)   ((pointer)[0] << 8 + (pointer)[1])
-
-/* Get a UTF-16-LE character. */
-#define UTF16LE_CHAR(pointer)   ((pointer)[0] + (pointer)[1] << 8)
-
 /*
- * From http://www.ietf.org/rfc/rfc3629.txt:
- *
- *    Char. number range  |        UTF-8 octet sequence
- *      (hexadecimal)    |              (binary)
- *   --------------------+---------------------------------------------
- *   0000 0000-0000 007F | 0xxxxxxx
- *   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
- *   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
- *   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+ * Set the reader error and return 0.
  */
 
-/* Get the length of a UTF-8 character (0 on error). */
-#define UTF8_LENGTH(pointer)    \
-    ((pointer)[0] < 0x80 ? 1 :  \
-    (pointer)[0] < 0xC0 ? 0 :   \
-    (pointer)[0] < 0xE0 ? 2 :   \
-    (pointer)[0] < 0xF0 ? 3 :   \
-    (pointer)[0] < 0xF8 ? 4 : 0)
-
-/* Get the value of the first byte of a UTF-8 sequence (0xFF on error). */
-#define UTF8_FIRST_CHUNK(pointer)   \
-    ((pointer)[0] < 0x80 ? (pointer)[0] & 0x7F :    \
-    (pointer)[0] < 0xC0 ? 0xFF :    \
-    (pointer)[0] < 0xE0 ? (pointer)[0] & 0x1F : \
-    (pointer)[0] < 0xF0 ? (pointer)[0] & 0x0F : \
-    (pointer)[0] < 0xF8 ? (pointer)[0] & 0x07 : 0xFF)
+int
+yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem)
+{
+    parser->error = YAML_READER_ERROR;
+    parser->problem = problem;
+    parser->problem_offset = parser->offset;
 
-/* Get the value of a non-first byte of a UTF-8 sequence (0xFF on error). */
-#define UTF8_NEXT_CHUNK(pointer)    \
-    ((pointer)[0] >= 0x80 && (pointer)[0] < 0xC0 ? (pointer)[0] & 0x3F : 0xFF)
+    return 0;
+}
 
-/* Determine the length of a UTF-8 character. */
 
 /*
  * Ensure that the buffer contains at least length characters.
@@ -102,12 +71,19 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
 
         if (!yaml_parser_update_raw_buffer(parser)) return 0;
 
+        /* If the raw buffer is empty, it is EOF. */
+
+        if (!parser->raw_unread) return 1;
+
         /* Decode the raw buffer. */
 
         while (parser->raw_unread)
         {
-            unsigned int ch;
+            unsigned int value, value2;
             int incomplete = 0;
+            unsigned char utf8_octet;
+            unsigned int utf8_length;
+            int k, low, high;
 
             /* Decode the next character. */
 
@@ -115,81 +91,200 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
             {
                 case YAML_UTF8_ENCODING:
 
-                    unsigned int utf8_length = UTF8_LENGTH(parser->raw_pointer);
-                    unsigned int utf8_chunk;
+                    /*
+                     * Decode a UTF-8 character.  Check RFC 3629
+                     * (http://www.ietf.org/rfc/rfc3629.txt) for more details.
+                     *
+                     * The following table (taken from the RFC) is used for
+                     * decoding.
+                     *
+                     *    Char. number range |        UTF-8 octet sequence
+                     *      (hexadecimal)    |              (binary)
+                     *   --------------------+------------------------------------
+                     *   0000 0000-0000 007F | 0xxxxxxx
+                     *   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
+                     *   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
+                     *   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                     *
+                     * Additionally, the characters in the range 0xD800-0xDFFF
+                     * are prohibited as they are reserved for use with UTF-16
+                     * surrogate pairs.
+                     */
+
+                    /* Determine the length of the UTF-8 sequence. */
+
+                    utf8_octet = parser->raw_pointer[0];
+                    utf8_length = (
+                            (utf8_octet & 0x80) == 0x00 ? 1 :
+                            (utf8_octet & 0xE0) == 0xC0 ? 2 :
+                            (utf8_octet & 0xF0) == 0xE0 ? 3 :
+                            (utf8_octet & 0xF8) == 0xF0 ? 4 : 0);
+
+                    /* Check if the leading octet is valid. */
+
+                    if (!utf8_length)
+                        return yaml_parser_set_reader_error(parser,
+                                "Invalid leading UTF-8 octet");
 
                     /* Check if the raw buffer contains an incomplete character. */
 
                     if (utf8_length > parser->raw_unread) {
                         if (parser->eof) {
-                            parser->error = YAML_READER_ERROR;
-                            return 0;
+                            return yaml_parser_set_reader_error(parser,
+                                    "Incomplete UTF-8 octet sequence");
                         }
                         incomplete = 1;
+                        break;
                     }
 
-                    /* Get the character checking it for validity. */
+                    /* Decode the leading octet. */
 
-                    utf8_chunk = UTF8_FIRST_CHUNK(parser->raw_pointer ++);
-                    if (utf8_chunk == 0xFF) {
-                        parser->error = YAML_READER_ERROR;
-                        return 0;
-                    }
-                    ch = utf8_chunk;
-                    parser->raw_unread --;
-                    while (-- utf8_length) {
-                        utf8_chunk = UTF8_NEXT_CHUNK(parser->raw_pointer ++);
-                        if (utf8_chunk == 0xFF) {
-                            parser->error = YAML_READER_ERROR;
-                            return 0;
-                        }
-                        ch = ch << 6 + utf8_chunk;
-                        parser->raw_unread --;
+                    value = (
+                        (utf8_octet & 0x80) == 0x00 ? utf8_octet & 0x7F :
+                        (utf8_octet & 0xE0) == 0xC0 ? utf8_octet & 0x1F :
+                        (utf8_octet & 0xF0) == 0xE0 ? utf8_octet & 0x0F :
+                        (utf8_octet & 0xF8) == 0xF0 ? utf8_octet & 0x07 : 0);
+
+                    /* Check and decode the trailing octets. */
+
+                    for (k = 1; k < utf8_length; k ++)
+                    {
+                        utf8_octet = parser->raw_pointer[k];
+
+                        /* Check if the octet is valid. */
+
+                        if ((utf8_octet & 0xC0) != 0x80)
+                            return yaml_parser_set_reader_error(parser,
+                                    "Invalid trailing UTF-8 octet");
+
+                        /* Decode the octet. */
+
+                        value = (value << 6) + (utf8_octet & 0x3F);
                     }
 
+                    /* Check the length of the sequence against the value. */
+
+                    if (!((utf8_length == 1) ||
+                            (utf8_length == 2 && value >= 0x80) ||
+                            (utf8_length == 3 && value >= 0x800) ||
+                            (utf8_length == 4 && value >= 0x10000)))
+                        return yaml_parser_set_reader_error(parser,
+                                "Invalid length of a UTF-8 sequence");
+
+                    /* Check the range of the value. */
+
+                    if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
+                        return yaml_parser_set_reader_error(parser,
+                                "Invalid Unicode character");
+
+                    parser->raw_pointer += utf8_length;
+                    parser->raw_unread -= utf8_length;
+                    parser->offset += utf8_length;
+
                     break;
                 
                 case YAML_UTF16LE_ENCODING:
+                case YAML_UTF16BE_ENCODING:
 
-                    /* Check if the raw buffer contains an incomplete character. */
+                    low = (parser->encoding == YAML_UTF16LE_ENCODING ? 0 : 1);
+                    high = (parser->encoding == YAML_UTF16LE_ENCODING ? 1 : 0);
+
+                    /*
+                     * The UTF-16 encoding is not as simple as one might
+                     * naively think.  Check RFC 2781
+                     * (http://www.ietf.org/rfc/rfc2781.txt).
+                     *
+                     * Normally, two subsequent bytes describe a Unicode
+                     * character.  However a special technique (called a
+                     * surrogate pair) is used for specifying character
+                     * values larger than 0xFFFF.
+                     *
+                     * A surrogate pair consists of two pseudo-characters:
+                     *      high surrogate area (0xD800-0xDBFF)
+                     *      low surrogate area (0xDC00-0xDFFF)
+                     *
+                     * The following formulas are used for decoding
+                     * and encoding characters using surrogate pairs:
+                     * 
+                     *  U  = U' + 0x10000   (0x01 00 00 <= U <= 0x10 FF FF)
+                     *  U' = yyyyyyyyyyxxxxxxxxxx   (0 <= U' <= 0x0F FF FF)
+                     *  W1 = 110110yyyyyyyyyy
+                     *  W2 = 110111xxxxxxxxxx
+                     *
+                     * where U is the character value, W1 is the high surrogate
+                     * area, W2 is the low surrogate area.
+                     */
+
+                    /* Check for incomplete UTF-16 character. */
 
                     if (parser->raw_unread < 2) {
                         if (parser->eof) {
-                            parser->error = YAML_READER_ERROR;
-                            return 0;
+                            return yaml_parser_set_reader_error(parser,
+                                    "Incomplete UTF-16 character");
                         }
                         incomplete = 1;
+                        break;
                     }
 
-                    /* Get the current character. */
+                    /* Get the character. */
 
-                    ch = UTF16LE_CHAR(parser->raw_pointer);
-                    parser->raw_pointer += 2;
-                    parser->raw_unread -= 2;
+                    value = parser->raw_pointer[low]
+                        + (parser->raw_pointer[high] << 8);
 
-                    break;
+                    /* Check for unexpected low surrogate area. */
 
-                case YAML_UTF16BE_ENCODING:
+                    if ((value & 0xFC00) == 0xDC00)
+                        return yaml_parser_set_reader_error(parser,
+                                "Unexpected low surrogate area");
 
-                    /* Check if the raw buffer contains an incomplete character. */
+                    /* Check for a high surrogate area. */
 
-                    if (parser->raw_unread < 2) {
-                        if (parser->eof) {
-                            parser->error = YAML_READER_ERROR;
-                            return 0;
+                    if ((value & 0xFC00) == 0xD800) {
+
+                        /* Check for incomplete surrogate pair. */
+
+                        if (parser->raw_unread < 4) {
+                            if (parser->eof) {
+                                return yaml_parser_set_reader_error(parser,
+                                        "Incomplete UTF-16 surrogate pair");
+                            }
+                            incomplete = 1;
+                            break;
                         }
-                        incomplete = 1;
-                    }
 
-                    /* Get the current character. */
+                        /* Get the next character. */
 
-                    ch = UTF16BE_CHAR(parser->raw_pointer);
-                    parser->raw_pointer += 2;
-                    parser->raw_unread -= 2;
+                        unsigned int value2 = parser->raw_pointer[low+2]
+                            + (parser->raw_pointer[high+2] << 8);
+
+                        /* Check for a low surrogate area. */
+
+                        if ((value2 & 0xFC00) != 0xDC00)
+                            return yaml_parser_set_reader_error(parser,
+                                    "Expected low surrogate area");
+
+                        /* Generate the value of the surrogate pair. */
+
+                        value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
+
+                        parser->raw_pointer += 4;
+                        parser->raw_unread -= 4;
+                        parser->offset += 4;
+                    }
+
+                    else {
+                        parser->raw_pointer += 2;
+                        parser->raw_unread -= 2;
+                        parser->offset += 4;
+                    }
 
                     break;
             }
 
+            /* Check if the raw buffer contains enough bytes to form a character. */
+
+            if (incomplete) break;
+
             /*
              * Check if the character is in the allowed range:
              *      #x9 | #xA | #xD | [#x20-#x7E]               (8 bit)
@@ -197,39 +292,42 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
              *      | [#x10000-#x10FFFF]                        (32 bit)
              */
 
-            if (! (ch == 0x09 || ch == 0x0A || ch == 0x0D
-                        || (ch >= 0x20 && ch <= 0x7E)
-                        || (ch == 0x85) || (ch >= 0xA0 && ch <= 0xD7FF)
-                        || (ch >= 0xE000 && ch <= 0xFFFD)
-                        || (ch >= 0x10000 && ch <= 0x10FFFF))) {
-                parser->error = YAML_READER_ERROR;
-                return 0;
-            }
+            if (! (value == 0x09 || value == 0x0A || value == 0x0D
+                        || (value >= 0x20 && value <= 0x7E)
+                        || (value == 0x85) || (value >= 0xA0 && value <= 0xD7FF)
+                        || (value >= 0xE000 && value <= 0xFFFD)
+                        || (value >= 0x10000 && value <= 0x10FFFF)))
+                return yaml_parser_set_reader_error(parser,
+                        "Control characters are not allowed");
 
             /* Finally put the character into the buffer. */
 
             /* 0000 0000-0000 007F -> 0xxxxxxx */
-            if (ch <= 0x7F) {
-                *(parser->buffer_end++) = ch;
+            if (value <= 0x7F) {
+                *(parser->buffer_end++) = value;
             }
             /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
-            else if (ch <= 0x7FF) {
-                *(parser->buffer_end++) = 0xC0 + (ch >> 6) & 0x1F;
-                *(parser->buffer_end++) = 0x80 + ch & 0x3F;
+            else if (value <= 0x7FF) {
+                *(parser->buffer_end++) = 0xC0 + (value >> 6);
+                *(parser->buffer_end++) = 0x80 + value & 0x3F;
             }
             /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
-            else if (ch <= 0xFFFF) {
-                *(parser->buffer_end++) = 0x80 + ch & 0x3F;
-                *(parser->buffer_end++) = 0xC0 + (ch >> 6) & 0x1F;
-                
+            else if (value <= 0xFFFF) {
+                *(parser->buffer_end++) = 0xE0 + (value >> 12);
+                *(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F;
+                *(parser->buffer_end++) = 0x80 + value & 0x3F;
             }
             /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
             else {
+                *(parser->buffer_end++) = 0xF0 + (value >> 18);
+                *(parser->buffer_end++) = 0x80 + (value >> 12) & 0x3F;
+                *(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F;
+                *(parser->buffer_end++) = 0x80 + value & 0x3F;
             }
         }
-        
     }
 
+    return 1;
 }
 
 /*
@@ -237,12 +335,16 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
  * found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
  */
 
+#define BOM_UTF8    "\xef\xbb\xbf"
+#define BOM_UTF16LE "\xff\xfe"
+#define BOM_UTF16BE "\xfe\xff"
+
 int
 yaml_parser_determine_encoding(yaml_parser_t *parser)
 {
     /* Ensure that we had enough bytes in the raw buffer. */
 
-    while (!parser->eof && parser->raw_unread < 2) {
+    while (!parser->eof && parser->raw_unread < 3) {
         if (!yaml_parser_update_raw_buffer(parser)) {
             return 0;
         }
@@ -250,15 +352,29 @@ yaml_parser_determine_encoding(yaml_parser_t *parser)
 
     /* Determine the encoding. */
 
-    if (parser->raw_unread >= 2 && IS_UTF16BE_BOM(parser->raw_pointer)) {
+    if (parser->raw_unread >= 2
+            && !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
+        parser->encoding = YAML_UTF16LE_ENCODING;
+        parser->raw_pointer += 2;
+        parser->raw_unread -= 2;
+    }
+    else if (parser->raw_unread >= 2
+            && !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
         parser->encoding = YAML_UTF16BE_ENCODING;
+        parser->raw_pointer += 2;
+        parser->raw_unread -= 2;
     }
-    else if (parser->raw_unread >= 2 && IS_UTF16LE_BOM(parser->raw_pointer)) {
-        parser->encoding = YAML_UTF16LE_ENCODING;
+    else if (parser->raw_unread >= 3
+            && !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
+        parser->encoding = YAML_UTF8_ENCODING;
+        parser->raw_pointer += 3;
+        parser->raw_unread -= 3;
     }
     else {
         parser->encoding = YAML_UTF8_ENCODING;
     }
+
+    return 1;
 }
 
 /*
@@ -291,8 +407,7 @@ yaml_parser_update_raw_buffer(yaml_parser_t *parser)
                 parser->raw_buffer + parser->raw_unread,
                 YAML_RAW_BUFFER_SIZE - parser->raw_unread,
                 &size_read)) {
-        parser->error = YAML_READER_ERROR;
-        return 0;
+        return yaml_parser_set_reader_error(parser, "Input error");
     }
     parser->raw_unread += size_read;
     if (!size_read) {
diff --git a/tests/Makefile.am b/tests/Makefile.am
index ea70ba5..8f69907 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,4 +1,4 @@
 AM_CPPFLAGS = -I$(top_srcdir)/include
 LDADD = $(top_builddir)/src/libyaml.la
-TESTS = test-version
-check_PROGRAMS = test-version
+TESTS = test-version test-reader
+check_PROGRAMS = test-version test-reader
diff --git a/tests/test-reader.c b/tests/test-reader.c
new file mode 100644
index 0000000..b1a7426
--- /dev/null
+++ b/tests/test-reader.c
@@ -0,0 +1,144 @@
+#include <yaml/yaml.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+/*
+ * Test cases are stolen from
+ * http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+ */
+
+typedef struct {
+    char *title;
+    char *test;
+    int result;
+} test_case;
+
+test_case utf8_sequences[] = {
+        /* {"title", "test 1|test 2|...|test N!", (0 or 1)}, */
+
+        {"a simple test", "'test' is '\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb2\xd0\xb5\xd1\x80\xd0\xba\xd0\xb0' in Russian!", 1},
+        {"an empty line", "!", 1},
+
+        {"u-0 is a control character", "\x00!", 0},
+        {"u-80 is a control character", "\xc2\x80!", 0},
+        {"u-800 is valid", "\xe0\xa0\x80!", 1},
+        {"u-10000 is valid", "\xf0\x90\x80\x80!", 1},
+        {"5 bytes sequences are not allowed", "\xf8\x88\x80\x80\x80!", 0},
+        {"6 bytes sequences are not allowed", "\xfc\x84\x80\x80\x80\x80!", 0},
+
+        {"u-7f is a control character", "\x7f!", 0},
+        {"u-7FF is valid", "\xdf\xbf!", 1},
+        {"u-FFFF is a control character", "\xef\xbf\xbf!", 0},
+        {"u-1FFFFF is too large", "\xf7\xbf\xbf\xbf!", 0},
+        {"u-3FFFFFF is 5 bytes", "\xfb\xbf\xbf\xbf\xbf!", 0},
+        {"u-7FFFFFFF is 6 bytes", "\xfd\xbf\xbf\xbf\xbf\xbf!", 0},
+
+        {"u-D7FF", "\xed\x9f\xbf!", 1},
+        {"u-E000", "\xee\x80\x80!", 1},
+        {"u-FFFD", "\xef\xbf\xbd!", 1},
+        {"u-10FFFF", "\xf4\x8f\xbf\xbf!", 1},
+        {"u-110000", "\xf4\x90\x80\x80!", 0},
+
+        {"first continuation byte", "\x80!", 0},
+        {"last continuation byte", "\xbf!", 0},
+
+        {"2 continuation bytes", "\x80\xbf!", 0},
+        {"3 continuation bytes", "\x80\xbf\x80!", 0},
+        {"4 continuation bytes", "\x80\xbf\x80\xbf!", 0},
+        {"5 continuation bytes", "\x80\xbf\x80\xbf\x80!", 0},
+        {"6 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf!", 0},
+        {"7 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf\x80!", 0},
+
+        {"sequence of all 64 possible continuation bytes",
+         "\x80|\x81|\x82|\x83|\x84|\x85|\x86|\x87|\x88|\x89|\x8a|\x8b|\x8c|\x8d|\x8e|\x8f|"
+         "\x90|\x91|\x92|\x93|\x94|\x95|\x96|\x97|\x98|\x99|\x9a|\x9b|\x9c|\x9d|\x9e|\x9f|"
+         "\xa0|\xa1|\xa2|\xa3|\xa4|\xa5|\xa6|\xa7|\xa8|\xa9|\xaa|\xab|\xac|\xad|\xae|\xaf|"
+         "\xb0|\xb1|\xb2|\xb3|\xb4|\xb5|\xb6|\xb7|\xb8|\xb9|\xba|\xbb|\xbc|\xbd|\xbe|\xbf!", 0},
+        {"32 first bytes of 2-byte sequences {0xc0-0xdf}",
+         "\xc0 |\xc1 |\xc2 |\xc3 |\xc4 |\xc5 |\xc6 |\xc7 |\xc8 |\xc9 |\xca |\xcb |\xcc |\xcd |\xce |\xcf |"
+         "\xd0 |\xd1 |\xd2 |\xd3 |\xd4 |\xd5 |\xd6 |\xd7 |\xd8 |\xd9 |\xda |\xdb |\xdc |\xdd |\xde |\xdf !", 0},
+        {"16 first bytes of 3-byte sequences {0xe0-0xef}",
+         "\xe0 |\xe1 |\xe2 |\xe3 |\xe4 |\xe5 |\xe6 |\xe7 |\xe8 |\xe9 |\xea |\xeb |\xec |\xed |\xee |\xef !", 0},
+        {"8 first bytes of 4-byte sequences {0xf0-0xf7}", "\xf0 |\xf1 |\xf2 |\xf3 |\xf4 |\xf5 |\xf6 |\xf7 !", 0},
+        {"4 first bytes of 5-byte sequences {0xf8-0xfb}", "\xf8 |\xf9 |\xfa |\xfb !", 0},
+        {"2 first bytes of 6-byte sequences {0xfc-0xfd}", "\xfc |\xfd !", 0},
+
+        {"sequences with last byte missing {u-0}",
+         "\xc0|\xe0\x80|\xf0\x80\x80|\xf8\x80\x80\x80|\xfc\x80\x80\x80\x80!", 0},
+        {"sequences with last byte missing {u-...FF}",
+         "\xdf|\xef\xbf|\xf7\xbf\xbf|\xfb\xbf\xbf\xbf|\xfd\xbf\xbf\xbf\xbf!", 0},
+
+        {"impossible bytes", "\xfe|\xff|\xfe\xfe\xff\xff!", 0},
+
+        {"overlong sequences {u-2f}",
+         "\xc0\xaf|\xe0\x80\xaf|\xf0\x80\x80\xaf|\xf8\x80\x80\x80\xaf|\xfc\x80\x80\x80\x80\xaf!", 0},
+
+        {"maximum overlong sequences",
+         "\xc1\xbf|\xe0\x9f\xbf|\xf0\x8f\xbf\xbf|\xf8\x87\xbf\xbf\xbf|\xfc\x83\xbf\xbf\xbf\xbf!", 0},
+
+        {"overlong representation of the NUL character",
+         "\xc0\x80|\xe0\x80\x80|\xf0\x80\x80\x80|\xf8\x80\x80\x80\x80|\xfc\x80\x80\x80\x80\x80!", 0},
+
+        {"single UTF-16 surrogates",
+         "\xed\xa0\x80|\xed\xad\xbf|\xed\xae\x80|\xed\xaf\xbf|\xed\xb0\x80|\xed\xbe\x80|\xed\xbf\xbf!", 0},
+
+        {"paired UTF-16 surrogates",
+         "\xed\xa0\x80\xed\xb0\x80|\xed\xa0\x80\xed\xbf\xbf|\xed\xad\xbf\xed\xb0\x80|"
+         "\xed\xad\xbf\xed\xbf\xbf|\xed\xae\x80\xed\xb0\x80|\xed\xae\x80\xed\xbf\xbf|"
+         "\xed\xaf\xbf\xed\xb0\x80|\xed\xaf\xbf\xed\xbf\xbf!", 0},
+
+        {"other illegal code positions", "\xef\xbf\xbe|\xef\xbf\xbf!", 0},
+
+        {NULL, NULL, 0}
+};
+
+int check_utf8_sequences(void)
+{
+    yaml_parser_t *parser;
+    int failed = 0;
+    int k;
+    printf("checking utf-8 sequences...\n");
+    for (k = 0; utf8_sequences[k].test; k++) {
+        char *title = utf8_sequences[k].title;
+        int check = utf8_sequences[k].result;
+        int result;
+        char *start = utf8_sequences[k].test;
+        char *end = start;
+        printf("\t%s:\n", title);
+        while(1) {
+            while (*end != '|' && *end != '!') end++;
+            parser = yaml_parser_new();
+            assert(parser);
+            yaml_parser_set_input_string(parser, (unsigned char *)start, end-start);
+            result = yaml_parser_update_buffer(parser, end-start);
+            if (result != check) {
+                printf("\t\t- ");
+                failed ++;
+            }
+            else {
+                printf("\t\t+ ");
+            }
+            if (!parser->error) {
+                printf("(no error)\n");
+            }
+            else if (parser->error == YAML_READER_ERROR) {
+                printf("(reader error: %s at %d)\n", parser->problem, parser->problem_offset);
+            }
+            if (*end == '!') break;
+            start = ++end;
+            yaml_parser_delete(parser);
+        };
+        printf("\n");
+    }
+    printf("checking utf-8 sequences: %d fail(s)\n", failed);
+    return failed;
+}
+
+
+int
+main(void)
+{
+    return check_utf8_sequences();
+}