Add token constructors and destructors.

Add YAML_DECLARE to the API declarations (Thanks to Peter Murphy for suggestion).


git-svn-id: http://svn.pyyaml.org/libyaml/trunk@183 18f92427-320e-0410-9341-c67f048884a3

4 files changed, 615 insertions, 156 deletions

diff --git a/doc/doxygen.cfg b/doc/doxygen.cfg
index f7a9aef..a58bb17 100644
--- a/doc/doxygen.cfg
+++ b/doc/doxygen.cfg
@@ -174,12 +174,12 @@ PERLMOD_MAKEVAR_PREFIX =
 # Configuration options related to the preprocessor   
 #---------------------------------------------------------------------------
 ENABLE_PREPROCESSING   = YES
-MACRO_EXPANSION        = NO
-EXPAND_ONLY_PREDEF     = NO
+MACRO_EXPANSION        = YES
+EXPAND_ONLY_PREDEF     = YES
 SEARCH_INCLUDES        = YES
 INCLUDE_PATH           = 
 INCLUDE_FILE_PATTERNS  = 
-PREDEFINED             = 
+PREDEFINED             = "YAML_DECLARE(type)=type"
 EXPAND_AS_DEFINED      = 
 SKIP_FUNCTION_MACROS   = YES
 #---------------------------------------------------------------------------
diff --git a/include/yaml/yaml.h b/include/yaml/yaml.h
index cfdeacf..6faf7df 100644
--- a/include/yaml/yaml.h
+++ b/include/yaml/yaml.h
@@ -20,6 +20,27 @@ extern "C" {
 #include <string.h>
 
 /**
+ * @defgroup Export Definitions
+ * @{
+ */
+
+/** The public API declaration. */
+
+#ifdef WIN32
+#   if defined(YAML_DECLARE_STATIC)
+#       define  YAML_DECLARE(type)  type
+#   elif defined(YAML_DECLARE_EXPORT)
+#       define  YAML_DECLARE(type)  __declspec(dllexport) type
+#   else
+#       define  YAML_DECLARE(type)  __declspec(dllimport) type
+#   endif
+#else
+#   define  YAML_DECLARE(type)  type
+#endif
+
+/** @} */
+
+/**
  * @defgroup version Version Information
  * @{
  */
@@ -32,7 +53,7 @@ extern "C" {
  * number, and @c Z is the patch version number.
  */
 
-const char *
+YAML_DECLARE(const char *)
 yaml_get_version_string(void);
 
 /**
@@ -43,7 +64,7 @@ yaml_get_version_string(void);
  * @param[out]  patch   Patch version number.
  */
 
-void
+YAML_DECLARE(void)
 yaml_get_version(int *major, int *minor, int *patch);
 
 /** @} */
@@ -53,7 +74,7 @@ yaml_get_version(int *major, int *minor, int *patch);
  * @{
  */
 
-/** The character type. */
+/** The character type (UTF-8 octet). */
 typedef unsigned char yaml_char_t;
 
 /** The stream encoding. */
@@ -78,31 +99,63 @@ typedef enum {
     YAML_EMITTER_ERROR
 } yaml_error_type_t;
 
+/** The pointer position. */
+typedef struct {
+    /** The position index. */
+    size_t index;
+
+    /** The position line. */
+    size_t line;
+
+    /** The position column. */
+    size_t column;
+} yaml_mark_t;
+
 /** @} */
 
-/*
+/**
+ * @defgroup Node Styles
+ * @{
+ */
 
+/** Scalar styles. */
 typedef enum {
     YAML_ANY_SCALAR_STYLE,
+
     YAML_PLAIN_SCALAR_STYLE,
+
     YAML_SINGLE_QUOTED_SCALAR_STYLE,
     YAML_DOUBLE_QUOTED_SCALAR_STYLE,
+
     YAML_LITERAL_SCALAR_STYLE,
     YAML_FOLDED_SCALAR_STYLE
 } yaml_scalar_style_t;
 
+
+/** Sequence styles. */
 typedef enum {
     YAML_ANY_SEQUENCE_STYLE,
+
     YAML_BLOCK_SEQUENCE_STYLE,
     YAML_FLOW_SEQUENCE_STYLE
 } yaml_sequence_style_t;
 
+/** Mapping styles. */
 typedef enum {
     YAML_ANY_MAPPING_STYLE,
+
     YAML_BLOCK_MAPPING_STYLE,
     YAML_FLOW_MAPPING_STYLE
 } yaml_mapping_style_t;
 
+/** @} */
+
+/**
+ * @defgroup Tokens
+ * @{
+ */
+
+/** Token types. */
 typedef enum {
     YAML_STREAM_START_TOKEN,
     YAML_STREAM_END_TOKEN,
@@ -132,62 +185,259 @@ typedef enum {
     YAML_SCALAR_TOKEN
 } yaml_token_type_t;
 
-typedef enum {
-    YAML_STREAM_START_EVENT,
-    YAML_STREAM_END_EVENT,
+/** The token structure. */
+typedef struct {
 
-    YAML_DOCUMENT_START_EVENT,
-    YAML_DOCUMENT_END_EVENT,
+    /** The token type. */
+    yaml_token_type_t type;
 
-    YAML_ALIAS_EVENT,
-    YAML_SCALAR_EVENT,
+    /** The token data. */
+    union {
 
-    YAML_SEQUENCE_START_EVENT,
-    YAML_SEQUENCE_END_EVENT,
+        /** The stream encoding (for @c YAML_STREAM_START_TOKEN). */
+        yaml_encoding_t encoding;
 
-    YAML_MAPPING_START_EVENT,
-    YAML_MAPPING_END_EVENT
-} yaml_event_type_t;
+        /** The anchor (for @c YAML_ALIAS_TOKEN and @c YAML_ANCHOR_TOKEN). */
+        yaml_char_t *anchor;
 
-typedef struct {
-    size_t offset;
-    size_t index;
-    size_t line;
-    size_t column;
-} yaml_mark_t;
+        /** The tag (for @c YAML_TAG_TOKEN). */
+        struct {
+            /** The tag handle. */
+            yaml_char_t *handle;
 
-typedef struct {
-    yaml_error_type_t type;
-    char *context;
-    yaml_mark_t context_mark;
-    char *problem;
-    yaml_mark_t problem_mark;
-} yaml_error_t;
+            /** The tag suffix. */
+            yaml_char_t *suffix;
+        } tag;
 
-typedef struct {
-    yaml_token_type_t type;
-    union {
-        yaml_encoding_t encoding;
-        char *anchor;
-        char *tag;
+        /** The scalar value (for @c YAML_SCALAR_TOKEN). */
         struct {
-            char *value;
+
+            /** The scalar value. */
+            yaml_char_t *value;
+
+            /** The length of the scalar value. */
             size_t length;
+
+            /** The scalar style. */
             yaml_scalar_style_t style;
         } scalar;
+
+        /** The version directive (for @c YAML_VERSION_DIRECTIVE_TOKEN). */
         struct {
+            /** The major version number. */
             int major;
+
+            /** The minor version number. */
             int minor;
-        } version;
+        } version_directive;
+
+        /** The tag directive (for @c YAML_TAG_DIRECTIVE_TOKEN). */
         struct {
-          char *handle;
-          char *prefix;
-        } tag_pair;
+            /** The tag handle. */
+            yaml_char_t *handle;
+
+            /** The tag prefix. */
+            yaml_char_t *prefix;
+        } tag_directive;
     } data;
+
+    /** The beginning of the token. */
     yaml_mark_t start_mark;
+
+    /** The end of the token. */
     yaml_mark_t end_mark;
+
 } yaml_token_t;
 
+/**
+ * Create a new token without assigning any data.
+ *
+ * This function can be used for constructing indicator tokens:
+ * @c YAML_DOCUMENT_START, @c YAML_DOCUMENT_END,
+ * @c YAML_BLOCK_SEQUENCE_START_TOKEN, @c YAML_BLOCK_MAPPING_START_TOKEN,
+ * @c YAML_BLOCK_END_TOKEN,
+ * @c YAML_FLOW_SEQUENCE_START_TOKEN, @c YAML_FLOW_SEQUENCE_END_TOKEN,
+ * @c YAML_FLOW_MAPPING_START_TOKEN, @c YAML_FLOW_MAPPING_END_TOKEN,
+ * @c YAML_BLOCK_ENTRY_TOKEN, @c YAML_FLOW_ENTRY_TOKEN,
+ * @c YAML_KEY_TOKEN, @c YAML_VALUE_TOKEN.
+ *
+ * @param[in]   type        The token type.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_token_new(yaml_token_type_t type,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_STREAM_START_TOKEN token with the specified encoding.
+ *
+ * @param[in]   encoding    The stream encoding.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_stream_start_token_new(yaml_encoding_t encoding,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_STREAM_END_TOKEN token.
+ *
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_stream_end_token_new(yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_VERSION_DIRECTIVE_TOKEN token with the specified
+ * version numbers.
+ *
+ * @param[in]   major       The major version number.
+ * @param[in]   minor       The minor version number.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_version_directive_token_new(int major, int minor,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_TAG_DIRECTIVE_TOKEN token with the specified tag
+ * handle and prefix.
+ *
+ * Note that the @a handle and the @a prefix pointers will be freed by
+ * the token descructor.
+ *
+ * @param[in]   handle      The tag handle.
+ * @param[in]   prefix      The tag prefix.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_tag_directive_token_new(yaml_char_t *handle, yaml_char_t *prefix,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_ALIAS_TOKEN token with the specified anchor.
+ *
+ * Note that the @a anchor pointer will be freed by the token descructor.
+ *
+ * @param[in]   anchor      The anchor.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_alias_token_new(yaml_char_t *anchor,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_ANCHOR_TOKEN token with the specified anchor.
+ *
+ * Note that the @a anchor pointer will be freed by the token descructor.
+ *
+ * @param[in]   anchor      The anchor.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_anchor_token_new(yaml_char_t *anchor,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_TAG_TOKEN token with the specified tag handle and
+ * suffix.
+ *
+ * Note that the @a handle and the @a suffix pointers will be freed by
+ * the token descructor.
+ *
+ * @param[in]   handle      The tag handle.
+ * @param[in]   suffix      The tag suffix.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_tag_token_new(yaml_char_t *handle, yaml_char_t *suffix,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Create a new @c YAML_SCALAR_TOKEN token with the specified scalar value,
+ * length, and style.
+ *
+ * Note that the scalar value may contain the @c NUL character, therefore
+ * the value length is also required.  The scalar value always ends with
+ * @c NUL.
+ *
+ * Note that the @a value pointer will be freed by the token descructor.
+ *
+ * @param[in]   value       The scalar value.
+ * @param[in]   length      The value length.
+ * @param[in]   style       The scalar style.
+ * @param[in]   start_mark  The beginning of the token.
+ * @param[in]   end_mark    The end of the token.
+ *
+ * @returns A new token object, or @c NULL on error.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_scalar_token_new(yaml_char_t *value, size_t length,
+        yaml_scalar_style_t style,
+        yaml_mark_t start_mark, yaml_mark_t end_mark);
+
+/**
+ * Destroy a token object.
+ *
+ * @param[in]   token   A token object.
+ */
+
+YAML_DECLARE(void)
+yaml_token_delete(yaml_token_t *token);
+
+/** @} */
+
+/*
+
+typedef enum {
+    YAML_STREAM_START_EVENT,
+    YAML_STREAM_END_EVENT,
+
+    YAML_DOCUMENT_START_EVENT,
+    YAML_DOCUMENT_END_EVENT,
+
+    YAML_ALIAS_EVENT,
+    YAML_SCALAR_EVENT,
+
+    YAML_SEQUENCE_START_EVENT,
+    YAML_SEQUENCE_END_EVENT,
+
+    YAML_MAPPING_START_EVENT,
+    YAML_MAPPING_END_EVENT
+} yaml_event_type_t;
+
 typedef struct {
     yaml_event_type_t type;
     union {
@@ -272,8 +522,13 @@ typedef int yaml_read_handler_t(void *data, unsigned char *buffer, size_t size,
  */
 
 typedef struct {
+    /** The string start pointer. */
     unsigned char *start;
+
+    /** The string end pointer. */
     unsigned char *end;
+
+    /** The string current position. */
     unsigned char *current;
 } yaml_string_input_t;
 
@@ -375,7 +630,7 @@ typedef struct {
  * @returns A new parser object; @c NULL on error.
  */
 
-yaml_parser_t *
+YAML_DECLARE(yaml_parser_t *)
 yaml_parser_new(void);
 
 /**
@@ -384,7 +639,7 @@ yaml_parser_new(void);
  * @param[in]   parser  A parser object.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_delete(yaml_parser_t *parser);
 
 /**
@@ -396,10 +651,10 @@ yaml_parser_delete(yaml_parser_t *parser);
  *
  * @param[in]   parser  A parser object.
  * @param[in]   input   A source data.
- * @param[in]   length  The length of the source data in bytes.
+ * @param[in]   size    The length of the source data in bytes.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_input_string(yaml_parser_t *parser,
         unsigned char *input, size_t size);
 
@@ -414,7 +669,7 @@ yaml_parser_set_input_string(yaml_parser_t *parser,
  * @param[in]   file    An open file.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_input_file(yaml_parser_t *parser, FILE *file);
 
 /**
@@ -425,17 +680,18 @@ yaml_parser_set_input_file(yaml_parser_t *parser, FILE *file);
  * @param[in]   data    Any application data for passing to the read handler.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_input(yaml_parser_t *parser,
         yaml_read_handler_t *handler, void *data);
 
 /**
  * Set the source encoding.
  *
+ * @param[in]   parser      A parser object.
  * @param[in]   encoding    The source encoding.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_encoding(yaml_parser_t *parser, yaml_encoding_t encoding);
 
 /** @} */
@@ -459,7 +715,7 @@ typedef struct {
  * or @c NULL if it failed.
  */
 
-void *
+YAML_DECLARE(void *)
 yaml_malloc(size_t size);
 
 /**
@@ -473,7 +729,7 @@ yaml_malloc(size_t size);
  * or @c NULL if it failed.
  */
 
-void *
+YAML_DECLARE(void *)
 yaml_realloc(void *ptr, size_t size);
 
 /**
@@ -483,7 +739,7 @@ yaml_realloc(void *ptr, size_t size);
  *                      valid.
  */
 
-void
+YAML_DECLARE(void)
 yaml_free(void *ptr);
 
 /** The size of the raw buffer. */
@@ -507,7 +763,7 @@ yaml_free(void *ptr);
  * @returns @c 1 on success, @c 0 on error.
  */
 
-int
+YAML_DECLARE(int)
 yaml_parser_update_buffer(yaml_parser_t *parser, size_t length);
 
 /** @} */
diff --git a/src/api.c b/src/api.c
index aa183af..50f118a 100644
--- a/src/api.c
+++ b/src/api.c
@@ -11,7 +11,7 @@
  * Allocate a dynamic memory block.
  */
 
-void *
+YAML_DECLARE(void *)
 yaml_malloc(size_t size)
 {
     return malloc(size ? size : 1);
@@ -21,7 +21,7 @@ yaml_malloc(size_t size)
  * Reallocate a dynamic memory block.
  */
 
-void *
+YAML_DECLARE(void *)
 yaml_realloc(void *ptr, size_t size)
 {
     return ptr ? realloc(ptr, size ? size : 1) : malloc(size ? size : 1);
@@ -31,7 +31,7 @@ yaml_realloc(void *ptr, size_t size)
  * Free a dynamic memory block.
  */
 
-void
+YAML_DECLARE(void)
 yaml_free(void *ptr)
 {
     if (ptr) free(ptr);
@@ -41,7 +41,7 @@ yaml_free(void *ptr)
  * Create a new parser object.
  */
 
-yaml_parser_t *
+YAML_DECLARE(yaml_parser_t *)
 yaml_parser_new(void)
 {
     yaml_parser_t *parser;
@@ -82,7 +82,7 @@ yaml_parser_new(void)
  * Destroy a parser object.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_delete(yaml_parser_t *parser)
 {
     assert(parser); /* Non-NULL parser object expected. */
@@ -136,7 +136,7 @@ yaml_file_read_handler(void *data, unsigned char *buffer, size_t size,
  * Set a string input.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_input_string(yaml_parser_t *parser,
         unsigned char *input, size_t size)
 {
@@ -156,7 +156,7 @@ yaml_parser_set_input_string(yaml_parser_t *parser,
  * Set a file input.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_input_file(yaml_parser_t *parser, FILE *file)
 {
     assert(parser); /* Non-NULL parser object expected. */
@@ -171,7 +171,7 @@ yaml_parser_set_input_file(yaml_parser_t *parser, FILE *file)
  * Set a generic input.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_input(yaml_parser_t *parser,
         yaml_read_handler_t *handler, void *data)
 {
@@ -187,7 +187,7 @@ yaml_parser_set_input(yaml_parser_t *parser,
  * Set the source encoding.
  */
 
-void
+YAML_DECLARE(void)
 yaml_parser_set_encoding(yaml_parser_t *parser, yaml_encoding_t encoding)
 {
     assert(parser); /* Non-NULL parser object expected. */
@@ -196,3 +196,207 @@ yaml_parser_set_encoding(yaml_parser_t *parser, yaml_encoding_t encoding)
     parser->encoding = encoding;
 }
 
+/*
+ * Create a token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_token_new(yaml_token_type_t type,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_malloc(sizeof(yaml_token_t));
+
+    if (!token) return NULL;
+
+    memset(token, 0, sizeof(yaml_token_t));
+
+    token->type = type;
+    token->start_mark = start_mark;
+    token->end_mark = end_mark;
+
+    return token;
+}
+
+/*
+ * Create a STREAM-START token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_stream_start_token(yaml_encoding_t encoding,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_STREAM_START_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    token->data.encoding = encoding;
+
+    return token;
+}
+
+/*
+ * Create a STREAM-END token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_stream_end_token(yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_STREAM_END_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    return token;
+}
+
+/*
+ * Create a VERSION-DIRECTIVE token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_version_directive_token_new(int major, int minor,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_VERSION_DIRECTIVE_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    token->data.version_directive.major = major;
+    token->data.version_directive.minor = minor;
+
+    return token;
+}
+
+/*
+ * Create a TAG-DIRECTIVE token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_tag_directive_token_new(yaml_char_t *handle, yaml_char_t *prefix,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_TAG_DIRECTIVE_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    token->data.tag_directive.handle = handle;
+    token->data.tag_directive.prefix = prefix;
+
+    return token;
+}
+
+/*
+ * Create an ALIAS token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_alias_token_new(yaml_char_t *anchor,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_ALIAS_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    token->data.anchor = anchor;
+
+    return token;
+}
+
+/*
+ * Create an ANCHOR token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_anchor_token_new(yaml_char_t *anchor,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_ANCHOR_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    token->data.anchor = anchor;
+
+    return token;
+}
+
+/*
+ * Create a TAG token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_tag_token_new(yaml_char_t *handle, yaml_char_t *suffix,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_TAG_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    token->data.tag.handle = handle;
+    token->data.tag.suffix = suffix;
+
+    return token;
+}
+
+/*
+ * Create a SCALAR token.
+ */
+
+YAML_DECLARE(yaml_token_t *)
+yaml_scalar_token_new(yaml_char_t *value, size_t length,
+        yaml_scalar_style_t style,
+        yaml_mark_t start_mark, yaml_mark_t end_mark)
+{
+    yaml_token_t *token = yaml_token_new(YAML_SCALAR_TOKEN,
+            start_mark, end_mark);
+
+    if (!token) return NULL;
+
+    token->data.scalar.value = value;
+    token->data.scalar.length = length;
+    token->data.scalar.style = style;
+
+    return token;
+}
+
+/*
+ * Destroy a token object.
+ */
+
+YAML_DECLARE(void)
+yaml_token_delete(yaml_token_t *token)
+{
+    assert(token);  /* Non-NULL token object expected. */
+
+    switch (token->type)
+    {
+        case YAML_TAG_DIRECTIVE_TOKEN:
+            yaml_free(token->data.tag_directive.handle);
+            yaml_free(token->data.tag_directive.prefix);
+            break;
+
+        case YAML_ALIAS_TOKEN:
+        case YAML_ANCHOR_TOKEN:
+            yaml_free(token->data.anchor);
+            break;
+
+        case YAML_TAG_TOKEN:
+            yaml_free(token->data.tag.handle);
+            yaml_free(token->data.tag.suffix);
+            break;
+
+        case YAML_SCALAR_TOKEN:
+            yaml_free(token->data.scalar.value);
+            break;
+    }
+
+    memset(token, 0, sizeof(yaml_token_t));
+
+    yaml_free(token);
+}
+
diff --git a/src/reader.c b/src/reader.c
index ac11323..e4e6f1a 100644
--- a/src/reader.c
+++ b/src/reader.c
@@ -11,7 +11,7 @@
  * Set the reader error and return 0.
  */
 
-int
+static int
 yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem,
         size_t offset, int value)
 {
@@ -23,6 +23,96 @@ yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem,
     return 0;
 }
 
+/*
+ * Update the raw buffer.
+ */
+
+static int
+yaml_parser_update_raw_buffer(yaml_parser_t *parser)
+{
+    size_t size_read = 0;
+
+    /* Return if the raw buffer is full. */
+
+    if (parser->raw_unread == YAML_RAW_BUFFER_SIZE) return 1;
+
+    /* Return on EOF. */
+
+    if (parser->eof) return 1;
+
+    /* Move the remaining bytes in the raw buffer to the beginning. */
+
+    if (parser->raw_unread && parser->raw_buffer < parser->raw_pointer) {
+        memmove(parser->raw_buffer, parser->raw_pointer, parser->raw_unread);
+    }
+    parser->raw_pointer = parser->raw_buffer;
+
+    /* Call the read handler to fill the buffer. */
+
+    if (!parser->read_handler(parser->read_handler_data,
+                parser->raw_buffer + parser->raw_unread,
+                YAML_RAW_BUFFER_SIZE - parser->raw_unread,
+                &size_read)) {
+        return yaml_parser_set_reader_error(parser, "Input error",
+                parser->offset, -1);
+    }
+    parser->raw_unread += size_read;
+    if (!size_read) {
+        parser->eof = 1;
+    }
+
+    return 1;
+}
+
+/*
+ * Determine the input stream encoding by checking the BOM symbol. If no BOM is
+ * found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
+ */
+
+#define BOM_UTF8    "\xef\xbb\xbf"
+#define BOM_UTF16LE "\xff\xfe"
+#define BOM_UTF16BE "\xfe\xff"
+
+static int
+yaml_parser_determine_encoding(yaml_parser_t *parser)
+{
+    /* Ensure that we had enough bytes in the raw buffer. */
+
+    while (!parser->eof && parser->raw_unread < 3) {
+        if (!yaml_parser_update_raw_buffer(parser)) {
+            return 0;
+        }
+    }
+
+    /* Determine the encoding. */
+
+    if (parser->raw_unread >= 2
+            && !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
+        parser->encoding = YAML_UTF16LE_ENCODING;
+        parser->raw_pointer += 2;
+        parser->raw_unread -= 2;
+        parser->offset += 2;
+    }
+    else if (parser->raw_unread >= 2
+            && !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
+        parser->encoding = YAML_UTF16BE_ENCODING;
+        parser->raw_pointer += 2;
+        parser->raw_unread -= 2;
+        parser->offset += 2;
+    }
+    else if (parser->raw_unread >= 3
+            && !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
+        parser->encoding = YAML_UTF8_ENCODING;
+        parser->raw_pointer += 3;
+        parser->raw_unread -= 3;
+        parser->offset += 3;
+    }
+    else {
+        parser->encoding = YAML_UTF8_ENCODING;
+    }
+
+    return 1;
+}
 
 /*
  * Ensure that the buffer contains at least length characters.
@@ -31,7 +121,7 @@ yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem,
  * The length is supposed to be significantly less that the buffer size.
  */
 
-int
+YAML_DECLARE(int)
 yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
 {
     /* If the EOF flag is set and the raw buffer is empty, do nothing. */
@@ -345,94 +435,3 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
     return 1;
 }
 
-/*
- * Determine the input stream encoding by checking the BOM symbol. If no BOM is
- * found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
- */
-
-#define BOM_UTF8    "\xef\xbb\xbf"
-#define BOM_UTF16LE "\xff\xfe"
-#define BOM_UTF16BE "\xfe\xff"
-
-int
-yaml_parser_determine_encoding(yaml_parser_t *parser)
-{
-    /* Ensure that we had enough bytes in the raw buffer. */
-
-    while (!parser->eof && parser->raw_unread < 3) {
-        if (!yaml_parser_update_raw_buffer(parser)) {
-            return 0;
-        }
-    }
-
-    /* Determine the encoding. */
-
-    if (parser->raw_unread >= 2
-            && !memcmp(parser->raw_pointer, BOM_UTF16LE, 2)) {
-        parser->encoding = YAML_UTF16LE_ENCODING;
-        parser->raw_pointer += 2;
-        parser->raw_unread -= 2;
-        parser->offset += 2;
-    }
-    else if (parser->raw_unread >= 2
-            && !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
-        parser->encoding = YAML_UTF16BE_ENCODING;
-        parser->raw_pointer += 2;
-        parser->raw_unread -= 2;
-        parser->offset += 2;
-    }
-    else if (parser->raw_unread >= 3
-            && !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
-        parser->encoding = YAML_UTF8_ENCODING;
-        parser->raw_pointer += 3;
-        parser->raw_unread -= 3;
-        parser->offset += 3;
-    }
-    else {
-        parser->encoding = YAML_UTF8_ENCODING;
-    }
-
-    return 1;
-}
-
-/*
- * Update the raw buffer.
- */
-
-int
-yaml_parser_update_raw_buffer(yaml_parser_t *parser)
-{
-    size_t size_read = 0;
-
-    /* Return if the raw buffer is full. */
-
-    if (parser->raw_unread == YAML_RAW_BUFFER_SIZE) return 1;
-
-    /* Return on EOF. */
-
-    if (parser->eof) return 1;
-
-    /* Move the remaining bytes in the raw buffer to the beginning. */
-
-    if (parser->raw_unread && parser->raw_buffer < parser->raw_pointer) {
-        memmove(parser->raw_buffer, parser->raw_pointer, parser->raw_unread);
-    }
-    parser->raw_pointer = parser->raw_buffer;
-
-    /* Call the read handler to fill the buffer. */
-
-    if (!parser->read_handler(parser->read_handler_data,
-                parser->raw_buffer + parser->raw_unread,
-                YAML_RAW_BUFFER_SIZE - parser->raw_unread,
-                &size_read)) {
-        return yaml_parser_set_reader_error(parser, "Input error",
-                parser->offset, -1);
-    }
-    parser->raw_unread += size_read;
-    if (!size_read) {
-        parser->eof = 1;
-    }
-
-    return 1;
-}
-