Add a generator feature to validate UTF8 strings as they are written to output. closes #25

5 files changed, 78 insertions, 23 deletions

diff --git a/reformatter/json_reformat.c b/reformatter/json_reformat.c
index d55ddaa..ba55748 100644
--- a/reformatter/json_reformat.c
+++ b/reformatter/json_reformat.c
@@ -24,67 +24,58 @@
 static int reformat_null(void * ctx)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_null(g);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_null(g);
 }
 
 static int reformat_boolean(void * ctx, int boolean)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_bool(g, boolean);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_bool(g, boolean);
 }
 
 static int reformat_number(void * ctx, const char * s, size_t l)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_number(g, s, l);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_number(g, s, l);
 }
 
 static int reformat_string(void * ctx, const unsigned char * stringVal,
                            size_t stringLen)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_string(g, stringVal, stringLen);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_string(g, stringVal, stringLen);
 }
 
 static int reformat_map_key(void * ctx, const unsigned char * stringVal,
                             size_t stringLen)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_string(g, stringVal, stringLen);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_string(g, stringVal, stringLen);
 }
 
 static int reformat_start_map(void * ctx)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_map_open(g);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_map_open(g);
 }
 
 
 static int reformat_end_map(void * ctx)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_map_close(g);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_map_close(g);
 }
 
 static int reformat_start_array(void * ctx)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_array_open(g);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_array_open(g);
 }
 
 static int reformat_end_array(void * ctx)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_array_close(g);
-    return 1;
+    return yajl_gen_status_ok == yajl_gen_array_close(g);
 }
 
 static yajl_callbacks callbacks = {
@@ -126,6 +117,7 @@ main(int argc, char ** argv)
 
     g = yajl_gen_alloc(NULL);
     yajl_gen_config(g, yajl_gen_beautify, 1);
+    yajl_gen_config(g, yajl_gen_validate_utf8, 1);
 
     /* ok.  open file.  let's read and parse */
     hand = yajl_alloc(&callbacks, NULL, (void *) g);
@@ -145,7 +137,8 @@ main(int argc, char ** argv)
                     yajl_config(hand, yajl_dont_validate_strings, 1);
                     break;
                 default:
-                    fprintf(stderr, "unrecognized option: '%c'\n\n", argv[a][i]);
+                    fprintf(stderr, "unrecognized option: '%c'\n\n",
+                            argv[a][i]);
                     usage(argv[0]);
             }
         }
@@ -156,8 +149,7 @@ main(int argc, char ** argv)
     }
 
 
-
-	for (;;) {
+    for (;;) {
         rd = fread((void *) fileData, 1, sizeof(fileData) - 1, stdin);
 
         if (rd == 0) {
diff --git a/src/api/yajl_gen.h b/src/api/yajl_gen.h
index b7c475f..61c0bd1 100644
--- a/src/api/yajl_gen.h
+++ b/src/api/yajl_gen.h
@@ -49,7 +49,11 @@ extern "C" {
         yajl_gen_invalid_number,
         /** A print callback was passed in, so there is no internal
          * buffer to get from */
-        yajl_gen_no_buf
+        yajl_gen_no_buf,
+        /** returned from yajl_gen_string() when the yajl_gen_validate_utf8
+         *  option is enabled and an invalid was passed by client code.
+         */
+        yajl_gen_invalid_string
     } yajl_gen_status;
 
     /** an opaque handle to a generator */
@@ -81,7 +85,13 @@ extern "C" {
          * example:
          *   yajl_gen_config(g, yajl_gen_print_callback, myFunc, myVoidPtr);
          */
-        yajl_gen_print_callback = 0x04
+        yajl_gen_print_callback = 0x04,
+        /**
+         * Normally the generator does not validate that strings you
+         * pass to it via yajl_gen_string() are valid UTF8.  Enabling
+         * this option will cause it to do so.
+         */
+        yajl_gen_validate_utf8 = 0x08
     } yajl_gen_option;
 
     /** allow the modification of generator options subsequent to handle
diff --git a/src/yajl_encode.c b/src/yajl_encode.c
index eaa2214..a0ab8d1 100644
--- a/src/yajl_encode.c
+++ b/src/yajl_encode.c
@@ -177,3 +177,45 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str,
     }
     yajl_buf_append(buf, str + beg, end - beg);
 }
+
+#define ADV_PTR s++; if (!(len--)) return 0;
+
+int yajl_string_validate_utf8(const unsigned char * s, size_t len)
+{
+    if (!len) return 1;
+    if (!s) return 0;
+    
+    while (len--) {
+        /* single byte */
+        if (*s <= 0x7f) {
+            /* noop */
+        }
+        /* two byte */ 
+        else if ((*s >> 5) == 0x6) {
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+        }
+        /* three byte */
+        else if ((*s >> 4) == 0x0e) {
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+        }
+        /* four byte */        
+        else if ((*s >> 3) == 0x1e) {
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+        } else {
+            return 0;
+        }
+        
+        s++;
+    }
+    
+    return 1;
+}
diff --git a/src/yajl_encode.h b/src/yajl_encode.h
index d605e31..25fbd70 100644
--- a/src/yajl_encode.h
+++ b/src/yajl_encode.h
@@ -31,4 +31,6 @@ void yajl_string_encode(yajl_buf buf, const unsigned char * str,
 void yajl_string_decode(yajl_buf buf, const unsigned char * str,
                         size_t length);
 
+int yajl_string_validate_utf8(const unsigned char * s, size_t len);
+
 #endif
diff --git a/src/yajl_gen.c b/src/yajl_gen.c
index d50b492..dd693e6 100644
--- a/src/yajl_gen.c
+++ b/src/yajl_gen.c
@@ -56,6 +56,7 @@ yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...)
 
     switch(opt) {
         case yajl_gen_beautify:
+        case yajl_gen_validate_utf8:
             if (va_arg(ap, int)) g->flags |= opt;
             else g->flags &= ~opt;
             break;
@@ -241,6 +242,14 @@ yajl_gen_status
 yajl_gen_string(yajl_gen g, const unsigned char * str,
                 size_t len)
 {
+    // if validation is enabled, check that the string is valid utf8
+    // XXX: This checking could be done a little faster, in the same pass as
+    // the string encoding
+    if (g->flags & yajl_gen_validate_utf8) {
+        if (!yajl_string_validate_utf8(str, len)) {
+            return yajl_gen_invalid_string;
+        }
+    }
     ENSURE_VALID_STATE; INSERT_SEP; INSERT_WHITESPACE;
     g->print(g->ctx, "\"", 1);
     yajl_string_encode2(g->print, g->ctx, str, len);