introduction of yajl_number callback to allow clients to handle arbitrary

precision numbers in JSON data.


git-svn-id: http://yajl-c.googlecode.com/svn/trunk@102 e775cfb5-b74b-0410-aad5-5bebe4a96390

11 files changed, 104 insertions, 51 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6bf9df7..3384791 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,7 +32,7 @@ PROJECT(YetAnotherJSONParser)
 
 SET (YAJL_MAJOR 0)
 SET (YAJL_MINOR 3)
-SET (YAJL_MICRO 0)
+SET (YAJL_MICRO 1)
 
 SET (YAJL_DIST_NAME "yajl-${YAJL_MAJOR}.${YAJL_MINOR}.${YAJL_MICRO}")
 
diff --git a/TODO b/TODO
index 7f18f3d..336f2d3 100644
--- a/TODO
+++ b/TODO
@@ -1,3 +1,5 @@
+* add a test for 0x1F bug
+* numeric overflow in integers and double
 * line and char offsets in the lexer and in error messages
 * numbers at eof bug. (lth: still pertinent?)
 * testing:
diff --git a/reformatter/json_reformat.c b/reformatter/json_reformat.c
index 73685b7..b630bc9 100644
--- a/reformatter/json_reformat.c
+++ b/reformatter/json_reformat.c
@@ -44,24 +44,17 @@ int reformat_null(void * ctx)
     return 1;
 }
 
-int reformat_boolean(void * ctx, int boolVal)
+int reformat_boolean(void * ctx, int boolean)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_bool(g, boolVal);
+    yajl_gen_bool(g, boolean);
     return 1;
 }
 
-int reformat_integer(void * ctx, long long integerVal)
+int reformat_number(void * ctx, const char * s, unsigned int l)
 {
     yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_integer(g, integerVal);
-    return 1;
-}
-
-int reformat_double(void * ctx, double doubleVal)
-{
-    yajl_gen g = (yajl_gen) ctx;
-    yajl_gen_double(g, doubleVal);
+    yajl_gen_number(g, s, l);
     return 1;
 }
 
@@ -113,8 +106,9 @@ int reformat_end_array(void * ctx)
 static yajl_callbacks callbacks = {
     reformat_null,
     reformat_boolean,
-    reformat_integer,
-    reformat_double,
+    NULL,
+    NULL,
+    reformat_number,
     reformat_string,
     reformat_start_map,
     reformat_map_key,
diff --git a/src/api/yajl_gen.h b/src/api/yajl_gen.h
index 145c2aa..334373b 100644
--- a/src/api/yajl_gen.h
+++ b/src/api/yajl_gen.h
@@ -79,9 +79,11 @@ extern "C" {
     /** free a generator handle */    
     void YAJL_API yajl_gen_free(yajl_gen handle);
 
-    yajl_gen_status YAJL_API yajl_gen_integer(yajl_gen hand,
-                                              long long int number);
+    yajl_gen_status YAJL_API yajl_gen_integer(yajl_gen hand, long int number);
     yajl_gen_status YAJL_API yajl_gen_double(yajl_gen hand, double number);
+    yajl_gen_status YAJL_API yajl_gen_number(yajl_gen hand,
+                                             const char * num,
+                                             unsigned int len);
     yajl_gen_status YAJL_API yajl_gen_string(yajl_gen hand,
                                              const unsigned char * str,
                                              unsigned int len);
diff --git a/src/api/yajl_parse.h b/src/api/yajl_parse.h
index 52f4d76..a6d5b1a 100644
--- a/src/api/yajl_parse.h
+++ b/src/api/yajl_parse.h
@@ -74,12 +74,27 @@ extern "C" {
      *  All callbacks return an integer.  If non-zero, the parse will
      *  continue.  If zero, the parse will be canceled and
      *  yajl_status_client_canceled will be returned from the parse.
+     *
+     *  Note about handling of numbers:
+     *    yajl will only convert numbers that can be represented in a double
+     *    or a long int.  All other numbers will be passed to the client
+     *    in string form using the yajl_number callback.  Furthermore, if
+     *    yajl_number is not NULL, it will always be used to return numbers,
+     *    that is yajl_integer and yajl_double will be ignored.  If
+     *    yajl_number is NULL but one of yajl_integer or yajl_double are
+     *    defined, parsing of a number larger than is representable
+     *    in a double or long int will result in a parse error.
      */
     typedef struct {
         int (* yajl_null)(void * ctx);
         int (* yajl_boolean)(void * ctx, int boolVal);
-        int (* yajl_integer)(void * ctx, long long integerVal);
+        int (* yajl_integer)(void * ctx, long integerVal);
         int (* yajl_double)(void * ctx, double doubleVal);
+        /** A callback which passes the string representation of the number
+         *  back to the client.  Will be used for all numbers when present */
+        int (* yajl_number)(void * ctx, const char * numberVal,
+                            unsigned int numberLen);
+
         /** strings are returned as pointers into the JSON text when,
          * possible, as a result, they are _not_ null padded */
         int (* yajl_string)(void * ctx, const unsigned char * stringVal,
diff --git a/src/yajl_encode.c b/src/yajl_encode.c
index 827cfbf..6804b54 100644
--- a/src/yajl_encode.c
+++ b/src/yajl_encode.c
@@ -39,7 +39,7 @@
 
 static void CharToHex(unsigned char c, char * hexBuf)
 {
-    const char * hexchar = "0123456789ABCDE";
+    const char * hexchar = "0123456789ABCDEF";
     hexBuf[0] = hexchar[c >> 4];
     hexBuf[1] = hexchar[c & 0x0F];
 }
diff --git a/src/yajl_gen.c b/src/yajl_gen.c
index f15b027..a753bdb 100644
--- a/src/yajl_gen.c
+++ b/src/yajl_gen.c
@@ -139,11 +139,11 @@ yajl_gen_free(yajl_gen g)
         yajl_buf_append(g->buf, "\n", 1);        
     
 yajl_gen_status
-yajl_gen_integer(yajl_gen g, long long int number)
+yajl_gen_integer(yajl_gen g, long int number)
 {
     char i[32];
     ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
-    sprintf(i, "%lld", number);
+    sprintf(i, "%ld", number);
     yajl_buf_append(g->buf, i, strlen(i));
     APPENDED_ATOM;
     FINAL_NEWLINE;
@@ -158,6 +158,17 @@ yajl_gen_double(yajl_gen g, double number)
     sprintf(i, "%lf", number);
     yajl_buf_append(g->buf, i, strlen(i));
     APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_number(yajl_gen g, const char * s, unsigned int l)
+{
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+    yajl_buf_append(g->buf, s, l);
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
     return yajl_gen_status_ok;
 }
 
diff --git a/src/yajl_parser.c b/src/yajl_parser.c
index b7564ce..2574941 100644
--- a/src/yajl_parser.c
+++ b/src/yajl_parser.c
@@ -34,10 +34,13 @@
 #include "yajl_parser.h"
 #include "yajl_encode.h"
 #include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
 #include <stdio.h>
 #include <string.h>
 #include <ctype.h>
 #include <assert.h>
+#include <math.h>
 
 unsigned char *
 yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
@@ -145,7 +148,6 @@ yajl_do_parse(yajl_handle hand, unsigned int * offset,
         case yajl_state_parse_error:            
             hand->errorOffset = *offset;
             return yajl_status_error;
-
         case yajl_state_start:
         case yajl_state_map_need_val:
         case yajl_state_array_need_val:
@@ -207,7 +209,7 @@ yajl_do_parse(yajl_handle hand, unsigned int * offset,
                     break;
                 case yajl_tok_integer:
                     /*
-                     * note.  sscanf does not respect the length of
+                     * note.  strtol does not respect the length of
                      * the lexical token.  in a corner case where the
                      * lexed number is a integer with a trailing zero,
                      * immediately followed by the end of buffer,
@@ -215,36 +217,61 @@ yajl_do_parse(yajl_handle hand, unsigned int * offset,
                      * crash.  for this reason we copy the integer
                      * (and doubles), into our parse buffer (the same
                      * one used for unescaping strings), before
-                     * calling sscanf.  yajl_buf ensures null padding,
+                     * calling strtol.  yajl_buf ensures null padding,
                      * so we're safe.
                      */
-                    if (hand->callbacks && hand->callbacks->yajl_integer) {
-                        long long int i = 0;
-                        int neg = 0;
-                        yajl_buf_clear(hand->decodeBuf);
-                        yajl_buf_append(hand->decodeBuf, buf, bufLen);
-                        buf = yajl_buf_data(hand->decodeBuf);
-                        if (*buf == '-') {
-                            buf++; neg = 1;
+                    if (hand->callbacks) {
+                        if (hand->callbacks->yajl_number) {
+                            _CC_CHK(hand->callbacks->yajl_number(hand->ctx,
+                                                                 (char *) buf,
+                                                                 bufLen));
+                        } else if (hand->callbacks->yajl_integer) {
+                            long int i = 0;
+                            yajl_buf_clear(hand->decodeBuf);
+                            yajl_buf_append(hand->decodeBuf, buf, bufLen);
+                            buf = yajl_buf_data(hand->decodeBuf);
+                            i = strtol((char *) buf, NULL, 10);
+                            if ((i == LONG_MIN || i == LONG_MAX) &&
+                                errno == ERANGE)
+                            {
+                                yajl_state_set(hand, yajl_state_parse_error);
+                                hand->parseError = "integer overflow" ;
+                                /* try to restore error offset */
+                                if (*offset >= bufLen) *offset -= bufLen;
+                                else *offset = 0;
+                                goto around_again;
+                            }
+                            _CC_CHK(hand->callbacks->yajl_integer(hand->ctx,
+                                                                  i));
                         }
-                        sscanf((char *) buf, "%lld", &i);
-                        if (neg) i -= (i<<1);
-                        _CC_CHK(hand->callbacks->yajl_integer(hand->ctx, i));
                     }
                     break;
                 case yajl_tok_double:
-                    if (hand->callbacks && hand->callbacks->yajl_double) {
-                        double d;
-                        int neg = 0;
-                        yajl_buf_clear(hand->decodeBuf);
-                        yajl_buf_append(hand->decodeBuf, buf, bufLen);
-                        buf = yajl_buf_data(hand->decodeBuf);
-                        if (*buf == '-') {
-                            buf++; neg = 1;
+                    if (hand->callbacks) {
+                        if (hand->callbacks->yajl_number) {
+                            _CC_CHK(hand->callbacks->yajl_number(hand->ctx,
+                                                                 (char *) buf,
+                                                                 bufLen));
+                        } else if (hand->callbacks->yajl_double) {
+                            double d = 0.0;
+                            yajl_buf_clear(hand->decodeBuf);
+                            yajl_buf_append(hand->decodeBuf, buf, bufLen);
+                            buf = yajl_buf_data(hand->decodeBuf);
+                            d = strtod((char *) buf, NULL);
+                            if ((d == HUGE_VAL || d == -HUGE_VAL) &&
+                                errno == ERANGE)
+                            {
+                                yajl_state_set(hand, yajl_state_parse_error);
+                                hand->parseError = "numeric (floating point) "
+                                    "overflow";
+                                /* try to restore error offset */
+                                if (*offset >= bufLen) *offset -= bufLen;
+                                else *offset = 0;
+                                goto around_again;
+                            }
+                            _CC_CHK(hand->callbacks->yajl_double(hand->ctx,
+                                                                 d));
                         }
-                        sscanf((char *) buf, "%lf", &d);
-                        if (neg) d *= -1.0;
-                        _CC_CHK(hand->callbacks->yajl_double(hand->ctx, d));
                     }
                     break;
                 case yajl_tok_right_brace: {
diff --git a/test/cases/integers.json b/test/cases/integers.json
index ca1393a..a4bbd7a 100644
--- a/test/cases/integers.json
+++ b/test/cases/integers.json
@@ -1,3 +1,4 @@
 [ 1,2,3,4,5,6,7,
   123456789 , -123456789,
-  9223372036854775807, -9223372036854775807 ]
+  2147483647, -2147483647,
+  2147483648 ]
diff --git a/test/cases/integers.json.gold b/test/cases/integers.json.gold
index f44b283..7341a2d 100644
--- a/test/cases/integers.json.gold
+++ b/test/cases/integers.json.gold
@@ -8,6 +8,6 @@ integer: 6
 integer: 7
 integer: 123456789
 integer: -123456789
-integer: 9223372036854775807
-integer: -9223372036854775807
-array close ']'
+integer: 2147483647
+integer: -2147483647
+parse error: integer overflow
diff --git a/test/yajl_test.c b/test/yajl_test.c
index a98b906..667f822 100644
--- a/test/yajl_test.c
+++ b/test/yajl_test.c
@@ -51,9 +51,9 @@ int test_yajl_boolean(void * ctx, int boolVal)
     return 1;
 }
 
-int test_yajl_integer(void *ctx, long long integerVal)
+int test_yajl_integer(void *ctx, long integerVal)
 {
-    printf("integer: %lld\n", integerVal);
+    printf("integer: %ld\n", integerVal);
     return 1;
 }
 
@@ -113,6 +113,7 @@ static yajl_callbacks callbacks = {
     test_yajl_boolean,
     test_yajl_integer,
     test_yajl_double,
+    NULL,
     test_yajl_string,
     test_yajl_start_map,
     test_yajl_map_key,