From 99ab562a92c05896ffb54216602507c56ea4e101 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 1 Oct 2020 19:18:26 +0200 Subject: - Make possible to allocate work space larger than 4GB All variables handling sizes that were uint are now size_t. The variable connect_work_size is now ulong (was uint); Also make Json functiosn to allocate a larger memory (M=9 was 7) modified: storage/connect/global.h modified: storage/connect/ha_connect.cc modified: storage/connect/json.cpp modified: storage/connect/jsonudf.cpp modified: storage/connect/plgdbutl.cpp modified: storage/connect/plugutil.cpp modified: storage/connect/user_connect.cc - Fix uninitialised variable (pretty) in Json_File. Make Jbin_file accept the same arguments as Json_File ones. modified: storage/connect/jsonudf.cpp - Change the Level option to Depth (the word currently used) (Level being still accepted) modified: storage/connect/mongo.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp - Suppress 2nd argument default value for MYSQLtoPLG function modified: storage/connect/myutil.h - Allow REST tables to be create not specifying a file_name modified: storage/connect/tabrest.cpp --- storage/connect/tabjson.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 3b0d458a7a6..08040884449 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -177,8 +177,8 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) bool mgo = (GetTypeID(topt->type) == TAB_MONGO); PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - if (level) { - lvl = atoi(level); + if (level = GetStringTableOption(g, topt, "Depth", level)) { + lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else lvl = 0; -- cgit v1.2.1 From 307258c8eec73522805345178aad906aa86a61af Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 5 Oct 2020 12:29:51 +0200 Subject: - Use BIN type when charset='binary' modified: storage/connect/ha_connect.cc - Allow JSON columns to be "binary" By setting their type as VARBINAY(132) and their name begin with Jbin_ modified: storage/connect/json.h modified: storage/connect/jsonudf.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/value.cpp modified: storage/connect/value.h - CHARSET BINARY cannot be used for text columns modified: storage/connect/mysql-test/connect/r/updelx.result modified: storage/connect/mysql-test/connect/t/updelx.test --- storage/connect/tabjson.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 08040884449..3ab750e67f8 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1486,7 +1486,18 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) if (Value->IsTypeNum()) { strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); - } else + } else if (Value->GetType() == TYPE_BIN) { + if (Value->GetClen() >= sizeof(BSON)) { + ULONG len = Tjp->Lrecl ? Tjp->Lrecl : 500; + PBSON bsp = JbinAlloc(g, NULL, len, jsp); + + strcat(bsp->Msg, " column"); + ((BINVAL*)Value)->SetBinValue(bsp, sizeof(BSON)); + } else { + strcpy(g->Message, "Column size too small"); + Value->SetValue_char(NULL, 0); + } // endif Clen + } else Value->SetValue_psz(Serialize(g, jsp, NULL, 0)); return Value; -- cgit v1.2.1 From d4138e7eedada2e079f76dca797bf7137088753a Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 5 Oct 2020 15:21:58 +0200 Subject: Fix compile error in tabjson.cpp (ULONG -> ulong) --- storage/connect/tabjson.cpp | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 3ab750e67f8..5fa2021331b 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -177,7 +177,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) bool mgo = (GetTypeID(topt->type) == TAB_MONGO); PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - if (level = GetStringTableOption(g, topt, "Depth", level)) { + if ((level = GetStringTableOption(g, topt, "Depth", level))) { lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else @@ -254,12 +254,14 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL; } else { - if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) - if (!mgo) { - sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); - return 0; - } else - tdp->Lrecl = 8192; // Should be enough + if (!((tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))) { + if (!mgo) { + sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); + return 0; + } else + tdp->Lrecl = 8192; // Should be enough + + } // endif Lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); @@ -1329,7 +1331,7 @@ bool JSONCOL::ParseJpath(PGLOBAL g) { char *p, *p1 = NULL, *p2 = NULL, *pbuf = NULL; int i; - bool a, mul = false; + bool a; if (Parsed) return false; // Already done @@ -1487,8 +1489,8 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); } else if (Value->GetType() == TYPE_BIN) { - if (Value->GetClen() >= sizeof(BSON)) { - ULONG len = Tjp->Lrecl ? Tjp->Lrecl : 500; + if ((unsigned)Value->GetClen() >= sizeof(BSON)) { + ulong len = Tjp->Lrecl ? Tjp->Lrecl : 500; PBSON bsp = JbinAlloc(g, NULL, len, jsp); strcat(bsp->Msg, " column"); @@ -1570,7 +1572,6 @@ void JSONCOL::ReadColumn(PGLOBAL g) PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i) { int n = Nod - 1; - bool expd = false; PJAR arp; PJVAL val = NULL; @@ -2130,13 +2131,15 @@ int TDBJSON::Cardinality(PGLOBAL g) { if (!g) return (Xcol || Multiple) ? 0 : 1; - else if (Cardinal < 0) - if (!Multiple) { - if (MakeDocument(g) == RC_OK) - Cardinal = Doc->size(); + else if (Cardinal < 0) { + if (!Multiple) { + if (MakeDocument(g) == RC_OK) + Cardinal = Doc->size(); - } else - return 10; + } else + return 10; + + } // endif Cardinal return Cardinal; } // end of Cardinality -- cgit v1.2.1 From 5d2ddef26e3fce9784e4befb9e2e0960ff2434cd Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Tue, 6 Oct 2020 12:50:12 +0200 Subject: Fix search for json subtable in tabjson.cpp --- storage/connect/tabjson.cpp | 48 +++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 15 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 5fa2021331b..fcf7f4d182c 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1995,8 +1995,9 @@ int TDBJSON::MakeNewDoc(PGLOBAL g) /***********************************************************************/ int TDBJSON::MakeDocument(PGLOBAL g) { - char *p, *memory, *objpath, *key = NULL; + char *p, *p1, *p2, *memory, *objpath, *key = NULL; int len, i = 0; + my_bool a; MODE mode = Mode; PJSON jsp; PJOB objp = NULL; @@ -2039,22 +2040,39 @@ int TDBJSON::MakeDocument(PGLOBAL g) if ((objpath = PlugDup(g, Objname))) { if (*objpath == '$') objpath++; if (*objpath == '.') objpath++; + p1 = p2 = NULL; /*********************************************************************/ /* Find the table in the tree structure. */ /*********************************************************************/ - for (; jsp && objpath; objpath = p) { - if ((p = strchr(objpath, Sep))) - *p++ = 0; - - if (*objpath != '[' && !IsNum(objpath)) { - // objpass is a key + for (p = objpath; jsp && p; p = (p2 ? p2 : NULL)) { + a = (p1 != NULL); + p1 = strchr(p, '['); + p2 = strchr(p, '.'); + + if (!p2) + p2 = p1; + else if (p1) { + if (p1 < p2) + p2 = p1; + else if (p1 == p2 + 1) + *p2++ = 0; // Old syntax .[ + else + p1 = NULL; + + } // endif p1 + + if (p2) + *p2++ = 0; + + if (!a && *p && *p != '[' && !IsNum(p)) { + // obj is a key if (jsp->GetType() != TYPE_JOB) { strcpy(g->Message, "Table path does not match the json file"); return RC_FX; } // endif Type - key = objpath; + key = p; objp = jsp->GetObject(); arp = NULL; val = objp->GetValue(key); @@ -2065,15 +2083,15 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif val } else { - if (*objpath == '[') { + if (*p == '[') { // Old style - if (objpath[strlen(objpath) - 1] != ']') { - sprintf(g->Message, "Invalid Table path %s", Objname); + if (p[strlen(p) - 1] != ']') { + sprintf(g->Message, "Invalid Table path near %s", p); return RC_FX; } else - objpath++; + p++; - } // endif objpath + } // endif p if (jsp->GetType() != TYPE_JAR) { strcpy(g->Message, "Table path does not match the json file"); @@ -2082,7 +2100,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) arp = jsp->GetArray(); objp = NULL; - i = atoi(objpath) - B; + i = atoi(p) - B; val = arp->GetValue(i); if (!val) { @@ -2093,7 +2111,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif jsp = val->GetJson(); - } // endfor objpath + } // endfor p } // endif objpath -- cgit v1.2.1 From dc3a693b7039cb4b1d46d797ca86364ac3173be7 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 18 Oct 2020 17:20:44 +0200 Subject: - Inline MakePtr and MakeOff with OFFSET as size_t Also add a new member Saved_Size in the Global structure. modified: storage/connect/global.h modified: storage/connect/plugutil.cpp modified: storage/connect/user_connect.cc modified: storage/connect/jsonudf.cpp - Add session variables json_all_path and default_depth modified: storage/connect/ha_connect.cc modified: storage/connect/mongo.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp - ADD column options JPATH and XPATH Work as FIELD_FORMAT but are more readable modified: storage/connect/ha_connect.cc modified: storage/connect/ha_connect.h modified: storage/connect/mysql-test/connect/r/json_java_2.result modified: storage/connect/mysql-test/connect/r/json_java_3.result modified: storage/connect/mysql-test/connect/r/json_mongo_c.result - Handle negative numbes in the option list modified: storage/connect/ha_connect.cc - Fix Json parse that could crash the server. Was because it could use THROW out of the TRY block. Also handle all error by THROW. It is now done by a new class JSON. modified: storage/connect/json.cpp modified: storage/connect/json.h - Add a new UDF function jfile_translate. It translate a Json file to pretty = 0. Fast because it does not a real parse of the file. modified: storage/connect/jsonudf.cpp modified: storage/connect/jsonudf.h - Add a now options JSIZE and STRINGIFY to Json tables. STRINGIFY makes Objects or Arrays to be returned by their json representation instead of by their concatenated values. JSIZE allows to specify the LRECL (was 256) defaults to 1024. Also fix a bug about locating the sub-table by its path. modified: storage/connect/tabjson.cpp modified: storage/connect/tabjson.h --- storage/connect/tabjson.cpp | 51 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 26 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index fcf7f4d182c..cdf9e40f97c 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -52,19 +52,10 @@ /* External functions. */ /***********************************************************************/ USETEMP UseTemp(void); +bool JsonAllPath(void); +int GetDefaultDepth(void); char *GetJsonNull(void); -//typedef struct _jncol { -// struct _jncol *Next; -// char *Name; -// char *Fmt; -// int Type; -// int Len; -// int Scale; -// bool Cbn; -// bool Found; -//} JCOL, *PJCL; - /***********************************************************************/ /* JSONColumns: construct the result blocks containing the description */ /* of all the columns of a table contained inside a JSON file. */ @@ -167,23 +158,20 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg) jsp = NULL; row = NULL; sep = NULL; - i = n = bf = ncol = lvl = 0; - all = false; + i = n = bf = ncol = lvl = sz = 0; + all = strfy = false; } // end of JSONDISC constructor int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) { char filename[_MAX_PATH]; bool mgo = (GetTypeID(topt->type) == TAB_MONGO); - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - - if ((level = GetStringTableOption(g, topt, "Depth", level))) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - sep = GetStringTableOption(g, topt, "Separator", "."); + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); + sep = GetStringTableOption(g, topt, "Separator", "."); + sz = GetIntegerTableOption(g, topt, "Jsize", 1024); + strfy = GetBooleanTableOption(g, topt, "Stringify", false); /*********************************************************************/ /* Open the input file. */ @@ -306,7 +294,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) // Allocate the parse work memory PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); memset(G, 0, sizeof(GLOBAL)); - G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea_Size = (size_t)tdp->Lrecl * 10; G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); PlugSubSet(G->Sarea, G->Sarea_Size); G->jump_level = 0; @@ -403,7 +391,10 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) PJAR jar; if ((valp = jvp ? jvp->GetValue() : NULL)) { - jcol.Type = valp->GetType(); + if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = valp->GetType(); jcol.Len = valp->GetValLen(); jcol.Scale = valp->GetValPrec(); jcol.Cbn = valp->IsNull(); @@ -482,8 +473,16 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) } // endswitch Type } else if (lvl >= 0) { - jcol.Type = TYPE_STRING; - jcol.Len = 256; + if (strfy) { + if (!fmt[bf]) + strcat(fmt, colname); + + strcat(fmt, ".*"); + } else if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = TYPE_STRING; + jcol.Len = sz; jcol.Scale = 0; jcol.Cbn = true; } else @@ -2040,7 +2039,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) if ((objpath = PlugDup(g, Objname))) { if (*objpath == '$') objpath++; if (*objpath == '.') objpath++; - p1 = p2 = NULL; + p1 = (*objpath == '[') ? objpath++ : NULL; /*********************************************************************/ /* Find the table in the tree structure. */ -- cgit v1.2.1