From 599d8cc2deee615526838ddc962778e51cd3a15a Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Fri, 2 Dec 2016 23:03:43 +0100 Subject: - MDEV-11366 SIGBUS errors in Connect Storage Engine for ArmHF and MIPS. Fix includes launchpad fix plus more to cover writing BIN tables. modified: storage/connect/tabfix.cpp modified: storage/connect/value.cpp modified: storage/connect/value.h - Typo: Change the name of filamzip to filamgz to prepare future ZIP tables. modified: storage/connect/CMakeLists.txt added: storage/connect/filamgz.cpp added: storage/connect/filamgz.h deleted: storage/connect/filamzip.cpp deleted: storage/connect/filamzip.h modified: storage/connect/plgdbsem.h modified: storage/connect/reldef.cpp modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfix.cpp modified: storage/connect/tabfmt.cpp modified: storage/connect/tabjson.cpp --- storage/connect/tabjson.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 17260836371..5f864f0bd48 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,6 +1,6 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.1 */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2015 */ +/* PROGRAM NAME: tabjson Version 1.2 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -25,9 +25,9 @@ //#include "resource.h" // for IDS_COLUMNS #include "tabjson.h" #include "filamap.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabmul.h" #include "checklvl.h" #include "resource.h" @@ -396,15 +396,15 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) (m == MODE_UPDATE || m == MODE_DELETE)); if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else -- cgit v1.2.1 From d44723e62153d9fb4165d038e9448c20a3ad890b Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 12 Dec 2016 10:57:19 +0100 Subject: - MDEV-11295: developing handling files contained in ZIP file. A first experimental and limited implementation. modified: storage/connect/CMakeLists.txt modified: storage/connect/filamap.cpp new file: storage/connect/filamzip.cpp new file: storage/connect/filamzip.h modified: storage/connect/ha_connect.cc new file: storage/connect/ioapi.c new file: storage/connect/ioapi.h modified: storage/connect/mycat.cc modified: storage/connect/plgdbsem.h modified: storage/connect/plgdbutl.cpp modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfmt.cpp modified: storage/connect/tabfmt.h modified: storage/connect/tabjson.cpp modified: storage/connect/tabjson.h new file: storage/connect/tabzip.cpp new file: storage/connect/tabzip.h new file: storage/connect/unzip.c new file: storage/connect/unzip.h new file: storage/connect/zip.c --- storage/connect/tabjson.cpp | 72 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 16 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 5f864f0bd48..73c6a6d85a4 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,5 +1,5 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.2 */ +/* PROGRAM NAME: tabjson Version 1.3 */ /* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -28,6 +28,9 @@ #if defined(GZ_SUPPORT) #include "filamgz.h" #endif // GZ_SUPPORT +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT #include "tabmul.h" #include "checklvl.h" #include "resource.h" @@ -67,7 +70,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT}; static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0}; - char *fn, colname[65], fmt[129]; + char colname[65], fmt[129]; int i, j, lvl, n = 0; int ncol = sizeof(buftyp) / sizeof(int); PVAL valp; @@ -94,16 +97,21 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ - if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; - } // endif fn + lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; tdp = new(g) JSONDEF; - tdp->Fn = fn; +#if defined(ZIP_SUPPORT) + tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL); + tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0); +#endif // ZIP_SUPPORT + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + tdp->Database = SetPath(g, db); tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; @@ -114,7 +122,10 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); + if (tdp->Zipfn) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); + else + tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) return NULL; @@ -127,10 +138,28 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) } // endif lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + + if (tdp->Zipfn) + tjnp = new(g) TDBJSN(tdp, new(g) ZIPFAM(tdp)); + else + tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + tjnp->SetMode(MODE_READ); - if (tjnp->OpenDB(g)) +#if USE_G + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G, G->Sarea, G->Sarea_Size); + G->jump_level = -1; + tjnp->SetG(G); +#else + tjnp->SetG(g); +#endif + + if (tjnp->OpenDB(g)) return NULL; switch (tjnp->ReadDB(g)) { @@ -395,7 +424,14 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Compressed) { + if (Zipfn) { +#if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Compressed) { #if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) GZFAM(this); @@ -426,12 +462,16 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - txfp = new(g) MAPFAM(this); + if (Zipfn) + txfp = new(g) ZIPFAM(this); + else + txfp = new(g) MAPFAM(this); + tdbp = new(g) TDBJSON(this, txfp); ((TDBJSON*)tdbp)->G = g; } // endif Pretty - if (Multiple) + if (Multiple && !Zipfn) tdbp = new(g) TDBMUL(tdbp); return tdbp; -- cgit v1.2.1 From 952306502ebf1b26c627c5dc8b141581eeb30671 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Wed, 14 Dec 2016 14:20:23 +0100 Subject: - MDEV-11295: developing handling files contained in ZIP file. Enable using multiple zip files modified: storage/connect/filamzip.cpp modified: storage/connect/ha_connect.cc modified: storage/connect/ha_connect.h modified: storage/connect/mycat.h modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfmt.cpp modified: storage/connect/tabjson.cpp --- storage/connect/tabjson.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 73c6a6d85a4..eff95445a3a 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -94,7 +94,12 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ lvl = GetIntegerTableOption(g, topt, "Level", 0); @@ -102,12 +107,12 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp = new(g) JSONDEF; #if defined(ZIP_SUPPORT) - tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL); - tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); #endif // ZIP_SUPPORT tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); - if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) { + if (!tdp->Fn) { strcpy(g->Message, MSG(MISSING_FNAME)); return NULL; } // endif Fn @@ -122,7 +127,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - if (tdp->Zipfn) + if (tdp->Zipped) tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); else tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); @@ -139,7 +144,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - if (tdp->Zipfn) + if (tdp->Zipped) tjnp = new(g) TDBJSN(tdp, new(g) ZIPFAM(tdp)); else tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); @@ -424,7 +429,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Zipfn) { + if (Zipped) { #if defined(ZIP_SUPPORT) txfp = new(g) ZIPFAM(this); #else // !ZIP_SUPPORT @@ -462,7 +467,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - if (Zipfn) + if (Zipped) txfp = new(g) ZIPFAM(this); else txfp = new(g) MAPFAM(this); @@ -471,7 +476,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSON*)tdbp)->G = g; } // endif Pretty - if (Multiple && !Zipfn) + if (Multiple) tdbp = new(g) TDBMUL(tdbp); return tdbp; -- cgit v1.2.1 From e6b563f8be68d57df2a4c9b8e2b6c130855b18e4 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Fri, 23 Dec 2016 16:58:32 +0100 Subject: Fix some XML table type bugs: - in DOMNODELIST::DropItem if (Listp == NULL || Listp->length <= n) return true; is wrong, should be: if (Listp == NULL || Listp->length < n) return true; - Crash in discovery with libxml2 in XMLColumns because: if (!tdp->Usedom) // nl was destroyed vp->nl = vp->pn->GetChildElements(g); is executed with vp->pn uninitialized. Fixed by adding: vp->pn = node; line 264. -In discovery with libxml2 some columns are not found. Because list was not recovered properly, nodes being modified and not reallocated. Fixed lines 214 and 277. modified: storage/connect/domdoc.cpp modified: storage/connect/tabxml.cpp Add support for zipped table files modified: storage/connect/domdoc.cpp modified: storage/connect/domdoc.h modified: storage/connect/filamap.cpp modified: storage/connect/filamap.h modified: storage/connect/filamzip.cpp modified: storage/connect/filamzip.h modified: storage/connect/ha_connect.cc modified: storage/connect/libdoc.cpp modified: storage/connect/plgdbutl.cpp modified: storage/connect/plgxml.cpp modified: storage/connect/plgxml.h modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfmt.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp --- storage/connect/tabjson.cpp | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) (limited to 'storage/connect/tabjson.cpp') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index eff95445a3a..1b9ce8b64c9 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -127,9 +127,14 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - if (tdp->Zipped) - tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); - else + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) @@ -144,9 +149,14 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - if (tdp->Zipped) - tjnp = new(g) TDBJSN(tdp, new(g) ZIPFAM(tdp)); - else + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjnp = new(g)TDBJSN(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); tjnp->SetMode(MODE_READ); @@ -467,9 +477,14 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - if (Zipped) - txfp = new(g) ZIPFAM(this); - else + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g)ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else txfp = new(g) MAPFAM(this); tdbp = new(g) TDBJSON(this, txfp); -- cgit v1.2.1