From 99ab562a92c05896ffb54216602507c56ea4e101 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 1 Oct 2020 19:18:26 +0200 Subject: - Make possible to allocate work space larger than 4GB All variables handling sizes that were uint are now size_t. The variable connect_work_size is now ulong (was uint); Also make Json functiosn to allocate a larger memory (M=9 was 7) modified: storage/connect/global.h modified: storage/connect/ha_connect.cc modified: storage/connect/json.cpp modified: storage/connect/jsonudf.cpp modified: storage/connect/plgdbutl.cpp modified: storage/connect/plugutil.cpp modified: storage/connect/user_connect.cc - Fix uninitialised variable (pretty) in Json_File. Make Jbin_file accept the same arguments as Json_File ones. modified: storage/connect/jsonudf.cpp - Change the Level option to Depth (the word currently used) (Level being still accepted) modified: storage/connect/mongo.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp - Suppress 2nd argument default value for MYSQLtoPLG function modified: storage/connect/myutil.h - Allow REST tables to be create not specifying a file_name modified: storage/connect/tabrest.cpp --- storage/connect/global.h | 20 +++++++++---------- storage/connect/ha_connect.cc | 21 ++++++++++++-------- storage/connect/json.cpp | 2 +- storage/connect/jsonudf.cpp | 44 +++++++++++++++++++++++------------------ storage/connect/mongo.cpp | 2 +- storage/connect/myutil.h | 4 ++-- storage/connect/plgdbutl.cpp | 20 +++++++++---------- storage/connect/plugutil.cpp | 24 +++++++++++----------- storage/connect/tabjson.cpp | 4 ++-- storage/connect/tabrest.cpp | 29 +++++++++++++++++++-------- storage/connect/tabxml.cpp | 3 ++- storage/connect/user_connect.cc | 17 ++++++++-------- 12 files changed, 108 insertions(+), 82 deletions(-) (limited to 'storage') diff --git a/storage/connect/global.h b/storage/connect/global.h index fd26c87b800..548d047ccc9 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -1,7 +1,7 @@ /***********************************************************************/ /* GLOBAL.H: Declaration file used by all CONNECT implementations. */ /* (C) Copyright MariaDB Corporation Ab */ -/* Author Olivier Bertrand 1993-2018 */ +/* Author Olivier Bertrand 1993-2020 */ /***********************************************************************/ /***********************************************************************/ @@ -112,8 +112,8 @@ extern "C" { /***********************************************************************/ #include "os.h" -typedef uint OFFSET; -typedef char NAME[9]; +typedef size_t OFFSET; +typedef char NAME[9]; typedef struct { ushort Length; @@ -136,7 +136,7 @@ typedef struct _parm *PPARM; /***********************************************************************/ typedef struct { /* Plug Area SubAlloc header */ OFFSET To_Free; /* Offset of next free block */ - uint FreeBlk; /* Size of remaining free memory */ + size_t FreeBlk; /* Size of remaining free memory */ } POOLHEADER, *PPOOLHEADER; /***********************************************************************/ @@ -188,7 +188,7 @@ typedef struct _parm { /***********************************************************************/ typedef struct _global { /* Global structure */ void *Sarea; /* Points to work area */ - uint Sarea_Size; /* Work area size */ + size_t Sarea_Size; /* Work area size */ PACTIVITY Activityp; char Message[MAX_STR]; ulong More; /* Used by jsonudf */ @@ -210,16 +210,16 @@ DllExport char *PlugReadMessage(PGLOBAL, int, char *); DllExport char *PlugGetMessage(PGLOBAL, int); #endif // XMSG || NEWMSG #if defined(__WIN__) -DllExport short GetLineLength(PGLOBAL); // Console line length +DllExport short GetLineLength(PGLOBAL); // Console line length #endif // __WIN__ -DllExport PGLOBAL PlugInit(LPCSTR, uint); // Plug global initialization -DllExport int PlugExit(PGLOBAL); // Plug global termination +DllExport PGLOBAL PlugInit(LPCSTR, size_t); // Plug global initialization +DllExport int PlugExit(PGLOBAL); // Plug global termination DllExport LPSTR PlugRemoveType(LPSTR, LPCSTR); DllExport LPCSTR PlugSetPath(LPSTR to, LPCSTR prefix, LPCSTR name, LPCSTR dir); DllExport BOOL PlugIsAbsolutePath(LPCSTR path); -DllExport bool AllocSarea(PGLOBAL, uint); +DllExport bool AllocSarea(PGLOBAL, size_t); DllExport void FreeSarea(PGLOBAL); -DllExport BOOL PlugSubSet(void *, uint); +DllExport BOOL PlugSubSet(void *, size_t); DllExport void *PlugSubAlloc(PGLOBAL, void *, size_t); DllExport char *PlugDup(PGLOBAL g, const char *str); DllExport void *MakePtr(void *, OFFSET); diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 7d53f287f74..e39b5cebaf4 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,7 +170,7 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.07.0001 November 12, 2019"; + char version[]= "Version 1.07.0001 September 30, 2020"; #if defined(__WIN__) char compver[]= "Version 1.07.0001 " __DATE__ " " __TIME__; char slash= '\\'; @@ -254,8 +254,8 @@ TYPCONV GetTypeConv(void); char *GetJsonNull(void); uint GetJsonGrpSize(void); char *GetJavaWrapper(void); -uint GetWorkSize(void); -void SetWorkSize(uint); +ulong GetWorkSize(void); +void SetWorkSize(ulong); extern "C" const char *msglang(void); static char *strz(PGLOBAL g, LEX_STRING &ls); @@ -348,10 +348,10 @@ static MYSQL_THDVAR_ENUM( &usetemp_typelib); // typelib // Size used for g->Sarea_Size -static MYSQL_THDVAR_UINT(work_size, +static MYSQL_THDVAR_ULONG(work_size, PLUGIN_VAR_RQCMDARG, "Size of the CONNECT work area.", - NULL, NULL, SZWORK, SZWMIN, UINT_MAX, 1); + NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1); // Size used when converting TEXT columns to VARCHAR static MYSQL_THDVAR_INT(conv_size, @@ -461,8 +461,8 @@ char *GetJsonNull(void) {return connect_hton ? THDVAR(current_thd, json_null) : NULL;} uint GetJsonGrpSize(void) {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;} -uint GetWorkSize(void) {return THDVAR(current_thd, work_size);} -void SetWorkSize(uint) +ulong GetWorkSize(void) {return THDVAR(current_thd, work_size);} +void SetWorkSize(ulong) { // Changing the session variable value seems to be impossible here // and should be done in a check function @@ -5326,7 +5326,12 @@ static bool add_field(String *sql, const char *field_name, int typ, int len, int dec, char *key, uint tm, const char *rem, char *dft, char *xtra, char *fmt, int flag, bool dbf, char v) { - char var= (len > 255) ? 'V' : v; +#if defined(DEVELOPMENT) + // Some client programs regard CHAR(36) as GUID + char var= (len > 255 || len == 36) ? 'V' : v; +#else + char var = (len > 255) ? 'V' : v; +#endif bool q, error= false; const char *type= PLGtoMYSQLtype(typ, dbf, var); diff --git a/storage/connect/json.cpp b/storage/connect/json.cpp index 98a4659cea8..a38e6487025 100644 --- a/storage/connect/json.cpp +++ b/storage/connect/json.cpp @@ -425,7 +425,7 @@ char *ParseString(PGLOBAL g, int& i, STRG& src) int n = 0, len = src.len; // Be sure of memory availability - if (len + 1 - i > (signed)((PPOOLHEADER)g->Sarea)->FreeBlk) { + if ((unsigned)(len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk) { strcpy(g->Message, "ParseString: Out of memory"); return NULL; } // endif len diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index dad86d51040..a1c6a77d1f2 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -25,7 +25,7 @@ #else #define PUSH_WARNING(M) htrc(M) #endif -#define M 7 +#define M 9 bool IsNum(PSZ s); char *NextChr(PSZ s, char sep); @@ -4423,13 +4423,15 @@ char *json_file(UDF_INIT *initid, UDF_ARGS *args, char *result, fn = MakePSZ(g, args, 0); if (args->arg_count > 1) { - int len, pretty, pty = 3; + int len, pretty = 3, pty = 3; PJSON jsp; PJVAL jvp = NULL; - pretty = (args->arg_type[1] == INT_RESULT) ? (int)*(longlong*)args->args[1] - : (args->arg_count > 2 && args->arg_type[2] == INT_RESULT) - ? (int)*(longlong*)args->args[2] : 3; + for (unsigned int i = 1; i < args->arg_count; i++) + if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) { + pretty = (int) * (longlong*)args->args[i]; + break; + } // endif type /*******************************************************************************/ /* Parse the json file and allocate its tree structure. */ @@ -5626,20 +5628,19 @@ my_bool jbin_file_init(UDF_INIT *initid, UDF_ARGS *args, char *message) } else if (args->arg_type[0] != STRING_RESULT || !args->args[0]) { strcpy(message, "First argument must be a constant string (file name)"); return true; - } else if (args->arg_count > 1 && args->arg_type[1] != STRING_RESULT) { - strcpy(message, "Second argument is not a string (path)"); - return true; - } else if (args->arg_count > 2 && args->arg_type[2] != INT_RESULT) { - strcpy(message, "Third argument is not an integer (pretty)"); - return true; - } else if (args->arg_count > 3) { - if (args->arg_type[3] != INT_RESULT) { - strcpy(message, "Fourth argument is not an integer (memory)"); + } // endifs + + for (unsigned int i = 1; i < args->arg_count; i++) { + if (!(args->arg_type[i] == INT_RESULT || args->arg_type[i] == STRING_RESULT)) { + sprintf(message, "Argument %d is not an integer or a string (pretty or path)", i); return true; - } else - more += (ulong)*(longlong*)args->args[3]; + } // endif arg_type - } // endifs + // Take care of eventual memory argument + if (args->arg_type[i] == INT_RESULT && args->args[i]) + more += (ulong) * (longlong*)args->args[i]; + + } // endfor i initid->maybe_null = 1; CalcLen(args, false, reslen, memlen); @@ -5654,7 +5655,7 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result, unsigned long *res_length, char *is_null, char *error) { char *fn; - int pretty, len = 0, pty = 3; + int pretty = 3, len = 0, pty = 3; PJSON jsp; PJVAL jvp = NULL; PGLOBAL g = (PGLOBAL)initid->ptr; @@ -5666,7 +5667,12 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result, PlugSubSet(g->Sarea, g->Sarea_Size); g->Xchk = NULL; fn = MakePSZ(g, args, 0); - pretty = (args->arg_count > 2 && args->args[2]) ? (int)*(longlong*)args->args[2] : 3; + + for (unsigned int i = 1; i < args->arg_count; i++) + if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) { + pretty = (int) * (longlong*)args->args[i]; + break; + } // endif type /*********************************************************************************/ /* Parse the json file and allocate its tree structure. */ diff --git a/storage/connect/mongo.cpp b/storage/connect/mongo.cpp index bd3d3b893c1..e821440a0c3 100644 --- a/storage/connect/mongo.cpp +++ b/storage/connect/mongo.cpp @@ -251,7 +251,7 @@ int MGODISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt) PCSZ level = GetStringTableOption(g, topt, "Level", NULL); PMGODEF tdp; - if (level) { + if (level = GetStringTableOption(g, topt, "Depth", level)) { lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else diff --git a/storage/connect/myutil.h b/storage/connect/myutil.h index 6991172b39e..fa41fa47d61 100644 --- a/storage/connect/myutil.h +++ b/storage/connect/myutil.h @@ -6,8 +6,8 @@ enum enum_field_types PLGtoMYSQL(int type, bool dbf, char var = 0); const char *PLGtoMYSQLtype(int type, bool dbf, char var = 0); -int MYSQLtoPLG(char *typname, char *var = NULL); -int MYSQLtoPLG(int mytype, char *var = NULL); +int MYSQLtoPLG(char *typname, char *var); +int MYSQLtoPLG(int mytype, char *var); PCSZ MyDateFmt(int mytype); PCSZ MyDateFmt(char *typname); diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index e296553d8e2..7bc7c5f592b 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -5,7 +5,7 @@ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2018 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -1242,7 +1242,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp) mp.Sub = mp.Size <= ((mp.Sub) ? maxsub : (maxsub >> 2)); if (trace(2)) - htrc("PlgDBalloc: in %p size=%d used=%d free=%d sub=%d\n", + htrc("PlgDBalloc: in %p size=%zd used=%zd free=%zd sub=%d\n", arp, mp.Size, pph->To_Free, pph->FreeBlk, mp.Sub); if (!mp.Sub) { @@ -1258,7 +1258,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp) mp.Memp = malloc(mp.Size); if (trace(8)) - htrc("PlgDBalloc: %s(%d) at %p\n", v, mp.Size, mp.Memp); + htrc("PlgDBalloc: %s(%zd) at %p\n", v, mp.Size, mp.Memp); if (!mp.Inlist && mp.Memp) { // New allocated block, put it in the memory block chain. @@ -1290,7 +1290,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize) #endif if (trace(2)) - htrc("PlgDBrealloc: %p size=%d sub=%d\n", mp.Memp, mp.Size, mp.Sub); + htrc("PlgDBrealloc: %p size=%zd sub=%d\n", mp.Memp, mp.Size, mp.Sub); if (newsize == mp.Size) return mp.Memp; // Nothing to do @@ -1340,7 +1340,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize) } // endif's if (trace(8)) - htrc(" newsize=%d newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub); + htrc(" newsize=%zd newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub); return mp.Memp; } // end of PlgDBrealloc @@ -1392,13 +1392,13 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size) pph = (PPOOLHEADER)memp; if (trace(16)) - htrc("PlgDBSubAlloc: memp=%p size=%d used=%d free=%d\n", + htrc("PlgDBSubAlloc: memp=%p size=%zd used=%zd free=%zd\n", memp, size, pph->To_Free, pph->FreeBlk); - if ((uint)size > pph->FreeBlk) { /* Not enough memory left in pool */ + if (size > pph->FreeBlk) { /* Not enough memory left in pool */ sprintf(g->Message, - "Not enough memory in Work area for request of %d (used=%d free=%d)", - (int) size, pph->To_Free, pph->FreeBlk); + "Not enough memory in Work area for request of %zd (used=%zd free=%zd)", + size, pph->To_Free, pph->FreeBlk); if (trace(1)) htrc("%s\n", g->Message); @@ -1414,7 +1414,7 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size) pph->FreeBlk -= size; // New size of pool free block if (trace(16)) - htrc("Done memp=%p used=%d free=%d\n", + htrc("Done memp=%p used=%zd free=%zd\n", memp, pph->To_Free, pph->FreeBlk); return (memp); diff --git a/storage/connect/plugutil.cpp b/storage/connect/plugutil.cpp index e74937b942a..2517b76cd3e 100644 --- a/storage/connect/plugutil.cpp +++ b/storage/connect/plugutil.cpp @@ -6,7 +6,7 @@ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1993-2019 */ +/* (C) Copyright to the author Olivier BERTRAND 1993-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -142,7 +142,7 @@ void htrc(char const* fmt, ...) /* Language points on initial language name and eventual path. */ /* Return value is the pointer to the Global structure. */ /***********************************************************************/ -PGLOBAL PlugInit(LPCSTR Language, uint worksize) +PGLOBAL PlugInit(LPCSTR Language, size_t worksize) { PGLOBAL g; @@ -459,7 +459,7 @@ short GetLineLength(PGLOBAL g) /***********************************************************************/ /* Program for memory allocation of work and language areas. */ /***********************************************************************/ -bool AllocSarea(PGLOBAL g, uint size) +bool AllocSarea(PGLOBAL g, size_t size) { /*********************************************************************/ /* This is the allocation routine for the WIN32/UNIX/AIX version. */ @@ -483,7 +483,7 @@ bool AllocSarea(PGLOBAL g, uint size) if (trace(8)) { #endif if (g->Sarea) - htrc("Work area of %u allocated at %p\n", size, g->Sarea); + htrc("Work area of %zd allocated at %p\n", size, g->Sarea); else htrc("SareaAlloc: %s\n", g->Message); @@ -510,7 +510,7 @@ void FreeSarea(PGLOBAL g) #else if (trace(8)) #endif - htrc("Freeing Sarea at %p size = %d\n", g->Sarea, g->Sarea_Size); + htrc("Freeing Sarea at %p size = %zd\n", g->Sarea, g->Sarea_Size); g->Sarea = NULL; g->Sarea_Size = 0; @@ -524,7 +524,7 @@ void FreeSarea(PGLOBAL g) /* Here there should be some verification done such as validity of */ /* the address and size not larger than memory size. */ /***********************************************************************/ -BOOL PlugSubSet(void *memp, uint size) +BOOL PlugSubSet(void *memp, size_t size) { PPOOLHEADER pph = (PPOOLHEADER)memp; @@ -560,15 +560,15 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) pph = (PPOOLHEADER)memp; if (trace(16)) - htrc("SubAlloc in %p size=%d used=%d free=%d\n", + htrc("SubAlloc in %p size=%zd used=%zd free=%zd\n", memp, size, pph->To_Free, pph->FreeBlk); - if ((uint)size > pph->FreeBlk) { /* Not enough memory left in pool */ + if (size > pph->FreeBlk) { /* Not enough memory left in pool */ PCSZ pname = "Work"; sprintf(g->Message, - "Not enough memory in %s area for request of %u (used=%d free=%d)", - pname, (uint)size, pph->To_Free, pph->FreeBlk); + "Not enough memory in %s area for request of %zd (used=%zd free=%zd)", + pname, size, pph->To_Free, pph->FreeBlk); if (trace(1)) htrc("PlugSubAlloc: %s\n", g->Message); @@ -581,10 +581,10 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) /*********************************************************************/ memp = MakePtr(memp, pph->To_Free); /* Points to suballocated block */ pph->To_Free += (OFFSET)size; /* New offset of pool free block */ - pph->FreeBlk -= (uint)size; /* New size of pool free block */ + pph->FreeBlk -= size; /* New size of pool free block */ if (trace(16)) - htrc("Done memp=%p used=%d free=%d\n", + htrc("Done memp=%p used=%zd free=%zd\n", memp, pph->To_Free, pph->FreeBlk); return (memp); diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 3b0d458a7a6..08040884449 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -177,8 +177,8 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) bool mgo = (GetTypeID(topt->type) == TAB_MONGO); PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - if (level) { - lvl = atoi(level); + if (level = GetStringTableOption(g, topt, "Depth", level)) { + lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else lvl = 0; diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index 9c6b724973f..b1bdeffc880 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -158,17 +158,32 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) http = GetStringTableOption(g, tp, "Http", NULL); uri = GetStringTableOption(g, tp, "Uri", NULL); - fn = GetStringTableOption(g, tp, "Filename", "rest.json"); #if defined(MARIADB) ftype = GetStringTableOption(g, tp, "Type", "JSON"); #else // !MARIADB // OEM tables must specify the file type ftype = GetStringTableOption(g, tp, "Ftype", "JSON"); #endif // !MARIADB + fn = GetStringTableOption(g, tp, "Filename", NULL); + + if (!fn) { + int n, m = strlen(ftype) + 1; + + strcat(strcpy(filename, tab), "."); + n = strlen(filename); + + // Fold ftype to lower case + for (int i = 0; i < m; i++) + filename[n + i] = tolower(ftype[i]); + + fn = filename; + tp->filename = PlugDup(g, fn); + } // endif fn // We used the file name relative to recorded datapath - strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); - strncat(filename, fn, _MAX_PATH - strlen(filename)); + PlugSetPath(filename, fn, db); + //strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); + //strncat(filename, fn, _MAX_PATH - strlen(filename)); // Retrieve the file from the web and copy it locally if (http && grf(g->Message, trace(515), http, uri, filename)) { @@ -227,12 +242,10 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Http = GetStringCatInfo(g, "Http", NULL); Uri = GetStringCatInfo(g, "Uri", NULL); - Fn = GetStringCatInfo(g, "Filename", "rest.json"); + Fn = GetStringCatInfo(g, "Filename", NULL); // We used the file name relative to recorded datapath - //PlugSetPath(filename, Fn, GetPath()); - strcpy(filename, GetPath()); - strncat(filename, Fn, _MAX_PATH - strlen(filename)); + PlugSetPath(filename, Fn, GetPath()); // Retrieve the file from the web and copy it locally rc = grf(g->Message, xt, Http, Uri, filename); @@ -270,7 +283,7 @@ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) if (trace(515)) htrc("REST GetTable mode=%d\n", m); - if (m != MODE_READ && m != MODE_READX) { + if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) { strcpy(g->Message, "REST tables are currently read only"); return NULL; } // endif m diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 68941c67be8..fa140ea0699 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -150,7 +150,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) return NULL; } else { lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; } // endif fn if (trace(1)) diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index c25443ef7ef..c061294c57e 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -28,7 +28,7 @@ */ /****************************************************************************/ -/* Author: Olivier Bertrand -- bertrandop@gmail.com -- 2004-2015 */ +/* Author: Olivier Bertrand -- bertrandop@gmail.com -- 2004-2020 */ /****************************************************************************/ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation @@ -58,8 +58,8 @@ PCONNECT user_connect::to_users= NULL; /****************************************************************************/ /* Get the work_size SESSION variable value . */ /****************************************************************************/ -uint GetWorkSize(void); -void SetWorkSize(uint); +ulong GetWorkSize(void); +void SetWorkSize(ulong); /* -------------------------- class user_connect -------------------------- */ @@ -97,14 +97,14 @@ user_connect::~user_connect() bool user_connect::user_init() { // Initialize Plug-like environment - uint worksize= GetWorkSize(); + ulong worksize= GetWorkSize(); PACTIVITY ap= NULL; PDBUSER dup= NULL; // Areasize= 64M because of VEC tables. Should be parameterisable //g= PlugInit(NULL, 67108864); //g= PlugInit(NULL, 134217728); // 128M was because of old embedded tests - g= PlugInit(NULL, worksize); + g= PlugInit(NULL, (size_t)worksize); // Check whether the initialization is complete if (!g || !g->Sarea || PlugSubSet(g->Sarea, g->Sarea_Size) @@ -157,15 +157,16 @@ void user_connect::SetHandler(ha_connect *hc) bool user_connect::CheckCleanup(bool force) { if (thdp->query_id > last_query_id || force) { - uint worksize= GetWorkSize(), size = g->Sarea_Size; + ulong worksize = GetWorkSize(); + size_t size = g->Sarea_Size; PlugCleanup(g, true); - if (size != worksize) { + if (size != (size_t)worksize) { FreeSarea(g); // Check whether the work area could be allocated - if (AllocSarea(g, worksize)) { + if (AllocSarea(g, (size_t)worksize)) { AllocSarea(g, size); SetWorkSize(g->Sarea_Size); // Was too big } // endif sarea -- cgit v1.2.1 From c6eb127ca88eb998fc3a7be122845d869a8d3ffc Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sat, 3 Oct 2020 19:06:05 +0200 Subject: - Fix allocating work space larger than 4GB The variable connect_work_size is now ulong or ulonglong for 64bit machines. modified: storage/connect/ha_connect.cc modified: storage/connect/user_connect.cc --- storage/connect/ha_connect.cc | 31 ++++++++++++++++++++----------- storage/connect/user_connect.cc | 12 ++++++------ 2 files changed, 26 insertions(+), 17 deletions(-) (limited to 'storage') diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index e39b5cebaf4..cf2a86185ef 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,9 +170,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.07.0001 September 30, 2020"; + char version[]= "Version 1.07.0002 October 03, 2020"; #if defined(__WIN__) - char compver[]= "Version 1.07.0001 " __DATE__ " " __TIME__; + char compver[]= "Version 1.07.0002 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -254,8 +254,8 @@ TYPCONV GetTypeConv(void); char *GetJsonNull(void); uint GetJsonGrpSize(void); char *GetJavaWrapper(void); -ulong GetWorkSize(void); -void SetWorkSize(ulong); +size_t GetWorkSize(void); +void SetWorkSize(size_t); extern "C" const char *msglang(void); static char *strz(PGLOBAL g, LEX_STRING &ls); @@ -347,11 +347,19 @@ static MYSQL_THDVAR_ENUM( 1, // def (AUTO) &usetemp_typelib); // typelib +#ifdef _WIN64 +// Size used for g->Sarea_Size +static MYSQL_THDVAR_ULONGLONG(work_size, + PLUGIN_VAR_RQCMDARG, + "Size of the CONNECT work area.", + NULL, NULL, SZWORK, SZWMIN, ULONGLONG_MAX, 1); +#else // Size used for g->Sarea_Size static MYSQL_THDVAR_ULONG(work_size, - PLUGIN_VAR_RQCMDARG, - "Size of the CONNECT work area.", - NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1); + PLUGIN_VAR_RQCMDARG, + "Size of the CONNECT work area.", + NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1); +#endif // Size used when converting TEXT columns to VARCHAR static MYSQL_THDVAR_INT(conv_size, @@ -461,8 +469,8 @@ char *GetJsonNull(void) {return connect_hton ? THDVAR(current_thd, json_null) : NULL;} uint GetJsonGrpSize(void) {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;} -ulong GetWorkSize(void) {return THDVAR(current_thd, work_size);} -void SetWorkSize(ulong) +size_t GetWorkSize(void) {return (size_t)THDVAR(current_thd, work_size);} +void SetWorkSize(size_t) { // Changing the session variable value seems to be impossible here // and should be done in a check function @@ -472,7 +480,8 @@ void SetWorkSize(ulong) #if defined(JAVA_SUPPORT) char *GetJavaWrapper(void) -{return connect_hton ? THDVAR(current_thd, java_wrapper) : (char*)"wrappers/JdbcInterface";} +{return connect_hton ? THDVAR(current_thd, java_wrapper) + : (char*)"wrappers/JdbcInterface";} #endif // JAVA_SUPPORT #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) @@ -7376,7 +7385,7 @@ maria_declare_plugin(connect) 0x0107, /* version number (1.07) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.07.0001", /* string version */ + "1.07.0002", /* string version */ MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index c061294c57e..5dbdb8f56f0 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -58,8 +58,8 @@ PCONNECT user_connect::to_users= NULL; /****************************************************************************/ /* Get the work_size SESSION variable value . */ /****************************************************************************/ -ulong GetWorkSize(void); -void SetWorkSize(ulong); +size_t GetWorkSize(void); +void SetWorkSize(size_t); /* -------------------------- class user_connect -------------------------- */ @@ -97,7 +97,7 @@ user_connect::~user_connect() bool user_connect::user_init() { // Initialize Plug-like environment - ulong worksize= GetWorkSize(); + size_t worksize= GetWorkSize(); PACTIVITY ap= NULL; PDBUSER dup= NULL; @@ -157,16 +157,16 @@ void user_connect::SetHandler(ha_connect *hc) bool user_connect::CheckCleanup(bool force) { if (thdp->query_id > last_query_id || force) { - ulong worksize = GetWorkSize(); + size_t worksize = GetWorkSize(); size_t size = g->Sarea_Size; PlugCleanup(g, true); - if (size != (size_t)worksize) { + if (size != worksize) { FreeSarea(g); // Check whether the work area could be allocated - if (AllocSarea(g, (size_t)worksize)) { + if (AllocSarea(g, worksize)) { AllocSarea(g, size); SetWorkSize(g->Sarea_Size); // Was too big } // endif sarea -- cgit v1.2.1 From 307258c8eec73522805345178aad906aa86a61af Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 5 Oct 2020 12:29:51 +0200 Subject: - Use BIN type when charset='binary' modified: storage/connect/ha_connect.cc - Allow JSON columns to be "binary" By setting their type as VARBINAY(132) and their name begin with Jbin_ modified: storage/connect/json.h modified: storage/connect/jsonudf.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/value.cpp modified: storage/connect/value.h - CHARSET BINARY cannot be used for text columns modified: storage/connect/mysql-test/connect/r/updelx.result modified: storage/connect/mysql-test/connect/t/updelx.test --- storage/connect/ha_connect.cc | 9 +++++--- storage/connect/json.h | 24 +++++++++++++++++++++- storage/connect/jsonudf.cpp | 10 +++++---- storage/connect/mysql-test/connect/r/updelx.result | 4 ++-- storage/connect/mysql-test/connect/t/updelx.test | 4 ++-- storage/connect/tabjson.cpp | 13 +++++++++++- storage/connect/value.cpp | 9 ++++++++ storage/connect/value.h | 7 ++++--- 8 files changed, 64 insertions(+), 16 deletions(-) (limited to 'storage') diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index cf2a86185ef..fd00b4878a9 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1586,6 +1586,9 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) chset= (char *)fp->charset()->name; + if (!strcmp(chset, "binary")) + v = 'B'; // Binary string + switch (fp->type()) { case MYSQL_TYPE_BLOB: case MYSQL_TYPE_VARCHAR: @@ -1595,7 +1598,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) default: pcf->Type= MYSQLtoPLG(fp->type(), &v); break; - } // endswitch SQL type + } // endswitch SQL type switch (pcf->Type) { case TYPE_STRING: @@ -1649,7 +1652,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) break; default: break; - } // endswitch type + } // endswitch type if (fp->flags & UNSIGNED_FLAG) pcf->Flags |= U_UNSIGNED; @@ -2222,7 +2225,7 @@ int ha_connect::MakeRecord(char *buf) case TYPE_BIN: p= value->GetCharValue(); charset= &my_charset_bin; - rc= fp->store(p, strlen(p), charset, CHECK_FIELD_WARN); + rc= fp->store(p, value->GetSize(), charset, CHECK_FIELD_WARN); break; case TYPE_DOUBLE: rc= fp->store(value->GetFloatValue()); diff --git a/storage/connect/json.h b/storage/connect/json.h index 1d058ad575f..d949f244e21 100644 --- a/storage/connect/json.h +++ b/storage/connect/json.h @@ -1,10 +1,11 @@ /**************** json H Declares Source Code File (.H) ****************/ /* Name: json.h Version 1.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2020 */ /* */ /* This file contains the JSON classes declares. */ /***********************************************************************/ +#include #include "value.h" #if defined(_DEBUG) @@ -44,6 +45,27 @@ typedef struct { int len; } STRG, *PSG; +// BSON size should be equal on Linux and Windows +#define BMX 255 +typedef struct BSON* PBSON; + +/***********************************************************************/ +/* Structure used to return binary json to Json UDF functions. */ +/***********************************************************************/ +struct BSON { + char Msg[BMX + 1]; + char *Filename; + PGLOBAL G; + int Pretty; + ulong Reslen; + my_bool Changed; + PJSON Top; + PJSON Jsp; + PBSON Bsp; +}; // end of struct BSON + +PBSON JbinAlloc(PGLOBAL g, UDF_ARGS* args, ulong len, PJSON jsp); + char *NextChr(PSZ s, char sep); char *GetJsonNull(void); diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index a1c6a77d1f2..0a862d2917d 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -1076,6 +1076,7 @@ my_bool JSNX::AddPath(void) /* --------------------------------- JSON UDF ---------------------------------- */ +#if 0 // Moved to json.h // BSON size should be equal on Linux and Windows #define BMX 255 typedef struct BSON *PBSON; @@ -1094,11 +1095,12 @@ struct BSON { PJSON Jsp; PBSON Bsp; }; // end of struct BSON +#endif // 0 /*********************************************************************************/ /* Allocate and initialize a BSON structure. */ /*********************************************************************************/ -static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) +PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) { PBSON bsp = (PBSON)PlgDBSubAlloc(g, NULL, sizeof(BSON)); @@ -1111,7 +1113,7 @@ static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) bsp->Reslen = len; bsp->Changed = false; bsp->Top = bsp->Jsp = jsp; - bsp->Bsp = (IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL; + bsp->Bsp = (args && IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL; } else PUSH_WARNING(g->Message); @@ -1422,7 +1424,7 @@ static int IsJson(UDF_ARGS *args, uint i, bool b) n = 2; // arg is a json file name } else if (b) { char *sap; - PGLOBAL g = PlugInit(NULL, args->lengths[i] * M + 1024); + PGLOBAL g = PlugInit(NULL, (size_t)args->lengths[i] * M + 1024); JsonSubSet(g); sap = MakePSZ(g, args, i); @@ -5763,7 +5765,7 @@ char *json_serialize(UDF_INIT *initid, UDF_ARGS *args, char *result, // Keep result of constant function g->Xchk = (initid->const_item) ? str : NULL; } else { - *error = 1; + // *error = 1; str = strcpy(result, "Argument is not a Jbin tree"); } // endif diff --git a/storage/connect/mysql-test/connect/r/updelx.result b/storage/connect/mysql-test/connect/r/updelx.result index 2aed1e06928..bb82afcc1a8 100644 --- a/storage/connect/mysql-test/connect/r/updelx.result +++ b/storage/connect/mysql-test/connect/r/updelx.result @@ -978,7 +978,7 @@ DROP TABLE t1; # FIX table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4; Warnings: Warning 1105 No file name. Table will use t1.fix @@ -1345,7 +1345,7 @@ DROP TABLE t1; # BIN table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8; Warnings: Warning 1105 No file name. Table will use t1.bin diff --git a/storage/connect/mysql-test/connect/t/updelx.test b/storage/connect/mysql-test/connect/t/updelx.test index 19d0d790a30..f6291432e48 100644 --- a/storage/connect/mysql-test/connect/t/updelx.test +++ b/storage/connect/mysql-test/connect/t/updelx.test @@ -36,7 +36,7 @@ DROP TABLE t1; --echo # FIX table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4; -- source updelx.inc ALTER TABLE t1 MAPPED=YES; @@ -48,7 +48,7 @@ DROP TABLE t1; --echo # BIN table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8; -- source updelx.inc ALTER TABLE t1 MAPPED=YES; diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 08040884449..3ab750e67f8 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1486,7 +1486,18 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) if (Value->IsTypeNum()) { strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); - } else + } else if (Value->GetType() == TYPE_BIN) { + if (Value->GetClen() >= sizeof(BSON)) { + ULONG len = Tjp->Lrecl ? Tjp->Lrecl : 500; + PBSON bsp = JbinAlloc(g, NULL, len, jsp); + + strcat(bsp->Msg, " column"); + ((BINVAL*)Value)->SetBinValue(bsp, sizeof(BSON)); + } else { + strcpy(g->Message, "Column size too small"); + Value->SetValue_char(NULL, 0); + } // endif Clen + } else Value->SetValue_psz(Serialize(g, jsp, NULL, 0)); return Value; diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index df75722d0e8..de04f7678f9 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -2250,6 +2250,15 @@ void BINVAL::SetBinValue(void *p) Len = Clen; } // end of SetBinValue +/***********************************************************************/ +/* BINVAL SetBinValue: fill string with len bytes. */ +/***********************************************************************/ +void BINVAL::SetBinValue(void* p, ulong len) +{ + memcpy(Binp, p, len); + Len = len; +} // end of SetBinValue + /***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ diff --git a/storage/connect/value.h b/storage/connect/value.h index 4f7d9a440fa..ee7a1c8032f 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -115,8 +115,8 @@ class DllExport VALUE : public BLOCK { virtual void SetValue(ulonglong) {assert(false);} virtual void SetValue(double) {assert(false);} virtual void SetValue_pvblk(PVBLK blk, int n) = 0; - virtual void SetBinValue(void *p) = 0; - virtual bool GetBinValue(void *buf, int buflen, bool go) = 0; + virtual void SetBinValue(void* p) = 0; + virtual bool GetBinValue(void *buf, int buflen, bool go) = 0; virtual int ShowValue(char *buf, int len) = 0; virtual char *GetCharString(char *p) = 0; virtual bool IsEqual(PVAL vp, bool chktype) = 0; @@ -385,7 +385,8 @@ class DllExport BINVAL: public VALUE { virtual void SetValue(ulonglong n); virtual void SetValue(double f); virtual void SetBinValue(void *p); - virtual bool GetBinValue(void *buf, int buflen, bool go); + virtual void SetBinValue(void* p, ulong len); + virtual bool GetBinValue(void *buf, int buflen, bool go); virtual int CompareValue(PVAL) {assert(false); return 0;} virtual int ShowValue(char *buf, int len); virtual char *GetCharString(char *p); -- cgit v1.2.1 From d4138e7eedada2e079f76dca797bf7137088753a Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 5 Oct 2020 15:21:58 +0200 Subject: Fix compile error in tabjson.cpp (ULONG -> ulong) --- storage/connect/tabjson.cpp | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'storage') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 3ab750e67f8..5fa2021331b 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -177,7 +177,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) bool mgo = (GetTypeID(topt->type) == TAB_MONGO); PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - if (level = GetStringTableOption(g, topt, "Depth", level)) { + if ((level = GetStringTableOption(g, topt, "Depth", level))) { lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else @@ -254,12 +254,14 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL; } else { - if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) - if (!mgo) { - sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); - return 0; - } else - tdp->Lrecl = 8192; // Should be enough + if (!((tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))) { + if (!mgo) { + sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); + return 0; + } else + tdp->Lrecl = 8192; // Should be enough + + } // endif Lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); @@ -1329,7 +1331,7 @@ bool JSONCOL::ParseJpath(PGLOBAL g) { char *p, *p1 = NULL, *p2 = NULL, *pbuf = NULL; int i; - bool a, mul = false; + bool a; if (Parsed) return false; // Already done @@ -1487,8 +1489,8 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); } else if (Value->GetType() == TYPE_BIN) { - if (Value->GetClen() >= sizeof(BSON)) { - ULONG len = Tjp->Lrecl ? Tjp->Lrecl : 500; + if ((unsigned)Value->GetClen() >= sizeof(BSON)) { + ulong len = Tjp->Lrecl ? Tjp->Lrecl : 500; PBSON bsp = JbinAlloc(g, NULL, len, jsp); strcat(bsp->Msg, " column"); @@ -1570,7 +1572,6 @@ void JSONCOL::ReadColumn(PGLOBAL g) PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i) { int n = Nod - 1; - bool expd = false; PJAR arp; PJVAL val = NULL; @@ -2130,13 +2131,15 @@ int TDBJSON::Cardinality(PGLOBAL g) { if (!g) return (Xcol || Multiple) ? 0 : 1; - else if (Cardinal < 0) - if (!Multiple) { - if (MakeDocument(g) == RC_OK) - Cardinal = Doc->size(); + else if (Cardinal < 0) { + if (!Multiple) { + if (MakeDocument(g) == RC_OK) + Cardinal = Doc->size(); - } else - return 10; + } else + return 10; + + } // endif Cardinal return Cardinal; } // end of Cardinality -- cgit v1.2.1 From 5d2ddef26e3fce9784e4befb9e2e0960ff2434cd Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Tue, 6 Oct 2020 12:50:12 +0200 Subject: Fix search for json subtable in tabjson.cpp --- storage/connect/tabjson.cpp | 48 +++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 15 deletions(-) (limited to 'storage') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 5fa2021331b..fcf7f4d182c 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1995,8 +1995,9 @@ int TDBJSON::MakeNewDoc(PGLOBAL g) /***********************************************************************/ int TDBJSON::MakeDocument(PGLOBAL g) { - char *p, *memory, *objpath, *key = NULL; + char *p, *p1, *p2, *memory, *objpath, *key = NULL; int len, i = 0; + my_bool a; MODE mode = Mode; PJSON jsp; PJOB objp = NULL; @@ -2039,22 +2040,39 @@ int TDBJSON::MakeDocument(PGLOBAL g) if ((objpath = PlugDup(g, Objname))) { if (*objpath == '$') objpath++; if (*objpath == '.') objpath++; + p1 = p2 = NULL; /*********************************************************************/ /* Find the table in the tree structure. */ /*********************************************************************/ - for (; jsp && objpath; objpath = p) { - if ((p = strchr(objpath, Sep))) - *p++ = 0; - - if (*objpath != '[' && !IsNum(objpath)) { - // objpass is a key + for (p = objpath; jsp && p; p = (p2 ? p2 : NULL)) { + a = (p1 != NULL); + p1 = strchr(p, '['); + p2 = strchr(p, '.'); + + if (!p2) + p2 = p1; + else if (p1) { + if (p1 < p2) + p2 = p1; + else if (p1 == p2 + 1) + *p2++ = 0; // Old syntax .[ + else + p1 = NULL; + + } // endif p1 + + if (p2) + *p2++ = 0; + + if (!a && *p && *p != '[' && !IsNum(p)) { + // obj is a key if (jsp->GetType() != TYPE_JOB) { strcpy(g->Message, "Table path does not match the json file"); return RC_FX; } // endif Type - key = objpath; + key = p; objp = jsp->GetObject(); arp = NULL; val = objp->GetValue(key); @@ -2065,15 +2083,15 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif val } else { - if (*objpath == '[') { + if (*p == '[') { // Old style - if (objpath[strlen(objpath) - 1] != ']') { - sprintf(g->Message, "Invalid Table path %s", Objname); + if (p[strlen(p) - 1] != ']') { + sprintf(g->Message, "Invalid Table path near %s", p); return RC_FX; } else - objpath++; + p++; - } // endif objpath + } // endif p if (jsp->GetType() != TYPE_JAR) { strcpy(g->Message, "Table path does not match the json file"); @@ -2082,7 +2100,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) arp = jsp->GetArray(); objp = NULL; - i = atoi(objpath) - B; + i = atoi(p) - B; val = arp->GetValue(i); if (!val) { @@ -2093,7 +2111,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif jsp = val->GetJson(); - } // endfor objpath + } // endfor p } // endif objpath -- cgit v1.2.1 From dc3a693b7039cb4b1d46d797ca86364ac3173be7 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 18 Oct 2020 17:20:44 +0200 Subject: - Inline MakePtr and MakeOff with OFFSET as size_t Also add a new member Saved_Size in the Global structure. modified: storage/connect/global.h modified: storage/connect/plugutil.cpp modified: storage/connect/user_connect.cc modified: storage/connect/jsonudf.cpp - Add session variables json_all_path and default_depth modified: storage/connect/ha_connect.cc modified: storage/connect/mongo.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp - ADD column options JPATH and XPATH Work as FIELD_FORMAT but are more readable modified: storage/connect/ha_connect.cc modified: storage/connect/ha_connect.h modified: storage/connect/mysql-test/connect/r/json_java_2.result modified: storage/connect/mysql-test/connect/r/json_java_3.result modified: storage/connect/mysql-test/connect/r/json_mongo_c.result - Handle negative numbes in the option list modified: storage/connect/ha_connect.cc - Fix Json parse that could crash the server. Was because it could use THROW out of the TRY block. Also handle all error by THROW. It is now done by a new class JSON. modified: storage/connect/json.cpp modified: storage/connect/json.h - Add a new UDF function jfile_translate. It translate a Json file to pretty = 0. Fast because it does not a real parse of the file. modified: storage/connect/jsonudf.cpp modified: storage/connect/jsonudf.h - Add a now options JSIZE and STRINGIFY to Json tables. STRINGIFY makes Objects or Arrays to be returned by their json representation instead of by their concatenated values. JSIZE allows to specify the LRECL (was 256) defaults to 1024. Also fix a bug about locating the sub-table by its path. modified: storage/connect/tabjson.cpp modified: storage/connect/tabjson.h --- storage/connect/global.h | 40 +- storage/connect/ha_connect.cc | 210 +++++---- storage/connect/ha_connect.h | 4 +- storage/connect/json.cpp | 164 +++---- storage/connect/json.h | 33 +- storage/connect/jsonudf.cpp | 500 ++++++++++++++++++++- storage/connect/jsonudf.h | 39 ++ storage/connect/mongo.cpp | 10 +- .../mysql-test/connect/r/json_java_2.result | 36 +- .../mysql-test/connect/r/json_java_3.result | 36 +- .../mysql-test/connect/r/json_mongo_c.result | 36 +- storage/connect/plugutil.cpp | 43 +- storage/connect/tabjson.cpp | 51 ++- storage/connect/tabjson.h | 4 +- storage/connect/tabxml.cpp | 6 +- storage/connect/user_connect.cc | 9 +- 16 files changed, 848 insertions(+), 373 deletions(-) (limited to 'storage') diff --git a/storage/connect/global.h b/storage/connect/global.h index 548d047ccc9..d17620861fa 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -89,14 +89,10 @@ extern "C" { #define PAT_LOG "log" #if defined(UNIX) || defined(LINUX) || defined(UNIV_LINUX) - /*********************************************************************/ - /* printf does not accept null pointer for %s target. */ - /*********************************************************************/ + // printf does not accept null pointer for %s target #define SVP(S) ((S) ? S : "") #else - /*********************************************************************/ - /* printf accepts null pointer for %s target. */ - /*********************************************************************/ + // printf accepts null pointer for %s target #define SVP(S) S #endif @@ -112,9 +108,6 @@ extern "C" { /***********************************************************************/ #include "os.h" -typedef size_t OFFSET; -typedef char NAME[9]; - typedef struct { ushort Length; char String[2]; @@ -127,6 +120,7 @@ typedef struct _global *PGLOBAL; typedef struct _globplg *PGS; typedef struct _activity *PACTIVITY; typedef struct _parm *PPARM; +typedef char NAME[9]; /***********************************************************************/ /* Segment Sub-Allocation block structure declares. */ @@ -135,7 +129,7 @@ typedef struct _parm *PPARM; /* restore them if needed. This scheme implies that no SubFree be used */ /***********************************************************************/ typedef struct { /* Plug Area SubAlloc header */ - OFFSET To_Free; /* Offset of next free block */ + size_t To_Free; /* Offset of next free block */ size_t FreeBlk; /* Size of remaining free memory */ } POOLHEADER, *PPOOLHEADER; @@ -190,9 +184,10 @@ typedef struct _global { /* Global structure */ void *Sarea; /* Points to work area */ size_t Sarea_Size; /* Work area size */ PACTIVITY Activityp; - char Message[MAX_STR]; + char Message[MAX_STR]; /* Message (result, error, trace) */ ulong More; /* Used by jsonudf */ - int Createas; /* To pass multi to ext tables */ + size_t Saved_Size; /* Saved work area to_free */ + bool Createas; /* To pass multi to ext tables */ void *Xchk; /* indexes in create/alter */ short Alchecked; /* Checked for ALTER */ short Mrr; /* True when doing mrr */ @@ -222,7 +217,6 @@ DllExport void FreeSarea(PGLOBAL); DllExport BOOL PlugSubSet(void *, size_t); DllExport void *PlugSubAlloc(PGLOBAL, void *, size_t); DllExport char *PlugDup(PGLOBAL g, const char *str); -DllExport void *MakePtr(void *, OFFSET); DllExport void htrc(char const *fmt, ...); DllExport void xtrc(uint, char const* fmt, ...); DllExport uint GetTraceValue(void); @@ -232,8 +226,24 @@ DllExport uint GetTraceValue(void); #endif /***********************************************************************/ -/* Non exported routine declarations. */ +/* Inline routine definitions. */ +/***********************************************************************/ +/***********************************************************************/ +/* This routine makes a pointer from an offset to a memory pointer. */ +/***********************************************************************/ +inline void* MakePtr(void* memp, size_t offset) { + // return ((offset == 0) ? NULL : &((char*)memp)[offset]); + return (!offset) ? NULL : (char *)memp + offset; +} /* end of MakePtr */ + +/***********************************************************************/ +/* This routine makes an offset from a pointer new format. */ /***********************************************************************/ -//void *PlugSubAlloc(PGLOBAL, void *, size_t); // Does throw +inline size_t MakeOff(void* memp, void* ptr) { +#if defined(_DEBUG) + assert(ptr > memp); +#endif // _DEBUG + return ((!ptr) ? 0 : (size_t)((char*)ptr - (size_t)memp)); +} /* end of MakeOff */ /*-------------------------- End of Global.H --------------------------*/ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index fd00b4878a9..a3dfc50562d 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,7 +170,7 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.07.0002 October 03, 2020"; + char version[]= "Version 1.07.0002 October 18, 2020"; #if defined(__WIN__) char compver[]= "Version 1.07.0002 " __DATE__ " " __TIME__; char slash= '\\'; @@ -251,6 +251,7 @@ bool ExactInfo(void); USETEMP UseTemp(void); int GetConvSize(void); TYPCONV GetTypeConv(void); +bool JsonAllPath(void); char *GetJsonNull(void); uint GetJsonGrpSize(void); char *GetJavaWrapper(void); @@ -394,6 +395,11 @@ static MYSQL_THDVAR_ENUM( 1, // def (yes) &xconv_typelib); // typelib +// Adding JPATH to all Json table columns +static MYSQL_THDVAR_BOOL(json_all_path, PLUGIN_VAR_RQCMDARG, + "Adding JPATH to all Json table columns", + NULL, NULL, 0); // NO by default + // Null representation for JSON values static MYSQL_THDVAR_STR(json_null, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, @@ -401,6 +407,12 @@ static MYSQL_THDVAR_STR(json_null, // check_json_null, update_json_null, NULL, NULL, ""); +// Default Json, XML or Mongo depth +static MYSQL_THDVAR_INT(default_depth, + PLUGIN_VAR_RQCMDARG, + "Default depth used by Json, XML and Mongo discovery", + NULL, NULL, 0, -1, 16, 1); + // Estimate max number of rows for JSON aggregate functions static MYSQL_THDVAR_UINT(json_grp_size, PLUGIN_VAR_RQCMDARG, // opt @@ -462,11 +474,13 @@ uint GetTraceValue(void) {return (uint)(connect_hton ? THDVAR(current_thd, xtrace) : 0);} bool ExactInfo(void) {return THDVAR(current_thd, exact_info);} static bool CondPushEnabled(void) {return THDVAR(current_thd, cond_push);} +bool JsonAllPath(void) {return THDVAR(current_thd, json_all_path);} USETEMP UseTemp(void) {return (USETEMP)THDVAR(current_thd, use_tempfile);} int GetConvSize(void) {return THDVAR(current_thd, conv_size);} TYPCONV GetTypeConv(void) {return (TYPCONV)THDVAR(current_thd, type_conv);} char *GetJsonNull(void) {return connect_hton ? THDVAR(current_thd, json_null) : NULL;} +int GetDefaultDepth(void) {return THDVAR(current_thd, default_depth);} uint GetJsonGrpSize(void) {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;} size_t GetWorkSize(void) {return (size_t)THDVAR(current_thd, work_size);} @@ -630,8 +644,10 @@ ha_create_table_option connect_field_option_list[]= HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), - HA_FOPTION_STRING("SPECIAL", special), - HA_FOPTION_ENUM("DISTRIB", opt, "scattered,clustered,sorted", 0), + HA_FOPTION_STRING("JPATH", jsonpath), + HA_FOPTION_STRING("XPATH", xmlpath), + HA_FOPTION_STRING("SPECIAL", special), + HA_FOPTION_ENUM("DISTRIB", opt, "scattered,clustered,sorted", 0), HA_FOPTION_END }; @@ -1322,9 +1338,10 @@ int GetIntegerTableOption(PGLOBAL g, PTOS options, PCSZ opname, int idef) if ((ulonglong) opval == (ulonglong)NO_IVAL) { PCSZ pv; - if ((pv= GetListOption(g, opname, options->oplist))) - opval= CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, true); - else + if ((pv = GetListOption(g, opname, options->oplist))) { + // opval = CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, false); + return atoi(pv); + } else return idef; } // endif opval @@ -1576,8 +1593,9 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) pcf->Offset= (int)fop->offset; pcf->Freq= (int)fop->freq; pcf->Datefmt= (char*)fop->dateformat; - pcf->Fieldfmt= (char*)fop->fieldformat; - } else { + pcf->Fieldfmt = fop->fieldformat ? (char*)fop->fieldformat + : fop->jsonpath ? (char*)fop->jsonpath : (char*)fop->xmlpath; + } else { pcf->Offset= -1; pcf->Freq= 0; pcf->Datefmt= NULL; @@ -4984,7 +5002,7 @@ int ha_connect::check_stmt(PGLOBAL g, MODE newmode, bool cras) } // endif CheckCleanup if (cras) - g->Createas= 1; // To tell external tables of a multi-table command + g->Createas= true; // To tell external tables of a multi-table command if (trace(1)) htrc("Calling CntCheckDB db=%s cras=%d\n", GetDBName(NULL), cras); @@ -5334,96 +5352,100 @@ static char *encode(PGLOBAL g, const char *cnm) @return Return 0 if ok */ -static bool add_field(String *sql, const char *field_name, int typ, int len, - int dec, char *key, uint tm, const char *rem, char *dft, - char *xtra, char *fmt, int flag, bool dbf, char v) -{ +static bool add_field(String* sql, TABTYPE ttp, const char* field_name, int typ, + int len, int dec, char* key, uint tm, const char* rem, + char* dft, char* xtra, char* fmt, int flag, bool dbf, char v) { #if defined(DEVELOPMENT) // Some client programs regard CHAR(36) as GUID - char var= (len > 255 || len == 36) ? 'V' : v; + char var = (len > 255 || len == 36) ? 'V' : v; #else char var = (len > 255) ? 'V' : v; #endif - bool q, error= false; - const char *type= PLGtoMYSQLtype(typ, dbf, var); + bool q, error = false; + const char* type = PLGtoMYSQLtype(typ, dbf, var); - error|= sql->append('`'); - error|= sql->append(field_name); - error|= sql->append("` "); - error|= sql->append(type); + error |= sql->append('`'); + error |= sql->append(field_name); + error |= sql->append("` "); + error |= sql->append(type); - if (typ == TYPE_STRING || - (len && typ != TYPE_DATE && (typ != TYPE_DOUBLE || dec >= 0))) { - error|= sql->append('('); - error|= sql->append_ulonglong(len); + if (typ == TYPE_STRING || + (len && typ != TYPE_DATE && (typ != TYPE_DOUBLE || dec >= 0))) { + error |= sql->append('('); + error |= sql->append_ulonglong(len); if (typ == TYPE_DOUBLE) { - error|= sql->append(','); - // dec must be < len and < 31 - error|= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1))); - } else if (dec > 0 && !strcmp(type, "DECIMAL")) { - error|= sql->append(','); - // dec must be < len - error|= sql->append_ulonglong(MY_MIN(dec, len - 1)); - } // endif dec - - error|= sql->append(')'); - } // endif len - - if (v == 'U') - error|= sql->append(" UNSIGNED"); - else if (v == 'Z') - error|= sql->append(" ZEROFILL"); - - if (key && *key) { - error|= sql->append(" "); - error|= sql->append(key); - } // endif key - - if (tm) - error|= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info); - - if (dft && *dft) { - error|= sql->append(" DEFAULT "); - - if (typ == TYPE_DATE) - q= (strspn(dft, "0123456789 -:/") == strlen(dft)); - else - q= !IsTypeNum(typ); + error |= sql->append(','); + // dec must be < len and < 31 + error |= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1))); + } else if (dec > 0 && !strcmp(type, "DECIMAL")) { + error |= sql->append(','); + // dec must be < len + error |= sql->append_ulonglong(MY_MIN(dec, len - 1)); + } // endif dec + + error |= sql->append(')'); + } // endif len + + if (v == 'U') + error |= sql->append(" UNSIGNED"); + else if (v == 'Z') + error |= sql->append(" ZEROFILL"); + + if (key && *key) { + error |= sql->append(" "); + error |= sql->append(key); + } // endif key + + if (tm) + error |= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info); + + if (dft && *dft) { + error |= sql->append(" DEFAULT "); + + if (typ == TYPE_DATE) + q = (strspn(dft, "0123456789 -:/") == strlen(dft)); + else + q = !IsTypeNum(typ); - if (q) { - error|= sql->append("'"); - error|= sql->append_for_single_quote(dft, strlen(dft)); - error|= sql->append("'"); - } else - error|= sql->append(dft); - - } // endif dft - - if (xtra && *xtra) { - error|= sql->append(" "); - error|= sql->append(xtra); - } // endif rem - - if (rem && *rem) { - error|= sql->append(" COMMENT '"); - error|= sql->append_for_single_quote(rem, strlen(rem)); - error|= sql->append("'"); - } // endif rem - - if (fmt && *fmt) { - error|= sql->append(" FIELD_FORMAT='"); - error|= sql->append_for_single_quote(fmt, strlen(fmt)); - error|= sql->append("'"); - } // endif flag - - if (flag) { - error|= sql->append(" FLAG="); - error|= sql->append_ulonglong(flag); - } // endif flag - - error|= sql->append(','); - return error; + if (q) { + error |= sql->append("'"); + error |= sql->append_for_single_quote(dft, strlen(dft)); + error |= sql->append("'"); + } else + error |= sql->append(dft); + + } // endif dft + + if (xtra && *xtra) { + error |= sql->append(" "); + error |= sql->append(xtra); + } // endif rem + + if (rem && *rem) { + error |= sql->append(" COMMENT '"); + error |= sql->append_for_single_quote(rem, strlen(rem)); + error |= sql->append("'"); + } // endif rem + + if (fmt && *fmt) { + switch (ttp) { + case TAB_JSON: error |= sql->append(" JPATH='"); break; + case TAB_XML: error |= sql->append(" XPATH='"); break; + default: error |= sql->append(" FIELD_FORMAT='"); + } // endswitch ttp + + error |= sql->append_for_single_quote(fmt, strlen(fmt)); + error |= sql->append("'"); + } // endif flag + + if (flag) { + error |= sql->append(" FLAG="); + error |= sql->append_ulonglong(flag); + } // endif flag + + error |= sql->append(','); + return error; } // end of add_field /** @@ -6045,7 +6067,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, len= 256; // STRBLK's have 0 length // Now add the field - if (add_field(&sql, cnm, typ, len, dec, NULL, tm, + if (add_field(&sql, ttp, cnm, typ, len, dec, NULL, tm, NULL, NULL, NULL, NULL, flg, dbf, v)) rc= HA_ERR_OUT_OF_MEM; } // endfor crp @@ -6239,7 +6261,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, prec= 0; // Now add the field - if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra, + if (add_field(&sql, ttp, cnm, typ, prec, dec, key, tm, rem, dft, xtra, fmt, flg, dbf, v)) rc= HA_ERR_OUT_OF_MEM; } // endfor i @@ -6992,7 +7014,7 @@ bool ha_connect::NoFieldOptionChange(TABLE *tab) fop1->fldlen == fop2->fldlen && CheckString(fop1->dateformat, fop2->dateformat) && CheckString(fop1->fieldformat, fop2->fieldformat) && - CheckString(fop1->special, fop2->special)); + CheckString(fop1->special, fop2->special)); } // endfor fld return rc; @@ -7362,7 +7384,9 @@ static struct st_mysql_sys_var* connect_system_variables[]= { MYSQL_SYSVAR(errmsg_dir_path), #endif // XMSG MYSQL_SYSVAR(json_null), - MYSQL_SYSVAR(json_grp_size), + MYSQL_SYSVAR(json_all_path), + MYSQL_SYSVAR(default_depth), + MYSQL_SYSVAR(json_grp_size), #if defined(JAVA_SUPPORT) MYSQL_SYSVAR(jvm_path), MYSQL_SYSVAR(class_path), diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 9a12ef94431..21e81b141a8 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -104,7 +104,9 @@ struct ha_field_option_struct uint opt; const char *dateformat; const char *fieldformat; - char *special; + const char* jsonpath; + const char* xmlpath; + char *special; }; /* diff --git a/storage/connect/json.cpp b/storage/connect/json.cpp index a38e6487025..ea3ea18da0b 100644 --- a/storage/connect/json.cpp +++ b/storage/connect/json.cpp @@ -93,9 +93,8 @@ char *NextChr(PSZ s, char sep) PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) { int i, pretty = (ptyp) ? *ptyp : 3; - bool b = false, pty[3] = {true, true, true}; - PJSON jsp = NULL; - STRG src; + bool b = false, pty[3] = {true,true,true}; + PJSON jsp = NULL, jp = NULL; if (trace(1)) htrc("ParseJson: s=%.10s len=%d\n", s, len); @@ -106,27 +105,29 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) } else if (comma) *comma = false; - src.str = s; - src.len = len; - // Trying to guess the pretty format if (s[0] == '[' && (s[1] == '\n' || (s[1] == '\r' && s[2] == '\n'))) pty[0] = false; try { - for (i = 0; i < len; i++) + jp = new(g) JSON(); + jp->s = s; + jp->len = len; + jp->pty = pty; + + for (i = 0; i < jp->len; i++) switch (s[i]) { case '[': if (jsp) - goto tryit; - else if (!(jsp = ParseArray(g, ++i, src, pty))) - throw 1; + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else + jsp = jp->ParseArray(g, ++i); break; case '{': if (jsp) - goto tryit; - else if (!(jsp = ParseObject(g, ++i, src, pty))) + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else if (!(jsp = jp->ParseObject(g, ++i))) throw 2; break; @@ -157,8 +158,8 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) default: if (jsp) - goto tryit; - else if (!(jsp = ParseValue(g, i, src, pty))) + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else if (!(jsp = jp->ParseValue(g, i))) throw 4; break; @@ -187,10 +188,17 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) } // end catch return jsp; +} // end of ParseJson -tryit: +/***********************************************************************/ +/* Parse several items as being in an array. */ +/***********************************************************************/ +PJAR JSON::ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp) +{ if (pty[0] && (!pretty || pretty > 2)) { - if ((jsp = ParseArray(g, (i = 0), src, pty)) && ptyp && pretty == 3) + PJAR jsp; + + if ((jsp = ParseArray(g, (i = 0))) && ptyp && pretty == 3) *ptyp = (pty[0]) ? 0 : 3; return jsp; @@ -198,26 +206,23 @@ tryit: strcpy(g->Message, "More than one item in file"); return NULL; -} // end of ParseJson +} // end of ParseAsArray /***********************************************************************/ /* Parse a JSON Array. */ /***********************************************************************/ -PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) +PJAR JSON::ParseArray(PGLOBAL g, int& i) { - char *s = src.str; - int len = src.len; - int level = 0; - bool b = (!i); - PJAR jarp = new(g) JARRAY; - PJVAL jvp = NULL; + int level = 0; + bool b = (!i); + PJAR jarp = new(g) JARRAY; for (; i < len; i++) switch (s[i]) { case ',': if (level < 2) { sprintf(g->Message, "Unexpected ',' near %.*s",ARGS); - return NULL; + throw 1; } else level = 1; @@ -225,8 +230,8 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) case ']': if (level == 1) { sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS); - return NULL; - } // endif level + throw 1; + } // endif level jarp->InitArray(g); return jarp; @@ -240,11 +245,9 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) default: if (level == 2) { sprintf(g->Message, "Unexpected value near %.*s", ARGS); - return NULL; - } else if ((jvp = ParseValue(g, i, src, pty))) - jarp->AddValue(g, jvp); - else - return NULL; + throw 1; + } else + jarp->AddValue(g, ParseValue(g, i)); level = (b) ? 1 : 2; break; @@ -256,18 +259,15 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) return jarp; } // endif b - strcpy(g->Message, "Unexpected EOF in array"); - return NULL; + throw ("Unexpected EOF in array"); } // end of ParseArray /***********************************************************************/ /* Parse a JSON Object. */ /***********************************************************************/ -PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) +PJOB JSON::ParseObject(PGLOBAL g, int& i) { PSZ key; - char *s = src.str; - int len = src.len; int level = 0; PJOB jobp = new(g) JOBJECT; PJPR jpp = NULL; @@ -276,42 +276,37 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) switch (s[i]) { case '"': if (level < 2) { - if ((key = ParseString(g, ++i, src))) { - jpp = jobp->AddPair(g, key); - level = 1; - } else - return NULL; - + key = ParseString(g, ++i); + jpp = jobp->AddPair(g, key); + level = 1; } else { sprintf(g->Message, "misplaced string near %.*s", ARGS); - return NULL; + throw 2; } // endif level break; case ':': if (level == 1) { - if (!(jpp->Val = ParseValue(g, ++i, src, pty))) - return NULL; - + jpp->Val = ParseValue(g, ++i); level = 2; } else { sprintf(g->Message, "Unexpected ':' near %.*s", ARGS); - return NULL; + throw 2; } // endif level break; case ',': if (level < 2) { sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); - return NULL; + throw 2; } else - level = 1; + level = 0; break; case '}': - if (level == 1) { + if (level < 2) { sprintf(g->Message, "Unexpected '}' near %.*s", ARGS); - return NULL; + throw 2; } // endif level return jobp; @@ -324,20 +319,19 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) default: sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); - return NULL; + throw 2; }; // endswitch s[i] strcpy(g->Message, "Unexpected EOF in Object"); - return NULL; + throw 2; } // end of ParseObject /***********************************************************************/ /* Parse a JSON Value. */ /***********************************************************************/ -PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) +PJVAL JSON::ParseValue(PGLOBAL g, int& i) { - char *strval, *s = src.str; - int n, len = src.len; + int n; PJVAL jvp = new(g) JVALUE; for (; i < len; i++) @@ -355,21 +349,13 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) suite: switch (s[i]) { case '[': - if (!(jvp->Jsp = ParseArray(g, ++i, src, pty))) - return NULL; - + jvp->Jsp = ParseArray(g, ++i); break; case '{': - if (!(jvp->Jsp = ParseObject(g, ++i, src, pty))) - return NULL; - + jvp->Jsp = ParseObject(g, ++i); break; case '"': - if ((strval = ParseString(g, ++i, src))) - jvp->Value = AllocateValue(g, strval, TYPE_STRING); - else - return NULL; - + jvp->Value = AllocateValue(g, ParseString(g, ++i), TYPE_STRING); break; case 't': if (!strncmp(s + i, "true", 4)) { @@ -398,11 +384,9 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) break; case '-': default: - if (s[i] == '-' || isdigit(s[i])) { - if (!(jvp->Value = ParseNumeric(g, i, src))) - goto err; - - } else + if (s[i] == '-' || isdigit(s[i])) + jvp->Value = ParseNumeric(g, i); + else goto err; }; // endswitch s[i] @@ -410,25 +394,21 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) return jvp; err: - sprintf(g->Message, "Unexpected character '%c' near %.*s", - s[i], ARGS); - return NULL; + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 3; } // end of ParseValue /***********************************************************************/ /* Unescape and parse a JSON string. */ /***********************************************************************/ -char *ParseString(PGLOBAL g, int& i, STRG& src) +char *JSON::ParseString(PGLOBAL g, int& i) { - char *s = src.str; uchar *p; - int n = 0, len = src.len; + int n = 0; // Be sure of memory availability - if ((unsigned)(len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk) { - strcpy(g->Message, "ParseString: Out of memory"); - return NULL; - } // endif len + if (((size_t)len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk) + throw("ParseString: Out of memory"); // The size to allocate is not known yet p = (uchar*)PlugSubAlloc(g, NULL, 0); @@ -502,17 +482,16 @@ char *ParseString(PGLOBAL g, int& i, STRG& src) }; // endswitch s[i] err: - strcpy(g->Message, "Unexpected EOF in String"); - return NULL; + throw("Unexpected EOF in String"); } // end of ParseString /***********************************************************************/ /* Parse a JSON numeric value. */ /***********************************************************************/ -PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src) +PVAL JSON::ParseNumeric(PGLOBAL g, int& i) { - char *s = src.str, buf[50]; - int n = 0, len = src.len; + char buf[50]; + int n = 0; short nd = 0; bool has_dot = false; bool has_e = false; @@ -575,14 +554,11 @@ PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src) i--; // Unstack following character return valp; - } else { - strcpy(g->Message, "No digit found"); - return NULL; - } // endif found_digit + } else + throw("No digit found"); err: - strcpy(g->Message, "Unexpected EOF in number"); - return NULL; + throw("Unexpected EOF in number"); } // end of ParseNumeric /***********************************************************************/ diff --git a/storage/connect/json.h b/storage/connect/json.h index d949f244e21..bc94b372133 100644 --- a/storage/connect/json.h +++ b/storage/connect/json.h @@ -69,12 +69,7 @@ PBSON JbinAlloc(PGLOBAL g, UDF_ARGS* args, ulong len, PJSON jsp); char *NextChr(PSZ s, char sep); char *GetJsonNull(void); -PJSON ParseJson(PGLOBAL g, char *s, int n, int *prty = NULL, bool *b = NULL); -PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty); -PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty); -PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty); -char *ParseString(PGLOBAL g, int& i, STRG& src); -PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src); +PJSON ParseJson(PGLOBAL g, char* s, int n, int* prty = NULL, bool* b = NULL); PSZ Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty); bool SerializeArray(JOUT *js, PJAR jarp, bool b); bool SerializeObject(JOUT *js, PJOB jobp); @@ -152,7 +147,7 @@ class JOUTPRT : public JOUTFILE { class JPAIR : public BLOCK { friend class JOBJECT; friend class JSNX; - friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*); + friend class JSON; friend bool SerializeObject(JOUT *, PJOB); public: JPAIR(PCSZ key) : BLOCK() {Key = key; Val = NULL; Next = NULL;} @@ -171,8 +166,9 @@ class JPAIR : public BLOCK { /* Class JSON. The base class for all other json classes. */ /***********************************************************************/ class JSON : public BLOCK { + friend PJSON ParseJson(PGLOBAL, char*, int, int*, bool*); public: - JSON(void) {Size = 0;} + JSON(void) : s(NULL), len(0), pty(NULL) {Size = 0;} int size(void) {return Size;} virtual int GetSize(bool b) {return Size;} @@ -209,14 +205,27 @@ class JSON : public BLOCK { virtual bool IsNull(void) {X return true;} protected: - int Size; + PJAR ParseArray(PGLOBAL g, int& i); + PJOB ParseObject(PGLOBAL g, int& i); + PJVAL ParseValue(PGLOBAL g, int& i); + char *ParseString(PGLOBAL g, int& i); + PVAL ParseNumeric(PGLOBAL g, int& i); + PJAR ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp); + + // Members + int Size; + + // Only used when parsing + private: + char *s; + int len; + bool *pty; }; // end of class JSON /***********************************************************************/ /* Class JOBJECT: contains a list of value pairs. */ /***********************************************************************/ class JOBJECT : public JSON { - friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*); friend bool SerializeObject(JOUT *, PJOB); friend class JSNX; public: @@ -282,8 +291,8 @@ class JVALUE : public JSON { friend class JARRAY; friend class JSNX; friend class JSONCOL; - friend PJVAL ParseValue(PGLOBAL, int&, STRG&, bool*); - friend bool SerializeValue(JOUT *, PJVAL); + friend class JSON; + friend bool SerializeValue(JOUT*, PJVAL); public: JVALUE(void) : JSON() {Clear();} JVALUE(PJSON jsp); diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 0a862d2917d..06164f4ed78 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -1076,27 +1076,6 @@ my_bool JSNX::AddPath(void) /* --------------------------------- JSON UDF ---------------------------------- */ -#if 0 // Moved to json.h -// BSON size should be equal on Linux and Windows -#define BMX 255 -typedef struct BSON *PBSON; - -/*********************************************************************************/ -/* Structure used to return binary json. */ -/*********************************************************************************/ -struct BSON { - char Msg[BMX + 1]; - char *Filename; - PGLOBAL G; - int Pretty; - ulong Reslen; - my_bool Changed; - PJSON Top; - PJSON Jsp; - PBSON Bsp; -}; // end of struct BSON -#endif // 0 - /*********************************************************************************/ /* Allocate and initialize a BSON structure. */ /*********************************************************************************/ @@ -1146,7 +1125,7 @@ static my_bool JsonSubSet(PGLOBAL g) { PPOOLHEADER pph = (PPOOLHEADER)g->Sarea; - pph->To_Free = (OFFSET)((g->Createas) ? g->Createas : sizeof(POOLHEADER)); + pph->To_Free = (g->Saved_Size) ? g->Saved_Size : (size_t)sizeof(POOLHEADER); pph->FreeBlk = g->Sarea_Size - pph->To_Free; return FALSE; } /* end of JsonSubSet */ @@ -1156,7 +1135,7 @@ static my_bool JsonSubSet(PGLOBAL g) /*********************************************************************************/ inline void JsonMemSave(PGLOBAL g) { - g->Createas = (int)((PPOOLHEADER)g->Sarea)->To_Free; + g->Saved_Size = ((PPOOLHEADER)g->Sarea)->To_Free; } /* end of JsonMemSave */ /*********************************************************************************/ @@ -1627,7 +1606,7 @@ static my_bool CheckMemory(PGLOBAL g, UDF_INIT *initid, UDF_ARGS *args, uint n, return true; } // endif SareaAlloc - g->Createas = 0; + g->Saved_Size = 0; g->Xchk = NULL; initid->max_length = rl; } // endif Size @@ -4501,6 +4480,7 @@ my_bool jfile_make_init(UDF_INIT *initid, UDF_ARGS *args, char *message) } // endif CalcLen(args, false, reslen, memlen); + memlen = memlen + 5000; // To take care of not pretty files return JsonInit(initid, args, message, true, reslen, memlen); } // end of jfile_make_init @@ -5781,6 +5761,478 @@ void json_serialize_deinit(UDF_INIT* initid) JsonFreeMem((PGLOBAL)initid->ptr); } // end of json_serialize_deinit +/*********************************************************************************/ +/* Convert a prettiest Json file to Pretty=0. */ +/*********************************************************************************/ +my_bool jfile_convert_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { + unsigned long reslen, memlen; + + if (args->arg_count != 3) { + strcpy(message, "This function must have 3 arguments"); + return true; + } else if (args->arg_type[2] != INT_RESULT) { + strcpy(message, "Third Argument must be an integer (LRECL)"); + return true; + } else for (int i = 0; i < 2; i++) + if (args->arg_type[i] != STRING_RESULT) { + sprintf(message, "Arguments %d must be a string (file name)", i+1); + return true; + } // endif args + + CalcLen(args, false, reslen, memlen); + return JsonInit(initid, args, message, false, reslen, memlen); +} // end of jfile_convert_init + +char *jfile_convert(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long *res_length, char *, char *error) { + char *str, *fn, *ofn; + int lrecl = (int)*(longlong*)args->args[2]; + PGLOBAL g = (PGLOBAL)initid->ptr; + + PlugSubSet(g->Sarea, g->Sarea_Size); + fn = MakePSZ(g, args, 0); + ofn = MakePSZ(g, args, 1); + + if (!g->Xchk) { + JUP* jup = new(g) JUP(g); + + str = jup->UnprettyJsonFile(g, fn, ofn, lrecl); + g->Xchk = str; + } else + str = (char*)g->Xchk; + + if (!str) { + if (g->Message) + str = PlugDup(g, g->Message); + else + str = PlugDup(g, "Unexpected error"); + + } // endif str + + *res_length = strlen(str); + return str; +} // end of jfile_convert + +void jfile_convert_deinit(UDF_INIT* initid) { + JsonFreeMem((PGLOBAL)initid->ptr); +} // end of jfile_convert_deinit + +/* --------------------------------- Class JUP --------------------------------- */ + +#define ARGS MY_MIN(24,len-i),s+MY_MAX(i-3,0) + +/*********************************************************************************/ +/* JUP public constructor. */ +/*********************************************************************************/ +JUP::JUP(PGLOBAL g) { + fs = NULL; + s = buff = NULL; + i = k = len = recl = 0; +} // end of JUP constructor + +/*********************************************************************************/ +/* Copy a json file to another with pretty = 0. */ +/*********************************************************************************/ +char* JUP::UnprettyJsonFile(PGLOBAL g, char *fn, char *outfn, int lrecl) { + char *ret = NULL; + HANDLE hFile; + MEMMAP mm; + + /*******************************************************************************/ + /* Create the mapping file object. */ + /*******************************************************************************/ + hFile = CreateFileMap(g, fn, &mm, MODE_READ, false); + + if (hFile == INVALID_HANDLE_VALUE) { + DWORD rc = GetLastError(); + + if (!(*g->Message)) + sprintf(g->Message, MSG(OPEN_MODE_ERROR), "map", (int)rc, fn); + + return NULL; + } // endif hFile + + /*******************************************************************************/ + /* Get the file size (assuming file is smaller than 4 GB) */ + /*******************************************************************************/ + if (!mm.lenL) { // Empty or deleted file + CloseFileHandle(hFile); + return NULL; + } else + len = (int)mm.lenL; + + if (!mm.memory) { + CloseFileHandle(hFile); + sprintf(g->Message, MSG(MAP_VIEW_ERROR), fn, GetLastError()); + return NULL; + } else + s = (char*)mm.memory; + + CloseFileHandle(hFile); // Not used anymore + + /*********************************************************************************/ + /* Parse the json file and allocate its tree structure. */ + /*********************************************************************************/ + if (!(fs = fopen(outfn, "wb"))) { + sprintf(g->Message, MSG(OPEN_MODE_ERROR), + "w", (int)errno, outfn); + strcat(strcat(g->Message, ": "), strerror(errno)); + CloseMemMap(mm.memory, (size_t)mm.lenL); + return NULL; + } // endif fs + + g->Message[0] = 0; + + if (!unPretty(g, lrecl)) + ret = outfn; + + CloseMemMap(mm.memory, (size_t)mm.lenL); + fclose(fs); + return ret; +} // end of UnprettyJsonFile + +/***********************************************************************/ +/* Translate a json file to pretty = 0. */ +/***********************************************************************/ +bool JUP::unPretty(PGLOBAL g, int lrecl) { + bool go, next, rc = false; + + if (trace(1)) + htrc("UnPretty: s=%.10s len=%zd lrecl=%d\n", s, len, lrecl); + + if (!s || !len) { + strcpy(g->Message, "Void JSON file"); + return true; + } else if (*s != '[') { + // strcpy(g->Message, "JSON file is not an array"); + s = strchr(s, '['); + // return true; + } // endif s + + i = 1; + go = next = true; + + try { + // Allocate the record + buff = (char*)PlugSubAlloc(g, NULL, (size_t)lrecl + 3); + recl = lrecl; + + do { + for (k = 0; go && i < len; i++) + switch (s[i]) { + case '{': + buff[k++] = s[i++]; + CopyObject(g); + break; + case '[': + throw "JSON file is not an array of objects"; + break; + case ' ': + case '\t': + case '\n': + case '\r': + break; + case ',': + go = false; + break; + case ']': + go = next = false; + break; + default: + sprintf(g->Message, "Unexpected '%c' near %.*s", s[i], ARGS); + throw 4; + break; + }; // endswitch s[i] + + // Write the record +#ifdef __win_ + buff[k++] = '\r'; +#endif + buff[k++] = '\n'; + buff[k] = 0; + + if ((fputs(buff, fs)) == EOF) { + sprintf(g->Message, MSG(FPUTS_ERROR), strerror(errno)); + throw 5; + } // endif EOF + + go = true; + } while (next); + + } catch (int n) { + if (trace(1)) + htrc("Exception %d: %s\n", n, g->Message); + rc = true; + } catch (const char* msg) { + strcpy(g->Message, msg); + rc = true; + } // end catch + + return rc; +} // end of unPretty + +/***********************************************************************/ +/* Copy a JSON Object. */ +/***********************************************************************/ +void JUP::CopyObject(PGLOBAL g) { + int level = 0; + + for (; i < len; i++) + switch (s[i]) { + case '"': + AddBuff(s[i++]); + + if (level < 2) { + CopyString(g); + level = 1; + } else { + sprintf(g->Message, "misplaced string near %.*s", ARGS); + throw 3; + } // endif level + + break; + case ':': + AddBuff(s[i++]); + + if (level == 1) { + CopyValue(g); + level = 2; + } else { + sprintf(g->Message, "Unexpected ':' near %.*s", ARGS); + throw 3; + } // endif level + + break; + case ',': + AddBuff(s[i]); + + if (level < 2) { + sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); + throw 3; + } else + level = 0; + + break; + case '}': + AddBuff(s[i]); + + if (level == 1) { + sprintf(g->Message, "Unexpected '}' near %.*s", ARGS); + throw 3; + } // endif level + + return; + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 3; + }; // endswitch s[i] + + throw "Unexpected EOF in Object"; +} // end of CopyObject + +/***********************************************************************/ +/* Copy a JSON Array. */ +/***********************************************************************/ +void JUP::CopyArray(PGLOBAL g) { + int level = 0; + + for (; i < len; i++) + switch (s[i]) { + case ',': + if (level < 2) { + sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); + throw 2; + } else + level = 1; + + AddBuff(s[i]); + break; + case ']': + if (level == 1) { + sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS); + throw 2; + } // endif level + + AddBuff(s[i]); + return; + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + if (level == 2) { + sprintf(g->Message, "Unexpected value near %.*s", ARGS); + throw 2; + } // endif level + + CopyValue(g); + level = 2; + break; + }; // endswitch s[i] + + throw "Unexpected EOF in array"; +} // end of CopyArray + +/***********************************************************************/ +/* Copy a JSON Value. */ +/***********************************************************************/ +void JUP::CopyValue(PGLOBAL g) { + for (; i < len; i++) + switch (s[i]) { + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + goto suite; + } // endswitch + +suite: + switch (s[i]) { + case '[': + AddBuff(s[i++]); + CopyArray(g); + break; + case '{': + AddBuff(s[i++]); + CopyObject(g); + break; + case '"': + AddBuff(s[i++]); + CopyString(g); + break; + case 't': + if (!strncmp(s + i, "true", 4)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + case 'f': + if (!strncmp(s + i, "false", 5)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + case 'n': + if (!strncmp(s + i, "null", 4)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + default: + if (s[i] == '-' || isdigit(s[i])) + CopyNumeric(g); + else + goto err; + + }; // endswitch s[i] + + return; + +err: + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 1; +} // end of CopyValue + +/***********************************************************************/ +/* Unescape and parse a JSON string. */ +/***********************************************************************/ +void JUP::CopyString(PGLOBAL g) { + for (; i < len; i++) { + AddBuff(s[i]); + + switch (s[i]) { + case '"': + return; + case '\\': + AddBuff(s[++i]); + break; + default: + break; + }; // endswitch s[i] + + } // endfor i + + throw "Unexpected EOF in String"; +} // end of CopyString + +/***********************************************************************/ +/* Copy a JSON numeric value. */ +/***********************************************************************/ +void JUP::CopyNumeric(PGLOBAL g) { + bool has_dot = false; + bool has_e = false; + bool found_digit = false; + + for (; i < len; i++) { + switch (s[i]) { + case '.': + if (!found_digit || has_dot || has_e) + goto err; + + has_dot = true; + break; + case 'e': + case 'E': + if (!found_digit || has_e) + goto err; + + has_e = true; + found_digit = false; + break; + case '+': + if (!has_e) + goto err; + + // fall through + case '-': + if (found_digit) + goto err; + + break; + default: + if (isdigit(s[i])) { + found_digit = true; + } else + goto fin; + + }; // endswitch s[i] + + AddBuff(s[i]); + } // endfor i + +fin: + if (!found_digit) + throw "No digit found"; + else + i--; + + return; + +err: + throw "Unexpected EOF in number"; +} // end of CopyNumeric + /*********************************************************************************/ /* Utility function returning an environment variable value. */ /*********************************************************************************/ diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index ee56869a111..897b0fe9919 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -235,6 +235,10 @@ extern "C" { DllExport char *json_serialize(UDF_EXEC_ARGS); DllExport void json_serialize_deinit(UDF_INIT*); + DllExport my_bool jfile_convert_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport char* jfile_convert(UDF_EXEC_ARGS); + DllExport void jfile_convert_deinit(UDF_INIT*); + DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); @@ -324,3 +328,38 @@ protected: my_bool Wr; // Write mode my_bool Jb; // Must return json item }; // end of class JSNX + +/*********************************************************************************/ +/* Class JUP: used by jfile_convert to make a json file pretty = 0. */ +/*********************************************************************************/ +class JUP : public BLOCK { +public: + // Constructor + JUP(PGLOBAL g); + + // Implementation + void AddBuff(char c) { + if (k < recl) + buff[k++] = c; + else + throw "Record size is too small"; + } // end of AddBuff + + // Methods + char *UnprettyJsonFile(PGLOBAL g, char* fn, char* outfn, int lrecl); + bool unPretty(PGLOBAL g, int lrecl); + void CopyObject(PGLOBAL g); + void CopyArray(PGLOBAL g); + void CopyValue(PGLOBAL g); + void CopyString(PGLOBAL g); + void CopyNumeric(PGLOBAL g); + + // Members + FILE* fs; + char* s; + char* buff; + int len; + int recl; + int i, k; +}; // end of class JUP + diff --git a/storage/connect/mongo.cpp b/storage/connect/mongo.cpp index e821440a0c3..5f10a89ee67 100644 --- a/storage/connect/mongo.cpp +++ b/storage/connect/mongo.cpp @@ -35,6 +35,7 @@ bool MakeSelector(PGLOBAL g, PFIL fp, PSTRG s); bool IsNum(PSZ s); +int GetDefaultDepth(void); /***********************************************************************/ /* Make selector json representation for Mongo tables. */ @@ -248,15 +249,10 @@ MGODISC::MGODISC(PGLOBAL g, int *lg) { /***********************************************************************/ int MGODISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt) { - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); PMGODEF tdp; - if (level = GetStringTableOption(g, topt, "Depth", level)) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); all = GetBooleanTableOption(g, topt, "Fullarray", false); /*********************************************************************/ diff --git a/storage/connect/mysql-test/connect/r/json_java_2.result b/storage/connect/mysql-test/connect/r/json_java_2.result index 4bbac236200..47fc4abbd28 100644 --- a/storage/connect/mysql-test/connect/r/json_java_2.result +++ b/storage/connect/mysql-test/connect/r/json_java_2.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=2' CONNECTION='mongodb://localhost:2701 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=2' `DATA_CHARSET`='utf8' `LRECL`=4096 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=2'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` char(24) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` char(24) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=2' `LRECL`=4096 diff --git a/storage/connect/mysql-test/connect/r/json_java_3.result b/storage/connect/mysql-test/connect/r/json_java_3.result index eb8bfc022d6..720c82cd7f9 100644 --- a/storage/connect/mysql-test/connect/r/json_java_3.result +++ b/storage/connect/mysql-test/connect/r/json_java_3.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=3' CONNECTION='mongodb://localhost:2701 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=3' `DATA_CHARSET`='utf8' `LRECL`=4096 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=3'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=3' `LRECL`=4096 diff --git a/storage/connect/mysql-test/connect/r/json_mongo_c.result b/storage/connect/mysql-test/connect/r/json_mongo_c.result index 550e94f286e..f9bfc01763e 100644 --- a/storage/connect/mysql-test/connect/r/json_mongo_c.result +++ b/storage/connect/mysql-test/connect/r/json_mongo_c.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=C,Version=0' CONNECTION='mongodb://localhost:27017' SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=C,Version=0' `DATA_CHARSET`='utf8' `LRECL`=1024 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=C,level=2,version=0'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(23,20) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(23,20) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"projection":{"cuisine":0}}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=C,level=2,version=0' `LRECL`=1024 diff --git a/storage/connect/plugutil.cpp b/storage/connect/plugutil.cpp index 2517b76cd3e..0ab594f5533 100644 --- a/storage/connect/plugutil.cpp +++ b/storage/connect/plugutil.cpp @@ -158,13 +158,14 @@ PGLOBAL PlugInit(LPCSTR Language, size_t worksize) } // end try/catch g->Sarea = NULL; - g->Createas = 0; + g->Createas = false; g->Alchecked = 0; g->Mrr = 0; g->Activityp = NULL; g->Xchk = NULL; g->N = 0; g->More = 0; + g->Saved_Size = 0; strcpy(g->Message, ""); /*******************************************************************/ @@ -528,7 +529,7 @@ BOOL PlugSubSet(void *memp, size_t size) { PPOOLHEADER pph = (PPOOLHEADER)memp; - pph->To_Free = (OFFSET)sizeof(POOLHEADER); + pph->To_Free = (size_t)sizeof(POOLHEADER); pph->FreeBlk = size - pph->To_Free; return FALSE; } /* end of PlugSubSet */ @@ -580,7 +581,7 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) /* Do the suballocation the simplest way. */ /*********************************************************************/ memp = MakePtr(memp, pph->To_Free); /* Points to suballocated block */ - pph->To_Free += (OFFSET)size; /* New offset of pool free block */ + pph->To_Free += size; /* New offset of pool free block */ pph->FreeBlk -= size; /* New size of pool free block */ if (trace(16)) @@ -605,40 +606,4 @@ char *PlugDup(PGLOBAL g, const char *str) } // end of PlugDup -#if 0 -/***********************************************************************/ -/* This routine suballocate a copy of the passed string. */ -/***********************************************************************/ -char *PlugDup(PGLOBAL g, const char *str) - { - char *buf; - size_t len; - - if (str && (len = strlen(str))) { - buf = (char*)PlugSubAlloc(g, NULL, len + 1); - strcpy(buf, str); - } else - buf = NULL; - - return(buf); - } /* end of PlugDup */ -#endif // 0 - -/***********************************************************************/ -/* This routine makes a pointer from an offset to a memory pointer. */ -/***********************************************************************/ -void *MakePtr(void *memp, OFFSET offset) - { - return ((offset == 0) ? NULL : &((char *)memp)[offset]); - } /* end of MakePtr */ - -/***********************************************************************/ -/* This routine makes an offset from a pointer new format. */ -/***********************************************************************/ -#if 0 -OFFSET MakeOff(void *memp, void *ptr) - { - return ((!ptr) ? 0 : (OFFSET)((char *)ptr - (char *)memp)); - } /* end of MakeOff */ -#endif /*--------------------- End of PLUGUTIL program -----------------------*/ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index fcf7f4d182c..cdf9e40f97c 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -52,19 +52,10 @@ /* External functions. */ /***********************************************************************/ USETEMP UseTemp(void); +bool JsonAllPath(void); +int GetDefaultDepth(void); char *GetJsonNull(void); -//typedef struct _jncol { -// struct _jncol *Next; -// char *Name; -// char *Fmt; -// int Type; -// int Len; -// int Scale; -// bool Cbn; -// bool Found; -//} JCOL, *PJCL; - /***********************************************************************/ /* JSONColumns: construct the result blocks containing the description */ /* of all the columns of a table contained inside a JSON file. */ @@ -167,23 +158,20 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg) jsp = NULL; row = NULL; sep = NULL; - i = n = bf = ncol = lvl = 0; - all = false; + i = n = bf = ncol = lvl = sz = 0; + all = strfy = false; } // end of JSONDISC constructor int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) { char filename[_MAX_PATH]; bool mgo = (GetTypeID(topt->type) == TAB_MONGO); - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - - if ((level = GetStringTableOption(g, topt, "Depth", level))) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - sep = GetStringTableOption(g, topt, "Separator", "."); + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); + sep = GetStringTableOption(g, topt, "Separator", "."); + sz = GetIntegerTableOption(g, topt, "Jsize", 1024); + strfy = GetBooleanTableOption(g, topt, "Stringify", false); /*********************************************************************/ /* Open the input file. */ @@ -306,7 +294,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) // Allocate the parse work memory PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); memset(G, 0, sizeof(GLOBAL)); - G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea_Size = (size_t)tdp->Lrecl * 10; G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); PlugSubSet(G->Sarea, G->Sarea_Size); G->jump_level = 0; @@ -403,7 +391,10 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) PJAR jar; if ((valp = jvp ? jvp->GetValue() : NULL)) { - jcol.Type = valp->GetType(); + if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = valp->GetType(); jcol.Len = valp->GetValLen(); jcol.Scale = valp->GetValPrec(); jcol.Cbn = valp->IsNull(); @@ -482,8 +473,16 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) } // endswitch Type } else if (lvl >= 0) { - jcol.Type = TYPE_STRING; - jcol.Len = 256; + if (strfy) { + if (!fmt[bf]) + strcat(fmt, colname); + + strcat(fmt, ".*"); + } else if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = TYPE_STRING; + jcol.Len = sz; jcol.Scale = 0; jcol.Cbn = true; } else @@ -2040,7 +2039,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) if ((objpath = PlugDup(g, Objname))) { if (*objpath == '$') objpath++; if (*objpath == '.') objpath++; - p1 = p2 = NULL; + p1 = (*objpath == '[') ? objpath++ : NULL; /*********************************************************************/ /* Find the table in the tree structure. */ diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index 8c3f1013919..88aa5e2ee8b 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -68,8 +68,8 @@ public: PCSZ sep; char colname[65], fmt[129], buf[16]; uint *length; - int i, n, bf, ncol, lvl; - bool all; + int i, n, bf, ncol, lvl, sz; + bool all, strfy; }; // end of JSONDISC /***********************************************************************/ diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index fa140ea0699..6c9e9597cec 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -3,7 +3,7 @@ /* ------------- */ /* Version 3.0 */ /* */ -/* Author Olivier BERTRAND 2007 - 2017 */ +/* Author Olivier BERTRAND 2007 - 2020 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -62,6 +62,8 @@ extern "C" char version[]; #define TYPE_UNKNOWN 12 /* Must be greater than other types */ #define XLEN(M) sizeof(M) - strlen(M) - 1 /* To avoid overflow*/ +int GetDefaultDepth(void); + /***********************************************************************/ /* Class and structure used by XMLColumns. */ /***********************************************************************/ @@ -149,7 +151,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) strcpy(g->Message, MSG(MISSING_FNAME)); return NULL; } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); lvl = GetIntegerTableOption(g, topt, "Depth", lvl); lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; } // endif fn diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index 5dbdb8f56f0..09d6db1ad27 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -158,16 +158,16 @@ bool user_connect::CheckCleanup(bool force) { if (thdp->query_id > last_query_id || force) { size_t worksize = GetWorkSize(); - size_t size = g->Sarea_Size; PlugCleanup(g, true); - if (size != worksize) { + if (worksize != g->Sarea_Size) { FreeSarea(g); + g->Saved_Size = g->Sarea_Size; // Check whether the work area could be allocated if (AllocSarea(g, worksize)) { - AllocSarea(g, size); + AllocSarea(g, g->Saved_Size); SetWorkSize(g->Sarea_Size); // Was too big } // endif sarea @@ -175,10 +175,11 @@ bool user_connect::CheckCleanup(bool force) PlugSubSet(g->Sarea, g->Sarea_Size); g->Xchk = NULL; - g->Createas = 0; + g->Createas = false; g->Alchecked = 0; g->Mrr = 0; g->More = 0; + g->Saved_Size = 0; last_query_id= thdp->query_id; if (trace(65) && !force) -- cgit v1.2.1 From 54c521ca338322ea08a6aba583be3b99649fb666 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 22 Oct 2020 16:49:57 +0300 Subject: MDEV-23672: incorrect v_indexes access fix v_indexes was wrongly copied from the source table. --- storage/innobase/handler/handler0alter.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 21918fbbe41..fef2d9650c0 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -592,6 +592,7 @@ inline bool dict_table_t::instant_column(const dict_table_t& table, mem_heap_alloc(heap, table.n_v_cols * sizeof(*v_cols))); for (ulint i = table.n_v_cols; i--; ) { new (&v_cols[i]) dict_v_col_t(table.v_cols[i]); + v_cols[i].v_indexes.clear(); } } else { ut_ad(table.n_v_cols == 0); -- cgit v1.2.1 From 985ede92034696d544d484a29b45828d56a031a5 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Tue, 20 Oct 2020 13:05:58 +0300 Subject: MDEV-20755 InnoDB: Database page corruption on disk or a failed file read of tablespace upon prepare of mariabackup incremental backup The problem: When incremental backup is taken, delta files are created for innodb tables which are marked as new tables during innodb ddl tracking. When such tablespace is tried to be opened during prepare in xb_delta_open_matching_space(), it is "created", i.e. xb_space_create_file() is invoked, instead of opening, even if a tablespace with the same name exists in the base backup directory. xb_space_create_file() writes page 0 header the tablespace. This header does not contain crypt data, as mariabackup does not have any information about crypt data in delta file metadata for tablespaces. After delta file is applied, recovery process is started. As the sequence of recovery for different pages is not defined, there can be the situation when crypt data redo log event is executed after some other page is read for recovery. When some page is read for recovery, it's decrypted using crypt data stored in tablespace header in page 0, if there is no crypt data, the page is not decryped and does not pass corruption test. This causes error for incremental backup --prepare for encrypted tablespaces. The error is not stable because crypt data redo log event updates crypt data on page 0, and recovery for different pages can be executed in undefined order. The fix: When delta file is created, the corresponding write filter copies only the pages which LSN is greater then some incremental LSN. When new file is created during incremental backup, the LSN of all it's pages must be greater then incremental LSN, so there is no need to create delta for such table, we can just copy it completely. The fix is to copy the whole file which was tracked during incremental backup with innodb ddl tracker, and copy it to base directory during --prepare instead of delta applying. There is also DBUG_EXECUTE_IF() in innodb code to avoid writing redo log record for crypt data updating on page 0 to make the test case stable. Note: The issue is not reproducible in 10.5 as optimized DDL's are deprecated in 10.5. But the fix is still useful because it allows to decrease data copy size during backup, as delta file contains some extra info. The test case should be removed for 10.5 as it will always pass. --- storage/innobase/fil/fil0crypt.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'storage') diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 16b2093b691..68a8a9be261 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -407,6 +407,8 @@ fil_space_crypt_t::write_page0( mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len + 8, encryption, MLOG_1BYTE, mtr); + DBUG_EXECUTE_IF("ib_do_not_log_crypt_data", return;); + byte* log_ptr = mlog_open(mtr, 11 + 17 + len); if (log_ptr != NULL) { -- cgit v1.2.1 From c4f8ccc0f53a4de9ca0cac54392b35348494040c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 18 Sep 2020 15:34:24 +0200 Subject: MDEV-21954 mysqld got signal 11 Fatal signal 6 while backtracing on parallel show global status protect global toku_global_status_rows with a mutex --- storage/tokudb/hatoku_hton.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'storage') diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 81355ea20ff..6002a3efd88 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -26,6 +26,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "hatoku_hton.h" #include "PerconaFT/src/ydb.h" +#include #define TOKU_METADB_NAME "tokudb_meta" @@ -239,6 +240,7 @@ tokudb::thread::rwlock_t tokudb_hton_initialized_lock; static SHOW_VAR *toku_global_status_variables = NULL; static uint64_t toku_global_status_max_rows; static TOKU_ENGINE_STATUS_ROW_S* toku_global_status_rows = NULL; +static std::mutex toku_global_status_rows_mutex; static void handle_ydb_error(int error) { switch (error) { @@ -1902,6 +1904,7 @@ static int show_tokudb_vars(TOKUDB_UNUSED(THD* thd), const int panic_string_len = 1024; char panic_string[panic_string_len] = {'\0'}; fs_redzone_state redzone_state; + const std::lock_guard lock(toku_global_status_rows_mutex); uint64_t num_rows; error = db_env->get_engine_status( -- cgit v1.2.1 From 641f81baf45212569ec8a98072d62fd29e3fc8f0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 1 Aug 2020 13:19:59 +0200 Subject: cleanup: use my_multi_malloc(), etc --- storage/heap/ha_heap.cc | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'storage') diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc index 2ffe2da06b2..396887df1b1 100644 --- a/storage/heap/ha_heap.cc +++ b/storage/heap/ha_heap.cc @@ -26,12 +26,8 @@ #include "ha_heap.h" #include "sql_base.h" // enum_tdc_remove_table_type -static handler *heap_create_handler(handlerton *hton, - TABLE_SHARE *table, - MEM_ROOT *mem_root); -static int -heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table, - HP_CREATE_INFO *hp_create_info); +static handler *heap_create_handler(handlerton *, TABLE_SHARE *, MEM_ROOT *); +static int heap_prepare_hp_create_info(TABLE *, bool, HP_CREATE_INFO *); int heap_panic(handlerton *hton, ha_panic_function flag) @@ -603,16 +599,15 @@ ha_rows ha_heap::records_in_range(uint inx, key_range *min_key, } -static int -heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table, - HP_CREATE_INFO *hp_create_info) +static int heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table, + HP_CREATE_INFO *hp_create_info) { - uint key, parts, mem_per_row= 0, keys= table_arg->s->keys; + TABLE_SHARE *share= table_arg->s; + uint key, parts, mem_per_row= 0, keys= share->keys; uint auto_key= 0, auto_key_type= 0; ha_rows max_rows; HP_KEYDEF *keydef; HA_KEYSEG *seg; - TABLE_SHARE *share= table_arg->s; bool found_real_auto_increment= 0; bzero(hp_create_info, sizeof(*hp_create_info)); @@ -620,11 +615,11 @@ heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table, for (key= parts= 0; key < keys; key++) parts+= table_arg->key_info[key].user_defined_key_parts; - if (!(keydef= (HP_KEYDEF*) my_malloc(keys * sizeof(HP_KEYDEF) + - parts * sizeof(HA_KEYSEG), - MYF(MY_WME | MY_THREAD_SPECIFIC)))) + if (!my_multi_malloc(MYF(MY_WME | MY_THREAD_SPECIFIC), + &keydef, keys * sizeof(HP_KEYDEF), + &seg, parts * sizeof(HA_KEYSEG), + NULL)) return my_errno; - seg= reinterpret_cast(keydef + keys); for (key= 0; key < keys; key++) { KEY *pos= table_arg->key_info+key; -- cgit v1.2.1 From 6cefe7d31ef43cadb905d71be743462825c7b4ff Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 2 Aug 2020 10:30:46 +0200 Subject: cleanup: use predefined CMAKE_DL_LIBS instead of, say, MY_SEARCH_LIBS(dlopen dl LIBDL) --- storage/mroonga/vendor/groonga/CMakeLists.txt | 1 - storage/mroonga/vendor/groonga/lib/CMakeLists.txt | 2 +- storage/rocksdb/build_rocksdb.cmake | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/mroonga/vendor/groonga/CMakeLists.txt b/storage/mroonga/vendor/groonga/CMakeLists.txt index 564e859d146..d271d4c4eb9 100644 --- a/storage/mroonga/vendor/groonga/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/CMakeLists.txt @@ -240,7 +240,6 @@ include(build/ac_macros/check_functions.m4) ac_check_symbols(fpclassify math.h) ac_check_lib(m fpclassify) -ac_check_lib(dl dlopen) ac_check_lib(execinfo backtrace) if(HAVE_LIBEXECINFO) set(HAVE_BACKTRACE TRUE) diff --git a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt index 21628b26ce2..8c71563f722 100644 --- a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt @@ -97,7 +97,7 @@ set(GRN_ALL_LIBRARIES ${LZ4_LIBS} ${LIBZSTD_LIBS} ${MESSAGE_PACK_LIBS} - ${DL_LIBS} + ${CMAKE_DL_LIBS} ${M_LIBS} ${WS2_32_LIBS} ${MRUBY_LIBS} diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index 8a981b8ec89..1099b51f850 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -160,7 +160,7 @@ find_package(Threads REQUIRED) if(WIN32) set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib) else() - set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} ${LIBRT} ${LIBDL}) + set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} ${LIBRT} ${CMAKE_DL_LIBS}) endif() set(ROCKSDB_LIBS rocksdblib}) -- cgit v1.2.1 From e64084d5a3a72462fa6263d1d0a86e72c0ba0d47 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 1 Aug 2020 13:12:50 +0200 Subject: MDEV-21201 No records produced in information_schema query, depending on projection Reimplement MDEV-14275 Improving memory utilization for information schema Postpone temp table instantiation until after setup_fields(). Replace all unused (not marked in read_set) columns in an I_S table with CHAR(0). This can drastically reduce the footprint of a MEMORY table (a TABLE_CATALOG alone is 1538 bytes per row). This does not change the engine. If the table was decided to be Aria (because of, say, blobs) then after optimization it'll stay Aria even if all blobs were removed. Note 1: when transforming table structure, share->blob_fields is preserved, otherwise Aria might switch from DYNAMIC to STATIC row format and expect a special field for a deleted mark, which create_tmp_tabe didn't provide. Note 2: optimizer was doing handler::info() (to know the number of rows) before the temp table is populated. That didn't make much sense. Now it's done before the table is even instantiated. Preserve the old behavior and report 0 rows. This reverts e2664ee8362 and a8458a2345e --- storage/heap/ha_heap.cc | 2 +- storage/maria/ha_maria.cc | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc index 396887df1b1..a40f4b8a464 100644 --- a/storage/heap/ha_heap.cc +++ b/storage/heap/ha_heap.cc @@ -364,7 +364,7 @@ int ha_heap::info(uint flag) HEAPINFO hp_info; if (!table) - return 1; + return 0; (void) heap_info(file,&hp_info,flag); diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index c0163473f3a..23d2becbb04 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -2440,6 +2440,9 @@ int ha_maria::info(uint flag) MARIA_INFO maria_info; char name_buff[FN_REFLEN]; + if (!table) + return 0; + (void) maria_status(file, &maria_info, flag); if (flag & HA_STATUS_VARIABLE) { -- cgit v1.2.1 From 987df9b37a2198ab70fb12076fa983d2efe408bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sun, 25 Oct 2020 11:48:34 +0200 Subject: MDEV-23720 Change innodb_log_optimize_ddl=OFF by default MariaDB 10.2.2 inherited from MySQL 5.7 a perceived optimization of ALTER TABLE, which skips the writing of redo log records. In MDEV-16809 we introduced a parameter that allows the redo log to be written, so that Mariabackup would not be impacted, but we kept the MySQL 5.7 behaviour enabled by default (innodb_log_optimize_ddl=ON). As noted in MDEV-19747 (Deprecate and ignore innodb_log_optimize_ddl, implemented in MariaDB 10.5.1), omitting the redo log writes can actually reduce performance, because we will have to wait for the data pages to be written out. When the redo log file is configured to be large enough, it actually can be much faster to write the redo log and avoid the extra page flushing. When the redo log is omitted (innodb_log_optimize_ddl=ON), also Mariabackup may have to perform a lot of extra work, to re-copy the entire data file if it is possible that any log was omitted during the backup. Starting with MariaDB 10.5.1, the parameter innodb_log_optimize_ddl is deprecated and ignored. We hereby deprecate (but will not ignore) the parameter in earlier versions as well. --- storage/innobase/handler/ha_innodb.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 637c66612ce..e36dcbad8ff 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -20245,10 +20245,10 @@ static MYSQL_SYSVAR_BOOL(log_compressed_pages, page_zip_log_pages, static MYSQL_SYSVAR_BOOL(log_optimize_ddl, innodb_log_optimize_ddl, PLUGIN_VAR_OPCMDARG, - "Reduce redo logging when natively creating indexes or rebuilding tables." - " Setting this OFF avoids delay due to page flushing and" - " allows concurrent backup.", - NULL, NULL, TRUE); + "DEPRECATED. Ignored in MariaDB 10.5." + " Reduce redo logging when natively creating indexes or rebuilding tables." + " Enabling this may slow down backup and cause delay due to page flushing.", + NULL, NULL, FALSE); static MYSQL_SYSVAR_ULONG(autoextend_increment, sys_tablespace_auto_extend_increment, -- cgit v1.2.1 From 3ba8f619e42bf8e7239ba03400c7a1a09b115e45 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Sun, 25 Oct 2020 11:47:16 +0530 Subject: MDEV-23370 innodb_fts.innodb_fts_misc failed in buildbot, server crashed in dict_table_autoinc_destroy This issue is caused by MDEV-22456 ad6171b91cac33e70bb28fa6865488b2c65e858c. Fix involves the backported version of 10.4 patch MDEV-22778 5f2628d1eea21d9732f582b77782b072e5e04014 and few parts of MDEV-17441 (e9a5f288f21c15ec6b4d2dd3d654a320904bb1bf). dict_table_t::stats_latch_created: Removed dict_table_t::stats_latch: make value member and always lock it for simplicity even for stats cloned table. zip_pad_info_t::mutex_created: Removed zip_pad_info_t::mutex: make member value instead of pointer os0once.h: Removed dict_table_remove_from_cache_low(): Ensure that fts_free() is always called, even if dict_mem_table_free() is deferred until btr_search_lazy_free(). InnoDB would always zip_pad_info_t::mutex and dict_table_t::autoinc_mutex, even for tables are not in ROW_FORMAT=COMPRESSED nor include any AUTO_INCREMENT column. --- storage/innobase/dict/dict0dict.cc | 150 ++++------------------------------- storage/innobase/dict/dict0mem.cc | 42 +++++----- storage/innobase/dict/dict0stats.cc | 10 +-- storage/innobase/ibuf/ibuf0ibuf.cc | 2 +- storage/innobase/include/dict0dict.h | 25 +----- storage/innobase/include/dict0mem.h | 118 ++++++--------------------- storage/innobase/include/os0once.h | 118 --------------------------- storage/innobase/page/page0zip.cc | 2 +- 8 files changed, 68 insertions(+), 399 deletions(-) delete mode 100644 storage/innobase/include/os0once.h (limited to 'storage') diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index ba0cb9e4d4e..37a8a4387d1 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -60,7 +60,6 @@ extern uint ibuf_debug; #include "lock0lock.h" #include "mach0data.h" #include "mem0mem.h" -#include "os0once.h" #include "page0page.h" #include "page0zip.h" #include "pars0pars.h" @@ -268,76 +267,6 @@ dict_mutex_exit_for_mysql(void) mutex_exit(&dict_sys->mutex); } -/** Allocate and init a dict_table_t's stats latch. -This function must not be called concurrently on the same table object. -@param[in,out] table_void table whose stats latch to create */ -static -void -dict_table_stats_latch_alloc( - void* table_void) -{ - dict_table_t* table = static_cast(table_void); - - /* Note: rw_lock_create() will call the constructor */ - - table->stats_latch = static_cast( - ut_malloc_nokey(sizeof(rw_lock_t))); - - ut_a(table->stats_latch != NULL); - - rw_lock_create(dict_table_stats_key, table->stats_latch, - SYNC_INDEX_TREE); -} - -/** Deinit and free a dict_table_t's stats latch. -This function must not be called concurrently on the same table object. -@param[in,out] table table whose stats latch to free */ -static -void -dict_table_stats_latch_free( - dict_table_t* table) -{ - rw_lock_free(table->stats_latch); - ut_free(table->stats_latch); -} - -/** Create a dict_table_t's stats latch or delay for lazy creation. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] table table whose stats latch to create -@param[in] enabled if false then the latch is disabled -and dict_table_stats_lock()/unlock() become noop on this table. */ -void -dict_table_stats_latch_create( - dict_table_t* table, - bool enabled) -{ - if (!enabled) { - table->stats_latch = NULL; - table->stats_latch_created = os_once::DONE; - return; - } - - /* We create this lazily the first time it is used. */ - table->stats_latch = NULL; - table->stats_latch_created = os_once::NEVER_DONE; -} - -/** Destroy a dict_table_t's stats latch. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] table table whose stats latch to destroy */ -void -dict_table_stats_latch_destroy( - dict_table_t* table) -{ - if (table->stats_latch_created == os_once::DONE - && table->stats_latch != NULL) { - - dict_table_stats_latch_free(table); - } -} - /** Lock the appropriate latch to protect a given table's statistics. @param[in] table table whose stats to lock @param[in] latch_mode RW_S_LATCH or RW_X_LATCH */ @@ -349,23 +278,12 @@ dict_table_stats_lock( ut_ad(table != NULL); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - os_once::do_or_wait_for_done( - &table->stats_latch_created, - dict_table_stats_latch_alloc, table); - - if (table->stats_latch == NULL) { - /* This is a dummy table object that is private in the current - thread and is not shared between multiple threads, thus we - skip any locking. */ - return; - } - switch (latch_mode) { case RW_S_LATCH: - rw_lock_s_lock(table->stats_latch); + rw_lock_s_lock(&table->stats_latch); break; case RW_X_LATCH: - rw_lock_x_lock(table->stats_latch); + rw_lock_x_lock(&table->stats_latch); break; case RW_NO_LATCH: /* fall through */ @@ -385,19 +303,12 @@ dict_table_stats_unlock( ut_ad(table != NULL); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - if (table->stats_latch == NULL) { - /* This is a dummy table object that is private in the current - thread and is not shared between multiple threads, thus we - skip any locking. */ - return; - } - switch (latch_mode) { case RW_S_LATCH: - rw_lock_s_unlock(table->stats_latch); + rw_lock_s_unlock(&table->stats_latch); break; case RW_X_LATCH: - rw_lock_x_unlock(table->stats_latch); + rw_lock_x_unlock(&table->stats_latch); break; case RW_NO_LATCH: /* fall through */ @@ -737,34 +648,6 @@ dict_table_get_nth_v_col_mysql( return(dict_table_get_nth_v_col(table, i)); } -/** Allocate and init the autoinc latch of a given table. -This function must not be called concurrently on the same table object. -@param[in,out] table_void table whose autoinc latch to create */ -static -void -dict_table_autoinc_alloc( - void* table_void) -{ - dict_table_t* table = static_cast(table_void); - table->autoinc_mutex = UT_NEW_NOKEY(ib_mutex_t()); - ut_a(table->autoinc_mutex != NULL); - mutex_create(LATCH_ID_AUTOINC, table->autoinc_mutex); -} - -/** Allocate and init the zip_pad_mutex of a given index. -This function must not be called concurrently on the same index object. -@param[in,out] index_void index whose zip_pad_mutex to create */ -static -void -dict_index_zip_pad_alloc( - void* index_void) -{ - dict_index_t* index = static_cast(index_void); - index->zip_pad.mutex = UT_NEW_NOKEY(SysMutex()); - ut_a(index->zip_pad.mutex != NULL); - mutex_create(LATCH_ID_ZIP_PAD_MUTEX, index->zip_pad.mutex); -} - /********************************************************************//** Acquire the autoinc lock. */ void @@ -772,11 +655,7 @@ dict_table_autoinc_lock( /*====================*/ dict_table_t* table) /*!< in/out: table */ { - os_once::do_or_wait_for_done( - &table->autoinc_mutex_created, - dict_table_autoinc_alloc, table); - - mutex_enter(table->autoinc_mutex); + mysql_mutex_lock(&table->autoinc_mutex); } /** Acquire the zip_pad_mutex latch. @@ -786,11 +665,7 @@ void dict_index_zip_pad_lock( dict_index_t* index) { - os_once::do_or_wait_for_done( - &index->zip_pad.mutex_created, - dict_index_zip_pad_alloc, index); - - mutex_enter(index->zip_pad.mutex); + mysql_mutex_lock(&index->zip_pad.mutex); } /** Get all the FTS indexes on a table. @@ -825,7 +700,7 @@ dict_table_autoinc_unlock( /*======================*/ dict_table_t* table) /*!< in/out: table */ { - mutex_exit(table->autoinc_mutex); + mysql_mutex_unlock(&table->autoinc_mutex); } /** Looks for column n in an index. @@ -1276,6 +1151,8 @@ dict_table_add_to_cache( dict_table_add_system_columns(table, heap); + mysql_mutex_init(0, &table->autoinc_mutex, NULL); + table->cached = TRUE; fold = ut_fold_string(table->name.m_name); @@ -1419,7 +1296,7 @@ dict_index_t *dict_index_t::clone() const (mem_heap_zalloc(heap, n_uniq * sizeof *stat_n_sample_sizes)); index->stat_n_non_null_key_vals= static_cast (mem_heap_zalloc(heap, n_uniq * sizeof *stat_n_non_null_key_vals)); - memset(&index->zip_pad, 0, sizeof index->zip_pad); + mysql_mutex_init(0, &index->zip_pad.mutex, NULL); return index; } @@ -2133,8 +2010,15 @@ dict_table_remove_from_cache_low( UT_DELETE(table->vc_templ); } + mysql_mutex_destroy(&table->autoinc_mutex); #ifdef BTR_CUR_HASH_ADAPT if (UNIV_UNLIKELY(UT_LIST_GET_LEN(table->freed_indexes) != 0)) { + if (table->fts) { + fts_optimize_remove_table(table); + fts_free(table); + table->fts = NULL; + } + table->vc_templ = NULL; table->id = 0; return; diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index 639fdbac22b..11d362d32c6 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -118,20 +118,15 @@ static bool dict_mem_table_is_system(char *name) } } -/**********************************************************************//** -Creates a table memory object. -@return own: table object */ dict_table_t* dict_mem_table_create( -/*==================*/ - const char* name, /*!< in: table name */ - ulint space, /*!< in: space where the clustered index of - the table is placed */ - ulint n_cols, /*!< in: total number of columns including - virtual and non-virtual columns */ - ulint n_v_cols,/*!< in: number of virtual columns */ - ulint flags, /*!< in: table flags */ - ulint flags2) /*!< in: table flags2 */ + const char* name, + ulint space, + ulint n_cols, + ulint n_v_cols, + ulint flags, + ulint flags2, + bool init_stats_latch) { dict_table_t* table; mem_heap_t* heap; @@ -170,16 +165,9 @@ dict_mem_table_create( table->v_cols = static_cast( mem_heap_alloc(heap, n_v_cols * sizeof(*table->v_cols))); - /* true means that the stats latch will be enabled - - dict_table_stats_lock() will not be noop. */ - dict_table_stats_latch_create(table, true); - table->autoinc_lock = static_cast( mem_heap_alloc(heap, lock_get_size())); - /* lazy creation of table autoinc latch */ - dict_table_autoinc_create_lazy(table); - /* If the table has an FTS index or we are in the process of building one, create the table->fts */ if (dict_table_has_fts_index(table) @@ -194,6 +182,12 @@ dict_mem_table_create( new(&table->foreign_set) dict_foreign_set(); new(&table->referenced_set) dict_foreign_set(); + if (init_stats_latch) { + rw_lock_create(dict_table_stats_key, &table->stats_latch, + SYNC_INDEX_TREE); + table->stats_latch_inited = true; + } + return(table); } @@ -222,9 +216,7 @@ dict_mem_table_free( } } - dict_table_autoinc_destroy(table); dict_mem_table_free_foreign_vcol_set(table); - dict_table_stats_latch_destroy(table); table->foreign_set.~dict_foreign_set(); table->referenced_set.~dict_foreign_set(); @@ -245,6 +237,10 @@ dict_mem_table_free( UT_DELETE(table->s_cols); } + if (table->stats_latch_inited) { + rw_lock_free(&table->stats_latch); + } + mem_heap_free(table->heap); } @@ -767,7 +763,7 @@ dict_mem_index_create( dict_mem_fill_index_struct(index, heap, table_name, index_name, space, type, n_fields); - dict_index_zip_pad_mutex_create_lazy(index); + mysql_mutex_init(0, &index->zip_pad.mutex, NULL); if (type & DICT_SPATIAL) { index->rtr_track = static_cast( @@ -1082,7 +1078,7 @@ dict_mem_index_free( ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); - dict_index_zip_pad_mutex_destroy(index); + mysql_mutex_destroy(&index->zip_pad.mutex); if (dict_index_is_spatial(index)) { rtr_info_active::iterator it; diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 563729cd560..5083fda48ee 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -33,6 +33,7 @@ Created Jan 06, 2010 Vasil Dimov #include "pars0pars.h" #include #include "btr0btr.h" +#include "sync0sync.h" #include #include @@ -418,11 +419,6 @@ dict_stats_table_clone_create( t->corrupted = table->corrupted; - /* This private object "t" is not shared with other threads, so - we do not need the stats_latch (thus we pass false below). The - dict_table_stats_lock()/unlock() routines will do nothing. */ - dict_table_stats_latch_create(t, false); - UT_LIST_INIT(t->indexes, &dict_index_t::indexes); #ifdef BTR_CUR_HASH_ADAPT UT_LIST_INIT(t->freed_indexes, &dict_index_t::indexes); @@ -490,6 +486,8 @@ dict_stats_table_clone_create( ut_d(t->magic_n = DICT_TABLE_MAGIC_N); + rw_lock_create(dict_table_stats_key, &t->stats_latch, SYNC_INDEX_TREE); + return(t); } @@ -502,7 +500,7 @@ dict_stats_table_clone_free( /*========================*/ dict_table_t* t) /*!< in: dummy table object to free */ { - dict_table_stats_latch_destroy(t); + rw_lock_free(&t->stats_latch); mem_heap_free(t->heap); } diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 7068dab77a4..37666cdf372 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -1447,7 +1447,7 @@ ibuf_dummy_index_create( table = dict_mem_table_create("IBUF_DUMMY", DICT_HDR_SPACE, n, 0, - comp ? DICT_TF_COMPACT : 0, 0); + comp ? DICT_TF_COMPACT : 0, 0, false); index = dict_mem_index_create("IBUF_DUMMY", "IBUF_DUMMY", DICT_HDR_SPACE, 0, n); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 565ea77374d..49e884d064d 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -302,7 +302,7 @@ UNIV_INLINE void dict_table_autoinc_initialize(dict_table_t* table, ib_uint64_t value) { - ut_ad(dict_table_autoinc_own(table)); + mysql_mutex_assert_owner(&table->autoinc_mutex); table->autoinc = value; } @@ -315,7 +315,7 @@ UNIV_INLINE ib_uint64_t dict_table_autoinc_read(const dict_table_t* table) { - ut_ad(dict_table_autoinc_own(table)); + mysql_mutex_assert_owner(&table->autoinc_mutex); return(table->autoinc); } @@ -329,7 +329,7 @@ UNIV_INLINE bool dict_table_autoinc_update_if_greater(dict_table_t* table, ib_uint64_t value) { - ut_ad(dict_table_autoinc_own(table)); + mysql_mutex_assert_owner(&table->autoinc_mutex); if (value > table->autoinc) { @@ -1524,25 +1524,6 @@ void dict_mutex_exit_for_mysql(void); /*===========================*/ -/** Create a dict_table_t's stats latch or delay for lazy creation. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] table table whose stats latch to create -@param[in] enabled if false then the latch is disabled -and dict_table_stats_lock()/unlock() become noop on this table. */ -void -dict_table_stats_latch_create( - dict_table_t* table, - bool enabled); - -/** Destroy a dict_table_t's stats latch. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] table table whose stats latch to destroy */ -void -dict_table_stats_latch_destroy( - dict_table_t* table); - /** Lock the appropriate latch to protect a given table's statistics. @param[in] table table whose stats to lock @param[in] latch_mode RW_S_LATCH or RW_X_LATCH */ diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index e8005cbc352..3c7913d7f39 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -44,7 +44,6 @@ Created 1/8/1996 Heikki Tuuri #include "fts0fts.h" #include "buf0buf.h" #include "gis0type.h" -#include "os0once.h" #include "fil0fil.h" #include #include "fil0crypt.h" @@ -299,21 +298,27 @@ parent table will fail, and user has to drop excessive foreign constraint before proceeds. */ #define FK_MAX_CASCADE_DEL 15 -/**********************************************************************//** -Creates a table memory object. +/** Creates a table memory object. +@param[in] name table name +@param[in] space space where the clustered index + of the table is placed +@param[in] n_cols total number of columns including + virtual and non-virtual columns +@param[in] n_v_cols number of virtual columns +@param[in] flags table flags +@param[in] flags2 table flags2 +@param[in] init_stats_latch whether to init the stats latch @return own: table object */ dict_table_t* dict_mem_table_create( -/*==================*/ - const char* name, /*!< in: table name */ - ulint space, /*!< in: space where the clustered index - of the table is placed */ - ulint n_cols, /*!< in: total number of columns - including virtual and non-virtual - columns */ - ulint n_v_cols, /*!< in: number of virtual columns */ - ulint flags, /*!< in: table flags */ - ulint flags2); /*!< in: table flags2 */ + const char* name, + ulint space, + ulint n_cols, + ulint n_v_cols, + ulint flags, + ulint flags2, + bool init_stats_latch=true); + /****************************************************************//** Free a table memory object. */ void @@ -792,7 +797,7 @@ extern ulong zip_pad_max; an uncompressed page should be left as padding to avoid compression failures. This estimate is based on a self-adapting heuristic. */ struct zip_pad_info_t { - SysMutex* mutex; /*!< mutex protecting the info */ + mysql_mutex_t mutex; /*!< mutex protecting the info */ ulint pad; /*!< number of bytes used as pad */ ulint success;/*!< successful compression ops during current round */ @@ -800,9 +805,6 @@ struct zip_pad_info_t { current round */ ulint n_rounds;/*!< number of currently successful rounds */ - volatile os_once::state_t - mutex_created; - /*!< Creation state of mutex member */ }; /** Number of samples of data size kept when page compression fails for @@ -1692,7 +1694,7 @@ struct dict_table_t { /** Statistics for query optimization. @{ */ /** Creation state of 'stats_latch'. */ - volatile os_once::state_t stats_latch_created; + bool stats_latch_inited; /** This latch protects: dict_table_t::stat_initialized, @@ -1705,7 +1707,7 @@ struct dict_table_t { dict_table_t::indexes*::stat_n_leaf_pages. (*) Those are not always protected for performance reasons. */ - rw_lock_t* stats_latch; + rw_lock_t stats_latch; /** TRUE if statistics have been calculated the first time after database startup or table creation. */ @@ -1829,11 +1831,8 @@ struct dict_table_t { from a select. */ lock_t* autoinc_lock; - /** Creation state of autoinc_mutex member */ - volatile os_once::state_t autoinc_mutex_created; - /** Mutex protecting the autoincrement counter. */ - ib_mutex_t* autoinc_mutex; + mysql_mutex_t autoinc_mutex; /** Autoinc counter value to give to the next inserted row. */ ib_uint64_t autoinc; @@ -1927,64 +1926,6 @@ struct dict_foreign_add_to_referenced_table { } }; -/** Destroy the autoinc latch of the given table. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] table table whose stats latch to destroy */ -inline -void -dict_table_autoinc_destroy( - dict_table_t* table) -{ - if (table->autoinc_mutex_created == os_once::DONE - && table->autoinc_mutex != NULL) { - mutex_free(table->autoinc_mutex); - UT_DELETE(table->autoinc_mutex); - } -} - -/** Request for lazy creation of the autoinc latch of a given table. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] table table whose autoinc latch is to be created. */ -inline -void -dict_table_autoinc_create_lazy( - dict_table_t* table) -{ - table->autoinc_mutex = NULL; - table->autoinc_mutex_created = os_once::NEVER_DONE; -} - -/** Request a lazy creation of dict_index_t::zip_pad::mutex. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] index index whose zip_pad mutex is to be created */ -inline -void -dict_index_zip_pad_mutex_create_lazy( - dict_index_t* index) -{ - index->zip_pad.mutex = NULL; - index->zip_pad.mutex_created = os_once::NEVER_DONE; -} - -/** Destroy the zip_pad_mutex of the given index. -This function is only called from either single threaded environment -or from a thread that has not shared the table object with other threads. -@param[in,out] table table whose stats latch to destroy */ -inline -void -dict_index_zip_pad_mutex_destroy( - dict_index_t* index) -{ - if (index->zip_pad.mutex_created == os_once::DONE - && index->zip_pad.mutex != NULL) { - mutex_free(index->zip_pad.mutex); - UT_DELETE(index->zip_pad.mutex); - } -} - /** Release the zip_pad_mutex of a given index. @param[in,out] index index whose zip_pad_mutex is to be released */ inline @@ -1992,21 +1933,8 @@ void dict_index_zip_pad_unlock( dict_index_t* index) { - mutex_exit(index->zip_pad.mutex); -} - -#ifdef UNIV_DEBUG -/** Check if the current thread owns the autoinc_mutex of a given table. -@param[in] table the autoinc_mutex belongs to this table -@return true, if the current thread owns the autoinc_mutex, false otherwise.*/ -inline -bool -dict_table_autoinc_own( - const dict_table_t* table) -{ - return(mutex_own(table->autoinc_mutex)); + mysql_mutex_unlock(&index->zip_pad.mutex); } -#endif /* UNIV_DEBUG */ /** Check whether the col is used in spatial index or regular index. @param[in] col column to check diff --git a/storage/innobase/include/os0once.h b/storage/innobase/include/os0once.h deleted file mode 100644 index ad72370eefa..00000000000 --- a/storage/innobase/include/os0once.h +++ /dev/null @@ -1,118 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2014, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/os0once.h -A class that aids executing a given function exactly once in a multi-threaded -environment. - -Created Feb 20, 2014 Vasil Dimov -*******************************************************/ - -#ifndef os0once_h -#define os0once_h - -#include "univ.i" -#include "ut0ut.h" - -/** Execute a given function exactly once in a multi-threaded environment -or wait for the function to be executed by another thread. - -Example usage: -First the user must create a control variable of type os_once::state_t and -assign it os_once::NEVER_DONE. -Then the user must pass this variable, together with a function to be -executed to os_once::do_or_wait_for_done(). - -Multiple threads can call os_once::do_or_wait_for_done() simultaneously with -the same (os_once::state_t) control variable. The provided function will be -called exactly once and when os_once::do_or_wait_for_done() returns then this -function has completed execution, by this or another thread. In other words -os_once::do_or_wait_for_done() will either execute the provided function or -will wait for its execution to complete if it is already called by another -thread or will do nothing if the function has already completed its execution -earlier. - -This mimics pthread_once(3), but unfortunatelly pthread_once(3) does not -support passing arguments to the init_routine() function. We should use -std::call_once() when we start compiling with C++11 enabled. */ -class os_once { -public: - /** Control variables' state type */ - typedef ib_uint32_t state_t; - - /** Not yet executed. */ - static const state_t NEVER_DONE = 0; - - /** Currently being executed by this or another thread. */ - static const state_t IN_PROGRESS = 1; - - /** Finished execution. */ - static const state_t DONE = 2; - - /** Call a given function or wait its execution to complete if it is - already called by another thread. - @param[in,out] state control variable - @param[in] do_func function to call - @param[in,out] do_func_arg an argument to pass to do_func(). */ - static - void - do_or_wait_for_done( - volatile state_t* state, - void (*do_func)(void*), - void* do_func_arg) - { - int32 oldval = NEVER_DONE; - - /* Avoid calling my_atomic_cas32() in the most common case. */ - if (*state == DONE) { - return; - } - - if (my_atomic_cas32((int32*) state, &oldval, IN_PROGRESS)) { - /* We are the first. Call the function. */ - - do_func(do_func_arg); - - my_atomic_store32((int32*) state, DONE); - } else { - /* The state is not NEVER_DONE, so either it is - IN_PROGRESS (somebody is calling the function right - now or DONE (it has already been called and completed). - Wait for it to become DONE. */ - for (;;) { - const state_t s = *state; - - switch (s) { - case DONE: - return; - case IN_PROGRESS: - break; - case NEVER_DONE: - /* fall through */ - default: - ut_error; - } - - UT_RELAX_CPU(); - } - } - } -}; - -#endif /* os0once_h */ diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index c722cdd619a..9664bda6fea 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -1656,7 +1656,7 @@ page_zip_fields_free( { if (index) { dict_table_t* table = index->table; - dict_index_zip_pad_mutex_destroy(index); + mysql_mutex_destroy(&index->zip_pad.mutex); mem_heap_free(index->heap); dict_mem_table_free(table); -- cgit v1.2.1 From e391417f0fdbd746e23808b3d15d6cbe5a3b0aac Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Mon, 26 Oct 2020 12:21:29 +0200 Subject: Bug #30933728 INNODB FTS PHRASE SEARCH HIT AN ASSERT Problem: In Full-text phrase search, we filter out row that do not contain all the tokens in the phrase. If we do not filter out doc_id that doesn't appear in all the token's doc_id lists then we hit an assert. Fix: if any of the token has last doc_id equal to ith doc_id of the first token doc_id list then filter out rest of the higher doc_ids. RB: 24909 Reviewed by : Annamalai Gurusami This is a cherry-pick of mysql/mysql-server@5aa075277dfe84a17a0331c57a6fe9b91dafb4cf but without a test case, because the test case depends on an n-gram tokenizer that will be missing from MariaDB until MDEV-10267 is added. --- storage/innobase/fts/fts0que.cc | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index be758c13d52..df2b330fe4b 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -4421,24 +4421,27 @@ fts_phrase_or_proximity_search( if (k == ib_vector_size(query->match_array[j])) { end_list = TRUE; - if (match[j]->doc_id != match[0]->doc_id) { - /* no match */ - if (query->flags & FTS_PHRASE) { - ulint s; + if (query->flags & FTS_PHRASE) { + ulint s; + /* Since i is the last doc id in the + match_array[j], remove all doc ids > i + from the match_array[0]. */ + fts_match_t* match_temp; + for (s = i + 1; s < n_matched; s++) { + match_temp = static_cast< + fts_match_t*>(ib_vector_get( + query->match_array[0], s)); + match_temp->doc_id = 0; + } + if (match[j]->doc_id != + match[0]->doc_id) { + /* no match */ match[0]->doc_id = 0; - - for (s = i + 1; s < n_matched; - s++) { - match[0] = static_cast< - fts_match_t*>( - ib_vector_get( - query->match_array[0], - s)); - match[0]->doc_id = 0; - } } + } + if (match[j]->doc_id != match[0]->doc_id) { goto func_exit; } } -- cgit v1.2.1 From 1ff8588c3f1f07e7f6e1b00fb5f485488324cc0a Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Fri, 28 Aug 2020 18:18:25 +0530 Subject: Bug #31228694 FTS QUERY WITH LIMIT HIT AN ASSERT Problem: 1. The server terminates abnormally when phrase search doesn't filter out doc_ids correctly. This problem has been fixed in bug 2. Wrong query result: It's a regression from the bug #22709692 fix. This fix optimize full-text search query with limit clause. when FTS expression involves only union operation, we fetch only number of doc_ids specified with the limit clause. Fulltext phrase search is not an union operation and we consider phrase search with plugin parser a union operation. In phrase search with limit clause, we fetch limited doc_ids for each token and if any of the selected doc_id does not contain all tokens in correct order then we do not include that row_id in the result set. Therefore phrase search gets fewer number of rows than the qualified rows exist in the table. Fix: Added a condition that phrase search with plugin parser is not a union operation. RB: 24925 Reviewed by : Annamalai Gurusami This is a cherry-pick of mysql/mysql-server@5549920b7a33ef33034461d973a9ecb17ce49799 without a test case, because the test case depends on an n-gram tokenizer that will be missing from MariaDB until MDEV-10267 is added. --- storage/innobase/fts/fts0ast.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fts/fts0ast.cc b/storage/innobase/fts/fts0ast.cc index e22613a265b..6be4fb0d52b 100644 --- a/storage/innobase/fts/fts0ast.cc +++ b/storage/innobase/fts/fts0ast.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2007, 2020, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -557,8 +557,7 @@ fts_ast_node_check_union( fts_ast_node_t* node) { if (node->type == FTS_AST_LIST - || node->type == FTS_AST_SUBEXP_LIST - || node->type == FTS_AST_PARSER_PHRASE_LIST) { + || node->type == FTS_AST_SUBEXP_LIST) { for (node = node->list.head; node; node = node->next) { if (!fts_ast_node_check_union(node)) { @@ -566,6 +565,9 @@ fts_ast_node_check_union( } } + } else if (node->type == FTS_AST_PARSER_PHRASE_LIST) { + /* Phrase search for plugin parser */ + return(false); } else if (node->type == FTS_AST_OPER && (node->oper == FTS_IGNORE || node->oper == FTS_EXIST)) { -- cgit v1.2.1 From 31cde275c26ba5009d16dfc62654884b94b22322 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 20 Oct 2020 20:10:40 +0300 Subject: MDEV-23356 InnoDB: Failing assertion: field->col->mtype == type, crash or ASAN failures in row_sel_convert_mysql_key_to_innobase, InnoDB indexes are inconsistent after INDEX changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit innobase_rename_indexes_cache(): fix corruption of index cache. Index ids help distinguish indexes when their names clash. innobase_rename_indexes_cache(): fix corruption of index statistics table. Use unique temporary names to avoid names clashing. Reviewed by: Marko Mäkelä --- storage/innobase/handler/handler0alter.cc | 73 ++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 15 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index fef2d9650c0..5700bdcacb6 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -57,6 +57,8 @@ Smart ALTER TABLE #include "ha_innodb.h" #include "ut0stage.h" #include "span.h" +#include +#include using st_::span; /** File format constraint for ALTER TABLE */ @@ -7450,22 +7452,28 @@ innobase_rename_index_cache(dict_index_t* index, const char* new_name) /** Rename the index name in cache. @param[in] ctx alter context @param[in] ha_alter_info Data used during inplace alter. */ -static void innobase_rename_indexes_cache( - const ha_innobase_inplace_ctx* ctx, - const Alter_inplace_info* ha_alter_info) +static void +innobase_rename_indexes_cache(const ha_innobase_inplace_ctx *ctx, + const Alter_inplace_info *ha_alter_info) { - DBUG_ASSERT(ha_alter_info->handler_flags & ALTER_RENAME_INDEX); + DBUG_ASSERT(ha_alter_info->handler_flags & ALTER_RENAME_INDEX); - for (const Alter_inplace_info::Rename_key_pair& pair : - ha_alter_info->rename_keys) { - dict_index_t* index = dict_table_get_index_on_name( - ctx->old_table, pair.old_key->name.str); - ut_ad(index); + std::vector> rename_info; + rename_info.reserve(ha_alter_info->rename_keys.size()); - innobase_rename_index_cache(index, pair.new_key->name.str); - } -} + for (const Alter_inplace_info::Rename_key_pair &pair : + ha_alter_info->rename_keys) + { + dict_index_t *index= + dict_table_get_index_on_name(ctx->old_table, pair.old_key->name.str); + ut_ad(index); + rename_info.emplace_back(index, pair.new_key->name.str); + } + + for (const auto &pair : rename_info) + innobase_rename_index_cache(pair.first, pair.second); +} /** Fill the stored column information in s_cols list. @param[in] altered_table mysql table object @@ -10600,11 +10608,18 @@ alter_stats_norebuild( } } - for (const Alter_inplace_info::Rename_key_pair& pair : - ha_alter_info->rename_keys) { + for (size_t i = 0; i < ha_alter_info->rename_keys.size(); i++) { + const Alter_inplace_info::Rename_key_pair& pair + = ha_alter_info->rename_keys[i]; + + std::stringstream ss; + ss << TEMP_FILE_PREFIX_INNODB << std::this_thread::get_id() + << i; + auto tmp_name = ss.str(); + dberr_t err = dict_stats_rename_index(ctx->new_table, pair.old_key->name.str, - pair.new_key->name.str); + tmp_name.c_str()); if (err != DB_SUCCESS) { push_warning_printf( @@ -10616,6 +10631,34 @@ alter_stats_norebuild( " statistics storage: %s", ctx->new_table->name.m_name, pair.old_key->name.str, + tmp_name.c_str(), + ut_strerr(err)); + } + } + + for (size_t i = 0; i < ha_alter_info->rename_keys.size(); i++) { + const Alter_inplace_info::Rename_key_pair& pair + = ha_alter_info->rename_keys[i]; + + std::stringstream ss; + ss << TEMP_FILE_PREFIX_INNODB << std::this_thread::get_id() + << i; + auto tmp_name = ss.str(); + + dberr_t err = dict_stats_rename_index(ctx->new_table, + tmp_name.c_str(), + pair.new_key->name.str); + + if (err != DB_SUCCESS) { + push_warning_printf( + thd, + Sql_condition::WARN_LEVEL_WARN, + ER_ERROR_ON_RENAME, + "Error renaming an index of table '%s'" + " from '%s' to '%s' in InnoDB persistent" + " statistics storage: %s", + ctx->new_table->name.m_name, + tmp_name.c_str(), pair.new_key->name.str, ut_strerr(err)); } -- cgit v1.2.1 From bc540b8706a404c8aec81a599c7e7ec1a46b5ad1 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 27 Oct 2020 17:56:49 +0530 Subject: MDEV-23693 Failing assertion: my_atomic_load32_explicit(&lock->lock_word, MY_MEMORY_ORDER_RELAXED) == X_LOCK_DECR InnoDB frees the block lock during buffer pool shrinking when other thread is yet to release the block lock. While shrinking the buffer pool, InnoDB allows the page to be freed unless it is buffer fixed. In some cases, InnoDB releases the latch after unfixing the block. Fix: ==== - InnoDB should unfix the block after releases the latch. - Add more assertion to check buffer fix while accessing the page. - Introduced block_hint structure to store buf_block_t pointer and allow accessing the buf_block_t pointer only by passing a functor. It returns original buf_block_t* pointer if it is valid or nullptr if the pointer become stale. - Replace buf_block_is_uncompressed() with buf_pool_t::is_block_pointer() This change is motivated by a change in mysql-5.7.32: mysql/mysql-server@46e60de444a8fbd876cc6778a7e64a1d3426a48d Bug #31036301 ASSERTION FAILURE: SYNC0RW.IC:429:LOCK->LOCK_WORD --- storage/innobase/CMakeLists.txt | 1 + storage/innobase/btr/btr0bulk.cc | 4 ++ storage/innobase/btr/btr0cur.cc | 43 +++++++---------- storage/innobase/btr/btr0pcur.cc | 39 +++++++++++----- storage/innobase/buf/buf0block_hint.cc | 78 +++++++++++++++++++++++++++++++ storage/innobase/buf/buf0buf.cc | 76 +----------------------------- storage/innobase/gis/gis0sea.cc | 25 ++++++++-- storage/innobase/include/btr0pcur.h | 11 ++--- storage/innobase/include/buf0block_hint.h | 77 ++++++++++++++++++++++++++++++ storage/innobase/include/buf0buf.h | 17 ++----- storage/innobase/include/buf0buf.ic | 35 ++++++++------ storage/innobase/include/mtr0mtr.ic | 4 +- storage/innobase/include/trx0undo.h | 2 - storage/innobase/mtr/mtr0mtr.cc | 4 +- storage/innobase/trx/trx0rec.cc | 10 ++-- storage/innobase/trx/trx0undo.cc | 1 - 16 files changed, 262 insertions(+), 165 deletions(-) create mode 100644 storage/innobase/buf/buf0block_hint.cc create mode 100644 storage/innobase/include/buf0block_hint.h (limited to 'storage') diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 1b5345cbddc..08e8e3ff5ab 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -28,6 +28,7 @@ SET(INNOBASE_SOURCES btr/btr0scrub.cc btr/btr0sea.cc btr/btr0defragment.cc + buf/buf0block_hint.cc buf/buf0buddy.cc buf/buf0buf.cc buf/buf0dblwr.cc diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc index 96ff81c9fab..100e48cf537 100644 --- a/storage/innobase/btr/btr0bulk.cc +++ b/storage/innobase/btr/btr0bulk.cc @@ -695,6 +695,8 @@ PageBulk::latch() m_mtr.set_named_space(m_index->space); } + ut_ad(m_block->page.buf_fix_count); + /* In case the block is S-latched by page_cleaner. */ if (!buf_page_optimistic_get(RW_X_LATCH, m_block, m_modify_clock, __FILE__, __LINE__, &m_mtr)) { @@ -713,6 +715,8 @@ PageBulk::latch() buf_block_buf_fix_dec(m_block); + ut_ad(m_block->page.buf_fix_count); + ut_ad(m_cur_rec > m_page && m_cur_rec < m_heap_top); return (m_err); diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 770c6d73585..6ae6bad10c8 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -416,6 +416,8 @@ btr_cur_optimistic_latch_leaves( ulint mode; ulint left_page_no; ulint curr_page_no; + ut_ad(block->page.buf_fix_count); + ut_ad(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); switch (*latch_mode) { case BTR_SEARCH_LEAF: @@ -427,20 +429,10 @@ btr_cur_optimistic_latch_leaves( mode = *latch_mode == BTR_SEARCH_PREV ? RW_S_LATCH : RW_X_LATCH; - buf_page_mutex_enter(block); - if (buf_block_get_state(block) != BUF_BLOCK_FILE_PAGE) { - buf_page_mutex_exit(block); - return(false); - } - /* pin the block not to be relocated */ - buf_block_buf_fix_inc(block, file, line); - buf_page_mutex_exit(block); - rw_lock_s_lock(&block->lock); if (block->modify_clock != modify_clock) { rw_lock_s_unlock(&block->lock); - - goto unpin_failed; + return false; } curr_page_no = block->page.id.page_no(); @@ -470,7 +462,7 @@ btr_cur_optimistic_latch_leaves( /* release the left block */ btr_leaf_page_release( cursor->left_block, mode, mtr); - goto unpin_failed; + return false; } } else { cursor->left_block = NULL; @@ -480,23 +472,28 @@ btr_cur_optimistic_latch_leaves( file, line, mtr)) { if (btr_page_get_prev(buf_block_get_frame(block)) == left_page_no) { - buf_block_buf_fix_dec(block); + /* block was already buffer-fixed while + entering the function and + buf_page_optimistic_get() buffer-fixes + it again. */ + ut_ad(2 <= block->page.buf_fix_count); *latch_mode = mode; return(true); } else { - /* release the block */ + /* release the block and decrement of + buf_fix_count which was incremented + in buf_page_optimistic_get() */ btr_leaf_page_release(block, mode, mtr); } } + ut_ad(block->page.buf_fix_count); /* release the left block */ if (cursor->left_block != NULL) { btr_leaf_page_release(cursor->left_block, mode, mtr); } -unpin_failed: - /* unpin the block */ - buf_block_buf_fix_dec(block); + return(false); default: @@ -1066,12 +1063,7 @@ btr_cur_search_to_nth_level_func( guess = NULL; #else info = btr_search_get_info(index); - - if (!buf_pool_is_obsolete(info->withdraw_clock)) { - guess = info->root_guess; - } else { - guess = NULL; - } + guess = info->root_guess; #ifdef BTR_CUR_HASH_ADAPT rw_lock_t* const search_latch = btr_get_search_latch(index); @@ -1509,10 +1501,7 @@ retry_page_get: } #ifdef BTR_CUR_ADAPT - if (block != guess) { - info->root_guess = block; - info->withdraw_clock = buf_withdraw_clock; - } + info->root_guess = block; #endif } diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 4340c2f32b0..560774c715e 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -165,11 +165,10 @@ btr_pcur_store_position( index, rec, &cursor->old_n_fields, &cursor->old_rec_buf, &cursor->buf_size); - cursor->block_when_stored = block; + cursor->block_when_stored.store(block); /* Function try to check if block is S/X latch. */ cursor->modify_clock = buf_block_get_modify_clock(block); - cursor->withdraw_clock = buf_withdraw_clock; } /**************************************************************//** @@ -199,6 +198,26 @@ btr_pcur_copy_stored_position( pcur_receive->old_n_fields = pcur_donate->old_n_fields; } +/** Structure acts as functor to do the latching of leaf pages. +It returns true if latching of leaf pages succeeded and false +otherwise. */ +struct optimistic_latch_leaves +{ + btr_pcur_t *const cursor; + ulint *latch_mode; + mtr_t *const mtr; + + optimistic_latch_leaves(btr_pcur_t *cursor, ulint *latch_mode, mtr_t *mtr) + :cursor(cursor), latch_mode(latch_mode), mtr(mtr) {} + + bool operator() (buf_block_t *hint) const + { + return hint && btr_cur_optimistic_latch_leaves( + hint, cursor->modify_clock, latch_mode, + btr_pcur_get_btr_cur(cursor), __FILE__, __LINE__, mtr); + } +}; + /**************************************************************//** Restores the stored position of a persistent cursor bufferfixing the page and obtaining the specified latches. If the cursor position was saved when the @@ -261,7 +280,7 @@ btr_pcur_restore_position_func( cursor->latch_mode = BTR_LATCH_MODE_WITHOUT_INTENTION(latch_mode); cursor->pos_state = BTR_PCUR_IS_POSITIONED; - cursor->block_when_stored = btr_pcur_get_block(cursor); + cursor->block_when_stored.clear(); return(FALSE); } @@ -276,12 +295,9 @@ btr_pcur_restore_position_func( case BTR_MODIFY_PREV: /* Try optimistic restoration. */ - if (!buf_pool_is_obsolete(cursor->withdraw_clock) - && btr_cur_optimistic_latch_leaves( - cursor->block_when_stored, cursor->modify_clock, - &latch_mode, btr_pcur_get_btr_cur(cursor), - file, line, mtr)) { - + if (cursor->block_when_stored.run_with_hint( + optimistic_latch_leaves(cursor, &latch_mode, + mtr))) { cursor->pos_state = BTR_PCUR_IS_POSITIONED; cursor->latch_mode = latch_mode; @@ -378,11 +394,10 @@ btr_pcur_restore_position_func( since the cursor can now be on a different page! But we can retain the value of old_rec */ - cursor->block_when_stored = btr_pcur_get_block(cursor); + cursor->block_when_stored.store(btr_pcur_get_block(cursor)); cursor->modify_clock = buf_block_get_modify_clock( - cursor->block_when_stored); + cursor->block_when_stored.block()); cursor->old_stored = true; - cursor->withdraw_clock = buf_withdraw_clock; mem_heap_free(heap); diff --git a/storage/innobase/buf/buf0block_hint.cc b/storage/innobase/buf/buf0block_hint.cc new file mode 100644 index 00000000000..9f974e8304d --- /dev/null +++ b/storage/innobase/buf/buf0block_hint.cc @@ -0,0 +1,78 @@ +/***************************************************************************** + +Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2020, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ + +#include "buf0block_hint.h" +namespace buf { + +void Block_hint::buffer_fix_block_if_still_valid() +{ + /* We need to check if m_block points to one of chunks. For this to be + meaningful we need to prevent freeing memory while we check, and until we + buffer-fix the block. For this purpose it is enough to latch any of the many + latches taken by buf_resize(). + However, for buffer-fixing to be meaningful, the block has to contain a page + (as opposed to being already empty, which might mean that buf_pool_resize() + can proceed and free it once we free the s-latch), so we confirm that the + block contains a page. However, it is not sufficient to check that this is + just any page, because just after we check it could get freed, unless we + have a latch which prevents this. This is tricky because page_hash latches + are sharded by page_id and we don't know the page_id until we look into the + block. To solve this chicken-and-egg problem somewhat, we latch the shard + for the m_page_id and compare block->page.id to it - so if is equal then we + can be reasonably sure that we have the correct latch. + There is still a theoretical problem here, where other threads might try + to modify the m_block->page.id while we are comparing it, but the chance of + accidentally causing the old space_id == m_page_id.m_space and the new + page_no == m_page_id.m_page_no is minimal as compilers emit a single 8-byte + comparison instruction to compare both at the same time atomically, and f() + will probably double-check the block->page.id again, anyway. + Finally, assuming that we have correct hash bucket latched, we should check if + the state of the block is BUF_BLOCK_FILE_PAGE before buffer-fixing the block, + as otherwise we risk buffer-fixing and operating on a block, which is already + meant to be freed. In particular, buf_LRU_free_page() first calls + buf_LRU_block_remove_hashed() under hash bucket latch protection to change the + state to BUF_BLOCK_REMOVE_HASH and then releases the latch. Later it calls + buf_LRU_block_free_hashed_page() without any latch to change the state to + BUF_BLOCK_MEMORY and reset the page's id, which means buf_resize() can free it + regardless of our buffer-fixing. */ + if (m_block) + { + const buf_pool_t *const buf_pool= buf_pool_get(m_page_id); + rw_lock_t *latch= buf_page_hash_lock_get(buf_pool, m_page_id); + rw_lock_s_lock(latch); + /* If not own buf_pool_mutex, page_hash can be changed. */ + latch= buf_page_hash_lock_s_confirm(latch, buf_pool, m_page_id); + if (buf_pool->is_block_field(m_block) && + m_page_id == m_block->page.id && + buf_block_get_state(m_block) == BUF_BLOCK_FILE_PAGE) + buf_block_buf_fix_inc(m_block, __FILE__, __LINE__); + else + clear(); + rw_lock_s_unlock(latch); + } +} +} // namespace buf diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index cfc79868482..e2b68011077 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -339,14 +339,6 @@ buf_pool_t* buf_pool_ptr; /** true when resizing buffer pool is in the critical path. */ volatile bool buf_pool_resizing; -/** true when withdrawing buffer pool pages might cause page relocation */ -volatile bool buf_pool_withdrawing; - -/** the clock is incremented every time a pointer to a page may become obsolete; -if the withdrwa clock has not changed, the pointer is still valid in buffer -pool. if changed, the pointer might not be in buffer pool any more. */ -volatile ulint buf_withdraw_clock; - /** Map of buffer pool chunks by its first frame address This is newly made by initialization of buffer pool and buf_resize_thread. Currently, no need mutex protection for update. */ @@ -2068,8 +2060,6 @@ buf_pool_init( NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE; buf_pool_resizing = false; - buf_pool_withdrawing = false; - buf_withdraw_clock = 0; buf_pool_ptr = (buf_pool_t*) ut_zalloc_nokey( n_instances * sizeof *buf_pool_ptr); @@ -2129,7 +2119,6 @@ buf_page_realloc( { buf_block_t* new_block; - ut_ad(buf_pool_withdrawing); ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); @@ -2551,9 +2540,6 @@ buf_pool_withdraw_blocks( ib::info() << "buffer pool " << i << " : withdrawn target " << UT_LIST_GET_LEN(buf_pool->withdraw) << " blocks."; - /* retry is not needed */ - ++buf_withdraw_clock; - return(false); } @@ -2650,7 +2636,6 @@ buf_pool_resize() NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE; ut_ad(!buf_pool_resizing); - ut_ad(!buf_pool_withdrawing); ut_ad(srv_buf_pool_chunk_unit > 0); new_instance_size = srv_buf_pool_size / srv_buf_pool_instances; @@ -2717,7 +2702,6 @@ buf_pool_resize() ut_ad(buf_pool->withdraw_target == 0); buf_pool->withdraw_target = withdraw_target; - buf_pool_withdrawing = true; } } @@ -2742,7 +2726,6 @@ withdraw_retry: if (srv_shutdown_state != SRV_SHUTDOWN_NONE) { /* abort to resize for shutdown. */ - buf_pool_withdrawing = false; return; } @@ -2804,7 +2787,6 @@ withdraw_retry: goto withdraw_retry; } - buf_pool_withdrawing = false; buf_resize_status("Latching whole of buffer pool."); @@ -3981,37 +3963,6 @@ buf_block_from_ahi(const byte* ptr) } #endif /* BTR_CUR_HASH_ADAPT */ -/********************************************************************//** -Find out if a pointer belongs to a buf_block_t. It can be a pointer to -the buf_block_t itself or a member of it. This functions checks one of -the buffer pool instances. -@return TRUE if ptr belongs to a buf_block_t struct */ -static -ibool -buf_pointer_is_block_field_instance( -/*================================*/ - buf_pool_t* buf_pool, /*!< in: buffer pool instance */ - const void* ptr) /*!< in: pointer not dereferenced */ -{ - const buf_chunk_t* chunk = buf_pool->chunks; - const buf_chunk_t* const echunk = chunk + ut_min( - buf_pool->n_chunks, buf_pool->n_chunks_new); - - /* TODO: protect buf_pool->chunks with a mutex (the older pointer will - currently remain while during buf_pool_resize()) */ - while (chunk < echunk) { - if (ptr >= (void*) chunk->blocks - && ptr < (void*) (chunk->blocks + chunk->size)) { - - return(TRUE); - } - - chunk++; - } - - return(FALSE); -} - /********************************************************************//** Find out if a pointer belongs to a buf_block_t. It can be a pointer to the buf_block_t itself or a member of it @@ -4024,11 +3975,7 @@ buf_pointer_is_block_field( ulint i; for (i = 0; i < srv_buf_pool_instances; i++) { - ibool found; - - found = buf_pointer_is_block_field_instance( - buf_pool_from_array(i), ptr); - if (found) { + if (buf_pool_from_array(i)->is_block_field(ptr)) { return(TRUE); } } @@ -4036,25 +3983,6 @@ buf_pointer_is_block_field( return(FALSE); } -/********************************************************************//** -Find out if a buffer block was created by buf_chunk_init(). -@return TRUE if "block" has been added to buf_pool->free by buf_chunk_init() */ -static -ibool -buf_block_is_uncompressed( -/*======================*/ - buf_pool_t* buf_pool, /*!< in: buffer pool instance */ - const buf_block_t* block) /*!< in: pointer to block, - not dereferenced */ -{ - if ((((ulint) block) % sizeof *block) != 0) { - /* The pointer should be aligned. */ - return(FALSE); - } - - return(buf_pointer_is_block_field_instance(buf_pool, (void*) block)); -} - #if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG /********************************************************************//** Return true if probe is enabled. @@ -4293,7 +4221,7 @@ loop: has been allocated by buf_page_alloc_descriptor(), it may have been freed by buf_relocate(). */ - if (!buf_block_is_uncompressed(buf_pool, block) + if (!buf_pool->is_block_field(block) || page_id != block->page.id || buf_block_get_state(block) != BUF_BLOCK_FILE_PAGE) { diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index c372a3f1cd4..5ea3f328d5c 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -1257,6 +1257,24 @@ rtr_check_discard_page( lock_mutex_exit(); } +/** Structure acts as functor to get the optimistic access of the page. +It returns true if it successfully gets the page. */ +struct optimistic_get +{ + btr_pcur_t *const r_cursor; + mtr_t *const mtr; + + optimistic_get(btr_pcur_t *r_cursor,mtr_t *mtr) + :r_cursor(r_cursor), mtr(mtr) {} + + bool operator()(buf_block_t *hint) const + { + return hint && buf_page_optimistic_get( + RW_X_LATCH, hint, r_cursor->modify_clock, __FILE__, + __LINE__, mtr); + } +}; + /** Restore the stored position of a persistent cursor bufferfixing the page */ static bool @@ -1290,11 +1308,8 @@ rtr_cur_restore_position( ut_ad(latch_mode == BTR_CONT_MODIFY_TREE); - if (!buf_pool_is_obsolete(r_cursor->withdraw_clock) - && buf_page_optimistic_get(RW_X_LATCH, - r_cursor->block_when_stored, - r_cursor->modify_clock, - __FILE__, __LINE__, mtr)) { + if (r_cursor->block_when_stored.run_with_hint( + optimistic_get(r_cursor, mtr))) { ut_ad(r_cursor->pos_state == BTR_PCUR_IS_POSITIONED); ut_ad(r_cursor->rel_pos == BTR_PCUR_ON); diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index 26c7ebcee6d..9e3b4fc20a6 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -29,6 +29,7 @@ Created 2/23/1996 Heikki Tuuri #include "dict0dict.h" #include "btr0cur.h" +#include "buf0block_hint.h" #include "btr0btr.h" #include "gis0rtree.h" @@ -514,13 +515,10 @@ struct btr_pcur_t{ whether cursor was on, before, or after the old_rec record */ enum btr_pcur_pos_t rel_pos; /** buffer block when the position was stored */ - buf_block_t* block_when_stored; + buf::Block_hint block_when_stored; /** the modify clock value of the buffer block when the cursor position was stored */ ib_uint64_t modify_clock; - /** the withdraw clock value of the buffer pool when the cursor - position was stored */ - ulint withdraw_clock; /** btr_pcur_store_position() and btr_pcur_restore_position() state. */ enum pcur_pos_t pos_state; /** PAGE_CUR_G, ... */ @@ -540,9 +538,8 @@ struct btr_pcur_t{ btr_pcur_t() : btr_cur(), latch_mode(0), old_stored(false), old_rec(NULL), old_n_fields(0), rel_pos(btr_pcur_pos_t(0)), - block_when_stored(NULL), - modify_clock(0), withdraw_clock(0), - pos_state(BTR_PCUR_NOT_POSITIONED), + block_when_stored(), + modify_clock(0), pos_state(BTR_PCUR_NOT_POSITIONED), search_mode(PAGE_CUR_UNSUPP), trx_if_known(NULL), old_rec_buf(NULL), buf_size(0) { diff --git a/storage/innobase/include/buf0block_hint.h b/storage/innobase/include/buf0block_hint.h new file mode 100644 index 00000000000..2d681175b25 --- /dev/null +++ b/storage/innobase/include/buf0block_hint.h @@ -0,0 +1,77 @@ +/***************************************************************************** + +Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2020, MariaDB Corporation. +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ +#pragma once +#include "buf0buf.h" + +namespace buf { +class Block_hint { + public: + Block_hint():m_block(NULL),m_page_id(0,0) {} + /** Stores the pointer to the block, which is currently buffer-fixed. + @param block a pointer to a buffer-fixed block to be stored */ + inline void store(buf_block_t *block) + { + ut_ad(block->page.buf_fix_count); + m_block= block; + m_page_id= block->page.id; + } + + /** Clears currently stored pointer. */ + inline void clear() { m_block= NULL; } + + /** Invoke f on m_block(which may be null) + @param f The function to be executed. It will be passed the pointer. + If you wish to use the block pointer subsequently, + you need to ensure you buffer-fix it before returning from f. + @return the return value of f + */ + template + bool run_with_hint(const F &f) + { + buffer_fix_block_if_still_valid(); + /* m_block could be changed during f() call, so we use local + variable to remember which block we need to unfix */ + buf_block_t *block= m_block; + bool res= f(block); + if (block) + buf_block_buf_fix_dec(block); + return res; + } + + buf_block_t *block() const { return m_block; } + + private: + /** The block pointer stored by store(). */ + buf_block_t *m_block; + /** If m_block is non-null, the m_block->page.id at time it was stored. */ + page_id_t m_page_id; + + /** A helper function which checks if m_block is not a dangling pointer and + still points to block with page with m_page_id and if so, buffer-fixes it, + otherwise clear()s it */ + void buffer_fix_block_if_still_valid(); +}; +} // namespace buf diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 5f390d2761d..40d2e6b2023 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -101,10 +101,6 @@ extern buf_pool_t* buf_pool_ptr; /*!< The buffer pools extern volatile bool buf_pool_withdrawing; /*!< true when withdrawing buffer pool pages might cause page relocation */ -extern volatile ulint buf_withdraw_clock; /*!< the clock is incremented - every time a pointer to a page may - become obsolete */ - # ifdef UNIV_DEBUG extern my_bool buf_disable_resize_buffer_pool_debug; /*!< if TRUE, resizing buffer pool is not allowed. */ @@ -1373,14 +1369,6 @@ buf_get_nth_chunk_block( ulint n, /*!< in: nth chunk in the buffer pool */ ulint* chunk_size); /*!< in: chunk size */ -/** Verify the possibility that a stored page is not in buffer pool. -@param[in] withdraw_clock withdraw clock when stored the page -@retval true if the page might be relocated */ -UNIV_INLINE -bool -buf_pool_is_obsolete( - ulint withdraw_clock); - /** Calculate aligned buffer pool size based on srv_buf_pool_chunk_unit, if needed. @param[in] size size in bytes @@ -2207,6 +2195,11 @@ struct buf_pool_t{ buf_tmp_array_t* tmp_arr; /*!< Array for temporal memory used in compression and encryption */ + /** Determine if a pointer belongs to a buf_block_t. + It can be a pointer to the buf_block_t itself or a member of it. + @param ptr a pointer that will not be dereferenced + @return whether the ptr belongs to a buf_block_t struct */ + inline bool is_block_field(const void *ptr) const; #if BUF_BUDDY_LOW > UNIV_ZIP_SIZE_MIN # error "BUF_BUDDY_LOW > UNIV_ZIP_SIZE_MIN" diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index 3df17e8a978..49b741ab5c8 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -54,6 +54,25 @@ struct buf_chunk_t{ } }; +bool buf_pool_t::is_block_field(const void *ptr) const +{ + const buf_chunk_t* chunk= chunks; + const buf_chunk_t *const echunk= chunk + ut_min(n_chunks, + n_chunks_new); + /* TODO: protect chunks with a mutex (the older pointer will + currently remain during resize()) */ + while (chunk < echunk) + { + if (ptr >= reinterpret_cast(chunk->blocks) && + ptr < reinterpret_cast( + chunk->blocks + chunk->size)) + return true; + chunk++; + } + + return false; +} + /*********************************************************************//** Gets the current size of buffer buf_pool in bytes. @return size in bytes */ @@ -1056,8 +1075,6 @@ buf_block_buf_fix_dec( /*==================*/ buf_block_t* block) /*!< in/out: block to bufferunfix */ { - buf_block_unfix(block); - #ifdef UNIV_DEBUG /* No debug latch is acquired if block belongs to system temporary. Debug latch is not of much help if access to block is single @@ -1066,6 +1083,8 @@ buf_block_buf_fix_dec( rw_lock_s_unlock(&block->debug_latch); } #endif /* UNIV_DEBUG */ + + buf_block_unfix(block); } /** Returns the buffer pool instance given a page id. @@ -1439,18 +1458,6 @@ buf_page_get_frame( } } -/** Verify the possibility that a stored page is not in buffer pool. -@param[in] withdraw_clock withdraw clock when stored the page -@retval true if the page might be relocated */ -UNIV_INLINE -bool -buf_pool_is_obsolete( - ulint withdraw_clock) -{ - return(buf_pool_withdrawing - || buf_withdraw_clock != withdraw_clock); -} - /** Calculate aligned buffer pool size based on srv_buf_pool_chunk_unit, if needed. @param[in] size size in bytes diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index 8d68affb1cb..a45d088d5d7 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -170,10 +170,10 @@ mtr_t::release_block_at_savepoint( ut_a(slot->object == block); - buf_block_unfix(reinterpret_cast(block)); - buf_page_release_latch(block, slot->type); + buf_block_unfix(reinterpret_cast(block)); + slot->object = NULL; } diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index bf2d3c7f7f7..99330453c33 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -424,8 +424,6 @@ struct trx_undo_t { undo_no_t top_undo_no; /*!< undo number of the latest record */ buf_block_t* guess_block; /*!< guess for the buffer block where the top page might reside */ - ulint withdraw_clock; /*!< the withdraw clock value of the - buffer pool when guess_block was stored */ /*-----------------------------*/ UT_LIST_NODE_T(trx_undo_t) undo_list; /*!< undo log objects in the rollback diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 424ac0b53f0..bc30d6efd55 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -233,8 +233,8 @@ static void memo_slot_release(mtr_memo_slot_t *slot) case MTR_MEMO_PAGE_SX_FIX: case MTR_MEMO_PAGE_X_FIX: buf_block_t *block= reinterpret_cast(slot->object); - buf_block_unfix(block); buf_page_release_latch(block, slot->type); + buf_block_unfix(block); break; } slot->object= NULL; @@ -276,8 +276,8 @@ struct ReleaseLatches { case MTR_MEMO_PAGE_SX_FIX: case MTR_MEMO_PAGE_X_FIX: buf_block_t *block= reinterpret_cast(slot->object); - buf_block_unfix(block); buf_page_release_latch(block, slot->type); + buf_block_unfix(block); break; } slot->object= NULL; diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 623d8990381..4aecc8ae610 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1950,8 +1950,7 @@ dberr_t trx_undo_report_rename(trx_t* trx, const dict_table_t* table) buf_block_t* block = buf_page_get_gen( page_id_t(undo->space, undo->last_page_no), univ_page_size, RW_X_LATCH, - buf_pool_is_obsolete(undo->withdraw_clock) - ? NULL : undo->guess_block, + undo->guess_block, BUF_GET, __FILE__, __LINE__, &mtr, &err); ut_ad((err == DB_SUCCESS) == !!block); @@ -1962,7 +1961,6 @@ dberr_t trx_undo_report_rename(trx_t* trx, const dict_table_t* table) if (ulint offset = trx_undo_page_report_rename( trx, table, block, &mtr)) { - undo->withdraw_clock = buf_withdraw_clock; undo->empty = FALSE; undo->top_page_no = undo->last_page_no; undo->top_offset = offset; @@ -2084,8 +2082,7 @@ trx_undo_report_row_operation( undo_block = buf_page_get_gen( page_id_t(undo->space, page_no), univ_page_size, RW_X_LATCH, - buf_pool_is_obsolete(undo->withdraw_clock) - ? NULL : undo->guess_block, BUF_GET, __FILE__, __LINE__, + undo->guess_block, BUF_GET, __FILE__, __LINE__, &mtr, &err); buf_block_dbg_add_level(undo_block, SYNC_TRX_UNDO_PAGE); @@ -2138,14 +2135,13 @@ trx_undo_report_row_operation( mtr_commit(&mtr); } else { /* Success */ - undo->withdraw_clock = buf_withdraw_clock; + undo->guess_block = undo_block; mtr_commit(&mtr); undo->empty = FALSE; undo->top_page_no = page_no; undo->top_offset = offset; undo->top_undo_no = trx->undo_no++; - undo->guess_block = undo_block; trx->undo_rseg_space = rseg->space; diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 5d4fcdcf53f..5e8467af298 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -1338,7 +1338,6 @@ trx_undo_mem_create( undo->empty = TRUE; undo->top_page_no = page_no; undo->guess_block = NULL; - undo->withdraw_clock = 0; return(undo); } -- cgit v1.2.1 From 42e1815ad850384fad292534665ff61b78dd96f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 27 Oct 2020 15:35:04 +0200 Subject: MDEV-16952 Introduce SET GLOBAL innodb_max_purge_lag_wait Let us introduce a dummy variable innodb_max_purge_lag_wait for waiting that the InnoDB history list length is below the user-specified limit. Specifically, SET GLOBAL innodb_max_purge_lag_wait=0; should wait for all history to be purged. This could be useful when upgrading from an older version to MariaDB 10.3 or later, to avoid hitting MDEV-15912. Note: the history cannot be purged if there exist transactions that may see old versions. Reviewed by: Vladislav Vaintroub --- storage/innobase/handler/ha_innodb.cc | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e36dcbad8ff..3db462ee92d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -271,6 +271,27 @@ enum default_row_format_enum { DEFAULT_ROW_FORMAT_DYNAMIC = 2, }; +/** A dummy variable */ +static uint innodb_max_purge_lag_wait; + +/** Wait for trx_sys_t::rseg_history_len to be below a limit. */ +static void innodb_max_purge_lag_wait_update(THD *thd, st_mysql_sys_var *, + void *, const void *limit) +{ + const uint l= *static_cast(limit); + if (trx_sys->rseg_history_len <= l) + return; + mysql_mutex_unlock(&LOCK_global_system_variables); + while (trx_sys->rseg_history_len > l) + { + if (thd_kill_level(thd)) + break; + srv_wake_purge_thread_if_not_active(); + os_thread_sleep(100000); + } + mysql_mutex_lock(&LOCK_global_system_variables); +} + static void set_my_errno(int err) { @@ -20147,6 +20168,11 @@ static MYSQL_SYSVAR_ULONG(max_purge_lag_delay, srv_max_purge_lag_delay, 0L, /* Minimum value */ 10000000UL, 0); /* Maximum value */ +static MYSQL_SYSVAR_UINT(max_purge_lag_wait, innodb_max_purge_lag_wait, + PLUGIN_VAR_RQCMDARG, + "Wait until History list length is below the specified limit", + NULL, innodb_max_purge_lag_wait_update, UINT_MAX, 0, UINT_MAX, 0); + static MYSQL_SYSVAR_BOOL(rollback_on_timeout, innobase_rollback_on_timeout, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, "Roll back the complete transaction on lock wait timeout, for 4.x compatibility (disabled by default)", @@ -21195,6 +21221,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(flushing_avg_loops), MYSQL_SYSVAR(max_purge_lag), MYSQL_SYSVAR(max_purge_lag_delay), + MYSQL_SYSVAR(max_purge_lag_wait), MYSQL_SYSVAR(old_blocks_pct), MYSQL_SYSVAR(old_blocks_time), MYSQL_SYSVAR(open_files), -- cgit v1.2.1 From afc9d00c66db946c8240fe1fa6b345a3a8b6fec1 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 27 Oct 2020 12:24:55 +0300 Subject: MDEV-23991 dict_table_stats_lock() has unnecessarily long scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch removes dict_index_t::stats_latch. Table/index statistics now protected with dict_sys->mutex. That way statistics computation can happen in parallel in several threads and dict_sys->mutex will be locked only for a short period of time. This patch is a joint work with Marko Mäkelä dict_index_t::lock: make mutable which allows to pass const pointer when only lock is touched in an object btr_height_get() btr_get_size(): make index argument const for better type safety btr_estimate_number_of_different_key_vals(): now returns computed values instead of setting fields in dict_index_t directly remove everything related to dict_index_t::stats_latch dict_stats_index_set_n_diff(): now returns computed values instead of setting fields in dict_index_t directly dict_stats_analyze_index(): now returns computed values instead of setting fields in dict_index_t directly Reviewed by: Marko Mäkelä --- storage/innobase/btr/btr0btr.cc | 4 +- storage/innobase/btr/btr0cur.cc | 46 ++++---- storage/innobase/dict/dict0dict.cc | 50 -------- storage/innobase/dict/dict0mem.cc | 13 +- storage/innobase/dict/dict0stats.cc | 210 +++++++++++++++++++++------------ storage/innobase/handler/ha_innodb.cc | 52 ++++---- storage/innobase/handler/i_s.cc | 47 ++++---- storage/innobase/ibuf/ibuf0ibuf.cc | 2 +- storage/innobase/include/btr0btr.h | 4 +- storage/innobase/include/btr0cur.h | 31 +++-- storage/innobase/include/dict0dict.h | 18 +-- storage/innobase/include/dict0dict.ic | 2 +- storage/innobase/include/dict0mem.h | 25 +--- storage/innobase/include/dict0stats.ic | 7 +- storage/innobase/include/sync0sync.h | 1 - storage/innobase/row/row0mysql.cc | 6 +- storage/innobase/row/row0uins.cc | 2 +- storage/innobase/sync/sync0debug.cc | 3 - storage/innobase/sync/sync0sync.cc | 1 - 19 files changed, 244 insertions(+), 280 deletions(-) (limited to 'storage') diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 3b0e755bc91..511ac66d58c 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -283,7 +283,7 @@ the index. ulint btr_height_get( /*===========*/ - dict_index_t* index, /*!< in: index tree */ + const dict_index_t* index, /*!< in: index tree */ mtr_t* mtr) /*!< in/out: mini-transaction */ { ulint height=0; @@ -592,7 +592,7 @@ Gets the number of pages in a B-tree. ulint btr_get_size( /*=========*/ - dict_index_t* index, /*!< in: index */ + const dict_index_t* index, /*!< in: index */ ulint flag, /*!< in: BTR_N_LEAF_PAGES or BTR_TOTAL_SIZE */ mtr_t* mtr) /*!< in/out: mini-transaction where index is s-latched */ diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 6ae6bad10c8..7fd34c5d652 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -6133,21 +6133,19 @@ btr_record_not_null_field_in_rec( } } -/*******************************************************************//** -Estimates the number of different key values in a given index, for +/** Estimates the number of different key values in a given index, for each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). The estimates are stored in the array index->stat_n_diff_key_vals[] (indexed 0..n_uniq-1) and the number of pages that were sampled is saved in -index->stat_n_sample_sizes[]. +result.n_sample_sizes[]. If innodb_stats_method is nulls_ignored, we also record the number of non-null values for each prefix and stored the estimates in -array index->stat_n_non_null_key_vals. -@return true if the index is available and we get the estimated numbers, -false if the index is unavailable. */ -bool -btr_estimate_number_of_different_key_vals( -/*======================================*/ - dict_index_t* index) /*!< in: index */ +array result.n_non_null_key_vals. +@param[in] index index +@return vector with statistics information +empty vector if the index is unavailable. */ +std::vector +btr_estimate_number_of_different_key_vals(dict_index_t* index) { btr_cur_t cursor; page_t* page; @@ -6167,11 +6165,11 @@ btr_estimate_number_of_different_key_vals( rec_offs* offsets_rec = NULL; rec_offs* offsets_next_rec = NULL; + std::vector result; + /* For spatial index, there is no such stats can be fetched. */ - if (dict_index_is_spatial(index)) { - return(false); - } + ut_ad(!dict_index_is_spatial(index)); n_cols = dict_index_get_n_unique(index); @@ -6280,7 +6278,7 @@ btr_estimate_number_of_different_key_vals( mtr_commit(&mtr); mem_heap_free(heap); - return(false); + return result; } /* Count the number of different key values for each prefix of @@ -6386,8 +6384,12 @@ exit_loop: also the pages used for external storage of fields (those pages are included in index->stat_n_leaf_pages) */ + result.reserve(n_cols); + for (j = 0; j < n_cols; j++) { - index->stat_n_diff_key_vals[j] + index_field_stats_t stat; + + stat.n_diff_key_vals = BTR_TABLE_STATS_FROM_SAMPLE( n_diff[j], index, n_sample_pages, total_external_size, not_empty_flag); @@ -6408,25 +6410,23 @@ exit_loop: add_on = n_sample_pages; } - index->stat_n_diff_key_vals[j] += add_on; + stat.n_diff_key_vals += add_on; - index->stat_n_sample_sizes[j] = n_sample_pages; + stat.n_sample_sizes = n_sample_pages; - /* Update the stat_n_non_null_key_vals[] with our - sampled result. stat_n_non_null_key_vals[] is created - and initialized to zero in dict_index_add_to_cache(), - along with stat_n_diff_key_vals[] array */ if (n_not_null != NULL) { - index->stat_n_non_null_key_vals[j] = + stat.n_non_null_key_vals = BTR_TABLE_STATS_FROM_SAMPLE( n_not_null[j], index, n_sample_pages, total_external_size, not_empty_flag); } + + result.push_back(stat); } mem_heap_free(heap); - return(true); + return result; } /*================== EXTERNAL STORAGE OF BIG FIELDS ===================*/ diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 37a8a4387d1..e3ce13b5b27 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -267,56 +267,6 @@ dict_mutex_exit_for_mysql(void) mutex_exit(&dict_sys->mutex); } -/** Lock the appropriate latch to protect a given table's statistics. -@param[in] table table whose stats to lock -@param[in] latch_mode RW_S_LATCH or RW_X_LATCH */ -void -dict_table_stats_lock( - dict_table_t* table, - ulint latch_mode) -{ - ut_ad(table != NULL); - ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - - switch (latch_mode) { - case RW_S_LATCH: - rw_lock_s_lock(&table->stats_latch); - break; - case RW_X_LATCH: - rw_lock_x_lock(&table->stats_latch); - break; - case RW_NO_LATCH: - /* fall through */ - default: - ut_error; - } -} - -/** Unlock the latch that has been locked by dict_table_stats_lock(). -@param[in] table table whose stats to unlock -@param[in] latch_mode RW_S_LATCH or RW_X_LATCH */ -void -dict_table_stats_unlock( - dict_table_t* table, - ulint latch_mode) -{ - ut_ad(table != NULL); - ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - - switch (latch_mode) { - case RW_S_LATCH: - rw_lock_s_unlock(&table->stats_latch); - break; - case RW_X_LATCH: - rw_lock_x_unlock(&table->stats_latch); - break; - case RW_NO_LATCH: - /* fall through */ - default: - ut_error; - } -} - /**********************************************************************//** Try to drop any indexes after an aborted index creation. This can also be after a server kill during DROP INDEX. */ diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index 11d362d32c6..efefa40e69f 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -125,8 +125,7 @@ dict_mem_table_create( ulint n_cols, ulint n_v_cols, ulint flags, - ulint flags2, - bool init_stats_latch) + ulint flags2) { dict_table_t* table; mem_heap_t* heap; @@ -182,12 +181,6 @@ dict_mem_table_create( new(&table->foreign_set) dict_foreign_set(); new(&table->referenced_set) dict_foreign_set(); - if (init_stats_latch) { - rw_lock_create(dict_table_stats_key, &table->stats_latch, - SYNC_INDEX_TREE); - table->stats_latch_inited = true; - } - return(table); } @@ -237,10 +230,6 @@ dict_mem_table_free( UT_DELETE(table->s_cols); } - if (table->stats_latch_inited) { - rw_lock_free(&table->stats_latch); - } - mem_heap_free(table->heap); } diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 5083fda48ee..bf2645fd65c 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -486,8 +486,6 @@ dict_stats_table_clone_create( ut_d(t->magic_n = DICT_TABLE_MAGIC_N); - rw_lock_create(dict_table_stats_key, &t->stats_latch, SYNC_INDEX_TREE); - return(t); } @@ -500,15 +498,12 @@ dict_stats_table_clone_free( /*========================*/ dict_table_t* t) /*!< in: dummy table object to free */ { - rw_lock_free(&t->stats_latch); mem_heap_free(t->heap); } /*********************************************************************//** Write all zeros (or 1 where it makes sense) into an index -statistics members. The resulting stats correspond to an empty index. -The caller must own index's table stats latch in X mode -(dict_table_stats_lock(table, RW_X_LATCH)) */ +statistics members. The resulting stats correspond to an empty index. */ static void dict_stats_empty_index( @@ -519,6 +514,7 @@ dict_stats_empty_index( { ut_ad(!(index->type & DICT_FTS)); ut_ad(!dict_index_is_ibuf(index)); + ut_ad(mutex_own(&dict_sys->mutex)); ulint n_uniq = index->n_uniq; @@ -548,10 +544,9 @@ dict_stats_empty_table( bool empty_defrag_stats) /*!< in: whether to empty defrag stats */ { - /* Zero the stats members */ - - dict_table_stats_lock(table, RW_X_LATCH); + mutex_enter(&dict_sys->mutex); + /* Zero the stats members */ table->stat_n_rows = 0; table->stat_clustered_index_size = 1; /* 1 page for each index, not counting the clustered */ @@ -575,8 +570,7 @@ dict_stats_empty_table( } table->stat_initialized = TRUE; - - dict_table_stats_unlock(table, RW_X_LATCH); + mutex_exit(&dict_sys->mutex); } /*********************************************************************//** @@ -675,6 +669,8 @@ dict_stats_copy( to have the same statistics as if the table was empty */ { + ut_ad(mutex_own(&dict_sys->mutex)); + dst->stats_last_recalc = src->stats_last_recalc; dst->stat_n_rows = src->stat_n_rows; dst->stat_clustered_index_size = src->stat_clustered_index_size; @@ -792,8 +788,6 @@ dict_stats_snapshot_create( { mutex_enter(&dict_sys->mutex); - dict_table_stats_lock(table, RW_S_LATCH); - dict_stats_assert_initialized(table); dict_table_t* t; @@ -807,8 +801,6 @@ dict_stats_snapshot_create( t->stats_sample_pages = table->stats_sample_pages; t->stats_bg_flag = table->stats_bg_flag; - dict_table_stats_unlock(table, RW_S_LATCH); - mutex_exit(&dict_sys->mutex); return(t); @@ -848,10 +840,14 @@ dict_stats_update_transient_for_index( Initialize some bogus index cardinality statistics, so that the data can be queried in various means, also via secondary indexes. */ + mutex_enter(&dict_sys->mutex); dict_stats_empty_index(index, false); + mutex_exit(&dict_sys->mutex); #if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG } else if (ibuf_debug && !dict_index_is_clust(index)) { + mutex_enter(&dict_sys->mutex); dict_stats_empty_index(index, false); + mutex_exit(&dict_sys->mutex); #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ } else { mtr_t mtr; @@ -874,7 +870,9 @@ dict_stats_update_transient_for_index( switch (size) { case ULINT_UNDEFINED: + mutex_enter(&dict_sys->mutex); dict_stats_empty_index(index, false); + mutex_exit(&dict_sys->mutex); return; case 0: /* The root node of the tree is a leaf */ @@ -886,11 +884,23 @@ dict_stats_update_transient_for_index( /* Do not continue if table decryption has failed or table is already marked as corrupted. */ if (index->is_readable()) { - /* We don't handle the return value since it - will be false only when some thread is - dropping the table and we don't have to empty - the statistics of the to be dropped index */ - btr_estimate_number_of_different_key_vals(index); + std::vector stats + = btr_estimate_number_of_different_key_vals( + index); + + if (!stats.empty()) { + ut_ad(!mutex_own(&dict_sys->mutex)); + mutex_enter(&dict_sys->mutex); + for (size_t i = 0; i < stats.size(); ++i) { + index->stat_n_diff_key_vals[i] + = stats[i].n_diff_key_vals; + index->stat_n_sample_sizes[i] + = stats[i].n_sample_sizes; + index->stat_n_non_null_key_vals[i] + = stats[i].n_non_null_key_vals; + } + mutex_exit(&dict_sys->mutex); + } } } } @@ -907,6 +917,8 @@ dict_stats_update_transient( /*========================*/ dict_table_t* table) /*!< in/out: table */ { + ut_ad(!mutex_own(&dict_sys->mutex)); + dict_index_t* index; ulint sum_of_index_sizes = 0; @@ -932,27 +944,25 @@ dict_stats_update_transient( ut_ad(!dict_index_is_ibuf(index)); - if (index->type & DICT_FTS || dict_index_is_spatial(index)) { + if (index->type & (DICT_FTS | DICT_SPATIAL)) { continue; } - dict_stats_empty_index(index, false); - - if (dict_stats_should_ignore_index(index)) { + if (dict_stats_should_ignore_index(index) + || !index->is_readable()) { + mutex_enter(&dict_sys->mutex); + dict_stats_empty_index(index, false); + mutex_exit(&dict_sys->mutex); continue; } - /* Do not continue if table decryption has failed or - table is already marked as corrupted. */ - if (!index->is_readable()) { - break; - } - dict_stats_update_transient_for_index(index); sum_of_index_sizes += index->stat_index_size; } + mutex_enter(&dict_sys->mutex); + index = dict_table_get_first_index(table); table->stat_n_rows = index->stat_n_diff_key_vals[ @@ -968,6 +978,8 @@ dict_stats_update_transient( table->stat_modified_counter = 0; table->stat_initialized = TRUE; + + mutex_exit(&dict_sys->mutex); } /* @{ Pseudo code about the relation between the following functions @@ -1808,16 +1820,31 @@ dict_stats_analyze_index_for_n_prefix( btr_pcur_close(&pcur); } +/** statistics for an index */ +struct index_stats_t +{ + std::vector stats; + ulint index_size; + ulint n_leaf_pages; + + index_stats_t(ulint n_uniq) : index_size(1), n_leaf_pages(1) + { + stats.reserve(n_uniq); + for (ulint i= 0; i < n_uniq; ++i) + stats.push_back(index_field_stats_t(0, 1, 0)); + } +}; + /** Set dict_index_t::stat_n_diff_key_vals[] and stat_n_sample_sizes[]. @param[in] n_diff_data input data to use to derive the results -@param[in,out] index index whose stat_n_diff_key_vals[] to set */ +@param[in,out] index_stats index stats to set */ UNIV_INLINE void dict_stats_index_set_n_diff( const n_diff_data_t* n_diff_data, - dict_index_t* index) + index_stats_t& index_stats) { - for (ulint n_prefix = dict_index_get_n_unique(index); + for (ulint n_prefix = index_stats.stats.size(); n_prefix >= 1; n_prefix--) { /* n_diff_all_analyzed_pages can be 0 here if @@ -1848,14 +1875,14 @@ dict_stats_index_set_n_diff( that the total number of ordinary leaf pages is T * D / (D + E). */ n_ordinary_leaf_pages - = index->stat_n_leaf_pages + = index_stats.n_leaf_pages * data->n_leaf_pages_to_analyze / (data->n_leaf_pages_to_analyze + data->n_external_pages_sum); } /* See REF01 for an explanation of the algorithm */ - index->stat_n_diff_key_vals[n_prefix - 1] + index_stats.stats[n_prefix - 1].n_diff_key_vals = n_ordinary_leaf_pages * data->n_diff_on_level @@ -1864,7 +1891,7 @@ dict_stats_index_set_n_diff( * data->n_diff_all_analyzed_pages / data->n_leaf_pages_to_analyze; - index->stat_n_sample_sizes[n_prefix - 1] + index_stats.stats[n_prefix - 1].n_sample_sizes = data->n_leaf_pages_to_analyze; DEBUG_PRINTF(" %s(): n_diff=" UINT64PF @@ -1873,9 +1900,9 @@ dict_stats_index_set_n_diff( " * " UINT64PF " / " UINT64PF " * " UINT64PF " / " UINT64PF ")\n", __func__, - index->stat_n_diff_key_vals[n_prefix - 1], + index_stats.stats[n_prefix - 1].n_diff_key_vals, n_prefix, - index->stat_n_leaf_pages, + index_stats.n_leaf_pages, data->n_diff_on_level, data->n_recs_on_level, data->n_diff_all_analyzed_pages, @@ -1883,15 +1910,12 @@ dict_stats_index_set_n_diff( } } -/*********************************************************************//** -Calculates new statistics for a given index and saves them to the index +/** Calculates new statistics for a given index and saves them to the index members stat_n_diff_key_vals[], stat_n_sample_sizes[], stat_index_size and -stat_n_leaf_pages. This function could be slow. */ -static -void -dict_stats_analyze_index( -/*=====================*/ - dict_index_t* index) /*!< in/out: index to analyze */ +stat_n_leaf_pages. This function can be slow. +@param[in] index index to analyze +@return index stats */ +static index_stats_t dict_stats_analyze_index(dict_index_t* index) { ulint root_level; ulint level; @@ -1902,20 +1926,22 @@ dict_stats_analyze_index( ib_uint64_t total_pages; mtr_t mtr; ulint size; + index_stats_t result(index->n_uniq); DBUG_ENTER("dict_stats_analyze_index"); DBUG_PRINT("info", ("index: %s, online status: %d", index->name(), dict_index_get_online_status(index))); + ut_ad(!mutex_own(&dict_sys->mutex)); // because this function is slow + ut_ad(index->table->get_ref_count()); + /* Disable update statistic for Rtree */ if (dict_index_is_spatial(index)) { - DBUG_VOID_RETURN; + DBUG_RETURN(result); } DEBUG_PRINTF(" %s(index=%s)\n", __func__, index->name()); - dict_stats_empty_index(index, false); - mtr_start(&mtr); mtr_s_lock(dict_index_get_lock(index), &mtr); @@ -1923,7 +1949,7 @@ dict_stats_analyze_index( size = btr_get_size(index, BTR_TOTAL_SIZE, &mtr); if (size != ULINT_UNDEFINED) { - index->stat_index_size = size; + result.index_size = size; size = btr_get_size(index, BTR_N_LEAF_PAGES, &mtr); } @@ -1933,13 +1959,13 @@ dict_stats_analyze_index( switch (size) { case ULINT_UNDEFINED: dict_stats_assert_initialized_index(index); - DBUG_VOID_RETURN; + DBUG_RETURN(result); case 0: /* The root node of the tree is a leaf */ size = 1; } - index->stat_n_leaf_pages = size; + result.n_leaf_pages = size; mtr_start(&mtr); @@ -1980,14 +2006,18 @@ dict_stats_analyze_index( NULL /* boundaries not needed */, &mtr); + mtr_commit(&mtr); + + mutex_enter(&dict_sys->mutex); for (ulint i = 0; i < n_uniq; i++) { - index->stat_n_sample_sizes[i] = total_pages; + result.stats[i].n_diff_key_vals = index->stat_n_diff_key_vals[i]; + result.stats[i].n_sample_sizes = total_pages; + result.stats[i].n_non_null_key_vals = index->stat_n_non_null_key_vals[i]; } + result.n_leaf_pages = index->stat_n_leaf_pages; + mutex_exit(&dict_sys->mutex); - mtr_commit(&mtr); - - dict_stats_assert_initialized_index(index); - DBUG_VOID_RETURN; + DBUG_RETURN(result); } /* For each level that is being scanned in the btree, this contains the @@ -2179,13 +2209,12 @@ found_level: /* n_prefix == 0 means that the above loop did not end up prematurely due to tree being changed and so n_diff_data[] is set up. */ if (n_prefix == 0) { - dict_stats_index_set_n_diff(n_diff_data, index); + dict_stats_index_set_n_diff(n_diff_data, result); } UT_DELETE_ARRAY(n_diff_data); - dict_stats_assert_initialized_index(index); - DBUG_VOID_RETURN; + DBUG_RETURN(result); } /*********************************************************************//** @@ -2203,7 +2232,7 @@ dict_stats_update_persistent( DEBUG_PRINTF("%s(table=%s)\n", __func__, table->name); - dict_table_stats_lock(table, RW_X_LATCH); + DEBUG_SYNC_C("dict_stats_update_persistent"); /* analyze the clustered index first */ @@ -2214,7 +2243,6 @@ dict_stats_update_persistent( || (index->type | DICT_UNIQUE) != (DICT_CLUSTERED | DICT_UNIQUE)) { /* Table definition is corrupt */ - dict_table_stats_unlock(table, RW_X_LATCH); dict_stats_empty_table(table, true); return(DB_CORRUPTION); @@ -2222,7 +2250,16 @@ dict_stats_update_persistent( ut_ad(!dict_index_is_ibuf(index)); - dict_stats_analyze_index(index); + index_stats_t stats = dict_stats_analyze_index(index); + + mutex_enter(&dict_sys->mutex); + index->stat_index_size = stats.index_size; + index->stat_n_leaf_pages = stats.n_leaf_pages; + for (size_t i = 0; i < stats.stats.size(); ++i) { + index->stat_n_diff_key_vals[i] = stats.stats[i].n_diff_key_vals; + index->stat_n_sample_sizes[i] = stats.stats[i].n_sample_sizes; + index->stat_n_non_null_key_vals[i] = stats.stats[i].n_non_null_key_vals; + } ulint n_unique = dict_index_get_n_unique(index); @@ -2240,7 +2277,7 @@ dict_stats_update_persistent( ut_ad(!dict_index_is_ibuf(index)); - if (index->type & DICT_FTS || dict_index_is_spatial(index)) { + if (index->type & (DICT_FTS | DICT_SPATIAL)) { continue; } @@ -2251,7 +2288,20 @@ dict_stats_update_persistent( } if (!(table->stats_bg_flag & BG_STAT_SHOULD_QUIT)) { - dict_stats_analyze_index(index); + mutex_exit(&dict_sys->mutex); + stats = dict_stats_analyze_index(index); + mutex_enter(&dict_sys->mutex); + + index->stat_index_size = stats.index_size; + index->stat_n_leaf_pages = stats.n_leaf_pages; + for (size_t i = 0; i < stats.stats.size(); ++i) { + index->stat_n_diff_key_vals[i] + = stats.stats[i].n_diff_key_vals; + index->stat_n_sample_sizes[i] + = stats.stats[i].n_sample_sizes; + index->stat_n_non_null_key_vals[i] + = stats.stats[i].n_non_null_key_vals; + } } table->stat_sum_of_other_index_sizes @@ -2266,7 +2316,7 @@ dict_stats_update_persistent( dict_stats_assert_initialized(table); - dict_table_stats_unlock(table, RW_X_LATCH); + mutex_exit(&dict_sys->mutex); return(DB_SUCCESS); } @@ -3094,11 +3144,22 @@ dict_stats_update_for_index( if (dict_stats_is_persistent_enabled(index->table)) { if (dict_stats_persistent_storage_check(false)) { - dict_table_stats_lock(index->table, RW_X_LATCH); - dict_stats_analyze_index(index); + index_stats_t stats = dict_stats_analyze_index(index); + mutex_enter(&dict_sys->mutex); + index->stat_index_size = stats.index_size; + index->stat_n_leaf_pages = stats.n_leaf_pages; + for (size_t i = 0; i < stats.stats.size(); ++i) { + index->stat_n_diff_key_vals[i] + = stats.stats[i].n_diff_key_vals; + index->stat_n_sample_sizes[i] + = stats.stats[i].n_sample_sizes; + index->stat_n_non_null_key_vals[i] + = stats.stats[i].n_non_null_key_vals; + } index->table->stat_sum_of_other_index_sizes += index->stat_index_size; - dict_table_stats_unlock(index->table, RW_X_LATCH); + mutex_exit(&dict_sys->mutex); + dict_stats_save(index->table, &index->id); DBUG_VOID_RETURN; } @@ -3119,9 +3180,7 @@ dict_stats_update_for_index( } } - dict_table_stats_lock(index->table, RW_X_LATCH); dict_stats_update_transient_for_index(index); - dict_table_stats_unlock(index->table, RW_X_LATCH); DBUG_VOID_RETURN; } @@ -3275,7 +3334,7 @@ dict_stats_update( switch (err) { case DB_SUCCESS: - dict_table_stats_lock(table, RW_X_LATCH); + mutex_enter(&dict_sys->mutex); /* Pass reset_ignored_indexes=true as parameter to dict_stats_copy. This will cause statictics @@ -3284,7 +3343,7 @@ dict_stats_update( dict_stats_assert_initialized(table); - dict_table_stats_unlock(table, RW_X_LATCH); + mutex_exit(&dict_sys->mutex); dict_stats_table_clone_free(t); @@ -3338,13 +3397,8 @@ dict_stats_update( } transient: - - dict_table_stats_lock(table, RW_X_LATCH); - dict_stats_update_transient(table); - dict_table_stats_unlock(table, RW_X_LATCH); - return(DB_SUCCESS); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3db462ee92d..0617005a213 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -672,7 +672,6 @@ static PSI_rwlock_info all_innodb_rwlocks[] = { PSI_RWLOCK_KEY(trx_purge_latch), PSI_RWLOCK_KEY(index_tree_rw_lock), PSI_RWLOCK_KEY(index_online_log), - PSI_RWLOCK_KEY(dict_table_stats), PSI_RWLOCK_KEY(hash_table_locks) }; # endif /* UNIV_PFS_RWLOCK */ @@ -14341,6 +14340,8 @@ ha_innobase::info_low( DEBUG_SYNC_C("ha_innobase_info_low"); + ut_ad(!mutex_own(&dict_sys->mutex)); + /* If we are forcing recovery at a high level, we will suppress statistics calculation on tables, because that may crash the server if an index is badly corrupted. */ @@ -14377,7 +14378,6 @@ ha_innobase::info_low( opt = DICT_STATS_RECALC_TRANSIENT; } - ut_ad(!mutex_own(&dict_sys->mutex)); ret = dict_stats_update(ib_table, opt); if (ret != DB_SUCCESS) { @@ -14398,9 +14398,7 @@ ha_innobase::info_low( ulint stat_clustered_index_size; ulint stat_sum_of_other_index_sizes; - if (!(flag & HA_STATUS_NO_LOCK)) { - dict_table_stats_lock(ib_table, RW_S_LATCH); - } + mutex_enter(&dict_sys->mutex); ut_a(ib_table->stat_initialized); @@ -14412,9 +14410,7 @@ ha_innobase::info_low( stat_sum_of_other_index_sizes = ib_table->stat_sum_of_other_index_sizes; - if (!(flag & HA_STATUS_NO_LOCK)) { - dict_table_stats_unlock(ib_table, RW_S_LATCH); - } + mutex_exit(&dict_sys->mutex); /* The MySQL optimizer seems to assume in a left join that n_rows @@ -14563,10 +14559,27 @@ ha_innobase::info_low( ib_push_frm_error(m_user_thd, ib_table, table, num_innodb_index, true); } - if (!(flag & HA_STATUS_NO_LOCK)) { - dict_table_stats_lock(ib_table, RW_S_LATCH); + + snprintf(path, sizeof(path), "%s/%s%s", + mysql_data_home, table->s->normalized_path.str, + reg_ext); + + unpack_filename(path,path); + + /* Note that we do not know the access time of the table, + nor the CHECK TABLE time, nor the UPDATE or INSERT time. */ + + if (os_file_get_status( + path, &stat_info, false, + srv_read_only_mode) == DB_SUCCESS) { + stats.create_time = (ulong) stat_info.ctime; } + struct Locking { + Locking() { mutex_enter(&dict_sys->mutex); } + ~Locking() { mutex_exit(&dict_sys->mutex); } + } locking; + ut_a(ib_table->stat_initialized); for (i = 0; i < table->s->keys; i++) { @@ -14644,25 +14657,6 @@ ha_innobase::info_low( key->rec_per_key[j] = rec_per_key_int; } } - - if (!(flag & HA_STATUS_NO_LOCK)) { - dict_table_stats_unlock(ib_table, RW_S_LATCH); - } - - snprintf(path, sizeof(path), "%s/%s%s", - mysql_data_home, table->s->normalized_path.str, - reg_ext); - - unpack_filename(path,path); - - /* Note that we do not know the access time of the table, - nor the CHECK TABLE time, nor the UPDATE or INSERT time. */ - - if (os_file_get_status( - path, &stat_info, false, - srv_read_only_mode) == DB_SUCCESS) { - stats.create_time = (ulong) stat_info.ctime; - } } if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 1140b8b252a..003760f80b1 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -6317,38 +6317,43 @@ i_s_dict_fill_sys_tablestats( OK(field_store_string(fields[SYS_TABLESTATS_NAME], table->name.m_name)); - dict_table_stats_lock(table, RW_S_LATCH); + { + struct Locking + { + Locking() { mutex_enter(&dict_sys->mutex); } + ~Locking() { mutex_exit(&dict_sys->mutex); } + } locking; - if (table->stat_initialized) { - OK(field_store_string(fields[SYS_TABLESTATS_INIT], - "Initialized")); + if (table->stat_initialized) { + OK(field_store_string(fields[SYS_TABLESTATS_INIT], + "Initialized")); - OK(fields[SYS_TABLESTATS_NROW]->store(table->stat_n_rows, - true)); + OK(fields[SYS_TABLESTATS_NROW]->store( + table->stat_n_rows, true)); - OK(fields[SYS_TABLESTATS_CLUST_SIZE]->store( - table->stat_clustered_index_size, true)); + OK(fields[SYS_TABLESTATS_CLUST_SIZE]->store( + table->stat_clustered_index_size, true)); - OK(fields[SYS_TABLESTATS_INDEX_SIZE]->store( - table->stat_sum_of_other_index_sizes, true)); + OK(fields[SYS_TABLESTATS_INDEX_SIZE]->store( + table->stat_sum_of_other_index_sizes, + true)); - OK(fields[SYS_TABLESTATS_MODIFIED]->store( - table->stat_modified_counter, true)); - } else { - OK(field_store_string(fields[SYS_TABLESTATS_INIT], - "Uninitialized")); + OK(fields[SYS_TABLESTATS_MODIFIED]->store( + table->stat_modified_counter, true)); + } else { + OK(field_store_string(fields[SYS_TABLESTATS_INIT], + "Uninitialized")); - OK(fields[SYS_TABLESTATS_NROW]->store(0, true)); + OK(fields[SYS_TABLESTATS_NROW]->store(0, true)); - OK(fields[SYS_TABLESTATS_CLUST_SIZE]->store(0, true)); + OK(fields[SYS_TABLESTATS_CLUST_SIZE]->store(0, true)); - OK(fields[SYS_TABLESTATS_INDEX_SIZE]->store(0, true)); + OK(fields[SYS_TABLESTATS_INDEX_SIZE]->store(0, true)); - OK(fields[SYS_TABLESTATS_MODIFIED]->store(0, true)); + OK(fields[SYS_TABLESTATS_MODIFIED]->store(0, true)); + } } - dict_table_stats_unlock(table, RW_S_LATCH); - OK(fields[SYS_TABLESTATS_AUTONINC]->store(table->autoinc, true)); OK(fields[SYS_TABLESTATS_TABLE_REF_COUNT]->store(ref_count, true)); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 37666cdf372..7068dab77a4 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -1447,7 +1447,7 @@ ibuf_dummy_index_create( table = dict_mem_table_create("IBUF_DUMMY", DICT_HDR_SPACE, n, 0, - comp ? DICT_TF_COMPACT : 0, 0, false); + comp ? DICT_TF_COMPACT : 0, 0); index = dict_mem_index_create("IBUF_DUMMY", "IBUF_DUMMY", DICT_HDR_SPACE, 0, n); diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 5bc0b70714c..29ece955702 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -213,7 +213,7 @@ the index. ulint btr_height_get( /*===========*/ - dict_index_t* index, /*!< in: index tree */ + const dict_index_t* index, /*!< in: index tree */ mtr_t* mtr) /*!< in/out: mini-transaction */ MY_ATTRIBUTE((warn_unused_result)); @@ -585,7 +585,7 @@ Gets the number of pages in a B-tree. ulint btr_get_size( /*=========*/ - dict_index_t* index, /*!< in: index */ + const dict_index_t* index, /*!< in: index */ ulint flag, /*!< in: BTR_N_LEAF_PAGES or BTR_TOTAL_SIZE */ mtr_t* mtr) /*!< in/out: mini-transaction where index is s-latched */ diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index f70fe687182..2f49ac6d12f 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -592,8 +592,24 @@ btr_estimate_n_rows_in_range( const dtuple_t* tuple2, page_cur_mode_t mode2); -/*******************************************************************//** -Estimates the number of different key values in a given index, for + +/** Statistics for one field of an index. */ +struct index_field_stats_t +{ + ib_uint64_t n_diff_key_vals; + ib_uint64_t n_sample_sizes; + ib_uint64_t n_non_null_key_vals; + + index_field_stats_t(ib_uint64_t n_diff_key_vals= 0, + ib_uint64_t n_sample_sizes= 0, + ib_uint64_t n_non_null_key_vals= 0) + : n_diff_key_vals(n_diff_key_vals), n_sample_sizes(n_sample_sizes), + n_non_null_key_vals(n_non_null_key_vals) + { + } +}; + +/** Estimates the number of different key values in a given index, for each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). The estimates are stored in the array index->stat_n_diff_key_vals[] (indexed 0..n_uniq-1) and the number of pages that were sampled is saved in @@ -601,12 +617,11 @@ index->stat_n_sample_sizes[]. If innodb_stats_method is nulls_ignored, we also record the number of non-null values for each prefix and stored the estimates in array index->stat_n_non_null_key_vals. -@return true if the index is available and we get the estimated numbers, -false if the index is unavailable. */ -bool -btr_estimate_number_of_different_key_vals( -/*======================================*/ - dict_index_t* index); /*!< in: index */ +@param[in] index index +@return stat vector if the index is available and we get the estimated numbers, +empty vector if the index is unavailable. */ +std::vector +btr_estimate_number_of_different_key_vals(dict_index_t* index); /** Gets the externally stored size of a record, in units of a database page. @param[in] rec record diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 49e884d064d..7351b5fb682 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1459,7 +1459,7 @@ UNIV_INLINE rw_lock_t* dict_index_get_lock( /*================*/ - dict_index_t* index) /*!< in: index */ + const dict_index_t* index) /*!< in: index */ MY_ATTRIBUTE((nonnull, warn_unused_result)); /********************************************************************//** Returns free space reserved for future updates of records. This is @@ -1524,22 +1524,6 @@ void dict_mutex_exit_for_mysql(void); /*===========================*/ -/** Lock the appropriate latch to protect a given table's statistics. -@param[in] table table whose stats to lock -@param[in] latch_mode RW_S_LATCH or RW_X_LATCH */ -void -dict_table_stats_lock( - dict_table_t* table, - ulint latch_mode); - -/** Unlock the latch that has been locked by dict_table_stats_lock(). -@param[in] table table whose stats to unlock -@param[in] latch_mode RW_S_LATCH or RW_X_LATCH */ -void -dict_table_stats_unlock( - dict_table_t* table, - ulint latch_mode); - /********************************************************************//** Checks if the database name in two table names is the same. @return TRUE if same db name */ diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 3b68e113aa5..554bfdd50ac 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1181,7 +1181,7 @@ UNIV_INLINE rw_lock_t* dict_index_get_lock( /*================*/ - dict_index_t* index) /*!< in: index */ + const dict_index_t* index) /*!< in: index */ { ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 3c7913d7f39..a4defe4d92a 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -307,7 +307,6 @@ before proceeds. */ @param[in] n_v_cols number of virtual columns @param[in] flags table flags @param[in] flags2 table flags2 -@param[in] init_stats_latch whether to init the stats latch @return own: table object */ dict_table_t* dict_mem_table_create( @@ -316,8 +315,7 @@ dict_mem_table_create( ulint n_cols, ulint n_v_cols, ulint flags, - ulint flags2, - bool init_stats_latch=true); + ulint flags2); /****************************************************************//** Free a table memory object. */ @@ -970,7 +968,7 @@ struct dict_index_t{ when InnoDB was started up */ zip_pad_info_t zip_pad;/*!< Information about state of compression failures and successes */ - rw_lock_t lock; /*!< read-write lock protecting the + mutable rw_lock_t lock; /*!< read-write lock protecting the upper levels of the index tree */ /** Determine if the index has been committed to the @@ -1691,23 +1689,8 @@ struct dict_table_t { /*!< set of foreign key constraints which refer to this table */ dict_foreign_set referenced_set; - /** Statistics for query optimization. @{ */ - - /** Creation state of 'stats_latch'. */ - bool stats_latch_inited; - - /** This latch protects: - dict_table_t::stat_initialized, - dict_table_t::stat_n_rows (*), - dict_table_t::stat_clustered_index_size, - dict_table_t::stat_sum_of_other_index_sizes, - dict_table_t::stat_modified_counter (*), - dict_table_t::indexes*::stat_n_diff_key_vals[], - dict_table_t::indexes*::stat_index_size, - dict_table_t::indexes*::stat_n_leaf_pages. - (*) Those are not always protected for - performance reasons. */ - rw_lock_t stats_latch; + /** Statistics for query optimization. Mostly protected by + dict_sys->mutex. @{ */ /** TRUE if statistics have been calculated the first time after database startup or table creation. */ diff --git a/storage/innobase/include/dict0stats.ic b/storage/innobase/include/dict0stats.ic index 98024935e16..c467ee1f3ac 100644 --- a/storage/innobase/include/dict0stats.ic +++ b/storage/innobase/include/dict0stats.ic @@ -75,7 +75,7 @@ dict_stats_is_persistent_enabled(const dict_table_t* table) + dict_stats_update(DICT_STATS_RECALC_TRANSIENT) on a table that has just been PS-enabled. This is acceptable. Avoiding this would mean that we would have to - protect the ::stat_persistent with dict_table_stats_lock() like the + protect the ::stat_persistent with dict_sys->mutex like the other ::stat_ members which would be too big performance penalty, especially when this function is called from dict_stats_update_if_needed(). */ @@ -178,10 +178,7 @@ dict_stats_deinit( ut_a(table->get_ref_count() == 0); - dict_table_stats_lock(table, RW_X_LATCH); - if (!table->stat_initialized) { - dict_table_stats_unlock(table, RW_X_LATCH); return; } @@ -221,6 +218,4 @@ dict_stats_deinit( sizeof(index->stat_n_leaf_pages)); } #endif /* HAVE_valgrind_or_MSAN */ - - dict_table_stats_unlock(table, RW_X_LATCH); } diff --git a/storage/innobase/include/sync0sync.h b/storage/innobase/include/sync0sync.h index 0438d9d0c92..037bf8047cd 100644 --- a/storage/innobase/include/sync0sync.h +++ b/storage/innobase/include/sync0sync.h @@ -126,7 +126,6 @@ extern mysql_pfs_key_t trx_i_s_cache_lock_key; extern mysql_pfs_key_t trx_purge_latch_key; extern mysql_pfs_key_t index_tree_rw_lock_key; extern mysql_pfs_key_t index_online_log_key; -extern mysql_pfs_key_t dict_table_stats_key; extern mysql_pfs_key_t trx_sys_rw_lock_key; extern mysql_pfs_key_t hash_table_locks_key; #endif /* UNIV_PFS_RWLOCK */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 6704c5cff00..312c35c58a3 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1512,7 +1512,7 @@ error_exit: srv_stats.n_rows_inserted.inc(size_t(trx->id)); } - /* Not protected by dict_table_stats_lock() for performance + /* Not protected by dict_sys->mutex for performance reasons, we would rather get garbage in stat_n_rows (which is just an estimate anyway) than protecting the following code with a latch. */ @@ -1870,7 +1870,7 @@ row_update_for_mysql(row_prebuilt_t* prebuilt) ut_ad(node->is_delete == is_delete); if (/*node->*/is_delete) { - /* Not protected by dict_table_stats_lock() for performance + /* Not protected by dict_sys->mutex for performance reasons, we would rather get garbage in stat_n_rows (which is just an estimate anyway) than protecting the following code with a latch. */ @@ -2131,7 +2131,7 @@ row_update_cascade_for_mysql( if (node->is_delete) { /* Not protected by - dict_table_stats_lock() for + dict_sys->mutex for performance reasons, we would rather get garbage in stat_n_rows (which is just an estimate anyway) than diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index 54fa244fca6..915d4b99d16 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -524,7 +524,7 @@ row_undo_ins( } if (err == DB_SUCCESS && node->table->stat_initialized) { - /* Not protected by dict_table_stats_lock() for + /* Not protected by dict_sys->mutex for performance reasons, we would rather get garbage in stat_n_rows (which is just an estimate anyway) than protecting the following code with a latch. */ diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index edcb886fe5c..89db512da2a 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -1473,9 +1473,6 @@ sync_latch_meta_init() LATCH_ADD_RWLOCK(INDEX_TREE, SYNC_INDEX_TREE, index_tree_rw_lock_key); - LATCH_ADD_RWLOCK(DICT_TABLE_STATS, SYNC_INDEX_TREE, - dict_table_stats_key); - LATCH_ADD_RWLOCK(HASH_TABLE_RW_LOCK, SYNC_BUF_PAGE_HASH, hash_table_locks_key); diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc index 2e5558a3294..81bce39df33 100644 --- a/storage/innobase/sync/sync0sync.cc +++ b/storage/innobase/sync/sync0sync.cc @@ -102,7 +102,6 @@ mysql_pfs_key_t buf_block_debug_latch_key; # endif /* UNIV_DEBUG */ mysql_pfs_key_t checkpoint_lock_key; mysql_pfs_key_t dict_operation_lock_key; -mysql_pfs_key_t dict_table_stats_key; mysql_pfs_key_t hash_table_locks_key; mysql_pfs_key_t index_tree_rw_lock_key; mysql_pfs_key_t index_online_log_key; -- cgit v1.2.1 From 527ade25909bf63655ba1f8965096d105ff55368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 Oct 2020 07:27:18 +0200 Subject: MDEV-23163 Merge new release of InnoDB 5.7.32 to 10.2 All relevant InnoDB changes from MySQL 5.7.32 have been applied in preceding commits. --- storage/innobase/include/univ.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index c7474cff4ea..7819b0ae92b 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -41,7 +41,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 7 -#define INNODB_VERSION_BUGFIX 31 +#define INNODB_VERSION_BUGFIX 32 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; -- cgit v1.2.1 From cc5f4428b8b568f98e967a57178efcaf78702168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 Oct 2020 08:13:06 +0200 Subject: MDEV-23693 fixup: Remove unused btr_search_t::withdraw_clock --- storage/innobase/include/btr0sea.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index f2e208df6eb..a9781c65491 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -230,8 +230,6 @@ struct btr_search_t{ the machine word, i.e., they cannot be turned into bit-fields. */ buf_block_t* root_guess;/*!< the root page frame when it was last time fetched, or NULL */ - ulint withdraw_clock; /*!< the withdraw clock value of the buffer - pool when root_guess was stored */ #ifdef BTR_CUR_HASH_ADAPT ulint hash_analysis; /*!< when this exceeds BTR_SEARCH_HASH_ANALYSIS, the hash -- cgit v1.2.1